k1nq
63d065011b
- Updated timestamps and AST hashes in manifest.json for various backend and frontend files to reflect recent changes. - Added new files related to MCC categories and optional hold reasons in the manifest. - Modified the README in the infra directory to correct the linting tool from 'mypy' to 'ty' for type checking in the backend project.
531 lines
20 KiB
Python
531 lines
20 KiB
Python
from __future__ import annotations
|
||
|
||
import re
|
||
from datetime import date, datetime
|
||
from typing import TypedDict
|
||
|
||
from openpyxl.workbook import Workbook
|
||
|
||
from app.errors import AppError
|
||
from app.imports.adapters.base import (
|
||
ALFA_DEBIT_XLSX_VERSION,
|
||
BANK_STATUS_HOLD,
|
||
BANK_STATUS_POSTED,
|
||
DIRECTION_IN,
|
||
DIRECTION_OUT,
|
||
PARSER_VERSION,
|
||
SourceRow,
|
||
StatementDraft,
|
||
UnsupportedStatementLayout,
|
||
account_fingerprint,
|
||
amount_to_kopecks,
|
||
money_to_kopecks,
|
||
)
|
||
from app.imports.readers import open_xlsx
|
||
|
||
DATE_HEADERS = ("дата операции", "дата")
|
||
AMOUNT_HEADERS = ("сумма в валюте счёта", "сумма операции", "сумма")
|
||
DESCRIPTION_HEADERS = ("описание", "назначение", "комментарий")
|
||
CATEGORY_HEADERS = ("категория",)
|
||
ACCOUNT_HEADERS = ("номер карты", "номер счёта", "карта")
|
||
|
||
# Alfa «Копилка для сдачи» round-up service. The rule fails closed: only an
|
||
# exact normalized phrase with a source account equal to the statement account
|
||
# and an outgoing posted row produces the savings_transfer decision hint. The
|
||
# account numbers in the pattern are part of the bank's phrasing contract and
|
||
# are matched against the statement header, never hardcoded per user.
|
||
SAVINGS_SERVICE_RE = re.compile(
|
||
r"перечисление средств в рамках услуги \"копилка для сдачи\" "
|
||
r"со счета (\d{20}) на счет (\d{20})"
|
||
)
|
||
|
||
|
||
def _savings_rule_text(value: str) -> str:
|
||
text = value.replace("\u00a0", " ").replace("ё", "е").replace("Ё", "Е").lower()
|
||
text = re.sub(r"[«»„“”\"']", '"', text)
|
||
return " ".join(text.split())
|
||
|
||
|
||
def detect_savings_transfer(
|
||
description: str | None,
|
||
*,
|
||
direction: str,
|
||
bank_status: str,
|
||
statement_account: str,
|
||
) -> bool:
|
||
if direction != DIRECTION_OUT or bank_status != BANK_STATUS_POSTED or not description:
|
||
return False
|
||
match = SAVINGS_SERVICE_RE.search(_savings_rule_text(description))
|
||
if match is None:
|
||
return False
|
||
statement_digits = re.sub(r"\D", "", statement_account or "")
|
||
if not statement_digits:
|
||
return False
|
||
return match.group(1) == statement_digits
|
||
|
||
|
||
class AlfaXlsxAdapterBase:
|
||
"""Alfa statement contract: XLSX with a header row and bank marker 'Альфа'."""
|
||
|
||
adapter_id: str = ""
|
||
product: str = ""
|
||
product_markers: tuple[str, ...] = ()
|
||
|
||
def detect(self, raw: bytes) -> bool:
|
||
try:
|
||
text = _scan_text(raw)
|
||
except AppError:
|
||
return False
|
||
lowered = text.lower()
|
||
return "альфа" in lowered and any(m in lowered for m in self.product_markers)
|
||
|
||
def parse(self, raw: bytes) -> StatementDraft:
|
||
with open_xlsx(raw) as workbook:
|
||
sheet = workbook.worksheets[0]
|
||
rows = []
|
||
for index, row in enumerate(sheet.iter_rows(values_only=True)):
|
||
if index > get_settings_row_cap():
|
||
raise UnsupportedStatementLayout("row quota exceeded during parse")
|
||
rows.append(row)
|
||
header_idx, columns = _find_header(rows)
|
||
full_text = _rows_text(rows).lower()
|
||
if "альфа" not in full_text:
|
||
raise UnsupportedStatementLayout("Alfa bank marker not found")
|
||
if not any(m in full_text for m in self.product_markers):
|
||
raise UnsupportedStatementLayout(
|
||
f"expected {self.product} statement markers not found"
|
||
)
|
||
fingerprint = None
|
||
masked = _find_account_value(rows)
|
||
if masked:
|
||
fingerprint = account_fingerprint("alfa", masked)
|
||
parsed: list[SourceRow] = []
|
||
for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 1):
|
||
if all(cell is None for cell in row):
|
||
continue
|
||
occurred = _parse_date(row[columns["date"]])
|
||
amount_raw = row[columns["amount"]]
|
||
warnings: list[str] = []
|
||
if occurred is None:
|
||
warnings.append("occurred_date_missing")
|
||
try:
|
||
amount = amount_to_kopecks(amount_raw)
|
||
except UnsupportedStatementLayout:
|
||
amount = 0
|
||
warnings.append("amount_unparseable")
|
||
description = (
|
||
_cell_str(row[columns["description"]]) if columns.get("description") else None
|
||
)
|
||
bank_category = (
|
||
_cell_str(row[columns["category"]]) if columns.get("category") else None
|
||
)
|
||
external_id = (
|
||
_cell_str(row[columns["external"]]) if columns.get("external") else None
|
||
)
|
||
parsed.append(
|
||
SourceRow(
|
||
external_id=external_id,
|
||
occurred_on=occurred,
|
||
posted_on=None,
|
||
amount_kopecks=amount,
|
||
direction=DIRECTION_IN if amount >= 0 else DIRECTION_OUT,
|
||
description=description,
|
||
mcc=None,
|
||
bank_category=bank_category,
|
||
location=None,
|
||
row_ref=f"sheet0:row{offset + 1}",
|
||
warnings=tuple(warnings),
|
||
)
|
||
)
|
||
return StatementDraft(
|
||
adapter_id=self.adapter_id,
|
||
parser_version=PARSER_VERSION,
|
||
bank_code="alfa",
|
||
product=self.product,
|
||
account_fingerprint=fingerprint,
|
||
currency="RUB",
|
||
period_start=min((r.occurred_on for r in parsed if r.occurred_on), default=None),
|
||
period_end=max((r.occurred_on for r in parsed if r.occurred_on), default=None),
|
||
opening_balance_kopecks=None,
|
||
closing_balance_kopecks=None,
|
||
rows=tuple(parsed),
|
||
)
|
||
|
||
|
||
def get_settings_row_cap() -> int:
|
||
from app.config import get_settings
|
||
|
||
return get_settings().max_xlsx_rows
|
||
|
||
|
||
def _scan_text(raw: bytes) -> str:
|
||
with open_xlsx(raw) as workbook:
|
||
return _rows_text(
|
||
[
|
||
row
|
||
for index, row in enumerate(workbook.worksheets[0].iter_rows(values_only=True))
|
||
if index < 40
|
||
]
|
||
)
|
||
|
||
|
||
def _rows_text(rows: list[tuple]) -> str:
|
||
parts = []
|
||
for row in rows:
|
||
for cell in row:
|
||
if cell is not None:
|
||
parts.append(str(cell))
|
||
return " | ".join(parts)
|
||
|
||
|
||
def _cell_str(value: object) -> str | None:
|
||
if value is None:
|
||
return None
|
||
text = str(value).strip()
|
||
return text or None
|
||
|
||
|
||
def _parse_date(value: object) -> date | None:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, datetime):
|
||
return value.date()
|
||
if isinstance(value, date):
|
||
return value
|
||
text = str(value).strip()
|
||
for fmt in ("%d.%m.%Y", "%Y-%m-%d", "%d.%m.%Y %H:%M:%S"):
|
||
try:
|
||
return datetime.strptime(text, fmt).date()
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _find_header(rows: list[tuple]) -> tuple[int, dict[str, int]]:
|
||
for index, row in enumerate(rows[:20]):
|
||
cells = [str(c).strip().lower() if c is not None else "" for c in row]
|
||
date_col = next((i for i, c in enumerate(cells) if c in DATE_HEADERS), None)
|
||
amount_col = next((i for i, c in enumerate(cells) if c in AMOUNT_HEADERS), None)
|
||
if date_col is None or amount_col is None:
|
||
continue
|
||
columns = {"date": date_col, "amount": amount_col}
|
||
for key, synonyms in (
|
||
("description", DESCRIPTION_HEADERS),
|
||
("category", CATEGORY_HEADERS),
|
||
("external", ("id операции", "номер операции")),
|
||
):
|
||
optional_col = next((i for i, c in enumerate(cells) if c in synonyms), None)
|
||
if optional_col is not None:
|
||
columns[key] = optional_col
|
||
return index, columns
|
||
raise UnsupportedStatementLayout("statement header row with date/amount columns not found")
|
||
|
||
|
||
def _find_account_value(rows: list[tuple]) -> str | None:
|
||
for row in rows[:20]:
|
||
for index, cell in enumerate(row):
|
||
if cell is None:
|
||
continue
|
||
header = str(row[index - 1]).strip().lower() if index > 0 and row[index - 1] else ""
|
||
if header in ACCOUNT_HEADERS:
|
||
return str(cell).strip()
|
||
masked = None
|
||
for row in rows[:20]:
|
||
for cell in row:
|
||
if cell is None:
|
||
continue
|
||
text = str(cell)
|
||
if len(text) >= 4 and any(ch.isdigit() for ch in text) and ("*" in text or "•" in text):
|
||
masked = text.strip()
|
||
return masked
|
||
|
||
|
||
class AlfaDebitXlsxAdapter:
|
||
"""Alfa debit statement contract, evidenced against the real export layout.
|
||
|
||
Workbook shape: a 'Выписка по счету' title block, account metadata with
|
||
the exact statement period ('За период с ... по ...') and header control
|
||
totals (Поступления / Расходы / Неподтвержденные операции / Текущий
|
||
баланс), an operation header row (Дата операции, Дата проводки, Код,
|
||
Категория, Описание, Сумма в валюте счета, Статус), then operation rows
|
||
terminated by the signature footer. A HOLD posting-date cell marks an
|
||
immutable hold row; the Код column becomes the external id; a trailing
|
||
MCC token in the description becomes the row MCC. The raw Расходы control
|
||
includes hold amounts, so declared_outflow_kopecks is canonicalized to
|
||
posted-only (raw outflow minus declared hold). Anything else fails closed.
|
||
"""
|
||
|
||
adapter_id = "alfa_debit_xlsx"
|
||
product = "debit"
|
||
parser_version = ALFA_DEBIT_XLSX_VERSION
|
||
bank_code = "alfa"
|
||
currency = "RUB"
|
||
|
||
detect_scan_rows = 60
|
||
header_scan_rows = 30
|
||
|
||
title_label = "выписка по счету"
|
||
period_re = re.compile(
|
||
r"за период с\s*(\d{2}\.\d{2}\.\d{4})\s*по\s*(\d{2}\.\d{2}\.\d{4})", re.IGNORECASE
|
||
)
|
||
mcc_re = re.compile(r"\bMCC\s*:?\s*(\d{4})\b", re.IGNORECASE)
|
||
header_labels = {
|
||
"date": "дата операции",
|
||
"posted": "дата проводки",
|
||
"code": "код",
|
||
"category": "категория",
|
||
"description": "описание",
|
||
"status": "статус",
|
||
}
|
||
amount_label_prefix = "сумма в"
|
||
meta_labels = {
|
||
"account": "номер счета",
|
||
"currency": "валюта счета",
|
||
"inflow": "поступления",
|
||
"outflow": "расходы",
|
||
"hold": "неподтвержденные операции",
|
||
"closing": "текущий баланс",
|
||
}
|
||
footer_markers = ("подпись сотрудника", "уполномоченное лицо")
|
||
footer_page_prefix = "страница"
|
||
hold_token = "hold"
|
||
|
||
def detect(self, raw: bytes) -> bool:
|
||
try:
|
||
with open_xlsx(raw) as workbook:
|
||
rows = _debit_read_rows(workbook, self.detect_scan_rows)
|
||
except Exception:
|
||
return False
|
||
cells = [_debit_norm(cell) for row in rows for cell in row]
|
||
if self.title_label not in cells:
|
||
return False
|
||
required = (self.header_labels["date"], self.header_labels["posted"], "статус")
|
||
return all(label in cells for label in required)
|
||
|
||
def parse(self, raw: bytes) -> StatementDraft:
|
||
with open_xlsx(raw) as workbook:
|
||
cap = get_settings_row_cap()
|
||
rows = _debit_read_rows(workbook, cap + 1)
|
||
if len(rows) > cap:
|
||
raise UnsupportedStatementLayout("row quota exceeded during parse")
|
||
header_idx, columns = _debit_find_header(self, rows)
|
||
meta = _debit_header_metadata(self, rows, header_idx)
|
||
period_start, period_end = _debit_statement_period(self, rows, header_idx)
|
||
parsed = _debit_operation_rows(
|
||
self, rows, header_idx, columns, statement_account=meta["account"]
|
||
)
|
||
if not parsed:
|
||
raise UnsupportedStatementLayout("no operation rows found in Alfa statement")
|
||
return StatementDraft(
|
||
adapter_id=self.adapter_id,
|
||
parser_version=self.parser_version,
|
||
bank_code=self.bank_code,
|
||
product=self.product,
|
||
account_fingerprint=account_fingerprint(self.bank_code, meta["account"]),
|
||
currency=meta["currency"],
|
||
period_start=period_start,
|
||
period_end=period_end,
|
||
opening_balance_kopecks=None,
|
||
closing_balance_kopecks=meta["closing"],
|
||
rows=tuple(parsed),
|
||
declared_inflow_kopecks=meta["inflow"],
|
||
declared_outflow_kopecks=meta["outflow"],
|
||
declared_hold_kopecks=meta["hold"],
|
||
)
|
||
|
||
|
||
def _debit_norm(cell: object) -> str:
|
||
if cell is None:
|
||
return ""
|
||
return str(cell).replace("\u00a0", " ").strip().lower()
|
||
|
||
|
||
def _debit_read_rows(workbook: Workbook, limit: int) -> list[tuple]:
|
||
sheet = workbook.worksheets[0]
|
||
rows: list[tuple] = []
|
||
for index, row in enumerate(sheet.iter_rows(values_only=True)):
|
||
if index >= limit:
|
||
break
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def _debit_find_header(
|
||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple]
|
||
) -> tuple[int, dict[str, int]]:
|
||
for index, row in enumerate(rows[: adapter.header_scan_rows]):
|
||
cells = [_debit_norm(cell) for cell in row]
|
||
columns: dict[str, int] = {}
|
||
for key, label in adapter.header_labels.items():
|
||
column = next((i for i, cell in enumerate(cells) if cell == label), None)
|
||
if column is None:
|
||
break
|
||
columns[key] = column
|
||
if len(columns) != len(adapter.header_labels):
|
||
continue
|
||
amount_column = next(
|
||
(i for i, cell in enumerate(cells) if cell.startswith(adapter.amount_label_prefix)),
|
||
None,
|
||
)
|
||
if amount_column is None:
|
||
continue
|
||
columns["amount"] = amount_column
|
||
return index, columns
|
||
raise UnsupportedStatementLayout("Alfa operation header row not found")
|
||
|
||
|
||
def _debit_header_value(rows: list[tuple], label: str) -> str:
|
||
for row in rows:
|
||
for index, cell in enumerate(row):
|
||
if _debit_norm(cell) != label:
|
||
continue
|
||
for candidate in row[index + 1 :]:
|
||
text = _cell_str(candidate)
|
||
if text:
|
||
return text
|
||
raise UnsupportedStatementLayout(f"Alfa header metadata not found: {label}")
|
||
|
||
|
||
class _AlfaHeaderMeta(TypedDict):
|
||
account: str
|
||
currency: str
|
||
inflow: int
|
||
outflow: int
|
||
hold: int
|
||
closing: int
|
||
|
||
|
||
def _debit_statement_period(
|
||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||
) -> tuple[date, date]:
|
||
text = "\n".join(_debit_norm(cell) for row in rows[:header_idx] for cell in row)
|
||
period_match = adapter.period_re.search(text)
|
||
if period_match is None:
|
||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||
period_start = _parse_date(period_match.group(1))
|
||
period_end = _parse_date(period_match.group(2))
|
||
if period_start is None or period_end is None:
|
||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||
return period_start, period_end
|
||
|
||
|
||
def _debit_header_metadata(
|
||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||
) -> _AlfaHeaderMeta:
|
||
meta: _AlfaHeaderMeta = {
|
||
"account": _debit_header_value(rows[:header_idx], adapter.meta_labels["account"]),
|
||
"currency": "RUB",
|
||
"inflow": 0,
|
||
"outflow": 0,
|
||
"hold": 0,
|
||
"closing": 0,
|
||
}
|
||
currency = _debit_header_value(rows[:header_idx], adapter.meta_labels["currency"])
|
||
if currency.replace(" ", "").upper() not in {"RUB", "RUR"}:
|
||
raise UnsupportedStatementLayout("unsupported Alfa statement currency")
|
||
raw_outflow = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["outflow"])
|
||
)
|
||
meta["hold"] = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["hold"])
|
||
)
|
||
# The raw expense control includes pending (hold) amounts; canonicalize to
|
||
# posted-only so service reconciliation can compare like with like.
|
||
canonical_outflow = raw_outflow - meta["hold"]
|
||
if canonical_outflow < 0:
|
||
raise UnsupportedStatementLayout("inconsistent Alfa expense header controls")
|
||
meta["outflow"] = canonical_outflow
|
||
meta["inflow"] = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["inflow"])
|
||
)
|
||
meta["closing"] = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["closing"])
|
||
)
|
||
return meta
|
||
|
||
|
||
def _debit_operation_rows(
|
||
adapter: AlfaDebitXlsxAdapter,
|
||
rows: list[tuple],
|
||
header_idx: int,
|
||
columns: dict[str, int],
|
||
statement_account: str = "",
|
||
) -> list[SourceRow]:
|
||
parsed: list[SourceRow] = []
|
||
separated_from_operations = False
|
||
for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 2):
|
||
norm_cells = [_debit_norm(cell) for cell in row]
|
||
if any(marker in cell for cell in norm_cells for marker in adapter.footer_markers):
|
||
break
|
||
if any(cell.startswith(adapter.footer_page_prefix) for cell in norm_cells):
|
||
break
|
||
if all(cell == "" for cell in norm_cells):
|
||
separated_from_operations = bool(parsed)
|
||
continue
|
||
occurred = _parse_date(row[columns["date"]])
|
||
posted_cell = _cell_str(row[columns["posted"]]) or ""
|
||
status_cell = _cell_str(row[columns["status"]]) or ""
|
||
bank_status = BANK_STATUS_POSTED
|
||
if adapter.hold_token in posted_cell.lower() or adapter.hold_token in status_cell.lower():
|
||
bank_status = BANK_STATUS_HOLD
|
||
posted_on = None
|
||
else:
|
||
posted_on = _parse_date(posted_cell)
|
||
amount_text = _cell_str(row[columns["amount"]])
|
||
amount: int | None = None
|
||
if amount_text is not None:
|
||
try:
|
||
amount = abs(money_to_kopecks(amount_text))
|
||
except UnsupportedStatementLayout:
|
||
amount = None
|
||
if occurred is None and amount is None:
|
||
later_has_operation = any(
|
||
_parse_date(later[columns["date"]]) is not None
|
||
for later in rows[offset:]
|
||
if len(later) > columns["date"]
|
||
)
|
||
if separated_from_operations and not later_has_operation:
|
||
break
|
||
raise UnsupportedStatementLayout("unexpected row in Alfa operation table")
|
||
warnings: list[str] = []
|
||
if occurred is None:
|
||
warnings.append("occurred_date_missing")
|
||
if amount is None:
|
||
amount = 0
|
||
warnings.append("amount_unparseable")
|
||
if bank_status == BANK_STATUS_POSTED and posted_on is None and posted_cell:
|
||
warnings.append("posted_date_unparseable")
|
||
direction = DIRECTION_OUT if (amount_text or "").startswith("-") else DIRECTION_IN
|
||
description = _cell_str(row[columns["description"]])
|
||
mcc_match = adapter.mcc_re.search(description) if description else None
|
||
parsed.append(
|
||
SourceRow(
|
||
external_id=_cell_str(row[columns["code"]]),
|
||
occurred_on=occurred,
|
||
posted_on=posted_on,
|
||
amount_kopecks=amount,
|
||
direction=direction,
|
||
description=description,
|
||
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
|
||
bank_category=_cell_str(row[columns["category"]]),
|
||
location=None,
|
||
row_ref=f"sheet0:row{offset}",
|
||
warnings=tuple(warnings),
|
||
bank_status=bank_status,
|
||
savings_transfer=detect_savings_transfer(
|
||
description,
|
||
direction=direction,
|
||
bank_status=bank_status,
|
||
statement_account=statement_account,
|
||
),
|
||
)
|
||
)
|
||
separated_from_operations = False
|
||
return parsed
|
||
|
||
|
||
class AlfaCreditXlsxAdapter(AlfaXlsxAdapterBase):
|
||
adapter_id = "alfa_credit_xlsx"
|
||
product = "credit"
|
||
product_markers = ("кредит", "кредитная")
|