k1nq
531 lines
20 KiB
Python
531 lines
20 KiB
Python
from __future__ import annotations
|
||
|
||
import re
|
||
from datetime import date, datetime
|
||
from typing import TypedDict
|
||
|
||
from openpyxl.workbook import Workbook
|
||
|
||
from app.errors import AppError
|
||
from app.imports.adapters.base import (
|
||
ALFA_DEBIT_XLSX_VERSION,
|
||
BANK_STATUS_HOLD,
|
||
BANK_STATUS_POSTED,
|
||
DIRECTION_IN,
|
||
DIRECTION_OUT,
|
||
PARSER_VERSION,
|
||
SourceRow,
|
||
StatementDraft,
|
||
UnsupportedStatementLayout,
|
||
account_fingerprint,
|
||
amount_to_kopecks,
|
||
money_to_kopecks,
|
||
)
|
||
from app.imports.readers import open_xlsx
|
||
|
||
DATE_HEADERS = ("дата операции", "дата")
|
||
AMOUNT_HEADERS = ("сумма в валюте счёта", "сумма операции", "сумма")
|
||
DESCRIPTION_HEADERS = ("описание", "назначение", "комментарий")
|
||
CATEGORY_HEADERS = ("категория",)
|
||
ACCOUNT_HEADERS = ("номер карты", "номер счёта", "карта")
|
||
|
||
# Alfa «Копилка для сдачи» round-up service. The rule fails closed: only an
|
||
# exact normalized phrase with a source account equal to the statement account
|
||
# and an outgoing posted row produces the savings_transfer decision hint. The
|
||
# account numbers in the pattern are part of the bank's phrasing contract and
|
||
# are matched against the statement header, never hardcoded per user.
|
||
SAVINGS_SERVICE_RE = re.compile(
|
||
r"перечисление средств в рамках услуги \"копилка для сдачи\" "
|
||
r"со счета (\d{20}) на счет (\d{20})"
|
||
)
|
||
|
||
|
||
def _savings_rule_text(value: str) -> str:
|
||
text = value.replace("\u00a0", " ").replace("ё", "е").replace("Ё", "Е").lower()
|
||
text = re.sub(r"[«»„“”\"']", '"', text)
|
||
return " ".join(text.split())
|
||
|
||
|
||
def detect_savings_transfer(
|
||
description: str | None,
|
||
*,
|
||
direction: str,
|
||
bank_status: str,
|
||
statement_account: str,
|
||
) -> bool:
|
||
if direction != DIRECTION_OUT or bank_status != BANK_STATUS_POSTED or not description:
|
||
return False
|
||
match = SAVINGS_SERVICE_RE.search(_savings_rule_text(description))
|
||
if match is None:
|
||
return False
|
||
statement_digits = re.sub(r"\D", "", statement_account or "")
|
||
if not statement_digits:
|
||
return False
|
||
return match.group(1) == statement_digits
|
||
|
||
|
||
class AlfaXlsxAdapterBase:
|
||
"""Alfa statement contract: XLSX with a header row and bank marker 'Альфа'."""
|
||
|
||
adapter_id: str = ""
|
||
product: str = ""
|
||
product_markers: tuple[str, ...] = ()
|
||
|
||
def detect(self, raw: bytes) -> bool:
|
||
try:
|
||
text = _scan_text(raw)
|
||
except AppError:
|
||
return False
|
||
lowered = text.lower()
|
||
return "альфа" in lowered and any(m in lowered for m in self.product_markers)
|
||
|
||
def parse(self, raw: bytes) -> StatementDraft:
|
||
with open_xlsx(raw) as workbook:
|
||
sheet = workbook.worksheets[0]
|
||
rows = []
|
||
for index, row in enumerate(sheet.iter_rows(values_only=True)):
|
||
if index > get_settings_row_cap():
|
||
raise UnsupportedStatementLayout("row quota exceeded during parse")
|
||
rows.append(row)
|
||
header_idx, columns = _find_header(rows)
|
||
full_text = _rows_text(rows).lower()
|
||
if "альфа" not in full_text:
|
||
raise UnsupportedStatementLayout("Alfa bank marker not found")
|
||
if not any(m in full_text for m in self.product_markers):
|
||
raise UnsupportedStatementLayout(
|
||
f"expected {self.product} statement markers not found"
|
||
)
|
||
fingerprint = None
|
||
masked = _find_account_value(rows)
|
||
if masked:
|
||
fingerprint = account_fingerprint("alfa", masked)
|
||
parsed: list[SourceRow] = []
|
||
for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 1):
|
||
if all(cell is None for cell in row):
|
||
continue
|
||
occurred = _parse_date(row[columns["date"]])
|
||
amount_raw = row[columns["amount"]]
|
||
warnings: list[str] = []
|
||
if occurred is None:
|
||
warnings.append("occurred_date_missing")
|
||
try:
|
||
amount = amount_to_kopecks(amount_raw)
|
||
except UnsupportedStatementLayout:
|
||
amount = 0
|
||
warnings.append("amount_unparseable")
|
||
description = (
|
||
_cell_str(row[columns["description"]]) if columns.get("description") else None
|
||
)
|
||
bank_category = (
|
||
_cell_str(row[columns["category"]]) if columns.get("category") else None
|
||
)
|
||
external_id = (
|
||
_cell_str(row[columns["external"]]) if columns.get("external") else None
|
||
)
|
||
parsed.append(
|
||
SourceRow(
|
||
external_id=external_id,
|
||
occurred_on=occurred,
|
||
posted_on=None,
|
||
amount_kopecks=amount,
|
||
direction=DIRECTION_IN if amount >= 0 else DIRECTION_OUT,
|
||
description=description,
|
||
mcc=None,
|
||
bank_category=bank_category,
|
||
location=None,
|
||
row_ref=f"sheet0:row{offset + 1}",
|
||
warnings=tuple(warnings),
|
||
)
|
||
)
|
||
return StatementDraft(
|
||
adapter_id=self.adapter_id,
|
||
parser_version=PARSER_VERSION,
|
||
bank_code="alfa",
|
||
product=self.product,
|
||
account_fingerprint=fingerprint,
|
||
currency="RUB",
|
||
period_start=min((r.occurred_on for r in parsed if r.occurred_on), default=None),
|
||
period_end=max((r.occurred_on for r in parsed if r.occurred_on), default=None),
|
||
opening_balance_kopecks=None,
|
||
closing_balance_kopecks=None,
|
||
rows=tuple(parsed),
|
||
)
|
||
|
||
|
||
def get_settings_row_cap() -> int:
|
||
from app.config import get_settings
|
||
|
||
return get_settings().max_xlsx_rows
|
||
|
||
|
||
def _scan_text(raw: bytes) -> str:
|
||
with open_xlsx(raw) as workbook:
|
||
return _rows_text(
|
||
[
|
||
row
|
||
for index, row in enumerate(workbook.worksheets[0].iter_rows(values_only=True))
|
||
if index < 40
|
||
]
|
||
)
|
||
|
||
|
||
def _rows_text(rows: list[tuple]) -> str:
|
||
parts = []
|
||
for row in rows:
|
||
for cell in row:
|
||
if cell is not None:
|
||
parts.append(str(cell))
|
||
return " | ".join(parts)
|
||
|
||
|
||
def _cell_str(value: object) -> str | None:
|
||
if value is None:
|
||
return None
|
||
text = str(value).strip()
|
||
return text or None
|
||
|
||
|
||
def _parse_date(value: object) -> date | None:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, datetime):
|
||
return value.date()
|
||
if isinstance(value, date):
|
||
return value
|
||
text = str(value).strip()
|
||
for fmt in ("%d.%m.%Y", "%Y-%m-%d", "%d.%m.%Y %H:%M:%S"):
|
||
try:
|
||
return datetime.strptime(text, fmt).date()
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _find_header(rows: list[tuple]) -> tuple[int, dict[str, int]]:
|
||
for index, row in enumerate(rows[:20]):
|
||
cells = [str(c).strip().lower() if c is not None else "" for c in row]
|
||
date_col = next((i for i, c in enumerate(cells) if c in DATE_HEADERS), None)
|
||
amount_col = next((i for i, c in enumerate(cells) if c in AMOUNT_HEADERS), None)
|
||
if date_col is None or amount_col is None:
|
||
continue
|
||
columns = {"date": date_col, "amount": amount_col}
|
||
for key, synonyms in (
|
||
("description", DESCRIPTION_HEADERS),
|
||
("category", CATEGORY_HEADERS),
|
||
("external", ("id операции", "номер операции")),
|
||
):
|
||
columns[key] = next( # type: ignore[assignment]
|
||
(i for i, c in enumerate(cells) if c in synonyms), None
|
||
)
|
||
return index, columns
|
||
raise UnsupportedStatementLayout("statement header row with date/amount columns not found")
|
||
|
||
|
||
def _find_account_value(rows: list[tuple]) -> str | None:
|
||
for row in rows[:20]:
|
||
for index, cell in enumerate(row):
|
||
if cell is None:
|
||
continue
|
||
header = str(row[index - 1]).strip().lower() if index > 0 and row[index - 1] else ""
|
||
if header in ACCOUNT_HEADERS:
|
||
return str(cell).strip()
|
||
masked = None
|
||
for row in rows[:20]:
|
||
for cell in row:
|
||
if cell is None:
|
||
continue
|
||
text = str(cell)
|
||
if len(text) >= 4 and any(ch.isdigit() for ch in text) and ("*" in text or "•" in text):
|
||
masked = text.strip()
|
||
return masked
|
||
|
||
|
||
class AlfaDebitXlsxAdapter:
|
||
"""Alfa debit statement contract, evidenced against the real export layout.
|
||
|
||
Workbook shape: a 'Выписка по счету' title block, account metadata with
|
||
the exact statement period ('За период с ... по ...') and header control
|
||
totals (Поступления / Расходы / Неподтвержденные операции / Текущий
|
||
баланс), an operation header row (Дата операции, Дата проводки, Код,
|
||
Категория, Описание, Сумма в валюте счета, Статус), then operation rows
|
||
terminated by the signature footer. A HOLD posting-date cell marks an
|
||
immutable hold row; the Код column becomes the external id; a trailing
|
||
MCC token in the description becomes the row MCC. The raw Расходы control
|
||
includes hold amounts, so declared_outflow_kopecks is canonicalized to
|
||
posted-only (raw outflow minus declared hold). Anything else fails closed.
|
||
"""
|
||
|
||
adapter_id = "alfa_debit_xlsx"
|
||
product = "debit"
|
||
parser_version = ALFA_DEBIT_XLSX_VERSION
|
||
bank_code = "alfa"
|
||
currency = "RUB"
|
||
|
||
detect_scan_rows = 60
|
||
header_scan_rows = 30
|
||
|
||
title_label = "выписка по счету"
|
||
period_re = re.compile(
|
||
r"за период с\s*(\d{2}\.\d{2}\.\d{4})\s*по\s*(\d{2}\.\d{2}\.\d{4})", re.IGNORECASE
|
||
)
|
||
mcc_re = re.compile(r"\bMCC\s*:?\s*(\d{4})\b", re.IGNORECASE)
|
||
header_labels = {
|
||
"date": "дата операции",
|
||
"posted": "дата проводки",
|
||
"code": "код",
|
||
"category": "категория",
|
||
"description": "описание",
|
||
"status": "статус",
|
||
}
|
||
amount_label_prefix = "сумма в"
|
||
meta_labels = {
|
||
"account": "номер счета",
|
||
"currency": "валюта счета",
|
||
"inflow": "поступления",
|
||
"outflow": "расходы",
|
||
"hold": "неподтвержденные операции",
|
||
"closing": "текущий баланс",
|
||
}
|
||
footer_markers = ("подпись сотрудника", "уполномоченное лицо")
|
||
footer_page_prefix = "страница"
|
||
hold_token = "hold"
|
||
|
||
def detect(self, raw: bytes) -> bool:
|
||
try:
|
||
with open_xlsx(raw) as workbook:
|
||
rows = _debit_read_rows(workbook, self.detect_scan_rows)
|
||
except Exception:
|
||
return False
|
||
cells = [_debit_norm(cell) for row in rows for cell in row]
|
||
if self.title_label not in cells:
|
||
return False
|
||
required = (self.header_labels["date"], self.header_labels["posted"], "статус")
|
||
return all(label in cells for label in required)
|
||
|
||
def parse(self, raw: bytes) -> StatementDraft:
|
||
with open_xlsx(raw) as workbook:
|
||
cap = get_settings_row_cap()
|
||
rows = _debit_read_rows(workbook, cap + 1)
|
||
if len(rows) > cap:
|
||
raise UnsupportedStatementLayout("row quota exceeded during parse")
|
||
header_idx, columns = _debit_find_header(self, rows)
|
||
meta = _debit_header_metadata(self, rows, header_idx)
|
||
period_start, period_end = _debit_statement_period(self, rows, header_idx)
|
||
parsed = _debit_operation_rows(
|
||
self, rows, header_idx, columns, statement_account=meta["account"]
|
||
)
|
||
if not parsed:
|
||
raise UnsupportedStatementLayout("no operation rows found in Alfa statement")
|
||
return StatementDraft(
|
||
adapter_id=self.adapter_id,
|
||
parser_version=self.parser_version,
|
||
bank_code=self.bank_code,
|
||
product=self.product,
|
||
account_fingerprint=account_fingerprint(self.bank_code, meta["account"]),
|
||
currency=meta["currency"],
|
||
period_start=period_start,
|
||
period_end=period_end,
|
||
opening_balance_kopecks=None,
|
||
closing_balance_kopecks=meta["closing"],
|
||
rows=tuple(parsed),
|
||
declared_inflow_kopecks=meta["inflow"],
|
||
declared_outflow_kopecks=meta["outflow"],
|
||
declared_hold_kopecks=meta["hold"],
|
||
)
|
||
|
||
|
||
def _debit_norm(cell: object) -> str:
|
||
if cell is None:
|
||
return ""
|
||
return str(cell).replace("\u00a0", " ").strip().lower()
|
||
|
||
|
||
def _debit_read_rows(workbook: Workbook, limit: int) -> list[tuple]:
|
||
sheet = workbook.worksheets[0]
|
||
rows: list[tuple] = []
|
||
for index, row in enumerate(sheet.iter_rows(values_only=True)):
|
||
if index >= limit:
|
||
break
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def _debit_find_header(
|
||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple]
|
||
) -> tuple[int, dict[str, int]]:
|
||
for index, row in enumerate(rows[: adapter.header_scan_rows]):
|
||
cells = [_debit_norm(cell) for cell in row]
|
||
columns: dict[str, int] = {}
|
||
for key, label in adapter.header_labels.items():
|
||
column = next((i for i, cell in enumerate(cells) if cell == label), None)
|
||
if column is None:
|
||
break
|
||
columns[key] = column
|
||
if len(columns) != len(adapter.header_labels):
|
||
continue
|
||
amount_column = next(
|
||
(i for i, cell in enumerate(cells) if cell.startswith(adapter.amount_label_prefix)),
|
||
None,
|
||
)
|
||
if amount_column is None:
|
||
continue
|
||
columns["amount"] = amount_column
|
||
return index, columns
|
||
raise UnsupportedStatementLayout("Alfa operation header row not found")
|
||
|
||
|
||
def _debit_header_value(rows: list[tuple], label: str) -> str:
|
||
for row in rows:
|
||
for index, cell in enumerate(row):
|
||
if _debit_norm(cell) != label:
|
||
continue
|
||
for candidate in row[index + 1 :]:
|
||
text = _cell_str(candidate)
|
||
if text:
|
||
return text
|
||
raise UnsupportedStatementLayout(f"Alfa header metadata not found: {label}")
|
||
|
||
|
||
class _AlfaHeaderMeta(TypedDict):
|
||
account: str
|
||
currency: str
|
||
inflow: int
|
||
outflow: int
|
||
hold: int
|
||
closing: int
|
||
|
||
|
||
def _debit_statement_period(
|
||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||
) -> tuple[date, date]:
|
||
text = "\n".join(_debit_norm(cell) for row in rows[:header_idx] for cell in row)
|
||
period_match = adapter.period_re.search(text)
|
||
if period_match is None:
|
||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||
period_start = _parse_date(period_match.group(1))
|
||
period_end = _parse_date(period_match.group(2))
|
||
if period_start is None or period_end is None:
|
||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||
return period_start, period_end
|
||
|
||
|
||
def _debit_header_metadata(
|
||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||
) -> _AlfaHeaderMeta:
|
||
meta: _AlfaHeaderMeta = {
|
||
"account": _debit_header_value(rows[:header_idx], adapter.meta_labels["account"]),
|
||
"currency": "RUB",
|
||
"inflow": 0,
|
||
"outflow": 0,
|
||
"hold": 0,
|
||
"closing": 0,
|
||
}
|
||
currency = _debit_header_value(rows[:header_idx], adapter.meta_labels["currency"])
|
||
if currency.replace(" ", "").upper() not in {"RUB", "RUR"}:
|
||
raise UnsupportedStatementLayout("unsupported Alfa statement currency")
|
||
raw_outflow = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["outflow"])
|
||
)
|
||
meta["hold"] = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["hold"])
|
||
)
|
||
# The raw expense control includes pending (hold) amounts; canonicalize to
|
||
# posted-only so service reconciliation can compare like with like.
|
||
canonical_outflow = raw_outflow - meta["hold"]
|
||
if canonical_outflow < 0:
|
||
raise UnsupportedStatementLayout("inconsistent Alfa expense header controls")
|
||
meta["outflow"] = canonical_outflow
|
||
meta["inflow"] = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["inflow"])
|
||
)
|
||
meta["closing"] = money_to_kopecks(
|
||
_debit_header_value(rows[:header_idx], adapter.meta_labels["closing"])
|
||
)
|
||
return meta
|
||
|
||
|
||
def _debit_operation_rows(
|
||
adapter: AlfaDebitXlsxAdapter,
|
||
rows: list[tuple],
|
||
header_idx: int,
|
||
columns: dict[str, int],
|
||
statement_account: str = "",
|
||
) -> list[SourceRow]:
|
||
parsed: list[SourceRow] = []
|
||
separated_from_operations = False
|
||
for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 2):
|
||
norm_cells = [_debit_norm(cell) for cell in row]
|
||
if any(marker in cell for cell in norm_cells for marker in adapter.footer_markers):
|
||
break
|
||
if any(cell.startswith(adapter.footer_page_prefix) for cell in norm_cells):
|
||
break
|
||
if all(cell == "" for cell in norm_cells):
|
||
separated_from_operations = bool(parsed)
|
||
continue
|
||
occurred = _parse_date(row[columns["date"]])
|
||
posted_cell = _cell_str(row[columns["posted"]]) or ""
|
||
status_cell = _cell_str(row[columns["status"]]) or ""
|
||
bank_status = BANK_STATUS_POSTED
|
||
if adapter.hold_token in posted_cell.lower() or adapter.hold_token in status_cell.lower():
|
||
bank_status = BANK_STATUS_HOLD
|
||
posted_on = None
|
||
else:
|
||
posted_on = _parse_date(posted_cell)
|
||
amount_text = _cell_str(row[columns["amount"]])
|
||
amount: int | None = None
|
||
if amount_text is not None:
|
||
try:
|
||
amount = abs(money_to_kopecks(amount_text))
|
||
except UnsupportedStatementLayout:
|
||
amount = None
|
||
if occurred is None and amount is None:
|
||
later_has_operation = any(
|
||
_parse_date(later[columns["date"]]) is not None
|
||
for later in rows[offset:]
|
||
if len(later) > columns["date"]
|
||
)
|
||
if separated_from_operations and not later_has_operation:
|
||
break
|
||
raise UnsupportedStatementLayout("unexpected row in Alfa operation table")
|
||
warnings: list[str] = []
|
||
if occurred is None:
|
||
warnings.append("occurred_date_missing")
|
||
if amount is None:
|
||
amount = 0
|
||
warnings.append("amount_unparseable")
|
||
if bank_status == BANK_STATUS_POSTED and posted_on is None and posted_cell:
|
||
warnings.append("posted_date_unparseable")
|
||
direction = DIRECTION_OUT if (amount_text or "").startswith("-") else DIRECTION_IN
|
||
description = _cell_str(row[columns["description"]])
|
||
mcc_match = adapter.mcc_re.search(description) if description else None
|
||
parsed.append(
|
||
SourceRow(
|
||
external_id=_cell_str(row[columns["code"]]),
|
||
occurred_on=occurred,
|
||
posted_on=posted_on,
|
||
amount_kopecks=amount,
|
||
direction=direction,
|
||
description=description,
|
||
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
|
||
bank_category=_cell_str(row[columns["category"]]),
|
||
location=None,
|
||
row_ref=f"sheet0:row{offset}",
|
||
warnings=tuple(warnings),
|
||
bank_status=bank_status,
|
||
savings_transfer=detect_savings_transfer(
|
||
description,
|
||
direction=direction,
|
||
bank_status=bank_status,
|
||
statement_account=statement_account,
|
||
),
|
||
)
|
||
)
|
||
separated_from_operations = False
|
||
return parsed
|
||
|
||
|
||
class AlfaCreditXlsxAdapter(AlfaXlsxAdapterBase):
|
||
adapter_id = "alfa_credit_xlsx"
|
||
product = "credit"
|
||
product_markers = ("кредит", "кредитная")
|