From e43428c1a299b7d930f5694f6e4cdf552944da1e Mon Sep 17 00:00:00 2001 From: k1nq Date: Fri, 18 Sep 2026 17:59:02 +0500 Subject: [PATCH] Implement Gitea CI and real statement imports --- {.github => .gitea}/workflows/ci.yml | 64 +- .gitignore | 4 +- ...7434954-gitea-ci-real-statement-imports.md | 84 + .../versions/20260918_0002_hold_lifecycle.py | 133 + backend/app/budgeting/service.py | 24 +- backend/app/imports/adapters/__init__.py | 12 +- backend/app/imports/adapters/alfa_xlsx.py | 281 +- backend/app/imports/adapters/base.py | 45 + backend/app/imports/adapters/ozon_pdf.py | 175 +- .../app/imports/adapters/rocketbank_pdf.py | 286 +- backend/app/imports/models.py | 63 +- backend/app/imports/router.py | 82 +- backend/app/imports/service.py | 351 +- backend/pyproject.toml | 4 + backend/tests/test_adapter_contracts.py | 825 + backend/tests/test_import_holds.py | 945 + backend/tests/test_postgres_and_s3.py | 47 +- backend/tests/test_private_fixtures.py | 224 + docs/adr/0003-versioned-statement-imports.md | 14 +- docs/parsers/private-fixture-gate.md | 30 +- graphify-out/.graphify_labels.json | 139 +- graphify-out/GRAPH_REPORT.md | 462 +- graphify-out/graph.html | 8 +- graphify-out/graph.json | 20988 +++++++++++----- graphify-out/manifest.json | 174 +- 25 files changed, 19256 insertions(+), 6208 deletions(-) rename {.github => .gitea}/workflows/ci.yml (56%) create mode 100644 .kilo/plans/1789727434954-gitea-ci-real-statement-imports.md create mode 100644 backend/alembic/versions/20260918_0002_hold_lifecycle.py create mode 100644 backend/tests/test_adapter_contracts.py create mode 100644 backend/tests/test_import_holds.py create mode 100644 backend/tests/test_private_fixtures.py diff --git a/.github/workflows/ci.yml b/.gitea/workflows/ci.yml similarity index 56% rename from .github/workflows/ci.yml rename to .gitea/workflows/ci.yml index 091328f..a1f11f0 100644 --- a/.github/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -1,13 +1,15 @@ -name: CI +name: Gitea Actions CI on: push: branches: [master, main] pull_request: +# Single persistent vm-runner with host-network job containers: serialize runs +# so PostgreSQL/Redis service ports never collide across concurrent runs. concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true + group: gitea-actions-ci + cancel-in-progress: false jobs: # Jobs activate only when their locked dependency manifests exist. @@ -36,7 +38,7 @@ jobs: image: postgres:17-alpine env: POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: ${{ secrets.CI_POSTGRES_PASSWORD }} POSTGRES_DB: finance_test ports: - 5432:5432 @@ -55,15 +57,17 @@ jobs: --health-timeout 5s --health-retries 12 env: - DATABASE_URL: postgresql+psycopg://postgres:postgres@localhost:5432/finance_test + DATABASE_URL: postgresql+psycopg://postgres:${{ secrets.CI_POSTGRES_PASSWORD }}@localhost:5432/finance_test REDIS_URL: redis://localhost:6379/0 CELERY_BROKER_URL: redis://localhost:6379/1 - IMPORTS_KEK_B64: MDAxMTIyMzM0NDU1NjY3Nzg4ODlhYWJiY2NkZGVlZg== - RLS_DATABASE_URL: postgresql+psycopg://finance_app_test:app-test@localhost:5432/finance_test + IMPORTS_KEK_B64: ${{ secrets.CI_IMPORTS_KEK_B64 }} + CI_RLS_DATABASE_PASSWORD: ${{ secrets.CI_RLS_DATABASE_PASSWORD }} + CI_MINIO_ROOT_PASSWORD: ${{ secrets.CI_MINIO_ROOT_PASSWORD }} + RLS_DATABASE_URL: postgresql+psycopg://finance_app_test:${{ secrets.CI_RLS_DATABASE_PASSWORD }}@localhost:5432/finance_test S3_ENDPOINT_URL: http://localhost:9000 S3_REGION: us-east-1 S3_ACCESS_KEY_ID: minioadmin - S3_SECRET_ACCESS_KEY: minioadmin + S3_SECRET_ACCESS_KEY: ${{ secrets.CI_MINIO_ROOT_PASSWORD }} S3_BUCKET_STATEMENTS: finance-statements steps: - uses: actions/checkout@v4 @@ -83,22 +87,31 @@ jobs: run: uv run mypy . - name: Create restricted app role and migrate run: | - uv run python -c "from sqlalchemy import create_engine,text; e=create_engine('${DATABASE_URL}'); c=e.connect(); c.execute(text(\"CREATE ROLE finance_app_test LOGIN PASSWORD 'app-test' NOSUPERUSER NOBYPASSRLS\")); c.execute(text('GRANT CONNECT ON DATABASE finance_test TO finance_app_test')); c.execute(text('GRANT USAGE ON SCHEMA public TO finance_app_test')); c.execute(text('ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO finance_app_test')); c.commit(); c.close(); e.dispose()" + uv run python -c "import os; from psycopg import sql; from sqlalchemy import create_engine,text; e=create_engine(os.environ['DATABASE_URL']); c=e.connect(); raw=c.connection.driver_connection; role=sql.SQL('CREATE ROLE finance_app_test LOGIN PASSWORD {} NOSUPERUSER NOBYPASSRLS').format(sql.Literal(os.environ['CI_RLS_DATABASE_PASSWORD'])).as_string(raw); c.exec_driver_sql(role); c.execute(text('GRANT CONNECT ON DATABASE finance_test TO finance_app_test')); c.execute(text('GRANT USAGE ON SCHEMA public TO finance_app_test')); c.execute(text('ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO finance_app_test')); c.commit(); c.close(); e.dispose()" uv run alembic upgrade head + # docker.io denies minio/minio from this network (run 246); quay.io serves it. + # Unique container name keeps parallel/retried runs off each other. - name: Start MinIO (S3-compatible test storage) run: | - docker run -d --name minio-test -p 9000:9000 \ - -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin \ - minio/minio:latest server /data + docker run -d --name minio-ci-${{ github.run_id }}-${{ github.run_attempt }} -p 9000:9000 \ + -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD \ + quay.io/minio/minio:latest server /data for i in $(seq 1 30); do - curl -sf http://localhost:9000/minio/health/live && exit 0 + if curl -sf http://localhost:9000/minio/health/live > /dev/null; then + exit 0 + fi sleep 2 done echo "MinIO did not become healthy" && exit 1 - name: Test (pytest) run: uv run pytest -ra - name: Build provider-neutral image - run: docker build -f ../infra/docker/Dockerfile.backend -t finance-api:ci . + run: docker build -f ../infra/docker/Dockerfile.backend -t finance-api:ci-${{ github.run_id }}-${{ github.run_attempt }} . + - name: Cleanup Docker resources + if: always() + run: | + docker rm -f minio-ci-${{ github.run_id }}-${{ github.run_attempt }} || true + docker rmi -f finance-api:ci-${{ github.run_id }}-${{ github.run_attempt }} || true frontend: name: Frontend (lint / type / test / build) @@ -110,9 +123,11 @@ jobs: working-directory: frontend steps: - uses: actions/checkout@v4 + # pnpm 10 is required: frontend/pnpm-workspace.yaml is config-only + # (allowBuilds) and pnpm 9 rejects it with "packages field missing or empty". - uses: pnpm/action-setup@v4 with: - version: 9 + version: 10 - uses: actions/setup-node@v4 with: node-version: 22 @@ -132,12 +147,31 @@ jobs: compose: name: Compose & nginx validation runs-on: ubuntu-latest + env: + # Unique project name per run: this runner reuses the host, and the + # compose file pins "name: finance", which would collide across runs. + COMPOSE_PROJECT_NAME: finance-ci-${{ github.run_id }}-${{ github.run_attempt }} + # Backend job binds 5432/6379/9000 on the shared host network in the + # same run, so compose services use non-overlapping host ports. + POSTGRES_HOST_PORT: "15432" + REDIS_HOST_PORT: "16379" + MINIO_API_HOST_PORT: "19000" + MINIO_CONSOLE_HOST_PORT: "19001" + MAILPIT_SMTP_HOST_PORT: "11025" + MAILPIT_UI_HOST_PORT: "18025" steps: - uses: actions/checkout@v4 - name: Prepare env from example (placeholders only) run: cp infra/.env.example infra/.env - name: Validate compose file run: docker compose -f infra/docker-compose.yml config -q + # docker.io denies minio/minio from this network (run 246); pre-pull from + # quay.io and retag so compose finds the image locally and never pulls + # the blocked reference. + - name: Pre-pull MinIO via quay.io + run: | + docker pull quay.io/minio/minio:latest + docker tag quay.io/minio/minio:latest minio/minio:latest - name: Boot infra services and wait for healthchecks run: docker compose -f infra/docker-compose.yml up -d --wait --wait-timeout 180 postgres redis minio mailpit - name: Show service status diff --git a/.gitignore b/.gitignore index b4253b2..5843cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -207,4 +207,6 @@ test-results/ .graphify_python .graphify_root cost.json -graphify-out/cache/ \ No newline at end of file +graphify-out/cache/ + +examples/ diff --git a/.kilo/plans/1789727434954-gitea-ci-real-statement-imports.md b/.kilo/plans/1789727434954-gitea-ci-real-statement-imports.md new file mode 100644 index 0000000..aeb3b48 --- /dev/null +++ b/.kilo/plans/1789727434954-gitea-ci-real-statement-imports.md @@ -0,0 +1,84 @@ +# Gitea CI and real statement imports + +## Goal and constraints + +- Move CI to the canonical Gitea Actions location and make the currently failing pipeline pass on the existing `vm-runner`. +- Store only CI-only credentials in Gitea repository Actions secrets; never copy local or future production secrets into Gitea. +- Replace guessed behavior for Alfa debit XLSX, Ozon debit PDF, and Rocketbank debit PDF with contracts proven against the ignored files in `examples/`. +- Never commit real statements, extracted text, full parser snapshots, or private expected values. +- Keep `alfa_credit_xlsx` and `ozon_credit_pdf` code untouched but remove them from the active adapter registry until real fixtures exist. +- Do not implement or redesign frontend UI. Retain its existing CI checks, and document the future HOLD UI contract in ADR 0003. + +## Confirmed decisions + +- `examples/ozon.pdf` is an Ozon debit statement. +- Repository-local `examples/` is an allowed private-fixture location because it is ignored; external `PRIVATE_FIXTURES_DIR` remains supported. +- Alfa `HOLD` rows are immutable source records, but they do not create ledger operations during normal import confirmation. +- An unresolved HOLD blocks closing the budget month containing its `occurred_on` date. +- A later confirmed statement can resolve a HOLD automatically when exactly one conservative match becomes posted. +- Manual HOLD resolution has two backend actions, both with a required reason: `confirm_as_posted` creates a normal operation; `discard_as_cancelled` creates none. +- Gitea secrets to create are `CI_POSTGRES_PASSWORD`, `CI_RLS_DATABASE_PASSWORD`, `CI_MINIO_ROOT_PASSWORD`, and `CI_IMPORTS_KEK_B64`. + +## Implementation plan + +1. **Lock the private-data boundary before parser work.** + - Preserve the existing user change adding `/examples/` to `.gitignore`; verify `git ls-files examples` remains empty. + - Update `docs/parsers/private-fixture-gate.md` to allow an ignored repo-local fixture directory as well as an external one. + - Add a private manifest contract (`manifest.json` beside the statements) containing adapter ID, parser version, period, row counts, control totals, balances, and HOLD totals. Create the current manifest under ignored `examples/`; never commit it. + - Ensure private-test failures report only filenames, adapter IDs, counts, dates, and aggregate mismatches, not raw descriptions, names, account/card numbers, or extracted pages. + +2. **Move and repair Gitea Actions CI.** + - Move `.github/workflows/ci.yml` to `.gitea/workflows/ci.yml` and rename the workflow to make Gitea ownership explicit. Keep compatibility names such as `${{ github.ref }}` and `$GITHUB_OUTPUT` where Gitea deliberately exposes the GitHub-compatible context. + - Use the configured `origin` (`k1nq/finance` on `git.k1nq.space`) and installed `tea` login. List existing Actions secret names first, generate new independent CI-only values, and create/update the four agreed secrets through `tea actions secrets`; do not print, persist, or reuse values from `infra/.env`. + - Replace hard-coded PostgreSQL, restricted-role, MinIO, and KEK values in the workflow with `${{ secrets.* }}`. Keep non-sensitive usernames, database names, endpoints, and bucket names in YAML. + - Retrieve the authenticated logs for Gitea run `246` before changing job mechanics. The observable failures are frontend dependency installation, backend MinIO startup, and Compose infrastructure startup; fix the concrete causes rather than assuming GitHub runner behavior. + - Make Docker resources safe on the persistent VM runner: unique container/project names, an `if: always()` cleanup step, and either serialized backend/Compose jobs or non-overlapping host ports if logs confirm shared-port collisions. Do not weaken tests to obtain a green run. + - Keep the frontend job limited to its existing install/lint/type/test/build checks. If its install failure is lockfile/tooling-related, make only the minimal package-manager/CI correction; do not change UI code. + +3. **Make the parser boundary explicit and internally consistent.** + - In `backend/app/imports/adapters/base.py`, make adapter output canonical: `amount_kopecks` is always positive, `direction` carries the sign, and each row has an immutable bank status (`posted` or `hold`). + - Replace the shared global parser version for changed adapters with per-contract versions. Use a new version for the three rewritten contracts so existing parse runs can never be confused with the old guessed output. + - Extend `StatementDraft` with independent bank control totals: declared inflow, declared outflow, and optional pending/HOLD total. Persist them in the existing parse-run draft JSON. + - Refactor reconciliation in `imports/service.py`: compare parsed posted inflow/outflow with declared totals; compare HOLD totals separately; where both balances exist, also require `opening + inflow - outflow == closing`. A mismatch stays review-blocking with stable diagnostic fields. + - Preserve raw descriptions only in encrypted/import-domain storage already designed for them; logs and test diagnostics must remain redacted. + +4. **Rewrite the three evidenced adapters against their actual layouts.** + - **Alfa debit XLSX:** detect the exact workbook structure (`Выписка по счету`, account metadata, period fields, and operation headers) rather than a nonexistent `Альфа`/`дебет` marker. Parse operation date, posting date, code, bank category, description, amount, status, MCC, and location. Treat a `HOLD` posting-date cell as `bank_status=hold`; stop before signatures/footer. Parse current balance, posted totals, and unconfirmed total from the header. + - **Ozon debit PDF:** detect the legal-bank/title/account/table structure without requiring a nonexistent debit marker. Extract table rows by PDF table/word geometry so multiline document IDs and descriptions remain attached to one transaction. Parse period, account fingerprint input, opening/closing balances, declared totals, external document ID, timestamp date, direction, and amount. + - **Rocketbank debit PDF:** replace inheritance from `OzonPdfAdapterBase` with an independent state-machine parser. Track Russian day headings across pages, accumulate each `time + operation code + multiline description + signed RUR amount` record, ignore repeated headers/footers and daily totals, and extract MCC/location where present. Parse statement-level period, account, balances, inflow/outflow controls. + - Keep the Alfa/Ozon credit classes present but unregistered. `detect_adapter` must only select the three verified-format development contracts and must still fail closed for unknown layouts. + +5. **Model HOLD lifecycle without mutating extracted facts.** + - Add an Alembic migration and model field `SourceRecord.bank_status` with constrained values `posted|hold`, defaulting existing records to `posted`. + - Add a tenant-scoped, one-per-HOLD `HoldResolution` record rather than overwriting the extracted source row. Store resolution kind (`matched_posted`, `confirmed_as_posted`, `discarded_as_cancelled`), optional resolving source record/operation/user, required manual reason, and timestamp. + - During candidate building, match a new posted record to unresolved confirmed HOLD records only within the same space and account/statement contract. Prefer a stable external ID; otherwise require one unambiguous date, amount, direction, and normalized-description fingerprint match. Ambiguity remains review-required. Create the resolution only when the new statement is confirmed. + - Skip unresolved HOLD rows when applying a confirmed import to the ledger, while retaining them in the active parse run and API detail response. + - Add a CSRF-protected imports endpoint/service action for `confirm_as_posted` and `discard_as_cancelled`. Require a mapped account and reason; use ledger services for operation creation, add provenance links, honor closed-period rules, and write audit events. + - Add unresolved active HOLDs from confirmed imports to budgeting close blockers for their `occurred_on` month. Manual confirmation may then expose the normal uncategorized-expense blocker until the created operation is categorized. + +6. **Build layered tests without leaking fixtures.** + - Add tracked synthetic tests for canonical amount semantics, stable detection failures, exact Alfa header mapping, Ozon multiline table grouping, Rocket day/page state transitions, totals reconciliation, HOLD parsing, and malformed/oversized/no-text cases. Generate sanitized XLSX data in tests and test PDF text/table cores with sanitized extracted structures; do not commit derivatives of the real statements. + - Add `private_fixtures` to pytest markers and a separately collected private test module. It reads `PRIVATE_FIXTURES_DIR/manifest.json`, fully parses each real file twice, asserts deterministic output and private expectations, and never runs in normal CI. + - Add service/database tests for: HOLD does not create an operation; confirmed unresolved HOLD blocks only its month; a unique later posted record resolves it on confirmation; ambiguous overlap remains unresolved; both manual actions require a reason; manual confirmation creates exactly one operation and late reimport links rather than duplicates it; tenant isolation and closed-period rules hold. + - Keep the beta gate as **not passed** for all three adapters after this work because only one real statement file per contract is available. Record that one private fixture passes and that a second independent real layout/period is still required. + +7. **Document the product contract and defer UI deliberately.** + - Amend `docs/adr/0003-versioned-statement-imports.md` with posted/HOLD source status, append-only resolution metadata, automatic settlement matching, close blocking, both manual actions, and the future UI requirement to show and resolve HOLD rows. + - Update parser gate/status documentation with the new parser versions and local validation result, without embedding private counts, totals, identifiers, or descriptions. + - Make no changes under `frontend/src`. Do not regenerate or redesign frontend API/UI in this workstream; backend response additions remain backward-compatible optional fields for the later design-first frontend project. + +## Validation and rollout + +1. Run focused adapter synthetic tests, HOLD service tests, import service tests, migration upgrade, Ruff, formatting, and mypy. +2. Run the private suite locally with `PRIVATE_FIXTURES_DIR` pointing to `examples/`; confirm the manifest and all statement files remain ignored and untracked afterward. +3. Run the complete backend suite. Run existing frontend checks only to validate the CI tooling correction, with no UI modifications. +4. Validate Compose and image builds on the same Docker topology used by the Gitea VM runner, including cleanup after forced failure. +5. Push only after local validation, then inspect the new Gitea Actions run with `tea`: all jobs green, secrets present by name, no secret values or private fixture paths/content in logs, and no `.github/workflows` workflow left active. +6. Do not mark any adapter beta-supported until a second independent private fixture for that contract passes the full gate. + +## Principal risks + +- PDF text order can drift even when the visual table looks unchanged; table/word grouping must fail closed and bump parser version on layout drift. +- Alfa HOLD rows may receive a new code or changed description when posted; only unique conservative matching may auto-resolve them. +- Gitea VM runners can retain Docker containers and share host ports across jobs; cleanup and isolation are acceptance criteria, not optional hygiene. +- Assertions, action logs, parser exceptions, and private manifests are all possible financial-data leak paths and must be reviewed before the first Gitea run. diff --git a/backend/alembic/versions/20260918_0002_hold_lifecycle.py b/backend/alembic/versions/20260918_0002_hold_lifecycle.py new file mode 100644 index 0000000..3d694b4 --- /dev/null +++ b/backend/alembic/versions/20260918_0002_hold_lifecycle.py @@ -0,0 +1,133 @@ +"""Hold lifecycle: source bank status and append-only hold resolutions.""" + +import sqlalchemy as sa + +from alembic import op + +revision = "0002" +down_revision = "0001" +branch_labels = None +depends_on = None + + +def _has_column(bind, table: str, column: str) -> bool: + inspector = sa.inspect(bind) + return column in {item["name"] for item in inspector.get_columns(table)} + + +def _has_check_constraint(bind, table: str, suffix: str) -> bool: + inspector = sa.inspect(bind) + return any( + (item.get("name") or "").endswith(suffix) for item in inspector.get_check_constraints(table) + ) + + +def upgrade() -> None: + bind = op.get_bind() + # Revision 0001 materializes the schema from live model metadata, so a + # database created fresh already carries these objects; guard every step. + if not _has_column(bind, "source_records", "bank_status"): + op.add_column( + "source_records", + sa.Column("bank_status", sa.String(length=8), nullable=False, server_default="posted"), + ) + if bind.dialect.name == "postgresql" and not _has_check_constraint( + bind, "source_records", "bank_status_valid" + ): + op.create_check_constraint( + "bank_status_valid", + "source_records", + "bank_status IN ('posted', 'hold')", + ) + inspector = sa.inspect(bind) + if not inspector.has_table("hold_resolutions"): + op.create_table( + "hold_resolutions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("space_id", sa.Uuid(), nullable=False), + sa.Column("hold_source_record_id", sa.Uuid(), nullable=False), + sa.Column("resolution", sa.String(length=24), nullable=False), + sa.Column("resolving_source_record_id", sa.Uuid(), nullable=True), + sa.Column("operation_id", sa.Uuid(), nullable=True), + sa.Column("resolved_by_user_id", sa.Uuid(), nullable=True), + sa.Column("reason", sa.String(length=500), nullable=True), + sa.Column("resolved_at", sa.DateTime(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.CheckConstraint( + "resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')", + name="ck_hold_resolutions_resolution_valid", + ), + sa.CheckConstraint( + "NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') " + "AND (reason IS NULL OR reason = ''))", + name="ck_hold_resolutions_manual_reason_required", + ), + sa.ForeignKeyConstraint( + ["space_id"], + ["financial_spaces.id"], + ondelete="CASCADE", + name="fk_hold_resolutions_space_id_financial_spaces", + ), + sa.ForeignKeyConstraint( + ["space_id", "hold_source_record_id"], + ["source_records.space_id", "source_records.id"], + ondelete="CASCADE", + name="fk_hold_resolutions_hold_tenant", + ), + sa.ForeignKeyConstraint( + ["space_id", "resolving_source_record_id"], + ["source_records.space_id", "source_records.id"], + name="fk_hold_resolutions_resolving_tenant", + ), + sa.ForeignKeyConstraint( + ["space_id", "operation_id"], + ["operations.space_id", "operations.id"], + name="fk_hold_resolutions_operation_tenant", + ), + sa.ForeignKeyConstraint( + ["resolved_by_user_id"], + ["users.id"], + ondelete="SET NULL", + name="fk_hold_resolutions_resolved_by_user_id_users", + ), + sa.PrimaryKeyConstraint("id", name="pk_hold_resolutions"), + sa.UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"), + sa.UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"), + ) + op.create_index("ix_hold_resolutions_space_id", "hold_resolutions", ["space_id"]) + op.create_index( + "ix_hold_resolutions_hold_source_record_id", + "hold_resolutions", + ["hold_source_record_id"], + ) + op.create_index("ix_hold_resolutions_operation_id", "hold_resolutions", ["operation_id"]) + if bind.dialect.name == "postgresql": + # Revision 0001 creates tables from live metadata, so a fresh database + # already has hold_resolutions before this revision runs. RLS setup must + # therefore be independent of the create-table branch above. + op.execute("ALTER TABLE hold_resolutions ENABLE ROW LEVEL SECURITY") + op.execute("ALTER TABLE hold_resolutions FORCE ROW LEVEL SECURITY") + op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions") + op.execute( + "CREATE POLICY tenant_isolation ON hold_resolutions " + "USING (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid) " + "WITH CHECK (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid)" + ) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if inspector.has_table("hold_resolutions"): + if bind.dialect.name == "postgresql": + op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions") + op.drop_index("ix_hold_resolutions_operation_id", table_name="hold_resolutions") + op.drop_index("ix_hold_resolutions_hold_source_record_id", table_name="hold_resolutions") + op.drop_index("ix_hold_resolutions_space_id", table_name="hold_resolutions") + op.drop_table("hold_resolutions") + if _has_column(bind, "source_records", "bank_status"): + with op.batch_alter_table("source_records") as batch_op: + if _has_check_constraint(bind, "source_records", "bank_status_valid"): + batch_op.drop_constraint("bank_status_valid", type_="check") + batch_op.drop_column("bank_status") diff --git a/backend/app/budgeting/service.py b/backend/app/budgeting/service.py index 263eb0c..66a6b8a 100644 --- a/backend/app/budgeting/service.py +++ b/backend/app/budgeting/service.py @@ -440,7 +440,7 @@ def _close_blockers( uncategorized_count: int, negative_envelopes: list[str], ) -> list[dict]: - from app.imports.models import StatementDocument + from app.imports.models import HoldResolution, SourceRecord, StatementDocument blockers = [] unresolved = ( @@ -455,6 +455,28 @@ def _close_blockers( ) if unresolved: blockers.append({"code": "IMPORTS_UNRESOLVED", "count": unresolved}) + start, end = period_bounds(period.year, period.month) + unresolved_holds = ( + db.query(SourceRecord.id) + .join( + StatementDocument, + (StatementDocument.space_id == SourceRecord.space_id) + & (StatementDocument.id == SourceRecord.document_id), + ) + .outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id) + .filter( + SourceRecord.space_id == space_id, + SourceRecord.bank_status == "hold", + StatementDocument.status == "confirmed", + StatementDocument.active_parse_run_id == SourceRecord.parse_run_id, + HoldResolution.id.is_(None), + SourceRecord.occurred_on >= start, + SourceRecord.occurred_on < end, + ) + .count() + ) + if unresolved_holds: + blockers.append({"code": "IMPORT_UNRESOLVED_HOLDS", "count": unresolved_holds}) if uncategorized_count: blockers.append({"code": "BUDGET_UNCATEGORIZED_EXPENSES", "count": uncategorized_count}) if negative_envelopes: diff --git a/backend/app/imports/adapters/__init__.py b/backend/app/imports/adapters/__init__.py index d268ad8..1781d39 100644 --- a/backend/app/imports/adapters/__init__.py +++ b/backend/app/imports/adapters/__init__.py @@ -1,13 +1,17 @@ -from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter +from app.imports.adapters.alfa_xlsx import ( + AlfaCreditXlsxAdapter as AlfaCreditXlsxAdapter, +) +from app.imports.adapters.alfa_xlsx import AlfaDebitXlsxAdapter from app.imports.adapters.base import StatementAdapter, UnsupportedStatementLayout -from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter +from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter as OzonCreditPdfAdapter +from app.imports.adapters.ozon_pdf import OzonDebitPdfAdapter from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter +# Only the three evidenced-format development contracts are active. The credit +# adapters stay present but unregistered until real fixtures exist for them. ADAPTERS: tuple[StatementAdapter, ...] = ( AlfaDebitXlsxAdapter(), - AlfaCreditXlsxAdapter(), OzonDebitPdfAdapter(), - OzonCreditPdfAdapter(), RocketbankDebitPdfAdapter(), ) diff --git a/backend/app/imports/adapters/alfa_xlsx.py b/backend/app/imports/adapters/alfa_xlsx.py index db06046..cfc188a 100644 --- a/backend/app/imports/adapters/alfa_xlsx.py +++ b/backend/app/imports/adapters/alfa_xlsx.py @@ -1,9 +1,16 @@ from __future__ import annotations +import re from datetime import date, datetime +from typing import TypedDict + +from openpyxl.workbook import Workbook from app.errors import AppError from app.imports.adapters.base import ( + ALFA_DEBIT_XLSX_VERSION, + BANK_STATUS_HOLD, + BANK_STATUS_POSTED, DIRECTION_IN, DIRECTION_OUT, PARSER_VERSION, @@ -12,6 +19,7 @@ from app.imports.adapters.base import ( UnsupportedStatementLayout, account_fingerprint, amount_to_kopecks, + money_to_kopecks, ) from app.imports.readers import open_xlsx @@ -198,10 +206,279 @@ def _find_account_value(rows: list[tuple]) -> str | None: return masked -class AlfaDebitXlsxAdapter(AlfaXlsxAdapterBase): +class AlfaDebitXlsxAdapter: + """Alfa debit statement contract, evidenced against the real export layout. + + Workbook shape: a 'Выписка по счету' title block, account metadata with + the exact statement period ('За период с ... по ...') and header control + totals (Поступления / Расходы / Неподтвержденные операции / Текущий + баланс), an operation header row (Дата операции, Дата проводки, Код, + Категория, Описание, Сумма в валюте счета, Статус), then operation rows + terminated by the signature footer. A HOLD posting-date cell marks an + immutable hold row; the Код column becomes the external id; a trailing + MCC token in the description becomes the row MCC. The raw Расходы control + includes hold amounts, so declared_outflow_kopecks is canonicalized to + posted-only (raw outflow minus declared hold). Anything else fails closed. + """ + adapter_id = "alfa_debit_xlsx" product = "debit" - product_markers = ("дебет", "дебетовая") + parser_version = ALFA_DEBIT_XLSX_VERSION + bank_code = "alfa" + currency = "RUB" + + detect_scan_rows = 60 + header_scan_rows = 30 + + title_label = "выписка по счету" + period_re = re.compile( + r"за период с\s*(\d{2}\.\d{2}\.\d{4})\s*по\s*(\d{2}\.\d{2}\.\d{4})", re.IGNORECASE + ) + mcc_re = re.compile(r"\bMCC\s*:?\s*(\d{4})\b", re.IGNORECASE) + header_labels = { + "date": "дата операции", + "posted": "дата проводки", + "code": "код", + "category": "категория", + "description": "описание", + "status": "статус", + } + amount_label_prefix = "сумма в" + meta_labels = { + "account": "номер счета", + "currency": "валюта счета", + "inflow": "поступления", + "outflow": "расходы", + "hold": "неподтвержденные операции", + "closing": "текущий баланс", + } + footer_markers = ("подпись сотрудника", "уполномоченное лицо") + footer_page_prefix = "страница" + hold_token = "hold" + + def detect(self, raw: bytes) -> bool: + try: + with open_xlsx(raw) as workbook: + rows = _debit_read_rows(workbook, self.detect_scan_rows) + except Exception: + return False + cells = [_debit_norm(cell) for row in rows for cell in row] + if self.title_label not in cells: + return False + required = (self.header_labels["date"], self.header_labels["posted"], "статус") + return all(label in cells for label in required) + + def parse(self, raw: bytes) -> StatementDraft: + with open_xlsx(raw) as workbook: + cap = get_settings_row_cap() + rows = _debit_read_rows(workbook, cap + 1) + if len(rows) > cap: + raise UnsupportedStatementLayout("row quota exceeded during parse") + header_idx, columns = _debit_find_header(self, rows) + meta = _debit_header_metadata(self, rows, header_idx) + period_start, period_end = _debit_statement_period(self, rows, header_idx) + parsed = _debit_operation_rows(self, rows, header_idx, columns) + if not parsed: + raise UnsupportedStatementLayout("no operation rows found in Alfa statement") + return StatementDraft( + adapter_id=self.adapter_id, + parser_version=self.parser_version, + bank_code=self.bank_code, + product=self.product, + account_fingerprint=account_fingerprint(self.bank_code, meta["account"]), + currency=meta["currency"], + period_start=period_start, + period_end=period_end, + opening_balance_kopecks=None, + closing_balance_kopecks=meta["closing"], + rows=tuple(parsed), + declared_inflow_kopecks=meta["inflow"], + declared_outflow_kopecks=meta["outflow"], + declared_hold_kopecks=meta["hold"], + ) + + +def _debit_norm(cell: object) -> str: + if cell is None: + return "" + return str(cell).replace("\u00a0", " ").strip().lower() + + +def _debit_read_rows(workbook: Workbook, limit: int) -> list[tuple]: + sheet = workbook.worksheets[0] + rows: list[tuple] = [] + for index, row in enumerate(sheet.iter_rows(values_only=True)): + if index >= limit: + break + rows.append(row) + return rows + + +def _debit_find_header( + adapter: AlfaDebitXlsxAdapter, rows: list[tuple] +) -> tuple[int, dict[str, int]]: + for index, row in enumerate(rows[: adapter.header_scan_rows]): + cells = [_debit_norm(cell) for cell in row] + columns: dict[str, int] = {} + for key, label in adapter.header_labels.items(): + column = next((i for i, cell in enumerate(cells) if cell == label), None) + if column is None: + break + columns[key] = column + if len(columns) != len(adapter.header_labels): + continue + amount_column = next( + (i for i, cell in enumerate(cells) if cell.startswith(adapter.amount_label_prefix)), + None, + ) + if amount_column is None: + continue + columns["amount"] = amount_column + return index, columns + raise UnsupportedStatementLayout("Alfa operation header row not found") + + +def _debit_header_value(rows: list[tuple], label: str) -> str: + for row in rows: + for index, cell in enumerate(row): + if _debit_norm(cell) != label: + continue + for candidate in row[index + 1 :]: + text = _cell_str(candidate) + if text: + return text + raise UnsupportedStatementLayout(f"Alfa header metadata not found: {label}") + + +class _AlfaHeaderMeta(TypedDict): + account: str + currency: str + inflow: int + outflow: int + hold: int + closing: int + + +def _debit_statement_period( + adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int +) -> tuple[date, date]: + text = "\n".join(_debit_norm(cell) for row in rows[:header_idx] for cell in row) + period_match = adapter.period_re.search(text) + if period_match is None: + raise UnsupportedStatementLayout("Alfa statement period not found") + period_start = _parse_date(period_match.group(1)) + period_end = _parse_date(period_match.group(2)) + if period_start is None or period_end is None: + raise UnsupportedStatementLayout("Alfa statement period not found") + return period_start, period_end + + +def _debit_header_metadata( + adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int +) -> _AlfaHeaderMeta: + meta: _AlfaHeaderMeta = { + "account": _debit_header_value(rows[:header_idx], adapter.meta_labels["account"]), + "currency": "RUB", + "inflow": 0, + "outflow": 0, + "hold": 0, + "closing": 0, + } + currency = _debit_header_value(rows[:header_idx], adapter.meta_labels["currency"]) + if currency.replace(" ", "").upper() not in {"RUB", "RUR"}: + raise UnsupportedStatementLayout("unsupported Alfa statement currency") + raw_outflow = money_to_kopecks( + _debit_header_value(rows[:header_idx], adapter.meta_labels["outflow"]) + ) + meta["hold"] = money_to_kopecks( + _debit_header_value(rows[:header_idx], adapter.meta_labels["hold"]) + ) + # The raw expense control includes pending (hold) amounts; canonicalize to + # posted-only so service reconciliation can compare like with like. + canonical_outflow = raw_outflow - meta["hold"] + if canonical_outflow < 0: + raise UnsupportedStatementLayout("inconsistent Alfa expense header controls") + meta["outflow"] = canonical_outflow + meta["inflow"] = money_to_kopecks( + _debit_header_value(rows[:header_idx], adapter.meta_labels["inflow"]) + ) + meta["closing"] = money_to_kopecks( + _debit_header_value(rows[:header_idx], adapter.meta_labels["closing"]) + ) + return meta + + +def _debit_operation_rows( + adapter: AlfaDebitXlsxAdapter, + rows: list[tuple], + header_idx: int, + columns: dict[str, int], +) -> list[SourceRow]: + parsed: list[SourceRow] = [] + separated_from_operations = False + for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 2): + norm_cells = [_debit_norm(cell) for cell in row] + if any(marker in cell for cell in norm_cells for marker in adapter.footer_markers): + break + if any(cell.startswith(adapter.footer_page_prefix) for cell in norm_cells): + break + if all(cell == "" for cell in norm_cells): + separated_from_operations = bool(parsed) + continue + occurred = _parse_date(row[columns["date"]]) + posted_cell = _cell_str(row[columns["posted"]]) or "" + status_cell = _cell_str(row[columns["status"]]) or "" + bank_status = BANK_STATUS_POSTED + if adapter.hold_token in posted_cell.lower() or adapter.hold_token in status_cell.lower(): + bank_status = BANK_STATUS_HOLD + posted_on = None + else: + posted_on = _parse_date(posted_cell) + amount_text = _cell_str(row[columns["amount"]]) + amount: int | None = None + if amount_text is not None: + try: + amount = abs(money_to_kopecks(amount_text)) + except UnsupportedStatementLayout: + amount = None + if occurred is None and amount is None: + later_has_operation = any( + _parse_date(later[columns["date"]]) is not None + for later in rows[offset:] + if len(later) > columns["date"] + ) + if separated_from_operations and not later_has_operation: + break + raise UnsupportedStatementLayout("unexpected row in Alfa operation table") + warnings: list[str] = [] + if occurred is None: + warnings.append("occurred_date_missing") + if amount is None: + amount = 0 + warnings.append("amount_unparseable") + if bank_status == BANK_STATUS_POSTED and posted_on is None and posted_cell: + warnings.append("posted_date_unparseable") + direction = DIRECTION_OUT if (amount_text or "").startswith("-") else DIRECTION_IN + description = _cell_str(row[columns["description"]]) + mcc_match = adapter.mcc_re.search(description) if description else None + parsed.append( + SourceRow( + external_id=_cell_str(row[columns["code"]]), + occurred_on=occurred, + posted_on=posted_on, + amount_kopecks=amount, + direction=direction, + description=description, + mcc=int(mcc_match.group(1)) if mcc_match is not None else None, + bank_category=_cell_str(row[columns["category"]]), + location=None, + row_ref=f"sheet0:row{offset}", + warnings=tuple(warnings), + bank_status=bank_status, + ) + ) + separated_from_operations = False + return parsed class AlfaCreditXlsxAdapter(AlfaXlsxAdapterBase): diff --git a/backend/app/imports/adapters/base.py b/backend/app/imports/adapters/base.py index 8f1ca26..f446b23 100644 --- a/backend/app/imports/adapters/base.py +++ b/backend/app/imports/adapters/base.py @@ -1,4 +1,5 @@ import hashlib +import re from dataclasses import dataclass from datetime import date from decimal import Decimal, InvalidOperation @@ -8,7 +9,16 @@ from app.errors import AppError DIRECTION_IN = "in" DIRECTION_OUT = "out" +BANK_STATUS_POSTED = "posted" +BANK_STATUS_HOLD = "hold" +BANK_STATUSES = (BANK_STATUS_POSTED, BANK_STATUS_HOLD) +# Legacy shared version kept only for the untouched, unregistered credit adapters. PARSER_VERSION = "1" +# Per-contract versions: bumped for the rewritten evidenced contracts so old +# parse runs can never be confused with the new canonical output. +ALFA_DEBIT_XLSX_VERSION = "2" +OZON_DEBIT_PDF_VERSION = "2" +ROCKETBANK_DEBIT_PDF_VERSION = "2" @dataclass(frozen=True) @@ -24,6 +34,15 @@ class SourceRow: location: str | None row_ref: str warnings: tuple[str, ...] = () + bank_status: str = BANK_STATUS_POSTED + + def __post_init__(self) -> None: + if self.amount_kopecks < 0: + raise UnsupportedStatementLayout( + "amount_kopecks must be positive; direction carries the sign" + ) + if self.bank_status not in BANK_STATUSES: + raise UnsupportedStatementLayout("bank_status must be posted or hold") @dataclass(frozen=True) @@ -39,6 +58,9 @@ class StatementDraft: opening_balance_kopecks: int | None closing_balance_kopecks: int | None rows: tuple[SourceRow, ...] + declared_inflow_kopecks: int | None = None + declared_outflow_kopecks: int | None = None + declared_hold_kopecks: int | None = None class UnsupportedStatementLayout(AppError): @@ -77,6 +99,29 @@ def amount_to_kopecks(value: object) -> int: raise UnsupportedStatementLayout(f"unsupported amount cell type: {type(value).__name__}") +def _mask_digits(text: str) -> str: + return re.sub(r"\d", "#", text) + + +CURRENCY_SUFFIX_RE = re.compile(r"(?:₽|RUR|rub|руб\.?)\s*$", re.IGNORECASE) +PLAIN_AMOUNT_RE = re.compile(r"^[+-]?\s?\d[\d\s]*(?:[.,]\d{1,2})?$") + + +def money_to_kopecks(text: object) -> int: + """Parse statement money strings like '-1 234,56 RUR', '+ 500.00 ₽', '12 345'. + + Accepts space/NBSP thousand separators and comma or dot decimal marks. + Fails closed with a digit-masked diagnostic; never leaks the raw value. + """ + if not isinstance(text, str): + raise UnsupportedStatementLayout(f"unsupported money cell type: {type(text).__name__}") + cleaned = text.replace("\u00a0", " ").strip() + cleaned = CURRENCY_SUFFIX_RE.sub("", cleaned).strip() + if not PLAIN_AMOUNT_RE.fullmatch(cleaned): + raise UnsupportedStatementLayout(f"unparseable money amount: {_mask_digits(text)}") + return amount_to_kopecks(cleaned) + + def account_fingerprint(bank_code: str, masked_identifier: str) -> str: digest = hashlib.sha256(f"{bank_code}|{masked_identifier}".encode()).hexdigest() return digest[:64] diff --git a/backend/app/imports/adapters/ozon_pdf.py b/backend/app/imports/adapters/ozon_pdf.py index bff02bf..47220fc 100644 --- a/backend/app/imports/adapters/ozon_pdf.py +++ b/backend/app/imports/adapters/ozon_pdf.py @@ -6,12 +6,14 @@ from datetime import date, datetime from app.imports.adapters.base import ( DIRECTION_IN, DIRECTION_OUT, + OZON_DEBIT_PDF_VERSION, PARSER_VERSION, SourceRow, StatementDraft, UnsupportedStatementLayout, account_fingerprint, amount_to_kopecks, + money_to_kopecks, ) from app.imports.readers import open_pdf @@ -122,10 +124,179 @@ def _first_page_text(raw: bytes) -> str: return pdf.pages[0].extract_text() or "" -class OzonDebitPdfAdapter(OzonPdfAdapterBase): +class OzonDebitPdfAdapter: + """Ozon debit statement contract, evidenced against the real document. + + A 'Справка о движении средств' text-layer PDF: a legal-bank header block + with account/period/opening balance, per-page tables whose rows keep + multiline document IDs and descriptions attached to one transaction, and a + closing block with declared inflow/outflow totals and the closing balance. + Amounts are signed strings ('- 500.00 ₽'); the sign maps to direction and + the stored amount is always positive. + """ + adapter_id = "ozon_debit_pdf" product = "debit" - product_markers = ("дебетов",) + parser_version = OZON_DEBIT_PDF_VERSION + bank_code = "ozon" + currency = "RUB" + + bank_marker = "озон банк" + title_marker = "справка о движении средств" + account_marker_re = re.compile(r"лицевого\s+сч[её]та:?\s*№\s*([\d*•]+)", re.IGNORECASE) + period_re = re.compile( + r"период\s+выписки:?\s*(\d{2}\.\d{2}\.\d{4})\s*[–—-]\s*(\d{2}\.\d{2}\.\d{4})", + re.IGNORECASE, + ) + opening_re = re.compile(r"входящий\s+остаток:?\s*(.+)", re.IGNORECASE) + inflow_re = re.compile(r"итого\s+зачислений\s+за\s+период:?\s*(.+)", re.IGNORECASE) + outflow_re = re.compile(r"итого\s+списаний\s+за\s+период:?\s*(.+)", re.IGNORECASE) + closing_re = re.compile(r"исходящий\s+остаток:?\s*(.+)", re.IGNORECASE) + amount_re = re.compile(r"^([+-])\s*(\d[\d\s]*(?:[.,]\d{2})?)\s*(?:₽|rur|rub)?$", re.IGNORECASE) + table_header_labels = ("дата операции", "документ", "назначение платежа") + table_subheader_tokens = ("рубли", "валюта") + + def detect(self, raw: bytes) -> bool: + try: + text = _first_page_text(raw) + except Exception: + return False + lowered = _ozon_normalize(text) + return ( + self.bank_marker in lowered + and self.title_marker in lowered + and "лицевого сч" in lowered + ) + + def parse(self, raw: bytes) -> StatementDraft: + pages_text: list[str] = [] + pages_tables: list[list[list[list[object]]]] = [] + with open_pdf(raw) as pdf: + if not pdf.pages: + raise UnsupportedStatementLayout("PDF has no pages") + for page_index, page in enumerate(pdf.pages): + text = page.extract_text() or "" + if page_index == 0 and not text.strip(): + raise UnsupportedStatementLayout( + "PDF has no extractable text layer; OCR is not supported" + ) + pages_text.append(text) + extracted = page.extract_tables() + pages_tables.append(extracted if extracted else []) + return self._draft_from_extracted(pages_text, pages_tables) + + def _draft_from_extracted( + self, pages_text: list[str], pages_tables: list[list[list[list[object]]]] + ) -> StatementDraft: + page_zero = _ozon_normalize(pages_text[0]) + if self.bank_marker not in page_zero or self.title_marker not in page_zero: + raise UnsupportedStatementLayout("Ozon statement markers not found") + account_match = self.account_marker_re.search(page_zero) + if account_match is None: + raise UnsupportedStatementLayout("Ozon account line not found") + period_match = self.period_re.search(page_zero) + if period_match is None: + raise UnsupportedStatementLayout("Ozon period line not found") + period_start = datetime.strptime(period_match.group(1), "%d.%m.%Y").date() + period_end = datetime.strptime(period_match.group(2), "%d.%m.%Y").date() + opening = money_to_kopecks(self._required_line(self.opening_re, page_zero, "opening")) + full_text = _ozon_normalize("\n".join(pages_text)) + declared_inflow = money_to_kopecks(self._required_line(self.inflow_re, full_text, "inflow")) + declared_outflow = money_to_kopecks( + self._required_line(self.outflow_re, full_text, "outflow") + ) + closing = money_to_kopecks(self._required_line(self.closing_re, full_text, "closing")) + rows: list[SourceRow] = [] + for page_index, tables in enumerate(pages_tables, start=1): + for table_index, table in enumerate(tables): + for row_index, cells in enumerate(table): + row = self._table_row( + cells, f"page{page_index}:table{table_index}:row{row_index}" + ) + if row is not None: + rows.append(row) + if not rows: + raise UnsupportedStatementLayout("no statement rows matched the Ozon contract") + return StatementDraft( + adapter_id=self.adapter_id, + parser_version=self.parser_version, + bank_code=self.bank_code, + product=self.product, + account_fingerprint=account_fingerprint(self.bank_code, account_match.group(1)), + currency=self.currency, + period_start=period_start, + period_end=period_end, + opening_balance_kopecks=opening, + closing_balance_kopecks=closing, + rows=tuple(rows), + declared_inflow_kopecks=declared_inflow, + declared_outflow_kopecks=declared_outflow, + declared_hold_kopecks=None, + ) + + def _required_line(self, pattern: re.Pattern[str], text: str, what: str) -> str: + match = pattern.search(text) + if match is None: + raise UnsupportedStatementLayout(f"Ozon {what} total line not found") + return match.group(1) + + def _table_row(self, cells: list[object], row_ref: str) -> SourceRow | None: + if cells is None: + raise UnsupportedStatementLayout("unexpected Ozon table row shape") + texts = [_ozon_cell(cell) for cell in cells] + if not any(texts): + return None + lowered = [text.lower() for text in texts] + joined = " ".join(text for text in lowered if text) + if all(label in joined for label in self.table_header_labels): + return None + if lowered[0] == "" and any(token in joined for token in self.table_subheader_tokens): + return None + if lowered[0] == "" or len(texts) < 4: + raise UnsupportedStatementLayout("unexpected Ozon table row shape") + occurred = None + date_match = re.match(r"^(\d{2}\.\d{2}\.\d{4})", texts[0]) + if date_match is not None: + occurred = datetime.strptime(date_match.group(1), "%d.%m.%Y").date() + external_id = re.sub(r"\s+", "", texts[1]) if len(texts) > 1 else "" + description = " ".join(texts[2].split()) if len(texts) > 2 else "" + warnings: list[str] = [] + if occurred is None: + warnings.append("occurred_date_missing") + if not external_id: + warnings.append("external_id_missing") + amount_match = self.amount_re.match(texts[3]) if len(texts) > 3 else None + if amount_match is None: + amount = 0 + warnings.append("amount_unparseable") + direction = DIRECTION_OUT + else: + sign, body = amount_match.group(1), amount_match.group(2) + amount = amount_to_kopecks(body.replace(" ", "").replace(",", ".")) + direction = DIRECTION_IN if sign == "+" else DIRECTION_OUT + return SourceRow( + external_id=external_id or None, + occurred_on=occurred, + posted_on=None, + amount_kopecks=amount, + direction=direction, + description=description or None, + mcc=None, + bank_category=None, + location=None, + row_ref=row_ref, + warnings=tuple(warnings), + ) + + +def _ozon_normalize(text: str) -> str: + return text.replace("\u00a0", " ").lower() + + +def _ozon_cell(cell: object) -> str: + if cell is None: + return "" + return str(cell).replace("\u00a0", " ").strip() class OzonCreditPdfAdapter(OzonPdfAdapterBase): diff --git a/backend/app/imports/adapters/rocketbank_pdf.py b/backend/app/imports/adapters/rocketbank_pdf.py index d039731..3b7dd44 100644 --- a/backend/app/imports/adapters/rocketbank_pdf.py +++ b/backend/app/imports/adapters/rocketbank_pdf.py @@ -1,12 +1,288 @@ from __future__ import annotations -from app.imports.adapters.ozon_pdf import OzonPdfAdapterBase +import re +from datetime import date + +from app.imports.adapters.base import ( + DIRECTION_IN, + DIRECTION_OUT, + ROCKETBANK_DEBIT_PDF_VERSION, + SourceRow, + StatementDraft, + UnsupportedStatementLayout, + account_fingerprint, + money_to_kopecks, +) +from app.imports.readers import open_pdf + +MONTH_NAMES = { + "января": 1, + "февраля": 2, + "марта": 3, + "апреля": 4, + "мая": 5, + "июня": 6, + "июля": 7, + "августа": 8, + "сентября": 9, + "октября": 10, + "ноября": 11, + "декабря": 12, +} + +DAY_HEADING_RE = re.compile( + r"^(\d{1,2})\s+(января|февраля|марта|апреля|мая|июня|июля|августа|сентября|" + r"октября|ноября|декабря)\s+(\d{4})\s+баланс на начало дня:\s*(-?[\d\s]+,\d{2})\s*rur$", + re.IGNORECASE, +) +RECORD_RE = re.compile( + r"^(\d{1,2}:\d{2})\s+(\d{6,})\s+(.*?)([+-]?\s?[\d\s]+,\d{2})\s*rur$", + re.IGNORECASE, +) +HEADER_VALUES_RE = re.compile( + r"^(-?[\d\s]+,\d{2})\s*rur\s+(-?[\d\s]+,\d{2})\s*rur(?:\s+(\d{2}\.\d{2}\.\d{4}))?$", + re.IGNORECASE, +) +PERIOD_RE = re.compile(r"^с\s+(\d{2}\.\d{2}\.\d{4})\s+по\s+(\d{2}\.\d{2}\.\d{4})$", re.IGNORECASE) +AMOUNT_RE = re.compile(r"^([+-]?)\s*(\d[\d\s]*,\d{2})$") +MCC_RE = re.compile(r"\bmcc\s*:?\s*(\d{4})\b", re.IGNORECASE) +LOCATION_RE = re.compile(r"rur,\s*([^,]+?),\s*mcc\b", re.IGNORECASE) -class RocketbankDebitPdfAdapter(OzonPdfAdapterBase): - """Rocketbank debit statement contract: same text-layer line scanning shape.""" +class RocketbankDebitPdfAdapter: + """Rocketbank debit statement contract, evidenced against the real document. + + Independent text-layer state machine: every page repeats the statement + header and column captions; day sections start with a Russian day heading + ('DD MONTH YYYY БАЛАНС НА НАЧАЛО ДНЯ: X RUR'); each record is a + 'time + operation code + multiline description + signed RUR amount' line + whose description column starts slightly above and continues below that + line, so the surrounding non-structural lines are folded into one + description. Daily totals and repeated captions are ignored. Statement + period, account, opening/closing balances and declared inflow/outflow + totals come from the first-page header block. + """ adapter_id = "rocketbank_debit_pdf" product = "debit" - bank_markers = ("рокетбанк", "rocketbank") - product_markers = ("дебетов", "дебетовая") + parser_version = ROCKETBANK_DEBIT_PDF_VERSION + bank_code = "rocketbank" + currency = "RUB" + + title_marker = "выписка по счету" + card_re = re.compile(r"дебетовая карта рокет\s*(\d{10,30})\s*rur", re.IGNORECASE) + column_caption = "время код операции описание сумма" + issued_prefix = "дата выдачи выписки:" + page_footer_prefix = "страница" + daily_total_prefixes = ("всего поступлений:", "всего расходов:") + opening_label = "входящий остаток" + closing_label = "исходящий остаток" + max_fold_lines = 6 + + def detect(self, raw: bytes) -> bool: + try: + with open_pdf(raw) as pdf: + if not pdf.pages: + return False + text = pdf.pages[0].extract_text() or "" + except Exception: + return False + lowered = text.lower() + return self.title_marker in lowered and "дебетовая карта рокет" in lowered + + def parse(self, raw: bytes) -> StatementDraft: + lines: list[tuple[int, str]] = [] + with open_pdf(raw) as pdf: + if not pdf.pages: + raise UnsupportedStatementLayout("PDF has no pages") + for page_number, page in enumerate(pdf.pages, start=1): + text = page.extract_text() or "" + if page_number == 1 and not text.strip(): + raise UnsupportedStatementLayout( + "PDF has no extractable text layer; OCR is not supported" + ) + for line in text.splitlines(): + stripped = line.strip() + if stripped: + lines.append((page_number, stripped)) + return self._draft_from_lines(lines) + + def _draft_from_lines(self, lines: list[tuple[int, str]]) -> StatementDraft: + blob = "\n".join(line.lower() for _, line in lines) + if self.title_marker not in blob: + raise UnsupportedStatementLayout("Rocketbank statement marker not found") + card_match = self.card_re.search(blob) + if card_match is None: + raise UnsupportedStatementLayout("Rocketbank card line not found") + + period_start: date | None = None + period_end: date | None = None + opening: int | None = None + closing: int | None = None + declared_inflow: int | None = None + declared_outflow: int | None = None + rows: list[SourceRow] = [] + current_day: date | None = None + pending: _PendingRecord | None = None + pre_lines: list[tuple[int, str]] = [] + record_ordinals: dict[int, int] = {} + + index = 0 + while index < len(lines): + page, line = lines[index] + low = line.lower() + if ( + low == self.title_marker + or low.startswith(self.issued_prefix) + or low.startswith(self.page_footer_prefix) + or low == self.column_caption + or low.startswith(self.daily_total_prefixes) + ): + index += 1 + continue + period_match = PERIOD_RE.match(low) + if period_match is not None and period_start is None: + period_start = _date_from(period_match.group(1)) + period_end = _date_from(period_match.group(2)) + index += 1 + continue + if self.opening_label in low or self.closing_label in low: + if index + 1 >= len(lines): + raise UnsupportedStatementLayout("Rocketbank balance values missing") + values_match = HEADER_VALUES_RE.match(lines[index + 1][1].lower()) + if values_match is None: + raise UnsupportedStatementLayout("Rocketbank balance values missing") + first = money_to_kopecks(values_match.group(1)) + second = money_to_kopecks(values_match.group(2)) + if self.opening_label in low: + opening, declared_inflow = first, second + else: + closing, declared_outflow = first, second + index += 2 + continue + day_match = DAY_HEADING_RE.match(low) + if day_match is not None: + if pending is not None: + rows.append(self._record(pending, current_day)) + pending = None + pre_lines = [] + current_day = date( + int(day_match.group(3)), + MONTH_NAMES[day_match.group(2).lower()], + int(day_match.group(1)), + ) + index += 1 + continue + record_match = RECORD_RE.match(line) + if record_match is not None: + if pending is not None: + rows.append(self._record(pending, current_day)) + ordinal = record_ordinals.get(page, 0) + 1 + record_ordinals[page] = ordinal + pending = _PendingRecord( + row_ref=f"page{page}:record{ordinal}", + code=record_match.group(2), + description=record_match.group(3).strip(), + amount_text=record_match.group(4), + pre_lines=pre_lines, + ) + pre_lines = [] + index += 1 + continue + if pending is not None: + pending.post_lines.append((page, line)) + index += 1 + continue + pre_lines.append((page, line)) + if len(pre_lines) > self.max_fold_lines: + raise UnsupportedStatementLayout("unrecognized content between Rocketbank sections") + index += 1 + if pending is not None: + rows.append(self._record(pending, current_day)) + pre_lines = [] + if pre_lines: + raise UnsupportedStatementLayout("unrecognized trailing Rocketbank content") + if ( + period_start is None + or opening is None + or closing is None + or declared_inflow is None + or declared_outflow is None + ): + raise UnsupportedStatementLayout("Rocketbank statement header block not found") + if not rows: + raise UnsupportedStatementLayout("no Rocketbank operation rows found") + return StatementDraft( + adapter_id=self.adapter_id, + parser_version=self.parser_version, + bank_code=self.bank_code, + product=self.product, + account_fingerprint=account_fingerprint(self.bank_code, card_match.group(1)), + currency=self.currency, + period_start=period_start, + period_end=period_end, + opening_balance_kopecks=opening, + closing_balance_kopecks=closing, + rows=tuple(rows), + declared_inflow_kopecks=declared_inflow, + declared_outflow_kopecks=declared_outflow, + declared_hold_kopecks=None, + ) + + def _record( + self, + pending: _PendingRecord, + current_day: date | None, + ) -> SourceRow: + if current_day is None: + raise UnsupportedStatementLayout("Rocketbank record outside a day section") + chunks = [text for _, text in pending.pre_lines] + chunks.append(pending.description) + chunks.extend(text for _, text in pending.post_lines) + description = " ".join(" ".join(chunk.split()) for chunk in chunks if chunk.strip()) + amount_match = AMOUNT_RE.match(" ".join(pending.amount_text.split())) + if amount_match is None: + raise UnsupportedStatementLayout("Rocketbank record amount missing") + sign = amount_match.group(1) + amount = money_to_kopecks(amount_match.group(2)) + direction = DIRECTION_OUT if sign == "-" else DIRECTION_IN + mcc_match = MCC_RE.search(description) + location_match = LOCATION_RE.search(description) + return SourceRow( + external_id=pending.code, + occurred_on=current_day, + posted_on=None, + amount_kopecks=amount, + direction=direction, + description=description or None, + mcc=int(mcc_match.group(1)) if mcc_match is not None else None, + bank_category=None, + location=location_match.group(1).strip() if location_match is not None else None, + row_ref=pending.row_ref, + warnings=(), + ) + + +class _PendingRecord: + __slots__ = ("row_ref", "code", "description", "amount_text", "pre_lines", "post_lines") + + def __init__( + self, + *, + row_ref: str, + code: str, + description: str, + amount_text: str, + pre_lines: list[tuple[int, str]], + ) -> None: + self.row_ref = row_ref + self.code = code + self.description = description + self.amount_text = amount_text + self.pre_lines = pre_lines + self.post_lines: list[tuple[int, str]] = [] + + +def _date_from(text: str) -> date: + day, month, year = (int(part) for part in text.split(".")) + return date(year, month, day) diff --git a/backend/app/imports/models.py b/backend/app/imports/models.py index d50632b..884c1bd 100644 --- a/backend/app/imports/models.py +++ b/backend/app/imports/models.py @@ -1,5 +1,5 @@ import uuid -from datetime import date +from datetime import date, datetime from typing import Any from sqlalchemy import ( @@ -18,7 +18,7 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Mapped, mapped_column -from app.db.base import Base, PkMixin, TimestampMixin +from app.db.base import Base, PkMixin, TimestampMixin, UTCDateTime, utcnow DOCUMENT_STATUSES = ( "uploaded", @@ -34,6 +34,9 @@ RUN_STATUSES = ("pending", "running", "succeeded", "failed") CANDIDATE_KINDS = ("duplicate", "transfer") CANDIDATE_CONFIDENCES = ("unique", "possible") CANDIDATE_RESOLUTIONS = ("pending", "is_duplicate", "is_new") +BANK_STATUSES = ("posted", "hold") +HOLD_RESOLUTIONS = ("matched_posted", "confirmed_as_posted", "discarded_as_cancelled") +HOLD_MANUAL_RESOLUTIONS = ("confirmed_as_posted", "discarded_as_cancelled") class StatementDocument(Base, PkMixin, TimestampMixin): @@ -99,6 +102,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin): __tablename__ = "source_records" __table_args__ = ( + CheckConstraint("bank_status IN ('posted', 'hold')", name="bank_status_valid"), ForeignKeyConstraint( ["space_id", "document_id"], ["statement_documents.space_id", "statement_documents.id"], @@ -128,6 +132,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin): posted_on: Mapped[date | None] = mapped_column(Date, nullable=True) amount_kopecks: Mapped[int] = mapped_column(BigInteger, nullable=False) direction: Mapped[str] = mapped_column(String(8), nullable=False) + bank_status: Mapped[str] = mapped_column(String(8), nullable=False, default="posted") description: Mapped[str | None] = mapped_column(String(500), nullable=True) mcc: Mapped[int | None] = mapped_column(Integer, nullable=True) bank_category: Mapped[str | None] = mapped_column(String(100), nullable=True) @@ -209,3 +214,57 @@ class DedupCandidate(Base, PkMixin, TimestampMixin): decided_by_user_id: Mapped[uuid.UUID | None] = mapped_column( Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True ) + + +class HoldResolution(Base, PkMixin, TimestampMixin): + """Append-only, tenant-scoped settlement of one HOLD source record. + + The extracted HOLD row itself is never mutated. At most one resolution + exists per hold source record; automatic matches (`matched_posted`) are + created only when a later statement is confirmed, and manual actions + (`confirmed_as_posted`, `discarded_as_cancelled`) always carry a reason. + """ + + __tablename__ = "hold_resolutions" + __table_args__ = ( + CheckConstraint( + "resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')", + name="resolution_valid", + ), + CheckConstraint( + "NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') " + "AND (reason IS NULL OR reason = ''))", + name="manual_reason_required", + ), + ForeignKeyConstraint( + ["space_id", "hold_source_record_id"], + ["source_records.space_id", "source_records.id"], + ondelete="CASCADE", + name="fk_hold_resolutions_hold_tenant", + ), + ForeignKeyConstraint( + ["space_id", "resolving_source_record_id"], + ["source_records.space_id", "source_records.id"], + name="fk_hold_resolutions_resolving_tenant", + ), + ForeignKeyConstraint( + ["space_id", "operation_id"], + ["operations.space_id", "operations.id"], + name="fk_hold_resolutions_operation_tenant", + ), + UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"), + UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"), + ) + + space_id: Mapped[uuid.UUID] = mapped_column( + Uuid, ForeignKey("financial_spaces.id", ondelete="CASCADE"), nullable=False, index=True + ) + hold_source_record_id: Mapped[uuid.UUID] = mapped_column(Uuid, nullable=False, index=True) + resolution: Mapped[str] = mapped_column(String(24), nullable=False) + resolving_source_record_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True) + operation_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True, index=True) + resolved_by_user_id: Mapped[uuid.UUID | None] = mapped_column( + Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + reason: Mapped[str | None] = mapped_column(String(500), nullable=True) + resolved_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False, default=utcnow) diff --git a/backend/app/imports/router.py b/backend/app/imports/router.py index 94d96b0..ceaaa52 100644 --- a/backend/app/imports/router.py +++ b/backend/app/imports/router.py @@ -33,6 +33,14 @@ class ConfirmBody(BaseModel): reason: str | None = None +class HoldResolveBody(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + action: str + reason: str + account_id: uuid.UUID | None = Field(default=None, alias="accountId") + + def summary(document: Any) -> dict: return { "id": document.id, @@ -100,21 +108,34 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict: document = service._document(db, space_id=context.space.id, document_id=document_id) base = summary(document) reconciliation = raw.get("reconciliation") or {} - rows = [ - { - "id": item["id"], - "externalId": item["external_id"], - "occurredOn": item["occurred_on"], - "amountKopecks": item["amount_kopecks"], - "direction": item["direction"], - "description": item["description"], - "payeeHint": None, - "warnings": (item["warnings"] or {}).get("codes", []), - "decision": "pending" if item["critical"] else item["review_decision"], - "duplicateOfOperationId": item["duplicate_operation_id"], - } - for item in raw["records"] - ] + rows = [] + for item in raw["records"]: + hold = item.get("hold_resolution") + rows.append( + { + "id": item["id"], + "externalId": item["external_id"], + "occurredOn": item["occurred_on"], + "amountKopecks": item["amount_kopecks"], + "direction": item["direction"], + "bankStatus": item.get("bank_status", "posted"), + "description": item["description"], + "payeeHint": None, + "warnings": (item["warnings"] or {}).get("codes", []), + "decision": "pending" if item["critical"] else item["review_decision"], + "duplicateOfOperationId": item["duplicate_operation_id"], + "holdResolution": None + if hold is None + else { + "resolution": hold["resolution"], + "resolvingSourceRecordId": hold["resolving_source_record_id"], + "operationId": hold["operation_id"], + "resolvedByUserId": hold["resolved_by_user_id"], + "reason": hold["reason"], + "resolvedAt": hold["resolved_at"], + }, + } + ) blockers = [ {"code": "IMPORT_FIELD_UNRESOLVED", "message": None, "rowId": str(item["id"])} for item in raw["records"] @@ -136,7 +157,16 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict: "status": status, "openingBalanceKopecks": reconciliation.get("opening_balance_kopecks"), "closingBalanceKopecks": reconciliation.get("closing_balance_kopecks"), - "computedClosingKopecks": None, + "computedClosingKopecks": reconciliation.get("computed_closing_kopecks"), + "declaredInflowKopecks": reconciliation.get("declared_inflow_kopecks"), + "declaredOutflowKopecks": reconciliation.get("declared_outflow_kopecks"), + "declaredHoldKopecks": reconciliation.get("declared_hold_kopecks"), + "parsedPostedInflowKopecks": reconciliation.get("parsed_posted_inflow_kopecks"), + "parsedPostedOutflowKopecks": reconciliation.get("parsed_posted_outflow_kopecks"), + "parsedHoldTotalKopecks": reconciliation.get("parsed_hold_total_kopecks"), + "postedTotalsMatched": reconciliation.get("posted_totals_matched"), + "holdTotalsMatched": reconciliation.get("hold_totals_matched"), + "balanceMatched": reconciliation.get("balance_matched"), "reason": reconciliation.get("reason"), }, "rows": rows, @@ -181,6 +211,26 @@ def decide_row( return detail(document_id, db, context) +@router.post("/{document_id}/rows/{row_id}/resolve") +def resolve_hold( + document_id: uuid.UUID, + row_id: uuid.UUID, + body: HoldResolveBody, + db: Db, + context: Context, + _csrf: Csrf, +) -> dict: + service.resolve_hold( + db, + space_id=context.space.id, + document_id=document_id, + source_record_id=row_id, + user=context.user, + **body.model_dump(by_alias=False), + ) + return detail(document_id, db, context) + + @router.post("/{document_id}/confirm") def confirm( document_id: uuid.UUID, diff --git a/backend/app/imports/service.py b/backend/app/imports/service.py index e904709..7c69577 100644 --- a/backend/app/imports/service.py +++ b/backend/app/imports/service.py @@ -9,8 +9,9 @@ from sqlalchemy.orm import Session from app.audit import service as audit from app.config import get_settings +from app.db.base import utcnow from app.db.session import session_scope -from app.errors import AppError, conflict, not_found +from app.errors import AppError, bad_request, conflict, not_found from app.identity.models import User from app.imports.adapters import detect_adapter, magic_kind from app.imports.adapters.base import SourceRow, StatementDraft, record_fingerprint @@ -18,6 +19,7 @@ from app.imports.blob import get_blob_store, new_object_key from app.imports.crypto import decrypt_file, encrypt_file from app.imports.models import ( DedupCandidate, + HoldResolution, ParseRun, SourceRecord, SourceRecordLink, @@ -31,6 +33,8 @@ from app.ledger.service import ( ensure_period_open_for_date, ) +HOLD_MANUAL_ACTIONS = ("confirm_as_posted", "discard_as_cancelled") + def _document(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID) -> StatementDocument: document = ( @@ -128,12 +132,16 @@ def _draft_json(draft: StatementDraft) -> dict[str, Any]: def _source_from_row( *, space_id: uuid.UUID, document_id: uuid.UUID, run_id: uuid.UUID, row: SourceRow ) -> SourceRecord: - critical_warnings = {"occurred_date_missing", "amount_unparseable"} + critical_warnings = {"occurred_date_missing", "amount_unparseable", "bank_status_invalid"} warnings = list(row.warnings) if row.direction not in {"in", "out"}: warnings.append("direction_invalid") if row.amount_kopecks == 0: warnings.append("amount_zero") + bank_status = getattr(row, "bank_status", "posted") + if bank_status not in {"posted", "hold"}: + warnings.append("bank_status_invalid") + bank_status = "posted" return SourceRecord( space_id=space_id, document_id=document_id, @@ -143,6 +151,7 @@ def _source_from_row( posted_on=row.posted_on, amount_kopecks=abs(row.amount_kopecks), direction=row.direction, + bank_status=bank_status, description=row.description, mcc=row.mcc, bank_category=row.bank_category, @@ -158,9 +167,66 @@ def _source_from_row( ) +def _reconciliation_summary(draft: Any) -> dict[str, Any]: + """Reconcile parsed rows against declared bank controls with stable diagnostics. + + Posted rows are compared with the declared inflow/outflow controls, HOLD + rows with the declared pending total, and where both balances exist the + balance equation must hold. Every check keeps an explicit tri-state field: + True (matched), False (mismatch, review-blocking), None (not declared). + """ + rows = list(draft.rows) + posted = [row for row in rows if getattr(row, "bank_status", "posted") == "posted"] + holds = [row for row in rows if getattr(row, "bank_status", "posted") == "hold"] + posted_inflow = sum(row.amount_kopecks for row in posted if row.direction == "in") + posted_outflow = sum(row.amount_kopecks for row in posted if row.direction == "out") + hold_total = sum(abs(row.amount_kopecks) for row in holds) + declared_inflow = getattr(draft, "declared_inflow_kopecks", None) + declared_outflow = getattr(draft, "declared_outflow_kopecks", None) + declared_hold = getattr(draft, "declared_hold_kopecks", None) + full = draft.opening_balance_kopecks is not None and draft.closing_balance_kopecks is not None + checks: list[bool] = [] + balance_matched: bool | None = None + computed_closing: int | None = None + if full: + computed_closing = int(draft.opening_balance_kopecks) + posted_inflow - posted_outflow + balance_matched = computed_closing == int(draft.closing_balance_kopecks) + checks.append(balance_matched) + posted_totals_matched: bool | None = None + if declared_inflow is not None or declared_outflow is not None: + posted_totals_matched = (declared_inflow is None or declared_inflow == posted_inflow) and ( + declared_outflow is None or declared_outflow == posted_outflow + ) + checks.append(posted_totals_matched) + hold_totals_matched: bool | None = None + if declared_hold is not None: + hold_totals_matched = declared_hold == hold_total + checks.append(hold_totals_matched) + return { + "currency": draft.currency, + "opening_balance_kopecks": draft.opening_balance_kopecks, + "closing_balance_kopecks": draft.closing_balance_kopecks, + "full": full, + "declared_inflow_kopecks": declared_inflow, + "declared_outflow_kopecks": declared_outflow, + "declared_hold_kopecks": declared_hold, + "parsed_posted_inflow_kopecks": posted_inflow, + "parsed_posted_outflow_kopecks": posted_outflow, + "parsed_hold_total_kopecks": hold_total, + "posted_totals_matched": posted_totals_matched, + "hold_totals_matched": hold_totals_matched, + "balance_matched": balance_matched, + "computed_closing_kopecks": computed_closing, + "matched": all(checks) if checks else None, + "confirmed_without_full_reconciliation": False, + } + + def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun) -> bool: pending = False for source in db.query(SourceRecord).filter_by(parse_run_id=run.id).all(): + if source.bank_status == "hold": + continue duplicate = None if source.external_id: matches = ( @@ -170,6 +236,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun SourceRecord.space_id == document.space_id, SourceRecord.id != source.id, SourceRecord.external_id == source.external_id, + SourceRecord.bank_status == "posted", ParseRun.adapter_id == run.adapter_id, ) .all() @@ -183,6 +250,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun SourceRecord.id != source.id, SourceRecord.record_fp == source.record_fp, SourceRecord.direction == source.direction, + SourceRecord.bank_status == "posted", ) .all() ) @@ -212,6 +280,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun SourceRecord.id != source.id, SourceRecord.amount_kopecks == source.amount_kopecks, SourceRecord.direction == opposite, + SourceRecord.bank_status == "posted", SourceRecord.occurred_on >= source.occurred_on - timedelta(days=1), SourceRecord.occurred_on <= source.occurred_on + timedelta(days=1), ) @@ -234,6 +303,105 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun return pending +def _match_hold_resolutions( + db: Session, *, document: StatementDocument, run: ParseRun +) -> dict[uuid.UUID, uuid.UUID]: + """Conservatively match unresolved confirmed-doc HOLDs to new posted records. + + Candidate pairs are searched only within the same space, mapped account, + and statement contract (adapter). A stable external ID wins; otherwise a + single unambiguous date/amount/direction/normalized-description + fingerprint match is required. Any overlap ambiguity leaves every + involved HOLD unresolved (review-required). Resolutions are written only + when the new statement is confirmed. + """ + if run.adapter_id is None or document.account_id is None: + return {} + holds = ( + db.query(SourceRecord) + .join( + StatementDocument, + (StatementDocument.space_id == SourceRecord.space_id) + & (StatementDocument.id == SourceRecord.document_id), + ) + .outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id) + .filter( + SourceRecord.space_id == document.space_id, + SourceRecord.bank_status == "hold", + SourceRecord.parse_run_id != run.id, + StatementDocument.status == "confirmed", + StatementDocument.active_parse_run_id == SourceRecord.parse_run_id, + StatementDocument.adapter_id == run.adapter_id, + StatementDocument.account_id == document.account_id, + HoldResolution.id.is_(None), + ) + .all() + ) + if not holds: + return {} + posted = ( + db.query(SourceRecord) + .filter_by(parse_run_id=run.id, bank_status="posted") + .filter(SourceRecord.review_decision != "discard") + .all() + ) + by_external: dict[str, list[SourceRecord]] = {} + by_fingerprint: dict[tuple[str, str], list[SourceRecord]] = {} + for record in posted: + if record.external_id: + by_external.setdefault(record.external_id, []).append(record) + by_fingerprint.setdefault((record.record_fp, record.direction), []).append(record) + candidates: dict[uuid.UUID, set[uuid.UUID]] = {} + for hold in holds: + matched: set[uuid.UUID] = set() + if hold.external_id: + matched.update(item.id for item in by_external.get(hold.external_id, [])) + matched.update(item.id for item in by_fingerprint.get((hold.record_fp, hold.direction), [])) + if len(matched) == 1: + candidates[hold.id] = matched + claimed: dict[uuid.UUID, int] = {} + for posted_ids in candidates.values(): + for posted_id in posted_ids: + claimed[posted_id] = claimed.get(posted_id, 0) + 1 + return { + hold_id: next(iter(posted_ids)) + for hold_id, posted_ids in candidates.items() + if all(claimed[posted_id] == 1 for posted_id in posted_ids) + } + + +def _apply_hold_matches( + db: Session, + *, + document: StatementDocument, + run: ParseRun, + user: User, + matches: dict[uuid.UUID, uuid.UUID], +) -> None: + for hold_id, posted_id in matches.items(): + db.add( + HoldResolution( + space_id=document.space_id, + hold_source_record_id=hold_id, + resolution="matched_posted", + resolving_source_record_id=posted_id, + resolved_at=utcnow(), + ) + ) + if matches: + db.flush() + audit.record( + db, + event_type="import.holds_matched_posted", + space_id=document.space_id, + user_id=user.id, + detail={ + "document_id": str(document.id), + "holds": sorted(str(hold_id) for hold_id in matches), + }, + ) + + def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> str: with session_scope(space_id) as db: document = _document(db, space_id=space_id, document_id=document_id) @@ -273,23 +441,8 @@ def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> s document.adapter_id = adapter.adapter_id document.account_fingerprint = draft.account_fingerprint document.active_parse_run_id = run.id - document.reconciliation = { - "currency": draft.currency, - "opening_balance_kopecks": draft.opening_balance_kopecks, - "closing_balance_kopecks": draft.closing_balance_kopecks, - "full": draft.opening_balance_kopecks is not None - and draft.closing_balance_kopecks is not None, - "confirmed_without_full_reconciliation": False, - } - if document.reconciliation["full"]: - signed_total = sum( - row.amount_kopecks if row.direction == "in" else -row.amount_kopecks - for row in draft.rows - ) - computed = int(draft.opening_balance_kopecks or 0) + signed_total - document.reconciliation["computed_closing_kopecks"] = computed - document.reconciliation["matched"] = computed == draft.closing_balance_kopecks - if draft.account_fingerprint and document.account_id is None: + document.reconciliation = _reconciliation_summary(draft) + if document.account_id is None and draft.account_fingerprint: matches = ( db.query(Account) .filter_by(space_id=space_id, fingerprint=draft.account_fingerprint) @@ -351,6 +504,25 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID) .order_by(SourceRecord.created_at) .all() ) + hold_ids = [record.id for record in records] + resolutions: dict[uuid.UUID, dict[str, Any]] = {} + if hold_ids: + for item in ( + db.query(HoldResolution) + .filter( + HoldResolution.space_id == space_id, + HoldResolution.hold_source_record_id.in_(hold_ids), + ) + .all() + ): + resolutions[item.hold_source_record_id] = { + "resolution": item.resolution, + "resolving_source_record_id": item.resolving_source_record_id, + "operation_id": item.operation_id, + "resolved_by_user_id": item.resolved_by_user_id, + "reason": item.reason, + "resolved_at": item.resolved_at, + } return { "id": document.id, "status": document.status, @@ -367,11 +539,13 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID) "posted_on": item.posted_on, "amount_kopecks": item.amount_kopecks, "direction": item.direction, + "bank_status": item.bank_status, "description": item.description, "critical": item.critical, "warnings": item.field_warnings, "review_decision": item.review_decision, "duplicate_operation_id": item.duplicate_operation_id, + "hold_resolution": resolutions.get(item.id), } for item in records ], @@ -489,6 +663,119 @@ def set_review_confirmation( return document +def resolve_hold( + db: Session, + *, + space_id: uuid.UUID, + document_id: uuid.UUID, + source_record_id: uuid.UUID, + user: User, + action: str, + reason: str | None, + account_id: uuid.UUID | None = None, +) -> HoldResolution: + """Manually settle one HOLD: book it as a normal operation or discard it.""" + document = _document(db, space_id=space_id, document_id=document_id) + if document.status != "confirmed": + raise conflict( + "IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED", + "Holds can only be resolved after the import is confirmed", + ) + if action not in HOLD_MANUAL_ACTIONS: + raise bad_request("IMPORT_HOLD_ACTION_INVALID", "Unknown hold resolution action") + if reason is None or not reason.strip(): + raise conflict("IMPORT_HOLD_REASON_REQUIRED", "A resolution reason is required") + source = ( + db.query(SourceRecord) + .filter_by( + id=source_record_id, + space_id=space_id, + parse_run_id=document.active_parse_run_id, + ) + .one_or_none() + ) + if source is None: + raise not_found("IMPORT_HOLD_NOT_FOUND", "Import row not found") + if source.bank_status != "hold": + raise conflict("IMPORT_HOLD_INVALID", "Only hold rows can be resolved manually") + existing = db.query(HoldResolution.id).filter_by(hold_source_record_id=source.id).one_or_none() + if existing is not None: + raise conflict("IMPORT_HOLD_ALREADY_RESOLVED", "Hold is already resolved") + if source.occurred_on is None: + raise conflict("IMPORT_HOLD_DATE_MISSING", "Hold has no usable operation date") + if source.direction not in {"in", "out"}: + raise conflict("IMPORT_HOLD_DIRECTION_INVALID", "Hold direction is invalid") + resolution = HoldResolution( + space_id=space_id, + hold_source_record_id=source.id, + resolution="confirmed_as_posted" + if action == "confirm_as_posted" + else "discarded_as_cancelled", + resolved_by_user_id=user.id, + reason=reason.strip(), + resolved_at=utcnow(), + ) + if action == "confirm_as_posted": + mapped_account_id = account_id or document.account_id + if mapped_account_id is None: + raise conflict( + "IMPORT_HOLD_ACCOUNT_REQUIRED", + "A mapped account is required to confirm a hold as posted", + ) + account = db.query(Account).filter_by(id=mapped_account_id, space_id=space_id).one_or_none() + if account is None: + raise not_found("LEDGER_ACCOUNT_NOT_FOUND", "Account not found") + if source.direction == "in": + operation = create_income( + db, + space_id=space_id, + user=user, + account_id=account.id, + amount_kopecks=source.amount_kopecks, + occurred_on=source.occurred_on, + posted_on=source.posted_on, + description=source.description, + request_key=f"import:{source.id}", + ) + else: + operation = create_expense( + db, + space_id=space_id, + user=user, + account_id=account.id, + amount_kopecks=source.amount_kopecks, + occurred_on=source.occurred_on, + posted_on=source.posted_on, + description=source.description, + request_key=f"import:{source.id}", + ) + db.add( + SourceRecordLink( + space_id=space_id, + source_record_id=source.id, + operation_id=operation.id, + kind="hold_confirmed", + ) + ) + resolution.operation_id = operation.id + db.add(resolution) + db.flush() + audit.record( + db, + event_type="import.hold_confirmed_as_posted" + if action == "confirm_as_posted" + else "import.hold_discarded_as_cancelled", + space_id=space_id, + user_id=user.id, + detail={ + "document_id": str(document.id), + "source_record_id": str(source.id), + "reason": reason.strip(), + }, + ) + return resolution + + def confirm_document( db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID, user: User ) -> StatementDocument: @@ -503,7 +790,7 @@ def confirm_document( "confirmed_without_full_reconciliation" ): raise conflict("IMPORT_RECONCILIATION_REQUIRED", "Reconciliation must be confirmed") - if reconciliation.get("full") and reconciliation.get("matched") is False: + if reconciliation.get("matched") is False: raise conflict("IMPORT_RECONCILIATION_MISMATCH", "Statement arithmetic does not reconcile") pending = ( db.query(DedupCandidate.id) @@ -519,7 +806,18 @@ def confirm_document( records = db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).all() if any(item.critical for item in records): raise conflict("IMPORT_FIELDS_UNRESOLVED", "Critical source fields remain unresolved") + run = db.get(ParseRun, document.active_parse_run_id) + if run is not None: + _apply_hold_matches( + db, + document=document, + run=run, + user=user, + matches=_match_hold_resolutions(db, document=document, run=run), + ) for source in records: + if source.bank_status == "hold": + continue if source.review_decision == "discard": continue if source.review_decision == "transfer": @@ -755,10 +1053,21 @@ def revert_document( document = _document(db, space_id=space_id, document_id=document_id) if document.status != "confirmed": return document + if document.active_parse_run_id: + db.query(HoldResolution).filter( + HoldResolution.space_id == space_id, + HoldResolution.resolution == "matched_posted", + HoldResolution.resolving_source_record_id.in_( + db.query(SourceRecord.id).filter_by(parse_run_id=document.active_parse_run_id) + ), + ).delete(synchronize_session=False) links = ( db.query(SourceRecordLink) .join(SourceRecord, SourceRecord.id == SourceRecordLink.source_record_id) - .filter(SourceRecord.parse_run_id == document.active_parse_run_id) + .filter( + SourceRecord.parse_run_id == document.active_parse_run_id, + SourceRecordLink.kind != "hold_confirmed", + ) .all() ) transfer_candidates = ( diff --git a/backend/pyproject.toml b/backend/pyproject.toml index aef84be..f0dca1d 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -39,6 +39,10 @@ package = false pythonpath = ["."] testpaths = ["tests"] filterwarnings = ["ignore::DeprecationWarning"] +addopts = "-m 'not private_fixtures'" +markers = [ + "private_fixtures: real-statement private suite; run explicitly with -m private_fixtures (never in CI)", +] [tool.ruff] line-length = 100 diff --git a/backend/tests/test_adapter_contracts.py b/backend/tests/test_adapter_contracts.py new file mode 100644 index 0000000..8592484 --- /dev/null +++ b/backend/tests/test_adapter_contracts.py @@ -0,0 +1,825 @@ +"""Sanitized synthetic contract tests for the evidenced statement adapters. + +These tests never touch real statements: every input below is synthetic and +tracked. Real-file validation lives in the separately collected private +suite (tests/test_private_fixtures.py, marker `private_fixtures`). +""" + +import io +from datetime import date + +import pytest +from openpyxl import Workbook + +from app.errors import AppError +from app.imports.adapters import ADAPTERS, detect_adapter, magic_kind +from app.imports.adapters import alfa_xlsx as alfa_module +from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter +from app.imports.adapters.base import ( + ALFA_DEBIT_XLSX_VERSION, + BANK_STATUS_HOLD, + OZON_DEBIT_PDF_VERSION, + PARSER_VERSION, + ROCKETBANK_DEBIT_PDF_VERSION, + SourceRow, + StatementDraft, + UnsupportedStatementLayout, + account_fingerprint, + money_to_kopecks, +) +from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter +from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter +from app.imports.service import _reconciliation_summary + +NBSP = "\u00a0" + + +def make_source_row(**overrides: object) -> SourceRow: + values: dict[str, object] = { + "external_id": None, + "occurred_on": date(2026, 1, 1), + "posted_on": date(2026, 1, 2), + "amount_kopecks": 100, + "direction": "out", + "description": None, + "mcc": None, + "bank_category": None, + "location": None, + "row_ref": "row1", + } + values.update(overrides) + return SourceRow(**values) # type: ignore[arg-type] + + +def make_draft(**overrides: object) -> StatementDraft: + values: dict[str, object] = { + "adapter_id": "alfa_debit_xlsx", + "parser_version": ALFA_DEBIT_XLSX_VERSION, + "bank_code": "alfa", + "product": "debit", + "account_fingerprint": None, + "currency": "RUB", + "period_start": None, + "period_end": None, + "opening_balance_kopecks": None, + "closing_balance_kopecks": None, + "rows": (), + } + values.update(overrides) + return StatementDraft(**values) # type: ignore[arg-type] + + +class TestBaseContract: + def test_source_row_defaults_to_posted(self) -> None: + assert make_source_row().bank_status == "posted" + + def test_negative_amount_is_rejected(self) -> None: + with pytest.raises(UnsupportedStatementLayout): + make_source_row(amount_kopecks=-1) + + def test_unknown_bank_status_is_rejected(self) -> None: + with pytest.raises(UnsupportedStatementLayout): + make_source_row(bank_status="pending") + + def test_draft_declared_totals_default_to_none(self) -> None: + draft = make_draft() + assert draft.declared_inflow_kopecks is None + assert draft.declared_outflow_kopecks is None + assert draft.declared_hold_kopecks is None + + def test_per_contract_parser_versions(self) -> None: + assert PARSER_VERSION == "1" + assert ALFA_DEBIT_XLSX_VERSION == "2" + assert OZON_DEBIT_PDF_VERSION == "2" + assert ROCKETBANK_DEBIT_PDF_VERSION == "2" + assert AlfaDebitXlsxAdapter.parser_version == ALFA_DEBIT_XLSX_VERSION + assert OzonDebitPdfAdapter.parser_version == OZON_DEBIT_PDF_VERSION + assert RocketbankDebitPdfAdapter.parser_version == ROCKETBANK_DEBIT_PDF_VERSION + + def test_registry_lists_only_verified_contracts(self) -> None: + assert [adapter.adapter_id for adapter in ADAPTERS] == [ + "alfa_debit_xlsx", + "ozon_debit_pdf", + "rocketbank_debit_pdf", + ] + + def test_credit_adapters_stay_present_but_unregistered(self) -> None: + assert AlfaCreditXlsxAdapter.adapter_id == "alfa_credit_xlsx" + assert OzonCreditPdfAdapter.adapter_id == "ozon_credit_pdf" + assert all( + adapter.adapter_id not in {"alfa_credit_xlsx", "ozon_credit_pdf"} + for adapter in ADAPTERS + ) + + def test_detect_adapter_fails_closed_on_unknown_xlsx(self) -> None: + buffer = io.BytesIO() + Workbook().save(buffer) + raw = buffer.getvalue() + with pytest.raises(UnsupportedStatementLayout) as exc: + detect_adapter(raw, "xlsx") + assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT" + + def test_detect_adapter_fails_closed_on_unknown_pdf(self) -> None: + with pytest.raises(UnsupportedStatementLayout): + detect_adapter(minimal_pdf(), "pdf") + + def test_magic_kind(self) -> None: + assert magic_kind(b"PK\x03\x04") == "xlsx" + assert magic_kind(b"%PDF-1.4") == "pdf" + assert magic_kind(b"not a statement") is None + + +class TestMoneyParsing: + def test_plain_and_separated_amounts(self) -> None: + assert money_to_kopecks("123") == 12300 + assert money_to_kopecks(f"1{NBSP}234,56") == 123456 + assert money_to_kopecks("1 234.56") == 123456 + assert money_to_kopecks("-1 234,56 RUR") == -123456 + assert money_to_kopecks("+ 500.00 \u20bd") == 50000 + + def test_malformed_amount_fails_closed_without_leaking_digits(self) -> None: + with pytest.raises(UnsupportedStatementLayout) as exc: + money_to_kopecks("12a3,45") + assert "#" in exc.value.message + assert "12" not in exc.value.message + + +def alfa_workbook_bytes(rows: list[list[object]]) -> bytes: + buffer = io.BytesIO() + workbook = Workbook() + sheet = workbook.worksheets[0] + for row in rows: + sheet.append(row) + workbook.save(buffer) + return buffer.getvalue() + + +def synthetic_alfa_rows() -> list[list[object]]: + return [ + [None, None, None], + ["Выписка по счету"], + ["За период с 01.09.2026 по 04.09.2026"], + ["Номер счета", None, "00000000000000000001"], + ["Дата открытия счета", None, "01.01.2025"], + ["Валюта счета", None, "RUR"], + ["Тип счета", None, "Текущий счёт"], + ["Дата формирования выписки", None, "10.09.2026"], + [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "Поступления", + None, + "1 000,00 RUR", + ], + [None, None, None, None, None, None, None, None, None, None, "Расходы", None, "293,45 RUR"], + [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "Неподтвержденные операции", + None, + "100,00 RUR", + ], + [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "Текущий баланс", + None, + "800,00 RUR", + ], + ["Операции по счету"], + [ + "Дата операции", + "Дата проводки", + None, + "Код", + "Категория", + None, + None, + None, + None, + None, + None, + "Описание", + f"Сумма в{NBSP}валюте{NBSP}счета", + None, + "Статус", + ], + [ + "01.09.2026", + "01.09.2026", + None, + "000000000000001", + "Покупки", + None, + None, + None, + None, + None, + None, + "Магазин 1 MCC: 5411", + "-123,45", + "Выполнен", + ], + [ + "02.09.2026", + "02.09.2026", + None, + "000000000000002", + "Переводы", + None, + None, + None, + None, + None, + None, + "Перевод 2 MCC5912", + "1 000,00", + "Выполнен", + ], + [ + "03.09.2026", + "HOLD", + None, + "000000000000003", + "Покупки", + None, + None, + None, + None, + None, + None, + "Покупка 3", + "-100,00", + ], + [ + "04.09.2026", + "04.09.2026", + None, + "000000000000004", + "Покупки", + None, + None, + None, + None, + None, + None, + "Покупка 4 MCC 5941", + "-50,00", + "Выполнен", + ], + [ + "05.09.2026", + "05.09.2026", + None, + "000000000000005", + "Покупки", + None, + None, + None, + None, + None, + None, + "Покупка 5", + "-20,00", + "Выполнен", + ], + [None], + [ + "(подпись сотрудника АО «БАНК»)", + None, + None, + None, + None, + None, + None, + "Уполномоченное лицо", + ], + ["Страница 1 из 1"], + ] + + +class TestAlfaDebitContract: + def setup_method(self) -> None: + self.adapter = AlfaDebitXlsxAdapter() + self.raw = alfa_workbook_bytes(synthetic_alfa_rows()) + + def test_detect(self) -> None: + assert self.adapter.detect(self.raw) is True + + def test_detect_rejects_workbook_without_contract_header(self) -> None: + raw = alfa_workbook_bytes([["Выписка по счету"], ["Что-то другое", "123"]]) + assert self.adapter.detect(raw) is False + + def test_parse_contract_fields(self) -> None: + draft = self.adapter.parse(self.raw) + assert draft.adapter_id == "alfa_debit_xlsx" + assert draft.parser_version == ALFA_DEBIT_XLSX_VERSION + assert draft.bank_code == "alfa" + assert draft.product == "debit" + assert draft.currency == "RUB" + assert draft.account_fingerprint == account_fingerprint("alfa", "00000000000000000001") + assert draft.declared_inflow_kopecks == 100000 + # canonical posted-only: raw 293,45 minus declared hold 100,00 + assert draft.declared_outflow_kopecks == 19345 + assert draft.declared_hold_kopecks == 10000 + assert draft.closing_balance_kopecks == 80000 + assert draft.opening_balance_kopecks is None + # exact header period fields, not min/max of transaction dates + assert draft.period_start == date(2026, 9, 1) + assert draft.period_end == date(2026, 9, 4) + + def test_missing_period_field_fails_closed(self) -> None: + rows = [list(row) for row in synthetic_alfa_rows()] + rows[2] = ["Что-то другое"] + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter.parse(alfa_workbook_bytes(rows)) + assert "period" in exc.value.message + + def test_inconsistent_expense_control_fails_closed(self) -> None: + rows = [list(row) for row in synthetic_alfa_rows()] + rows[9] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "Расходы", + None, + "50,00 RUR", + ] + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter.parse(alfa_workbook_bytes(rows)) + assert "expense" in exc.value.message + + def test_parse_rows_with_hold(self) -> None: + draft = self.adapter.parse(self.raw) + assert len(draft.rows) == 5 + first, second, hold, fourth, last = draft.rows + assert (first.amount_kopecks, first.direction, first.bank_status) == ( + 12345, + "out", + "posted", + ) + assert first.posted_on == date(2026, 9, 1) + assert (second.amount_kopecks, second.direction) == (100000, "in") + assert hold.bank_status == BANK_STATUS_HOLD + assert hold.posted_on is None + assert hold.amount_kopecks == 10000 + assert hold.direction == "out" + assert last.warnings == () + assert first.row_ref == "sheet0:row15" + assert first.bank_category == "Покупки" + + def test_external_ids_from_code_column(self) -> None: + draft = self.adapter.parse(self.raw) + assert [row.external_id for row in draft.rows] == [ + "000000000000001", + "000000000000002", + "000000000000003", + "000000000000004", + "000000000000005", + ] + + def test_mcc_from_description(self) -> None: + draft = self.adapter.parse(self.raw) + first, second, hold, fourth, last = draft.rows + assert first.mcc == 5411 + assert second.mcc == 5912 + assert hold.mcc is None + assert fourth.mcc == 5941 + assert last.mcc is None + assert all(row.location is None for row in draft.rows) + + def test_footer_is_excluded(self) -> None: + draft = self.adapter.parse(self.raw) + assert all(row.occurred_on is not None for row in draft.rows) + assert draft.rows[-1].occurred_on == date(2026, 9, 5) + + def test_parse_is_deterministic(self) -> None: + assert self.adapter.parse(self.raw) == self.adapter.parse(self.raw) + + def test_service_reconciliation_contract(self) -> None: + draft = self.adapter.parse(self.raw) + summary = _reconciliation_summary(draft) + assert summary["matched"] is True + assert summary["posted_totals_matched"] is True + assert summary["hold_totals_matched"] is True + + def test_service_reconciliation_flags_mismatch(self) -> None: + draft = make_draft( + declared_inflow_kopecks=1000, + declared_outflow_kopecks=1, + declared_hold_kopecks=0, + rows=(make_source_row(amount_kopecks=1000, direction="in"),), + ) + summary = _reconciliation_summary(draft) + assert summary["posted_totals_matched"] is False + assert summary["matched"] is False + + def test_malformed_layout_fails_closed(self) -> None: + raw = alfa_workbook_bytes([["Дата", "Сумма"], ["01.09.2026", "-10,00"]]) + assert self.adapter.detect(raw) is False + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter.parse(raw) + assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT" + + def test_unexpected_mid_table_row_fails_closed(self) -> None: + rows = [list(row) for row in synthetic_alfa_rows()] + rows.insert(16, ["unexpected layout drift"]) + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter.parse(alfa_workbook_bytes(rows)) + assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT" + + def test_unsupported_currency_fails_closed(self) -> None: + rows = [list(row) for row in synthetic_alfa_rows()] + rows[5][2] = "USD" + with pytest.raises(UnsupportedStatementLayout): + self.adapter.parse(alfa_workbook_bytes(rows)) + + def test_row_quota_is_enforced(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(alfa_module, "get_settings_row_cap", lambda: 3) + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter.parse(self.raw) + assert "quota" in exc.value.message + + +def ozon_extracted() -> tuple[list[str], list[list[list[object]]]]: + pages_text = [ + "\n".join( + [ + "ООО «ОЗОН Банк», ИНН 1234567890, город Москва", + "Справка о движении средств", + "Номер лицевого счёта: № 00000000000000000001, открыт 01.01.2025", + "Период выписки: 01.02.2026 – 28.02.2026", + "Входящий остаток: 1000.50 \u20bd", + ] + ), + "", + "\n".join( + [ + "Итого зачислений за период: 3000.00 \u20bd", + "Итого списаний за период: 2500.75 \u20bd", + "Исходящий остаток: 1499.75 \u20bd", + ] + ), + ] + pages_tables = [ + [ + [ + [ + "Дата операции", + "Документ", + "Назначение платежа", + "Сумма операции", + None, + ], + [None, None, None, "Российские рубли", "Валюта"], + [ + "01.02.2026 10:00:00", + "1234567890\n1", + "Оплата товаров\nпо карте 1234", + "- 500.00 \u20bd", + "- 500.00 \u20bd", + ], + ] + ], + [ + [ + [ + "02.02.2026 11:30:00", + "1234567891", + "Перевод\nчерез СБП", + "+ 2 000,00 \u20bd", + "+ 2 000,00 \u20bd", + ] + ] + ], + [], + ] + return pages_text, pages_tables + + +class TestOzonDebitContract: + def setup_method(self) -> None: + self.adapter = OzonDebitPdfAdapter() + self.pages_text, self.pages_tables = ozon_extracted() + + def test_core_contract_fields(self) -> None: + draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables) + assert draft.adapter_id == "ozon_debit_pdf" + assert draft.bank_code == "ozon" + assert draft.currency == "RUB" + assert draft.account_fingerprint == account_fingerprint("ozon", "00000000000000000001") + assert draft.period_start == date(2026, 2, 1) + assert draft.period_end == date(2026, 2, 28) + assert draft.opening_balance_kopecks == 100050 + assert draft.closing_balance_kopecks == 149975 + assert draft.declared_inflow_kopecks == 300000 + assert draft.declared_outflow_kopecks == 250075 + assert draft.declared_hold_kopecks is None + + def test_multiline_cells_stay_one_transaction(self) -> None: + draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables) + assert len(draft.rows) == 2 + first, second = draft.rows + assert first.external_id == "12345678901" + assert first.description == "Оплата товаров по карте 1234" + assert (first.amount_kopecks, first.direction) == (50000, "out") + assert first.occurred_on == date(2026, 2, 1) + assert (second.amount_kopecks, second.direction) == (200000, "in") + assert second.external_id == "1234567891" + assert first.warnings == () + + def test_missing_header_marker_fails_closed(self) -> None: + pages_text, pages_tables = ozon_extracted() + pages_text[0] = "Какой-то другой документ" + with pytest.raises(UnsupportedStatementLayout): + self.adapter._draft_from_extracted(pages_text, pages_tables) + + def test_missing_totals_fail_closed(self) -> None: + pages_text, pages_tables = ozon_extracted() + pages_text[2] = "" + with pytest.raises(UnsupportedStatementLayout): + self.adapter._draft_from_extracted(pages_text, pages_tables) + + def test_no_rows_fails_closed(self) -> None: + pages_text, _ = ozon_extracted() + with pytest.raises(UnsupportedStatementLayout): + self.adapter._draft_from_extracted(pages_text, [[]]) + + def test_unexpected_row_shape_fails_closed(self) -> None: + pages_text, pages_tables = ozon_extracted() + pages_tables[0][0].append([None, None, "текст без суммы"]) + with pytest.raises(UnsupportedStatementLayout): + self.adapter._draft_from_extracted(pages_text, pages_tables) + + def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None: + raw = minimal_pdf() + assert self.adapter.detect(raw) is False + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter.parse(raw) + assert "text layer" in exc.value.message + + def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None: + import app.imports.readers as readers_module + + monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings()) + with pytest.raises(AppError) as exc: + self.adapter.parse(minimal_pdf(page_count=2)) + assert exc.value.code == "IMPORT_QUOTA_EXCEEDED" + + +def same_page_rocket_lines() -> list[tuple[int, str]]: + return [ + (1, "ВЫПИСКА ПО СЧЕТУ"), + (1, "С 01.07.2026 ПО 10.09.2026"), + (1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"), + (1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ"), + (1, "0,00 RUR 0,00 RUR"), + (1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"), + (1, "0,00 RUR 0,00 RUR"), + (1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"), + (1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"), + (1, "10:00 00000000001 -10,00 RUR"), + (1, "11:00 00000000002 -20,00 RUR"), + (1, "12:00 00000000003 -30,00 RUR"), + (1, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"), + (1, "ВСЕГО РАСХОДОВ: 60,00 RUR"), + ] + + +def synthetic_rocket_lines() -> list[tuple[int, str]]: + return [ + (1, "ВЫПИСКА ПО СЧЕТУ"), + (1, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"), + (1, "С 01.07.2026 ПО 10.09.2026"), + (1, "ИВАНОВ ИВАН ИВАНОВИЧ"), + (1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"), + (1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ ДАТА ПОСЛЕДНЕГО ДВИЖЕНИЯ"), + (1, "0,00 RUR 1 000,00 RUR 01.07.2026"), + (1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"), + (1, "500,00 RUR 500,00 RUR"), + (1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"), + (1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"), + (1, "ЗАЧИСЛЕНИЕ ПЕРЕВОДА ДЕНЕЖНЫХ СРЕДСТВ/СБП/"), + (1, "12:00 00000000001 +1 000,00 RUR"), + (1, "СЧЕТ КОРРЕСПОНДЕНТА БАНК ВТБ (ПАО)."), + (1, "ВСЕГО ПОСТУПЛЕНИЙ: 1 000,00 RUR"), + (1, "ВСЕГО РАСХОДОВ: 0,00 RUR"), + (1, "СТРАНИЦА 1/2"), + (2, "ВЫПИСКА ПО СЧЕТУ"), + (2, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"), + (2, "С 01.07.2026 ПО 10.09.2026"), + (2, "02 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 1 000,00 RUR"), + (2, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"), + (2, "ПЛАТЕЖ АВТОРИЗАЦИЯ №123,01.07.2026 10:00,0000********0000,"), + (2, "13:30 00000000002 555RUR,EKATERINBURG,MCC 5411,SHOP\\RU\\CITY\\SHOP\\ -250,50 RUR"), + (2, "0000\\ ЗДАНИЕ."), + (2, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"), + (2, "ВСЕГО РАСХОДОВ: 250,50 RUR"), + (2, "СТРАНИЦА 2/2"), + ] + + +class TestRocketbankDebitContract: + def setup_method(self) -> None: + self.adapter = RocketbankDebitPdfAdapter() + + def test_day_and_page_state_machine(self) -> None: + draft = self.adapter._draft_from_lines(synthetic_rocket_lines()) + assert len(draft.rows) == 2 + first, second = draft.rows + assert first.occurred_on == date(2026, 7, 1) + assert second.occurred_on == date(2026, 7, 2) + + def test_row_refs_are_unique_within_a_page(self) -> None: + draft = self.adapter._draft_from_lines(synthetic_rocket_lines()) + first, second = draft.rows + assert first.row_ref == "page1:record1" + assert second.row_ref == "page2:record1" + same_page = self.adapter._draft_from_lines(same_page_rocket_lines()) + refs = [row.row_ref for row in same_page.rows] + assert refs == ["page1:record1", "page1:record2", "page1:record3"] + assert len(set(refs)) == len(refs) + + def test_statement_controls(self) -> None: + draft = self.adapter._draft_from_lines(synthetic_rocket_lines()) + assert draft.adapter_id == "rocketbank_debit_pdf" + assert draft.bank_code == "rocketbank" + assert draft.currency == "RUB" + assert draft.account_fingerprint == account_fingerprint( + "rocketbank", "00000000000000000001" + ) + assert draft.period_start == date(2026, 7, 1) + assert draft.period_end == date(2026, 9, 10) + assert draft.opening_balance_kopecks == 0 + assert draft.closing_balance_kopecks == 50000 + assert draft.declared_inflow_kopecks == 100000 + assert draft.declared_outflow_kopecks == 50000 + assert draft.declared_hold_kopecks is None + + def test_record_folding_and_enrichment(self) -> None: + draft = self.adapter._draft_from_lines(synthetic_rocket_lines()) + first, second = draft.rows + assert first.external_id == "00000000001" + assert (first.amount_kopecks, first.direction) == (100000, "in") + assert "ЗАЧИСЛЕНИЕ ПЕРЕВОДА" in (first.description or "") + assert "СЧЕТ КОРРЕСПОНДЕНТА" in (first.description or "") + assert first.mcc is None + assert second.external_id == "00000000002" + assert (second.amount_kopecks, second.direction) == (25050, "out") + assert second.mcc == 5411 + assert second.location == "EKATERINBURG" + assert (second.description or "").startswith("ПЛАТЕЖ АВТОРИЗАЦИЯ") + assert "555RUR" in (second.description or "") + assert "ЗДАНИЕ." in (second.description or "") + assert all(row.warnings == () for row in draft.rows) + + def test_repeated_headers_and_daily_totals_are_ignored(self) -> None: + draft = self.adapter._draft_from_lines(synthetic_rocket_lines()) + joined = " ".join(row.description or "" for row in draft.rows) + assert "ВСЕГО" not in joined + assert "ВЫПИСКА ПО СЧЕТУ" not in joined + assert "СТРАНИЦА" not in joined + + def test_parse_is_deterministic(self) -> None: + assert self.adapter._draft_from_lines( + synthetic_rocket_lines() + ) == self.adapter._draft_from_lines(synthetic_rocket_lines()) + + def test_record_before_day_section_fails_closed(self) -> None: + lines = [ + (1, "ВЫПИСКА ПО СЧЕТУ"), + (1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"), + (1, "12:00 00000000001 +1 000,00 RUR"), + ] + with pytest.raises(UnsupportedStatementLayout): + self.adapter._draft_from_lines(lines) + + def test_missing_header_block_fails_closed(self) -> None: + lines = [ + (1, "ВЫПИСКА ПО СЧЕТУ"), + (1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"), + (1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"), + (1, "12:00 00000000001 +1 000,00 RUR"), + ] + with pytest.raises(UnsupportedStatementLayout): + self.adapter._draft_from_lines(lines) + + def test_unrecognized_content_fails_closed(self) -> None: + lines = synthetic_rocket_lines()[:11] + [(1, "НЕИЗВЕСТНАЯ СТРОКА БЕЗ СТРУКТУРЫ")] + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter._draft_from_lines(lines) + assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT" + + def test_fold_line_cap_fails_closed(self) -> None: + junk = [(1, f"НЕИЗВЕСТНАЯ СТРОКА {index}") for index in range(7)] + lines = synthetic_rocket_lines()[:11] + junk + with pytest.raises(UnsupportedStatementLayout): + self.adapter._draft_from_lines(lines) + + def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None: + raw = minimal_pdf() + assert self.adapter.detect(raw) is False + with pytest.raises(UnsupportedStatementLayout) as exc: + self.adapter.parse(raw) + assert "text layer" in exc.value.message + + def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None: + import app.imports.readers as readers_module + + monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings()) + with pytest.raises(AppError) as exc: + self.adapter.parse(minimal_pdf(page_count=2)) + assert exc.value.code == "IMPORT_QUOTA_EXCEEDED" + + +class _QuotaSettings: + max_pdf_pages = 1 + + +def minimal_pdf(page_count: int = 1, page_text: str = "") -> bytes: + """Build a small valid single-font PDF with empty or ASCII-only pages.""" + objects: list[tuple[int, bytes]] = [] + page_ids: list[int] = [] + next_id = 3 + for _ in range(page_count): + page_id = next_id + content_id = next_id + 1 + next_id += 2 + page_ids.append(page_id) + content = ( + f"BT /F1 12 Tf 72 720 Td ({page_text}) Tj ET".encode("latin-1", "replace") + if page_text + else b"" + ) + objects.append( + ( + content_id, + b"<< /Length %d >>\nstream\n%s\nendstream" % (len(content), content), + ) + ) + objects.append( + ( + page_id, + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents %d 0 R >>" + % content_id, + ) + ) + font_id = next_id + objects.append((font_id, b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")) + objects.insert( + 0, + ( + 2, + b"<< /Type /Pages /Kids [" + + b" ".join(b"%d 0 R" % page_id for page_id in page_ids) + + b"] /Count %d >>" % page_count, + ), + ) + objects.insert(0, (1, b"<< /Type /Catalog /Pages 2 0 R >>")) + out = bytearray(b"%PDF-1.4\n") + offsets: dict[int, int] = {} + for number, body in objects: + offsets[number] = len(out) + out += b"%d 0 obj\n" % number + body + b"\nendobj\n" + xref_position = len(out) + size = max(offsets) + 1 + out += b"xref\n0 %d\n" % size + out += b"0000000000 65535 f \n" + for number in range(1, size): + out += b"%010d 00000 n \n" % offsets[number] + out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % ( + size, + xref_position, + ) + return bytes(out) diff --git a/backend/tests/test_import_holds.py b/backend/tests/test_import_holds.py new file mode 100644 index 0000000..1f8c25a --- /dev/null +++ b/backend/tests/test_import_holds.py @@ -0,0 +1,945 @@ +import uuid +from datetime import date +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.budgeting.service import close_period, compute_period_state, get_or_create_period +from app.errors import AppError +from app.identity.models import User +from app.imports import service +from app.imports.adapters.base import record_fingerprint +from app.imports.models import ( + HoldResolution, + ParseRun, + SourceRecord, + SourceRecordLink, + StatementDocument, +) +from app.ledger.models import Operation +from app.ledger.service import create_account +from app.spaces.models import Membership +from app.spaces.service import create_space + +MAY_10 = date(2026, 5, 10) + + +def seed(db: Session, *, name: str = "Holds"): + user = User(email=f"{uuid.uuid4().hex}@example.com", password_hash="unused") + db.add(user) + db.flush() + space = create_space(db, user=user, name=name) + account = create_account( + db, + space_id=space.id, + user=user, + name="Debit", + kind="debit", + opened_on=date(2026, 1, 1), + ) + return user, space, account + + +def make_document( + db: Session, + space, + account, + *, + user: User | None = None, + status: str = "ready", + adapter_id: str = "alfa_debit_xlsx", +) -> tuple[StatementDocument, ParseRun]: + document = StatementDocument( + space_id=space.id, + account_id=account.id, + original_filename=f"{uuid.uuid4().hex}.xlsx", + size_bytes=100, + content_sha256=uuid.uuid4().hex + uuid.uuid4().hex, + object_key=uuid.uuid4().hex, + status=status, + reconciliation={"full": True, "matched": True}, + uploaded_by_user_id=user.id if user else None, + ) + db.add(document) + db.flush() + run = ParseRun( + space_id=space.id, + document_id=document.id, + adapter_id=adapter_id, + status="succeeded", + ) + db.add(run) + db.flush() + document.active_parse_run_id = run.id + document.adapter_id = adapter_id + return document, run + + +def add_record( + db: Session, + document: StatementDocument, + run: ParseRun, + *, + direction: str = "out", + amount: int = 12_345, + occurred_on: date = MAY_10, + bank_status: str = "posted", + external_id: str | None = None, + description: str | None = None, +) -> SourceRecord: + record = SourceRecord( + space_id=document.space_id, + document_id=document.id, + parse_run_id=run.id, + external_id=external_id, + occurred_on=occurred_on, + amount_kopecks=amount, + direction=direction, + bank_status=bank_status, + description=description, + row_ref=f"sheet0:{uuid.uuid4().hex[:8]}", + record_fp=record_fingerprint(occurred_on, amount, description), + ) + db.add(record) + db.flush() + return record + + +def hold_document(db: Session, *, occurred_on: date = MAY_10, external_id: str | None = None): + """A confirmed import whose active run contains exactly one unresolved hold.""" + user, space, account = seed(db) + document, run = make_document(db, space, account, user=user) + hold = add_record( + db, + document, + run, + bank_status="hold", + occurred_on=occurred_on, + external_id=external_id, + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=document.id, user=user) + db.flush() + return user, space, account, document, hold + + +def test_bank_status_defaults_to_posted_and_is_constrained(db: Session) -> None: + user, space, account = seed(db) + document, run = make_document(db, space, account, user=user) + record = SourceRecord( + space_id=space.id, + document_id=document.id, + parse_run_id=run.id, + amount_kopecks=1_000, + direction="out", + row_ref="sheet0:row1", + record_fp="fp", + ) + db.add(record) + db.flush() + assert record.bank_status == "posted" + bad = SourceRecord( + space_id=space.id, + document_id=document.id, + parse_run_id=run.id, + amount_kopecks=1_000, + direction="out", + bank_status="pending", + row_ref="sheet0:row2", + record_fp="fp2", + ) + db.add(bad) + with pytest.raises(IntegrityError): + db.flush() + db.rollback() + + +def test_hold_row_creates_no_operation_on_confirm(db: Session) -> None: + user, space, account = seed(db) + document, run = make_document(db, space, account, user=user) + hold = add_record(db, document, run, bank_status="hold") + add_record( + db, + document, + run, + bank_status="posted", + occurred_on=date(2026, 5, 11), + amount=7_000, + ) + service.confirm_document(db, space_id=space.id, document_id=document.id, user=user) + db.flush() + operations = db.query(Operation).filter_by(space_id=space.id).all() + assert len(operations) == 1 + assert operations[0].amount_kopecks == 7_000 + assert db.query(SourceRecordLink).filter_by(source_record_id=hold.id).count() == 0 + assert document.status == "confirmed" + detail = service.document_detail(db, space_id=space.id, document_id=document.id) + hold_row = next(item for item in detail["records"] if item["id"] == hold.id) + assert hold_row["bank_status"] == "hold" + assert hold_row["hold_resolution"] is None + assert hold_row["critical"] is False + + +def test_unresolved_hold_blocks_only_its_occurred_month(db: Session) -> None: + user, space, _account, _document, _hold = hold_document(db) + april = get_or_create_period(db, space_id=space.id, year=2026, month=4) + close_period(db, space_id=space.id, user=user, period=april) + may = get_or_create_period(db, space_id=space.id, year=2026, month=5) + state = compute_period_state(db, space_id=space.id, period=may) + assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"]) + with pytest.raises(AppError) as exc: + close_period(db, space_id=space.id, user=user, period=may) + assert exc.value.code == "BUDGET_CLOSE_BLOCKED" + june = get_or_create_period(db, space_id=space.id, year=2026, month=6) + june_state = compute_period_state(db, space_id=space.id, period=june) + assert not any( + blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in june_state["close_blockers"] + ) + + +def test_discard_as_cancelled_requires_reason_and_unblocks_month(db: Session) -> None: + user, space, _account, document, hold = hold_document(db) + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="discard_as_cancelled", + reason=" ", + ) + assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED" + assert db.query(HoldResolution).count() == 0 + resolution = service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="discard_as_cancelled", + reason="Cancelled by the bank", + ) + db.flush() + assert resolution.resolution == "discarded_as_cancelled" + assert resolution.operation_id is None + assert resolution.resolved_by_user_id == user.id + assert db.query(Operation).filter_by(space_id=space.id).count() == 0 + detail = service.document_detail(db, space_id=space.id, document_id=document.id) + hold_row = next(item for item in detail["records"] if item["id"] == hold.id) + assert hold_row["hold_resolution"]["reason"] == "Cancelled by the bank" + april = get_or_create_period(db, space_id=space.id, year=2026, month=4) + close_period(db, space_id=space.id, user=user, period=april) + may = get_or_create_period(db, space_id=space.id, year=2026, month=5) + close_period(db, space_id=space.id, user=user, period=may) + assert may.status == "closed" + + +def test_confirm_as_posted_requires_reason(db: Session) -> None: + user, space, _account, document, hold = hold_document(db) + for reason in (None, ""): + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="confirm_as_posted", + reason=reason, + ) + assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED" + assert db.query(HoldResolution).count() == 0 + + +def test_hold_cannot_be_resolved_before_document_confirmation(db: Session) -> None: + user, space, account = seed(db) + document, run = make_document(db, space, account, user=user) + hold = add_record(db, document, run, bank_status="hold") + + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="confirm_as_posted", + reason="Not confirmed yet", + ) + + assert exc.value.code == "IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED" + assert db.query(Operation).filter_by(space_id=space.id).count() == 0 + assert db.query(HoldResolution).count() == 0 + + +def test_manual_confirm_rejects_bad_action_and_non_hold_row(db: Session) -> None: + user, space, account = seed(db) + document, run = make_document(db, space, account, user=user) + hold = add_record(db, document, run, bank_status="hold") + posted = add_record( + db, document, run, bank_status="posted", occurred_on=date(2026, 5, 11), amount=900 + ) + service.confirm_document(db, space_id=space.id, document_id=document.id, user=user) + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="delete_forever", + reason="why not", + ) + assert exc.value.code == "IMPORT_HOLD_ACTION_INVALID" + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=posted.id, + user=user, + action="discard_as_cancelled", + reason="posted row", + ) + assert exc.value.code == "IMPORT_HOLD_INVALID" + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=uuid.uuid4(), + user=user, + action="discard_as_cancelled", + reason="missing row", + ) + assert exc.value.code == "IMPORT_HOLD_NOT_FOUND" + + +def test_manual_confirm_creates_one_operation_and_late_import_dedupes(db: Session) -> None: + user, space, account, document, hold = hold_document(db, external_id="H-1") + resolution = service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="confirm_as_posted", + reason="Verified against the paper receipt", + account_id=account.id, + ) + db.flush() + operations = db.query(Operation).filter_by(space_id=space.id).all() + assert len(operations) == 1 + operation = operations[0] + assert operation.type == "expense" + assert operation.amount_kopecks == 12_345 + assert operation.occurred_on == MAY_10 + assert resolution.operation_id == operation.id + link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one_or_none() + assert link is not None and link.operation_id == operation.id + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="discard_as_cancelled", + reason="second try", + ) + assert exc.value.code == "IMPORT_HOLD_ALREADY_RESOLVED" + + later, later_run = make_document(db, space, account, user=user) + late_posted = add_record( + db, + later, + later_run, + bank_status="posted", + external_id="H-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=later.id, user=user) + db.flush() + assert db.query(Operation).filter_by(space_id=space.id).count() == 1 + late_link = db.query(SourceRecordLink).filter_by(source_record_id=late_posted.id).one_or_none() + assert late_link is not None + assert late_link.operation_id == operation.id + assert late_link.kind == "duplicate" + + +def test_manual_confirm_of_income_hold_creates_income(db: Session) -> None: + user, space, account = seed(db) + document, run = make_document(db, space, account, user=user) + hold = add_record( + db, + document, + run, + direction="in", + amount=45_000, + bank_status="hold", + description="Refund hold", + ) + service.confirm_document(db, space_id=space.id, document_id=document.id, user=user) + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="confirm_as_posted", + reason="Money arrived", + ) + db.flush() + operation = db.query(Operation).filter_by(space_id=space.id).one() + assert operation.type == "income" + assert operation.amount_kopecks == 45_000 + + +def test_manual_confirm_honors_closed_period(db: Session) -> None: + user, space, account = seed(db) + february = get_or_create_period(db, space_id=space.id, year=2026, month=2) + close_period(db, space_id=space.id, user=user, period=february) + march = get_or_create_period(db, space_id=space.id, year=2026, month=3) + close_period(db, space_id=space.id, user=user, period=march) + document, run = make_document(db, space, account, user=user) + hold = add_record(db, document, run, bank_status="hold", occurred_on=date(2026, 3, 15)) + service.confirm_document(db, space_id=space.id, document_id=document.id, user=user) + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="confirm_as_posted", + reason="Book it", + ) + assert exc.value.code == "BUDGET_PERIOD_CLOSED" + db.flush() + assert db.query(Operation).filter_by(space_id=space.id).count() == 0 + assert db.query(HoldResolution).count() == 0 + resolution = service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="discard_as_cancelled", + reason="Never posted", + ) + assert resolution.operation_id is None + + +def test_unique_posted_record_resolves_hold_on_confirmation(db: Session) -> None: + user, space, account = seed(db) + first, first_run = make_document(db, space, account, user=user) + hold = add_record( + db, + first, + first_run, + bank_status="hold", + external_id="ALFA-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=first.id, user=user) + assert db.query(HoldResolution).count() == 0 + + second, second_run = make_document(db, space, account, user=user) + posted = add_record( + db, + second, + second_run, + bank_status="posted", + external_id="ALFA-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=second.id, user=user) + db.flush() + resolution = db.query(HoldResolution).one() + assert resolution.hold_source_record_id == hold.id + assert resolution.resolving_source_record_id == posted.id + assert resolution.resolution == "matched_posted" + assert resolution.resolved_by_user_id is None + operations = db.query(Operation).filter_by(space_id=space.id).all() + assert len(operations) == 1 + assert operations[0].amount_kopecks == 12_345 + may = get_or_create_period(db, space_id=space.id, year=2026, month=5) + state = compute_period_state(db, space_id=space.id, period=may) + assert not any( + blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"] + ) + + +def test_external_id_match_survives_description_drift(db: Session) -> None: + user, space, account = seed(db) + first, first_run = make_document(db, space, account, user=user) + hold = add_record( + db, + first, + first_run, + bank_status="hold", + external_id="X-9", + description="HOLD COFFEE", + occurred_on=MAY_10, + ) + service.confirm_document(db, space_id=space.id, document_id=first.id, user=user) + second, second_run = make_document(db, space, account, user=user) + posted = add_record( + db, + second, + second_run, + bank_status="posted", + external_id="X-9", + description="Coffee llc prague", + occurred_on=date(2026, 5, 12), + ) + service.confirm_document(db, space_id=space.id, document_id=second.id, user=user) + db.flush() + resolution = db.query(HoldResolution).one() + assert resolution.hold_source_record_id == hold.id + assert resolution.resolving_source_record_id == posted.id + + +def test_fingerprint_match_without_external_id_resolves(db: Session) -> None: + user, space, account = seed(db) + first, first_run = make_document(db, space, account, user=user) + hold = add_record( + db, + first, + first_run, + bank_status="hold", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=first.id, user=user) + second, second_run = make_document(db, space, account, user=user) + posted = add_record( + db, + second, + second_run, + bank_status="posted", + description="Coffee shop", + ) + matches = service._match_hold_resolutions(db, document=second, run=second_run) + assert matches == {hold.id: posted.id} + service.confirm_document(db, space_id=space.id, document_id=second.id, user=user) + db.flush() + assert db.query(HoldResolution).count() == 1 + + +def test_ambiguous_overlap_remains_unresolved(db: Session) -> None: + user, space, account = seed(db) + first, first_run = make_document(db, space, account, user=user) + hold_a = add_record( + db, + first, + first_run, + bank_status="hold", + external_id="A-1", + description="Coffee shop", + ) + hold_b = add_record( + db, + first, + first_run, + bank_status="hold", + external_id="B-2", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=first.id, user=user) + assert hold_a.id != hold_b.id + second, second_run = make_document(db, space, account, user=user) + add_record( + db, + second, + second_run, + bank_status="posted", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=second.id, user=user) + db.flush() + assert db.query(HoldResolution).count() == 0 + may = get_or_create_period(db, space_id=space.id, year=2026, month=5) + state = compute_period_state(db, space_id=space.id, period=may) + assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"]) + + +def test_matcher_rejects_two_posted_candidates_for_one_hold(db: Session) -> None: + user, space, account = seed(db) + first, first_run = make_document(db, space, account, user=user) + add_record(db, first, first_run, bank_status="hold", description="Coffee shop") + service.confirm_document(db, space_id=space.id, document_id=first.id, user=user) + second, second_run = make_document(db, space, account, user=user) + add_record(db, second, second_run, bank_status="posted", description="Coffee shop") + add_record(db, second, second_run, bank_status="posted", description="Coffee shop") + matches = service._match_hold_resolutions(db, document=second, run=second_run) + assert matches == {} + assert db.query(HoldResolution).count() == 0 + + +def test_discarded_posted_record_does_not_resolve_hold(db: Session) -> None: + user, space, account = seed(db) + first, first_run = make_document(db, space, account, user=user) + hold = add_record( + db, + first, + first_run, + bank_status="hold", + external_id="DISCARDED-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=first.id, user=user) + + second, second_run = make_document(db, space, account, user=user) + posted = add_record( + db, + second, + second_run, + bank_status="posted", + external_id="DISCARDED-1", + description="Coffee shop", + ) + posted.review_decision = "discard" + service.confirm_document(db, space_id=space.id, document_id=second.id, user=user) + db.flush() + + assert db.query(HoldResolution).filter_by(hold_source_record_id=hold.id).count() == 0 + assert db.query(Operation).filter_by(space_id=space.id).count() == 0 + may = get_or_create_period(db, space_id=space.id, year=2026, month=5) + state = compute_period_state(db, space_id=space.id, period=may) + assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"]) + + +def test_hold_matching_is_tenant_scoped(db: Session) -> None: + user1, space1, account1 = seed(db, name="Space One") + user2, space2, account2 = seed(db, name="Space Two") + first, first_run = make_document(db, space1, account1, user=user1) + hold = add_record( + db, + first, + first_run, + bank_status="hold", + external_id="T-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space1.id, document_id=first.id, user=user1) + second, second_run = make_document(db, space2, account2, user=user2) + add_record( + db, + second, + second_run, + bank_status="posted", + external_id="T-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space2.id, document_id=second.id, user=user2) + db.flush() + assert db.query(HoldResolution).count() == 0 + may_two = get_or_create_period(db, space_id=space2.id, year=2026, month=5) + state_two = compute_period_state(db, space_id=space2.id, period=may_two) + assert not any( + blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_two["close_blockers"] + ) + may_one = get_or_create_period(db, space_id=space1.id, year=2026, month=5) + state_one = compute_period_state(db, space_id=space1.id, period=may_one) + assert any( + blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_one["close_blockers"] + ) + with pytest.raises(AppError) as exc: + service.resolve_hold( + db, + space_id=space2.id, + document_id=first.id, + source_record_id=hold.id, + user=user2, + action="discard_as_cancelled", + reason="cross tenant", + ) + assert exc.value.code == "IMPORT_NOT_FOUND" + + +def test_revert_clears_automatic_match_and_reblocks_month(db: Session) -> None: + user, space, account = seed(db) + first, first_run = make_document(db, space, account, user=user) + hold = add_record( + db, + first, + first_run, + bank_status="hold", + external_id="R-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=first.id, user=user) + second, second_run = make_document(db, space, account, user=user) + add_record( + db, + second, + second_run, + bank_status="posted", + external_id="R-1", + description="Coffee shop", + ) + service.confirm_document(db, space_id=space.id, document_id=second.id, user=user) + db.flush() + assert db.query(HoldResolution).count() == 1 + service.revert_document(db, space_id=space.id, document_id=second.id, user=user) + db.flush() + assert db.query(HoldResolution).count() == 0 + assert db.query(Operation).filter_by(space_id=space.id).count() == 0 + may = get_or_create_period(db, space_id=space.id, year=2026, month=5) + state = compute_period_state(db, space_id=space.id, period=may) + assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"]) + assert hold.bank_status == "hold" + + +def test_revert_preserves_manual_hold_operation_and_provenance(db: Session) -> None: + user, space, account, document, hold = hold_document(db) + resolution = service.resolve_hold( + db, + space_id=space.id, + document_id=document.id, + source_record_id=hold.id, + user=user, + action="confirm_as_posted", + reason="Verified in the bank app", + account_id=account.id, + ) + db.flush() + operation_id = resolution.operation_id + + service.revert_document(db, space_id=space.id, document_id=document.id, user=user) + db.flush() + + assert document.status == "ready" + assert operation_id is not None + assert db.get(Operation, operation_id) is not None + assert db.get(HoldResolution, resolution.id) is not None + link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one() + assert link.kind == "hold_confirmed" + assert link.operation_id == operation_id + + +def test_hold_resolution_rejects_unknown_kind(db: Session) -> None: + user, space, _account, _document, hold = hold_document(db) + db.add( + HoldResolution( + space_id=space.id, + hold_source_record_id=hold.id, + resolution="resolved_somehow", + ) + ) + with pytest.raises(IntegrityError): + db.flush() + db.rollback() + + +def test_manual_resolution_requires_reason_at_database_level(db: Session) -> None: + _user, space, _account, _document, hold = hold_document(db) + db.add( + HoldResolution( + space_id=space.id, + hold_source_record_id=hold.id, + resolution="confirmed_as_posted", + reason=None, + ) + ) + with pytest.raises(IntegrityError): + db.flush() + db.rollback() + _user, space, _account, _document, hold = hold_document(db) + db.add( + HoldResolution( + space_id=space.id, + hold_source_record_id=hold.id, + resolution="matched_posted", + reason=None, + ) + ) + db.flush() + + +def _draft(rows, *, opening=1_000, closing=1_200, **declared): + return SimpleNamespace( + currency="RUB", + opening_balance_kopecks=opening, + closing_balance_kopecks=closing, + declared_inflow_kopecks=declared.get("inflow"), + declared_outflow_kopecks=declared.get("outflow"), + declared_hold_kopecks=declared.get("hold"), + rows=tuple(rows), + ) + + +def _row(amount, direction, bank_status="posted"): + return SimpleNamespace( + amount_kopecks=amount, + direction=direction, + bank_status=bank_status, + ) + + +def test_reconciliation_separates_posted_and_hold_totals() -> None: + summary = service._reconciliation_summary( + _draft( + [ + _row(50_000, "in"), + _row(30_000, "out"), + _row(20_000, "out", bank_status="hold"), + ], + opening=100_000, + closing=120_000, + inflow=50_000, + outflow=30_000, + hold=20_000, + ) + ) + assert summary["parsed_posted_inflow_kopecks"] == 50_000 + assert summary["parsed_posted_outflow_kopecks"] == 30_000 + assert summary["parsed_hold_total_kopecks"] == 20_000 + assert summary["posted_totals_matched"] is True + assert summary["hold_totals_matched"] is True + assert summary["balance_matched"] is True + assert summary["matched"] is True + assert summary["computed_closing_kopecks"] == 120_000 + + +def test_reconciliation_declared_outflow_mismatch_blocks() -> None: + summary = service._reconciliation_summary( + _draft( + [_row(30_000, "out")], + opening=100_000, + closing=70_000, + inflow=None, + outflow=40_000, + ) + ) + assert summary["posted_totals_matched"] is False + assert summary["hold_totals_matched"] is None + assert summary["matched"] is False + + +def test_reconciliation_hold_mismatch_is_review_blocking() -> None: + summary = service._reconciliation_summary( + _draft([_row(25_000, "out", bank_status="hold")], opening=None, closing=None, hold=10_000) + ) + assert summary["full"] is False + assert summary["hold_totals_matched"] is False + assert summary["matched"] is False + assert summary["balance_matched"] is None + + +def test_reconciliation_balance_equation_ignores_holds() -> None: + summary = service._reconciliation_summary( + _draft( + [ + _row(50_000, "in"), + _row(30_000, "out"), + _row(20_000, "out", bank_status="hold"), + ], + opening=100_000, + closing=140_000, + ) + ) + assert summary["balance_matched"] is False + assert summary["matched"] is False + assert summary["computed_closing_kopecks"] == 120_000 + + +def test_reconciliation_without_declared_totals_keeps_legacy_semantics() -> None: + draft = SimpleNamespace( + currency="RUB", + opening_balance_kopecks=100, + closing_balance_kopecks=300, + rows=(_row(200, "in"),), + ) + summary = service._reconciliation_summary(draft) + assert summary["full"] is True + assert summary["posted_totals_matched"] is None + assert summary["hold_totals_matched"] is None + assert summary["balance_matched"] is True + assert summary["matched"] is True + + +def test_source_row_without_bank_status_defaults_posted() -> None: + from app.imports.adapters.base import SourceRow + + source = service._source_from_row( + space_id=uuid.uuid4(), + document_id=uuid.uuid4(), + run_id=uuid.uuid4(), + row=SourceRow( + external_id=None, + occurred_on=date(2026, 5, 10), + posted_on=None, + amount_kopecks=1_000, + direction="out", + description=None, + mcc=None, + bank_category=None, + location=None, + row_ref="row1", + ), + ) + assert source.bank_status == "posted" + assert source.critical is False + + +def test_resolve_endpoint_requires_csrf_and_resolves_hold(client: TestClient, db: Session) -> None: + invite = client.post( + "/api/v1/admin/invitations", + headers={"X-Admin-Token": "test-admin-token"}, + json={"email": "holds-user@example.com"}, + ) + assert invite.status_code == 201, invite.text + accepted = client.post( + "/api/v1/auth/invitations/accept", + json={ + "token": invite.json()["token"], + "password": "a-long-password-for-tests", + "fullName": "Holds User", + "locale": "en", + }, + ) + assert accepted.status_code == 201, accepted.text + csrf = client.cookies["csrf_token"] + db.commit() + + user = db.query(User).filter_by(email="holds-user@example.com").one() + space_id = db.query(Membership).filter_by(user_id=user.id).one().space_id + account = create_account( + db, + space_id=space_id, + user=user, + name="Debit", + kind="debit", + opened_on=date(2026, 1, 1), + ) + space = SimpleNamespace(id=space_id) + document, run = make_document(db, space, account, user=user) + hold = add_record(db, document, run, bank_status="hold") + service.confirm_document(db, space_id=space_id, document_id=document.id, user=user) + db.commit() + + rejected = client.post( + f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve", + json={"action": "confirm_as_posted", "reason": "no csrf"}, + ) + assert rejected.status_code == 403 + + resolved = client.post( + f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve", + headers={"X-CSRF-Token": csrf}, + json={"action": "confirm_as_posted", "reason": "Checked in the bank app"}, + ) + assert resolved.status_code == 200, resolved.text + rows = resolved.json()["rows"] + row = next(item for item in rows if item["id"] == str(hold.id)) + assert row["bankStatus"] == "hold" + assert row["holdResolution"]["resolution"] == "confirmed_as_posted" + assert row["holdResolution"]["reason"] == "Checked in the bank app" + assert row["holdResolution"]["operationId"] is not None + operations = db.query(Operation).filter_by(space_id=space_id).all() + assert len(operations) == 1 diff --git a/backend/tests/test_postgres_and_s3.py b/backend/tests/test_postgres_and_s3.py index 93b5fcf..67f93b5 100644 --- a/backend/tests/test_postgres_and_s3.py +++ b/backend/tests/test_postgres_and_s3.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import Session from app.db.session import set_space_context from app.imports.blob import get_blob_store +from app.imports.models import HoldResolution, ParseRun, SourceRecord, StatementDocument from app.ledger.models import Account from app.spaces.models import FinancialSpace @@ -28,11 +29,55 @@ def test_postgres_rls_requires_matching_transaction_context() -> None: db.rollback() set_space_context(db, space_id) - db.add(Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today())) + account = Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today()) + db.add(account) + db.flush() + document = StatementDocument( + space_id=space_id, + account_id=account.id, + original_filename="synthetic.xlsx", + size_bytes=1, + content_sha256=uuid.uuid4().hex + uuid.uuid4().hex, + object_key=uuid.uuid4().hex, + status="confirmed", + ) + db.add(document) + db.flush() + run = ParseRun( + space_id=space_id, + document_id=document.id, + adapter_id="alfa_debit_xlsx", + status="succeeded", + ) + db.add(run) + db.flush() + document.active_parse_run_id = run.id + hold = SourceRecord( + space_id=space_id, + document_id=document.id, + parse_run_id=run.id, + occurred_on=date.today(), + amount_kopecks=100, + direction="out", + bank_status="hold", + row_ref="sheet0:row1", + record_fp=uuid.uuid4().hex, + ) + db.add(hold) + db.flush() + db.add( + HoldResolution( + space_id=space_id, + hold_source_record_id=hold.id, + resolution="discarded_as_cancelled", + reason="synthetic test resolution", + ) + ) db.commit() set_space_context(db, uuid.uuid4()) assert db.query(Account).count() == 0 + assert db.query(HoldResolution).count() == 0 db.rollback() set_space_context(db, space_id) persisted_space = db.get(FinancialSpace, space_id) diff --git a/backend/tests/test_private_fixtures.py b/backend/tests/test_private_fixtures.py new file mode 100644 index 0000000..3ac20ea --- /dev/null +++ b/backend/tests/test_private_fixtures.py @@ -0,0 +1,224 @@ +"""Private real-statement suite (marker: private_fixtures). + +Runs only when invoked explicitly: + + PRIVATE_FIXTURES_DIR=/secure/path uv run pytest -m private_fixtures + +When PRIVATE_FIXTURES_DIR is unset, the ignored repository-local ``examples/`` +directory is used if it contains a ``manifest.json``. Normal CI never collects +this module (the default pytest addopts deselect the marker). + +Privacy contract (docs/parsers/private-fixture-gate.md): + +- fixture contents, parsed rows, descriptions, identifiers, and manifest + financial values never appear in assertion output, messages, or logs; +- failures report only the filename, adapter id, and the name of the + mismatched aggregate; +- every file is parsed twice and the results must be identical. +""" + +import json +import os +from pathlib import Path + +import pytest + +from app.imports.adapters import detect_adapter, magic_kind +from app.imports.service import _reconciliation_summary + +pytestmark = pytest.mark.private_fixtures + +REPO_EXAMPLES = Path(__file__).resolve().parents[2] / "examples" + + +def _load_entries() -> tuple[Path | None, list[dict], str | None]: + """Resolve fixture entries. + + Returns (directory, entries, explicit_error). An explicit error is only + produced when PRIVATE_FIXTURES_DIR was set on purpose: then a missing + directory, missing/malformed manifest, or empty manifest must fail loudly + (redacted message, no values). Without the variable, an absent or broken + repo-local examples directory is a silent skip. + """ + env = os.environ.get("PRIVATE_FIXTURES_DIR") + if env: + directory = Path(env) + if not directory.is_dir(): + return None, [], "PRIVATE_FIXTURES_DIR is set but the directory does not exist" + try: + manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8")) + except Exception: + return directory, [], "private manifest.json is missing or unreadable" + entries = manifest.get("fixtures") if isinstance(manifest, dict) else None + if not isinstance(entries, list) or not entries: + return directory, [], "private manifest declares no fixtures" + return directory, [entry for entry in entries if isinstance(entry, dict)], None + try: + if not REPO_EXAMPLES.is_dir(): + return None, [], None + manifest = json.loads((REPO_EXAMPLES / "manifest.json").read_text(encoding="utf-8")) + entries = manifest.get("fixtures") if isinstance(manifest, dict) else None + if not isinstance(entries, list): + return None, [], None + return REPO_EXAMPLES, [entry for entry in entries if isinstance(entry, dict)], None + except Exception: + return None, [], None + + +FIXTURE_DIR, ENTRIES, EXPLICIT_ERROR = _load_entries() +if EXPLICIT_ERROR: + raise pytest.fail.Exception(EXPLICIT_ERROR) +PARAMS = ENTRIES if ENTRIES else [None] + + +def _check(ok: bool, name: str, what: str) -> None: + assert ok, f"{name}: {what} mismatch" + + +def _fail(name: str, message: str) -> None: + pytest.fail(f"{name}: {message}", pytrace=False) + + +@pytest.fixture(params=PARAMS, ids=[entry.get("filename", "unavailable") for entry in PARAMS]) +def parsed(request: pytest.FixtureRequest): + entry = request.param + if entry is None: + pytest.skip("private fixtures are not available") + name = str(entry.get("filename", "unnamed")) + path = (FIXTURE_DIR / name) if FIXTURE_DIR is not None else None + if path is None or not path.is_file(): + _fail(name, "fixture file is missing") + raw = path.read_bytes() + kind = magic_kind(raw) + if kind not in {"xlsx", "pdf"}: + _fail(name, "unexpected file type") + try: + adapter = detect_adapter(raw, kind) + except Exception: + _fail(name, "adapter detection failed") + if adapter.adapter_id != entry.get("adapter_id"): + _fail(name, "adapter detection mismatch") + try: + first = adapter.parse(raw) + second = adapter.parse(raw) + except Exception: + _fail(name, "parse failed") + if first != second: + _fail(name, "parse result is not deterministic") + return name, entry, first + + +def test_manifest_contract(parsed) -> None: + name, entry, draft = parsed + _check(draft.parser_version == entry.get("parser_version"), name, "parser version") + for key, actual in (("period_start", draft.period_start), ("period_end", draft.period_end)): + expected = entry.get(key) + _check( + expected is None or (actual is not None and actual.isoformat() == expected), name, key + ) + + +def test_row_counts(parsed) -> None: + name, entry, draft = parsed + rows = draft.rows + _check(len(rows) == entry.get("row_count"), name, "row count") + posted = sum(1 for row in rows if row.bank_status == "posted") + holds = sum(1 for row in rows if row.bank_status == "hold") + _check(posted == entry.get("posted_count"), name, "posted count") + _check(holds == entry.get("hold_count"), name, "hold count") + _check( + sum(1 for row in rows if row.direction == "in") == entry.get("in_count"), + name, + "inflow row count", + ) + _check( + sum(1 for row in rows if row.direction == "out") == entry.get("out_count"), + name, + "outflow row count", + ) + + +def test_row_invariants(parsed) -> None: + name, _, draft = parsed + rows = draft.rows + _check(all(row.amount_kopecks >= 0 for row in rows), name, "positive amount invariant") + _check( + all(row.bank_status in {"posted", "hold"} for row in rows), + name, + "bank status invariant", + ) + + +def test_hold_totals(parsed) -> None: + name, entry, draft = parsed + hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold") + _check(hold_total == entry.get("hold_total_kopecks"), name, "hold total") + + +def test_control_totals(parsed) -> None: + name, entry, draft = parsed + _check( + draft.declared_inflow_kopecks == entry.get("declared_inflow_kopecks"), + name, + "declared inflow total", + ) + _check( + draft.declared_outflow_kopecks == entry.get("declared_outflow_kopecks"), + name, + "declared outflow total", + ) + _check( + draft.declared_hold_kopecks == entry.get("declared_hold_kopecks"), + name, + "declared hold total", + ) + + +def test_balances(parsed) -> None: + name, entry, draft = parsed + _check( + draft.opening_balance_kopecks == entry.get("opening_balance_kopecks"), + name, + "opening balance", + ) + _check( + draft.closing_balance_kopecks == entry.get("closing_balance_kopecks"), + name, + "closing balance", + ) + + +def test_declared_totals_match_parsed_rows(parsed) -> None: + name, _, draft = parsed + posted_in = sum( + row.amount_kopecks + for row in draft.rows + if row.bank_status == "posted" and row.direction == "in" + ) + posted_out = sum( + row.amount_kopecks + for row in draft.rows + if row.bank_status == "posted" and row.direction == "out" + ) + hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold") + # Declared controls are canonicalized to posted-only by the adapters; the + # hold control equals the parsed hold sum. + _check(posted_in == draft.declared_inflow_kopecks, name, "declared vs parsed inflow") + _check(posted_out == draft.declared_outflow_kopecks, name, "declared vs parsed outflow") + _check(hold_total == (draft.declared_hold_kopecks or 0), name, "declared vs parsed hold") + + +def test_service_reconciliation_matches(parsed) -> None: + name, _, draft = parsed + summary = _reconciliation_summary(draft) + _check(summary.get("matched") is True, name, "service reconciliation matched") + _check( + summary.get("posted_totals_matched") is True, + name, + "service reconciliation posted totals", + ) + _check( + summary.get("hold_totals_matched") is not False, + name, + "service reconciliation hold totals", + ) diff --git a/docs/adr/0003-versioned-statement-imports.md b/docs/adr/0003-versioned-statement-imports.md index a8db721..cb43b05 100644 --- a/docs/adr/0003-versioned-statement-imports.md +++ b/docs/adr/0003-versioned-statement-imports.md @@ -2,29 +2,39 @@ - **Status:** Accepted - **Date:** 2026-09-11 -- **Scope:** statement import pipeline, parse provenance, deduplication, apply/revert semantics +- **Scope:** statement import pipeline, parse provenance, deduplication, HOLD lifecycle, apply/revert semantics ## Context -Five bank statement contracts must be parsed (Alfa debit/credit XLSX, Ozon debit/credit PDF, Rocketbank debit PDF). Bank layouts drift over time. Money facts must never change until the user has reviewed the parse; losing a real operation is worse than holding a duplicate for review. The HTTP process must not parse files. +Five bank statement contracts must be parsed (Alfa debit/credit XLSX, Ozon debit/credit PDF, Rocketbank debit PDF). Bank layouts drift over time. Money facts must never change until the user has reviewed the parse; losing a real operation is worse than holding a duplicate for review. The HTTP process must not parse files. Banks additionally report pending (HOLD) transactions that are not yet booked; treating them as settled would misstate balances. ## Decision - An uploaded statement document is immutable: magic/MIME/size/hash checks, then storage encrypted per ADR 0004. Full extracted text is never persisted — only allowlisted statement fields plus page/row coordinates. - Every parse produces a versioned **parse run** (adapter + parser version) yielding immutable **source records** with document origin. A reparse creates a new run and diff; previous runs are never mutated. +- Each source row carries an immutable **bank status** (`posted` or `hold`). HOLD rows are preserved exactly as extracted and never create ledger operations during normal import confirmation. - State machine: `uploaded → queued → processing → review_required/ready → confirmed | failed`. Confirm applies the whole run atomically in a single DB transaction, creating only new operations/postings; existing duplicates only gain additional source links. Revert deletes exactly what that import created, with cascade reopen of affected periods. - Deduplication is conservative: auto-link only a stable bank identifier or a single unambiguous match; ambiguous candidates stay pending until the user decides. +- Reconciliation compares parsed posted inflow/outflow with the bank's declared totals, compares HOLD totals separately, and — where opening and closing balances exist — requires `opening + posted inflow − posted outflow = closing`. Any mismatch stays review-blocking with stable diagnostic fields. +- **HOLD lifecycle:** each HOLD source record carries at most one tenant-scoped, append-only `HoldResolution` (`matched_posted`, `confirmed_as_posted`, `discarded_as_cancelled`) with optional resolving source record, operation, user, and a required reason for manual actions. The extracted HOLD row itself is never overwritten. +- **Automatic settlement matching:** when a new statement is confirmed, its posted records are conservatively matched — same space, mapped account, and statement contract only — to unresolved HOLDs from confirmed imports. A stable external ID wins; otherwise a single unambiguous date/amount/direction/normalized-description fingerprint match is required. Any ambiguity leaves the HOLD unresolved; the resolution is written only at confirmation of the resolving statement. Reverting that statement removes the automatic match. +- **Manual resolution:** two CSRF-protected backend actions, both requiring a reason and (for booking) a mapped account: `confirm_as_posted` creates a normal ledger operation through the ledger services (honoring closed-period rules) with a provenance source-record link; `discard_as_cancelled` creates none. Both write audit events. A later import of the same posted row dedupes onto the manually created operation instead of duplicating it. +- **Close blocking:** unresolved HOLDs from confirmed imports block closing only the budget month containing their `occurred_on` date; manual confirmation may then expose the normal uncategorized-expense blocker until the created operation is categorized. - Bank layout drift produces a **new parser version** with an explicit review diff — historical data is never silently reinterpreted. An adapter is beta-eligible only after the private fixture gate (see `docs/parsers/private-fixture-gate.md`). - Parsing runs in the Celery worker, isolated and resource-limited (memory, time, pages, rows, cells). ## Consequences - No row influences the ledger before explicit confirm; this invariant is enforced by tests (import confirm/revert idempotence). +- Unresolved HOLD rows remain visible in the import detail API (with optional resolution metadata) and keep blocking their month until settled; a HOLD can only be cleared by a unique posted match or an explicit manual action. - Parse runs and source records grow storage; acceptable at beta scale, trimmed only with the source file deletion workflow. - Adapter code must explicitly detect its layout; a mismatch fails with stable diagnostics instead of guessing. +- **Future UI (deferred by design):** the frontend must show HOLD rows with their status and resolution state and offer both manual resolution actions. The backend response fields (`bankStatus`, `holdResolution`) are optional, backward-compatible additions; no UI changes ship in this workstream. ## Alternatives rejected - In-request synchronous parsing — blocks HTTP workers on untrusted files. - Mutable "latest parse" overwrites — destroys provenance and diffability. - Aggressive auto-dedup — risks dropping real operations. +- Mutating the extracted HOLD row in place when it posts — loses the original bank fact; append-only resolution metadata preserves it. +- Treating HOLDs as ledger operations on import — misstates balances before the bank books the transaction. diff --git a/docs/parsers/private-fixture-gate.md b/docs/parsers/private-fixture-gate.md index 69b86df..75fb599 100644 --- a/docs/parsers/private-fixture-gate.md +++ b/docs/parsers/private-fixture-gate.md @@ -1,20 +1,25 @@ # Parser Private Fixture Gate Real bank statements contain personal financial data and are **never committed** -to Git (`.gitignore` at the repo root covers this; keep fixture directories -outside the repository). CI runs synthetic fixtures only. +to Git. CI runs synthetic fixtures only. Private fixtures may live either in an +external directory or in the ignored repository-local `examples/` directory. ## Contract - Synthetic parser inputs live with the backend tests and run in CI. -- Real statement files stay in a local `PRIVATE_FIXTURES_DIR`, never in Git, never in shared storage without encryption. +- Real statement files stay in `PRIVATE_FIXTURES_DIR` or the ignored repository-local `examples/` directory, never in Git and never in shared storage without encryption. +- Each private fixture directory contains an ignored `manifest.json` beside the statements. For each fixture it records the filename, adapter ID, parser version, period, row counts, control totals, balances, and HOLD totals. +- The manifest and its expected values are private financial data. They must not appear in tracked test parameters, documentation, CI inputs, artifacts, or logs. +- Private assertions report only a filename, adapter ID, dates, counts, and the name of a mismatched aggregate. They never render parsed rows, descriptions, names, account/card numbers, extracted pages, or expected financial values. - The private suite runs explicitly, on the developer's machine: ```sh PRIVATE_FIXTURES_DIR=/secure/path/fixtures uv run pytest -m private_fixtures ``` - (Exact marker/task names are finalized by the backend workstream; the entry point contract is `PRIVATE_FIXTURES_DIR` + a dedicated pytest marker.) + When `PRIVATE_FIXTURES_DIR` is unset, the suite may use the ignored + repository-local `examples/` directory. The dedicated marker keeps this suite + out of normal CI collection. ## Beta gate per adapter/contract @@ -35,11 +40,14 @@ gate must not be enabled or advertised, even if synthetic tests pass (plan | Adapter / parser version | Private periods passed | Gate | | --- | --- | --- | -| Alfa debit XLSX | — | **not passed** | -| Alfa credit XLSX | — | **not passed** | -| Ozon debit PDF | — | **not passed** | -| Ozon credit PDF | — | **not passed** | -| Rocketbank debit PDF | — | **not passed** | +| Alfa debit XLSX / 2 | 1 | **not passed** | +| Alfa credit XLSX / 1 (unregistered) | 0 | **not passed** | +| Ozon debit PDF / 2 | 1 | **not passed** | +| Ozon credit PDF / 1 (unregistered) | 0 | **not passed** | +| Rocketbank debit PDF / 2 | 1 | **not passed** | -No adapter is currently gated. Bank layout drift resets the gate: it must create -a new parser version and a review diff, never reinterpret historical data. +The three debit contracts each pass one local private fixture deterministically, +but no adapter is currently gated. A second independent real period/layout is +still required for each contract. Bank layout drift resets the gate: it must +create a new parser version and a review diff, never reinterpret historical +data. diff --git a/graphify-out/.graphify_labels.json b/graphify-out/.graphify_labels.json index a2f2b6b..ed80f98 100644 --- a/graphify-out/.graphify_labels.json +++ b/graphify-out/.graphify_labels.json @@ -1,64 +1,77 @@ { - "0": "Identity and Configuration", - "1": "Budget Domain Services", - "2": "Database Models", - "3": "Database Context and Exports", - "4": "Statement Import Services", - "5": "Parser Adapters", - "6": "Ledger Services", - "7": "API Domain Types", - "8": "Authentication UI", - "9": "Ledger API", - "10": "Import Review UI", - "11": "TypeScript Configuration", - "12": "Analytics UI", - "13": "API and Analytics", - "14": "Frontend Development Dependencies", - "15": "Accounts UI", - "16": "Frontend Runtime Dependencies", - "17": "Import Detail Mutations", - "18": "Operations UI", - "19": "Query Keys and Operations", - "20": "Frontend Bootstrap", - "21": "Frontend Test Support", - "22": "Budget UI", - "23": "HTTP API Client", - "24": "Form Fields", - "25": "Infrastructure and CI", - "26": "Core Finance Domain", - "27": "Frontend Scripts", - "28": "Settings and Data Lifecycle", - "29": "Budget Allocation Model", - "30": "MVP and Import Design", - "31": "Data Security and Recovery", - "32": "Ledger Concepts", - "33": "Parser Versioning", - "34": "Frontend Architecture", - "35": "Beta Release Governance", - "36": "Package Metadata", - "37": "Tenant Isolation", - "38": "Deletion and Restore", - "39": "API Generation Script", - "40": "OpenAPI Schema", - "41": "API Integration Tests", - "42": "Localization Tests", - "43": "Income Classification", - "44": "Accessibility ESLint", - "45": "JavaScript Globals", - "46": "JSDOM Dependency", - "47": "React Type Definitions", - "48": "React DOM Types", - "49": "TypeScript ESLint", - "50": "Vite React Plugin", - "51": "Vitest Dependency", - "52": "Favicon Design", - "53": "Database Role Initialization", - "54": "Application Package", - "55": "Spaces Package", - "56": "Payee Concept", - "57": "ESLint Configuration", - "58": "Money Utilities Tests", - "59": "Vite Environment Types", - "60": "Vite Configuration", - "61": "Backend Package Metadata" -} \ No newline at end of file + "0": "identity/router.py", + "1": "budgeting/router.py", + "2": "Base", + "3": "set_space_context", + "4": "imports/router.py", + "5": "alfa_xlsx.py", + "6": "Session", + "7": "types.ts", + "8": "router.tsx", + "9": "ledger/router.py", + "10": "AccountsPage.tsx", + "11": "compilerOptions", + "12": "AnalyticsPage.tsx", + "13": "analytics/router.py", + "14": "devDependencies", + "15": "test_import_holds.py", + "16": "dependencies", + "17": "ImportDetailPage.tsx", + "18": "OperationsPage.tsx", + "19": "SettingsPage.tsx", + "20": "Toaster.tsx", + "21": "ozon_pdf.py", + "22": "BudgetPage.tsx", + "23": "client.ts", + "24": "fields.tsx", + "25": "Docker Compose Development Stack", + "26": "Financial Space", + "27": "scripts", + "28": "dependencies.py", + "29": "General Pool", + "30": "TestBaseContract", + "31": "Per-File Envelope Encryption", + "32": "Operation", + "33": "Versioned Statement Imports", + "34": "Finance Frontend SPA", + "35": "Closed Beta Launch Gates", + "36": "package.json", + "37": "Financial Tracker MVP", + "38": "test_adapter_contracts.py", + "39": "generate-api.mjs", + "40": "schema.d.ts", + "41": "test_api.py", + "42": "i18n-parity.test.ts", + "43": "Expected Income", + "44": "test_private_fixtures.py", + "45": "TestAlfaDebitContract", + "46": "jsdom", + "47": "graphify", + "48": "@types/react-dom", + "49": "typescript-eslint", + "50": "SourceRow", + "51": "vitest", + "52": "Favicon SVG (ledger lines icon)", + "53": "init-app-role.sh", + "54": "app/__init__.py", + "55": "spaces/__init__.py", + "56": "Payee", + "57": "eslint.config.js", + "58": "money.test.ts", + "59": "vite-env.d.ts", + "60": "vite.config.ts", + "61": "finance-backend", + "62": "LocalDirectoryBlobStore", + "63": "account_fingerprint", + "64": "Gitea CI and real statement imports", + "65": "20260918_0002_hold_lifecycle.py", + "66": "eslint-plugin-react-hooks", + "67": "openapi-typescript", + "68": "@testing-library/dom", + "69": "@testing-library/react", + "70": "vite", + "71": "spaces/router.py", + "72": "identity/service.py", + "73": "get_settings", + "74": "ColumnChart.tsx" +} diff --git a/graphify-out/GRAPH_REPORT.md b/graphify-out/GRAPH_REPORT.md index 8b3c14c..950e06d 100644 --- a/graphify-out/GRAPH_REPORT.md +++ b/graphify-out/GRAPH_REPORT.md @@ -1,293 +1,359 @@ -# Graph Report - finance (2026-09-14) +# Graph Report - finance (2026-09-18) ## Corpus Check -- 147 files · ~51,467 words +- 149 files · ~63,829 words - Verdict: corpus is large enough that graph structure adds value. ## Summary -- 1078 nodes · 3099 edges · 62 communities (42 shown, 14 thin omitted) -- Extraction: 98% EXTRACTED · 2% INFERRED · 0% AMBIGUOUS · INFERRED: 50 edges (avg confidence: 0.93) +- 1267 nodes · 3697 edges · 75 communities (54 shown, 15 thin omitted) +- Extraction: 98% EXTRACTED · 2% INFERRED · 0% AMBIGUOUS · INFERRED: 70 edges (avg confidence: 0.93) - Token cost: 0 input · 0 output +## Graph Freshness +- Built from commit: `c85b12b9` +- Run `git rev-parse HEAD` and compare to check if the graph is stale. +- Run `graphify update .` after code changes (no API cost). + ## Community Hubs (Navigation) -- Identity and Configuration -- Budget Domain Services -- Database Models -- Database Context and Exports -- Statement Import Services -- Parser Adapters -- Ledger Services -- API Domain Types -- Authentication UI -- Ledger API -- Import Review UI -- TypeScript Configuration -- Analytics UI -- API and Analytics -- Frontend Development Dependencies -- Accounts UI -- Frontend Runtime Dependencies -- Import Detail Mutations -- Operations UI -- Query Keys and Operations -- Frontend Bootstrap -- Frontend Test Support -- Budget UI -- HTTP API Client -- Form Fields -- Infrastructure and CI -- Core Finance Domain -- Frontend Scripts -- Settings and Data Lifecycle -- Budget Allocation Model -- MVP and Import Design -- Data Security and Recovery -- Ledger Concepts -- Parser Versioning -- Frontend Architecture -- Beta Release Governance -- Package Metadata -- Tenant Isolation -- Deletion and Restore -- API Generation Script -- OpenAPI Schema -- API Integration Tests -- Localization Tests -- Income Classification -- Accessibility ESLint -- JavaScript Globals -- JSDOM Dependency -- React Type Definitions -- React DOM Types -- TypeScript ESLint -- Vite React Plugin -- Vitest Dependency -- Favicon Design -- Database Role Initialization -- Payee Concept -- Backend Package Metadata +- identity/router.py +- budgeting/router.py +- Base +- set_space_context +- imports/router.py +- alfa_xlsx.py +- Session +- types.ts +- router.tsx +- ledger/router.py +- AccountsPage.tsx +- compilerOptions +- AnalyticsPage.tsx +- analytics/router.py +- devDependencies +- test_import_holds.py +- dependencies +- ImportDetailPage.tsx +- OperationsPage.tsx +- SettingsPage.tsx +- Toaster.tsx +- ozon_pdf.py +- BudgetPage.tsx +- client.ts +- fields.tsx +- Docker Compose Development Stack +- Financial Space +- scripts +- dependencies.py +- General Pool +- TestBaseContract +- Per-File Envelope Encryption +- Operation +- Versioned Statement Imports +- Finance Frontend SPA +- Closed Beta Launch Gates +- package.json +- Financial Tracker MVP +- test_adapter_contracts.py +- generate-api.mjs +- schema.d.ts +- test_api.py +- i18n-parity.test.ts +- Expected Income +- test_private_fixtures.py +- TestAlfaDebitContract +- jsdom +- graphify +- @types/react-dom +- typescript-eslint +- SourceRow +- vitest +- Favicon SVG (ledger lines icon) +- init-app-role.sh +- Payee +- finance-backend +- LocalDirectoryBlobStore +- account_fingerprint +- Gitea CI and real statement imports +- 20260918_0002_hold_lifecycle.py +- eslint-plugin-react-hooks +- openapi-typescript +- @testing-library/dom +- @testing-library/react +- vite +- spaces/router.py +- identity/service.py +- get_settings +- ColumnChart.tsx ## God Nodes (most connected - your core abstractions) -1. `Session` - 111 edges -2. `Base` - 46 edges -3. `AppError` - 41 edges -4. `get_settings()` - 38 edges -5. `PkMixin` - 38 edges -6. `TimestampMixin` - 37 edges -7. `conflict()` - 34 edges -8. `record()` - 32 edges -9. `bad_request()` - 27 edges -10. `not_found()` - 26 edges +1. `Session` - 140 edges +2. `AppError` - 53 edges +3. `Base` - 47 edges +4. `PkMixin` - 39 edges +5. `get_settings()` - 38 edges +6. `TimestampMixin` - 38 edges +7. `conflict()` - 35 edges +8. `record()` - 34 edges +9. `UnsupportedStatementLayout` - 30 edges +10. `bad_request()` - 29 edges ## Surprising Connections (you probably didn't know these) +- `Typed Same-Origin API Client` --conceptually_related_to--> `Same-Origin Reverse Proxy` [INFERRED] + frontend/README.md → infra/README.md +- `Financial Space Ownership` --conceptually_related_to--> `Financial Space` [INFERRED] + docs/adr/0001-financial-space-ownership-and-rls.md → CONTEXT.md - `Versioned Statement Imports` --conceptually_related_to--> `Statement Import` [INFERRED] docs/adr/0003-versioned-statement-imports.md → CONTEXT.md - `MVP Release Gates` --conceptually_related_to--> `Closed Beta Launch Gates` [INFERRED] .kilo/plans/1789034192214-financial-tracker-mvp.md → docs/operations/beta-release-runbook.md -- `Financial Space Ownership` --conceptually_related_to--> `Financial Space` [INFERRED] - docs/adr/0001-financial-space-ownership-and-rls.md → CONTEXT.md -- `Typed Same-Origin API Client` --conceptually_related_to--> `Same-Origin Reverse Proxy` [INFERRED] - frontend/README.md → infra/README.md -- `Compose and Nginx Validation` --references--> `Docker Compose Development Stack` [EXTRACTED] - .github/workflows/ci.yml → infra/docker-compose.yml +- `Carry-Over Budget Reserves` --references--> `Savings Envelope` [EXTRACTED] + docs/adr/0002-journal-and-budget-reserves.md → CONTEXT.md ## Import Cycles - None detected. ## Hyperedges (group relationships) -- **Financial Space Tenancy Boundary** — context_financial_space, context_membership, docs_adr_0001_financial_space_ownership_and_rls_defense_in_depth_authorization, docs_adr_0001_financial_space_ownership_and_rls_transaction_local_rls, docs_adr_0001_financial_space_ownership_and_rls_cross_space_integrity [EXTRACTED 1.00] - **Auditable Financial Model** — context_operation, context_posting, docs_adr_0002_journal_and_budget_reserves_derived_balances, docs_adr_0002_journal_and_budget_reserves_plan_funding_separation, docs_adr_0002_journal_and_budget_reserves_closure_snapshots [EXTRACTED 1.00] +- **Financial Space Tenancy Boundary** — context_financial_space, context_membership, docs_adr_0001_financial_space_ownership_and_rls_defense_in_depth_authorization, docs_adr_0001_financial_space_ownership_and_rls_transaction_local_rls, docs_adr_0001_financial_space_ownership_and_rls_cross_space_integrity [EXTRACTED 1.00] - **Safe Statement Import Flow** — context_statement_import, context_parse_run, context_source_record, docs_adr_0003_versioned_statement_imports_immutable_statement_document, docs_adr_0003_versioned_statement_imports_conservative_deduplication, docs_adr_0003_versioned_statement_imports_atomic_confirm_revert [EXTRACTED 1.00] -## Communities (62 total, 14 thin omitted) +## Communities (75 total, 15 thin omitted) -### Community 0 - "Identity and Configuration" -Cohesion: 0.05 -Nodes (87): get_settings(), Settings, current_context(), current_session(), alias, Context, CurrentSession, Db (+79 more) +### Community 0 - "identity/router.py" +Cohesion: 0.20 +Nodes (22): accept_invite(), AcceptInviteBody, confirm_password_reset(), create_invite(), InviteBody, login(), LoginBody, logout() (+14 more) -### Community 1 - "Budget Domain Services" -Cohesion: 0.09 -Nodes (84): balances(), cash_flow(), category_spend(), plan_fact(), date, UUID, BudgetPeriod, Calendar month budget period in the space timezone (month granularity). (+76 more) +### Community 1 - "budgeting/router.py" +Cohesion: 0.20 +Nodes (34): AssignmentItem, assignments(), AssignmentsBody, categories(), CategoryBody, CategoryFundingBody, close(), close_frontend() (+26 more) -### Community 2 - "Database Models" -Cohesion: 0.08 -Nodes (64): downgrade(), upgrade(), AuditEvent, Append-only domain/security event. Never stores raw financial payloads., ExpectedIncome, ExpectedIncomeMatch, FundingMovement, PeriodClosure (+56 more) - -### Community 3 - "Database Context and Exports" +### Community 2 - "Base" Cohesion: 0.06 -Nodes (46): get_db(), get_session_factory(), Any, sessionmaker, Transaction-local RLS context. No-op on SQLite (tests); fail-closed on…, Transactional scope for workers/endpoints that own the transaction lifecycle., session_scope(), set_space_context() (+38 more) +Nodes (87): downgrade(), upgrade(), AuditEvent, Append-only domain/security event. Never stores raw financial payloads., ExpectedIncome, ExpectedIncomeMatch, FundingMovement, PeriodClosure (+79 more) -### Community 4 - "Statement Import Services" -Cohesion: 0.10 -Nodes (55): detect_adapter(), magic_kind(), new_object_key(), decrypt_file(), encrypt_file(), EncryptedFile, Per-file random DEK; DEK wrapped with the KEK from the secret manager., Crypto-shred hook: deleting wrapped DEK rows in DB plus blobs removes access. (+47 more) - -### Community 5 - "Parser Adapters" -Cohesion: 0.08 -Nodes (42): AppError, Any, Domain error rendered as a machine-readable JSON error payload., unprocessable(), AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter, AlfaXlsxAdapterBase, _cell_str() (+34 more) - -### Community 6 - "Ledger Services" -Cohesion: 0.14 -Nodes (48): Any, record(), get_income_source(), conflict(), not_found(), confirm_document(), Operation, Posting (+40 more) - -### Community 7 - "API Domain Types" +### Community 3 - "set_space_context" Cohesion: 0.07 -Nodes (39): AccountStatus, BalanceItem, BalancesReport, BudgetBlocker, BudgetCategory, BudgetStatus, BudgetSummary, BudgetWarning (+31 more) +Nodes (43): get_db(), get_engine(), get_session_factory(), Any, sessionmaker, Transaction-local RLS context. No-op on SQLite (tests); fail-closed on…, Transactional scope for workers/endpoints that own the transaction lifecycle., session_scope() (+35 more) -### Community 8 - "Authentication UI" -Cohesion: 0.10 -Nodes (33): AcceptInvitePayload, LoginPayload, PasswordResetPayload, AcceptInvitePage, AccountsPage, BudgetPage, LoginPage, OperationsPage (+25 more) +### Community 4 - "imports/router.py" +Cohesion: 0.15 +Nodes (33): confirm(), ConfirmBody, decide_row(), DecisionBody, delete_source(), detail(), HoldResolveBody, imports() (+25 more) -### Community 9 - "Ledger API" +### Community 5 - "alfa_xlsx.py" +Cohesion: 0.14 +Nodes (29): AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter, _AlfaHeaderMeta, AlfaXlsxAdapterBase, _cell_str(), _debit_find_header(), _debit_header_metadata(), _debit_header_value() (+21 more) + +### Community 6 - "Session" +Cohesion: 0.07 +Nodes (112): balances(), plan_fact(), Any, record(), BudgetPeriod, Calendar month budget period in the space timezone (month granularity)., add_expected_income(), archive_category() (+104 more) + +### Community 7 - "types.ts" +Cohesion: 0.05 +Nodes (53): api, AcceptInvitePayload, Account, AccountStatus, BalanceItem, BalancesReport, BudgetBlocker, BudgetCategory (+45 more) + +### Community 8 - "router.tsx" +Cohesion: 0.09 +Nodes (35): AcceptInvitePage, AccountsPage, BudgetPage, ImportDetailPage, ImportsPage, LoginPage, OperationsPage, RequireAuth() (+27 more) + +### Community 9 - "ledger/router.py" Cohesion: 0.15 Nodes (31): account_json(), AccountBody, accounts(), archive_account(), create_account(), create_operation(), delete_account(), operation_json() (+23 more) -### Community 10 - "Import Review UI" -Cohesion: 0.11 -Nodes (21): Operation, ImportsPage, NotFoundPage, ReviewQueuePage, Badge(), BadgeTone, EmptyState(), EmptyStateProps (+13 more) +### Community 10 - "AccountsPage.tsx" +Cohesion: 0.21 +Nodes (13): AccountType, CreateAccountPayload, ConfirmDialog(), ConfirmDialogProps, Modal(), ModalProps, useAccounts(), useArchiveAccount() (+5 more) -### Community 11 - "TypeScript Configuration" +### Community 11 - "compilerOptions" Cohesion: 0.07 Nodes (28): compilerOptions, baseUrl, esModuleInterop, forceConsistentCasingInFileNames, isolatedModules, jsx, lib, module (+20 more) -### Community 12 - "Analytics UI" -Cohesion: 0.12 -Nodes (19): AnalyticsPage, CashFlowColumn, ColumnChart(), ColumnChartProps, BarRow, HorizontalBars(), HorizontalBarsProps, PlanFactBars() (+11 more) - -### Community 13 - "API and Analytics" +### Community 12 - "AnalyticsPage.tsx" Cohesion: 0.14 -Nodes (22): balances(), cash_flow(), category_spend(), plan_fact(), Context, date, Db, get (+14 more) +Nodes (15): AnalyticsPage, NotFoundPage, PlanFactBars(), PlanFactBarsProps, PlanFactRow, MonthSwitcher(), MonthSwitcherProps, PageHeader() (+7 more) -### Community 14 - "Frontend Development Dependencies" -Cohesion: 0.10 -Nodes (21): eslint, @eslint/js, eslint-plugin-react-hooks, devDependencies, eslint, @eslint/js, eslint-plugin-react-hooks, openapi-typescript (+13 more) +### Community 13 - "analytics/router.py" +Cohesion: 0.27 +Nodes (12): balances(), cash_flow(), category_spend(), plan_fact(), Context, date, Db, get (+4 more) -### Community 15 - "Accounts UI" -Cohesion: 0.16 -Nodes (15): Account, AccountType, ConfirmDialog(), ConfirmDialogProps, Modal(), ModalProps, useAccounts(), useArchiveAccount() (+7 more) +### Community 14 - "devDependencies" +Cohesion: 0.11 +Nodes (19): eslint, @eslint/js, eslint-plugin-jsx-a11y, devDependencies, eslint, @eslint/js, eslint-plugin-jsx-a11y, globals (+11 more) -### Community 16 - "Frontend Runtime Dependencies" +### Community 15 - "test_import_holds.py" +Cohesion: 0.13 +Nodes (43): AppError, Domain error rendered as a machine-readable JSON error payload., normalize_description(), date, record_fingerprint(), add_record(), _draft(), hold_document() (+35 more) + +### Community 16 - "dependencies" Cohesion: 0.11 Nodes (19): dependencies, @hookform/resolvers, i18next, react, react-dom, react-hook-form, react-i18next, react-router-dom (+11 more) -### Community 17 - "Import Detail Mutations" -Cohesion: 0.27 -Nodes (15): RowDecision, ImportDetailPage, useToast(), useConfirmImport(), useImport(), useImportErrorToaster(), useReparseImport(), useRevertImport() (+7 more) +### Community 17 - "ImportDetailPage.tsx" +Cohesion: 0.33 +Nodes (13): useToast(), useConfirmImport(), useImport(), useImportErrorToaster(), useReparseImport(), useRevertImport(), useSaveRowDecision(), useUploadImport() (+5 more) -### Community 18 - "Operations UI" -Cohesion: 0.27 -Nodes (14): useActiveAccounts(), useCreateOperation(), useLedgerMutation(), useOperations(), useVoidOperation(), amountField(), dateField(), ExpenseForm() (+6 more) +### Community 18 - "OperationsPage.tsx" +Cohesion: 0.14 +Nodes (27): CreateOperationPayload, Badge(), BadgeTone, EmptyState(), EmptyStateProps, QueryGate(), QueryGateProps, useActiveAccounts() (+19 more) -### Community 19 - "Query Keys and Operations" -Cohesion: 0.22 -Nodes (10): api, invalidationRoots, queryKeys, SESSION_ROOT, sessionInvalidationRoot, SPACE_ROOT, spaceInvalidationRoot, OperationFilters (+2 more) +### Community 19 - "SettingsPage.tsx" +Cohesion: 0.16 +Nodes (16): invalidationRoots, queryKeys, SESSION_ROOT, sessionInvalidationRoot, SPACE_ROOT, spaceInvalidationRoot, OperationFilters, useCancelDeletion() (+8 more) -### Community 20 - "Frontend Bootstrap" -Cohesion: 0.17 -Nodes (10): Locale, App(), queryClient, AppRoutes(), createI18n(), detectLocale(), i18n, LOCALES (+2 more) +### Community 20 - "Toaster.tsx" +Cohesion: 0.12 +Nodes (15): Locale, App(), queryClient, AppRoutes(), ToastApi, ToastContext, ToastItem, ToastKind (+7 more) -### Community 21 - "Frontend Test Support" -Cohesion: 0.17 -Nodes (12): User, ToastApi, ToastContext, ToastItem, ToastKind, ToastProvider(), createTestQueryClient(), renderWithProviders() (+4 more) +### Community 21 - "ozon_pdf.py" +Cohesion: 0.14 +Nodes (13): Protocol, StatementAdapter, StatementDraft, _first_page_text(), _ozon_cell(), _ozon_normalize(), OzonCreditPdfAdapter, OzonDebitPdfAdapter (+5 more) -### Community 22 - "Budget UI" +### Community 22 - "BudgetPage.tsx" Cohesion: 0.36 Nodes (12): useBudgetCategories(), useBudgetErrorToaster(), useBudgetSummary(), useCloseMonth(), useCopyPreviousMonth(), useFundCategory(), usePeriodMutation(), useReopenMonth() (+4 more) -### Community 23 - "HTTP API Client" +### Community 23 - "client.ts" Cohesion: 0.23 Nodes (10): apiClient, ApiError, buildUrl(), ErrorEnvelope, parseErrorEnvelope(), QueryValue, readCsrfToken(), request() (+2 more) -### Community 24 - "Form Fields" -Cohesion: 0.23 -Nodes (10): FieldChrome, InputProps, MoneyField(), MoneyFieldProps, SelectFieldProps, SelectProps, TextareaField(), TextareaFieldProps (+2 more) +### Community 24 - "fields.tsx" +Cohesion: 0.21 +Nodes (11): FieldChrome, InputProps, MoneyField(), MoneyFieldProps, SelectField(), SelectFieldProps, SelectProps, TextareaField() (+3 more) -### Community 25 - "Infrastructure and CI" -Cohesion: 0.18 -Nodes (12): Backend Quality Gate, CI Pipeline, Compose and Nginx Validation, Frontend Quality Gate, Commodity Infrastructure Services, Docker Compose Development Stack, Internal Backend Network, Same-Origin Frontend Service (+4 more) +### Community 25 - "Docker Compose Development Stack" +Cohesion: 0.29 +Nodes (8): Commodity Infrastructure Services, Docker Compose Development Stack, Internal Backend Network, Same-Origin Frontend Service, Worker Parsing Isolation, Finance Infrastructure, Same-Origin Reverse Proxy, Backend and Frontend Workstream Contracts -### Community 26 - "Core Finance Domain" -Cohesion: 0.22 -Nodes (9): Credit Reserve, Financial Account, Financial Space, Membership, User, Carry-Over Budget Reserves, Closed Beta Zero-Based Budgeting, Finance Tracker (+1 more) +### Community 26 - "Financial Space" +Cohesion: 0.17 +Nodes (12): Credit Reserve, Financial Account, Financial Space, Membership, User, Carry-Over Budget Reserves, Provider-Neutral Data Boundary, Russian Federation Data Residency (+4 more) -### Community 27 - "Frontend Scripts" +### Community 27 - "scripts" Cohesion: 0.22 Nodes (9): scripts, api:generate, build, dev, lint, preview, test, test:watch (+1 more) -### Community 28 - "Settings and Data Lifecycle" -Cohesion: 0.44 -Nodes (7): useCancelDeletion(), useRequestDeletion(), useRequestExport(), useSpaceDeletion(), DeletionForm(), DeletionFormValues, SettingsPage() +### Community 28 - "dependencies.py" +Cohesion: 0.15 +Nodes (21): current_context(), current_session(), alias, Context, CurrentSession, Db, Header, User (+13 more) -### Community 29 - "Budget Allocation Model" +### Community 29 - "General Pool" Cohesion: 0.29 Nodes (8): Budget Period, Funding, General Pool, Monthly Category, Plan Assignment, Savings Envelope, Immutable Closure Snapshots, Plan and Funding Separation -### Community 30 - "MVP and Import Design" -Cohesion: 0.25 -Nodes (8): Atomic Import Confirm and Revert, Conservative Deduplication, Versioned Statement Imports, Budget Invariants, Critical User Path, Financial Tracker MVP, Statement Import Pipeline, Modular Monolith +### Community 30 - "TestBaseContract" +Cohesion: 0.17 +Nodes (5): detect_adapter(), make_draft(), make_source_row(), TestBaseContract, Workbook -### Community 31 - "Data Security and Recovery" -Cohesion: 0.25 -Nodes (8): Immutable Statement Document, Per-File Envelope Encryption, Provider-Neutral Data Boundary, Russian Federation Data Residency, Backup and Restore Runbook, Mandatory KEK Backup, Quarterly Restore Drill, Provider-Neutral Infrastructure +### Community 31 - "Per-File Envelope Encryption" +Cohesion: 0.29 +Nodes (7): Immutable Statement Document, Per-File Envelope Encryption, Backup and Restore Runbook, Mandatory KEK Backup, Quarterly Restore Drill, Disaster Restore Procedure, Post-Restore Deletion Replay -### Community 32 - "Ledger Concepts" -Cohesion: 0.38 -Nodes (7): Internal Transfer, Operation, Posting, Refund, Derived Account Balances, Operations and Postings Journal, Ledger Invariants +### Community 32 - "Operation" +Cohesion: 0.47 +Nodes (6): Internal Transfer, Operation, Posting, Refund, Derived Account Balances, Operations and Postings Journal -### Community 33 - "Parser Versioning" -Cohesion: 0.33 -Nodes (7): Parse Run, Source Record, Statement Import, Parser Versioning and Review Diff, Layout Drift Re-Gating, Parser Private Fixture Gate, Private Statement Test Contract +### Community 33 - "Versioned Statement Imports" +Cohesion: 0.20 +Nodes (11): Parse Run, Source Record, Statement Import, Atomic Import Confirm and Revert, Conservative Deduplication, Parser Versioning and Review Diff, Versioned Statement Imports, Layout Drift Re-Gating (+3 more) -### Community 34 - "Frontend Architecture" +### Community 34 - "Finance Frontend SPA" Cohesion: 0.29 Nodes (7): Server-Side Secret Boundary, Grossbukh Application Shell, Accessible Responsive UI, Finance Frontend SPA, RU and EN Localization, Space-Scoped Query Keys, Typed Same-Origin API Client -### Community 35 - "Beta Release Governance" -Cohesion: 0.29 -Nodes (7): Closed Beta Release Runbook, Forward-Fix Rollback Policy, Financial Data Incident Response, Closed Beta Launch Gates, Production Launch Sequence, Adapter Beta Eligibility, MVP Release Gates +### Community 35 - "Closed Beta Launch Gates" +Cohesion: 0.20 +Nodes (11): Crypto-Shredded Deletion, Closed Beta Release Runbook, Forward-Fix Rollback Policy, Financial Data Incident Response, Closed Beta Launch Gates, Production Launch Sequence, Thirty-Day Space Deletion, Durable Deletion Ledger (+3 more) -### Community 36 - "Package Metadata" -Cohesion: 0.29 -Nodes (6): allowScripts, esbuild@0.25.12, name, private, type, version - -### Community 37 - "Tenant Isolation" -Cohesion: 0.33 -Nodes (6): Audited Break-Glass Access, Cross-Space Referential Integrity, Defense-in-Depth Authorization, Financial Space Ownership, Transaction-Local RLS Context, Tenant Boundary - -### Community 38 - "Deletion and Restore" +### Community 36 - "package.json" Cohesion: 0.40 -Nodes (6): Crypto-Shredded Deletion, Disaster Restore Procedure, Thirty-Day Space Deletion, Durable Deletion Ledger, Idempotent Resumable Purge, Post-Restore Deletion Replay +Nodes (4): name, private, type, version -### Community 39 - "API Generation Script" +### Community 37 - "Financial Tracker MVP" +Cohesion: 0.18 +Nodes (11): Audited Break-Glass Access, Cross-Space Referential Integrity, Defense-in-Depth Authorization, Financial Space Ownership, Transaction-Local RLS Context, Budget Invariants, Critical User Path, Financial Tracker MVP (+3 more) + +### Community 38 - "test_adapter_contracts.py" +Cohesion: 0.11 +Nodes (9): minimal_pdf(), _QuotaSettings, Sanitized synthetic contract tests for the evidenced statement adapters. These…, Build a small valid single-font PDF with empty or ASCII-only pages., same_page_rocket_lines(), synthetic_rocket_lines(), TestMoneyParsing, TestRocketbankDebitContract (+1 more) + +### Community 39 - "generate-api.mjs" Cohesion: 0.40 Nodes (5): firstReadable(), LOCAL_CANDIDATES, main(), OUT_DIR, REMOTE_CANDIDATES -### Community 40 - "OpenAPI Schema" +### Community 40 - "schema.d.ts" Cohesion: 0.33 Nodes (5): components, $defs, operations, paths, webhooks -### Community 41 - "API Integration Tests" +### Community 41 - "test_api.py" Cohesion: 0.67 Nodes (3): TestClient, test_invite_login_and_authenticated_space(), test_openapi_contains_critical_paths() +### Community 44 - "test_private_fixtures.py" +Cohesion: 0.12 +Nodes (22): magic_kind(), _draft_json(), Any, Reconcile parsed rows against declared bank controls with stable diagnostics.…, _reconciliation_summary(), _check(), _fail(), _load_entries() (+14 more) + +### Community 45 - "TestAlfaDebitContract" +Cohesion: 0.21 +Nodes (3): alfa_workbook_bytes(), synthetic_alfa_rows(), TestAlfaDebitContract + +### Community 47 - "graphify" +Cohesion: 0.17 +Nodes (11): command, enabled, timeout, type, mcp, graphify, plugin, $schema (+3 more) + +### Community 50 - "SourceRow" +Cohesion: 0.19 +Nodes (9): SourceRow, _date_from(), _PendingRecord, date, Rocketbank debit statement contract, evidenced against the real document.…, RocketbankDebitPdfAdapter, open_pdf(), _quota_error() (+1 more) + +### Community 62 - "LocalDirectoryBlobStore" +Cohesion: 0.39 +Nodes (3): LocalDirectoryBlobStore, Path, S3-neutral local store; production swaps in an S3-compatible client. + +### Community 63 - "account_fingerprint" +Cohesion: 0.25 +Nodes (3): account_fingerprint(), ozon_extracted(), TestOzonDebitContract + +### Community 64 - "Gitea CI and real statement imports" +Cohesion: 0.29 +Nodes (6): Confirmed decisions, Gitea CI and real statement imports, Goal and constraints, Implementation plan, Principal risks, Validation and rollout + +### Community 65 - "20260918_0002_hold_lifecycle.py" +Cohesion: 0.80 +Nodes (4): downgrade(), _has_check_constraint(), _has_column(), upgrade() + +### Community 71 - "spaces/router.py" +Cohesion: 0.16 +Nodes (19): send_token_email(), cancel_deletion(), current(), deletion_status(), DeletionBody, invite_member(), MemberInviteBody, alias (+11 more) + +### Community 72 - "identity/service.py" +Cohesion: 0.28 +Nodes (18): unauthorized(), accept_invitation(), authenticate(), create_invitation(), create_password_reset(), create_session(), get_active_session(), normalize_email() (+10 more) + +### Community 73 - "get_settings" +Cohesion: 0.16 +Nodes (12): get_settings(), Settings, create_celery_app(), encrypt_file(), EncryptedFile, Per-file random DEK; DEK wrapped with the KEK from the secret manager., Crypto-shred hook: deleting wrapped DEK rows in DB plus blobs removes access., shred_dek_material() (+4 more) + +### Community 74 - "ColumnChart.tsx" +Cohesion: 0.24 +Nodes (7): CashFlowColumn, ColumnChart(), ColumnChartProps, BarRow, HorizontalBars(), HorizontalBarsProps, rows + ## Knowledge Gaps -- **176 isolated node(s):** `finance-backend`, `name`, `private`, `version`, `type` (+171 more) - These have ≤1 connection - possible missing edges or undocumented components. (Counts symbols only; 296 node(s) total have ≤1 connection when file, concept and rationale nodes are included.) -- **14 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes. +- **185 isolated node(s):** `$schema`, `file:///home/k1nq/repos/finance/.kilo/plugins/graphify.js`, `type`, `/home/k1nq/.local/bin/graphify-mcp`, `/home/k1nq/repos/finance/graphify-out/graph.json` (+180 more) + These have ≤1 connection - possible missing edges or undocumented components. (Counts symbols only; 335 node(s) total have ≤1 connection when file, concept and rationale nodes are included.) +- **15 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes. ## Suggested Questions _Questions this graph is uniquely positioned to answer:_ -- **Why does `Session` connect `Budget Domain Services` to `Identity and Configuration`, `Database Models`, `Database Context and Exports`, `Statement Import Services`, `Ledger Services`?** - _High betweenness centrality (0.072) - this node is a cross-community bridge._ -- **Why does `AppError` connect `Parser Adapters` to `Identity and Configuration`, `Budget Domain Services`, `Database Models`, `Database Context and Exports`, `Statement Import Services`, `Ledger Services`, `API and Analytics`?** - _High betweenness centrality (0.035) - this node is a cross-community bridge._ -- **Why does `get_settings()` connect `Identity and Configuration` to `Database Models`, `Database Context and Exports`, `Statement Import Services`, `Parser Adapters`, `API and Analytics`?** - _High betweenness centrality (0.024) - this node is a cross-community bridge._ +- **Why does `AppError` connect `test_import_holds.py` to `identity/router.py`, `Base`, `set_space_context`, `imports/router.py`, `alfa_xlsx.py`, `Session`, `test_adapter_contracts.py`, `identity/service.py`, `get_settings`, `SourceRow`, `dependencies.py`, `account_fingerprint`?** + _High betweenness centrality (0.078) - this node is a cross-community bridge._ +- **Why does `Session` connect `Session` to `Base`, `set_space_context`, `identity/service.py`, `analytics/router.py`, `test_import_holds.py`, `dependencies.py`?** + _High betweenness centrality (0.074) - this node is a cross-community bridge._ +- **Why does `bad_request()` connect `Session` to `budgeting/router.py`, `spaces/router.py`, `identity/service.py`, `ledger/router.py`, `test_import_holds.py`?** + _High betweenness centrality (0.025) - this node is a cross-community bridge._ - **Are the 2 inferred relationships involving `Session` (e.g. with `UTCDateTime` and `test_postgres_rls_requires_matching_transaction_context()`) actually correct?** _`Session` has 2 INFERRED edges - model-reasoned connections that need verification._ +- **Are the 23 inferred relationships involving `AppError` (e.g. with `request_export()` and `login()`) actually correct?** + _`AppError` has 23 INFERRED edges - model-reasoned connections that need verification._ - **Are the 3 inferred relationships involving `Base` (e.g. with `downgrade()` and `upgrade()`) actually correct?** _`Base` has 3 INFERRED edges - model-reasoned connections that need verification._ -- **Are the 13 inferred relationships involving `AppError` (e.g. with `request_export()` and `login()`) actually correct?** - _`AppError` has 13 INFERRED edges - model-reasoned connections that need verification._ -- **What connects `finance-backend`, `name`, `private` to the rest of the system?** - _176 weakly-connected nodes found - possible documentation gaps or missing edges._ \ No newline at end of file +- **What connects `$schema`, `file:///home/k1nq/repos/finance/.kilo/plugins/graphify.js`, `type` to the rest of the system?** + _185 weakly-connected nodes found - possible documentation gaps or missing edges._ \ No newline at end of file diff --git a/graphify-out/graph.html b/graphify-out/graph.html index 22ec91c..5a48e76 100644 --- a/graphify-out/graph.html +++ b/graphify-out/graph.html @@ -63,12 +63,12 @@
-
1078 nodes · 3099 edges · 62 communities
+
1267 nodes · 3697 edges · 75 communities