Implement Gitea CI and real statement imports
Gitea Actions CI / detect (push) Successful in 3s
Gitea Actions CI / Compose & nginx validation (push) Failing after 20s
Gitea Actions CI / Frontend (lint / type / test / build) (push) Failing after 24s
Gitea Actions CI / Backend (lint / type / test / build) (push) Failing after 1m34s
Gitea Actions CI / detect (push) Successful in 3s
Gitea Actions CI / Compose & nginx validation (push) Failing after 20s
Gitea Actions CI / Frontend (lint / type / test / build) (push) Failing after 24s
Gitea Actions CI / Backend (lint / type / test / build) (push) Failing after 1m34s
This commit is contained in:
@@ -1,13 +1,15 @@
|
||||
name: CI
|
||||
name: Gitea Actions CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, main]
|
||||
pull_request:
|
||||
|
||||
# Single persistent vm-runner with host-network job containers: serialize runs
|
||||
# so PostgreSQL/Redis service ports never collide across concurrent runs.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
group: gitea-actions-ci
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# Jobs activate only when their locked dependency manifests exist.
|
||||
@@ -36,7 +38,7 @@ jobs:
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_PASSWORD: ${{ secrets.CI_POSTGRES_PASSWORD }}
|
||||
POSTGRES_DB: finance_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
@@ -55,15 +57,17 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 12
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://postgres:postgres@localhost:5432/finance_test
|
||||
DATABASE_URL: postgresql+psycopg://postgres:${{ secrets.CI_POSTGRES_PASSWORD }}@localhost:5432/finance_test
|
||||
REDIS_URL: redis://localhost:6379/0
|
||||
CELERY_BROKER_URL: redis://localhost:6379/1
|
||||
IMPORTS_KEK_B64: MDAxMTIyMzM0NDU1NjY3Nzg4ODlhYWJiY2NkZGVlZg==
|
||||
RLS_DATABASE_URL: postgresql+psycopg://finance_app_test:app-test@localhost:5432/finance_test
|
||||
IMPORTS_KEK_B64: ${{ secrets.CI_IMPORTS_KEK_B64 }}
|
||||
CI_RLS_DATABASE_PASSWORD: ${{ secrets.CI_RLS_DATABASE_PASSWORD }}
|
||||
CI_MINIO_ROOT_PASSWORD: ${{ secrets.CI_MINIO_ROOT_PASSWORD }}
|
||||
RLS_DATABASE_URL: postgresql+psycopg://finance_app_test:${{ secrets.CI_RLS_DATABASE_PASSWORD }}@localhost:5432/finance_test
|
||||
S3_ENDPOINT_URL: http://localhost:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_ACCESS_KEY_ID: minioadmin
|
||||
S3_SECRET_ACCESS_KEY: minioadmin
|
||||
S3_SECRET_ACCESS_KEY: ${{ secrets.CI_MINIO_ROOT_PASSWORD }}
|
||||
S3_BUCKET_STATEMENTS: finance-statements
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -83,22 +87,31 @@ jobs:
|
||||
run: uv run mypy .
|
||||
- name: Create restricted app role and migrate
|
||||
run: |
|
||||
uv run python -c "from sqlalchemy import create_engine,text; e=create_engine('${DATABASE_URL}'); c=e.connect(); c.execute(text(\"CREATE ROLE finance_app_test LOGIN PASSWORD 'app-test' NOSUPERUSER NOBYPASSRLS\")); c.execute(text('GRANT CONNECT ON DATABASE finance_test TO finance_app_test')); c.execute(text('GRANT USAGE ON SCHEMA public TO finance_app_test')); c.execute(text('ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO finance_app_test')); c.commit(); c.close(); e.dispose()"
|
||||
uv run python -c "import os; from psycopg import sql; from sqlalchemy import create_engine,text; e=create_engine(os.environ['DATABASE_URL']); c=e.connect(); raw=c.connection.driver_connection; role=sql.SQL('CREATE ROLE finance_app_test LOGIN PASSWORD {} NOSUPERUSER NOBYPASSRLS').format(sql.Literal(os.environ['CI_RLS_DATABASE_PASSWORD'])).as_string(raw); c.exec_driver_sql(role); c.execute(text('GRANT CONNECT ON DATABASE finance_test TO finance_app_test')); c.execute(text('GRANT USAGE ON SCHEMA public TO finance_app_test')); c.execute(text('ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO finance_app_test')); c.commit(); c.close(); e.dispose()"
|
||||
uv run alembic upgrade head
|
||||
# docker.io denies minio/minio from this network (run 246); quay.io serves it.
|
||||
# Unique container name keeps parallel/retried runs off each other.
|
||||
- name: Start MinIO (S3-compatible test storage)
|
||||
run: |
|
||||
docker run -d --name minio-test -p 9000:9000 \
|
||||
-e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin \
|
||||
minio/minio:latest server /data
|
||||
docker run -d --name minio-ci-${{ github.run_id }}-${{ github.run_attempt }} -p 9000:9000 \
|
||||
-e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD \
|
||||
quay.io/minio/minio:latest server /data
|
||||
for i in $(seq 1 30); do
|
||||
curl -sf http://localhost:9000/minio/health/live && exit 0
|
||||
if curl -sf http://localhost:9000/minio/health/live > /dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "MinIO did not become healthy" && exit 1
|
||||
- name: Test (pytest)
|
||||
run: uv run pytest -ra
|
||||
- name: Build provider-neutral image
|
||||
run: docker build -f ../infra/docker/Dockerfile.backend -t finance-api:ci .
|
||||
run: docker build -f ../infra/docker/Dockerfile.backend -t finance-api:ci-${{ github.run_id }}-${{ github.run_attempt }} .
|
||||
- name: Cleanup Docker resources
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f minio-ci-${{ github.run_id }}-${{ github.run_attempt }} || true
|
||||
docker rmi -f finance-api:ci-${{ github.run_id }}-${{ github.run_attempt }} || true
|
||||
|
||||
frontend:
|
||||
name: Frontend (lint / type / test / build)
|
||||
@@ -110,9 +123,11 @@ jobs:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# pnpm 10 is required: frontend/pnpm-workspace.yaml is config-only
|
||||
# (allowBuilds) and pnpm 9 rejects it with "packages field missing or empty".
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 9
|
||||
version: 10
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
@@ -132,12 +147,31 @@ jobs:
|
||||
compose:
|
||||
name: Compose & nginx validation
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Unique project name per run: this runner reuses the host, and the
|
||||
# compose file pins "name: finance", which would collide across runs.
|
||||
COMPOSE_PROJECT_NAME: finance-ci-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# Backend job binds 5432/6379/9000 on the shared host network in the
|
||||
# same run, so compose services use non-overlapping host ports.
|
||||
POSTGRES_HOST_PORT: "15432"
|
||||
REDIS_HOST_PORT: "16379"
|
||||
MINIO_API_HOST_PORT: "19000"
|
||||
MINIO_CONSOLE_HOST_PORT: "19001"
|
||||
MAILPIT_SMTP_HOST_PORT: "11025"
|
||||
MAILPIT_UI_HOST_PORT: "18025"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Prepare env from example (placeholders only)
|
||||
run: cp infra/.env.example infra/.env
|
||||
- name: Validate compose file
|
||||
run: docker compose -f infra/docker-compose.yml config -q
|
||||
# docker.io denies minio/minio from this network (run 246); pre-pull from
|
||||
# quay.io and retag so compose finds the image locally and never pulls
|
||||
# the blocked reference.
|
||||
- name: Pre-pull MinIO via quay.io
|
||||
run: |
|
||||
docker pull quay.io/minio/minio:latest
|
||||
docker tag quay.io/minio/minio:latest minio/minio:latest
|
||||
- name: Boot infra services and wait for healthchecks
|
||||
run: docker compose -f infra/docker-compose.yml up -d --wait --wait-timeout 180 postgres redis minio mailpit
|
||||
- name: Show service status
|
||||
@@ -208,3 +208,5 @@ test-results/
|
||||
.graphify_root
|
||||
cost.json
|
||||
graphify-out/cache/
|
||||
|
||||
examples/
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# Gitea CI and real statement imports
|
||||
|
||||
## Goal and constraints
|
||||
|
||||
- Move CI to the canonical Gitea Actions location and make the currently failing pipeline pass on the existing `vm-runner`.
|
||||
- Store only CI-only credentials in Gitea repository Actions secrets; never copy local or future production secrets into Gitea.
|
||||
- Replace guessed behavior for Alfa debit XLSX, Ozon debit PDF, and Rocketbank debit PDF with contracts proven against the ignored files in `examples/`.
|
||||
- Never commit real statements, extracted text, full parser snapshots, or private expected values.
|
||||
- Keep `alfa_credit_xlsx` and `ozon_credit_pdf` code untouched but remove them from the active adapter registry until real fixtures exist.
|
||||
- Do not implement or redesign frontend UI. Retain its existing CI checks, and document the future HOLD UI contract in ADR 0003.
|
||||
|
||||
## Confirmed decisions
|
||||
|
||||
- `examples/ozon.pdf` is an Ozon debit statement.
|
||||
- Repository-local `examples/` is an allowed private-fixture location because it is ignored; external `PRIVATE_FIXTURES_DIR` remains supported.
|
||||
- Alfa `HOLD` rows are immutable source records, but they do not create ledger operations during normal import confirmation.
|
||||
- An unresolved HOLD blocks closing the budget month containing its `occurred_on` date.
|
||||
- A later confirmed statement can resolve a HOLD automatically when exactly one conservative match becomes posted.
|
||||
- Manual HOLD resolution has two backend actions, both with a required reason: `confirm_as_posted` creates a normal operation; `discard_as_cancelled` creates none.
|
||||
- Gitea secrets to create are `CI_POSTGRES_PASSWORD`, `CI_RLS_DATABASE_PASSWORD`, `CI_MINIO_ROOT_PASSWORD`, and `CI_IMPORTS_KEK_B64`.
|
||||
|
||||
## Implementation plan
|
||||
|
||||
1. **Lock the private-data boundary before parser work.**
|
||||
- Preserve the existing user change adding `/examples/` to `.gitignore`; verify `git ls-files examples` remains empty.
|
||||
- Update `docs/parsers/private-fixture-gate.md` to allow an ignored repo-local fixture directory as well as an external one.
|
||||
- Add a private manifest contract (`manifest.json` beside the statements) containing adapter ID, parser version, period, row counts, control totals, balances, and HOLD totals. Create the current manifest under ignored `examples/`; never commit it.
|
||||
- Ensure private-test failures report only filenames, adapter IDs, counts, dates, and aggregate mismatches, not raw descriptions, names, account/card numbers, or extracted pages.
|
||||
|
||||
2. **Move and repair Gitea Actions CI.**
|
||||
- Move `.github/workflows/ci.yml` to `.gitea/workflows/ci.yml` and rename the workflow to make Gitea ownership explicit. Keep compatibility names such as `${{ github.ref }}` and `$GITHUB_OUTPUT` where Gitea deliberately exposes the GitHub-compatible context.
|
||||
- Use the configured `origin` (`k1nq/finance` on `git.k1nq.space`) and installed `tea` login. List existing Actions secret names first, generate new independent CI-only values, and create/update the four agreed secrets through `tea actions secrets`; do not print, persist, or reuse values from `infra/.env`.
|
||||
- Replace hard-coded PostgreSQL, restricted-role, MinIO, and KEK values in the workflow with `${{ secrets.* }}`. Keep non-sensitive usernames, database names, endpoints, and bucket names in YAML.
|
||||
- Retrieve the authenticated logs for Gitea run `246` before changing job mechanics. The observable failures are frontend dependency installation, backend MinIO startup, and Compose infrastructure startup; fix the concrete causes rather than assuming GitHub runner behavior.
|
||||
- Make Docker resources safe on the persistent VM runner: unique container/project names, an `if: always()` cleanup step, and either serialized backend/Compose jobs or non-overlapping host ports if logs confirm shared-port collisions. Do not weaken tests to obtain a green run.
|
||||
- Keep the frontend job limited to its existing install/lint/type/test/build checks. If its install failure is lockfile/tooling-related, make only the minimal package-manager/CI correction; do not change UI code.
|
||||
|
||||
3. **Make the parser boundary explicit and internally consistent.**
|
||||
- In `backend/app/imports/adapters/base.py`, make adapter output canonical: `amount_kopecks` is always positive, `direction` carries the sign, and each row has an immutable bank status (`posted` or `hold`).
|
||||
- Replace the shared global parser version for changed adapters with per-contract versions. Use a new version for the three rewritten contracts so existing parse runs can never be confused with the old guessed output.
|
||||
- Extend `StatementDraft` with independent bank control totals: declared inflow, declared outflow, and optional pending/HOLD total. Persist them in the existing parse-run draft JSON.
|
||||
- Refactor reconciliation in `imports/service.py`: compare parsed posted inflow/outflow with declared totals; compare HOLD totals separately; where both balances exist, also require `opening + inflow - outflow == closing`. A mismatch stays review-blocking with stable diagnostic fields.
|
||||
- Preserve raw descriptions only in encrypted/import-domain storage already designed for them; logs and test diagnostics must remain redacted.
|
||||
|
||||
4. **Rewrite the three evidenced adapters against their actual layouts.**
|
||||
- **Alfa debit XLSX:** detect the exact workbook structure (`Выписка по счету`, account metadata, period fields, and operation headers) rather than a nonexistent `Альфа`/`дебет` marker. Parse operation date, posting date, code, bank category, description, amount, status, MCC, and location. Treat a `HOLD` posting-date cell as `bank_status=hold`; stop before signatures/footer. Parse current balance, posted totals, and unconfirmed total from the header.
|
||||
- **Ozon debit PDF:** detect the legal-bank/title/account/table structure without requiring a nonexistent debit marker. Extract table rows by PDF table/word geometry so multiline document IDs and descriptions remain attached to one transaction. Parse period, account fingerprint input, opening/closing balances, declared totals, external document ID, timestamp date, direction, and amount.
|
||||
- **Rocketbank debit PDF:** replace inheritance from `OzonPdfAdapterBase` with an independent state-machine parser. Track Russian day headings across pages, accumulate each `time + operation code + multiline description + signed RUR amount` record, ignore repeated headers/footers and daily totals, and extract MCC/location where present. Parse statement-level period, account, balances, inflow/outflow controls.
|
||||
- Keep the Alfa/Ozon credit classes present but unregistered. `detect_adapter` must only select the three verified-format development contracts and must still fail closed for unknown layouts.
|
||||
|
||||
5. **Model HOLD lifecycle without mutating extracted facts.**
|
||||
- Add an Alembic migration and model field `SourceRecord.bank_status` with constrained values `posted|hold`, defaulting existing records to `posted`.
|
||||
- Add a tenant-scoped, one-per-HOLD `HoldResolution` record rather than overwriting the extracted source row. Store resolution kind (`matched_posted`, `confirmed_as_posted`, `discarded_as_cancelled`), optional resolving source record/operation/user, required manual reason, and timestamp.
|
||||
- During candidate building, match a new posted record to unresolved confirmed HOLD records only within the same space and account/statement contract. Prefer a stable external ID; otherwise require one unambiguous date, amount, direction, and normalized-description fingerprint match. Ambiguity remains review-required. Create the resolution only when the new statement is confirmed.
|
||||
- Skip unresolved HOLD rows when applying a confirmed import to the ledger, while retaining them in the active parse run and API detail response.
|
||||
- Add a CSRF-protected imports endpoint/service action for `confirm_as_posted` and `discard_as_cancelled`. Require a mapped account and reason; use ledger services for operation creation, add provenance links, honor closed-period rules, and write audit events.
|
||||
- Add unresolved active HOLDs from confirmed imports to budgeting close blockers for their `occurred_on` month. Manual confirmation may then expose the normal uncategorized-expense blocker until the created operation is categorized.
|
||||
|
||||
6. **Build layered tests without leaking fixtures.**
|
||||
- Add tracked synthetic tests for canonical amount semantics, stable detection failures, exact Alfa header mapping, Ozon multiline table grouping, Rocket day/page state transitions, totals reconciliation, HOLD parsing, and malformed/oversized/no-text cases. Generate sanitized XLSX data in tests and test PDF text/table cores with sanitized extracted structures; do not commit derivatives of the real statements.
|
||||
- Add `private_fixtures` to pytest markers and a separately collected private test module. It reads `PRIVATE_FIXTURES_DIR/manifest.json`, fully parses each real file twice, asserts deterministic output and private expectations, and never runs in normal CI.
|
||||
- Add service/database tests for: HOLD does not create an operation; confirmed unresolved HOLD blocks only its month; a unique later posted record resolves it on confirmation; ambiguous overlap remains unresolved; both manual actions require a reason; manual confirmation creates exactly one operation and late reimport links rather than duplicates it; tenant isolation and closed-period rules hold.
|
||||
- Keep the beta gate as **not passed** for all three adapters after this work because only one real statement file per contract is available. Record that one private fixture passes and that a second independent real layout/period is still required.
|
||||
|
||||
7. **Document the product contract and defer UI deliberately.**
|
||||
- Amend `docs/adr/0003-versioned-statement-imports.md` with posted/HOLD source status, append-only resolution metadata, automatic settlement matching, close blocking, both manual actions, and the future UI requirement to show and resolve HOLD rows.
|
||||
- Update parser gate/status documentation with the new parser versions and local validation result, without embedding private counts, totals, identifiers, or descriptions.
|
||||
- Make no changes under `frontend/src`. Do not regenerate or redesign frontend API/UI in this workstream; backend response additions remain backward-compatible optional fields for the later design-first frontend project.
|
||||
|
||||
## Validation and rollout
|
||||
|
||||
1. Run focused adapter synthetic tests, HOLD service tests, import service tests, migration upgrade, Ruff, formatting, and mypy.
|
||||
2. Run the private suite locally with `PRIVATE_FIXTURES_DIR` pointing to `examples/`; confirm the manifest and all statement files remain ignored and untracked afterward.
|
||||
3. Run the complete backend suite. Run existing frontend checks only to validate the CI tooling correction, with no UI modifications.
|
||||
4. Validate Compose and image builds on the same Docker topology used by the Gitea VM runner, including cleanup after forced failure.
|
||||
5. Push only after local validation, then inspect the new Gitea Actions run with `tea`: all jobs green, secrets present by name, no secret values or private fixture paths/content in logs, and no `.github/workflows` workflow left active.
|
||||
6. Do not mark any adapter beta-supported until a second independent private fixture for that contract passes the full gate.
|
||||
|
||||
## Principal risks
|
||||
|
||||
- PDF text order can drift even when the visual table looks unchanged; table/word grouping must fail closed and bump parser version on layout drift.
|
||||
- Alfa HOLD rows may receive a new code or changed description when posted; only unique conservative matching may auto-resolve them.
|
||||
- Gitea VM runners can retain Docker containers and share host ports across jobs; cleanup and isolation are acceptance criteria, not optional hygiene.
|
||||
- Assertions, action logs, parser exceptions, and private manifests are all possible financial-data leak paths and must be reviewed before the first Gitea run.
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Hold lifecycle: source bank status and append-only hold resolutions."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "0002"
|
||||
down_revision = "0001"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_column(bind, table: str, column: str) -> bool:
|
||||
inspector = sa.inspect(bind)
|
||||
return column in {item["name"] for item in inspector.get_columns(table)}
|
||||
|
||||
|
||||
def _has_check_constraint(bind, table: str, suffix: str) -> bool:
|
||||
inspector = sa.inspect(bind)
|
||||
return any(
|
||||
(item.get("name") or "").endswith(suffix) for item in inspector.get_check_constraints(table)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
# Revision 0001 materializes the schema from live model metadata, so a
|
||||
# database created fresh already carries these objects; guard every step.
|
||||
if not _has_column(bind, "source_records", "bank_status"):
|
||||
op.add_column(
|
||||
"source_records",
|
||||
sa.Column("bank_status", sa.String(length=8), nullable=False, server_default="posted"),
|
||||
)
|
||||
if bind.dialect.name == "postgresql" and not _has_check_constraint(
|
||||
bind, "source_records", "bank_status_valid"
|
||||
):
|
||||
op.create_check_constraint(
|
||||
"bank_status_valid",
|
||||
"source_records",
|
||||
"bank_status IN ('posted', 'hold')",
|
||||
)
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("hold_resolutions"):
|
||||
op.create_table(
|
||||
"hold_resolutions",
|
||||
sa.Column("id", sa.Uuid(), nullable=False),
|
||||
sa.Column("space_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("hold_source_record_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("resolution", sa.String(length=24), nullable=False),
|
||||
sa.Column("resolving_source_record_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("operation_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("resolved_by_user_id", sa.Uuid(), nullable=True),
|
||||
sa.Column("reason", sa.String(length=500), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
sa.CheckConstraint(
|
||||
"resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')",
|
||||
name="ck_hold_resolutions_resolution_valid",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') "
|
||||
"AND (reason IS NULL OR reason = ''))",
|
||||
name="ck_hold_resolutions_manual_reason_required",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id"],
|
||||
["financial_spaces.id"],
|
||||
ondelete="CASCADE",
|
||||
name="fk_hold_resolutions_space_id_financial_spaces",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id", "hold_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
ondelete="CASCADE",
|
||||
name="fk_hold_resolutions_hold_tenant",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id", "resolving_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
name="fk_hold_resolutions_resolving_tenant",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["space_id", "operation_id"],
|
||||
["operations.space_id", "operations.id"],
|
||||
name="fk_hold_resolutions_operation_tenant",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["resolved_by_user_id"],
|
||||
["users.id"],
|
||||
ondelete="SET NULL",
|
||||
name="fk_hold_resolutions_resolved_by_user_id_users",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_hold_resolutions"),
|
||||
sa.UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"),
|
||||
sa.UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"),
|
||||
)
|
||||
op.create_index("ix_hold_resolutions_space_id", "hold_resolutions", ["space_id"])
|
||||
op.create_index(
|
||||
"ix_hold_resolutions_hold_source_record_id",
|
||||
"hold_resolutions",
|
||||
["hold_source_record_id"],
|
||||
)
|
||||
op.create_index("ix_hold_resolutions_operation_id", "hold_resolutions", ["operation_id"])
|
||||
if bind.dialect.name == "postgresql":
|
||||
# Revision 0001 creates tables from live metadata, so a fresh database
|
||||
# already has hold_resolutions before this revision runs. RLS setup must
|
||||
# therefore be independent of the create-table branch above.
|
||||
op.execute("ALTER TABLE hold_resolutions ENABLE ROW LEVEL SECURITY")
|
||||
op.execute("ALTER TABLE hold_resolutions FORCE ROW LEVEL SECURITY")
|
||||
op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions")
|
||||
op.execute(
|
||||
"CREATE POLICY tenant_isolation ON hold_resolutions "
|
||||
"USING (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid) "
|
||||
"WITH CHECK (space_id = NULLIF(current_setting('app.space_id', true), '')::uuid)"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("hold_resolutions"):
|
||||
if bind.dialect.name == "postgresql":
|
||||
op.execute("DROP POLICY IF EXISTS tenant_isolation ON hold_resolutions")
|
||||
op.drop_index("ix_hold_resolutions_operation_id", table_name="hold_resolutions")
|
||||
op.drop_index("ix_hold_resolutions_hold_source_record_id", table_name="hold_resolutions")
|
||||
op.drop_index("ix_hold_resolutions_space_id", table_name="hold_resolutions")
|
||||
op.drop_table("hold_resolutions")
|
||||
if _has_column(bind, "source_records", "bank_status"):
|
||||
with op.batch_alter_table("source_records") as batch_op:
|
||||
if _has_check_constraint(bind, "source_records", "bank_status_valid"):
|
||||
batch_op.drop_constraint("bank_status_valid", type_="check")
|
||||
batch_op.drop_column("bank_status")
|
||||
@@ -440,7 +440,7 @@ def _close_blockers(
|
||||
uncategorized_count: int,
|
||||
negative_envelopes: list[str],
|
||||
) -> list[dict]:
|
||||
from app.imports.models import StatementDocument
|
||||
from app.imports.models import HoldResolution, SourceRecord, StatementDocument
|
||||
|
||||
blockers = []
|
||||
unresolved = (
|
||||
@@ -455,6 +455,28 @@ def _close_blockers(
|
||||
)
|
||||
if unresolved:
|
||||
blockers.append({"code": "IMPORTS_UNRESOLVED", "count": unresolved})
|
||||
start, end = period_bounds(period.year, period.month)
|
||||
unresolved_holds = (
|
||||
db.query(SourceRecord.id)
|
||||
.join(
|
||||
StatementDocument,
|
||||
(StatementDocument.space_id == SourceRecord.space_id)
|
||||
& (StatementDocument.id == SourceRecord.document_id),
|
||||
)
|
||||
.outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id)
|
||||
.filter(
|
||||
SourceRecord.space_id == space_id,
|
||||
SourceRecord.bank_status == "hold",
|
||||
StatementDocument.status == "confirmed",
|
||||
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
|
||||
HoldResolution.id.is_(None),
|
||||
SourceRecord.occurred_on >= start,
|
||||
SourceRecord.occurred_on < end,
|
||||
)
|
||||
.count()
|
||||
)
|
||||
if unresolved_holds:
|
||||
blockers.append({"code": "IMPORT_UNRESOLVED_HOLDS", "count": unresolved_holds})
|
||||
if uncategorized_count:
|
||||
blockers.append({"code": "BUDGET_UNCATEGORIZED_EXPENSES", "count": uncategorized_count})
|
||||
if negative_envelopes:
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter
|
||||
from app.imports.adapters.alfa_xlsx import (
|
||||
AlfaCreditXlsxAdapter as AlfaCreditXlsxAdapter,
|
||||
)
|
||||
from app.imports.adapters.alfa_xlsx import AlfaDebitXlsxAdapter
|
||||
from app.imports.adapters.base import StatementAdapter, UnsupportedStatementLayout
|
||||
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter
|
||||
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter as OzonCreditPdfAdapter
|
||||
from app.imports.adapters.ozon_pdf import OzonDebitPdfAdapter
|
||||
from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter
|
||||
|
||||
# Only the three evidenced-format development contracts are active. The credit
|
||||
# adapters stay present but unregistered until real fixtures exist for them.
|
||||
ADAPTERS: tuple[StatementAdapter, ...] = (
|
||||
AlfaDebitXlsxAdapter(),
|
||||
AlfaCreditXlsxAdapter(),
|
||||
OzonDebitPdfAdapter(),
|
||||
OzonCreditPdfAdapter(),
|
||||
RocketbankDebitPdfAdapter(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from typing import TypedDict
|
||||
|
||||
from openpyxl.workbook import Workbook
|
||||
|
||||
from app.errors import AppError
|
||||
from app.imports.adapters.base import (
|
||||
ALFA_DEBIT_XLSX_VERSION,
|
||||
BANK_STATUS_HOLD,
|
||||
BANK_STATUS_POSTED,
|
||||
DIRECTION_IN,
|
||||
DIRECTION_OUT,
|
||||
PARSER_VERSION,
|
||||
@@ -12,6 +19,7 @@ from app.imports.adapters.base import (
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
amount_to_kopecks,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.readers import open_xlsx
|
||||
|
||||
@@ -198,10 +206,279 @@ def _find_account_value(rows: list[tuple]) -> str | None:
|
||||
return masked
|
||||
|
||||
|
||||
class AlfaDebitXlsxAdapter(AlfaXlsxAdapterBase):
|
||||
class AlfaDebitXlsxAdapter:
|
||||
"""Alfa debit statement contract, evidenced against the real export layout.
|
||||
|
||||
Workbook shape: a 'Выписка по счету' title block, account metadata with
|
||||
the exact statement period ('За период с ... по ...') and header control
|
||||
totals (Поступления / Расходы / Неподтвержденные операции / Текущий
|
||||
баланс), an operation header row (Дата операции, Дата проводки, Код,
|
||||
Категория, Описание, Сумма в валюте счета, Статус), then operation rows
|
||||
terminated by the signature footer. A HOLD posting-date cell marks an
|
||||
immutable hold row; the Код column becomes the external id; a trailing
|
||||
MCC token in the description becomes the row MCC. The raw Расходы control
|
||||
includes hold amounts, so declared_outflow_kopecks is canonicalized to
|
||||
posted-only (raw outflow minus declared hold). Anything else fails closed.
|
||||
"""
|
||||
|
||||
adapter_id = "alfa_debit_xlsx"
|
||||
product = "debit"
|
||||
product_markers = ("дебет", "дебетовая")
|
||||
parser_version = ALFA_DEBIT_XLSX_VERSION
|
||||
bank_code = "alfa"
|
||||
currency = "RUB"
|
||||
|
||||
detect_scan_rows = 60
|
||||
header_scan_rows = 30
|
||||
|
||||
title_label = "выписка по счету"
|
||||
period_re = re.compile(
|
||||
r"за период с\s*(\d{2}\.\d{2}\.\d{4})\s*по\s*(\d{2}\.\d{2}\.\d{4})", re.IGNORECASE
|
||||
)
|
||||
mcc_re = re.compile(r"\bMCC\s*:?\s*(\d{4})\b", re.IGNORECASE)
|
||||
header_labels = {
|
||||
"date": "дата операции",
|
||||
"posted": "дата проводки",
|
||||
"code": "код",
|
||||
"category": "категория",
|
||||
"description": "описание",
|
||||
"status": "статус",
|
||||
}
|
||||
amount_label_prefix = "сумма в"
|
||||
meta_labels = {
|
||||
"account": "номер счета",
|
||||
"currency": "валюта счета",
|
||||
"inflow": "поступления",
|
||||
"outflow": "расходы",
|
||||
"hold": "неподтвержденные операции",
|
||||
"closing": "текущий баланс",
|
||||
}
|
||||
footer_markers = ("подпись сотрудника", "уполномоченное лицо")
|
||||
footer_page_prefix = "страница"
|
||||
hold_token = "hold"
|
||||
|
||||
def detect(self, raw: bytes) -> bool:
|
||||
try:
|
||||
with open_xlsx(raw) as workbook:
|
||||
rows = _debit_read_rows(workbook, self.detect_scan_rows)
|
||||
except Exception:
|
||||
return False
|
||||
cells = [_debit_norm(cell) for row in rows for cell in row]
|
||||
if self.title_label not in cells:
|
||||
return False
|
||||
required = (self.header_labels["date"], self.header_labels["posted"], "статус")
|
||||
return all(label in cells for label in required)
|
||||
|
||||
def parse(self, raw: bytes) -> StatementDraft:
|
||||
with open_xlsx(raw) as workbook:
|
||||
cap = get_settings_row_cap()
|
||||
rows = _debit_read_rows(workbook, cap + 1)
|
||||
if len(rows) > cap:
|
||||
raise UnsupportedStatementLayout("row quota exceeded during parse")
|
||||
header_idx, columns = _debit_find_header(self, rows)
|
||||
meta = _debit_header_metadata(self, rows, header_idx)
|
||||
period_start, period_end = _debit_statement_period(self, rows, header_idx)
|
||||
parsed = _debit_operation_rows(self, rows, header_idx, columns)
|
||||
if not parsed:
|
||||
raise UnsupportedStatementLayout("no operation rows found in Alfa statement")
|
||||
return StatementDraft(
|
||||
adapter_id=self.adapter_id,
|
||||
parser_version=self.parser_version,
|
||||
bank_code=self.bank_code,
|
||||
product=self.product,
|
||||
account_fingerprint=account_fingerprint(self.bank_code, meta["account"]),
|
||||
currency=meta["currency"],
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
opening_balance_kopecks=None,
|
||||
closing_balance_kopecks=meta["closing"],
|
||||
rows=tuple(parsed),
|
||||
declared_inflow_kopecks=meta["inflow"],
|
||||
declared_outflow_kopecks=meta["outflow"],
|
||||
declared_hold_kopecks=meta["hold"],
|
||||
)
|
||||
|
||||
|
||||
def _debit_norm(cell: object) -> str:
|
||||
if cell is None:
|
||||
return ""
|
||||
return str(cell).replace("\u00a0", " ").strip().lower()
|
||||
|
||||
|
||||
def _debit_read_rows(workbook: Workbook, limit: int) -> list[tuple]:
|
||||
sheet = workbook.worksheets[0]
|
||||
rows: list[tuple] = []
|
||||
for index, row in enumerate(sheet.iter_rows(values_only=True)):
|
||||
if index >= limit:
|
||||
break
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _debit_find_header(
|
||||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple]
|
||||
) -> tuple[int, dict[str, int]]:
|
||||
for index, row in enumerate(rows[: adapter.header_scan_rows]):
|
||||
cells = [_debit_norm(cell) for cell in row]
|
||||
columns: dict[str, int] = {}
|
||||
for key, label in adapter.header_labels.items():
|
||||
column = next((i for i, cell in enumerate(cells) if cell == label), None)
|
||||
if column is None:
|
||||
break
|
||||
columns[key] = column
|
||||
if len(columns) != len(adapter.header_labels):
|
||||
continue
|
||||
amount_column = next(
|
||||
(i for i, cell in enumerate(cells) if cell.startswith(adapter.amount_label_prefix)),
|
||||
None,
|
||||
)
|
||||
if amount_column is None:
|
||||
continue
|
||||
columns["amount"] = amount_column
|
||||
return index, columns
|
||||
raise UnsupportedStatementLayout("Alfa operation header row not found")
|
||||
|
||||
|
||||
def _debit_header_value(rows: list[tuple], label: str) -> str:
|
||||
for row in rows:
|
||||
for index, cell in enumerate(row):
|
||||
if _debit_norm(cell) != label:
|
||||
continue
|
||||
for candidate in row[index + 1 :]:
|
||||
text = _cell_str(candidate)
|
||||
if text:
|
||||
return text
|
||||
raise UnsupportedStatementLayout(f"Alfa header metadata not found: {label}")
|
||||
|
||||
|
||||
class _AlfaHeaderMeta(TypedDict):
|
||||
account: str
|
||||
currency: str
|
||||
inflow: int
|
||||
outflow: int
|
||||
hold: int
|
||||
closing: int
|
||||
|
||||
|
||||
def _debit_statement_period(
|
||||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||||
) -> tuple[date, date]:
|
||||
text = "\n".join(_debit_norm(cell) for row in rows[:header_idx] for cell in row)
|
||||
period_match = adapter.period_re.search(text)
|
||||
if period_match is None:
|
||||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||||
period_start = _parse_date(period_match.group(1))
|
||||
period_end = _parse_date(period_match.group(2))
|
||||
if period_start is None or period_end is None:
|
||||
raise UnsupportedStatementLayout("Alfa statement period not found")
|
||||
return period_start, period_end
|
||||
|
||||
|
||||
def _debit_header_metadata(
|
||||
adapter: AlfaDebitXlsxAdapter, rows: list[tuple], header_idx: int
|
||||
) -> _AlfaHeaderMeta:
|
||||
meta: _AlfaHeaderMeta = {
|
||||
"account": _debit_header_value(rows[:header_idx], adapter.meta_labels["account"]),
|
||||
"currency": "RUB",
|
||||
"inflow": 0,
|
||||
"outflow": 0,
|
||||
"hold": 0,
|
||||
"closing": 0,
|
||||
}
|
||||
currency = _debit_header_value(rows[:header_idx], adapter.meta_labels["currency"])
|
||||
if currency.replace(" ", "").upper() not in {"RUB", "RUR"}:
|
||||
raise UnsupportedStatementLayout("unsupported Alfa statement currency")
|
||||
raw_outflow = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["outflow"])
|
||||
)
|
||||
meta["hold"] = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["hold"])
|
||||
)
|
||||
# The raw expense control includes pending (hold) amounts; canonicalize to
|
||||
# posted-only so service reconciliation can compare like with like.
|
||||
canonical_outflow = raw_outflow - meta["hold"]
|
||||
if canonical_outflow < 0:
|
||||
raise UnsupportedStatementLayout("inconsistent Alfa expense header controls")
|
||||
meta["outflow"] = canonical_outflow
|
||||
meta["inflow"] = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["inflow"])
|
||||
)
|
||||
meta["closing"] = money_to_kopecks(
|
||||
_debit_header_value(rows[:header_idx], adapter.meta_labels["closing"])
|
||||
)
|
||||
return meta
|
||||
|
||||
|
||||
def _debit_operation_rows(
|
||||
adapter: AlfaDebitXlsxAdapter,
|
||||
rows: list[tuple],
|
||||
header_idx: int,
|
||||
columns: dict[str, int],
|
||||
) -> list[SourceRow]:
|
||||
parsed: list[SourceRow] = []
|
||||
separated_from_operations = False
|
||||
for offset, row in enumerate(rows[header_idx + 1 :], start=header_idx + 2):
|
||||
norm_cells = [_debit_norm(cell) for cell in row]
|
||||
if any(marker in cell for cell in norm_cells for marker in adapter.footer_markers):
|
||||
break
|
||||
if any(cell.startswith(adapter.footer_page_prefix) for cell in norm_cells):
|
||||
break
|
||||
if all(cell == "" for cell in norm_cells):
|
||||
separated_from_operations = bool(parsed)
|
||||
continue
|
||||
occurred = _parse_date(row[columns["date"]])
|
||||
posted_cell = _cell_str(row[columns["posted"]]) or ""
|
||||
status_cell = _cell_str(row[columns["status"]]) or ""
|
||||
bank_status = BANK_STATUS_POSTED
|
||||
if adapter.hold_token in posted_cell.lower() or adapter.hold_token in status_cell.lower():
|
||||
bank_status = BANK_STATUS_HOLD
|
||||
posted_on = None
|
||||
else:
|
||||
posted_on = _parse_date(posted_cell)
|
||||
amount_text = _cell_str(row[columns["amount"]])
|
||||
amount: int | None = None
|
||||
if amount_text is not None:
|
||||
try:
|
||||
amount = abs(money_to_kopecks(amount_text))
|
||||
except UnsupportedStatementLayout:
|
||||
amount = None
|
||||
if occurred is None and amount is None:
|
||||
later_has_operation = any(
|
||||
_parse_date(later[columns["date"]]) is not None
|
||||
for later in rows[offset:]
|
||||
if len(later) > columns["date"]
|
||||
)
|
||||
if separated_from_operations and not later_has_operation:
|
||||
break
|
||||
raise UnsupportedStatementLayout("unexpected row in Alfa operation table")
|
||||
warnings: list[str] = []
|
||||
if occurred is None:
|
||||
warnings.append("occurred_date_missing")
|
||||
if amount is None:
|
||||
amount = 0
|
||||
warnings.append("amount_unparseable")
|
||||
if bank_status == BANK_STATUS_POSTED and posted_on is None and posted_cell:
|
||||
warnings.append("posted_date_unparseable")
|
||||
direction = DIRECTION_OUT if (amount_text or "").startswith("-") else DIRECTION_IN
|
||||
description = _cell_str(row[columns["description"]])
|
||||
mcc_match = adapter.mcc_re.search(description) if description else None
|
||||
parsed.append(
|
||||
SourceRow(
|
||||
external_id=_cell_str(row[columns["code"]]),
|
||||
occurred_on=occurred,
|
||||
posted_on=posted_on,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
description=description,
|
||||
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
|
||||
bank_category=_cell_str(row[columns["category"]]),
|
||||
location=None,
|
||||
row_ref=f"sheet0:row{offset}",
|
||||
warnings=tuple(warnings),
|
||||
bank_status=bank_status,
|
||||
)
|
||||
)
|
||||
separated_from_operations = False
|
||||
return parsed
|
||||
|
||||
|
||||
class AlfaCreditXlsxAdapter(AlfaXlsxAdapterBase):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal, InvalidOperation
|
||||
@@ -8,7 +9,16 @@ from app.errors import AppError
|
||||
|
||||
DIRECTION_IN = "in"
|
||||
DIRECTION_OUT = "out"
|
||||
BANK_STATUS_POSTED = "posted"
|
||||
BANK_STATUS_HOLD = "hold"
|
||||
BANK_STATUSES = (BANK_STATUS_POSTED, BANK_STATUS_HOLD)
|
||||
# Legacy shared version kept only for the untouched, unregistered credit adapters.
|
||||
PARSER_VERSION = "1"
|
||||
# Per-contract versions: bumped for the rewritten evidenced contracts so old
|
||||
# parse runs can never be confused with the new canonical output.
|
||||
ALFA_DEBIT_XLSX_VERSION = "2"
|
||||
OZON_DEBIT_PDF_VERSION = "2"
|
||||
ROCKETBANK_DEBIT_PDF_VERSION = "2"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -24,6 +34,15 @@ class SourceRow:
|
||||
location: str | None
|
||||
row_ref: str
|
||||
warnings: tuple[str, ...] = ()
|
||||
bank_status: str = BANK_STATUS_POSTED
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.amount_kopecks < 0:
|
||||
raise UnsupportedStatementLayout(
|
||||
"amount_kopecks must be positive; direction carries the sign"
|
||||
)
|
||||
if self.bank_status not in BANK_STATUSES:
|
||||
raise UnsupportedStatementLayout("bank_status must be posted or hold")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -39,6 +58,9 @@ class StatementDraft:
|
||||
opening_balance_kopecks: int | None
|
||||
closing_balance_kopecks: int | None
|
||||
rows: tuple[SourceRow, ...]
|
||||
declared_inflow_kopecks: int | None = None
|
||||
declared_outflow_kopecks: int | None = None
|
||||
declared_hold_kopecks: int | None = None
|
||||
|
||||
|
||||
class UnsupportedStatementLayout(AppError):
|
||||
@@ -77,6 +99,29 @@ def amount_to_kopecks(value: object) -> int:
|
||||
raise UnsupportedStatementLayout(f"unsupported amount cell type: {type(value).__name__}")
|
||||
|
||||
|
||||
def _mask_digits(text: str) -> str:
|
||||
return re.sub(r"\d", "#", text)
|
||||
|
||||
|
||||
CURRENCY_SUFFIX_RE = re.compile(r"(?:₽|RUR|rub|руб\.?)\s*$", re.IGNORECASE)
|
||||
PLAIN_AMOUNT_RE = re.compile(r"^[+-]?\s?\d[\d\s]*(?:[.,]\d{1,2})?$")
|
||||
|
||||
|
||||
def money_to_kopecks(text: object) -> int:
|
||||
"""Parse statement money strings like '-1 234,56 RUR', '+ 500.00 ₽', '12 345'.
|
||||
|
||||
Accepts space/NBSP thousand separators and comma or dot decimal marks.
|
||||
Fails closed with a digit-masked diagnostic; never leaks the raw value.
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
raise UnsupportedStatementLayout(f"unsupported money cell type: {type(text).__name__}")
|
||||
cleaned = text.replace("\u00a0", " ").strip()
|
||||
cleaned = CURRENCY_SUFFIX_RE.sub("", cleaned).strip()
|
||||
if not PLAIN_AMOUNT_RE.fullmatch(cleaned):
|
||||
raise UnsupportedStatementLayout(f"unparseable money amount: {_mask_digits(text)}")
|
||||
return amount_to_kopecks(cleaned)
|
||||
|
||||
|
||||
def account_fingerprint(bank_code: str, masked_identifier: str) -> str:
|
||||
digest = hashlib.sha256(f"{bank_code}|{masked_identifier}".encode()).hexdigest()
|
||||
return digest[:64]
|
||||
|
||||
@@ -6,12 +6,14 @@ from datetime import date, datetime
|
||||
from app.imports.adapters.base import (
|
||||
DIRECTION_IN,
|
||||
DIRECTION_OUT,
|
||||
OZON_DEBIT_PDF_VERSION,
|
||||
PARSER_VERSION,
|
||||
SourceRow,
|
||||
StatementDraft,
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
amount_to_kopecks,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.readers import open_pdf
|
||||
|
||||
@@ -122,10 +124,179 @@ def _first_page_text(raw: bytes) -> str:
|
||||
return pdf.pages[0].extract_text() or ""
|
||||
|
||||
|
||||
class OzonDebitPdfAdapter(OzonPdfAdapterBase):
|
||||
class OzonDebitPdfAdapter:
|
||||
"""Ozon debit statement contract, evidenced against the real document.
|
||||
|
||||
A 'Справка о движении средств' text-layer PDF: a legal-bank header block
|
||||
with account/period/opening balance, per-page tables whose rows keep
|
||||
multiline document IDs and descriptions attached to one transaction, and a
|
||||
closing block with declared inflow/outflow totals and the closing balance.
|
||||
Amounts are signed strings ('- 500.00 ₽'); the sign maps to direction and
|
||||
the stored amount is always positive.
|
||||
"""
|
||||
|
||||
adapter_id = "ozon_debit_pdf"
|
||||
product = "debit"
|
||||
product_markers = ("дебетов",)
|
||||
parser_version = OZON_DEBIT_PDF_VERSION
|
||||
bank_code = "ozon"
|
||||
currency = "RUB"
|
||||
|
||||
bank_marker = "озон банк"
|
||||
title_marker = "справка о движении средств"
|
||||
account_marker_re = re.compile(r"лицевого\s+сч[её]та:?\s*№\s*([\d*•]+)", re.IGNORECASE)
|
||||
period_re = re.compile(
|
||||
r"период\s+выписки:?\s*(\d{2}\.\d{2}\.\d{4})\s*[–—-]\s*(\d{2}\.\d{2}\.\d{4})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
opening_re = re.compile(r"входящий\s+остаток:?\s*(.+)", re.IGNORECASE)
|
||||
inflow_re = re.compile(r"итого\s+зачислений\s+за\s+период:?\s*(.+)", re.IGNORECASE)
|
||||
outflow_re = re.compile(r"итого\s+списаний\s+за\s+период:?\s*(.+)", re.IGNORECASE)
|
||||
closing_re = re.compile(r"исходящий\s+остаток:?\s*(.+)", re.IGNORECASE)
|
||||
amount_re = re.compile(r"^([+-])\s*(\d[\d\s]*(?:[.,]\d{2})?)\s*(?:₽|rur|rub)?$", re.IGNORECASE)
|
||||
table_header_labels = ("дата операции", "документ", "назначение платежа")
|
||||
table_subheader_tokens = ("рубли", "валюта")
|
||||
|
||||
def detect(self, raw: bytes) -> bool:
|
||||
try:
|
||||
text = _first_page_text(raw)
|
||||
except Exception:
|
||||
return False
|
||||
lowered = _ozon_normalize(text)
|
||||
return (
|
||||
self.bank_marker in lowered
|
||||
and self.title_marker in lowered
|
||||
and "лицевого сч" in lowered
|
||||
)
|
||||
|
||||
def parse(self, raw: bytes) -> StatementDraft:
|
||||
pages_text: list[str] = []
|
||||
pages_tables: list[list[list[list[object]]]] = []
|
||||
with open_pdf(raw) as pdf:
|
||||
if not pdf.pages:
|
||||
raise UnsupportedStatementLayout("PDF has no pages")
|
||||
for page_index, page in enumerate(pdf.pages):
|
||||
text = page.extract_text() or ""
|
||||
if page_index == 0 and not text.strip():
|
||||
raise UnsupportedStatementLayout(
|
||||
"PDF has no extractable text layer; OCR is not supported"
|
||||
)
|
||||
pages_text.append(text)
|
||||
extracted = page.extract_tables()
|
||||
pages_tables.append(extracted if extracted else [])
|
||||
return self._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def _draft_from_extracted(
|
||||
self, pages_text: list[str], pages_tables: list[list[list[list[object]]]]
|
||||
) -> StatementDraft:
|
||||
page_zero = _ozon_normalize(pages_text[0])
|
||||
if self.bank_marker not in page_zero or self.title_marker not in page_zero:
|
||||
raise UnsupportedStatementLayout("Ozon statement markers not found")
|
||||
account_match = self.account_marker_re.search(page_zero)
|
||||
if account_match is None:
|
||||
raise UnsupportedStatementLayout("Ozon account line not found")
|
||||
period_match = self.period_re.search(page_zero)
|
||||
if period_match is None:
|
||||
raise UnsupportedStatementLayout("Ozon period line not found")
|
||||
period_start = datetime.strptime(period_match.group(1), "%d.%m.%Y").date()
|
||||
period_end = datetime.strptime(period_match.group(2), "%d.%m.%Y").date()
|
||||
opening = money_to_kopecks(self._required_line(self.opening_re, page_zero, "opening"))
|
||||
full_text = _ozon_normalize("\n".join(pages_text))
|
||||
declared_inflow = money_to_kopecks(self._required_line(self.inflow_re, full_text, "inflow"))
|
||||
declared_outflow = money_to_kopecks(
|
||||
self._required_line(self.outflow_re, full_text, "outflow")
|
||||
)
|
||||
closing = money_to_kopecks(self._required_line(self.closing_re, full_text, "closing"))
|
||||
rows: list[SourceRow] = []
|
||||
for page_index, tables in enumerate(pages_tables, start=1):
|
||||
for table_index, table in enumerate(tables):
|
||||
for row_index, cells in enumerate(table):
|
||||
row = self._table_row(
|
||||
cells, f"page{page_index}:table{table_index}:row{row_index}"
|
||||
)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
if not rows:
|
||||
raise UnsupportedStatementLayout("no statement rows matched the Ozon contract")
|
||||
return StatementDraft(
|
||||
adapter_id=self.adapter_id,
|
||||
parser_version=self.parser_version,
|
||||
bank_code=self.bank_code,
|
||||
product=self.product,
|
||||
account_fingerprint=account_fingerprint(self.bank_code, account_match.group(1)),
|
||||
currency=self.currency,
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
opening_balance_kopecks=opening,
|
||||
closing_balance_kopecks=closing,
|
||||
rows=tuple(rows),
|
||||
declared_inflow_kopecks=declared_inflow,
|
||||
declared_outflow_kopecks=declared_outflow,
|
||||
declared_hold_kopecks=None,
|
||||
)
|
||||
|
||||
def _required_line(self, pattern: re.Pattern[str], text: str, what: str) -> str:
|
||||
match = pattern.search(text)
|
||||
if match is None:
|
||||
raise UnsupportedStatementLayout(f"Ozon {what} total line not found")
|
||||
return match.group(1)
|
||||
|
||||
def _table_row(self, cells: list[object], row_ref: str) -> SourceRow | None:
|
||||
if cells is None:
|
||||
raise UnsupportedStatementLayout("unexpected Ozon table row shape")
|
||||
texts = [_ozon_cell(cell) for cell in cells]
|
||||
if not any(texts):
|
||||
return None
|
||||
lowered = [text.lower() for text in texts]
|
||||
joined = " ".join(text for text in lowered if text)
|
||||
if all(label in joined for label in self.table_header_labels):
|
||||
return None
|
||||
if lowered[0] == "" and any(token in joined for token in self.table_subheader_tokens):
|
||||
return None
|
||||
if lowered[0] == "" or len(texts) < 4:
|
||||
raise UnsupportedStatementLayout("unexpected Ozon table row shape")
|
||||
occurred = None
|
||||
date_match = re.match(r"^(\d{2}\.\d{2}\.\d{4})", texts[0])
|
||||
if date_match is not None:
|
||||
occurred = datetime.strptime(date_match.group(1), "%d.%m.%Y").date()
|
||||
external_id = re.sub(r"\s+", "", texts[1]) if len(texts) > 1 else ""
|
||||
description = " ".join(texts[2].split()) if len(texts) > 2 else ""
|
||||
warnings: list[str] = []
|
||||
if occurred is None:
|
||||
warnings.append("occurred_date_missing")
|
||||
if not external_id:
|
||||
warnings.append("external_id_missing")
|
||||
amount_match = self.amount_re.match(texts[3]) if len(texts) > 3 else None
|
||||
if amount_match is None:
|
||||
amount = 0
|
||||
warnings.append("amount_unparseable")
|
||||
direction = DIRECTION_OUT
|
||||
else:
|
||||
sign, body = amount_match.group(1), amount_match.group(2)
|
||||
amount = amount_to_kopecks(body.replace(" ", "").replace(",", "."))
|
||||
direction = DIRECTION_IN if sign == "+" else DIRECTION_OUT
|
||||
return SourceRow(
|
||||
external_id=external_id or None,
|
||||
occurred_on=occurred,
|
||||
posted_on=None,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
description=description or None,
|
||||
mcc=None,
|
||||
bank_category=None,
|
||||
location=None,
|
||||
row_ref=row_ref,
|
||||
warnings=tuple(warnings),
|
||||
)
|
||||
|
||||
|
||||
def _ozon_normalize(text: str) -> str:
|
||||
return text.replace("\u00a0", " ").lower()
|
||||
|
||||
|
||||
def _ozon_cell(cell: object) -> str:
|
||||
if cell is None:
|
||||
return ""
|
||||
return str(cell).replace("\u00a0", " ").strip()
|
||||
|
||||
|
||||
class OzonCreditPdfAdapter(OzonPdfAdapterBase):
|
||||
|
||||
@@ -1,12 +1,288 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.imports.adapters.ozon_pdf import OzonPdfAdapterBase
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
from app.imports.adapters.base import (
|
||||
DIRECTION_IN,
|
||||
DIRECTION_OUT,
|
||||
ROCKETBANK_DEBIT_PDF_VERSION,
|
||||
SourceRow,
|
||||
StatementDraft,
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.readers import open_pdf
|
||||
|
||||
MONTH_NAMES = {
|
||||
"января": 1,
|
||||
"февраля": 2,
|
||||
"марта": 3,
|
||||
"апреля": 4,
|
||||
"мая": 5,
|
||||
"июня": 6,
|
||||
"июля": 7,
|
||||
"августа": 8,
|
||||
"сентября": 9,
|
||||
"октября": 10,
|
||||
"ноября": 11,
|
||||
"декабря": 12,
|
||||
}
|
||||
|
||||
DAY_HEADING_RE = re.compile(
|
||||
r"^(\d{1,2})\s+(января|февраля|марта|апреля|мая|июня|июля|августа|сентября|"
|
||||
r"октября|ноября|декабря)\s+(\d{4})\s+баланс на начало дня:\s*(-?[\d\s]+,\d{2})\s*rur$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
RECORD_RE = re.compile(
|
||||
r"^(\d{1,2}:\d{2})\s+(\d{6,})\s+(.*?)([+-]?\s?[\d\s]+,\d{2})\s*rur$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
HEADER_VALUES_RE = re.compile(
|
||||
r"^(-?[\d\s]+,\d{2})\s*rur\s+(-?[\d\s]+,\d{2})\s*rur(?:\s+(\d{2}\.\d{2}\.\d{4}))?$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
PERIOD_RE = re.compile(r"^с\s+(\d{2}\.\d{2}\.\d{4})\s+по\s+(\d{2}\.\d{2}\.\d{4})$", re.IGNORECASE)
|
||||
AMOUNT_RE = re.compile(r"^([+-]?)\s*(\d[\d\s]*,\d{2})$")
|
||||
MCC_RE = re.compile(r"\bmcc\s*:?\s*(\d{4})\b", re.IGNORECASE)
|
||||
LOCATION_RE = re.compile(r"rur,\s*([^,]+?),\s*mcc\b", re.IGNORECASE)
|
||||
|
||||
|
||||
class RocketbankDebitPdfAdapter(OzonPdfAdapterBase):
|
||||
"""Rocketbank debit statement contract: same text-layer line scanning shape."""
|
||||
class RocketbankDebitPdfAdapter:
|
||||
"""Rocketbank debit statement contract, evidenced against the real document.
|
||||
|
||||
Independent text-layer state machine: every page repeats the statement
|
||||
header and column captions; day sections start with a Russian day heading
|
||||
('DD MONTH YYYY БАЛАНС НА НАЧАЛО ДНЯ: X RUR'); each record is a
|
||||
'time + operation code + multiline description + signed RUR amount' line
|
||||
whose description column starts slightly above and continues below that
|
||||
line, so the surrounding non-structural lines are folded into one
|
||||
description. Daily totals and repeated captions are ignored. Statement
|
||||
period, account, opening/closing balances and declared inflow/outflow
|
||||
totals come from the first-page header block.
|
||||
"""
|
||||
|
||||
adapter_id = "rocketbank_debit_pdf"
|
||||
product = "debit"
|
||||
bank_markers = ("рокетбанк", "rocketbank")
|
||||
product_markers = ("дебетов", "дебетовая")
|
||||
parser_version = ROCKETBANK_DEBIT_PDF_VERSION
|
||||
bank_code = "rocketbank"
|
||||
currency = "RUB"
|
||||
|
||||
title_marker = "выписка по счету"
|
||||
card_re = re.compile(r"дебетовая карта рокет\s*(\d{10,30})\s*rur", re.IGNORECASE)
|
||||
column_caption = "время код операции описание сумма"
|
||||
issued_prefix = "дата выдачи выписки:"
|
||||
page_footer_prefix = "страница"
|
||||
daily_total_prefixes = ("всего поступлений:", "всего расходов:")
|
||||
opening_label = "входящий остаток"
|
||||
closing_label = "исходящий остаток"
|
||||
max_fold_lines = 6
|
||||
|
||||
def detect(self, raw: bytes) -> bool:
|
||||
try:
|
||||
with open_pdf(raw) as pdf:
|
||||
if not pdf.pages:
|
||||
return False
|
||||
text = pdf.pages[0].extract_text() or ""
|
||||
except Exception:
|
||||
return False
|
||||
lowered = text.lower()
|
||||
return self.title_marker in lowered and "дебетовая карта рокет" in lowered
|
||||
|
||||
def parse(self, raw: bytes) -> StatementDraft:
|
||||
lines: list[tuple[int, str]] = []
|
||||
with open_pdf(raw) as pdf:
|
||||
if not pdf.pages:
|
||||
raise UnsupportedStatementLayout("PDF has no pages")
|
||||
for page_number, page in enumerate(pdf.pages, start=1):
|
||||
text = page.extract_text() or ""
|
||||
if page_number == 1 and not text.strip():
|
||||
raise UnsupportedStatementLayout(
|
||||
"PDF has no extractable text layer; OCR is not supported"
|
||||
)
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped:
|
||||
lines.append((page_number, stripped))
|
||||
return self._draft_from_lines(lines)
|
||||
|
||||
def _draft_from_lines(self, lines: list[tuple[int, str]]) -> StatementDraft:
|
||||
blob = "\n".join(line.lower() for _, line in lines)
|
||||
if self.title_marker not in blob:
|
||||
raise UnsupportedStatementLayout("Rocketbank statement marker not found")
|
||||
card_match = self.card_re.search(blob)
|
||||
if card_match is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank card line not found")
|
||||
|
||||
period_start: date | None = None
|
||||
period_end: date | None = None
|
||||
opening: int | None = None
|
||||
closing: int | None = None
|
||||
declared_inflow: int | None = None
|
||||
declared_outflow: int | None = None
|
||||
rows: list[SourceRow] = []
|
||||
current_day: date | None = None
|
||||
pending: _PendingRecord | None = None
|
||||
pre_lines: list[tuple[int, str]] = []
|
||||
record_ordinals: dict[int, int] = {}
|
||||
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
page, line = lines[index]
|
||||
low = line.lower()
|
||||
if (
|
||||
low == self.title_marker
|
||||
or low.startswith(self.issued_prefix)
|
||||
or low.startswith(self.page_footer_prefix)
|
||||
or low == self.column_caption
|
||||
or low.startswith(self.daily_total_prefixes)
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
period_match = PERIOD_RE.match(low)
|
||||
if period_match is not None and period_start is None:
|
||||
period_start = _date_from(period_match.group(1))
|
||||
period_end = _date_from(period_match.group(2))
|
||||
index += 1
|
||||
continue
|
||||
if self.opening_label in low or self.closing_label in low:
|
||||
if index + 1 >= len(lines):
|
||||
raise UnsupportedStatementLayout("Rocketbank balance values missing")
|
||||
values_match = HEADER_VALUES_RE.match(lines[index + 1][1].lower())
|
||||
if values_match is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank balance values missing")
|
||||
first = money_to_kopecks(values_match.group(1))
|
||||
second = money_to_kopecks(values_match.group(2))
|
||||
if self.opening_label in low:
|
||||
opening, declared_inflow = first, second
|
||||
else:
|
||||
closing, declared_outflow = first, second
|
||||
index += 2
|
||||
continue
|
||||
day_match = DAY_HEADING_RE.match(low)
|
||||
if day_match is not None:
|
||||
if pending is not None:
|
||||
rows.append(self._record(pending, current_day))
|
||||
pending = None
|
||||
pre_lines = []
|
||||
current_day = date(
|
||||
int(day_match.group(3)),
|
||||
MONTH_NAMES[day_match.group(2).lower()],
|
||||
int(day_match.group(1)),
|
||||
)
|
||||
index += 1
|
||||
continue
|
||||
record_match = RECORD_RE.match(line)
|
||||
if record_match is not None:
|
||||
if pending is not None:
|
||||
rows.append(self._record(pending, current_day))
|
||||
ordinal = record_ordinals.get(page, 0) + 1
|
||||
record_ordinals[page] = ordinal
|
||||
pending = _PendingRecord(
|
||||
row_ref=f"page{page}:record{ordinal}",
|
||||
code=record_match.group(2),
|
||||
description=record_match.group(3).strip(),
|
||||
amount_text=record_match.group(4),
|
||||
pre_lines=pre_lines,
|
||||
)
|
||||
pre_lines = []
|
||||
index += 1
|
||||
continue
|
||||
if pending is not None:
|
||||
pending.post_lines.append((page, line))
|
||||
index += 1
|
||||
continue
|
||||
pre_lines.append((page, line))
|
||||
if len(pre_lines) > self.max_fold_lines:
|
||||
raise UnsupportedStatementLayout("unrecognized content between Rocketbank sections")
|
||||
index += 1
|
||||
if pending is not None:
|
||||
rows.append(self._record(pending, current_day))
|
||||
pre_lines = []
|
||||
if pre_lines:
|
||||
raise UnsupportedStatementLayout("unrecognized trailing Rocketbank content")
|
||||
if (
|
||||
period_start is None
|
||||
or opening is None
|
||||
or closing is None
|
||||
or declared_inflow is None
|
||||
or declared_outflow is None
|
||||
):
|
||||
raise UnsupportedStatementLayout("Rocketbank statement header block not found")
|
||||
if not rows:
|
||||
raise UnsupportedStatementLayout("no Rocketbank operation rows found")
|
||||
return StatementDraft(
|
||||
adapter_id=self.adapter_id,
|
||||
parser_version=self.parser_version,
|
||||
bank_code=self.bank_code,
|
||||
product=self.product,
|
||||
account_fingerprint=account_fingerprint(self.bank_code, card_match.group(1)),
|
||||
currency=self.currency,
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
opening_balance_kopecks=opening,
|
||||
closing_balance_kopecks=closing,
|
||||
rows=tuple(rows),
|
||||
declared_inflow_kopecks=declared_inflow,
|
||||
declared_outflow_kopecks=declared_outflow,
|
||||
declared_hold_kopecks=None,
|
||||
)
|
||||
|
||||
def _record(
|
||||
self,
|
||||
pending: _PendingRecord,
|
||||
current_day: date | None,
|
||||
) -> SourceRow:
|
||||
if current_day is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank record outside a day section")
|
||||
chunks = [text for _, text in pending.pre_lines]
|
||||
chunks.append(pending.description)
|
||||
chunks.extend(text for _, text in pending.post_lines)
|
||||
description = " ".join(" ".join(chunk.split()) for chunk in chunks if chunk.strip())
|
||||
amount_match = AMOUNT_RE.match(" ".join(pending.amount_text.split()))
|
||||
if amount_match is None:
|
||||
raise UnsupportedStatementLayout("Rocketbank record amount missing")
|
||||
sign = amount_match.group(1)
|
||||
amount = money_to_kopecks(amount_match.group(2))
|
||||
direction = DIRECTION_OUT if sign == "-" else DIRECTION_IN
|
||||
mcc_match = MCC_RE.search(description)
|
||||
location_match = LOCATION_RE.search(description)
|
||||
return SourceRow(
|
||||
external_id=pending.code,
|
||||
occurred_on=current_day,
|
||||
posted_on=None,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
description=description or None,
|
||||
mcc=int(mcc_match.group(1)) if mcc_match is not None else None,
|
||||
bank_category=None,
|
||||
location=location_match.group(1).strip() if location_match is not None else None,
|
||||
row_ref=pending.row_ref,
|
||||
warnings=(),
|
||||
)
|
||||
|
||||
|
||||
class _PendingRecord:
|
||||
__slots__ = ("row_ref", "code", "description", "amount_text", "pre_lines", "post_lines")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
row_ref: str,
|
||||
code: str,
|
||||
description: str,
|
||||
amount_text: str,
|
||||
pre_lines: list[tuple[int, str]],
|
||||
) -> None:
|
||||
self.row_ref = row_ref
|
||||
self.code = code
|
||||
self.description = description
|
||||
self.amount_text = amount_text
|
||||
self.pre_lines = pre_lines
|
||||
self.post_lines: list[tuple[int, str]] = []
|
||||
|
||||
|
||||
def _date_from(text: str) -> date:
|
||||
day, month, year = (int(part) for part in text.split("."))
|
||||
return date(year, month, day)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import uuid
|
||||
from datetime import date
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
@@ -18,7 +18,7 @@ from sqlalchemy import (
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.db.base import Base, PkMixin, TimestampMixin
|
||||
from app.db.base import Base, PkMixin, TimestampMixin, UTCDateTime, utcnow
|
||||
|
||||
DOCUMENT_STATUSES = (
|
||||
"uploaded",
|
||||
@@ -34,6 +34,9 @@ RUN_STATUSES = ("pending", "running", "succeeded", "failed")
|
||||
CANDIDATE_KINDS = ("duplicate", "transfer")
|
||||
CANDIDATE_CONFIDENCES = ("unique", "possible")
|
||||
CANDIDATE_RESOLUTIONS = ("pending", "is_duplicate", "is_new")
|
||||
BANK_STATUSES = ("posted", "hold")
|
||||
HOLD_RESOLUTIONS = ("matched_posted", "confirmed_as_posted", "discarded_as_cancelled")
|
||||
HOLD_MANUAL_RESOLUTIONS = ("confirmed_as_posted", "discarded_as_cancelled")
|
||||
|
||||
|
||||
class StatementDocument(Base, PkMixin, TimestampMixin):
|
||||
@@ -99,6 +102,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin):
|
||||
|
||||
__tablename__ = "source_records"
|
||||
__table_args__ = (
|
||||
CheckConstraint("bank_status IN ('posted', 'hold')", name="bank_status_valid"),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "document_id"],
|
||||
["statement_documents.space_id", "statement_documents.id"],
|
||||
@@ -128,6 +132,7 @@ class SourceRecord(Base, PkMixin, TimestampMixin):
|
||||
posted_on: Mapped[date | None] = mapped_column(Date, nullable=True)
|
||||
amount_kopecks: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
direction: Mapped[str] = mapped_column(String(8), nullable=False)
|
||||
bank_status: Mapped[str] = mapped_column(String(8), nullable=False, default="posted")
|
||||
description: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
mcc: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
bank_category: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||
@@ -209,3 +214,57 @@ class DedupCandidate(Base, PkMixin, TimestampMixin):
|
||||
decided_by_user_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
|
||||
|
||||
class HoldResolution(Base, PkMixin, TimestampMixin):
|
||||
"""Append-only, tenant-scoped settlement of one HOLD source record.
|
||||
|
||||
The extracted HOLD row itself is never mutated. At most one resolution
|
||||
exists per hold source record; automatic matches (`matched_posted`) are
|
||||
created only when a later statement is confirmed, and manual actions
|
||||
(`confirmed_as_posted`, `discarded_as_cancelled`) always carry a reason.
|
||||
"""
|
||||
|
||||
__tablename__ = "hold_resolutions"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"resolution IN ('matched_posted', 'confirmed_as_posted', 'discarded_as_cancelled')",
|
||||
name="resolution_valid",
|
||||
),
|
||||
CheckConstraint(
|
||||
"NOT (resolution IN ('confirmed_as_posted', 'discarded_as_cancelled') "
|
||||
"AND (reason IS NULL OR reason = ''))",
|
||||
name="manual_reason_required",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "hold_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
ondelete="CASCADE",
|
||||
name="fk_hold_resolutions_hold_tenant",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "resolving_source_record_id"],
|
||||
["source_records.space_id", "source_records.id"],
|
||||
name="fk_hold_resolutions_resolving_tenant",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["space_id", "operation_id"],
|
||||
["operations.space_id", "operations.id"],
|
||||
name="fk_hold_resolutions_operation_tenant",
|
||||
),
|
||||
UniqueConstraint("hold_source_record_id", name="uq_hold_resolutions_hold_record"),
|
||||
UniqueConstraint("space_id", "id", name="uq_hold_resolutions_space_id"),
|
||||
)
|
||||
|
||||
space_id: Mapped[uuid.UUID] = mapped_column(
|
||||
Uuid, ForeignKey("financial_spaces.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
hold_source_record_id: Mapped[uuid.UUID] = mapped_column(Uuid, nullable=False, index=True)
|
||||
resolution: Mapped[str] = mapped_column(String(24), nullable=False)
|
||||
resolving_source_record_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True)
|
||||
operation_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, nullable=True, index=True)
|
||||
resolved_by_user_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
Uuid, ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
reason: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
resolved_at: Mapped[datetime] = mapped_column(UTCDateTime(), nullable=False, default=utcnow)
|
||||
|
||||
@@ -33,6 +33,14 @@ class ConfirmBody(BaseModel):
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
class HoldResolveBody(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
action: str
|
||||
reason: str
|
||||
account_id: uuid.UUID | None = Field(default=None, alias="accountId")
|
||||
|
||||
|
||||
def summary(document: Any) -> dict:
|
||||
return {
|
||||
"id": document.id,
|
||||
@@ -100,21 +108,34 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
document = service._document(db, space_id=context.space.id, document_id=document_id)
|
||||
base = summary(document)
|
||||
reconciliation = raw.get("reconciliation") or {}
|
||||
rows = [
|
||||
{
|
||||
"id": item["id"],
|
||||
"externalId": item["external_id"],
|
||||
"occurredOn": item["occurred_on"],
|
||||
"amountKopecks": item["amount_kopecks"],
|
||||
"direction": item["direction"],
|
||||
"description": item["description"],
|
||||
"payeeHint": None,
|
||||
"warnings": (item["warnings"] or {}).get("codes", []),
|
||||
"decision": "pending" if item["critical"] else item["review_decision"],
|
||||
"duplicateOfOperationId": item["duplicate_operation_id"],
|
||||
}
|
||||
for item in raw["records"]
|
||||
]
|
||||
rows = []
|
||||
for item in raw["records"]:
|
||||
hold = item.get("hold_resolution")
|
||||
rows.append(
|
||||
{
|
||||
"id": item["id"],
|
||||
"externalId": item["external_id"],
|
||||
"occurredOn": item["occurred_on"],
|
||||
"amountKopecks": item["amount_kopecks"],
|
||||
"direction": item["direction"],
|
||||
"bankStatus": item.get("bank_status", "posted"),
|
||||
"description": item["description"],
|
||||
"payeeHint": None,
|
||||
"warnings": (item["warnings"] or {}).get("codes", []),
|
||||
"decision": "pending" if item["critical"] else item["review_decision"],
|
||||
"duplicateOfOperationId": item["duplicate_operation_id"],
|
||||
"holdResolution": None
|
||||
if hold is None
|
||||
else {
|
||||
"resolution": hold["resolution"],
|
||||
"resolvingSourceRecordId": hold["resolving_source_record_id"],
|
||||
"operationId": hold["operation_id"],
|
||||
"resolvedByUserId": hold["resolved_by_user_id"],
|
||||
"reason": hold["reason"],
|
||||
"resolvedAt": hold["resolved_at"],
|
||||
},
|
||||
}
|
||||
)
|
||||
blockers = [
|
||||
{"code": "IMPORT_FIELD_UNRESOLVED", "message": None, "rowId": str(item["id"])}
|
||||
for item in raw["records"]
|
||||
@@ -136,7 +157,16 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
"status": status,
|
||||
"openingBalanceKopecks": reconciliation.get("opening_balance_kopecks"),
|
||||
"closingBalanceKopecks": reconciliation.get("closing_balance_kopecks"),
|
||||
"computedClosingKopecks": None,
|
||||
"computedClosingKopecks": reconciliation.get("computed_closing_kopecks"),
|
||||
"declaredInflowKopecks": reconciliation.get("declared_inflow_kopecks"),
|
||||
"declaredOutflowKopecks": reconciliation.get("declared_outflow_kopecks"),
|
||||
"declaredHoldKopecks": reconciliation.get("declared_hold_kopecks"),
|
||||
"parsedPostedInflowKopecks": reconciliation.get("parsed_posted_inflow_kopecks"),
|
||||
"parsedPostedOutflowKopecks": reconciliation.get("parsed_posted_outflow_kopecks"),
|
||||
"parsedHoldTotalKopecks": reconciliation.get("parsed_hold_total_kopecks"),
|
||||
"postedTotalsMatched": reconciliation.get("posted_totals_matched"),
|
||||
"holdTotalsMatched": reconciliation.get("hold_totals_matched"),
|
||||
"balanceMatched": reconciliation.get("balance_matched"),
|
||||
"reason": reconciliation.get("reason"),
|
||||
},
|
||||
"rows": rows,
|
||||
@@ -181,6 +211,26 @@ def decide_row(
|
||||
return detail(document_id, db, context)
|
||||
|
||||
|
||||
@router.post("/{document_id}/rows/{row_id}/resolve")
|
||||
def resolve_hold(
|
||||
document_id: uuid.UUID,
|
||||
row_id: uuid.UUID,
|
||||
body: HoldResolveBody,
|
||||
db: Db,
|
||||
context: Context,
|
||||
_csrf: Csrf,
|
||||
) -> dict:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=context.space.id,
|
||||
document_id=document_id,
|
||||
source_record_id=row_id,
|
||||
user=context.user,
|
||||
**body.model_dump(by_alias=False),
|
||||
)
|
||||
return detail(document_id, db, context)
|
||||
|
||||
|
||||
@router.post("/{document_id}/confirm")
|
||||
def confirm(
|
||||
document_id: uuid.UUID,
|
||||
|
||||
+330
-21
@@ -9,8 +9,9 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.audit import service as audit
|
||||
from app.config import get_settings
|
||||
from app.db.base import utcnow
|
||||
from app.db.session import session_scope
|
||||
from app.errors import AppError, conflict, not_found
|
||||
from app.errors import AppError, bad_request, conflict, not_found
|
||||
from app.identity.models import User
|
||||
from app.imports.adapters import detect_adapter, magic_kind
|
||||
from app.imports.adapters.base import SourceRow, StatementDraft, record_fingerprint
|
||||
@@ -18,6 +19,7 @@ from app.imports.blob import get_blob_store, new_object_key
|
||||
from app.imports.crypto import decrypt_file, encrypt_file
|
||||
from app.imports.models import (
|
||||
DedupCandidate,
|
||||
HoldResolution,
|
||||
ParseRun,
|
||||
SourceRecord,
|
||||
SourceRecordLink,
|
||||
@@ -31,6 +33,8 @@ from app.ledger.service import (
|
||||
ensure_period_open_for_date,
|
||||
)
|
||||
|
||||
HOLD_MANUAL_ACTIONS = ("confirm_as_posted", "discard_as_cancelled")
|
||||
|
||||
|
||||
def _document(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID) -> StatementDocument:
|
||||
document = (
|
||||
@@ -128,12 +132,16 @@ def _draft_json(draft: StatementDraft) -> dict[str, Any]:
|
||||
def _source_from_row(
|
||||
*, space_id: uuid.UUID, document_id: uuid.UUID, run_id: uuid.UUID, row: SourceRow
|
||||
) -> SourceRecord:
|
||||
critical_warnings = {"occurred_date_missing", "amount_unparseable"}
|
||||
critical_warnings = {"occurred_date_missing", "amount_unparseable", "bank_status_invalid"}
|
||||
warnings = list(row.warnings)
|
||||
if row.direction not in {"in", "out"}:
|
||||
warnings.append("direction_invalid")
|
||||
if row.amount_kopecks == 0:
|
||||
warnings.append("amount_zero")
|
||||
bank_status = getattr(row, "bank_status", "posted")
|
||||
if bank_status not in {"posted", "hold"}:
|
||||
warnings.append("bank_status_invalid")
|
||||
bank_status = "posted"
|
||||
return SourceRecord(
|
||||
space_id=space_id,
|
||||
document_id=document_id,
|
||||
@@ -143,6 +151,7 @@ def _source_from_row(
|
||||
posted_on=row.posted_on,
|
||||
amount_kopecks=abs(row.amount_kopecks),
|
||||
direction=row.direction,
|
||||
bank_status=bank_status,
|
||||
description=row.description,
|
||||
mcc=row.mcc,
|
||||
bank_category=row.bank_category,
|
||||
@@ -158,9 +167,66 @@ def _source_from_row(
|
||||
)
|
||||
|
||||
|
||||
def _reconciliation_summary(draft: Any) -> dict[str, Any]:
|
||||
"""Reconcile parsed rows against declared bank controls with stable diagnostics.
|
||||
|
||||
Posted rows are compared with the declared inflow/outflow controls, HOLD
|
||||
rows with the declared pending total, and where both balances exist the
|
||||
balance equation must hold. Every check keeps an explicit tri-state field:
|
||||
True (matched), False (mismatch, review-blocking), None (not declared).
|
||||
"""
|
||||
rows = list(draft.rows)
|
||||
posted = [row for row in rows if getattr(row, "bank_status", "posted") == "posted"]
|
||||
holds = [row for row in rows if getattr(row, "bank_status", "posted") == "hold"]
|
||||
posted_inflow = sum(row.amount_kopecks for row in posted if row.direction == "in")
|
||||
posted_outflow = sum(row.amount_kopecks for row in posted if row.direction == "out")
|
||||
hold_total = sum(abs(row.amount_kopecks) for row in holds)
|
||||
declared_inflow = getattr(draft, "declared_inflow_kopecks", None)
|
||||
declared_outflow = getattr(draft, "declared_outflow_kopecks", None)
|
||||
declared_hold = getattr(draft, "declared_hold_kopecks", None)
|
||||
full = draft.opening_balance_kopecks is not None and draft.closing_balance_kopecks is not None
|
||||
checks: list[bool] = []
|
||||
balance_matched: bool | None = None
|
||||
computed_closing: int | None = None
|
||||
if full:
|
||||
computed_closing = int(draft.opening_balance_kopecks) + posted_inflow - posted_outflow
|
||||
balance_matched = computed_closing == int(draft.closing_balance_kopecks)
|
||||
checks.append(balance_matched)
|
||||
posted_totals_matched: bool | None = None
|
||||
if declared_inflow is not None or declared_outflow is not None:
|
||||
posted_totals_matched = (declared_inflow is None or declared_inflow == posted_inflow) and (
|
||||
declared_outflow is None or declared_outflow == posted_outflow
|
||||
)
|
||||
checks.append(posted_totals_matched)
|
||||
hold_totals_matched: bool | None = None
|
||||
if declared_hold is not None:
|
||||
hold_totals_matched = declared_hold == hold_total
|
||||
checks.append(hold_totals_matched)
|
||||
return {
|
||||
"currency": draft.currency,
|
||||
"opening_balance_kopecks": draft.opening_balance_kopecks,
|
||||
"closing_balance_kopecks": draft.closing_balance_kopecks,
|
||||
"full": full,
|
||||
"declared_inflow_kopecks": declared_inflow,
|
||||
"declared_outflow_kopecks": declared_outflow,
|
||||
"declared_hold_kopecks": declared_hold,
|
||||
"parsed_posted_inflow_kopecks": posted_inflow,
|
||||
"parsed_posted_outflow_kopecks": posted_outflow,
|
||||
"parsed_hold_total_kopecks": hold_total,
|
||||
"posted_totals_matched": posted_totals_matched,
|
||||
"hold_totals_matched": hold_totals_matched,
|
||||
"balance_matched": balance_matched,
|
||||
"computed_closing_kopecks": computed_closing,
|
||||
"matched": all(checks) if checks else None,
|
||||
"confirmed_without_full_reconciliation": False,
|
||||
}
|
||||
|
||||
|
||||
def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun) -> bool:
|
||||
pending = False
|
||||
for source in db.query(SourceRecord).filter_by(parse_run_id=run.id).all():
|
||||
if source.bank_status == "hold":
|
||||
continue
|
||||
duplicate = None
|
||||
if source.external_id:
|
||||
matches = (
|
||||
@@ -170,6 +236,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
SourceRecord.space_id == document.space_id,
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.external_id == source.external_id,
|
||||
SourceRecord.bank_status == "posted",
|
||||
ParseRun.adapter_id == run.adapter_id,
|
||||
)
|
||||
.all()
|
||||
@@ -183,6 +250,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.record_fp == source.record_fp,
|
||||
SourceRecord.direction == source.direction,
|
||||
SourceRecord.bank_status == "posted",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
@@ -212,6 +280,7 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.amount_kopecks == source.amount_kopecks,
|
||||
SourceRecord.direction == opposite,
|
||||
SourceRecord.bank_status == "posted",
|
||||
SourceRecord.occurred_on >= source.occurred_on - timedelta(days=1),
|
||||
SourceRecord.occurred_on <= source.occurred_on + timedelta(days=1),
|
||||
)
|
||||
@@ -234,6 +303,105 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
return pending
|
||||
|
||||
|
||||
def _match_hold_resolutions(
|
||||
db: Session, *, document: StatementDocument, run: ParseRun
|
||||
) -> dict[uuid.UUID, uuid.UUID]:
|
||||
"""Conservatively match unresolved confirmed-doc HOLDs to new posted records.
|
||||
|
||||
Candidate pairs are searched only within the same space, mapped account,
|
||||
and statement contract (adapter). A stable external ID wins; otherwise a
|
||||
single unambiguous date/amount/direction/normalized-description
|
||||
fingerprint match is required. Any overlap ambiguity leaves every
|
||||
involved HOLD unresolved (review-required). Resolutions are written only
|
||||
when the new statement is confirmed.
|
||||
"""
|
||||
if run.adapter_id is None or document.account_id is None:
|
||||
return {}
|
||||
holds = (
|
||||
db.query(SourceRecord)
|
||||
.join(
|
||||
StatementDocument,
|
||||
(StatementDocument.space_id == SourceRecord.space_id)
|
||||
& (StatementDocument.id == SourceRecord.document_id),
|
||||
)
|
||||
.outerjoin(HoldResolution, HoldResolution.hold_source_record_id == SourceRecord.id)
|
||||
.filter(
|
||||
SourceRecord.space_id == document.space_id,
|
||||
SourceRecord.bank_status == "hold",
|
||||
SourceRecord.parse_run_id != run.id,
|
||||
StatementDocument.status == "confirmed",
|
||||
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
|
||||
StatementDocument.adapter_id == run.adapter_id,
|
||||
StatementDocument.account_id == document.account_id,
|
||||
HoldResolution.id.is_(None),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
if not holds:
|
||||
return {}
|
||||
posted = (
|
||||
db.query(SourceRecord)
|
||||
.filter_by(parse_run_id=run.id, bank_status="posted")
|
||||
.filter(SourceRecord.review_decision != "discard")
|
||||
.all()
|
||||
)
|
||||
by_external: dict[str, list[SourceRecord]] = {}
|
||||
by_fingerprint: dict[tuple[str, str], list[SourceRecord]] = {}
|
||||
for record in posted:
|
||||
if record.external_id:
|
||||
by_external.setdefault(record.external_id, []).append(record)
|
||||
by_fingerprint.setdefault((record.record_fp, record.direction), []).append(record)
|
||||
candidates: dict[uuid.UUID, set[uuid.UUID]] = {}
|
||||
for hold in holds:
|
||||
matched: set[uuid.UUID] = set()
|
||||
if hold.external_id:
|
||||
matched.update(item.id for item in by_external.get(hold.external_id, []))
|
||||
matched.update(item.id for item in by_fingerprint.get((hold.record_fp, hold.direction), []))
|
||||
if len(matched) == 1:
|
||||
candidates[hold.id] = matched
|
||||
claimed: dict[uuid.UUID, int] = {}
|
||||
for posted_ids in candidates.values():
|
||||
for posted_id in posted_ids:
|
||||
claimed[posted_id] = claimed.get(posted_id, 0) + 1
|
||||
return {
|
||||
hold_id: next(iter(posted_ids))
|
||||
for hold_id, posted_ids in candidates.items()
|
||||
if all(claimed[posted_id] == 1 for posted_id in posted_ids)
|
||||
}
|
||||
|
||||
|
||||
def _apply_hold_matches(
|
||||
db: Session,
|
||||
*,
|
||||
document: StatementDocument,
|
||||
run: ParseRun,
|
||||
user: User,
|
||||
matches: dict[uuid.UUID, uuid.UUID],
|
||||
) -> None:
|
||||
for hold_id, posted_id in matches.items():
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=document.space_id,
|
||||
hold_source_record_id=hold_id,
|
||||
resolution="matched_posted",
|
||||
resolving_source_record_id=posted_id,
|
||||
resolved_at=utcnow(),
|
||||
)
|
||||
)
|
||||
if matches:
|
||||
db.flush()
|
||||
audit.record(
|
||||
db,
|
||||
event_type="import.holds_matched_posted",
|
||||
space_id=document.space_id,
|
||||
user_id=user.id,
|
||||
detail={
|
||||
"document_id": str(document.id),
|
||||
"holds": sorted(str(hold_id) for hold_id in matches),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> str:
|
||||
with session_scope(space_id) as db:
|
||||
document = _document(db, space_id=space_id, document_id=document_id)
|
||||
@@ -273,23 +441,8 @@ def process_statement_document(document_id: uuid.UUID, space_id: uuid.UUID) -> s
|
||||
document.adapter_id = adapter.adapter_id
|
||||
document.account_fingerprint = draft.account_fingerprint
|
||||
document.active_parse_run_id = run.id
|
||||
document.reconciliation = {
|
||||
"currency": draft.currency,
|
||||
"opening_balance_kopecks": draft.opening_balance_kopecks,
|
||||
"closing_balance_kopecks": draft.closing_balance_kopecks,
|
||||
"full": draft.opening_balance_kopecks is not None
|
||||
and draft.closing_balance_kopecks is not None,
|
||||
"confirmed_without_full_reconciliation": False,
|
||||
}
|
||||
if document.reconciliation["full"]:
|
||||
signed_total = sum(
|
||||
row.amount_kopecks if row.direction == "in" else -row.amount_kopecks
|
||||
for row in draft.rows
|
||||
)
|
||||
computed = int(draft.opening_balance_kopecks or 0) + signed_total
|
||||
document.reconciliation["computed_closing_kopecks"] = computed
|
||||
document.reconciliation["matched"] = computed == draft.closing_balance_kopecks
|
||||
if draft.account_fingerprint and document.account_id is None:
|
||||
document.reconciliation = _reconciliation_summary(draft)
|
||||
if document.account_id is None and draft.account_fingerprint:
|
||||
matches = (
|
||||
db.query(Account)
|
||||
.filter_by(space_id=space_id, fingerprint=draft.account_fingerprint)
|
||||
@@ -351,6 +504,25 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
|
||||
.order_by(SourceRecord.created_at)
|
||||
.all()
|
||||
)
|
||||
hold_ids = [record.id for record in records]
|
||||
resolutions: dict[uuid.UUID, dict[str, Any]] = {}
|
||||
if hold_ids:
|
||||
for item in (
|
||||
db.query(HoldResolution)
|
||||
.filter(
|
||||
HoldResolution.space_id == space_id,
|
||||
HoldResolution.hold_source_record_id.in_(hold_ids),
|
||||
)
|
||||
.all()
|
||||
):
|
||||
resolutions[item.hold_source_record_id] = {
|
||||
"resolution": item.resolution,
|
||||
"resolving_source_record_id": item.resolving_source_record_id,
|
||||
"operation_id": item.operation_id,
|
||||
"resolved_by_user_id": item.resolved_by_user_id,
|
||||
"reason": item.reason,
|
||||
"resolved_at": item.resolved_at,
|
||||
}
|
||||
return {
|
||||
"id": document.id,
|
||||
"status": document.status,
|
||||
@@ -367,11 +539,13 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
|
||||
"posted_on": item.posted_on,
|
||||
"amount_kopecks": item.amount_kopecks,
|
||||
"direction": item.direction,
|
||||
"bank_status": item.bank_status,
|
||||
"description": item.description,
|
||||
"critical": item.critical,
|
||||
"warnings": item.field_warnings,
|
||||
"review_decision": item.review_decision,
|
||||
"duplicate_operation_id": item.duplicate_operation_id,
|
||||
"hold_resolution": resolutions.get(item.id),
|
||||
}
|
||||
for item in records
|
||||
],
|
||||
@@ -489,6 +663,119 @@ def set_review_confirmation(
|
||||
return document
|
||||
|
||||
|
||||
def resolve_hold(
|
||||
db: Session,
|
||||
*,
|
||||
space_id: uuid.UUID,
|
||||
document_id: uuid.UUID,
|
||||
source_record_id: uuid.UUID,
|
||||
user: User,
|
||||
action: str,
|
||||
reason: str | None,
|
||||
account_id: uuid.UUID | None = None,
|
||||
) -> HoldResolution:
|
||||
"""Manually settle one HOLD: book it as a normal operation or discard it."""
|
||||
document = _document(db, space_id=space_id, document_id=document_id)
|
||||
if document.status != "confirmed":
|
||||
raise conflict(
|
||||
"IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED",
|
||||
"Holds can only be resolved after the import is confirmed",
|
||||
)
|
||||
if action not in HOLD_MANUAL_ACTIONS:
|
||||
raise bad_request("IMPORT_HOLD_ACTION_INVALID", "Unknown hold resolution action")
|
||||
if reason is None or not reason.strip():
|
||||
raise conflict("IMPORT_HOLD_REASON_REQUIRED", "A resolution reason is required")
|
||||
source = (
|
||||
db.query(SourceRecord)
|
||||
.filter_by(
|
||||
id=source_record_id,
|
||||
space_id=space_id,
|
||||
parse_run_id=document.active_parse_run_id,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if source is None:
|
||||
raise not_found("IMPORT_HOLD_NOT_FOUND", "Import row not found")
|
||||
if source.bank_status != "hold":
|
||||
raise conflict("IMPORT_HOLD_INVALID", "Only hold rows can be resolved manually")
|
||||
existing = db.query(HoldResolution.id).filter_by(hold_source_record_id=source.id).one_or_none()
|
||||
if existing is not None:
|
||||
raise conflict("IMPORT_HOLD_ALREADY_RESOLVED", "Hold is already resolved")
|
||||
if source.occurred_on is None:
|
||||
raise conflict("IMPORT_HOLD_DATE_MISSING", "Hold has no usable operation date")
|
||||
if source.direction not in {"in", "out"}:
|
||||
raise conflict("IMPORT_HOLD_DIRECTION_INVALID", "Hold direction is invalid")
|
||||
resolution = HoldResolution(
|
||||
space_id=space_id,
|
||||
hold_source_record_id=source.id,
|
||||
resolution="confirmed_as_posted"
|
||||
if action == "confirm_as_posted"
|
||||
else "discarded_as_cancelled",
|
||||
resolved_by_user_id=user.id,
|
||||
reason=reason.strip(),
|
||||
resolved_at=utcnow(),
|
||||
)
|
||||
if action == "confirm_as_posted":
|
||||
mapped_account_id = account_id or document.account_id
|
||||
if mapped_account_id is None:
|
||||
raise conflict(
|
||||
"IMPORT_HOLD_ACCOUNT_REQUIRED",
|
||||
"A mapped account is required to confirm a hold as posted",
|
||||
)
|
||||
account = db.query(Account).filter_by(id=mapped_account_id, space_id=space_id).one_or_none()
|
||||
if account is None:
|
||||
raise not_found("LEDGER_ACCOUNT_NOT_FOUND", "Account not found")
|
||||
if source.direction == "in":
|
||||
operation = create_income(
|
||||
db,
|
||||
space_id=space_id,
|
||||
user=user,
|
||||
account_id=account.id,
|
||||
amount_kopecks=source.amount_kopecks,
|
||||
occurred_on=source.occurred_on,
|
||||
posted_on=source.posted_on,
|
||||
description=source.description,
|
||||
request_key=f"import:{source.id}",
|
||||
)
|
||||
else:
|
||||
operation = create_expense(
|
||||
db,
|
||||
space_id=space_id,
|
||||
user=user,
|
||||
account_id=account.id,
|
||||
amount_kopecks=source.amount_kopecks,
|
||||
occurred_on=source.occurred_on,
|
||||
posted_on=source.posted_on,
|
||||
description=source.description,
|
||||
request_key=f"import:{source.id}",
|
||||
)
|
||||
db.add(
|
||||
SourceRecordLink(
|
||||
space_id=space_id,
|
||||
source_record_id=source.id,
|
||||
operation_id=operation.id,
|
||||
kind="hold_confirmed",
|
||||
)
|
||||
)
|
||||
resolution.operation_id = operation.id
|
||||
db.add(resolution)
|
||||
db.flush()
|
||||
audit.record(
|
||||
db,
|
||||
event_type="import.hold_confirmed_as_posted"
|
||||
if action == "confirm_as_posted"
|
||||
else "import.hold_discarded_as_cancelled",
|
||||
space_id=space_id,
|
||||
user_id=user.id,
|
||||
detail={
|
||||
"document_id": str(document.id),
|
||||
"source_record_id": str(source.id),
|
||||
"reason": reason.strip(),
|
||||
},
|
||||
)
|
||||
return resolution
|
||||
|
||||
|
||||
def confirm_document(
|
||||
db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID, user: User
|
||||
) -> StatementDocument:
|
||||
@@ -503,7 +790,7 @@ def confirm_document(
|
||||
"confirmed_without_full_reconciliation"
|
||||
):
|
||||
raise conflict("IMPORT_RECONCILIATION_REQUIRED", "Reconciliation must be confirmed")
|
||||
if reconciliation.get("full") and reconciliation.get("matched") is False:
|
||||
if reconciliation.get("matched") is False:
|
||||
raise conflict("IMPORT_RECONCILIATION_MISMATCH", "Statement arithmetic does not reconcile")
|
||||
pending = (
|
||||
db.query(DedupCandidate.id)
|
||||
@@ -519,7 +806,18 @@ def confirm_document(
|
||||
records = db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).all()
|
||||
if any(item.critical for item in records):
|
||||
raise conflict("IMPORT_FIELDS_UNRESOLVED", "Critical source fields remain unresolved")
|
||||
run = db.get(ParseRun, document.active_parse_run_id)
|
||||
if run is not None:
|
||||
_apply_hold_matches(
|
||||
db,
|
||||
document=document,
|
||||
run=run,
|
||||
user=user,
|
||||
matches=_match_hold_resolutions(db, document=document, run=run),
|
||||
)
|
||||
for source in records:
|
||||
if source.bank_status == "hold":
|
||||
continue
|
||||
if source.review_decision == "discard":
|
||||
continue
|
||||
if source.review_decision == "transfer":
|
||||
@@ -755,10 +1053,21 @@ def revert_document(
|
||||
document = _document(db, space_id=space_id, document_id=document_id)
|
||||
if document.status != "confirmed":
|
||||
return document
|
||||
if document.active_parse_run_id:
|
||||
db.query(HoldResolution).filter(
|
||||
HoldResolution.space_id == space_id,
|
||||
HoldResolution.resolution == "matched_posted",
|
||||
HoldResolution.resolving_source_record_id.in_(
|
||||
db.query(SourceRecord.id).filter_by(parse_run_id=document.active_parse_run_id)
|
||||
),
|
||||
).delete(synchronize_session=False)
|
||||
links = (
|
||||
db.query(SourceRecordLink)
|
||||
.join(SourceRecord, SourceRecord.id == SourceRecordLink.source_record_id)
|
||||
.filter(SourceRecord.parse_run_id == document.active_parse_run_id)
|
||||
.filter(
|
||||
SourceRecord.parse_run_id == document.active_parse_run_id,
|
||||
SourceRecordLink.kind != "hold_confirmed",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
transfer_candidates = (
|
||||
|
||||
@@ -39,6 +39,10 @@ package = false
|
||||
pythonpath = ["."]
|
||||
testpaths = ["tests"]
|
||||
filterwarnings = ["ignore::DeprecationWarning"]
|
||||
addopts = "-m 'not private_fixtures'"
|
||||
markers = [
|
||||
"private_fixtures: real-statement private suite; run explicitly with -m private_fixtures (never in CI)",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
|
||||
@@ -0,0 +1,825 @@
|
||||
"""Sanitized synthetic contract tests for the evidenced statement adapters.
|
||||
|
||||
These tests never touch real statements: every input below is synthetic and
|
||||
tracked. Real-file validation lives in the separately collected private
|
||||
suite (tests/test_private_fixtures.py, marker `private_fixtures`).
|
||||
"""
|
||||
|
||||
import io
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
from openpyxl import Workbook
|
||||
|
||||
from app.errors import AppError
|
||||
from app.imports.adapters import ADAPTERS, detect_adapter, magic_kind
|
||||
from app.imports.adapters import alfa_xlsx as alfa_module
|
||||
from app.imports.adapters.alfa_xlsx import AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter
|
||||
from app.imports.adapters.base import (
|
||||
ALFA_DEBIT_XLSX_VERSION,
|
||||
BANK_STATUS_HOLD,
|
||||
OZON_DEBIT_PDF_VERSION,
|
||||
PARSER_VERSION,
|
||||
ROCKETBANK_DEBIT_PDF_VERSION,
|
||||
SourceRow,
|
||||
StatementDraft,
|
||||
UnsupportedStatementLayout,
|
||||
account_fingerprint,
|
||||
money_to_kopecks,
|
||||
)
|
||||
from app.imports.adapters.ozon_pdf import OzonCreditPdfAdapter, OzonDebitPdfAdapter
|
||||
from app.imports.adapters.rocketbank_pdf import RocketbankDebitPdfAdapter
|
||||
from app.imports.service import _reconciliation_summary
|
||||
|
||||
NBSP = "\u00a0"
|
||||
|
||||
|
||||
def make_source_row(**overrides: object) -> SourceRow:
|
||||
values: dict[str, object] = {
|
||||
"external_id": None,
|
||||
"occurred_on": date(2026, 1, 1),
|
||||
"posted_on": date(2026, 1, 2),
|
||||
"amount_kopecks": 100,
|
||||
"direction": "out",
|
||||
"description": None,
|
||||
"mcc": None,
|
||||
"bank_category": None,
|
||||
"location": None,
|
||||
"row_ref": "row1",
|
||||
}
|
||||
values.update(overrides)
|
||||
return SourceRow(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def make_draft(**overrides: object) -> StatementDraft:
|
||||
values: dict[str, object] = {
|
||||
"adapter_id": "alfa_debit_xlsx",
|
||||
"parser_version": ALFA_DEBIT_XLSX_VERSION,
|
||||
"bank_code": "alfa",
|
||||
"product": "debit",
|
||||
"account_fingerprint": None,
|
||||
"currency": "RUB",
|
||||
"period_start": None,
|
||||
"period_end": None,
|
||||
"opening_balance_kopecks": None,
|
||||
"closing_balance_kopecks": None,
|
||||
"rows": (),
|
||||
}
|
||||
values.update(overrides)
|
||||
return StatementDraft(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestBaseContract:
|
||||
def test_source_row_defaults_to_posted(self) -> None:
|
||||
assert make_source_row().bank_status == "posted"
|
||||
|
||||
def test_negative_amount_is_rejected(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
make_source_row(amount_kopecks=-1)
|
||||
|
||||
def test_unknown_bank_status_is_rejected(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
make_source_row(bank_status="pending")
|
||||
|
||||
def test_draft_declared_totals_default_to_none(self) -> None:
|
||||
draft = make_draft()
|
||||
assert draft.declared_inflow_kopecks is None
|
||||
assert draft.declared_outflow_kopecks is None
|
||||
assert draft.declared_hold_kopecks is None
|
||||
|
||||
def test_per_contract_parser_versions(self) -> None:
|
||||
assert PARSER_VERSION == "1"
|
||||
assert ALFA_DEBIT_XLSX_VERSION == "2"
|
||||
assert OZON_DEBIT_PDF_VERSION == "2"
|
||||
assert ROCKETBANK_DEBIT_PDF_VERSION == "2"
|
||||
assert AlfaDebitXlsxAdapter.parser_version == ALFA_DEBIT_XLSX_VERSION
|
||||
assert OzonDebitPdfAdapter.parser_version == OZON_DEBIT_PDF_VERSION
|
||||
assert RocketbankDebitPdfAdapter.parser_version == ROCKETBANK_DEBIT_PDF_VERSION
|
||||
|
||||
def test_registry_lists_only_verified_contracts(self) -> None:
|
||||
assert [adapter.adapter_id for adapter in ADAPTERS] == [
|
||||
"alfa_debit_xlsx",
|
||||
"ozon_debit_pdf",
|
||||
"rocketbank_debit_pdf",
|
||||
]
|
||||
|
||||
def test_credit_adapters_stay_present_but_unregistered(self) -> None:
|
||||
assert AlfaCreditXlsxAdapter.adapter_id == "alfa_credit_xlsx"
|
||||
assert OzonCreditPdfAdapter.adapter_id == "ozon_credit_pdf"
|
||||
assert all(
|
||||
adapter.adapter_id not in {"alfa_credit_xlsx", "ozon_credit_pdf"}
|
||||
for adapter in ADAPTERS
|
||||
)
|
||||
|
||||
def test_detect_adapter_fails_closed_on_unknown_xlsx(self) -> None:
|
||||
buffer = io.BytesIO()
|
||||
Workbook().save(buffer)
|
||||
raw = buffer.getvalue()
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
detect_adapter(raw, "xlsx")
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_detect_adapter_fails_closed_on_unknown_pdf(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
detect_adapter(minimal_pdf(), "pdf")
|
||||
|
||||
def test_magic_kind(self) -> None:
|
||||
assert magic_kind(b"PK\x03\x04") == "xlsx"
|
||||
assert magic_kind(b"%PDF-1.4") == "pdf"
|
||||
assert magic_kind(b"not a statement") is None
|
||||
|
||||
|
||||
class TestMoneyParsing:
|
||||
def test_plain_and_separated_amounts(self) -> None:
|
||||
assert money_to_kopecks("123") == 12300
|
||||
assert money_to_kopecks(f"1{NBSP}234,56") == 123456
|
||||
assert money_to_kopecks("1 234.56") == 123456
|
||||
assert money_to_kopecks("-1 234,56 RUR") == -123456
|
||||
assert money_to_kopecks("+ 500.00 \u20bd") == 50000
|
||||
|
||||
def test_malformed_amount_fails_closed_without_leaking_digits(self) -> None:
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
money_to_kopecks("12a3,45")
|
||||
assert "#" in exc.value.message
|
||||
assert "12" not in exc.value.message
|
||||
|
||||
|
||||
def alfa_workbook_bytes(rows: list[list[object]]) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
workbook = Workbook()
|
||||
sheet = workbook.worksheets[0]
|
||||
for row in rows:
|
||||
sheet.append(row)
|
||||
workbook.save(buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def synthetic_alfa_rows() -> list[list[object]]:
|
||||
return [
|
||||
[None, None, None],
|
||||
["Выписка по счету"],
|
||||
["За период с 01.09.2026 по 04.09.2026"],
|
||||
["Номер счета", None, "00000000000000000001"],
|
||||
["Дата открытия счета", None, "01.01.2025"],
|
||||
["Валюта счета", None, "RUR"],
|
||||
["Тип счета", None, "Текущий счёт"],
|
||||
["Дата формирования выписки", None, "10.09.2026"],
|
||||
[
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Поступления",
|
||||
None,
|
||||
"1 000,00 RUR",
|
||||
],
|
||||
[None, None, None, None, None, None, None, None, None, None, "Расходы", None, "293,45 RUR"],
|
||||
[
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Неподтвержденные операции",
|
||||
None,
|
||||
"100,00 RUR",
|
||||
],
|
||||
[
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Текущий баланс",
|
||||
None,
|
||||
"800,00 RUR",
|
||||
],
|
||||
["Операции по счету"],
|
||||
[
|
||||
"Дата операции",
|
||||
"Дата проводки",
|
||||
None,
|
||||
"Код",
|
||||
"Категория",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Описание",
|
||||
f"Сумма в{NBSP}валюте{NBSP}счета",
|
||||
None,
|
||||
"Статус",
|
||||
],
|
||||
[
|
||||
"01.09.2026",
|
||||
"01.09.2026",
|
||||
None,
|
||||
"000000000000001",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Магазин 1 MCC: 5411",
|
||||
"-123,45",
|
||||
"Выполнен",
|
||||
],
|
||||
[
|
||||
"02.09.2026",
|
||||
"02.09.2026",
|
||||
None,
|
||||
"000000000000002",
|
||||
"Переводы",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Перевод 2 MCC5912",
|
||||
"1 000,00",
|
||||
"Выполнен",
|
||||
],
|
||||
[
|
||||
"03.09.2026",
|
||||
"HOLD",
|
||||
None,
|
||||
"000000000000003",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Покупка 3",
|
||||
"-100,00",
|
||||
],
|
||||
[
|
||||
"04.09.2026",
|
||||
"04.09.2026",
|
||||
None,
|
||||
"000000000000004",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Покупка 4 MCC 5941",
|
||||
"-50,00",
|
||||
"Выполнен",
|
||||
],
|
||||
[
|
||||
"05.09.2026",
|
||||
"05.09.2026",
|
||||
None,
|
||||
"000000000000005",
|
||||
"Покупки",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Покупка 5",
|
||||
"-20,00",
|
||||
"Выполнен",
|
||||
],
|
||||
[None],
|
||||
[
|
||||
"(подпись сотрудника АО «БАНК»)",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Уполномоченное лицо",
|
||||
],
|
||||
["Страница 1 из 1"],
|
||||
]
|
||||
|
||||
|
||||
class TestAlfaDebitContract:
|
||||
def setup_method(self) -> None:
|
||||
self.adapter = AlfaDebitXlsxAdapter()
|
||||
self.raw = alfa_workbook_bytes(synthetic_alfa_rows())
|
||||
|
||||
def test_detect(self) -> None:
|
||||
assert self.adapter.detect(self.raw) is True
|
||||
|
||||
def test_detect_rejects_workbook_without_contract_header(self) -> None:
|
||||
raw = alfa_workbook_bytes([["Выписка по счету"], ["Что-то другое", "123"]])
|
||||
assert self.adapter.detect(raw) is False
|
||||
|
||||
def test_parse_contract_fields(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert draft.adapter_id == "alfa_debit_xlsx"
|
||||
assert draft.parser_version == ALFA_DEBIT_XLSX_VERSION
|
||||
assert draft.bank_code == "alfa"
|
||||
assert draft.product == "debit"
|
||||
assert draft.currency == "RUB"
|
||||
assert draft.account_fingerprint == account_fingerprint("alfa", "00000000000000000001")
|
||||
assert draft.declared_inflow_kopecks == 100000
|
||||
# canonical posted-only: raw 293,45 minus declared hold 100,00
|
||||
assert draft.declared_outflow_kopecks == 19345
|
||||
assert draft.declared_hold_kopecks == 10000
|
||||
assert draft.closing_balance_kopecks == 80000
|
||||
assert draft.opening_balance_kopecks is None
|
||||
# exact header period fields, not min/max of transaction dates
|
||||
assert draft.period_start == date(2026, 9, 1)
|
||||
assert draft.period_end == date(2026, 9, 4)
|
||||
|
||||
def test_missing_period_field_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows[2] = ["Что-то другое"]
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
assert "period" in exc.value.message
|
||||
|
||||
def test_inconsistent_expense_control_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows[9] = [
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"Расходы",
|
||||
None,
|
||||
"50,00 RUR",
|
||||
]
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
assert "expense" in exc.value.message
|
||||
|
||||
def test_parse_rows_with_hold(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert len(draft.rows) == 5
|
||||
first, second, hold, fourth, last = draft.rows
|
||||
assert (first.amount_kopecks, first.direction, first.bank_status) == (
|
||||
12345,
|
||||
"out",
|
||||
"posted",
|
||||
)
|
||||
assert first.posted_on == date(2026, 9, 1)
|
||||
assert (second.amount_kopecks, second.direction) == (100000, "in")
|
||||
assert hold.bank_status == BANK_STATUS_HOLD
|
||||
assert hold.posted_on is None
|
||||
assert hold.amount_kopecks == 10000
|
||||
assert hold.direction == "out"
|
||||
assert last.warnings == ()
|
||||
assert first.row_ref == "sheet0:row15"
|
||||
assert first.bank_category == "Покупки"
|
||||
|
||||
def test_external_ids_from_code_column(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert [row.external_id for row in draft.rows] == [
|
||||
"000000000000001",
|
||||
"000000000000002",
|
||||
"000000000000003",
|
||||
"000000000000004",
|
||||
"000000000000005",
|
||||
]
|
||||
|
||||
def test_mcc_from_description(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
first, second, hold, fourth, last = draft.rows
|
||||
assert first.mcc == 5411
|
||||
assert second.mcc == 5912
|
||||
assert hold.mcc is None
|
||||
assert fourth.mcc == 5941
|
||||
assert last.mcc is None
|
||||
assert all(row.location is None for row in draft.rows)
|
||||
|
||||
def test_footer_is_excluded(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
assert all(row.occurred_on is not None for row in draft.rows)
|
||||
assert draft.rows[-1].occurred_on == date(2026, 9, 5)
|
||||
|
||||
def test_parse_is_deterministic(self) -> None:
|
||||
assert self.adapter.parse(self.raw) == self.adapter.parse(self.raw)
|
||||
|
||||
def test_service_reconciliation_contract(self) -> None:
|
||||
draft = self.adapter.parse(self.raw)
|
||||
summary = _reconciliation_summary(draft)
|
||||
assert summary["matched"] is True
|
||||
assert summary["posted_totals_matched"] is True
|
||||
assert summary["hold_totals_matched"] is True
|
||||
|
||||
def test_service_reconciliation_flags_mismatch(self) -> None:
|
||||
draft = make_draft(
|
||||
declared_inflow_kopecks=1000,
|
||||
declared_outflow_kopecks=1,
|
||||
declared_hold_kopecks=0,
|
||||
rows=(make_source_row(amount_kopecks=1000, direction="in"),),
|
||||
)
|
||||
summary = _reconciliation_summary(draft)
|
||||
assert summary["posted_totals_matched"] is False
|
||||
assert summary["matched"] is False
|
||||
|
||||
def test_malformed_layout_fails_closed(self) -> None:
|
||||
raw = alfa_workbook_bytes([["Дата", "Сумма"], ["01.09.2026", "-10,00"]])
|
||||
assert self.adapter.detect(raw) is False
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(raw)
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_unexpected_mid_table_row_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows.insert(16, ["unexpected layout drift"])
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_unsupported_currency_fails_closed(self) -> None:
|
||||
rows = [list(row) for row in synthetic_alfa_rows()]
|
||||
rows[5][2] = "USD"
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter.parse(alfa_workbook_bytes(rows))
|
||||
|
||||
def test_row_quota_is_enforced(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(alfa_module, "get_settings_row_cap", lambda: 3)
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(self.raw)
|
||||
assert "quota" in exc.value.message
|
||||
|
||||
|
||||
def ozon_extracted() -> tuple[list[str], list[list[list[object]]]]:
|
||||
pages_text = [
|
||||
"\n".join(
|
||||
[
|
||||
"ООО «ОЗОН Банк», ИНН 1234567890, город Москва",
|
||||
"Справка о движении средств",
|
||||
"Номер лицевого счёта: № 00000000000000000001, открыт 01.01.2025",
|
||||
"Период выписки: 01.02.2026 – 28.02.2026",
|
||||
"Входящий остаток: 1000.50 \u20bd",
|
||||
]
|
||||
),
|
||||
"",
|
||||
"\n".join(
|
||||
[
|
||||
"Итого зачислений за период: 3000.00 \u20bd",
|
||||
"Итого списаний за период: 2500.75 \u20bd",
|
||||
"Исходящий остаток: 1499.75 \u20bd",
|
||||
]
|
||||
),
|
||||
]
|
||||
pages_tables = [
|
||||
[
|
||||
[
|
||||
[
|
||||
"Дата операции",
|
||||
"Документ",
|
||||
"Назначение платежа",
|
||||
"Сумма операции",
|
||||
None,
|
||||
],
|
||||
[None, None, None, "Российские рубли", "Валюта"],
|
||||
[
|
||||
"01.02.2026 10:00:00",
|
||||
"1234567890\n1",
|
||||
"Оплата товаров\nпо карте 1234",
|
||||
"- 500.00 \u20bd",
|
||||
"- 500.00 \u20bd",
|
||||
],
|
||||
]
|
||||
],
|
||||
[
|
||||
[
|
||||
[
|
||||
"02.02.2026 11:30:00",
|
||||
"1234567891",
|
||||
"Перевод\nчерез СБП",
|
||||
"+ 2 000,00 \u20bd",
|
||||
"+ 2 000,00 \u20bd",
|
||||
]
|
||||
]
|
||||
],
|
||||
[],
|
||||
]
|
||||
return pages_text, pages_tables
|
||||
|
||||
|
||||
class TestOzonDebitContract:
|
||||
def setup_method(self) -> None:
|
||||
self.adapter = OzonDebitPdfAdapter()
|
||||
self.pages_text, self.pages_tables = ozon_extracted()
|
||||
|
||||
def test_core_contract_fields(self) -> None:
|
||||
draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables)
|
||||
assert draft.adapter_id == "ozon_debit_pdf"
|
||||
assert draft.bank_code == "ozon"
|
||||
assert draft.currency == "RUB"
|
||||
assert draft.account_fingerprint == account_fingerprint("ozon", "00000000000000000001")
|
||||
assert draft.period_start == date(2026, 2, 1)
|
||||
assert draft.period_end == date(2026, 2, 28)
|
||||
assert draft.opening_balance_kopecks == 100050
|
||||
assert draft.closing_balance_kopecks == 149975
|
||||
assert draft.declared_inflow_kopecks == 300000
|
||||
assert draft.declared_outflow_kopecks == 250075
|
||||
assert draft.declared_hold_kopecks is None
|
||||
|
||||
def test_multiline_cells_stay_one_transaction(self) -> None:
|
||||
draft = self.adapter._draft_from_extracted(self.pages_text, self.pages_tables)
|
||||
assert len(draft.rows) == 2
|
||||
first, second = draft.rows
|
||||
assert first.external_id == "12345678901"
|
||||
assert first.description == "Оплата товаров по карте 1234"
|
||||
assert (first.amount_kopecks, first.direction) == (50000, "out")
|
||||
assert first.occurred_on == date(2026, 2, 1)
|
||||
assert (second.amount_kopecks, second.direction) == (200000, "in")
|
||||
assert second.external_id == "1234567891"
|
||||
assert first.warnings == ()
|
||||
|
||||
def test_missing_header_marker_fails_closed(self) -> None:
|
||||
pages_text, pages_tables = ozon_extracted()
|
||||
pages_text[0] = "Какой-то другой документ"
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def test_missing_totals_fail_closed(self) -> None:
|
||||
pages_text, pages_tables = ozon_extracted()
|
||||
pages_text[2] = ""
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def test_no_rows_fails_closed(self) -> None:
|
||||
pages_text, _ = ozon_extracted()
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, [[]])
|
||||
|
||||
def test_unexpected_row_shape_fails_closed(self) -> None:
|
||||
pages_text, pages_tables = ozon_extracted()
|
||||
pages_tables[0][0].append([None, None, "текст без суммы"])
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_extracted(pages_text, pages_tables)
|
||||
|
||||
def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None:
|
||||
raw = minimal_pdf()
|
||||
assert self.adapter.detect(raw) is False
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(raw)
|
||||
assert "text layer" in exc.value.message
|
||||
|
||||
def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import app.imports.readers as readers_module
|
||||
|
||||
monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings())
|
||||
with pytest.raises(AppError) as exc:
|
||||
self.adapter.parse(minimal_pdf(page_count=2))
|
||||
assert exc.value.code == "IMPORT_QUOTA_EXCEEDED"
|
||||
|
||||
|
||||
def same_page_rocket_lines() -> list[tuple[int, str]]:
|
||||
return [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "С 01.07.2026 ПО 10.09.2026"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ"),
|
||||
(1, "0,00 RUR 0,00 RUR"),
|
||||
(1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"),
|
||||
(1, "0,00 RUR 0,00 RUR"),
|
||||
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
|
||||
(1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
|
||||
(1, "10:00 00000000001 -10,00 RUR"),
|
||||
(1, "11:00 00000000002 -20,00 RUR"),
|
||||
(1, "12:00 00000000003 -30,00 RUR"),
|
||||
(1, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"),
|
||||
(1, "ВСЕГО РАСХОДОВ: 60,00 RUR"),
|
||||
]
|
||||
|
||||
|
||||
def synthetic_rocket_lines() -> list[tuple[int, str]]:
|
||||
return [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"),
|
||||
(1, "С 01.07.2026 ПО 10.09.2026"),
|
||||
(1, "ИВАНОВ ИВАН ИВАНОВИЧ"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "ВХОДЯЩИЙ ОСТАТОК ПОСТУПЛЕНИЯ ДАТА ПОСЛЕДНЕГО ДВИЖЕНИЯ"),
|
||||
(1, "0,00 RUR 1 000,00 RUR 01.07.2026"),
|
||||
(1, "ИСХОДЯЩИЙ ОСТАТОК РАСХОДЫ"),
|
||||
(1, "500,00 RUR 500,00 RUR"),
|
||||
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
|
||||
(1, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
|
||||
(1, "ЗАЧИСЛЕНИЕ ПЕРЕВОДА ДЕНЕЖНЫХ СРЕДСТВ/СБП/"),
|
||||
(1, "12:00 00000000001 +1 000,00 RUR"),
|
||||
(1, "СЧЕТ КОРРЕСПОНДЕНТА БАНК ВТБ (ПАО)."),
|
||||
(1, "ВСЕГО ПОСТУПЛЕНИЙ: 1 000,00 RUR"),
|
||||
(1, "ВСЕГО РАСХОДОВ: 0,00 RUR"),
|
||||
(1, "СТРАНИЦА 1/2"),
|
||||
(2, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(2, "ДАТА ВЫДАЧИ ВЫПИСКИ: 11.09.2026"),
|
||||
(2, "С 01.07.2026 ПО 10.09.2026"),
|
||||
(2, "02 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 1 000,00 RUR"),
|
||||
(2, "ВРЕМЯ КОД ОПЕРАЦИИ ОПИСАНИЕ СУММА"),
|
||||
(2, "ПЛАТЕЖ АВТОРИЗАЦИЯ №123,01.07.2026 10:00,0000********0000,"),
|
||||
(2, "13:30 00000000002 555RUR,EKATERINBURG,MCC 5411,SHOP\\RU\\CITY\\SHOP\\ -250,50 RUR"),
|
||||
(2, "0000\\ ЗДАНИЕ."),
|
||||
(2, "ВСЕГО ПОСТУПЛЕНИЙ: 0,00 RUR"),
|
||||
(2, "ВСЕГО РАСХОДОВ: 250,50 RUR"),
|
||||
(2, "СТРАНИЦА 2/2"),
|
||||
]
|
||||
|
||||
|
||||
class TestRocketbankDebitContract:
|
||||
def setup_method(self) -> None:
|
||||
self.adapter = RocketbankDebitPdfAdapter()
|
||||
|
||||
def test_day_and_page_state_machine(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
assert len(draft.rows) == 2
|
||||
first, second = draft.rows
|
||||
assert first.occurred_on == date(2026, 7, 1)
|
||||
assert second.occurred_on == date(2026, 7, 2)
|
||||
|
||||
def test_row_refs_are_unique_within_a_page(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
first, second = draft.rows
|
||||
assert first.row_ref == "page1:record1"
|
||||
assert second.row_ref == "page2:record1"
|
||||
same_page = self.adapter._draft_from_lines(same_page_rocket_lines())
|
||||
refs = [row.row_ref for row in same_page.rows]
|
||||
assert refs == ["page1:record1", "page1:record2", "page1:record3"]
|
||||
assert len(set(refs)) == len(refs)
|
||||
|
||||
def test_statement_controls(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
assert draft.adapter_id == "rocketbank_debit_pdf"
|
||||
assert draft.bank_code == "rocketbank"
|
||||
assert draft.currency == "RUB"
|
||||
assert draft.account_fingerprint == account_fingerprint(
|
||||
"rocketbank", "00000000000000000001"
|
||||
)
|
||||
assert draft.period_start == date(2026, 7, 1)
|
||||
assert draft.period_end == date(2026, 9, 10)
|
||||
assert draft.opening_balance_kopecks == 0
|
||||
assert draft.closing_balance_kopecks == 50000
|
||||
assert draft.declared_inflow_kopecks == 100000
|
||||
assert draft.declared_outflow_kopecks == 50000
|
||||
assert draft.declared_hold_kopecks is None
|
||||
|
||||
def test_record_folding_and_enrichment(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
first, second = draft.rows
|
||||
assert first.external_id == "00000000001"
|
||||
assert (first.amount_kopecks, first.direction) == (100000, "in")
|
||||
assert "ЗАЧИСЛЕНИЕ ПЕРЕВОДА" in (first.description or "")
|
||||
assert "СЧЕТ КОРРЕСПОНДЕНТА" in (first.description or "")
|
||||
assert first.mcc is None
|
||||
assert second.external_id == "00000000002"
|
||||
assert (second.amount_kopecks, second.direction) == (25050, "out")
|
||||
assert second.mcc == 5411
|
||||
assert second.location == "EKATERINBURG"
|
||||
assert (second.description or "").startswith("ПЛАТЕЖ АВТОРИЗАЦИЯ")
|
||||
assert "555RUR" in (second.description or "")
|
||||
assert "ЗДАНИЕ." in (second.description or "")
|
||||
assert all(row.warnings == () for row in draft.rows)
|
||||
|
||||
def test_repeated_headers_and_daily_totals_are_ignored(self) -> None:
|
||||
draft = self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
joined = " ".join(row.description or "" for row in draft.rows)
|
||||
assert "ВСЕГО" not in joined
|
||||
assert "ВЫПИСКА ПО СЧЕТУ" not in joined
|
||||
assert "СТРАНИЦА" not in joined
|
||||
|
||||
def test_parse_is_deterministic(self) -> None:
|
||||
assert self.adapter._draft_from_lines(
|
||||
synthetic_rocket_lines()
|
||||
) == self.adapter._draft_from_lines(synthetic_rocket_lines())
|
||||
|
||||
def test_record_before_day_section_fails_closed(self) -> None:
|
||||
lines = [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "12:00 00000000001 +1 000,00 RUR"),
|
||||
]
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_lines(lines)
|
||||
|
||||
def test_missing_header_block_fails_closed(self) -> None:
|
||||
lines = [
|
||||
(1, "ВЫПИСКА ПО СЧЕТУ"),
|
||||
(1, "ДЕБЕТОВАЯ КАРТА РОКЕТ 00000000000000000001 RUR"),
|
||||
(1, "01 ИЮЛЯ 2026 БАЛАНС НА НАЧАЛО ДНЯ: 0,00 RUR"),
|
||||
(1, "12:00 00000000001 +1 000,00 RUR"),
|
||||
]
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_lines(lines)
|
||||
|
||||
def test_unrecognized_content_fails_closed(self) -> None:
|
||||
lines = synthetic_rocket_lines()[:11] + [(1, "НЕИЗВЕСТНАЯ СТРОКА БЕЗ СТРУКТУРЫ")]
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter._draft_from_lines(lines)
|
||||
assert exc.value.code == "IMPORT_UNSUPPORTED_LAYOUT"
|
||||
|
||||
def test_fold_line_cap_fails_closed(self) -> None:
|
||||
junk = [(1, f"НЕИЗВЕСТНАЯ СТРОКА {index}") for index in range(7)]
|
||||
lines = synthetic_rocket_lines()[:11] + junk
|
||||
with pytest.raises(UnsupportedStatementLayout):
|
||||
self.adapter._draft_from_lines(lines)
|
||||
|
||||
def test_detect_and_parse_fail_closed_on_blank_pdf(self) -> None:
|
||||
raw = minimal_pdf()
|
||||
assert self.adapter.detect(raw) is False
|
||||
with pytest.raises(UnsupportedStatementLayout) as exc:
|
||||
self.adapter.parse(raw)
|
||||
assert "text layer" in exc.value.message
|
||||
|
||||
def test_oversized_pdf_is_rejected_by_quota(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import app.imports.readers as readers_module
|
||||
|
||||
monkeypatch.setattr(readers_module, "get_settings", lambda: _QuotaSettings())
|
||||
with pytest.raises(AppError) as exc:
|
||||
self.adapter.parse(minimal_pdf(page_count=2))
|
||||
assert exc.value.code == "IMPORT_QUOTA_EXCEEDED"
|
||||
|
||||
|
||||
class _QuotaSettings:
|
||||
max_pdf_pages = 1
|
||||
|
||||
|
||||
def minimal_pdf(page_count: int = 1, page_text: str = "") -> bytes:
|
||||
"""Build a small valid single-font PDF with empty or ASCII-only pages."""
|
||||
objects: list[tuple[int, bytes]] = []
|
||||
page_ids: list[int] = []
|
||||
next_id = 3
|
||||
for _ in range(page_count):
|
||||
page_id = next_id
|
||||
content_id = next_id + 1
|
||||
next_id += 2
|
||||
page_ids.append(page_id)
|
||||
content = (
|
||||
f"BT /F1 12 Tf 72 720 Td ({page_text}) Tj ET".encode("latin-1", "replace")
|
||||
if page_text
|
||||
else b""
|
||||
)
|
||||
objects.append(
|
||||
(
|
||||
content_id,
|
||||
b"<< /Length %d >>\nstream\n%s\nendstream" % (len(content), content),
|
||||
)
|
||||
)
|
||||
objects.append(
|
||||
(
|
||||
page_id,
|
||||
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents %d 0 R >>"
|
||||
% content_id,
|
||||
)
|
||||
)
|
||||
font_id = next_id
|
||||
objects.append((font_id, b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"))
|
||||
objects.insert(
|
||||
0,
|
||||
(
|
||||
2,
|
||||
b"<< /Type /Pages /Kids ["
|
||||
+ b" ".join(b"%d 0 R" % page_id for page_id in page_ids)
|
||||
+ b"] /Count %d >>" % page_count,
|
||||
),
|
||||
)
|
||||
objects.insert(0, (1, b"<< /Type /Catalog /Pages 2 0 R >>"))
|
||||
out = bytearray(b"%PDF-1.4\n")
|
||||
offsets: dict[int, int] = {}
|
||||
for number, body in objects:
|
||||
offsets[number] = len(out)
|
||||
out += b"%d 0 obj\n" % number + body + b"\nendobj\n"
|
||||
xref_position = len(out)
|
||||
size = max(offsets) + 1
|
||||
out += b"xref\n0 %d\n" % size
|
||||
out += b"0000000000 65535 f \n"
|
||||
for number in range(1, size):
|
||||
out += b"%010d 00000 n \n" % offsets[number]
|
||||
out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
|
||||
size,
|
||||
xref_position,
|
||||
)
|
||||
return bytes(out)
|
||||
@@ -0,0 +1,945 @@
|
||||
import uuid
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.budgeting.service import close_period, compute_period_state, get_or_create_period
|
||||
from app.errors import AppError
|
||||
from app.identity.models import User
|
||||
from app.imports import service
|
||||
from app.imports.adapters.base import record_fingerprint
|
||||
from app.imports.models import (
|
||||
HoldResolution,
|
||||
ParseRun,
|
||||
SourceRecord,
|
||||
SourceRecordLink,
|
||||
StatementDocument,
|
||||
)
|
||||
from app.ledger.models import Operation
|
||||
from app.ledger.service import create_account
|
||||
from app.spaces.models import Membership
|
||||
from app.spaces.service import create_space
|
||||
|
||||
MAY_10 = date(2026, 5, 10)
|
||||
|
||||
|
||||
def seed(db: Session, *, name: str = "Holds"):
|
||||
user = User(email=f"{uuid.uuid4().hex}@example.com", password_hash="unused")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
space = create_space(db, user=user, name=name)
|
||||
account = create_account(
|
||||
db,
|
||||
space_id=space.id,
|
||||
user=user,
|
||||
name="Debit",
|
||||
kind="debit",
|
||||
opened_on=date(2026, 1, 1),
|
||||
)
|
||||
return user, space, account
|
||||
|
||||
|
||||
def make_document(
|
||||
db: Session,
|
||||
space,
|
||||
account,
|
||||
*,
|
||||
user: User | None = None,
|
||||
status: str = "ready",
|
||||
adapter_id: str = "alfa_debit_xlsx",
|
||||
) -> tuple[StatementDocument, ParseRun]:
|
||||
document = StatementDocument(
|
||||
space_id=space.id,
|
||||
account_id=account.id,
|
||||
original_filename=f"{uuid.uuid4().hex}.xlsx",
|
||||
size_bytes=100,
|
||||
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
|
||||
object_key=uuid.uuid4().hex,
|
||||
status=status,
|
||||
reconciliation={"full": True, "matched": True},
|
||||
uploaded_by_user_id=user.id if user else None,
|
||||
)
|
||||
db.add(document)
|
||||
db.flush()
|
||||
run = ParseRun(
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
adapter_id=adapter_id,
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
document.active_parse_run_id = run.id
|
||||
document.adapter_id = adapter_id
|
||||
return document, run
|
||||
|
||||
|
||||
def add_record(
|
||||
db: Session,
|
||||
document: StatementDocument,
|
||||
run: ParseRun,
|
||||
*,
|
||||
direction: str = "out",
|
||||
amount: int = 12_345,
|
||||
occurred_on: date = MAY_10,
|
||||
bank_status: str = "posted",
|
||||
external_id: str | None = None,
|
||||
description: str | None = None,
|
||||
) -> SourceRecord:
|
||||
record = SourceRecord(
|
||||
space_id=document.space_id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
external_id=external_id,
|
||||
occurred_on=occurred_on,
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
bank_status=bank_status,
|
||||
description=description,
|
||||
row_ref=f"sheet0:{uuid.uuid4().hex[:8]}",
|
||||
record_fp=record_fingerprint(occurred_on, amount, description),
|
||||
)
|
||||
db.add(record)
|
||||
db.flush()
|
||||
return record
|
||||
|
||||
|
||||
def hold_document(db: Session, *, occurred_on: date = MAY_10, external_id: str | None = None):
|
||||
"""A confirmed import whose active run contains exactly one unresolved hold."""
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
document,
|
||||
run,
|
||||
bank_status="hold",
|
||||
occurred_on=occurred_on,
|
||||
external_id=external_id,
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
db.flush()
|
||||
return user, space, account, document, hold
|
||||
|
||||
|
||||
def test_bank_status_defaults_to_posted_and_is_constrained(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
record = SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
amount_kopecks=1_000,
|
||||
direction="out",
|
||||
row_ref="sheet0:row1",
|
||||
record_fp="fp",
|
||||
)
|
||||
db.add(record)
|
||||
db.flush()
|
||||
assert record.bank_status == "posted"
|
||||
bad = SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
amount_kopecks=1_000,
|
||||
direction="out",
|
||||
bank_status="pending",
|
||||
row_ref="sheet0:row2",
|
||||
record_fp="fp2",
|
||||
)
|
||||
db.add(bad)
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_hold_row_creates_no_operation_on_confirm(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
add_record(
|
||||
db,
|
||||
document,
|
||||
run,
|
||||
bank_status="posted",
|
||||
occurred_on=date(2026, 5, 11),
|
||||
amount=7_000,
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
db.flush()
|
||||
operations = db.query(Operation).filter_by(space_id=space.id).all()
|
||||
assert len(operations) == 1
|
||||
assert operations[0].amount_kopecks == 7_000
|
||||
assert db.query(SourceRecordLink).filter_by(source_record_id=hold.id).count() == 0
|
||||
assert document.status == "confirmed"
|
||||
detail = service.document_detail(db, space_id=space.id, document_id=document.id)
|
||||
hold_row = next(item for item in detail["records"] if item["id"] == hold.id)
|
||||
assert hold_row["bank_status"] == "hold"
|
||||
assert hold_row["hold_resolution"] is None
|
||||
assert hold_row["critical"] is False
|
||||
|
||||
|
||||
def test_unresolved_hold_blocks_only_its_occurred_month(db: Session) -> None:
|
||||
user, space, _account, _document, _hold = hold_document(db)
|
||||
april = get_or_create_period(db, space_id=space.id, year=2026, month=4)
|
||||
close_period(db, space_id=space.id, user=user, period=april)
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
with pytest.raises(AppError) as exc:
|
||||
close_period(db, space_id=space.id, user=user, period=may)
|
||||
assert exc.value.code == "BUDGET_CLOSE_BLOCKED"
|
||||
june = get_or_create_period(db, space_id=space.id, year=2026, month=6)
|
||||
june_state = compute_period_state(db, space_id=space.id, period=june)
|
||||
assert not any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in june_state["close_blockers"]
|
||||
)
|
||||
|
||||
|
||||
def test_discard_as_cancelled_requires_reason_and_unblocks_month(db: Session) -> None:
|
||||
user, space, _account, document, hold = hold_document(db)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason=" ",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED"
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="Cancelled by the bank",
|
||||
)
|
||||
db.flush()
|
||||
assert resolution.resolution == "discarded_as_cancelled"
|
||||
assert resolution.operation_id is None
|
||||
assert resolution.resolved_by_user_id == user.id
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
detail = service.document_detail(db, space_id=space.id, document_id=document.id)
|
||||
hold_row = next(item for item in detail["records"] if item["id"] == hold.id)
|
||||
assert hold_row["hold_resolution"]["reason"] == "Cancelled by the bank"
|
||||
april = get_or_create_period(db, space_id=space.id, year=2026, month=4)
|
||||
close_period(db, space_id=space.id, user=user, period=april)
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
close_period(db, space_id=space.id, user=user, period=may)
|
||||
assert may.status == "closed"
|
||||
|
||||
|
||||
def test_confirm_as_posted_requires_reason(db: Session) -> None:
|
||||
user, space, _account, document, hold = hold_document(db)
|
||||
for reason in (None, ""):
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason=reason,
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_REASON_REQUIRED"
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
|
||||
def test_hold_cannot_be_resolved_before_document_confirmation(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Not confirmed yet",
|
||||
)
|
||||
|
||||
assert exc.value.code == "IMPORT_HOLD_DOCUMENT_NOT_CONFIRMED"
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
|
||||
def test_manual_confirm_rejects_bad_action_and_non_hold_row(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
posted = add_record(
|
||||
db, document, run, bank_status="posted", occurred_on=date(2026, 5, 11), amount=900
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="delete_forever",
|
||||
reason="why not",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_ACTION_INVALID"
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=posted.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="posted row",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_INVALID"
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=uuid.uuid4(),
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="missing row",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_NOT_FOUND"
|
||||
|
||||
|
||||
def test_manual_confirm_creates_one_operation_and_late_import_dedupes(db: Session) -> None:
|
||||
user, space, account, document, hold = hold_document(db, external_id="H-1")
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Verified against the paper receipt",
|
||||
account_id=account.id,
|
||||
)
|
||||
db.flush()
|
||||
operations = db.query(Operation).filter_by(space_id=space.id).all()
|
||||
assert len(operations) == 1
|
||||
operation = operations[0]
|
||||
assert operation.type == "expense"
|
||||
assert operation.amount_kopecks == 12_345
|
||||
assert operation.occurred_on == MAY_10
|
||||
assert resolution.operation_id == operation.id
|
||||
link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one_or_none()
|
||||
assert link is not None and link.operation_id == operation.id
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="second try",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_HOLD_ALREADY_RESOLVED"
|
||||
|
||||
later, later_run = make_document(db, space, account, user=user)
|
||||
late_posted = add_record(
|
||||
db,
|
||||
later,
|
||||
later_run,
|
||||
bank_status="posted",
|
||||
external_id="H-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=later.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 1
|
||||
late_link = db.query(SourceRecordLink).filter_by(source_record_id=late_posted.id).one_or_none()
|
||||
assert late_link is not None
|
||||
assert late_link.operation_id == operation.id
|
||||
assert late_link.kind == "duplicate"
|
||||
|
||||
|
||||
def test_manual_confirm_of_income_hold_creates_income(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
document,
|
||||
run,
|
||||
direction="in",
|
||||
amount=45_000,
|
||||
bank_status="hold",
|
||||
description="Refund hold",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Money arrived",
|
||||
)
|
||||
db.flush()
|
||||
operation = db.query(Operation).filter_by(space_id=space.id).one()
|
||||
assert operation.type == "income"
|
||||
assert operation.amount_kopecks == 45_000
|
||||
|
||||
|
||||
def test_manual_confirm_honors_closed_period(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
february = get_or_create_period(db, space_id=space.id, year=2026, month=2)
|
||||
close_period(db, space_id=space.id, user=user, period=february)
|
||||
march = get_or_create_period(db, space_id=space.id, year=2026, month=3)
|
||||
close_period(db, space_id=space.id, user=user, period=march)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold", occurred_on=date(2026, 3, 15))
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Book it",
|
||||
)
|
||||
assert exc.value.code == "BUDGET_PERIOD_CLOSED"
|
||||
db.flush()
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="discard_as_cancelled",
|
||||
reason="Never posted",
|
||||
)
|
||||
assert resolution.operation_id is None
|
||||
|
||||
|
||||
def test_unique_posted_record_resolves_hold_on_confirmation(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="ALFA-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="ALFA-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
resolution = db.query(HoldResolution).one()
|
||||
assert resolution.hold_source_record_id == hold.id
|
||||
assert resolution.resolving_source_record_id == posted.id
|
||||
assert resolution.resolution == "matched_posted"
|
||||
assert resolution.resolved_by_user_id is None
|
||||
operations = db.query(Operation).filter_by(space_id=space.id).all()
|
||||
assert len(operations) == 1
|
||||
assert operations[0].amount_kopecks == 12_345
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert not any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"]
|
||||
)
|
||||
|
||||
|
||||
def test_external_id_match_survives_description_drift(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="X-9",
|
||||
description="HOLD COFFEE",
|
||||
occurred_on=MAY_10,
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="X-9",
|
||||
description="Coffee llc prague",
|
||||
occurred_on=date(2026, 5, 12),
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
resolution = db.query(HoldResolution).one()
|
||||
assert resolution.hold_source_record_id == hold.id
|
||||
assert resolution.resolving_source_record_id == posted.id
|
||||
|
||||
|
||||
def test_fingerprint_match_without_external_id_resolves(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
description="Coffee shop",
|
||||
)
|
||||
matches = service._match_hold_resolutions(db, document=second, run=second_run)
|
||||
assert matches == {hold.id: posted.id}
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 1
|
||||
|
||||
|
||||
def test_ambiguous_overlap_remains_unresolved(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold_a = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="A-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
hold_b = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="B-2",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
assert hold_a.id != hold_b.id
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
|
||||
|
||||
def test_matcher_rejects_two_posted_candidates_for_one_hold(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
add_record(db, first, first_run, bank_status="hold", description="Coffee shop")
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
add_record(db, second, second_run, bank_status="posted", description="Coffee shop")
|
||||
add_record(db, second, second_run, bank_status="posted", description="Coffee shop")
|
||||
matches = service._match_hold_resolutions(db, document=second, run=second_run)
|
||||
assert matches == {}
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
|
||||
|
||||
def test_discarded_posted_record_does_not_resolve_hold(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="DISCARDED-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
posted = add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="DISCARDED-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
posted.review_decision = "discard"
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
|
||||
assert db.query(HoldResolution).filter_by(hold_source_record_id=hold.id).count() == 0
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
|
||||
|
||||
def test_hold_matching_is_tenant_scoped(db: Session) -> None:
|
||||
user1, space1, account1 = seed(db, name="Space One")
|
||||
user2, space2, account2 = seed(db, name="Space Two")
|
||||
first, first_run = make_document(db, space1, account1, user=user1)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="T-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space1.id, document_id=first.id, user=user1)
|
||||
second, second_run = make_document(db, space2, account2, user=user2)
|
||||
add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="T-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space2.id, document_id=second.id, user=user2)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
may_two = get_or_create_period(db, space_id=space2.id, year=2026, month=5)
|
||||
state_two = compute_period_state(db, space_id=space2.id, period=may_two)
|
||||
assert not any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_two["close_blockers"]
|
||||
)
|
||||
may_one = get_or_create_period(db, space_id=space1.id, year=2026, month=5)
|
||||
state_one = compute_period_state(db, space_id=space1.id, period=may_one)
|
||||
assert any(
|
||||
blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state_one["close_blockers"]
|
||||
)
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.resolve_hold(
|
||||
db,
|
||||
space_id=space2.id,
|
||||
document_id=first.id,
|
||||
source_record_id=hold.id,
|
||||
user=user2,
|
||||
action="discard_as_cancelled",
|
||||
reason="cross tenant",
|
||||
)
|
||||
assert exc.value.code == "IMPORT_NOT_FOUND"
|
||||
|
||||
|
||||
def test_revert_clears_automatic_match_and_reblocks_month(db: Session) -> None:
|
||||
user, space, account = seed(db)
|
||||
first, first_run = make_document(db, space, account, user=user)
|
||||
hold = add_record(
|
||||
db,
|
||||
first,
|
||||
first_run,
|
||||
bank_status="hold",
|
||||
external_id="R-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first.id, user=user)
|
||||
second, second_run = make_document(db, space, account, user=user)
|
||||
add_record(
|
||||
db,
|
||||
second,
|
||||
second_run,
|
||||
bank_status="posted",
|
||||
external_id="R-1",
|
||||
description="Coffee shop",
|
||||
)
|
||||
service.confirm_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 1
|
||||
service.revert_document(db, space_id=space.id, document_id=second.id, user=user)
|
||||
db.flush()
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 0
|
||||
may = get_or_create_period(db, space_id=space.id, year=2026, month=5)
|
||||
state = compute_period_state(db, space_id=space.id, period=may)
|
||||
assert any(blocker["code"] == "IMPORT_UNRESOLVED_HOLDS" for blocker in state["close_blockers"])
|
||||
assert hold.bank_status == "hold"
|
||||
|
||||
|
||||
def test_revert_preserves_manual_hold_operation_and_provenance(db: Session) -> None:
|
||||
user, space, account, document, hold = hold_document(db)
|
||||
resolution = service.resolve_hold(
|
||||
db,
|
||||
space_id=space.id,
|
||||
document_id=document.id,
|
||||
source_record_id=hold.id,
|
||||
user=user,
|
||||
action="confirm_as_posted",
|
||||
reason="Verified in the bank app",
|
||||
account_id=account.id,
|
||||
)
|
||||
db.flush()
|
||||
operation_id = resolution.operation_id
|
||||
|
||||
service.revert_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
db.flush()
|
||||
|
||||
assert document.status == "ready"
|
||||
assert operation_id is not None
|
||||
assert db.get(Operation, operation_id) is not None
|
||||
assert db.get(HoldResolution, resolution.id) is not None
|
||||
link = db.query(SourceRecordLink).filter_by(source_record_id=hold.id).one()
|
||||
assert link.kind == "hold_confirmed"
|
||||
assert link.operation_id == operation_id
|
||||
|
||||
|
||||
def test_hold_resolution_rejects_unknown_kind(db: Session) -> None:
|
||||
user, space, _account, _document, hold = hold_document(db)
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space.id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="resolved_somehow",
|
||||
)
|
||||
)
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_manual_resolution_requires_reason_at_database_level(db: Session) -> None:
|
||||
_user, space, _account, _document, hold = hold_document(db)
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space.id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="confirmed_as_posted",
|
||||
reason=None,
|
||||
)
|
||||
)
|
||||
with pytest.raises(IntegrityError):
|
||||
db.flush()
|
||||
db.rollback()
|
||||
_user, space, _account, _document, hold = hold_document(db)
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space.id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="matched_posted",
|
||||
reason=None,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _draft(rows, *, opening=1_000, closing=1_200, **declared):
|
||||
return SimpleNamespace(
|
||||
currency="RUB",
|
||||
opening_balance_kopecks=opening,
|
||||
closing_balance_kopecks=closing,
|
||||
declared_inflow_kopecks=declared.get("inflow"),
|
||||
declared_outflow_kopecks=declared.get("outflow"),
|
||||
declared_hold_kopecks=declared.get("hold"),
|
||||
rows=tuple(rows),
|
||||
)
|
||||
|
||||
|
||||
def _row(amount, direction, bank_status="posted"):
|
||||
return SimpleNamespace(
|
||||
amount_kopecks=amount,
|
||||
direction=direction,
|
||||
bank_status=bank_status,
|
||||
)
|
||||
|
||||
|
||||
def test_reconciliation_separates_posted_and_hold_totals() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft(
|
||||
[
|
||||
_row(50_000, "in"),
|
||||
_row(30_000, "out"),
|
||||
_row(20_000, "out", bank_status="hold"),
|
||||
],
|
||||
opening=100_000,
|
||||
closing=120_000,
|
||||
inflow=50_000,
|
||||
outflow=30_000,
|
||||
hold=20_000,
|
||||
)
|
||||
)
|
||||
assert summary["parsed_posted_inflow_kopecks"] == 50_000
|
||||
assert summary["parsed_posted_outflow_kopecks"] == 30_000
|
||||
assert summary["parsed_hold_total_kopecks"] == 20_000
|
||||
assert summary["posted_totals_matched"] is True
|
||||
assert summary["hold_totals_matched"] is True
|
||||
assert summary["balance_matched"] is True
|
||||
assert summary["matched"] is True
|
||||
assert summary["computed_closing_kopecks"] == 120_000
|
||||
|
||||
|
||||
def test_reconciliation_declared_outflow_mismatch_blocks() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft(
|
||||
[_row(30_000, "out")],
|
||||
opening=100_000,
|
||||
closing=70_000,
|
||||
inflow=None,
|
||||
outflow=40_000,
|
||||
)
|
||||
)
|
||||
assert summary["posted_totals_matched"] is False
|
||||
assert summary["hold_totals_matched"] is None
|
||||
assert summary["matched"] is False
|
||||
|
||||
|
||||
def test_reconciliation_hold_mismatch_is_review_blocking() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft([_row(25_000, "out", bank_status="hold")], opening=None, closing=None, hold=10_000)
|
||||
)
|
||||
assert summary["full"] is False
|
||||
assert summary["hold_totals_matched"] is False
|
||||
assert summary["matched"] is False
|
||||
assert summary["balance_matched"] is None
|
||||
|
||||
|
||||
def test_reconciliation_balance_equation_ignores_holds() -> None:
|
||||
summary = service._reconciliation_summary(
|
||||
_draft(
|
||||
[
|
||||
_row(50_000, "in"),
|
||||
_row(30_000, "out"),
|
||||
_row(20_000, "out", bank_status="hold"),
|
||||
],
|
||||
opening=100_000,
|
||||
closing=140_000,
|
||||
)
|
||||
)
|
||||
assert summary["balance_matched"] is False
|
||||
assert summary["matched"] is False
|
||||
assert summary["computed_closing_kopecks"] == 120_000
|
||||
|
||||
|
||||
def test_reconciliation_without_declared_totals_keeps_legacy_semantics() -> None:
|
||||
draft = SimpleNamespace(
|
||||
currency="RUB",
|
||||
opening_balance_kopecks=100,
|
||||
closing_balance_kopecks=300,
|
||||
rows=(_row(200, "in"),),
|
||||
)
|
||||
summary = service._reconciliation_summary(draft)
|
||||
assert summary["full"] is True
|
||||
assert summary["posted_totals_matched"] is None
|
||||
assert summary["hold_totals_matched"] is None
|
||||
assert summary["balance_matched"] is True
|
||||
assert summary["matched"] is True
|
||||
|
||||
|
||||
def test_source_row_without_bank_status_defaults_posted() -> None:
|
||||
from app.imports.adapters.base import SourceRow
|
||||
|
||||
source = service._source_from_row(
|
||||
space_id=uuid.uuid4(),
|
||||
document_id=uuid.uuid4(),
|
||||
run_id=uuid.uuid4(),
|
||||
row=SourceRow(
|
||||
external_id=None,
|
||||
occurred_on=date(2026, 5, 10),
|
||||
posted_on=None,
|
||||
amount_kopecks=1_000,
|
||||
direction="out",
|
||||
description=None,
|
||||
mcc=None,
|
||||
bank_category=None,
|
||||
location=None,
|
||||
row_ref="row1",
|
||||
),
|
||||
)
|
||||
assert source.bank_status == "posted"
|
||||
assert source.critical is False
|
||||
|
||||
|
||||
def test_resolve_endpoint_requires_csrf_and_resolves_hold(client: TestClient, db: Session) -> None:
|
||||
invite = client.post(
|
||||
"/api/v1/admin/invitations",
|
||||
headers={"X-Admin-Token": "test-admin-token"},
|
||||
json={"email": "holds-user@example.com"},
|
||||
)
|
||||
assert invite.status_code == 201, invite.text
|
||||
accepted = client.post(
|
||||
"/api/v1/auth/invitations/accept",
|
||||
json={
|
||||
"token": invite.json()["token"],
|
||||
"password": "a-long-password-for-tests",
|
||||
"fullName": "Holds User",
|
||||
"locale": "en",
|
||||
},
|
||||
)
|
||||
assert accepted.status_code == 201, accepted.text
|
||||
csrf = client.cookies["csrf_token"]
|
||||
db.commit()
|
||||
|
||||
user = db.query(User).filter_by(email="holds-user@example.com").one()
|
||||
space_id = db.query(Membership).filter_by(user_id=user.id).one().space_id
|
||||
account = create_account(
|
||||
db,
|
||||
space_id=space_id,
|
||||
user=user,
|
||||
name="Debit",
|
||||
kind="debit",
|
||||
opened_on=date(2026, 1, 1),
|
||||
)
|
||||
space = SimpleNamespace(id=space_id)
|
||||
document, run = make_document(db, space, account, user=user)
|
||||
hold = add_record(db, document, run, bank_status="hold")
|
||||
service.confirm_document(db, space_id=space_id, document_id=document.id, user=user)
|
||||
db.commit()
|
||||
|
||||
rejected = client.post(
|
||||
f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve",
|
||||
json={"action": "confirm_as_posted", "reason": "no csrf"},
|
||||
)
|
||||
assert rejected.status_code == 403
|
||||
|
||||
resolved = client.post(
|
||||
f"/api/v1/imports/{document.id}/rows/{hold.id}/resolve",
|
||||
headers={"X-CSRF-Token": csrf},
|
||||
json={"action": "confirm_as_posted", "reason": "Checked in the bank app"},
|
||||
)
|
||||
assert resolved.status_code == 200, resolved.text
|
||||
rows = resolved.json()["rows"]
|
||||
row = next(item for item in rows if item["id"] == str(hold.id))
|
||||
assert row["bankStatus"] == "hold"
|
||||
assert row["holdResolution"]["resolution"] == "confirmed_as_posted"
|
||||
assert row["holdResolution"]["reason"] == "Checked in the bank app"
|
||||
assert row["holdResolution"]["operationId"] is not None
|
||||
operations = db.query(Operation).filter_by(space_id=space_id).all()
|
||||
assert len(operations) == 1
|
||||
@@ -9,6 +9,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import set_space_context
|
||||
from app.imports.blob import get_blob_store
|
||||
from app.imports.models import HoldResolution, ParseRun, SourceRecord, StatementDocument
|
||||
from app.ledger.models import Account
|
||||
from app.spaces.models import FinancialSpace
|
||||
|
||||
@@ -28,11 +29,55 @@ def test_postgres_rls_requires_matching_transaction_context() -> None:
|
||||
db.rollback()
|
||||
|
||||
set_space_context(db, space_id)
|
||||
db.add(Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today()))
|
||||
account = Account(space_id=space_id, name="Allowed", kind="cash", opened_on=date.today())
|
||||
db.add(account)
|
||||
db.flush()
|
||||
document = StatementDocument(
|
||||
space_id=space_id,
|
||||
account_id=account.id,
|
||||
original_filename="synthetic.xlsx",
|
||||
size_bytes=1,
|
||||
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
|
||||
object_key=uuid.uuid4().hex,
|
||||
status="confirmed",
|
||||
)
|
||||
db.add(document)
|
||||
db.flush()
|
||||
run = ParseRun(
|
||||
space_id=space_id,
|
||||
document_id=document.id,
|
||||
adapter_id="alfa_debit_xlsx",
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
document.active_parse_run_id = run.id
|
||||
hold = SourceRecord(
|
||||
space_id=space_id,
|
||||
document_id=document.id,
|
||||
parse_run_id=run.id,
|
||||
occurred_on=date.today(),
|
||||
amount_kopecks=100,
|
||||
direction="out",
|
||||
bank_status="hold",
|
||||
row_ref="sheet0:row1",
|
||||
record_fp=uuid.uuid4().hex,
|
||||
)
|
||||
db.add(hold)
|
||||
db.flush()
|
||||
db.add(
|
||||
HoldResolution(
|
||||
space_id=space_id,
|
||||
hold_source_record_id=hold.id,
|
||||
resolution="discarded_as_cancelled",
|
||||
reason="synthetic test resolution",
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
set_space_context(db, uuid.uuid4())
|
||||
assert db.query(Account).count() == 0
|
||||
assert db.query(HoldResolution).count() == 0
|
||||
db.rollback()
|
||||
set_space_context(db, space_id)
|
||||
persisted_space = db.get(FinancialSpace, space_id)
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Private real-statement suite (marker: private_fixtures).
|
||||
|
||||
Runs only when invoked explicitly:
|
||||
|
||||
PRIVATE_FIXTURES_DIR=/secure/path uv run pytest -m private_fixtures
|
||||
|
||||
When PRIVATE_FIXTURES_DIR is unset, the ignored repository-local ``examples/``
|
||||
directory is used if it contains a ``manifest.json``. Normal CI never collects
|
||||
this module (the default pytest addopts deselect the marker).
|
||||
|
||||
Privacy contract (docs/parsers/private-fixture-gate.md):
|
||||
|
||||
- fixture contents, parsed rows, descriptions, identifiers, and manifest
|
||||
financial values never appear in assertion output, messages, or logs;
|
||||
- failures report only the filename, adapter id, and the name of the
|
||||
mismatched aggregate;
|
||||
- every file is parsed twice and the results must be identical.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.imports.adapters import detect_adapter, magic_kind
|
||||
from app.imports.service import _reconciliation_summary
|
||||
|
||||
pytestmark = pytest.mark.private_fixtures
|
||||
|
||||
REPO_EXAMPLES = Path(__file__).resolve().parents[2] / "examples"
|
||||
|
||||
|
||||
def _load_entries() -> tuple[Path | None, list[dict], str | None]:
|
||||
"""Resolve fixture entries.
|
||||
|
||||
Returns (directory, entries, explicit_error). An explicit error is only
|
||||
produced when PRIVATE_FIXTURES_DIR was set on purpose: then a missing
|
||||
directory, missing/malformed manifest, or empty manifest must fail loudly
|
||||
(redacted message, no values). Without the variable, an absent or broken
|
||||
repo-local examples directory is a silent skip.
|
||||
"""
|
||||
env = os.environ.get("PRIVATE_FIXTURES_DIR")
|
||||
if env:
|
||||
directory = Path(env)
|
||||
if not directory.is_dir():
|
||||
return None, [], "PRIVATE_FIXTURES_DIR is set but the directory does not exist"
|
||||
try:
|
||||
manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return directory, [], "private manifest.json is missing or unreadable"
|
||||
entries = manifest.get("fixtures") if isinstance(manifest, dict) else None
|
||||
if not isinstance(entries, list) or not entries:
|
||||
return directory, [], "private manifest declares no fixtures"
|
||||
return directory, [entry for entry in entries if isinstance(entry, dict)], None
|
||||
try:
|
||||
if not REPO_EXAMPLES.is_dir():
|
||||
return None, [], None
|
||||
manifest = json.loads((REPO_EXAMPLES / "manifest.json").read_text(encoding="utf-8"))
|
||||
entries = manifest.get("fixtures") if isinstance(manifest, dict) else None
|
||||
if not isinstance(entries, list):
|
||||
return None, [], None
|
||||
return REPO_EXAMPLES, [entry for entry in entries if isinstance(entry, dict)], None
|
||||
except Exception:
|
||||
return None, [], None
|
||||
|
||||
|
||||
FIXTURE_DIR, ENTRIES, EXPLICIT_ERROR = _load_entries()
|
||||
if EXPLICIT_ERROR:
|
||||
raise pytest.fail.Exception(EXPLICIT_ERROR)
|
||||
PARAMS = ENTRIES if ENTRIES else [None]
|
||||
|
||||
|
||||
def _check(ok: bool, name: str, what: str) -> None:
|
||||
assert ok, f"{name}: {what} mismatch"
|
||||
|
||||
|
||||
def _fail(name: str, message: str) -> None:
|
||||
pytest.fail(f"{name}: {message}", pytrace=False)
|
||||
|
||||
|
||||
@pytest.fixture(params=PARAMS, ids=[entry.get("filename", "unavailable") for entry in PARAMS])
|
||||
def parsed(request: pytest.FixtureRequest):
|
||||
entry = request.param
|
||||
if entry is None:
|
||||
pytest.skip("private fixtures are not available")
|
||||
name = str(entry.get("filename", "unnamed"))
|
||||
path = (FIXTURE_DIR / name) if FIXTURE_DIR is not None else None
|
||||
if path is None or not path.is_file():
|
||||
_fail(name, "fixture file is missing")
|
||||
raw = path.read_bytes()
|
||||
kind = magic_kind(raw)
|
||||
if kind not in {"xlsx", "pdf"}:
|
||||
_fail(name, "unexpected file type")
|
||||
try:
|
||||
adapter = detect_adapter(raw, kind)
|
||||
except Exception:
|
||||
_fail(name, "adapter detection failed")
|
||||
if adapter.adapter_id != entry.get("adapter_id"):
|
||||
_fail(name, "adapter detection mismatch")
|
||||
try:
|
||||
first = adapter.parse(raw)
|
||||
second = adapter.parse(raw)
|
||||
except Exception:
|
||||
_fail(name, "parse failed")
|
||||
if first != second:
|
||||
_fail(name, "parse result is not deterministic")
|
||||
return name, entry, first
|
||||
|
||||
|
||||
def test_manifest_contract(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
_check(draft.parser_version == entry.get("parser_version"), name, "parser version")
|
||||
for key, actual in (("period_start", draft.period_start), ("period_end", draft.period_end)):
|
||||
expected = entry.get(key)
|
||||
_check(
|
||||
expected is None or (actual is not None and actual.isoformat() == expected), name, key
|
||||
)
|
||||
|
||||
|
||||
def test_row_counts(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
rows = draft.rows
|
||||
_check(len(rows) == entry.get("row_count"), name, "row count")
|
||||
posted = sum(1 for row in rows if row.bank_status == "posted")
|
||||
holds = sum(1 for row in rows if row.bank_status == "hold")
|
||||
_check(posted == entry.get("posted_count"), name, "posted count")
|
||||
_check(holds == entry.get("hold_count"), name, "hold count")
|
||||
_check(
|
||||
sum(1 for row in rows if row.direction == "in") == entry.get("in_count"),
|
||||
name,
|
||||
"inflow row count",
|
||||
)
|
||||
_check(
|
||||
sum(1 for row in rows if row.direction == "out") == entry.get("out_count"),
|
||||
name,
|
||||
"outflow row count",
|
||||
)
|
||||
|
||||
|
||||
def test_row_invariants(parsed) -> None:
|
||||
name, _, draft = parsed
|
||||
rows = draft.rows
|
||||
_check(all(row.amount_kopecks >= 0 for row in rows), name, "positive amount invariant")
|
||||
_check(
|
||||
all(row.bank_status in {"posted", "hold"} for row in rows),
|
||||
name,
|
||||
"bank status invariant",
|
||||
)
|
||||
|
||||
|
||||
def test_hold_totals(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold")
|
||||
_check(hold_total == entry.get("hold_total_kopecks"), name, "hold total")
|
||||
|
||||
|
||||
def test_control_totals(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
_check(
|
||||
draft.declared_inflow_kopecks == entry.get("declared_inflow_kopecks"),
|
||||
name,
|
||||
"declared inflow total",
|
||||
)
|
||||
_check(
|
||||
draft.declared_outflow_kopecks == entry.get("declared_outflow_kopecks"),
|
||||
name,
|
||||
"declared outflow total",
|
||||
)
|
||||
_check(
|
||||
draft.declared_hold_kopecks == entry.get("declared_hold_kopecks"),
|
||||
name,
|
||||
"declared hold total",
|
||||
)
|
||||
|
||||
|
||||
def test_balances(parsed) -> None:
|
||||
name, entry, draft = parsed
|
||||
_check(
|
||||
draft.opening_balance_kopecks == entry.get("opening_balance_kopecks"),
|
||||
name,
|
||||
"opening balance",
|
||||
)
|
||||
_check(
|
||||
draft.closing_balance_kopecks == entry.get("closing_balance_kopecks"),
|
||||
name,
|
||||
"closing balance",
|
||||
)
|
||||
|
||||
|
||||
def test_declared_totals_match_parsed_rows(parsed) -> None:
|
||||
name, _, draft = parsed
|
||||
posted_in = sum(
|
||||
row.amount_kopecks
|
||||
for row in draft.rows
|
||||
if row.bank_status == "posted" and row.direction == "in"
|
||||
)
|
||||
posted_out = sum(
|
||||
row.amount_kopecks
|
||||
for row in draft.rows
|
||||
if row.bank_status == "posted" and row.direction == "out"
|
||||
)
|
||||
hold_total = sum(row.amount_kopecks for row in draft.rows if row.bank_status == "hold")
|
||||
# Declared controls are canonicalized to posted-only by the adapters; the
|
||||
# hold control equals the parsed hold sum.
|
||||
_check(posted_in == draft.declared_inflow_kopecks, name, "declared vs parsed inflow")
|
||||
_check(posted_out == draft.declared_outflow_kopecks, name, "declared vs parsed outflow")
|
||||
_check(hold_total == (draft.declared_hold_kopecks or 0), name, "declared vs parsed hold")
|
||||
|
||||
|
||||
def test_service_reconciliation_matches(parsed) -> None:
|
||||
name, _, draft = parsed
|
||||
summary = _reconciliation_summary(draft)
|
||||
_check(summary.get("matched") is True, name, "service reconciliation matched")
|
||||
_check(
|
||||
summary.get("posted_totals_matched") is True,
|
||||
name,
|
||||
"service reconciliation posted totals",
|
||||
)
|
||||
_check(
|
||||
summary.get("hold_totals_matched") is not False,
|
||||
name,
|
||||
"service reconciliation hold totals",
|
||||
)
|
||||
@@ -2,29 +2,39 @@
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-09-11
|
||||
- **Scope:** statement import pipeline, parse provenance, deduplication, apply/revert semantics
|
||||
- **Scope:** statement import pipeline, parse provenance, deduplication, HOLD lifecycle, apply/revert semantics
|
||||
|
||||
## Context
|
||||
|
||||
Five bank statement contracts must be parsed (Alfa debit/credit XLSX, Ozon debit/credit PDF, Rocketbank debit PDF). Bank layouts drift over time. Money facts must never change until the user has reviewed the parse; losing a real operation is worse than holding a duplicate for review. The HTTP process must not parse files.
|
||||
Five bank statement contracts must be parsed (Alfa debit/credit XLSX, Ozon debit/credit PDF, Rocketbank debit PDF). Bank layouts drift over time. Money facts must never change until the user has reviewed the parse; losing a real operation is worse than holding a duplicate for review. The HTTP process must not parse files. Banks additionally report pending (HOLD) transactions that are not yet booked; treating them as settled would misstate balances.
|
||||
|
||||
## Decision
|
||||
|
||||
- An uploaded statement document is immutable: magic/MIME/size/hash checks, then storage encrypted per ADR 0004. Full extracted text is never persisted — only allowlisted statement fields plus page/row coordinates.
|
||||
- Every parse produces a versioned **parse run** (adapter + parser version) yielding immutable **source records** with document origin. A reparse creates a new run and diff; previous runs are never mutated.
|
||||
- Each source row carries an immutable **bank status** (`posted` or `hold`). HOLD rows are preserved exactly as extracted and never create ledger operations during normal import confirmation.
|
||||
- State machine: `uploaded → queued → processing → review_required/ready → confirmed | failed`. Confirm applies the whole run atomically in a single DB transaction, creating only new operations/postings; existing duplicates only gain additional source links. Revert deletes exactly what that import created, with cascade reopen of affected periods.
|
||||
- Deduplication is conservative: auto-link only a stable bank identifier or a single unambiguous match; ambiguous candidates stay pending until the user decides.
|
||||
- Reconciliation compares parsed posted inflow/outflow with the bank's declared totals, compares HOLD totals separately, and — where opening and closing balances exist — requires `opening + posted inflow − posted outflow = closing`. Any mismatch stays review-blocking with stable diagnostic fields.
|
||||
- **HOLD lifecycle:** each HOLD source record carries at most one tenant-scoped, append-only `HoldResolution` (`matched_posted`, `confirmed_as_posted`, `discarded_as_cancelled`) with optional resolving source record, operation, user, and a required reason for manual actions. The extracted HOLD row itself is never overwritten.
|
||||
- **Automatic settlement matching:** when a new statement is confirmed, its posted records are conservatively matched — same space, mapped account, and statement contract only — to unresolved HOLDs from confirmed imports. A stable external ID wins; otherwise a single unambiguous date/amount/direction/normalized-description fingerprint match is required. Any ambiguity leaves the HOLD unresolved; the resolution is written only at confirmation of the resolving statement. Reverting that statement removes the automatic match.
|
||||
- **Manual resolution:** two CSRF-protected backend actions, both requiring a reason and (for booking) a mapped account: `confirm_as_posted` creates a normal ledger operation through the ledger services (honoring closed-period rules) with a provenance source-record link; `discard_as_cancelled` creates none. Both write audit events. A later import of the same posted row dedupes onto the manually created operation instead of duplicating it.
|
||||
- **Close blocking:** unresolved HOLDs from confirmed imports block closing only the budget month containing their `occurred_on` date; manual confirmation may then expose the normal uncategorized-expense blocker until the created operation is categorized.
|
||||
- Bank layout drift produces a **new parser version** with an explicit review diff — historical data is never silently reinterpreted. An adapter is beta-eligible only after the private fixture gate (see `docs/parsers/private-fixture-gate.md`).
|
||||
- Parsing runs in the Celery worker, isolated and resource-limited (memory, time, pages, rows, cells).
|
||||
|
||||
## Consequences
|
||||
|
||||
- No row influences the ledger before explicit confirm; this invariant is enforced by tests (import confirm/revert idempotence).
|
||||
- Unresolved HOLD rows remain visible in the import detail API (with optional resolution metadata) and keep blocking their month until settled; a HOLD can only be cleared by a unique posted match or an explicit manual action.
|
||||
- Parse runs and source records grow storage; acceptable at beta scale, trimmed only with the source file deletion workflow.
|
||||
- Adapter code must explicitly detect its layout; a mismatch fails with stable diagnostics instead of guessing.
|
||||
- **Future UI (deferred by design):** the frontend must show HOLD rows with their status and resolution state and offer both manual resolution actions. The backend response fields (`bankStatus`, `holdResolution`) are optional, backward-compatible additions; no UI changes ship in this workstream.
|
||||
|
||||
## Alternatives rejected
|
||||
|
||||
- In-request synchronous parsing — blocks HTTP workers on untrusted files.
|
||||
- Mutable "latest parse" overwrites — destroys provenance and diffability.
|
||||
- Aggressive auto-dedup — risks dropping real operations.
|
||||
- Mutating the extracted HOLD row in place when it posts — loses the original bank fact; append-only resolution metadata preserves it.
|
||||
- Treating HOLDs as ledger operations on import — misstates balances before the bank books the transaction.
|
||||
|
||||
@@ -1,20 +1,25 @@
|
||||
# Parser Private Fixture Gate
|
||||
|
||||
Real bank statements contain personal financial data and are **never committed**
|
||||
to Git (`.gitignore` at the repo root covers this; keep fixture directories
|
||||
outside the repository). CI runs synthetic fixtures only.
|
||||
to Git. CI runs synthetic fixtures only. Private fixtures may live either in an
|
||||
external directory or in the ignored repository-local `examples/` directory.
|
||||
|
||||
## Contract
|
||||
|
||||
- Synthetic parser inputs live with the backend tests and run in CI.
|
||||
- Real statement files stay in a local `PRIVATE_FIXTURES_DIR`, never in Git, never in shared storage without encryption.
|
||||
- Real statement files stay in `PRIVATE_FIXTURES_DIR` or the ignored repository-local `examples/` directory, never in Git and never in shared storage without encryption.
|
||||
- Each private fixture directory contains an ignored `manifest.json` beside the statements. For each fixture it records the filename, adapter ID, parser version, period, row counts, control totals, balances, and HOLD totals.
|
||||
- The manifest and its expected values are private financial data. They must not appear in tracked test parameters, documentation, CI inputs, artifacts, or logs.
|
||||
- Private assertions report only a filename, adapter ID, dates, counts, and the name of a mismatched aggregate. They never render parsed rows, descriptions, names, account/card numbers, extracted pages, or expected financial values.
|
||||
- The private suite runs explicitly, on the developer's machine:
|
||||
|
||||
```sh
|
||||
PRIVATE_FIXTURES_DIR=/secure/path/fixtures uv run pytest -m private_fixtures
|
||||
```
|
||||
|
||||
(Exact marker/task names are finalized by the backend workstream; the entry point contract is `PRIVATE_FIXTURES_DIR` + a dedicated pytest marker.)
|
||||
When `PRIVATE_FIXTURES_DIR` is unset, the suite may use the ignored
|
||||
repository-local `examples/` directory. The dedicated marker keeps this suite
|
||||
out of normal CI collection.
|
||||
|
||||
## Beta gate per adapter/contract
|
||||
|
||||
@@ -35,11 +40,14 @@ gate must not be enabled or advertised, even if synthetic tests pass (plan
|
||||
|
||||
| Adapter / parser version | Private periods passed | Gate |
|
||||
| --- | --- | --- |
|
||||
| Alfa debit XLSX | — | **not passed** |
|
||||
| Alfa credit XLSX | — | **not passed** |
|
||||
| Ozon debit PDF | — | **not passed** |
|
||||
| Ozon credit PDF | — | **not passed** |
|
||||
| Rocketbank debit PDF | — | **not passed** |
|
||||
| Alfa debit XLSX / 2 | 1 | **not passed** |
|
||||
| Alfa credit XLSX / 1 (unregistered) | 0 | **not passed** |
|
||||
| Ozon debit PDF / 2 | 1 | **not passed** |
|
||||
| Ozon credit PDF / 1 (unregistered) | 0 | **not passed** |
|
||||
| Rocketbank debit PDF / 2 | 1 | **not passed** |
|
||||
|
||||
No adapter is currently gated. Bank layout drift resets the gate: it must create
|
||||
a new parser version and a review diff, never reinterpret historical data.
|
||||
The three debit contracts each pass one local private fixture deterministically,
|
||||
but no adapter is currently gated. A second independent real period/layout is
|
||||
still required for each contract. Bank layout drift resets the gate: it must
|
||||
create a new parser version and a review diff, never reinterpret historical
|
||||
data.
|
||||
|
||||
@@ -1,64 +1,77 @@
|
||||
{
|
||||
"0": "Identity and Configuration",
|
||||
"1": "Budget Domain Services",
|
||||
"2": "Database Models",
|
||||
"3": "Database Context and Exports",
|
||||
"4": "Statement Import Services",
|
||||
"5": "Parser Adapters",
|
||||
"6": "Ledger Services",
|
||||
"7": "API Domain Types",
|
||||
"8": "Authentication UI",
|
||||
"9": "Ledger API",
|
||||
"10": "Import Review UI",
|
||||
"11": "TypeScript Configuration",
|
||||
"12": "Analytics UI",
|
||||
"13": "API and Analytics",
|
||||
"14": "Frontend Development Dependencies",
|
||||
"15": "Accounts UI",
|
||||
"16": "Frontend Runtime Dependencies",
|
||||
"17": "Import Detail Mutations",
|
||||
"18": "Operations UI",
|
||||
"19": "Query Keys and Operations",
|
||||
"20": "Frontend Bootstrap",
|
||||
"21": "Frontend Test Support",
|
||||
"22": "Budget UI",
|
||||
"23": "HTTP API Client",
|
||||
"24": "Form Fields",
|
||||
"25": "Infrastructure and CI",
|
||||
"26": "Core Finance Domain",
|
||||
"27": "Frontend Scripts",
|
||||
"28": "Settings and Data Lifecycle",
|
||||
"29": "Budget Allocation Model",
|
||||
"30": "MVP and Import Design",
|
||||
"31": "Data Security and Recovery",
|
||||
"32": "Ledger Concepts",
|
||||
"33": "Parser Versioning",
|
||||
"34": "Frontend Architecture",
|
||||
"35": "Beta Release Governance",
|
||||
"36": "Package Metadata",
|
||||
"37": "Tenant Isolation",
|
||||
"38": "Deletion and Restore",
|
||||
"39": "API Generation Script",
|
||||
"40": "OpenAPI Schema",
|
||||
"41": "API Integration Tests",
|
||||
"42": "Localization Tests",
|
||||
"43": "Income Classification",
|
||||
"44": "Accessibility ESLint",
|
||||
"45": "JavaScript Globals",
|
||||
"46": "JSDOM Dependency",
|
||||
"47": "React Type Definitions",
|
||||
"48": "React DOM Types",
|
||||
"49": "TypeScript ESLint",
|
||||
"50": "Vite React Plugin",
|
||||
"51": "Vitest Dependency",
|
||||
"52": "Favicon Design",
|
||||
"53": "Database Role Initialization",
|
||||
"54": "Application Package",
|
||||
"55": "Spaces Package",
|
||||
"56": "Payee Concept",
|
||||
"57": "ESLint Configuration",
|
||||
"58": "Money Utilities Tests",
|
||||
"59": "Vite Environment Types",
|
||||
"60": "Vite Configuration",
|
||||
"61": "Backend Package Metadata"
|
||||
"0": "identity/router.py",
|
||||
"1": "budgeting/router.py",
|
||||
"2": "Base",
|
||||
"3": "set_space_context",
|
||||
"4": "imports/router.py",
|
||||
"5": "alfa_xlsx.py",
|
||||
"6": "Session",
|
||||
"7": "types.ts",
|
||||
"8": "router.tsx",
|
||||
"9": "ledger/router.py",
|
||||
"10": "AccountsPage.tsx",
|
||||
"11": "compilerOptions",
|
||||
"12": "AnalyticsPage.tsx",
|
||||
"13": "analytics/router.py",
|
||||
"14": "devDependencies",
|
||||
"15": "test_import_holds.py",
|
||||
"16": "dependencies",
|
||||
"17": "ImportDetailPage.tsx",
|
||||
"18": "OperationsPage.tsx",
|
||||
"19": "SettingsPage.tsx",
|
||||
"20": "Toaster.tsx",
|
||||
"21": "ozon_pdf.py",
|
||||
"22": "BudgetPage.tsx",
|
||||
"23": "client.ts",
|
||||
"24": "fields.tsx",
|
||||
"25": "Docker Compose Development Stack",
|
||||
"26": "Financial Space",
|
||||
"27": "scripts",
|
||||
"28": "dependencies.py",
|
||||
"29": "General Pool",
|
||||
"30": "TestBaseContract",
|
||||
"31": "Per-File Envelope Encryption",
|
||||
"32": "Operation",
|
||||
"33": "Versioned Statement Imports",
|
||||
"34": "Finance Frontend SPA",
|
||||
"35": "Closed Beta Launch Gates",
|
||||
"36": "package.json",
|
||||
"37": "Financial Tracker MVP",
|
||||
"38": "test_adapter_contracts.py",
|
||||
"39": "generate-api.mjs",
|
||||
"40": "schema.d.ts",
|
||||
"41": "test_api.py",
|
||||
"42": "i18n-parity.test.ts",
|
||||
"43": "Expected Income",
|
||||
"44": "test_private_fixtures.py",
|
||||
"45": "TestAlfaDebitContract",
|
||||
"46": "jsdom",
|
||||
"47": "graphify",
|
||||
"48": "@types/react-dom",
|
||||
"49": "typescript-eslint",
|
||||
"50": "SourceRow",
|
||||
"51": "vitest",
|
||||
"52": "Favicon SVG (ledger lines icon)",
|
||||
"53": "init-app-role.sh",
|
||||
"54": "app/__init__.py",
|
||||
"55": "spaces/__init__.py",
|
||||
"56": "Payee",
|
||||
"57": "eslint.config.js",
|
||||
"58": "money.test.ts",
|
||||
"59": "vite-env.d.ts",
|
||||
"60": "vite.config.ts",
|
||||
"61": "finance-backend",
|
||||
"62": "LocalDirectoryBlobStore",
|
||||
"63": "account_fingerprint",
|
||||
"64": "Gitea CI and real statement imports",
|
||||
"65": "20260918_0002_hold_lifecycle.py",
|
||||
"66": "eslint-plugin-react-hooks",
|
||||
"67": "openapi-typescript",
|
||||
"68": "@testing-library/dom",
|
||||
"69": "@testing-library/react",
|
||||
"70": "vite",
|
||||
"71": "spaces/router.py",
|
||||
"72": "identity/service.py",
|
||||
"73": "get_settings",
|
||||
"74": "ColumnChart.tsx"
|
||||
}
|
||||
+264
-198
@@ -1,293 +1,359 @@
|
||||
# Graph Report - finance (2026-09-14)
|
||||
# Graph Report - finance (2026-09-18)
|
||||
|
||||
## Corpus Check
|
||||
- 147 files · ~51,467 words
|
||||
- 149 files · ~63,829 words
|
||||
- Verdict: corpus is large enough that graph structure adds value.
|
||||
|
||||
## Summary
|
||||
- 1078 nodes · 3099 edges · 62 communities (42 shown, 14 thin omitted)
|
||||
- Extraction: 98% EXTRACTED · 2% INFERRED · 0% AMBIGUOUS · INFERRED: 50 edges (avg confidence: 0.93)
|
||||
- 1267 nodes · 3697 edges · 75 communities (54 shown, 15 thin omitted)
|
||||
- Extraction: 98% EXTRACTED · 2% INFERRED · 0% AMBIGUOUS · INFERRED: 70 edges (avg confidence: 0.93)
|
||||
- Token cost: 0 input · 0 output
|
||||
|
||||
## Graph Freshness
|
||||
- Built from commit: `c85b12b9`
|
||||
- Run `git rev-parse HEAD` and compare to check if the graph is stale.
|
||||
- Run `graphify update .` after code changes (no API cost).
|
||||
|
||||
## Community Hubs (Navigation)
|
||||
- Identity and Configuration
|
||||
- Budget Domain Services
|
||||
- Database Models
|
||||
- Database Context and Exports
|
||||
- Statement Import Services
|
||||
- Parser Adapters
|
||||
- Ledger Services
|
||||
- API Domain Types
|
||||
- Authentication UI
|
||||
- Ledger API
|
||||
- Import Review UI
|
||||
- TypeScript Configuration
|
||||
- Analytics UI
|
||||
- API and Analytics
|
||||
- Frontend Development Dependencies
|
||||
- Accounts UI
|
||||
- Frontend Runtime Dependencies
|
||||
- Import Detail Mutations
|
||||
- Operations UI
|
||||
- Query Keys and Operations
|
||||
- Frontend Bootstrap
|
||||
- Frontend Test Support
|
||||
- Budget UI
|
||||
- HTTP API Client
|
||||
- Form Fields
|
||||
- Infrastructure and CI
|
||||
- Core Finance Domain
|
||||
- Frontend Scripts
|
||||
- Settings and Data Lifecycle
|
||||
- Budget Allocation Model
|
||||
- MVP and Import Design
|
||||
- Data Security and Recovery
|
||||
- Ledger Concepts
|
||||
- Parser Versioning
|
||||
- Frontend Architecture
|
||||
- Beta Release Governance
|
||||
- Package Metadata
|
||||
- Tenant Isolation
|
||||
- Deletion and Restore
|
||||
- API Generation Script
|
||||
- OpenAPI Schema
|
||||
- API Integration Tests
|
||||
- Localization Tests
|
||||
- Income Classification
|
||||
- Accessibility ESLint
|
||||
- JavaScript Globals
|
||||
- JSDOM Dependency
|
||||
- React Type Definitions
|
||||
- React DOM Types
|
||||
- TypeScript ESLint
|
||||
- Vite React Plugin
|
||||
- Vitest Dependency
|
||||
- Favicon Design
|
||||
- Database Role Initialization
|
||||
- Payee Concept
|
||||
- Backend Package Metadata
|
||||
- identity/router.py
|
||||
- budgeting/router.py
|
||||
- Base
|
||||
- set_space_context
|
||||
- imports/router.py
|
||||
- alfa_xlsx.py
|
||||
- Session
|
||||
- types.ts
|
||||
- router.tsx
|
||||
- ledger/router.py
|
||||
- AccountsPage.tsx
|
||||
- compilerOptions
|
||||
- AnalyticsPage.tsx
|
||||
- analytics/router.py
|
||||
- devDependencies
|
||||
- test_import_holds.py
|
||||
- dependencies
|
||||
- ImportDetailPage.tsx
|
||||
- OperationsPage.tsx
|
||||
- SettingsPage.tsx
|
||||
- Toaster.tsx
|
||||
- ozon_pdf.py
|
||||
- BudgetPage.tsx
|
||||
- client.ts
|
||||
- fields.tsx
|
||||
- Docker Compose Development Stack
|
||||
- Financial Space
|
||||
- scripts
|
||||
- dependencies.py
|
||||
- General Pool
|
||||
- TestBaseContract
|
||||
- Per-File Envelope Encryption
|
||||
- Operation
|
||||
- Versioned Statement Imports
|
||||
- Finance Frontend SPA
|
||||
- Closed Beta Launch Gates
|
||||
- package.json
|
||||
- Financial Tracker MVP
|
||||
- test_adapter_contracts.py
|
||||
- generate-api.mjs
|
||||
- schema.d.ts
|
||||
- test_api.py
|
||||
- i18n-parity.test.ts
|
||||
- Expected Income
|
||||
- test_private_fixtures.py
|
||||
- TestAlfaDebitContract
|
||||
- jsdom
|
||||
- graphify
|
||||
- @types/react-dom
|
||||
- typescript-eslint
|
||||
- SourceRow
|
||||
- vitest
|
||||
- Favicon SVG (ledger lines icon)
|
||||
- init-app-role.sh
|
||||
- Payee
|
||||
- finance-backend
|
||||
- LocalDirectoryBlobStore
|
||||
- account_fingerprint
|
||||
- Gitea CI and real statement imports
|
||||
- 20260918_0002_hold_lifecycle.py
|
||||
- eslint-plugin-react-hooks
|
||||
- openapi-typescript
|
||||
- @testing-library/dom
|
||||
- @testing-library/react
|
||||
- vite
|
||||
- spaces/router.py
|
||||
- identity/service.py
|
||||
- get_settings
|
||||
- ColumnChart.tsx
|
||||
|
||||
## God Nodes (most connected - your core abstractions)
|
||||
1. `Session` - 111 edges
|
||||
2. `Base` - 46 edges
|
||||
3. `AppError` - 41 edges
|
||||
4. `get_settings()` - 38 edges
|
||||
5. `PkMixin` - 38 edges
|
||||
6. `TimestampMixin` - 37 edges
|
||||
7. `conflict()` - 34 edges
|
||||
8. `record()` - 32 edges
|
||||
9. `bad_request()` - 27 edges
|
||||
10. `not_found()` - 26 edges
|
||||
1. `Session` - 140 edges
|
||||
2. `AppError` - 53 edges
|
||||
3. `Base` - 47 edges
|
||||
4. `PkMixin` - 39 edges
|
||||
5. `get_settings()` - 38 edges
|
||||
6. `TimestampMixin` - 38 edges
|
||||
7. `conflict()` - 35 edges
|
||||
8. `record()` - 34 edges
|
||||
9. `UnsupportedStatementLayout` - 30 edges
|
||||
10. `bad_request()` - 29 edges
|
||||
|
||||
## Surprising Connections (you probably didn't know these)
|
||||
- `Typed Same-Origin API Client` --conceptually_related_to--> `Same-Origin Reverse Proxy` [INFERRED]
|
||||
frontend/README.md → infra/README.md
|
||||
- `Financial Space Ownership` --conceptually_related_to--> `Financial Space` [INFERRED]
|
||||
docs/adr/0001-financial-space-ownership-and-rls.md → CONTEXT.md
|
||||
- `Versioned Statement Imports` --conceptually_related_to--> `Statement Import` [INFERRED]
|
||||
docs/adr/0003-versioned-statement-imports.md → CONTEXT.md
|
||||
- `MVP Release Gates` --conceptually_related_to--> `Closed Beta Launch Gates` [INFERRED]
|
||||
.kilo/plans/1789034192214-financial-tracker-mvp.md → docs/operations/beta-release-runbook.md
|
||||
- `Financial Space Ownership` --conceptually_related_to--> `Financial Space` [INFERRED]
|
||||
docs/adr/0001-financial-space-ownership-and-rls.md → CONTEXT.md
|
||||
- `Typed Same-Origin API Client` --conceptually_related_to--> `Same-Origin Reverse Proxy` [INFERRED]
|
||||
frontend/README.md → infra/README.md
|
||||
- `Compose and Nginx Validation` --references--> `Docker Compose Development Stack` [EXTRACTED]
|
||||
.github/workflows/ci.yml → infra/docker-compose.yml
|
||||
- `Carry-Over Budget Reserves` --references--> `Savings Envelope` [EXTRACTED]
|
||||
docs/adr/0002-journal-and-budget-reserves.md → CONTEXT.md
|
||||
|
||||
## Import Cycles
|
||||
- None detected.
|
||||
|
||||
## Hyperedges (group relationships)
|
||||
- **Financial Space Tenancy Boundary** — context_financial_space, context_membership, docs_adr_0001_financial_space_ownership_and_rls_defense_in_depth_authorization, docs_adr_0001_financial_space_ownership_and_rls_transaction_local_rls, docs_adr_0001_financial_space_ownership_and_rls_cross_space_integrity [EXTRACTED 1.00]
|
||||
- **Auditable Financial Model** — context_operation, context_posting, docs_adr_0002_journal_and_budget_reserves_derived_balances, docs_adr_0002_journal_and_budget_reserves_plan_funding_separation, docs_adr_0002_journal_and_budget_reserves_closure_snapshots [EXTRACTED 1.00]
|
||||
- **Financial Space Tenancy Boundary** — context_financial_space, context_membership, docs_adr_0001_financial_space_ownership_and_rls_defense_in_depth_authorization, docs_adr_0001_financial_space_ownership_and_rls_transaction_local_rls, docs_adr_0001_financial_space_ownership_and_rls_cross_space_integrity [EXTRACTED 1.00]
|
||||
- **Safe Statement Import Flow** — context_statement_import, context_parse_run, context_source_record, docs_adr_0003_versioned_statement_imports_immutable_statement_document, docs_adr_0003_versioned_statement_imports_conservative_deduplication, docs_adr_0003_versioned_statement_imports_atomic_confirm_revert [EXTRACTED 1.00]
|
||||
|
||||
## Communities (62 total, 14 thin omitted)
|
||||
## Communities (75 total, 15 thin omitted)
|
||||
|
||||
### Community 0 - "Identity and Configuration"
|
||||
Cohesion: 0.05
|
||||
Nodes (87): get_settings(), Settings, current_context(), current_session(), alias, Context, CurrentSession, Db (+79 more)
|
||||
### Community 0 - "identity/router.py"
|
||||
Cohesion: 0.20
|
||||
Nodes (22): accept_invite(), AcceptInviteBody, confirm_password_reset(), create_invite(), InviteBody, login(), LoginBody, logout() (+14 more)
|
||||
|
||||
### Community 1 - "Budget Domain Services"
|
||||
Cohesion: 0.09
|
||||
Nodes (84): balances(), cash_flow(), category_spend(), plan_fact(), date, UUID, BudgetPeriod, Calendar month budget period in the space timezone (month granularity). (+76 more)
|
||||
### Community 1 - "budgeting/router.py"
|
||||
Cohesion: 0.20
|
||||
Nodes (34): AssignmentItem, assignments(), AssignmentsBody, categories(), CategoryBody, CategoryFundingBody, close(), close_frontend() (+26 more)
|
||||
|
||||
### Community 2 - "Database Models"
|
||||
Cohesion: 0.08
|
||||
Nodes (64): downgrade(), upgrade(), AuditEvent, Append-only domain/security event. Never stores raw financial payloads., ExpectedIncome, ExpectedIncomeMatch, FundingMovement, PeriodClosure (+56 more)
|
||||
|
||||
### Community 3 - "Database Context and Exports"
|
||||
### Community 2 - "Base"
|
||||
Cohesion: 0.06
|
||||
Nodes (46): get_db(), get_session_factory(), Any, sessionmaker, Transaction-local RLS context. No-op on SQLite (tests); fail-closed on…, Transactional scope for workers/endpoints that own the transaction lifecycle., session_scope(), set_space_context() (+38 more)
|
||||
Nodes (87): downgrade(), upgrade(), AuditEvent, Append-only domain/security event. Never stores raw financial payloads., ExpectedIncome, ExpectedIncomeMatch, FundingMovement, PeriodClosure (+79 more)
|
||||
|
||||
### Community 4 - "Statement Import Services"
|
||||
Cohesion: 0.10
|
||||
Nodes (55): detect_adapter(), magic_kind(), new_object_key(), decrypt_file(), encrypt_file(), EncryptedFile, Per-file random DEK; DEK wrapped with the KEK from the secret manager., Crypto-shred hook: deleting wrapped DEK rows in DB plus blobs removes access. (+47 more)
|
||||
|
||||
### Community 5 - "Parser Adapters"
|
||||
Cohesion: 0.08
|
||||
Nodes (42): AppError, Any, Domain error rendered as a machine-readable JSON error payload., unprocessable(), AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter, AlfaXlsxAdapterBase, _cell_str() (+34 more)
|
||||
|
||||
### Community 6 - "Ledger Services"
|
||||
Cohesion: 0.14
|
||||
Nodes (48): Any, record(), get_income_source(), conflict(), not_found(), confirm_document(), Operation, Posting (+40 more)
|
||||
|
||||
### Community 7 - "API Domain Types"
|
||||
### Community 3 - "set_space_context"
|
||||
Cohesion: 0.07
|
||||
Nodes (39): AccountStatus, BalanceItem, BalancesReport, BudgetBlocker, BudgetCategory, BudgetStatus, BudgetSummary, BudgetWarning (+31 more)
|
||||
Nodes (43): get_db(), get_engine(), get_session_factory(), Any, sessionmaker, Transaction-local RLS context. No-op on SQLite (tests); fail-closed on…, Transactional scope for workers/endpoints that own the transaction lifecycle., session_scope() (+35 more)
|
||||
|
||||
### Community 8 - "Authentication UI"
|
||||
Cohesion: 0.10
|
||||
Nodes (33): AcceptInvitePayload, LoginPayload, PasswordResetPayload, AcceptInvitePage, AccountsPage, BudgetPage, LoginPage, OperationsPage (+25 more)
|
||||
### Community 4 - "imports/router.py"
|
||||
Cohesion: 0.15
|
||||
Nodes (33): confirm(), ConfirmBody, decide_row(), DecisionBody, delete_source(), detail(), HoldResolveBody, imports() (+25 more)
|
||||
|
||||
### Community 9 - "Ledger API"
|
||||
### Community 5 - "alfa_xlsx.py"
|
||||
Cohesion: 0.14
|
||||
Nodes (29): AlfaCreditXlsxAdapter, AlfaDebitXlsxAdapter, _AlfaHeaderMeta, AlfaXlsxAdapterBase, _cell_str(), _debit_find_header(), _debit_header_metadata(), _debit_header_value() (+21 more)
|
||||
|
||||
### Community 6 - "Session"
|
||||
Cohesion: 0.07
|
||||
Nodes (112): balances(), plan_fact(), Any, record(), BudgetPeriod, Calendar month budget period in the space timezone (month granularity)., add_expected_income(), archive_category() (+104 more)
|
||||
|
||||
### Community 7 - "types.ts"
|
||||
Cohesion: 0.05
|
||||
Nodes (53): api, AcceptInvitePayload, Account, AccountStatus, BalanceItem, BalancesReport, BudgetBlocker, BudgetCategory (+45 more)
|
||||
|
||||
### Community 8 - "router.tsx"
|
||||
Cohesion: 0.09
|
||||
Nodes (35): AcceptInvitePage, AccountsPage, BudgetPage, ImportDetailPage, ImportsPage, LoginPage, OperationsPage, RequireAuth() (+27 more)
|
||||
|
||||
### Community 9 - "ledger/router.py"
|
||||
Cohesion: 0.15
|
||||
Nodes (31): account_json(), AccountBody, accounts(), archive_account(), create_account(), create_operation(), delete_account(), operation_json() (+23 more)
|
||||
|
||||
### Community 10 - "Import Review UI"
|
||||
Cohesion: 0.11
|
||||
Nodes (21): Operation, ImportsPage, NotFoundPage, ReviewQueuePage, Badge(), BadgeTone, EmptyState(), EmptyStateProps (+13 more)
|
||||
### Community 10 - "AccountsPage.tsx"
|
||||
Cohesion: 0.21
|
||||
Nodes (13): AccountType, CreateAccountPayload, ConfirmDialog(), ConfirmDialogProps, Modal(), ModalProps, useAccounts(), useArchiveAccount() (+5 more)
|
||||
|
||||
### Community 11 - "TypeScript Configuration"
|
||||
### Community 11 - "compilerOptions"
|
||||
Cohesion: 0.07
|
||||
Nodes (28): compilerOptions, baseUrl, esModuleInterop, forceConsistentCasingInFileNames, isolatedModules, jsx, lib, module (+20 more)
|
||||
|
||||
### Community 12 - "Analytics UI"
|
||||
Cohesion: 0.12
|
||||
Nodes (19): AnalyticsPage, CashFlowColumn, ColumnChart(), ColumnChartProps, BarRow, HorizontalBars(), HorizontalBarsProps, PlanFactBars() (+11 more)
|
||||
|
||||
### Community 13 - "API and Analytics"
|
||||
### Community 12 - "AnalyticsPage.tsx"
|
||||
Cohesion: 0.14
|
||||
Nodes (22): balances(), cash_flow(), category_spend(), plan_fact(), Context, date, Db, get (+14 more)
|
||||
Nodes (15): AnalyticsPage, NotFoundPage, PlanFactBars(), PlanFactBarsProps, PlanFactRow, MonthSwitcher(), MonthSwitcherProps, PageHeader() (+7 more)
|
||||
|
||||
### Community 14 - "Frontend Development Dependencies"
|
||||
Cohesion: 0.10
|
||||
Nodes (21): eslint, @eslint/js, eslint-plugin-react-hooks, devDependencies, eslint, @eslint/js, eslint-plugin-react-hooks, openapi-typescript (+13 more)
|
||||
### Community 13 - "analytics/router.py"
|
||||
Cohesion: 0.27
|
||||
Nodes (12): balances(), cash_flow(), category_spend(), plan_fact(), Context, date, Db, get (+4 more)
|
||||
|
||||
### Community 15 - "Accounts UI"
|
||||
Cohesion: 0.16
|
||||
Nodes (15): Account, AccountType, ConfirmDialog(), ConfirmDialogProps, Modal(), ModalProps, useAccounts(), useArchiveAccount() (+7 more)
|
||||
### Community 14 - "devDependencies"
|
||||
Cohesion: 0.11
|
||||
Nodes (19): eslint, @eslint/js, eslint-plugin-jsx-a11y, devDependencies, eslint, @eslint/js, eslint-plugin-jsx-a11y, globals (+11 more)
|
||||
|
||||
### Community 16 - "Frontend Runtime Dependencies"
|
||||
### Community 15 - "test_import_holds.py"
|
||||
Cohesion: 0.13
|
||||
Nodes (43): AppError, Domain error rendered as a machine-readable JSON error payload., normalize_description(), date, record_fingerprint(), add_record(), _draft(), hold_document() (+35 more)
|
||||
|
||||
### Community 16 - "dependencies"
|
||||
Cohesion: 0.11
|
||||
Nodes (19): dependencies, @hookform/resolvers, i18next, react, react-dom, react-hook-form, react-i18next, react-router-dom (+11 more)
|
||||
|
||||
### Community 17 - "Import Detail Mutations"
|
||||
Cohesion: 0.27
|
||||
Nodes (15): RowDecision, ImportDetailPage, useToast(), useConfirmImport(), useImport(), useImportErrorToaster(), useReparseImport(), useRevertImport() (+7 more)
|
||||
### Community 17 - "ImportDetailPage.tsx"
|
||||
Cohesion: 0.33
|
||||
Nodes (13): useToast(), useConfirmImport(), useImport(), useImportErrorToaster(), useReparseImport(), useRevertImport(), useSaveRowDecision(), useUploadImport() (+5 more)
|
||||
|
||||
### Community 18 - "Operations UI"
|
||||
Cohesion: 0.27
|
||||
Nodes (14): useActiveAccounts(), useCreateOperation(), useLedgerMutation(), useOperations(), useVoidOperation(), amountField(), dateField(), ExpenseForm() (+6 more)
|
||||
### Community 18 - "OperationsPage.tsx"
|
||||
Cohesion: 0.14
|
||||
Nodes (27): CreateOperationPayload, Badge(), BadgeTone, EmptyState(), EmptyStateProps, QueryGate(), QueryGateProps, useActiveAccounts() (+19 more)
|
||||
|
||||
### Community 19 - "Query Keys and Operations"
|
||||
Cohesion: 0.22
|
||||
Nodes (10): api, invalidationRoots, queryKeys, SESSION_ROOT, sessionInvalidationRoot, SPACE_ROOT, spaceInvalidationRoot, OperationFilters (+2 more)
|
||||
### Community 19 - "SettingsPage.tsx"
|
||||
Cohesion: 0.16
|
||||
Nodes (16): invalidationRoots, queryKeys, SESSION_ROOT, sessionInvalidationRoot, SPACE_ROOT, spaceInvalidationRoot, OperationFilters, useCancelDeletion() (+8 more)
|
||||
|
||||
### Community 20 - "Frontend Bootstrap"
|
||||
Cohesion: 0.17
|
||||
Nodes (10): Locale, App(), queryClient, AppRoutes(), createI18n(), detectLocale(), i18n, LOCALES (+2 more)
|
||||
### Community 20 - "Toaster.tsx"
|
||||
Cohesion: 0.12
|
||||
Nodes (15): Locale, App(), queryClient, AppRoutes(), ToastApi, ToastContext, ToastItem, ToastKind (+7 more)
|
||||
|
||||
### Community 21 - "Frontend Test Support"
|
||||
Cohesion: 0.17
|
||||
Nodes (12): User, ToastApi, ToastContext, ToastItem, ToastKind, ToastProvider(), createTestQueryClient(), renderWithProviders() (+4 more)
|
||||
### Community 21 - "ozon_pdf.py"
|
||||
Cohesion: 0.14
|
||||
Nodes (13): Protocol, StatementAdapter, StatementDraft, _first_page_text(), _ozon_cell(), _ozon_normalize(), OzonCreditPdfAdapter, OzonDebitPdfAdapter (+5 more)
|
||||
|
||||
### Community 22 - "Budget UI"
|
||||
### Community 22 - "BudgetPage.tsx"
|
||||
Cohesion: 0.36
|
||||
Nodes (12): useBudgetCategories(), useBudgetErrorToaster(), useBudgetSummary(), useCloseMonth(), useCopyPreviousMonth(), useFundCategory(), usePeriodMutation(), useReopenMonth() (+4 more)
|
||||
|
||||
### Community 23 - "HTTP API Client"
|
||||
### Community 23 - "client.ts"
|
||||
Cohesion: 0.23
|
||||
Nodes (10): apiClient, ApiError, buildUrl(), ErrorEnvelope, parseErrorEnvelope(), QueryValue, readCsrfToken(), request() (+2 more)
|
||||
|
||||
### Community 24 - "Form Fields"
|
||||
Cohesion: 0.23
|
||||
Nodes (10): FieldChrome, InputProps, MoneyField(), MoneyFieldProps, SelectFieldProps, SelectProps, TextareaField(), TextareaFieldProps (+2 more)
|
||||
### Community 24 - "fields.tsx"
|
||||
Cohesion: 0.21
|
||||
Nodes (11): FieldChrome, InputProps, MoneyField(), MoneyFieldProps, SelectField(), SelectFieldProps, SelectProps, TextareaField() (+3 more)
|
||||
|
||||
### Community 25 - "Infrastructure and CI"
|
||||
Cohesion: 0.18
|
||||
Nodes (12): Backend Quality Gate, CI Pipeline, Compose and Nginx Validation, Frontend Quality Gate, Commodity Infrastructure Services, Docker Compose Development Stack, Internal Backend Network, Same-Origin Frontend Service (+4 more)
|
||||
### Community 25 - "Docker Compose Development Stack"
|
||||
Cohesion: 0.29
|
||||
Nodes (8): Commodity Infrastructure Services, Docker Compose Development Stack, Internal Backend Network, Same-Origin Frontend Service, Worker Parsing Isolation, Finance Infrastructure, Same-Origin Reverse Proxy, Backend and Frontend Workstream Contracts
|
||||
|
||||
### Community 26 - "Core Finance Domain"
|
||||
Cohesion: 0.22
|
||||
Nodes (9): Credit Reserve, Financial Account, Financial Space, Membership, User, Carry-Over Budget Reserves, Closed Beta Zero-Based Budgeting, Finance Tracker (+1 more)
|
||||
### Community 26 - "Financial Space"
|
||||
Cohesion: 0.17
|
||||
Nodes (12): Credit Reserve, Financial Account, Financial Space, Membership, User, Carry-Over Budget Reserves, Provider-Neutral Data Boundary, Russian Federation Data Residency (+4 more)
|
||||
|
||||
### Community 27 - "Frontend Scripts"
|
||||
### Community 27 - "scripts"
|
||||
Cohesion: 0.22
|
||||
Nodes (9): scripts, api:generate, build, dev, lint, preview, test, test:watch (+1 more)
|
||||
|
||||
### Community 28 - "Settings and Data Lifecycle"
|
||||
Cohesion: 0.44
|
||||
Nodes (7): useCancelDeletion(), useRequestDeletion(), useRequestExport(), useSpaceDeletion(), DeletionForm(), DeletionFormValues, SettingsPage()
|
||||
### Community 28 - "dependencies.py"
|
||||
Cohesion: 0.15
|
||||
Nodes (21): current_context(), current_session(), alias, Context, CurrentSession, Db, Header, User (+13 more)
|
||||
|
||||
### Community 29 - "Budget Allocation Model"
|
||||
### Community 29 - "General Pool"
|
||||
Cohesion: 0.29
|
||||
Nodes (8): Budget Period, Funding, General Pool, Monthly Category, Plan Assignment, Savings Envelope, Immutable Closure Snapshots, Plan and Funding Separation
|
||||
|
||||
### Community 30 - "MVP and Import Design"
|
||||
Cohesion: 0.25
|
||||
Nodes (8): Atomic Import Confirm and Revert, Conservative Deduplication, Versioned Statement Imports, Budget Invariants, Critical User Path, Financial Tracker MVP, Statement Import Pipeline, Modular Monolith
|
||||
### Community 30 - "TestBaseContract"
|
||||
Cohesion: 0.17
|
||||
Nodes (5): detect_adapter(), make_draft(), make_source_row(), TestBaseContract, Workbook
|
||||
|
||||
### Community 31 - "Data Security and Recovery"
|
||||
Cohesion: 0.25
|
||||
Nodes (8): Immutable Statement Document, Per-File Envelope Encryption, Provider-Neutral Data Boundary, Russian Federation Data Residency, Backup and Restore Runbook, Mandatory KEK Backup, Quarterly Restore Drill, Provider-Neutral Infrastructure
|
||||
### Community 31 - "Per-File Envelope Encryption"
|
||||
Cohesion: 0.29
|
||||
Nodes (7): Immutable Statement Document, Per-File Envelope Encryption, Backup and Restore Runbook, Mandatory KEK Backup, Quarterly Restore Drill, Disaster Restore Procedure, Post-Restore Deletion Replay
|
||||
|
||||
### Community 32 - "Ledger Concepts"
|
||||
Cohesion: 0.38
|
||||
Nodes (7): Internal Transfer, Operation, Posting, Refund, Derived Account Balances, Operations and Postings Journal, Ledger Invariants
|
||||
### Community 32 - "Operation"
|
||||
Cohesion: 0.47
|
||||
Nodes (6): Internal Transfer, Operation, Posting, Refund, Derived Account Balances, Operations and Postings Journal
|
||||
|
||||
### Community 33 - "Parser Versioning"
|
||||
Cohesion: 0.33
|
||||
Nodes (7): Parse Run, Source Record, Statement Import, Parser Versioning and Review Diff, Layout Drift Re-Gating, Parser Private Fixture Gate, Private Statement Test Contract
|
||||
### Community 33 - "Versioned Statement Imports"
|
||||
Cohesion: 0.20
|
||||
Nodes (11): Parse Run, Source Record, Statement Import, Atomic Import Confirm and Revert, Conservative Deduplication, Parser Versioning and Review Diff, Versioned Statement Imports, Layout Drift Re-Gating (+3 more)
|
||||
|
||||
### Community 34 - "Frontend Architecture"
|
||||
### Community 34 - "Finance Frontend SPA"
|
||||
Cohesion: 0.29
|
||||
Nodes (7): Server-Side Secret Boundary, Grossbukh Application Shell, Accessible Responsive UI, Finance Frontend SPA, RU and EN Localization, Space-Scoped Query Keys, Typed Same-Origin API Client
|
||||
|
||||
### Community 35 - "Beta Release Governance"
|
||||
Cohesion: 0.29
|
||||
Nodes (7): Closed Beta Release Runbook, Forward-Fix Rollback Policy, Financial Data Incident Response, Closed Beta Launch Gates, Production Launch Sequence, Adapter Beta Eligibility, MVP Release Gates
|
||||
### Community 35 - "Closed Beta Launch Gates"
|
||||
Cohesion: 0.20
|
||||
Nodes (11): Crypto-Shredded Deletion, Closed Beta Release Runbook, Forward-Fix Rollback Policy, Financial Data Incident Response, Closed Beta Launch Gates, Production Launch Sequence, Thirty-Day Space Deletion, Durable Deletion Ledger (+3 more)
|
||||
|
||||
### Community 36 - "Package Metadata"
|
||||
Cohesion: 0.29
|
||||
Nodes (6): allowScripts, esbuild@0.25.12, name, private, type, version
|
||||
|
||||
### Community 37 - "Tenant Isolation"
|
||||
Cohesion: 0.33
|
||||
Nodes (6): Audited Break-Glass Access, Cross-Space Referential Integrity, Defense-in-Depth Authorization, Financial Space Ownership, Transaction-Local RLS Context, Tenant Boundary
|
||||
|
||||
### Community 38 - "Deletion and Restore"
|
||||
### Community 36 - "package.json"
|
||||
Cohesion: 0.40
|
||||
Nodes (6): Crypto-Shredded Deletion, Disaster Restore Procedure, Thirty-Day Space Deletion, Durable Deletion Ledger, Idempotent Resumable Purge, Post-Restore Deletion Replay
|
||||
Nodes (4): name, private, type, version
|
||||
|
||||
### Community 39 - "API Generation Script"
|
||||
### Community 37 - "Financial Tracker MVP"
|
||||
Cohesion: 0.18
|
||||
Nodes (11): Audited Break-Glass Access, Cross-Space Referential Integrity, Defense-in-Depth Authorization, Financial Space Ownership, Transaction-Local RLS Context, Budget Invariants, Critical User Path, Financial Tracker MVP (+3 more)
|
||||
|
||||
### Community 38 - "test_adapter_contracts.py"
|
||||
Cohesion: 0.11
|
||||
Nodes (9): minimal_pdf(), _QuotaSettings, Sanitized synthetic contract tests for the evidenced statement adapters. These…, Build a small valid single-font PDF with empty or ASCII-only pages., same_page_rocket_lines(), synthetic_rocket_lines(), TestMoneyParsing, TestRocketbankDebitContract (+1 more)
|
||||
|
||||
### Community 39 - "generate-api.mjs"
|
||||
Cohesion: 0.40
|
||||
Nodes (5): firstReadable(), LOCAL_CANDIDATES, main(), OUT_DIR, REMOTE_CANDIDATES
|
||||
|
||||
### Community 40 - "OpenAPI Schema"
|
||||
### Community 40 - "schema.d.ts"
|
||||
Cohesion: 0.33
|
||||
Nodes (5): components, $defs, operations, paths, webhooks
|
||||
|
||||
### Community 41 - "API Integration Tests"
|
||||
### Community 41 - "test_api.py"
|
||||
Cohesion: 0.67
|
||||
Nodes (3): TestClient, test_invite_login_and_authenticated_space(), test_openapi_contains_critical_paths()
|
||||
|
||||
### Community 44 - "test_private_fixtures.py"
|
||||
Cohesion: 0.12
|
||||
Nodes (22): magic_kind(), _draft_json(), Any, Reconcile parsed rows against declared bank controls with stable diagnostics.…, _reconciliation_summary(), _check(), _fail(), _load_entries() (+14 more)
|
||||
|
||||
### Community 45 - "TestAlfaDebitContract"
|
||||
Cohesion: 0.21
|
||||
Nodes (3): alfa_workbook_bytes(), synthetic_alfa_rows(), TestAlfaDebitContract
|
||||
|
||||
### Community 47 - "graphify"
|
||||
Cohesion: 0.17
|
||||
Nodes (11): command, enabled, timeout, type, mcp, graphify, plugin, $schema (+3 more)
|
||||
|
||||
### Community 50 - "SourceRow"
|
||||
Cohesion: 0.19
|
||||
Nodes (9): SourceRow, _date_from(), _PendingRecord, date, Rocketbank debit statement contract, evidenced against the real document.…, RocketbankDebitPdfAdapter, open_pdf(), _quota_error() (+1 more)
|
||||
|
||||
### Community 62 - "LocalDirectoryBlobStore"
|
||||
Cohesion: 0.39
|
||||
Nodes (3): LocalDirectoryBlobStore, Path, S3-neutral local store; production swaps in an S3-compatible client.
|
||||
|
||||
### Community 63 - "account_fingerprint"
|
||||
Cohesion: 0.25
|
||||
Nodes (3): account_fingerprint(), ozon_extracted(), TestOzonDebitContract
|
||||
|
||||
### Community 64 - "Gitea CI and real statement imports"
|
||||
Cohesion: 0.29
|
||||
Nodes (6): Confirmed decisions, Gitea CI and real statement imports, Goal and constraints, Implementation plan, Principal risks, Validation and rollout
|
||||
|
||||
### Community 65 - "20260918_0002_hold_lifecycle.py"
|
||||
Cohesion: 0.80
|
||||
Nodes (4): downgrade(), _has_check_constraint(), _has_column(), upgrade()
|
||||
|
||||
### Community 71 - "spaces/router.py"
|
||||
Cohesion: 0.16
|
||||
Nodes (19): send_token_email(), cancel_deletion(), current(), deletion_status(), DeletionBody, invite_member(), MemberInviteBody, alias (+11 more)
|
||||
|
||||
### Community 72 - "identity/service.py"
|
||||
Cohesion: 0.28
|
||||
Nodes (18): unauthorized(), accept_invitation(), authenticate(), create_invitation(), create_password_reset(), create_session(), get_active_session(), normalize_email() (+10 more)
|
||||
|
||||
### Community 73 - "get_settings"
|
||||
Cohesion: 0.16
|
||||
Nodes (12): get_settings(), Settings, create_celery_app(), encrypt_file(), EncryptedFile, Per-file random DEK; DEK wrapped with the KEK from the secret manager., Crypto-shred hook: deleting wrapped DEK rows in DB plus blobs removes access., shred_dek_material() (+4 more)
|
||||
|
||||
### Community 74 - "ColumnChart.tsx"
|
||||
Cohesion: 0.24
|
||||
Nodes (7): CashFlowColumn, ColumnChart(), ColumnChartProps, BarRow, HorizontalBars(), HorizontalBarsProps, rows
|
||||
|
||||
## Knowledge Gaps
|
||||
- **176 isolated node(s):** `finance-backend`, `name`, `private`, `version`, `type` (+171 more)
|
||||
These have ≤1 connection - possible missing edges or undocumented components. (Counts symbols only; 296 node(s) total have ≤1 connection when file, concept and rationale nodes are included.)
|
||||
- **14 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes.
|
||||
- **185 isolated node(s):** `$schema`, `file:///home/k1nq/repos/finance/.kilo/plugins/graphify.js`, `type`, `/home/k1nq/.local/bin/graphify-mcp`, `/home/k1nq/repos/finance/graphify-out/graph.json` (+180 more)
|
||||
These have ≤1 connection - possible missing edges or undocumented components. (Counts symbols only; 335 node(s) total have ≤1 connection when file, concept and rationale nodes are included.)
|
||||
- **15 thin communities (<3 nodes) omitted from report** — run `graphify query` to explore isolated nodes.
|
||||
|
||||
## Suggested Questions
|
||||
_Questions this graph is uniquely positioned to answer:_
|
||||
|
||||
- **Why does `Session` connect `Budget Domain Services` to `Identity and Configuration`, `Database Models`, `Database Context and Exports`, `Statement Import Services`, `Ledger Services`?**
|
||||
_High betweenness centrality (0.072) - this node is a cross-community bridge._
|
||||
- **Why does `AppError` connect `Parser Adapters` to `Identity and Configuration`, `Budget Domain Services`, `Database Models`, `Database Context and Exports`, `Statement Import Services`, `Ledger Services`, `API and Analytics`?**
|
||||
_High betweenness centrality (0.035) - this node is a cross-community bridge._
|
||||
- **Why does `get_settings()` connect `Identity and Configuration` to `Database Models`, `Database Context and Exports`, `Statement Import Services`, `Parser Adapters`, `API and Analytics`?**
|
||||
_High betweenness centrality (0.024) - this node is a cross-community bridge._
|
||||
- **Why does `AppError` connect `test_import_holds.py` to `identity/router.py`, `Base`, `set_space_context`, `imports/router.py`, `alfa_xlsx.py`, `Session`, `test_adapter_contracts.py`, `identity/service.py`, `get_settings`, `SourceRow`, `dependencies.py`, `account_fingerprint`?**
|
||||
_High betweenness centrality (0.078) - this node is a cross-community bridge._
|
||||
- **Why does `Session` connect `Session` to `Base`, `set_space_context`, `identity/service.py`, `analytics/router.py`, `test_import_holds.py`, `dependencies.py`?**
|
||||
_High betweenness centrality (0.074) - this node is a cross-community bridge._
|
||||
- **Why does `bad_request()` connect `Session` to `budgeting/router.py`, `spaces/router.py`, `identity/service.py`, `ledger/router.py`, `test_import_holds.py`?**
|
||||
_High betweenness centrality (0.025) - this node is a cross-community bridge._
|
||||
- **Are the 2 inferred relationships involving `Session` (e.g. with `UTCDateTime` and `test_postgres_rls_requires_matching_transaction_context()`) actually correct?**
|
||||
_`Session` has 2 INFERRED edges - model-reasoned connections that need verification._
|
||||
- **Are the 23 inferred relationships involving `AppError` (e.g. with `request_export()` and `login()`) actually correct?**
|
||||
_`AppError` has 23 INFERRED edges - model-reasoned connections that need verification._
|
||||
- **Are the 3 inferred relationships involving `Base` (e.g. with `downgrade()` and `upgrade()`) actually correct?**
|
||||
_`Base` has 3 INFERRED edges - model-reasoned connections that need verification._
|
||||
- **Are the 13 inferred relationships involving `AppError` (e.g. with `request_export()` and `login()`) actually correct?**
|
||||
_`AppError` has 13 INFERRED edges - model-reasoned connections that need verification._
|
||||
- **What connects `finance-backend`, `name`, `private` to the rest of the system?**
|
||||
_176 weakly-connected nodes found - possible documentation gaps or missing edges._
|
||||
- **What connects `$schema`, `file:///home/k1nq/repos/finance/.kilo/plugins/graphify.js`, `type` to the rest of the system?**
|
||||
_185 weakly-connected nodes found - possible documentation gaps or missing edges._
|
||||
File diff suppressed because one or more lines are too long
+15194
-5794
File diff suppressed because it is too large
Load Diff
+108
-66
@@ -72,10 +72,10 @@
|
||||
"semantic_hash": "c352243e4fb85ab49f9cf4532c11bd3f"
|
||||
},
|
||||
"backend/app/budgeting/service.py": {
|
||||
"mtime": 1789358018.8352346,
|
||||
"seen": 1789377303.0691924,
|
||||
"ast_hash": "3eac2370782c5606fa0ceb3b2b6b49c6",
|
||||
"semantic_hash": "3eac2370782c5606fa0ceb3b2b6b49c6"
|
||||
"mtime": 1789730971.072982,
|
||||
"seen": 1789735433.158112,
|
||||
"ast_hash": "c30d4c58608f88b81ff05be63a314247",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/categorization/__init__.py": {
|
||||
"mtime": 1789127372.62572,
|
||||
@@ -204,34 +204,34 @@
|
||||
"semantic_hash": "d41d8cd98f00b204e9800998ecf8427e"
|
||||
},
|
||||
"backend/app/imports/adapters/__init__.py": {
|
||||
"mtime": 1789128289.6891637,
|
||||
"seen": 1789377303.0692112,
|
||||
"ast_hash": "cb3d7cdba651c7e72e63b48e0dc1ada4",
|
||||
"semantic_hash": "cb3d7cdba651c7e72e63b48e0dc1ada4"
|
||||
"mtime": 1789733796.3306727,
|
||||
"seen": 1789735433.158127,
|
||||
"ast_hash": "0d3191f98e2602dcb37b96eefcb9d2ea",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/adapters/alfa_xlsx.py": {
|
||||
"mtime": 1789129314.8220334,
|
||||
"seen": 1789377303.069212,
|
||||
"ast_hash": "f134a338a84dabd4bf83fe9921fee6e1",
|
||||
"semantic_hash": "f134a338a84dabd4bf83fe9921fee6e1"
|
||||
"mtime": 1789736201.215202,
|
||||
"seen": 1789736272.9455209,
|
||||
"ast_hash": "31eee0cded845f9ddaff98d2127f4354",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/adapters/base.py": {
|
||||
"mtime": 1789128289.6891637,
|
||||
"seen": 1789377303.069213,
|
||||
"ast_hash": "d56aa3b5b32d0673ac4f2898d658327c",
|
||||
"semantic_hash": "d56aa3b5b32d0673ac4f2898d658327c"
|
||||
"mtime": 1789733778.7145572,
|
||||
"seen": 1789735433.1581285,
|
||||
"ast_hash": "4f6364ab07cf38ba2eea1ba94678d784",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/adapters/ozon_pdf.py": {
|
||||
"mtime": 1789128297.7359908,
|
||||
"seen": 1789377303.0692136,
|
||||
"ast_hash": "24fb6546e843e084f5b41ba134842c75",
|
||||
"semantic_hash": "24fb6546e843e084f5b41ba134842c75"
|
||||
"mtime": 1789733868.836172,
|
||||
"seen": 1789735433.158129,
|
||||
"ast_hash": "dfa6dc9ccf2c655e6c2eeaceea0cf06a",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/adapters/rocketbank_pdf.py": {
|
||||
"mtime": 1789127950.4610171,
|
||||
"seen": 1789377303.069214,
|
||||
"ast_hash": "7e3e787ffb32eb5a88b7f1af8627ba83",
|
||||
"semantic_hash": "7e3e787ffb32eb5a88b7f1af8627ba83"
|
||||
"mtime": 1789734555.2246265,
|
||||
"seen": 1789735433.1581297,
|
||||
"ast_hash": "b4f28ba73c56da5f5eb96eb4f129e098",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/blob.py": {
|
||||
"mtime": 1789129216.5581613,
|
||||
@@ -252,10 +252,10 @@
|
||||
"semantic_hash": "c72d7849c6f46e6c9a9136f3ae9dc92f"
|
||||
},
|
||||
"backend/app/imports/models.py": {
|
||||
"mtime": 1789358018.8338456,
|
||||
"seen": 1789377303.0692174,
|
||||
"ast_hash": "416d7e45875501344596ef7867f77d0c",
|
||||
"semantic_hash": "416d7e45875501344596ef7867f77d0c"
|
||||
"mtime": 1789730695.7355492,
|
||||
"seen": 1789735433.1581326,
|
||||
"ast_hash": "b99c93656a17603691838c9e7963d9ed",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/readers.py": {
|
||||
"mtime": 1789128289.6877162,
|
||||
@@ -264,16 +264,16 @@
|
||||
"semantic_hash": "0904afccafd9eb544b4d6846fc6384c8"
|
||||
},
|
||||
"backend/app/imports/router.py": {
|
||||
"mtime": 1789358168.7152903,
|
||||
"seen": 1789377303.0692196,
|
||||
"ast_hash": "9eba514885eb369b2756103d10d518a3",
|
||||
"semantic_hash": "9eba514885eb369b2756103d10d518a3"
|
||||
"mtime": 1789730952.1978154,
|
||||
"seen": 1789735433.1581337,
|
||||
"ast_hash": "5bd2a94019f9cbda6d0dc617dc82cd41",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/service.py": {
|
||||
"mtime": 1789358130.3195853,
|
||||
"seen": 1789377303.0692203,
|
||||
"ast_hash": "aa388826ac8d183731806f96c10da0f4",
|
||||
"semantic_hash": "aa388826ac8d183731806f96c10da0f4"
|
||||
"mtime": 1789736072.1450737,
|
||||
"seen": 1789736272.9455285,
|
||||
"ast_hash": "16c0182cdcfb0aee9a7a5a895e3c86b2",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/app/imports/tasks.py": {
|
||||
"mtime": 1789356921.2930734,
|
||||
@@ -354,10 +354,10 @@
|
||||
"semantic_hash": "4788addc575411056226ee892d1e2806"
|
||||
},
|
||||
"backend/pyproject.toml": {
|
||||
"mtime": 1789360098.861183,
|
||||
"seen": 1789377303.0692322,
|
||||
"ast_hash": "3af36a3ae7c03efc22fbaee92f71daaf",
|
||||
"semantic_hash": "3af36a3ae7c03efc22fbaee92f71daaf"
|
||||
"mtime": 1789731892.783645,
|
||||
"seen": 1789735433.1581438,
|
||||
"ast_hash": "6ae54b23d066a4a48b7edbbe964af75f",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/tests/conftest.py": {
|
||||
"mtime": 1789129509.9246948,
|
||||
@@ -390,10 +390,10 @@
|
||||
"semantic_hash": "fb7d5e4503c9482b5014e28528f75e2a"
|
||||
},
|
||||
"backend/tests/test_postgres_and_s3.py": {
|
||||
"mtime": 1789130360.2146895,
|
||||
"seen": 1789377303.0692372,
|
||||
"ast_hash": "c811cba6bd00ab76e3e64f977bc29e04",
|
||||
"semantic_hash": "c811cba6bd00ab76e3e64f977bc29e04"
|
||||
"mtime": 1789734220.6980176,
|
||||
"seen": 1789735433.1581545,
|
||||
"ast_hash": "1def023e0a52eed53bcb97557919484b",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"frontend/eslint.config.js": {
|
||||
"mtime": 1789128706.6116898,
|
||||
@@ -402,10 +402,10 @@
|
||||
"semantic_hash": "ed54799a921d10394f826893acb46876"
|
||||
},
|
||||
"frontend/package.json": {
|
||||
"mtime": 1789126987.143408,
|
||||
"seen": 1789377303.069239,
|
||||
"ast_hash": "d3b3d1b1c95c803bfa3f71241cdca23d",
|
||||
"semantic_hash": "d3b3d1b1c95c803bfa3f71241cdca23d"
|
||||
"mtime": 1789378006.2922633,
|
||||
"seen": 1789735433.1581702,
|
||||
"ast_hash": "10697e744e175f761f5ddb85c7925350",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"frontend/scripts/generate-api.mjs": {
|
||||
"mtime": 1789126898.3307452,
|
||||
@@ -779,12 +779,6 @@
|
||||
"ast_hash": "4c2b103cde06bc6df773f005c00b73d6",
|
||||
"semantic_hash": "4c2b103cde06bc6df773f005c00b73d6"
|
||||
},
|
||||
".github/workflows/ci.yml": {
|
||||
"mtime": 1789360098.861183,
|
||||
"seen": 1789377303.0692956,
|
||||
"ast_hash": "2088fd86568bef56ce17fd0cc3b0b944",
|
||||
"semantic_hash": "2088fd86568bef56ce17fd0cc3b0b944"
|
||||
},
|
||||
".kilo/plans/1789034192214-financial-tracker-mvp.md": {
|
||||
"mtime": 1789099088.960013,
|
||||
"seen": 1789377303.0692966,
|
||||
@@ -816,10 +810,10 @@
|
||||
"semantic_hash": "6ca9f9b8f91733ce37e15f339a656726"
|
||||
},
|
||||
"docs/adr/0003-versioned-statement-imports.md": {
|
||||
"mtime": 1789126790.5402813,
|
||||
"seen": 1789377303.0693007,
|
||||
"ast_hash": "6ed335eff108aaeaad91163d648825c1",
|
||||
"semantic_hash": "6ed335eff108aaeaad91163d648825c1"
|
||||
"mtime": 1789732137.8203447,
|
||||
"seen": 1789735433.1582236,
|
||||
"ast_hash": "e05b45d637e097a3898bc1b2f2160fe7",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"docs/adr/0004-provider-neutral-encrypted-data-boundary.md": {
|
||||
"mtime": 1789126790.5422814,
|
||||
@@ -846,10 +840,10 @@
|
||||
"semantic_hash": "1aff4ccf6a2dbcf86ed72fd34bb69c6f"
|
||||
},
|
||||
"docs/parsers/private-fixture-gate.md": {
|
||||
"mtime": 1789126842.518988,
|
||||
"seen": 1789377303.0693052,
|
||||
"ast_hash": "d4e23880cc1e29bfe416a3730ebbc76e",
|
||||
"semantic_hash": "d4e23880cc1e29bfe416a3730ebbc76e"
|
||||
"mtime": 1789734164.3005245,
|
||||
"seen": 1789735433.1582274,
|
||||
"ast_hash": "02086c7b911cf06d3d170ac04be7592e",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"frontend/README.md": {
|
||||
"mtime": 1789128756.9573848,
|
||||
@@ -870,15 +864,63 @@
|
||||
"semantic_hash": "0a7bb82cce04631f36f2d561a1665478"
|
||||
},
|
||||
"infra/docker-compose.yml": {
|
||||
"mtime": 1789358018.8362348,
|
||||
"seen": 1789377303.0693088,
|
||||
"ast_hash": "ee8600186391e2a0bf175a7e7ea7a2d2",
|
||||
"semantic_hash": "ee8600186391e2a0bf175a7e7ea7a2d2"
|
||||
"mtime": 1789400438.2433548,
|
||||
"seen": 1789735433.1582313,
|
||||
"ast_hash": "9f04dd1906326ddea306584fe68a2ae4",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"frontend/public/favicon.svg": {
|
||||
"mtime": 1789126886.1235795,
|
||||
"seen": 1789377303.0693095,
|
||||
"ast_hash": "83637b50c3736f9f0675b7f23ff5a51e",
|
||||
"semantic_hash": "83637b50c3736f9f0675b7f23ff5a51e"
|
||||
},
|
||||
".kilo/kilo.json": {
|
||||
"mtime": 1789377478.0622306,
|
||||
"seen": 1789735433.1580946,
|
||||
"ast_hash": "1839ad09cc2e72bfeb650219265f2e20",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/alembic/versions/20260918_0002_hold_lifecycle.py": {
|
||||
"mtime": 1789735291.8675542,
|
||||
"seen": 1789735433.1581025,
|
||||
"ast_hash": "2a669c07ecd8d473e703bc823cb26183",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/tests/test_adapter_contracts.py": {
|
||||
"mtime": 1789735944.366427,
|
||||
"seen": 1789736272.9455402,
|
||||
"ast_hash": "c3f63850c85b7c5262e66370ab7ab8a9",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/tests/test_import_holds.py": {
|
||||
"mtime": 1789736201.2172022,
|
||||
"seen": 1789736272.9455423,
|
||||
"ast_hash": "6f14226a83e65efbaaee6446d4ec4cb6",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"backend/tests/test_private_fixtures.py": {
|
||||
"mtime": 1789736201.2242022,
|
||||
"seen": 1789736272.9455452,
|
||||
"ast_hash": "e94f214a0d04cdd1f944699fdd32abe6",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
".gitea/workflows/ci.yml": {
|
||||
"mtime": 1789736072.1440737,
|
||||
"seen": 1789736272.945592,
|
||||
"ast_hash": "8e291c92f0eb7fe1ed1129c4fc55b188",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
".kilo/plans/1789727434954-gitea-ci-real-statement-imports.md": {
|
||||
"mtime": 1789729895.9594803,
|
||||
"seen": 1789735433.1582198,
|
||||
"ast_hash": "eb38e75994c5fc29d4137adb7e0e37f8",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"frontend/pnpm-workspace.yaml": {
|
||||
"mtime": 1789377936.9976747,
|
||||
"seen": 1789735433.1582298,
|
||||
"ast_hash": "2ae2370fa527b015cd7bd98dc7ca2a6b",
|
||||
"semantic_hash": ""
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user