Update manifest.json with new file metadata and hashes
- Updated timestamps and hashes for backend/app/imports/router.py and backend/app/imports/service.py to reflect recent changes. - Updated metadata for backend/tests/test_imports.py with new timestamps and hashes. - Added new entries for .kilo/plugins/graphify.js and AGENTS.md, including their respective timestamps and hashes.
This commit is contained in:
@@ -109,8 +109,24 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
base = summary(document)
|
||||
reconciliation = raw.get("reconciliation") or {}
|
||||
rows = []
|
||||
pending_candidate_row_ids: set[uuid.UUID] = set()
|
||||
for item in raw["records"]:
|
||||
hold = item.get("hold_resolution")
|
||||
candidates = [
|
||||
{
|
||||
"id": candidate["id"],
|
||||
"kind": candidate["kind"],
|
||||
"confidence": candidate["confidence"],
|
||||
"resolution": candidate["resolution"],
|
||||
"existingOperationId": candidate["existing_operation_id"],
|
||||
}
|
||||
for candidate in item.get("candidates", [])
|
||||
]
|
||||
has_pending_candidate = any(
|
||||
candidate["resolution"] == "pending" for candidate in candidates
|
||||
)
|
||||
if has_pending_candidate:
|
||||
pending_candidate_row_ids.add(item["id"])
|
||||
rows.append(
|
||||
{
|
||||
"id": item["id"],
|
||||
@@ -122,8 +138,11 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
"description": item["description"],
|
||||
"payeeHint": None,
|
||||
"warnings": (item["warnings"] or {}).get("codes", []),
|
||||
"decision": "pending" if item["critical"] else item["review_decision"],
|
||||
"decision": "pending"
|
||||
if item["critical"] or has_pending_candidate
|
||||
else item["review_decision"],
|
||||
"duplicateOfOperationId": item["duplicate_operation_id"],
|
||||
"candidates": candidates,
|
||||
"holdResolution": None
|
||||
if hold is None
|
||||
else {
|
||||
@@ -141,6 +160,15 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
for item in raw["records"]
|
||||
if item["critical"]
|
||||
]
|
||||
blockers.extend(
|
||||
{
|
||||
"code": "IMPORT_DECISION_REQUIRED",
|
||||
"message": "Possible duplicate or transfer requires a decision",
|
||||
"rowId": str(item["id"]),
|
||||
}
|
||||
for item in raw["records"]
|
||||
if item["id"] in pending_candidate_row_ids
|
||||
)
|
||||
status = (
|
||||
"matched"
|
||||
if reconciliation.get("full")
|
||||
@@ -176,6 +204,7 @@ def detail(document_id: uuid.UUID, db: Db, context: Context) -> dict:
|
||||
"discard": sum(row["decision"] == "discard" for row in rows),
|
||||
"duplicate": sum(row["decision"] == "duplicate" for row in rows),
|
||||
"transfer": sum(row["decision"] == "transfer" for row in rows),
|
||||
"pending": sum(row["decision"] == "pending" for row in rows),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -229,41 +229,61 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
continue
|
||||
duplicate = None
|
||||
if source.external_id:
|
||||
matches = (
|
||||
match_query = (
|
||||
db.query(SourceRecord)
|
||||
.join(ParseRun, ParseRun.id == SourceRecord.parse_run_id)
|
||||
.join(StatementDocument, StatementDocument.id == SourceRecord.document_id)
|
||||
.join(SourceRecordLink, SourceRecordLink.source_record_id == SourceRecord.id)
|
||||
.filter(
|
||||
SourceRecord.space_id == document.space_id,
|
||||
SourceRecord.document_id != document.id,
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.external_id == source.external_id,
|
||||
SourceRecord.bank_status == "posted",
|
||||
ParseRun.adapter_id == run.adapter_id,
|
||||
StatementDocument.status == "confirmed",
|
||||
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
if document.account_id is not None:
|
||||
match_query = match_query.filter(
|
||||
StatementDocument.account_id == document.account_id
|
||||
)
|
||||
matches = match_query.all()
|
||||
duplicate = matches[0] if len(matches) == 1 else None
|
||||
if duplicate is None:
|
||||
matches = (
|
||||
match_query = (
|
||||
db.query(SourceRecord)
|
||||
.join(ParseRun, ParseRun.id == SourceRecord.parse_run_id)
|
||||
.join(StatementDocument, StatementDocument.id == SourceRecord.document_id)
|
||||
.join(SourceRecordLink, SourceRecordLink.source_record_id == SourceRecord.id)
|
||||
.filter(
|
||||
SourceRecord.space_id == document.space_id,
|
||||
SourceRecord.document_id != document.id,
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.record_fp == source.record_fp,
|
||||
SourceRecord.direction == source.direction,
|
||||
SourceRecord.bank_status == "posted",
|
||||
ParseRun.adapter_id == run.adapter_id,
|
||||
StatementDocument.status == "confirmed",
|
||||
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
if document.account_id is not None:
|
||||
match_query = match_query.filter(
|
||||
StatementDocument.account_id == document.account_id
|
||||
)
|
||||
matches = match_query.all()
|
||||
duplicate = matches[0] if len(matches) == 1 else None
|
||||
if duplicate is not None:
|
||||
link = db.query(SourceRecordLink).filter_by(source_record_id=duplicate.id).one_or_none()
|
||||
link = db.query(SourceRecordLink).filter_by(source_record_id=duplicate.id).one()
|
||||
db.add(
|
||||
DedupCandidate(
|
||||
space_id=document.space_id,
|
||||
source_record_id=source.id,
|
||||
candidate_kind="duplicate",
|
||||
existing_source_record_id=duplicate.id,
|
||||
existing_operation_id=link.operation_id if link else None,
|
||||
existing_operation_id=link.operation_id,
|
||||
confidence="strong" if source.external_id else "possible",
|
||||
)
|
||||
)
|
||||
@@ -272,20 +292,28 @@ def _build_candidates(db: Session, *, document: StatementDocument, run: ParseRun
|
||||
if source.occurred_on is None:
|
||||
continue
|
||||
opposite = "in" if source.direction == "out" else "out"
|
||||
transfer_matches = (
|
||||
transfer_query = (
|
||||
db.query(SourceRecord)
|
||||
.join(SourceRecordLink, SourceRecordLink.source_record_id == SourceRecord.id)
|
||||
.join(StatementDocument, StatementDocument.id == SourceRecord.document_id)
|
||||
.filter(
|
||||
SourceRecord.space_id == document.space_id,
|
||||
SourceRecord.document_id != document.id,
|
||||
SourceRecord.id != source.id,
|
||||
SourceRecord.amount_kopecks == source.amount_kopecks,
|
||||
SourceRecord.direction == opposite,
|
||||
SourceRecord.bank_status == "posted",
|
||||
SourceRecord.occurred_on >= source.occurred_on - timedelta(days=1),
|
||||
SourceRecord.occurred_on <= source.occurred_on + timedelta(days=1),
|
||||
StatementDocument.status == "confirmed",
|
||||
StatementDocument.active_parse_run_id == SourceRecord.parse_run_id,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
if document.account_id is not None:
|
||||
transfer_query = transfer_query.filter(
|
||||
StatementDocument.account_id != document.account_id
|
||||
)
|
||||
transfer_matches = transfer_query.all()
|
||||
if len(transfer_matches) == 1:
|
||||
counterpart = transfer_matches[0]
|
||||
link = db.query(SourceRecordLink).filter_by(source_record_id=counterpart.id).one()
|
||||
@@ -504,14 +532,15 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
|
||||
.order_by(SourceRecord.created_at)
|
||||
.all()
|
||||
)
|
||||
hold_ids = [record.id for record in records]
|
||||
record_ids = [record.id for record in records]
|
||||
resolutions: dict[uuid.UUID, dict[str, Any]] = {}
|
||||
if hold_ids:
|
||||
candidates: dict[uuid.UUID, list[dict[str, Any]]] = {}
|
||||
if record_ids:
|
||||
for item in (
|
||||
db.query(HoldResolution)
|
||||
.filter(
|
||||
HoldResolution.space_id == space_id,
|
||||
HoldResolution.hold_source_record_id.in_(hold_ids),
|
||||
HoldResolution.hold_source_record_id.in_(record_ids),
|
||||
)
|
||||
.all()
|
||||
):
|
||||
@@ -523,6 +552,24 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
|
||||
"reason": item.reason,
|
||||
"resolved_at": item.resolved_at,
|
||||
}
|
||||
for candidate in (
|
||||
db.query(DedupCandidate)
|
||||
.filter(
|
||||
DedupCandidate.space_id == space_id,
|
||||
DedupCandidate.source_record_id.in_(record_ids),
|
||||
)
|
||||
.order_by(DedupCandidate.created_at)
|
||||
.all()
|
||||
):
|
||||
candidates.setdefault(candidate.source_record_id, []).append(
|
||||
{
|
||||
"id": candidate.id,
|
||||
"kind": candidate.candidate_kind,
|
||||
"confidence": candidate.confidence,
|
||||
"resolution": candidate.resolution,
|
||||
"existing_operation_id": candidate.existing_operation_id,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"id": document.id,
|
||||
"status": document.status,
|
||||
@@ -546,6 +593,7 @@ def document_detail(db: Session, *, space_id: uuid.UUID, document_id: uuid.UUID)
|
||||
"review_decision": item.review_decision,
|
||||
"duplicate_operation_id": item.duplicate_operation_id,
|
||||
"hold_resolution": resolutions.get(item.id),
|
||||
"candidates": candidates.get(item.id, []),
|
||||
}
|
||||
for item in records
|
||||
],
|
||||
@@ -783,15 +831,8 @@ def confirm_document(
|
||||
db.refresh(document, with_for_update=True)
|
||||
if document.status == "confirmed":
|
||||
return document
|
||||
if document.status != "ready" or document.account_id is None:
|
||||
if document.status not in {"ready", "review_required"} or document.account_id is None:
|
||||
raise conflict("IMPORT_NOT_READY", "Import review is not complete")
|
||||
reconciliation = document.reconciliation or {}
|
||||
if not reconciliation.get("full") and not reconciliation.get(
|
||||
"confirmed_without_full_reconciliation"
|
||||
):
|
||||
raise conflict("IMPORT_RECONCILIATION_REQUIRED", "Reconciliation must be confirmed")
|
||||
if reconciliation.get("matched") is False:
|
||||
raise conflict("IMPORT_RECONCILIATION_MISMATCH", "Statement arithmetic does not reconcile")
|
||||
pending = (
|
||||
db.query(DedupCandidate.id)
|
||||
.join(SourceRecord, SourceRecord.id == DedupCandidate.source_record_id)
|
||||
@@ -806,6 +847,15 @@ def confirm_document(
|
||||
records = db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).all()
|
||||
if any(item.critical for item in records):
|
||||
raise conflict("IMPORT_FIELDS_UNRESOLVED", "Critical source fields remain unresolved")
|
||||
reconciliation = document.reconciliation or {}
|
||||
if not reconciliation.get("full") and not reconciliation.get(
|
||||
"confirmed_without_full_reconciliation"
|
||||
):
|
||||
raise conflict("IMPORT_RECONCILIATION_REQUIRED", "Reconciliation must be confirmed")
|
||||
if reconciliation.get("matched") is False:
|
||||
raise conflict("IMPORT_RECONCILIATION_MISMATCH", "Statement arithmetic does not reconcile")
|
||||
if document.status != "ready":
|
||||
raise conflict("IMPORT_NOT_READY", "Import review is not complete")
|
||||
run = db.get(ParseRun, document.active_parse_run_id)
|
||||
if run is not None:
|
||||
_apply_hold_matches(
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.errors import AppError
|
||||
from app.identity.models import User
|
||||
from app.imports import router as imports_router
|
||||
from app.imports import service
|
||||
from app.imports.adapters.base import SourceRow
|
||||
from app.imports.models import (
|
||||
@@ -217,6 +219,259 @@ def test_duplicate_candidate_can_reuse_unique_operation_without_client_id(db: Se
|
||||
assert db.query(Operation).filter_by(space_id=space.id).count() == 1
|
||||
|
||||
|
||||
def test_duplicate_candidates_ignore_rows_from_same_parse_run(db: Session) -> None:
|
||||
user, space, document = ready_document(db)
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
first_source = (
|
||||
db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).one()
|
||||
)
|
||||
existing_link = db.query(SourceRecordLink).filter_by(source_record_id=first_source.id).one()
|
||||
second_source = SourceRecord(
|
||||
space_id=first_source.space_id,
|
||||
document_id=document.id,
|
||||
parse_run_id=document.active_parse_run_id,
|
||||
occurred_on=first_source.occurred_on,
|
||||
amount_kopecks=first_source.amount_kopecks,
|
||||
direction=first_source.direction,
|
||||
row_ref="sheet0:row3",
|
||||
record_fp=first_source.record_fp,
|
||||
)
|
||||
db.add(second_source)
|
||||
db.flush()
|
||||
db.add(
|
||||
SourceRecordLink(
|
||||
space_id=space.id,
|
||||
source_record_id=second_source.id,
|
||||
operation_id=existing_link.operation_id,
|
||||
kind="duplicate",
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
run = db.get(ParseRun, document.active_parse_run_id)
|
||||
assert run is not None
|
||||
assert service._build_candidates(db, document=document, run=run) is False
|
||||
assert db.query(DedupCandidate).filter_by(space_id=document.space_id).count() == 0
|
||||
|
||||
|
||||
def test_duplicate_candidates_ignore_inactive_parse_runs(db: Session) -> None:
|
||||
user, space, source_document = ready_document(db)
|
||||
service.confirm_document(db, space_id=space.id, document_id=source_document.id, user=user)
|
||||
active_source = (
|
||||
db.query(SourceRecord).filter_by(parse_run_id=source_document.active_parse_run_id).one()
|
||||
)
|
||||
active_link = db.query(SourceRecordLink).filter_by(source_record_id=active_source.id).one()
|
||||
inactive_run = ParseRun(
|
||||
space_id=space.id,
|
||||
document_id=source_document.id,
|
||||
adapter_id="alfa_debit_xlsx",
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(inactive_run)
|
||||
db.flush()
|
||||
inactive_source = SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=source_document.id,
|
||||
parse_run_id=inactive_run.id,
|
||||
occurred_on=active_source.occurred_on,
|
||||
amount_kopecks=active_source.amount_kopecks,
|
||||
direction=active_source.direction,
|
||||
row_ref="sheet0:old-row",
|
||||
record_fp="inactive-run-fingerprint",
|
||||
)
|
||||
db.add(inactive_source)
|
||||
db.flush()
|
||||
db.add(
|
||||
SourceRecordLink(
|
||||
space_id=space.id,
|
||||
source_record_id=inactive_source.id,
|
||||
operation_id=active_link.operation_id,
|
||||
kind="duplicate",
|
||||
)
|
||||
)
|
||||
target_document = StatementDocument(
|
||||
space_id=space.id,
|
||||
account_id=source_document.account_id,
|
||||
original_filename="target.xlsx",
|
||||
size_bytes=100,
|
||||
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
|
||||
object_key=uuid.uuid4().hex,
|
||||
status="review_required",
|
||||
reconciliation={"full": True, "matched": True},
|
||||
)
|
||||
db.add(target_document)
|
||||
db.flush()
|
||||
target_run = ParseRun(
|
||||
space_id=space.id,
|
||||
document_id=target_document.id,
|
||||
adapter_id="alfa_debit_xlsx",
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(target_run)
|
||||
db.flush()
|
||||
target_document.active_parse_run_id = target_run.id
|
||||
db.add(
|
||||
SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=target_document.id,
|
||||
parse_run_id=target_run.id,
|
||||
occurred_on=inactive_source.occurred_on,
|
||||
amount_kopecks=inactive_source.amount_kopecks,
|
||||
direction=inactive_source.direction,
|
||||
row_ref="sheet0:row2",
|
||||
record_fp=inactive_source.record_fp,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
assert service._build_candidates(db, document=target_document, run=target_run) is False
|
||||
assert db.query(DedupCandidate).filter_by(space_id=space.id).count() == 0
|
||||
|
||||
|
||||
def test_duplicate_candidates_ignore_unconfirmed_documents(db: Session) -> None:
|
||||
_user, space, first_document = ready_document(db)
|
||||
first_source = (
|
||||
db.query(SourceRecord).filter_by(parse_run_id=first_document.active_parse_run_id).one()
|
||||
)
|
||||
second_document = StatementDocument(
|
||||
space_id=space.id,
|
||||
account_id=first_document.account_id,
|
||||
original_filename="second.xlsx",
|
||||
size_bytes=100,
|
||||
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
|
||||
object_key=uuid.uuid4().hex,
|
||||
status="review_required",
|
||||
reconciliation={"full": True, "matched": True},
|
||||
)
|
||||
db.add(second_document)
|
||||
db.flush()
|
||||
second_run = ParseRun(
|
||||
space_id=space.id,
|
||||
document_id=second_document.id,
|
||||
adapter_id="alfa_debit_xlsx",
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(second_run)
|
||||
db.flush()
|
||||
second_document.active_parse_run_id = second_run.id
|
||||
db.add(
|
||||
SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=second_document.id,
|
||||
parse_run_id=second_run.id,
|
||||
occurred_on=first_source.occurred_on,
|
||||
amount_kopecks=first_source.amount_kopecks,
|
||||
direction=first_source.direction,
|
||||
row_ref="sheet0:row2",
|
||||
record_fp=first_source.record_fp,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
assert service._build_candidates(db, document=second_document, run=second_run) is False
|
||||
assert db.query(DedupCandidate).filter_by(space_id=space.id).count() == 0
|
||||
|
||||
|
||||
def test_duplicate_candidates_are_retained_before_account_mapping(db: Session) -> None:
|
||||
user, space, first_document = ready_document(db)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first_document.id, user=user)
|
||||
first_source = (
|
||||
db.query(SourceRecord).filter_by(parse_run_id=first_document.active_parse_run_id).one()
|
||||
)
|
||||
second_document = StatementDocument(
|
||||
space_id=space.id,
|
||||
account_id=None,
|
||||
original_filename="unmapped.xlsx",
|
||||
size_bytes=100,
|
||||
content_sha256=uuid.uuid4().hex + uuid.uuid4().hex,
|
||||
object_key=uuid.uuid4().hex,
|
||||
status="review_required",
|
||||
reconciliation={"full": True, "matched": True},
|
||||
)
|
||||
db.add(second_document)
|
||||
db.flush()
|
||||
second_run = ParseRun(
|
||||
space_id=space.id,
|
||||
document_id=second_document.id,
|
||||
adapter_id="alfa_debit_xlsx",
|
||||
status="succeeded",
|
||||
)
|
||||
db.add(second_run)
|
||||
db.flush()
|
||||
second_document.active_parse_run_id = second_run.id
|
||||
source = SourceRecord(
|
||||
space_id=space.id,
|
||||
document_id=second_document.id,
|
||||
parse_run_id=second_run.id,
|
||||
occurred_on=first_source.occurred_on,
|
||||
amount_kopecks=first_source.amount_kopecks,
|
||||
direction=first_source.direction,
|
||||
row_ref="sheet0:row2",
|
||||
record_fp=first_source.record_fp,
|
||||
)
|
||||
db.add(source)
|
||||
db.flush()
|
||||
|
||||
assert service._build_candidates(db, document=second_document, run=second_run) is True
|
||||
candidate = db.query(DedupCandidate).filter_by(source_record_id=source.id).one()
|
||||
assert candidate.existing_source_record_id == first_source.id
|
||||
|
||||
|
||||
def test_import_detail_exposes_pending_candidate_and_blocker(db: Session) -> None:
|
||||
_user, space, document = ready_document(db)
|
||||
document.status = "review_required"
|
||||
source = db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).one()
|
||||
candidate = DedupCandidate(
|
||||
space_id=space.id,
|
||||
source_record_id=source.id,
|
||||
candidate_kind="duplicate",
|
||||
confidence="possible",
|
||||
)
|
||||
db.add(candidate)
|
||||
db.flush()
|
||||
|
||||
response = imports_router.detail(document.id, db, SimpleNamespace(space=space))
|
||||
|
||||
assert response["rows"][0]["decision"] == "pending"
|
||||
assert response["rows"][0]["candidates"] == [
|
||||
{
|
||||
"id": candidate.id,
|
||||
"kind": "duplicate",
|
||||
"confidence": "possible",
|
||||
"resolution": "pending",
|
||||
"existingOperationId": None,
|
||||
}
|
||||
]
|
||||
assert response["blockers"] == [
|
||||
{
|
||||
"code": "IMPORT_DECISION_REQUIRED",
|
||||
"message": "Possible duplicate or transfer requires a decision",
|
||||
"rowId": str(source.id),
|
||||
}
|
||||
]
|
||||
assert response["totals"]["pending"] == 1
|
||||
|
||||
|
||||
def test_confirm_reports_pending_candidate_instead_of_generic_not_ready(db: Session) -> None:
|
||||
user, space, document = ready_document(db)
|
||||
document.status = "review_required"
|
||||
source = db.query(SourceRecord).filter_by(parse_run_id=document.active_parse_run_id).one()
|
||||
db.add(
|
||||
DedupCandidate(
|
||||
space_id=space.id,
|
||||
source_record_id=source.id,
|
||||
candidate_kind="duplicate",
|
||||
confidence="possible",
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
with pytest.raises(AppError) as exc:
|
||||
service.confirm_document(db, space_id=space.id, document_id=document.id, user=user)
|
||||
|
||||
assert exc.value.code == "IMPORT_DECISIONS_REQUIRED"
|
||||
|
||||
|
||||
def test_transfer_merge_revert_restores_counterpart_operation(db: Session) -> None:
|
||||
user, space, first_document = ready_document(db)
|
||||
service.confirm_document(db, space_id=space.id, document_id=first_document.id, user=user)
|
||||
|
||||
Reference in New Issue
Block a user