diff --git a/README.md b/README.md index a6e0646..b98fcfa 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ flowchart LR - **Local-first ledger** — SQLite is the source of truth; no hosted telemetry or cloud model requirement is implemented. - **Deterministic finance engine** — income, spending, savings rate, budgets, goals, recurring evidence, anomalies, trends, and scenarios are calculated in Python from integer minor units. -- **Statement import studio** — upload CSV or PDF statements in the browser, preview extracted rows, +- **Statement import studio** — upload CSV, HDFC India Delimited `.txt`, or PDF statements in the browser, preview extracted rows, review warnings/errors, exclude rows, state the statement's sign convention, and commit explicitly. Commit is blocked while any included row has a blocking error or the sign convention is unanswered. - **PDF + OCR path** — digital PDFs use `pdfplumber`; scanned PDF pages can fall back to local Tesseract OCR when installed. @@ -131,13 +131,13 @@ PFA is strict about money because small mistakes corrupt advice: | Path | Supported now | Notes | | --- | --- | --- | | CLI | Local UTF-8 CSV files | `uv run pfa import `; `--dry-run` validates without persistence. | -| Browser/API preview | CSV and PDF uploads | `POST /imports/preview` stages a bounded local upload, extracts candidates, and deletes raw uploaded bytes after extraction. | +| Browser/API preview | CSV, HDFC Delimited `.txt`, and PDF uploads | `POST /imports/preview` stages a bounded local upload, extracts candidates, and deletes raw uploaded bytes after extraction. | | Digital PDF | Yes, best-effort | Uses `pdfplumber` table/word extraction with source-page provenance. | | Scanned PDF | Basic local OCR fallback | Requires Tesseract installed on `PATH`; OCR-derived rows carry review warnings, and low-confidence date/amount fields block commit. | Upload limits default to 15 MiB, 100 PDF pages, 10,000 candidate rows, and a 24-hour TTL for uncommitted normalized batches. Committed batches keep metadata and transaction IDs, not raw statement bytes. -CSV imports accept common aliases for date, description, amount, account, and transaction ID. They support signed `amount` columns, debit/credit columns, comma/semicolon/tab delimiters, UTF-8 BOM, row-level errors, duplicate detection, and manual review for unresolved classifications. +CSV imports accept common aliases for date, description, amount, account, and transaction ID. They support signed `amount` columns, debit/credit columns, comma/semicolon/tab delimiters, UTF-8 BOM, row-level errors, duplicate detection, and manual review for unresolved classifications. HDFC India Delimited exports are content-detected from their exact seven-column header, require a confirmed INR Current/Savings account, and reconcile ordered closing balances before commit. For unsigned credit-card-style exports, the preview API supports an explicit `amount_sign` patch (`as_written` or `debit_positive`) so PFA does not silently guess whether positive values are purchases or credits. diff --git a/alembic/versions/0007_adapter_currency_metadata.py b/alembic/versions/0007_adapter_currency_metadata.py new file mode 100644 index 0000000..34b003a --- /dev/null +++ b/alembic/versions/0007_adapter_currency_metadata.py @@ -0,0 +1,21 @@ +"""persist generic statement-adapter currency metadata""" + +import sqlalchemy as sa +from alembic import op + +revision = "0007_adapter_currency_metadata" +down_revision = "0006_transfer_events" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("import_batches", sa.Column("suggested_currency", sa.String(length=3))) + op.add_column("import_batches", sa.Column("currency_evidence", sa.String(length=40))) + op.add_column("import_batches", sa.Column("compatible_account_types_json", sa.Text())) + + +def downgrade() -> None: + op.drop_column("import_batches", "compatible_account_types_json") + op.drop_column("import_batches", "currency_evidence") + op.drop_column("import_batches", "suggested_currency") diff --git a/src/pfa/api/app.py b/src/pfa/api/app.py index 6b0bd65..7643213 100644 --- a/src/pfa/api/app.py +++ b/src/pfa/api/app.py @@ -97,6 +97,12 @@ class NewAccountRequest(BaseModel): last4: str | None = Field(default=None, min_length=4, max_length=4, pattern=r"^\d{4}$") opening_balance_minor: int = 0 opening_balance_as_of: date | None = None + opening_balance_confirmed: bool = False + currency_confirmed: bool = False + + +class AccountMetadataUpdateRequest(BaseModel): + institution: Literal["hdfc_bank"] class CandidateIssueResponse(BaseModel): @@ -113,6 +119,7 @@ class CandidateResponse(BaseModel): normalized_description: str amount_minor: int | None direction: str | None + direction_explicit: bool currency: str account_hint: str | None account_id: int | None @@ -148,6 +155,9 @@ class ImportBatchResponse(BaseModel): detection_reason_codes: list[str] detected_institution: str | None detected_account_hint: str | None + suggested_currency: str | None + currency_evidence: str | None + compatible_account_types: list[str] reconciliation: dict[str, object] | None semantic_totals: dict[str, int] amount_sign: str | None @@ -170,6 +180,7 @@ class ImportBatchPatchRequest(BaseModel): account: str | None = None # deprecated label compatibility destination_account_id: int | None = Field(default=None, gt=0) new_account: NewAccountRequest | None = None + account_metadata_update: AccountMetadataUpdateRequest | None = None excluded_candidate_ids: list[str] | None = None @model_validator(mode="after") @@ -177,9 +188,13 @@ def one_binding(self) -> ImportBatchPatchRequest: if self.destination_account_id is not None and self.new_account is not None: raise ValueError("choose destination_account_id or new_account, not both") if self.account is not None and ( - self.destination_account_id is not None or self.new_account is not None + self.destination_account_id is not None + or self.new_account is not None + or self.account_metadata_update is not None ): raise ValueError("account is a legacy alias; use one stable binding") + if self.account_metadata_update is not None and self.destination_account_id is None: + raise ValueError("account_metadata_update requires destination_account_id") return self # Both are closed sets: an unrecognised value is a 422, not a silent no-op. @@ -264,6 +279,7 @@ def _candidate_response(candidate: CandidateTransaction) -> CandidateResponse: normalized_description=candidate.normalized_description, amount_minor=candidate.amount_minor, direction=candidate.direction, + direction_explicit=candidate.direction_explicit, currency=candidate.currency, account_hint=candidate.account_hint, account_id=candidate.account_id, @@ -309,6 +325,13 @@ def _batch_response(batch: ImportBatchModel) -> ImportBatchResponse: ), detected_institution=batch.detected_institution, detected_account_hint=batch.detected_account_hint, + suggested_currency=batch.suggested_currency, + currency_evidence=batch.currency_evidence, + compatible_account_types=( + json.loads(batch.compatible_account_types_json) + if batch.compatible_account_types_json + else [] + ), reconciliation=( json.loads(batch.reconciliation_json) if batch.reconciliation_json else None ), @@ -482,6 +505,11 @@ def patch_import_batch(batch_id: str, request: ImportBatchPatchRequest) -> Impor else None ), excluded_candidate_ids=request.excluded_candidate_ids, + account_metadata_update=( + request.account_metadata_update.model_dump() + if request.account_metadata_update + else None + ), amount_mode=request.amount_mode, amount_sign=request.amount_sign, ), diff --git a/src/pfa/db/models.py b/src/pfa/db/models.py index 5b97f5f..4e0ca1e 100644 --- a/src/pfa/db/models.py +++ b/src/pfa/db/models.py @@ -145,6 +145,9 @@ class ImportBatchModel(Base): detection_reason_codes_json: Mapped[str | None] = mapped_column(Text, nullable=True) detected_institution: Mapped[str | None] = mapped_column(String(120), nullable=True) detected_account_hint: Mapped[str | None] = mapped_column(String(40), nullable=True) + suggested_currency: Mapped[str | None] = mapped_column(String(3), nullable=True) + currency_evidence: Mapped[str | None] = mapped_column(String(40), nullable=True) + compatible_account_types_json: Mapped[str | None] = mapped_column(Text, nullable=True) reconciliation_json: Mapped[str | None] = mapped_column(Text, nullable=True) # The sign convention the user declared for this source, kept so the preview can be # rebuilt after a refresh and so a committed batch records how it read its amounts. diff --git a/src/pfa/db/repositories.py b/src/pfa/db/repositories.py index 5b95f1a..9600742 100644 --- a/src/pfa/db/repositories.py +++ b/src/pfa/db/repositories.py @@ -103,11 +103,17 @@ def create( raise ValueError(f"unsupported account currency {currency!r}") if last4 is not None and (len(last4) != 4 or not last4.isdigit()): raise ValueError("last4 must contain exactly four digits") + institution_value = institution.strip() if institution else None + if institution_value and institution_value.casefold().replace(" ", "_") in { + "hdfc", + "hdfc_bank", + }: + institution_value = "hdfc_bank" account = AccountModel( name=name.strip(), currency=currency, account_type=account_type, - institution=institution.strip() if institution else None, + institution=institution_value, last4=last4, opening_balance_minor=opening_balance_minor, opening_balance_as_of=opening_balance_as_of, diff --git a/src/pfa/ingestion/batches.py b/src/pfa/ingestion/batches.py index 330bb64..c33768b 100644 --- a/src/pfa/ingestion/batches.py +++ b/src/pfa/ingestion/batches.py @@ -28,10 +28,13 @@ from .candidates import ( ACCOUNT_CURRENCY_MISMATCH, ACCOUNT_INACTIVE, + ACCOUNT_INSTITUTION_MISMATCH, + ACCOUNT_INSTITUTION_REQUIRED, ACCOUNT_NOT_FOUND, ACCOUNT_REQUIRED, ACCOUNT_TYPE_MISMATCH, AMBIGUOUS_SIGN, + BALANCE_RECONCILIATION_FAILED, BATCH_ALREADY_COMMITTED, BATCH_EXPIRED, BATCH_HAS_BLOCKING_ERRORS, @@ -43,6 +46,7 @@ EXTRACTION_TIMEOUT, GENERIC_SIGN_CONFIRMATION_REQUIRED, INVALID_ACCOUNT_DRAFT, + INVALID_ACCOUNT_METADATA_UPDATE, NO_USABLE_ROWS, RECONCILIATION_INCOMPLETE, RECONCILIATION_MISMATCH, @@ -63,6 +67,7 @@ ) from .dialects import DIALECTS, Dialect, detect_adapter from .extractors.csv import CsvStatementExtractor +from .extractors.hdfc import HdfcDelimitedExtractor from .extractors.ocr import OcrFallbackPdfExtractor from .extractors.pdf import clean_amount_text @@ -86,6 +91,8 @@ class NewAccountDraft: last4: str | None = None opening_balance_minor: int = 0 opening_balance_as_of: date | None = None + opening_balance_confirmed: bool = False + currency_confirmed: bool = False def as_dict(self) -> dict[str, object]: return { @@ -98,6 +105,8 @@ def as_dict(self) -> dict[str, object]: "opening_balance_as_of": self.opening_balance_as_of.isoformat() if self.opening_balance_as_of else None, + "opening_balance_confirmed": self.opening_balance_confirmed, + "currency_confirmed": self.currency_confirmed, } @classmethod @@ -112,6 +121,8 @@ def from_dict(cls, value: dict[str, object]) -> NewAccountDraft: last4=str(value["last4"]) if value.get("last4") else None, opening_balance_minor=int(str(opening)), opening_balance_as_of=date.fromisoformat(str(as_of)) if as_of else None, + opening_balance_confirmed=bool(value.get("opening_balance_confirmed", False)), + currency_confirmed=bool(value.get("currency_confirmed", False)), ) @@ -123,6 +134,7 @@ class BatchPatch: excluded_candidate_ids: list[str] | None = None amount_mode: str | None = None amount_sign: str | None = None + account_metadata_update: dict[str, str] | None = None def _now() -> datetime: @@ -165,11 +177,18 @@ def batch_semantic_totals(batch: ImportBatchModel) -> dict[str, int]: saved = json.loads(batch.reconciliation_json).get("semantic_totals") if isinstance(saved, dict): return {str(key): int(value) for key, value in saved.items()} - spending = refunds = transfers = repayments = money_in = 0 + spending = refunds = transfers = repayments = money_in = money_out = 0 + money_in_count = money_out_count = 0 for candidate in batch_candidates(batch): signed = candidate.signed_amount_minor if signed is None or not candidate.included: continue + if signed > 0: + money_in += signed + money_in_count += 1 + elif signed < 0: + money_out += abs(signed) + money_out_count += 1 description = candidate.raw_description.upper() kind = candidate.kind if kind is None: @@ -192,10 +211,11 @@ def batch_semantic_totals(batch: ImportBatchModel) -> dict[str, int]: "PAYMENT RECEIVED" in description and signed > 0 ): repayments += abs(signed) - elif kind == "income": - money_in += max(signed, 0) return { "money_in_minor": money_in, + "money_out_minor": money_out, + "money_in_count": money_in_count, + "money_out_count": money_out_count, "spending_minor": spending, "refunds_minor": refunds, "transfers_minor": transfers, @@ -210,6 +230,11 @@ def _extractor_for( account_currency: str = "GBP", ) -> StatementExtractor: """Picks only the extraction engine; statement semantics come from content detection.""" + if dialect.adapter_id == "hdfc_in_delimited_v1": + return HdfcDelimitedExtractor( + max_candidate_rows=settings.max_candidate_rows, + dialect=dialect, + ) if source.path.suffix.lower() == ".pdf": return OcrFallbackPdfExtractor( settings=settings, @@ -302,8 +327,12 @@ def _normalize_dates( ACCOUNT_NOT_FOUND, ACCOUNT_REQUIRED, ACCOUNT_TYPE_MISMATCH, + ACCOUNT_INSTITUTION_MISMATCH, + ACCOUNT_INSTITUTION_REQUIRED, + BALANCE_RECONCILIATION_FAILED, DUPLICATE_ACCOUNT_SUSPECTED, INVALID_ACCOUNT_DRAFT, + INVALID_ACCOUNT_METADATA_UPDATE, GENERIC_SIGN_CONFIRMATION_REQUIRED, RECONCILIATION_INCOMPLETE, RECONCILIATION_MISMATCH, @@ -320,6 +349,20 @@ def _draft_from_batch(batch: ImportBatchModel) -> NewAccountDraft | None: return None +def _institution_key(value: str | None) -> str: + return (value or "").strip().casefold().replace(" ", "_") + + +def _hdfc_opening_suggestion(batch: ImportBatchModel) -> dict[str, object] | None: + if not batch.reconciliation_json: + return None + try: + value = json.loads(batch.reconciliation_json).get("opening_balance_suggestion") + except (TypeError, ValueError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + def _binding_issues( batch: ImportBatchModel, uow: UnitOfWork, dialect: Dialect ) -> list[CandidateIssue]: @@ -356,6 +399,30 @@ def _binding_issues( issues.append( CandidateIssue(INVALID_ACCOUNT_DRAFT, "account name and currency are invalid") ) + expected_currency = dialect.suggested_currency + if expected_currency and draft.currency.upper() != expected_currency: + issues.append( + CandidateIssue( + ACCOUNT_CURRENCY_MISMATCH, + f"this statement suggests {expected_currency}; confirm that currency", + ) + ) + if expected_currency and not draft.currency_confirmed: + issues.append( + CandidateIssue( + INVALID_ACCOUNT_DRAFT, + f"confirm {expected_currency} as the account currency before importing", + ) + ) + if dialect.institution and _institution_key(draft.institution) != _institution_key( + dialect.institution + ): + issues.append( + CandidateIssue( + ACCOUNT_INSTITUTION_MISMATCH, + "new account institution does not match the statement", + ) + ) if draft.last4 is not None and (len(draft.last4) != 4 or not draft.last4.isdigit()): issues.append( CandidateIssue( @@ -417,15 +484,48 @@ def _binding_issues( f"this statement is for {expected}; selected account is {account.account_type}", ) ) - detected_currency = (batch.detected_currency or "GBP").upper() - if account.currency.upper() != detected_currency: + expected_currency = (dialect.suggested_currency or batch.detected_currency or "GBP").upper() + if account.currency.upper() != expected_currency: issues.append( CandidateIssue( ACCOUNT_CURRENCY_MISMATCH, - f"statement currency {detected_currency} does not match " + f"statement currency {expected_currency} does not match " f"account currency {account.currency}", ) ) + if dialect.institution: + if not account.institution: + issues.append( + CandidateIssue( + ACCOUNT_INSTITUTION_REQUIRED, + "confirm that the selected legacy account belongs to the " + "statement institution", + ) + ) + elif _institution_key(account.institution) != _institution_key(dialect.institution): + issues.append( + CandidateIssue( + ACCOUNT_INSTITUTION_MISMATCH, + "the selected account belongs to a different institution", + ) + ) + if dialect.adapter_id == "hdfc_in_delimited_v1" and draft is not None: + suggestion = _hdfc_opening_suggestion(batch) + if suggestion is not None: + expected_date = suggestion.get("as_of") + expected_minor = suggestion.get("balance_minor") + if ( + not draft.opening_balance_confirmed + or draft.opening_balance_as_of is None + or draft.opening_balance_as_of.isoformat() != expected_date + or draft.opening_balance_minor != expected_minor + ): + issues.append( + CandidateIssue( + INVALID_ACCOUNT_DRAFT, + "confirm the derived opening balance and date before creating this account", + ) + ) if ( batch.adapter_id == "generic" and batch.amount_sign is None @@ -471,12 +571,25 @@ def _set_reconciliation( result = reconcile_candidates(candidates, _reconciliation_account_type(batch, uow)) batch.reconciliation_json = json.dumps(result) if batch.adapter_id not in (None, "generic"): - if result["status"] == "mismatch": - return [CandidateIssue(RECONCILIATION_MISMATCH, "statement balances do not reconcile")] - if result["status"] == "incomplete": - return [ + issues: list[CandidateIssue] = [] + if result["arithmetic_integrity"] == "mismatch": + mismatch_rows = result.get("mismatch_source_rows", []) + message = "statement balances do not reconcile" + if mismatch_rows: + message += "; check source row(s) " + ", ".join(map(str, mismatch_rows)) + issues.append( + CandidateIssue( + BALANCE_RECONCILIATION_FAILED + if batch.adapter_id == "hdfc_in_delimited_v1" + else RECONCILIATION_MISMATCH, + message, + ) + ) + if result["coverage_integrity"] == "incomplete": + issues.append( CandidateIssue(RECONCILIATION_INCOMPLETE, "not every statement row is included") - ] + ) + return issues return [] @@ -508,14 +621,14 @@ def create_batch( if account and selected is None: selected = uow.accounts.get_by_name(account) destination_account_id = selected.id if selected is not None else None - account_currency = ( - selected.currency - if selected is not None - else (new_account.currency if new_account else "GBP") - ) detection = detect_adapter(source.path, source.media_type) dialect = detection.dialect + account_currency = ( + detection.suggested_currency + or (selected.currency if selected is not None else None) + or (new_account.currency if new_account else "GBP") + ) extractor = _extractor_for(source, settings, dialect, account_currency=account_currency) batch = ImportBatchModel( @@ -536,6 +649,11 @@ def create_batch( detection_reason_codes_json=json.dumps(list(detection.reason_codes)), detected_institution=detection.institution, detected_account_hint=detection.account_hint, + suggested_currency=detection.suggested_currency or dialect.suggested_currency, + currency_evidence=detection.currency_evidence or dialect.currency_evidence, + compatible_account_types_json=json.dumps( + sorted(item.value for item in dialect.compatible_account_types) + ), amount_sign=dialect.default_sign, issues_json="[]", counts_json=json.dumps(_counts([])), @@ -604,9 +722,17 @@ def create_batch( batch.statement_end = max(parsed_dates) batch.detected_account = extraction.detected_account - batch.detected_currency = extraction.detected_currency or account_currency + # HDFC's INR is an adapter suggestion, not evidence read from the file. Keep the + # detected field empty so the UI must ask for confirmation. + batch.detected_currency = ( + extraction.detected_currency + if extraction.detected_currency is not None + else (None if detection.suggested_currency else account_currency) + ) batch.detected_institution = extraction.detected_institution or detection.institution batch.detected_account_hint = extraction.detected_account_hint or detection.account_hint + batch.suggested_currency = detection.suggested_currency or dialect.suggested_currency + batch.currency_evidence = detection.currency_evidence or dialect.currency_evidence batch.page_count = extraction.page_count batch.candidates_json = candidates_to_json(candidates) _set_batch_issues(batch, uow, dialect, extraction.issues) @@ -701,6 +827,30 @@ def apply_patch(uow: UnitOfWork, batch_id: str, patch: BatchPatch) -> ImportBatc 422, ) + if patch.account_metadata_update is not None: + update = patch.account_metadata_update + if ( + patch.destination_account_id is None + or batch.adapter_id != "hdfc_in_delimited_v1" + or set(update) != {"institution"} + or _institution_key(update.get("institution")) != "hdfc_bank" + ): + raise BatchError( + INVALID_ACCOUNT_METADATA_UPDATE, + "only a missing legacy institution may be marked as HDFC Bank", + 422, + ) + metadata_account = uow.accounts.get(patch.destination_account_id) + if metadata_account is None: + raise BatchError(ACCOUNT_NOT_FOUND, "select an existing account", 422) + if metadata_account.institution is not None: + raise BatchError( + INVALID_ACCOUNT_METADATA_UPDATE, + "institution correction is allowed only when the account institution is missing", + 422, + ) + metadata_account.institution = "hdfc_bank" + candidates = batch_candidates(batch) if patch.account is not None: diff --git a/src/pfa/ingestion/candidates.py b/src/pfa/ingestion/candidates.py index 8ad4845..5014b67 100644 --- a/src/pfa/ingestion/candidates.py +++ b/src/pfa/ingestion/candidates.py @@ -42,14 +42,22 @@ NO_HEADER_ROW = "NO_HEADER_ROW" HEADERLESS_CSV = "HEADERLESS_CSV" # warning: no header row, columns were read by position UNREADABLE_FILE = "UNREADABLE_FILE" -PDF_ENCRYPTED = "PDF_ENCRYPTED" +PDF_PASSWORD_REQUIRED = "PDF_PASSWORD_REQUIRED" +# Compatibility name retained for callers of the parent import slice. +PDF_ENCRYPTED = PDF_PASSWORD_REQUIRED PDF_TOO_MANY_PAGES = "PDF_TOO_MANY_PAGES" PDF_NOT_EXTRACTABLE = "PDF_NOT_EXTRACTABLE" TOO_MANY_ROWS = "TOO_MANY_ROWS" OCR_UNAVAILABLE = "OCR_UNAVAILABLE" +HDFC_HEADER_NOT_FOUND = "HDFC_HEADER_NOT_FOUND" +HDFC_ROW_WIDTH_INVALID = "HDFC_ROW_WIDTH_INVALID" +HDFC_AMOUNT_SIDES_INVALID = "HDFC_AMOUNT_SIDES_INVALID" # Upload and batch-lifecycle issue codes (T3). UNSUPPORTED_FILE_TYPE = "UNSUPPORTED_FILE_TYPE" +UNSUPPORTED_TEXT_FORMAT = "UNSUPPORTED_TEXT_FORMAT" +UNSUPPORTED_TEXT_LAYOUT = "UNSUPPORTED_TEXT_LAYOUT" +UNSUPPORTED_SPREADSHEET_FORMAT = "UNSUPPORTED_SPREADSHEET_FORMAT" INVALID_SIGNATURE = "INVALID_SIGNATURE" FILE_TOO_LARGE = "FILE_TOO_LARGE" UPLOAD_FAILED = "UPLOAD_FAILED" @@ -65,11 +73,15 @@ ACCOUNT_INACTIVE = "ACCOUNT_INACTIVE" ACCOUNT_TYPE_MISMATCH = "ACCOUNT_TYPE_MISMATCH" ACCOUNT_CURRENCY_MISMATCH = "ACCOUNT_CURRENCY_MISMATCH" +ACCOUNT_INSTITUTION_REQUIRED = "ACCOUNT_INSTITUTION_REQUIRED" +ACCOUNT_INSTITUTION_MISMATCH = "ACCOUNT_INSTITUTION_MISMATCH" +INVALID_ACCOUNT_METADATA_UPDATE = "INVALID_ACCOUNT_METADATA_UPDATE" DUPLICATE_ACCOUNT_SUSPECTED = "DUPLICATE_ACCOUNT_SUSPECTED" ACCOUNT_REQUIRED = "ACCOUNT_REQUIRED" INVALID_ACCOUNT_DRAFT = "INVALID_ACCOUNT_DRAFT" GENERIC_SIGN_CONFIRMATION_REQUIRED = "GENERIC_SIGN_CONFIRMATION_REQUIRED" RECONCILIATION_MISMATCH = "RECONCILIATION_MISMATCH" +BALANCE_RECONCILIATION_FAILED = "BALANCE_RECONCILIATION_FAILED" RECONCILIATION_INCOMPLETE = "RECONCILIATION_INCOMPLETE" UNDO_REQUIRES_CONFIRMATION = "UNDO_REQUIRES_CONFIRMATION" diff --git a/src/pfa/ingestion/dialects.py b/src/pfa/ingestion/dialects.py index 597a37e..e82bff5 100644 --- a/src/pfa/ingestion/dialects.py +++ b/src/pfa/ingestion/dialects.py @@ -1,6 +1,8 @@ from __future__ import annotations import csv +import io +import re from dataclasses import dataclass, replace from pathlib import Path @@ -27,6 +29,16 @@ class Dialect: two_column: bool = False compatible_account_types: frozenset[AccountType] = frozenset() institution: str | None = None + suggested_currency: str | None = None + currency_evidence: str | None = None + explicit_source_direction: bool = False + header_signature: tuple[str, ...] = () + + def header_matches(self, cells: list[str]) -> bool: + if not self.header_signature: + return False + normalized = tuple(re.sub(r"\s+", " ", cell.strip()).casefold() for cell in cells) + return normalized == self.header_signature @dataclass(frozen=True, slots=True) @@ -37,7 +49,19 @@ class AdapterDetection: institution: str | None = None account_hint: str | None = None currency: str | None = None - + suggested_currency: str | None = None + currency_evidence: str | None = None + + +HDFC_HEADERS = ( + "date", + "narration", + "value dat", + "debit amount", + "credit amount", + "chq/ref number", + "closing balance", +) GENERIC = Dialect() @@ -86,6 +110,18 @@ class AdapterDetection: compatible_account_types=frozenset({AccountType.CREDIT_CARD}), institution="Barclaycard", ) +HDFC_IN_DELIMITED = replace( + GENERIC, + name="hdfc_in_delimited", + adapter_id="hdfc_in_delimited_v1", + date_formats=("%d/%m/%Y", "%d/%m/%y"), + compatible_account_types=frozenset({AccountType.CURRENT, AccountType.SAVINGS}), + institution="hdfc_bank", + suggested_currency="INR", + currency_evidence="adapter_suggestion", + explicit_source_direction=True, + header_signature=HDFC_HEADERS, +) DIALECTS: dict[str, Dialect] = { "generic": GENERIC, @@ -96,6 +132,7 @@ class AdapterDetection: "hsbc_uk_current": HSBC_UK_CURRENT, "hsbc": HSBC_UK_CURRENT, "barclaycard": BARCLAYCARD, + "hdfc_in_delimited_v1": HDFC_IN_DELIMITED, } @@ -121,9 +158,19 @@ def _csv_detection(path: Path) -> AdapterDetection: text = _csv_text(path) lower = text.lower() try: - header = next(csv.reader(text.splitlines()), []) + reader = csv.reader(io.StringIO(text)) + header = next((row for row in reader if any(cell.strip() for cell in row)), []) except csv.Error: header = [] + if HDFC_IN_DELIMITED.header_matches(header): + return AdapterDetection( + HDFC_IN_DELIMITED, + 0.99, + ("hdfc_delimited_header", "explicit_source_columns"), + institution="hdfc_bank", + suggested_currency="INR", + currency_evidence="adapter_suggestion", + ) headers = {" ".join(cell.strip().lower().split()) for cell in header} if ( "card member" in lower diff --git a/src/pfa/ingestion/extractors/csv.py b/src/pfa/ingestion/extractors/csv.py index 5b90559..464fbaa 100644 --- a/src/pfa/ingestion/extractors/csv.py +++ b/src/pfa/ingestion/extractors/csv.py @@ -170,6 +170,10 @@ def __init__(self, dialect: Dialect = GENERIC, currency: str = "GBP") -> None: self.currency = currency def extract(self, source: StatementSource) -> ExtractionResult: + if self.dialect.adapter_id == "hdfc_in_delimited_v1": + from .hdfc import HdfcDelimitedExtractor + + return HdfcDelimitedExtractor(dialect=self.dialect).extract(source) result = ExtractionResult(extractor=self.name) positional = False try: diff --git a/src/pfa/ingestion/extractors/hdfc.py b/src/pfa/ingestion/extractors/hdfc.py new file mode 100644 index 0000000..2789da9 --- /dev/null +++ b/src/pfa/ingestion/extractors/hdfc.py @@ -0,0 +1,202 @@ +"""Strict HDFC India delimited statement extraction. + +HDFC's Delimited export is intentionally kept separate from the permissive generic CSV +reader. The seven-column header is the format contract; a file that does not match it is +not allowed to fall through to positional parsing. +""" + +from __future__ import annotations + +import csv +import io +import re +from collections.abc import Iterable +from decimal import Decimal, InvalidOperation + +from pfa.domain.errors import ValidationError +from pfa.domain.money import minor_units + +from ..candidates import ( + HDFC_AMOUNT_SIDES_INVALID, + HDFC_HEADER_NOT_FOUND, + HDFC_ROW_WIDTH_INVALID, + INVALID_AMOUNT, + NO_HEADER_ROW, + TOO_MANY_ROWS, + UNREADABLE_FILE, + CandidateIssue, + CandidateTransaction, + ExtractionResult, + StatementSource, +) +from ..dialects import HDFC_HEADERS, HDFC_IN_DELIMITED, Dialect + + +def _clean_decimal(value: str) -> tuple[Decimal, bool]: + text = value.strip() + negative = text.startswith("-") or text.startswith("−") + if text.startswith(("-", "−")): + text = text[1:].strip() + if text.startswith("(") and text.endswith(")"): + negative = True + text = text[1:-1].strip() + text = re.sub(r"(?i)^(?:inr|rs\.?|₹)", "", text).strip() + text = text.replace(",", "").replace("₹", "").strip() + try: + return Decimal(text), negative + except InvalidOperation as exc: + raise ValueError from exc + + +def _magnitude(value: str) -> int: + if not value.strip(): + return 0 + decimal, negative = _clean_decimal(value) + if negative or decimal < 0: + raise ValueError + return minor_units(decimal, "INR") + + +def _balance(value: str) -> int: + decimal, negative = _clean_decimal(value) + amount = minor_units(decimal, "INR") + return -amount if negative else amount + + +def _blank_row(cells: Iterable[str]) -> bool: + return not any(cell.strip() for cell in cells) + + +def _candidate( + cells: list[str], line_number: int, dialect: Dialect = HDFC_IN_DELIMITED +) -> CandidateTransaction: + date_text, narration, value_date, debit, credit, reference, closing = cells + raw_fields = { + "value_date": value_date.strip(), + "debit": debit.strip(), + "credit": credit.strip(), + "source_reference": reference.strip(), + "closing_balance": closing.strip(), + } + candidate = CandidateTransaction( + candidate_id=f"h{line_number}", + transaction_date=date_text.strip() or None, + posted_date=value_date.strip() or None, + raw_description=narration.strip(), + currency="INR", + source_format="csv", + source_line=line_number, + extraction_method="hdfc_in_delimited_v1", + raw_fields=raw_fields, + ) + + try: + debit_minor = _magnitude(debit) + credit_minor = _magnitude(credit) + _balance(closing) + except (ValueError, ValidationError): + candidate.add_issue( + INVALID_AMOUNT, + "debit, credit, and closing balance must be valid INR amounts", + ) + return candidate + + if (debit_minor > 0) == (credit_minor > 0): + candidate.add_issue( + HDFC_AMOUNT_SIDES_INVALID, + "exactly one of debit amount or credit amount must be positive", + ) + return candidate + + candidate.amount_minor = debit_minor or credit_minor + candidate.direction = "debit" if debit_minor > 0 else "credit" + candidate.direction_explicit = True + # ``direction`` is PFA's legacy normalized money-out/money-in value. The raw + # source columns remain in raw_fields for provenance, while the canonical sign + # is supplied by CandidateTransaction.signed_minor. + return candidate + + +class HdfcDelimitedExtractor: + """Reads only the exact seven-column HDFC Delimited export.""" + + name = "hdfc_in_delimited_v1" + + def __init__(self, *, max_candidate_rows: int = 10_000, dialect: Dialect = HDFC_IN_DELIMITED): + self.max_candidate_rows = max_candidate_rows + self.dialect = dialect + + def extract(self, source: StatementSource) -> ExtractionResult: + result = ExtractionResult(extractor=self.name, detected_institution="hdfc_bank") + try: + text = source.path.read_text(encoding="utf-8-sig") + except UnicodeDecodeError: + result.issues.append( + CandidateIssue(UNREADABLE_FILE, "file is not valid UTF-8 HDFC delimited text") + ) + return result + except OSError: + result.issues.append(CandidateIssue(UNREADABLE_FILE, "could not read the statement")) + return result + + reader = csv.reader(io.StringIO(text), strict=True) + header: list[str] | None = None + try: + for row in reader: + if not _blank_row(row): + header = row + break + except csv.Error: + result.issues.append(CandidateIssue(HDFC_HEADER_NOT_FOUND, "invalid CSV quoting")) + return result + if header is None: + result.issues.append(CandidateIssue(NO_HEADER_ROW, "CSV has no header row")) + return result + if not self.dialect.header_matches(header): + result.issues.append( + CandidateIssue( + HDFC_HEADER_NOT_FOUND, + "HDFC Delimited header was not found; download the Delimited format", + ) + ) + return result + + candidates: list[CandidateTransaction] = [] + try: + for row in reader: + line_number = reader.line_num + if _blank_row(row): + continue + if len(row) != len(HDFC_HEADERS): + candidate = CandidateTransaction( + candidate_id=f"h{line_number}", + source_format="csv", + source_line=line_number, + extraction_method=self.name, + raw_fields={"source_reference": row[5].strip() if len(row) > 5 else ""}, + ) + candidate.add_issue( + HDFC_ROW_WIDTH_INVALID, + "each HDFC Delimited data row must contain seven columns", + ) + else: + candidate = _candidate(row, line_number, self.dialect) + candidates.append(candidate) + if len(candidates) > self.max_candidate_rows: + result.issues.append( + CandidateIssue( + TOO_MANY_ROWS, + f"the statement exceeds the {self.max_candidate_rows}-row limit", + ) + ) + break + except csv.Error: + result.issues.append(CandidateIssue(HDFC_HEADER_NOT_FOUND, "invalid CSV quoting")) + return result + + result.candidates = candidates[: self.max_candidate_rows] + return result + + +def hdfc_delimited_header(cells: list[str]) -> bool: + return HDFC_IN_DELIMITED.header_matches(cells) diff --git a/src/pfa/ingestion/extractors/pdf.py b/src/pfa/ingestion/extractors/pdf.py index 1232071..d002608 100644 --- a/src/pfa/ingestion/extractors/pdf.py +++ b/src/pfa/ingestion/extractors/pdf.py @@ -623,7 +623,8 @@ def extract(self, source: StatementSource) -> ExtractionResult: result.issues.append( CandidateIssue( PDF_ENCRYPTED, - "PDF is password-protected; remove the password and re-upload", + "password-protected PDF import is not supported in this release; " + "remove the password and re-upload", ) ) return result diff --git a/src/pfa/ingestion/reconciliation.py b/src/pfa/ingestion/reconciliation.py index 83fa3f0..7b34c72 100644 --- a/src/pfa/ingestion/reconciliation.py +++ b/src/pfa/ingestion/reconciliation.py @@ -1,12 +1,14 @@ from __future__ import annotations +from datetime import timedelta from decimal import Decimal, InvalidOperation from typing import Any from pfa.domain.accounts import AccountType, account_nature +from pfa.domain.errors import ImportRowError from pfa.domain.money import minor_units -from .candidates import CandidateTransaction +from .candidates import CandidateTransaction, parse_date from .extractors.pdf import clean_amount_text @@ -19,19 +21,99 @@ def _balance_minor(value: str, currency: str) -> int | None: return -amount if negative else amount +def _coverage_pass(candidates: list[CandidateTransaction]) -> bool: + return all( + candidate.duplicate_of is not None or (candidate.included and candidate.state != "error") + for candidate in candidates + ) + + +def _hdfc_reconciliation(candidates: list[CandidateTransaction]) -> dict[str, Any]: + """Check HDFC's ordered asset-account closing-balance chain. + + The first row deterministically supplies a *suggested* end-of-day baseline. Every + later row is checked against the previous source closing balance; excluded rows are + still part of the arithmetic check, because removing one cannot repair bad evidence. + """ + coverage_pass = _coverage_pass(candidates) + rows: list[tuple[CandidateTransaction, int]] = [] + for candidate in candidates: + closing = _balance_minor(candidate.raw_fields.get("closing_balance", ""), "INR") + if closing is None or candidate.signed_amount_minor is None: + continue + rows.append((candidate, closing)) + + if len(rows) != len(candidates) or not rows: + return { + "arithmetic_integrity": "not_available", + "coverage_integrity": "pass" if coverage_pass else "incomplete", + "status": "not available" if coverage_pass else "incomplete", + "reconciled": False, + "checked_transition_count": 0, + "mismatch_count": 0, + "source_ordering": "preserved", + "coverage_complete": coverage_pass, + "evidence": "HDFC closing-balance evidence is incomplete", + } + + first, first_closing = rows[0] + try: + first_date = parse_date(first.transaction_date or "", "day_first") + except (ImportRowError, ValueError): + first_date = None + first_signed = first.signed_amount_minor + assert first_signed is not None + opening = first_closing - first_signed + suggestion = { + "balance_minor": opening, + "as_of": (first_date - timedelta(days=1)).isoformat() if first_date else None, + "provenance": "derived_from_first_row", + } + + mismatch_source_rows: list[int] = [] + for (_previous, previous_closing), (current, current_closing) in zip( + rows, rows[1:], strict=False + ): + signed = current.signed_amount_minor + assert signed is not None + if previous_closing + signed != current_closing: + if current.source_line is not None: + mismatch_source_rows.append(current.source_line) + + mismatch_count = len(mismatch_source_rows) + arithmetic_pass = mismatch_count == 0 + coverage = "pass" if coverage_pass else "incomplete" + status = "reconciled" if arithmetic_pass and coverage_pass else "mismatch" + if not coverage_pass: + status = "incomplete" + return { + "arithmetic_integrity": "pass" if arithmetic_pass else "mismatch", + "coverage_integrity": coverage, + "status": status, + "reconciled": arithmetic_pass and coverage_pass, + "checked_transition_count": max(len(rows) - 1, 0), + "mismatch_count": mismatch_count, + "mismatch_source_rows": mismatch_source_rows, + "source_ordering": "preserved", + "coverage_complete": coverage_pass, + "opening_balance_suggestion": suggestion, + "closing_balance_minor": rows[-1][1], + "currency": "INR", + "evidence": ( + f"{max(len(rows) - 1, 0)}/{max(len(rows) - 1, 0)} ordered balance transitions checked" + ), + } + + def reconcile_candidates( candidates: list[CandidateTransaction], account_type: AccountType | str, ) -> dict[str, Any]: - """Reconcile balance-chain evidence without changing the ledger. + """Reconcile balance-chain evidence without changing the ledger.""" + if any("closing_balance" in candidate.raw_fields for candidate in candidates): + return _hdfc_reconciliation(candidates) - A statement balance is a closing balance for its row. The first row therefore gives a - deterministic opening balance by subtracting that row's natural movement. - """ - coverage_pass = all( - candidate.included and (candidate.state != "error" or candidate.duplicate_of is not None) - for candidate in candidates - ) + coverage_pass = _coverage_pass(candidates) rows: list[tuple[CandidateTransaction, int]] = [] for candidate in candidates: if not candidate.included or candidate.duplicate_of is not None: @@ -41,9 +123,6 @@ def reconcile_candidates( continue if candidate.signed_amount_minor is None: continue - movement = candidate.signed_amount_minor - if account_nature(account_type) == "liability": - movement = -movement rows.append((candidate, balance)) if not rows: diff --git a/src/pfa/ingestion/upload.py b/src/pfa/ingestion/upload.py index 92fd409..7f03be4 100644 --- a/src/pfa/ingestion/upload.py +++ b/src/pfa/ingestion/upload.py @@ -1,6 +1,7 @@ """Multipart upload staging: bounded, streamed, and signature-checked. -Accepts CSV and PDF. Never trust the client-supplied filename as a path component; +Accepts CSV, UTF-8 delimited text, and PDF. Never trust the client-supplied filename as a +path component; the staged name is always generated. The extension decides which signature check runs and, downstream, which extractor the batch layer selects. """ @@ -20,13 +21,17 @@ FILE_TOO_LARGE, INVALID_SIGNATURE, UNSUPPORTED_FILE_TYPE, + UNSUPPORTED_SPREADSHEET_FORMAT, + UNSUPPORTED_TEXT_FORMAT, + UNSUPPORTED_TEXT_LAYOUT, UPLOAD_FAILED, StatementSource, ) +from .dialects import HDFC_IN_DELIMITED, detect_adapter CHUNK_SIZE = 64 * 1024 -SUPPORTED_EXTENSIONS = {".csv", ".pdf"} -DEFAULT_MEDIA_TYPES = {".csv": "text/csv", ".pdf": "application/pdf"} +SUPPORTED_EXTENSIONS = {".csv", ".pdf", ".txt"} +DEFAULT_MEDIA_TYPES = {".csv": "text/csv", ".pdf": "application/pdf", ".txt": "text/plain"} PDF_SIGNATURE = b"%PDF-" REJECTED_MEDIA_PREFIXES = ("image/",) @@ -46,10 +51,15 @@ def stage_upload( original_filename = Path(file.filename or "upload").name ext = Path(original_filename).suffix.lower() + if ext in {".xls", ".xlsx"}: + raise UploadRejected( + UNSUPPORTED_SPREADSHEET_FORMAT, + "Excel statements are not supported; for HDFC, download the Delimited format", + ) if ext not in SUPPORTED_EXTENSIONS: raise UploadRejected( UNSUPPORTED_FILE_TYPE, - f"unsupported file type {ext or '(none)'!r}; only .csv and .pdf are accepted", + f"unsupported file type {ext or '(none)'!r}; only .csv, .txt, and .pdf are accepted", ) media_type = file.content_type or "" # Lowercased: a blocklist that "Image/PNG" walks straight through is not a blocklist. @@ -90,10 +100,35 @@ def stage_upload( raise UploadRejected(INVALID_SIGNATURE, "file is not a valid PDF") else: try: - head.decode("utf-8-sig") + staged_path.read_text(encoding="utf-8-sig") except UnicodeDecodeError as exc: staged_path.unlink(missing_ok=True) - raise UploadRejected(INVALID_SIGNATURE, "file is not valid UTF-8 CSV text") from exc + raise UploadRejected( + INVALID_SIGNATURE, + "file is not valid UTF-8 delimited text", + ) from exc + if ext == ".txt": + detection = detect_adapter(staged_path, media_type) + if detection.dialect is not HDFC_IN_DELIMITED: + text = staged_path.read_text(encoding="utf-8-sig", errors="ignore")[:100_000] + lower = " ".join(text.lower().split()) + formatted_markers = ( + "statement of account" in lower + or "withdrawal amt" in lower + or "deposit amt" in lower + or "chq./ref.no" in lower + or ("value dt" in lower and "closing balance" in lower) + ) + staged_path.unlink(missing_ok=True) + if formatted_markers: + raise UploadRejected( + UNSUPPORTED_TEXT_LAYOUT, + "HDFC formatted text is not supported; download the Delimited format", + ) + raise UploadRejected( + UNSUPPORTED_TEXT_FORMAT, + "unrecognized text statement; for HDFC, download the Delimited format", + ) return StatementSource( path=staged_path, diff --git a/src/pfa/web/app.js b/src/pfa/web/app.js index 913f072..b8b8fda 100644 --- a/src/pfa/web/app.js +++ b/src/pfa/web/app.js @@ -394,9 +394,27 @@ function setupUploadHandlers() { const selected = $("destination-account-select").value; const newName = $("new-account-name").value.trim(); if ((!selected && !newName) || !state.activeBatch) return; + const isHdfc = state.activeBatch.adapter_id === "hdfc_in_delimited_v1"; + const opening = $("new-account-opening-balance")?.value; const body = selected - ? { destination_account_id: Number(selected) } - : { new_account: { name: newName, account_type: $("new-account-type").value, currency: state.activeBatch.detected_currency || "GBP" } }; + ? { + destination_account_id: Number(selected), + ...(isHdfc && $("mark-hdfc-account")?.checked + ? { account_metadata_update: { institution: "hdfc_bank" } } + : {}) + } + : { + new_account: { + name: newName, + account_type: $("new-account-type").value, + currency: isHdfc ? "INR" : (state.activeBatch.detected_currency || "GBP"), + institution: isHdfc ? "hdfc_bank" : null, + currency_confirmed: isHdfc ? $("confirm-account-currency").checked : false, + opening_balance_minor: isHdfc && opening ? Math.round(Number(opening) * 100) : 0, + opening_balance_as_of: isHdfc ? $("new-account-opening-as-of")?.value || null : null, + opening_balance_confirmed: isHdfc ? $("confirm-opening-balance").checked : false + } + }; try { const patched = await apiRequest(`/imports/${state.activeBatch.id}`, { method: "PATCH", @@ -404,7 +422,8 @@ function setupUploadHandlers() { body: JSON.stringify(body) }); state.activeBatch = patched; - showToast(`Assigned account "${acc}" to statement batch.`); + renderBatchInspector(patched); + showToast(`Assigned ${selected ? "the selected account" : `account "${newName}"`} to statement batch.`); } catch (err) { showToast(err.message, true); } @@ -430,6 +449,7 @@ function setupUploadHandlers() { if (!state.activeBatch) return; try { const committed = await apiRequest(`/imports/${state.activeBatch.id}/commit`, { method: "POST" }); + state.activeBatch = committed; $("batch-inspector").hidden = true; $("batch-success-card").hidden = false; $("nav-import-status").hidden = true; @@ -533,24 +553,31 @@ async function handleStatementUpload(file) { function renderBatchInspector(batch) { $("batch-filename").textContent = batch.original_filename; - $("batch-extractor").textContent = batch.extractor || "auto"; - $("batch-currency").textContent = batch.detected_currency || "GBP"; + const isHdfc = batch.adapter_id === "hdfc_in_delimited_v1"; + $("batch-extractor").textContent = isHdfc + ? "HDFC Bank · Delimited · High confidence" + : batch.extractor || "auto"; + const currency = batch.detected_currency || batch.suggested_currency || "GBP"; + $("batch-currency").textContent = isHdfc ? `${currency} — confirm` : currency; $("batch-pages").textContent = batch.page_count || "1"; $("batch-period").textContent = batch.statement_start && batch.statement_end ? `${batch.statement_start} to ${batch.statement_end}` : "Auto-detected"; $("destination-account-select").value = batch.destination_account_id ? String(batch.destination_account_id) : ""; $("new-account-name").value = batch.new_account?.name || ""; - $("new-account-type").value = batch.new_account?.account_type || "current"; + $("new-account-type").value = batch.new_account?.account_type || (isHdfc ? "current" : "current"); + renderHdfcBinding(batch); const semantic = batch.semantic_totals || {}; $("batch-semantic-summary").innerHTML = ` Import effect - Money in ${formatMoney(semantic.money_in_minor || 0, batch.detected_currency || "GBP")} - Spending ${formatMoney(semantic.spending_minor || 0, batch.detected_currency || "GBP")} - Refunds ${formatMoney(semantic.refunds_minor || 0, batch.detected_currency || "GBP")} - Transfers ${formatMoney(semantic.transfers_minor || 0, batch.detected_currency || "GBP")} - Repayments ${formatMoney(semantic.repayments_minor || 0, batch.detected_currency || "GBP")}`; - + ${semantic.money_out_count || 0} money out · ${formatMoney(semantic.money_out_minor || 0, currency)} + ${semantic.money_in_count || 0} money in · ${formatMoney(semantic.money_in_minor || 0, currency)} + Spending ${formatMoney(semantic.spending_minor || 0, currency)} + Refunds ${formatMoney(semantic.refunds_minor || 0, currency)} + Transfers ${formatMoney(semantic.transfers_minor || 0, currency)} + Repayments ${formatMoney(semantic.repayments_minor || 0, currency)}`; + + renderReconciliation(batch); // Amount sign selector: only generic formats may ask the user for semantics renderAmountSignSelector(batch); @@ -559,12 +586,69 @@ function renderBatchInspector(batch) { if (batch.issues && batch.issues.length > 0) { $("batch-issues-alert").hidden = false; - $("batch-issues-content").innerHTML = batch.issues.map((i) => `
${escapeHtml(i.code)}: ${escapeHtml(i.message)}
`).join(""); + $("batch-issues-content").innerHTML = batch.issues.map((i) => `
${escapeHtml(issueLabel(i))}: ${escapeHtml(i.message)}
`).join(""); } else { $("batch-issues-alert").hidden = true; } } +function renderHdfcBinding(batch) { + const isHdfc = batch.adapter_id === "hdfc_in_delimited_v1"; + const fields = $("hdfc-account-fields"); + const correction = $("hdfc-legacy-correction"); + if (!fields || !correction) return; + fields.hidden = !isHdfc; + correction.hidden = !isHdfc || !batch.destination_account_id || + Boolean(state.accounts.find((account) => account.id === batch.destination_account_id)?.institution); + if (!isHdfc) return; + + $("new-account-currency").value = batch.suggested_currency || "INR"; + $("new-account-institution").value = "hdfc_bank"; + const draft = batch.new_account || {}; + $("confirm-account-currency").checked = Boolean(draft.currency_confirmed); + $("confirm-opening-balance").checked = Boolean(draft.opening_balance_confirmed); + $("mark-hdfc-account").checked = false; + const suggestion = batch.reconciliation?.opening_balance_suggestion; + if (suggestion && !batch.new_account) { + $("new-account-opening-balance").value = (Number(suggestion.balance_minor) / 100).toFixed(2); + $("new-account-opening-as-of").value = suggestion.as_of || ""; + } else { + $("new-account-opening-balance").value = draft.opening_balance_minor === undefined ? "" : (Number(draft.opening_balance_minor) / 100).toFixed(2); + $("new-account-opening-as-of").value = draft.opening_balance_as_of || ""; + } + const type = $("new-account-type"); + if (type) { + type.innerHTML = ``; + type.value = draft.account_type || type.value || "current"; + } +} + +function renderReconciliation(batch) { + const target = $("batch-reconciliation"); + const result = batch.reconciliation; + if (!target || !result) return; + const status = result.status || "not available"; + const evidence = result.evidence || "No balance evidence available"; + const transitions = result.checked_transition_count === undefined ? "" : ` · ${result.checked_transition_count} transitions checked`; + target.innerHTML = `Reconciliation: ${escapeHtml(status)}${escapeHtml(evidence)}${escapeHtml(transitions)}`; +} + +function issueLabel(issue) { + const labels = { + ACCOUNT_REQUIRED: "Choose a compatible account", + ACCOUNT_TYPE_MISMATCH: "Account type does not match", + ACCOUNT_CURRENCY_MISMATCH: "Currency confirmation needed", + ACCOUNT_INSTITUTION_REQUIRED: "Confirm this account belongs to HDFC Bank", + ACCOUNT_INSTITUTION_MISMATCH: "The selected account belongs to another institution", + BALANCE_RECONCILIATION_FAILED: "Statement balance check failed", + RECONCILIATION_INCOMPLETE: "All statement rows must be included", + HDFC_AMOUNT_SIDES_INVALID: "Each row needs one money-out or money-in amount", + HDFC_ROW_WIDTH_INVALID: "A statement row has the wrong number of columns", + UNSUPPORTED_TEXT_LAYOUT: "Choose HDFC Delimited when downloading" + }; + return labels[issue.code] || issue.code; +} + function renderAmountSignSelector(batch) { const wrap = $("amount-sign-wrap"); if (!wrap) return; @@ -602,6 +686,7 @@ function updateBatchCounts(batch) { $("count-excluded").textContent = batch.counts.excluded || 0; const validToCommit = (batch.counts.valid || 0); + const duplicateOnlyCommit = (batch.counts.duplicate || 0) > 0 && batch.reconciliation?.status === "reconciled"; const candidates = batch.candidates || []; // Check for blocking errors on included candidates @@ -629,7 +714,7 @@ function updateBatchCounts(batch) { } else if (needsSign) { commitBtn.disabled = true; noteEl.textContent = "Set the amount sign convention before committing"; - } else if (validToCommit === 0) { + } else if (validToCommit === 0 && !duplicateOnlyCommit) { commitBtn.disabled = true; noteEl.textContent = "No valid transactions to commit"; } else { @@ -662,7 +747,7 @@ function renderCandidatesTable() { const isDebit = c.direction === "debit"; const amountStr = formatMoney(c.amount_minor, c.currency); const issues = c.issues || []; - const issueHtml = issues.map((i) => `${escapeHtml(i.code)}`).join(" "); + const issueHtml = issues.map((i) => `${escapeHtml(issueLabel(i))}`).join(" "); const dupHtml = c.duplicate_of ? `Duplicate of #${c.duplicate_of}` : ""; return ` @@ -680,7 +765,7 @@ function renderCandidatesTable() { ${isDebit ? `−${amountStr}` : `+${amountStr}`} - ${escapeHtml(c.extraction_method || "table")} + ${escapeHtml(c.extraction_method === "hdfc_in_delimited_v1" ? "Delimited" : c.extraction_method || "table")} ${issueHtml} ${dupHtml} diff --git a/src/pfa/web/index.html b/src/pfa/web/index.html index e48b2a6..d18dc3a 100644 --- a/src/pfa/web/index.html +++ b/src/pfa/web/index.html @@ -259,7 +259,8 @@

Import Statement

Drag & drop bank statement here

-

Supports PDF Statements, CSV exports, or scanned documents

+

Supports PDF statements, CSV exports, or HDFC Delimited text

+ HDFC customers: choose Delimited when downloading.