From 9b77c8b8372048537b429ab82102805d50bbe3f7 Mon Sep 17 00:00:00 2001 From: Illiyin Date: Tue, 11 Aug 2026 22:47:07 +0500 Subject: [PATCH 1/6] feat: initialize pakdocling core architecture, schemas, preprocessor, extractors, CLI, and test suite --- README.md | 131 ++++++++++++++++- pakdocling/__init__.py | 47 ++++++- pakdocling/cli.py | 98 ++++++++++++- pakdocling/extractors/__init__.py | 15 ++ pakdocling/extractors/base.py | 20 +++ pakdocling/extractors/cnic.py | 182 ++++++++++++++++++++++++ pakdocling/extractors/degree.py | 174 +++++++++++++++++++++++ pakdocling/extractors/intermediate.py | 193 ++++++++++++++++++++++++++ pakdocling/extractors/matric.py | 175 +++++++++++++++++++++++ pakdocling/models/__init__.py | 25 ++++ pakdocling/models/schema.py | 144 +++++++++++++++++++ pakdocling/ocr/__init__.py | 10 ++ pakdocling/ocr/engine.py | 113 +++++++++++++++ pakdocling/pipeline.py | 121 ++++++++++++++++ pakdocling/preprocessing/__init__.py | 5 + pakdocling/preprocessing/image.py | 134 ++++++++++++++++++ pyproject.toml | 8 +- tests/test_cli.py | 13 ++ tests/test_cnic_extractor.py | 58 ++++++++ tests/test_educational_extractors.py | 88 ++++++++++++ tests/test_models.py | 71 ++++++++++ tests/test_ocr.py | 32 +++++ tests/test_pipeline.py | 32 +++++ tests/test_placeholder.py | 3 - tests/test_preprocessing.py | 43 ++++++ 25 files changed, 1925 insertions(+), 10 deletions(-) create mode 100644 pakdocling/extractors/__init__.py create mode 100644 pakdocling/extractors/base.py create mode 100644 pakdocling/extractors/cnic.py create mode 100644 pakdocling/extractors/degree.py create mode 100644 pakdocling/extractors/intermediate.py create mode 100644 pakdocling/extractors/matric.py create mode 100644 pakdocling/models/__init__.py create mode 100644 pakdocling/models/schema.py create mode 100644 pakdocling/ocr/__init__.py create mode 100644 pakdocling/ocr/engine.py create mode 100644 pakdocling/pipeline.py create mode 100644 pakdocling/preprocessing/__init__.py create mode 100644 pakdocling/preprocessing/image.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_cnic_extractor.py create mode 100644 tests/test_educational_extractors.py create mode 100644 tests/test_models.py create mode 100644 tests/test_ocr.py create mode 100644 tests/test_pipeline.py delete mode 100644 tests/test_placeholder.py create mode 100644 tests/test_preprocessing.py diff --git a/README.md b/README.md index ec52096..99f87cf 100644 --- a/README.md +++ b/README.md @@ -1 +1,130 @@ -# pakdocling +# pakdocling: Pakistani Document Intelligence Library 🇵🇰 + +[![PyPI Version](https://img.shields.io/pypi/v/pakdocling.svg)](https://pypi.org/project/pakdocling/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) + +An open-source Python library for structured data extraction from Pakistani identity and educational documents. + +## 🎯 The Problem + +Existing international OCR engines (EasyOCR, Tesseract, AWS Textract) return unstructured, line-by-line raw text without understanding document layouts. They do not know what a Pakistani CNIC looks like, cannot parse field structures of Pakistani Board Matric/Intermediate certificates, and do not return structured JSON objects with named fields. + +Pakistani developers building **KYC pipelines**, **HR systems**, and **edtech platforms** currently solve this manually or with expensive proprietary APIs. + +`pakdocling` solves this by taking document images as input and returning validated, typed **Pydantic JSON objects**. + +--- + +## 📄 Documents Supported in v1 + +| Document Type | Document ID | Key Fields Extracted | +| :--- | :--- | :--- | +| **CNIC** | `cnic` | 13-digit CNIC number, Name, Father/Husband Name, Gender, DOB, Issue Date, Expiry Date, Card Format (`old_green` vs `new_blue` Smart Card) | +| **Matric Certificate** | `matric` | Roll No, Registration No, Student Name, Father Name, Board (BISE Lahore, Karachi, Rawalpindi, etc.), Passing Year, Total & Obtained Marks, Grade, Group | +| **Intermediate Certificate** | `intermediate` | Roll No, Reg No, Student Name, Father Name, BISE Board, Passing Year, Total/Obtained Marks, Grade, Group (Pre-Engineering, Pre-Medical, ICS, Commerce) | +| **University Degree / Transcript** | `degree` | Student Name, Father Name, Registration No, Degree Award Title, Major, Issuing University (NUST, FAST, QAU, LUMS, PU, etc.), Graduation Year, CGPA | + +--- + +## 🚀 Installation + +```bash +pip install pakdocling +``` + +For development mode: +```bash +git clone https://github.com/Epochry-Lab/pakdocling.git +cd pakdocling +pip install -e ".[dev]" +``` + +--- + +## 💻 Python API Usage + +### 1. Extracting Data from a CNIC Image + +```python +from pakdocling import extract_document + +# Auto-detect or specify doc_type='cnic' +result = extract_document("path/to/cnic_card.jpg", doc_type="cnic") + +if result.success: + cnic = result.data + print(f"CNIC Number: {cnic.cnic_number}") + print(f"Name: {cnic.full_name}") + print(f"Father Name: {cnic.father_name}") + print(f"Gender: {cnic.gender}") + print(f"Date of Birth: {cnic.date_of_birth}") + print(f"Card Variant: {cnic.variant}") +``` + +### 2. Outputting Validated JSON + +```python +from pakdocling import DocumentPipeline, DocumentType + +pipeline = DocumentPipeline() +result = pipeline.extract("matric_certificate.png", doc_type=DocumentType.MATRIC) + +# Export Pydantic model to JSON +print(result.model_dump_json(indent=2)) +``` + +### 3. Offline & Fast Testing with `MockOCREngine` + +```python +from pakdocling import DocumentPipeline, MockOCREngine + +mock_ocr = MockOCREngine(mock_text=""" +NATIONAL UNIVERSITY OF SCIENCES AND TECHNOLOGY (NUST) +Certified that Zainab Shah Registration No NUST-2019-BSCS-0042 +is awarded Bachelor of Science in Software Engineering +CGPA: 3.85 / 4.00 +Graduation Year: 2023 +""") + +pipeline = DocumentPipeline(ocr_engine=mock_ocr) +result = pipeline.extract("dummy.png", doc_type="degree") +print(result.data.degree_title) # "Bachelor of Science in Software Engineering" +print(result.data.cgpa) # 3.85 +``` + +--- + +## 🛠️ Command Line Interface (CLI) + +`pakdocling` comes with a CLI powered by Typer and Rich: + +```bash +# Check version & supported document schemas +pakdocling info + +# Extract data from an image file and print formatted JSON +pakdocling extract sample_cnic.jpg --doc-type cnic + +# Extract and save JSON to a file +pakdocling extract degree_transcript.png --doc-type auto -o output.json +``` + +--- + +## 🏗️ Core Technology Stack + +- **EasyOCR**: Deep learning OCR engine for multi-language text extraction. +- **OpenCV & NumPy**: Image preprocessing pipeline (deskewing, noise reduction, adaptive thresholding, contrast enhancement). +- **Pydantic v2**: Type safety, field validation, and JSON serialization. +- **Typer & Rich**: Modern terminal CLI interface. + +--- + +## 🤝 Contributing + +Contributions are welcome! Check out [CONTRIBUTING.md](CONTRIBUTING.md) to get started. + +## 📜 License + +Distributed under the MIT License. See `LICENSE` for details. diff --git a/pakdocling/__init__.py b/pakdocling/__init__.py index d3861cb..7f9ef9f 100644 --- a/pakdocling/__init__.py +++ b/pakdocling/__init__.py @@ -1,3 +1,48 @@ -"""pakdocling - Structured extraction from Pakistani documents.""" +"""pakdocling - Pakistani Document Intelligence Library.""" + +from pakdocling.extractors import ( + BaseExtractor, + CNICExtractor, + DegreeExtractor, + IntermediateExtractor, + MatricExtractor, +) +from pakdocling.models import ( + CNICData, + CNICVariant, + DocumentType, + ExtractionResult, + Gender, + IntermediateCertificateData, + MatricCertificateData, + UniversityDegreeData, +) +from pakdocling.ocr import BaseOCREngine, EasyOCREngine, MockOCREngine, OCRResultItem +from pakdocling.pipeline import DocumentPipeline, extract_document +from pakdocling.preprocessing import ImagePreprocessor __version__ = "0.0.1" + +__all__ = [ + "__version__", + "DocumentType", + "CNICVariant", + "Gender", + "CNICData", + "MatricCertificateData", + "IntermediateCertificateData", + "UniversityDegreeData", + "ExtractionResult", + "BaseExtractor", + "CNICExtractor", + "MatricExtractor", + "IntermediateExtractor", + "DegreeExtractor", + "OCRResultItem", + "BaseOCREngine", + "EasyOCREngine", + "MockOCREngine", + "ImagePreprocessor", + "DocumentPipeline", + "extract_document", +] diff --git a/pakdocling/cli.py b/pakdocling/cli.py index db6b128..70f30f1 100644 --- a/pakdocling/cli.py +++ b/pakdocling/cli.py @@ -1,10 +1,102 @@ +"""Command Line Interface for pakdocling using Typer.""" + +import json +from pathlib import Path +from typing import Optional import typer +from rich import print as rprint +from rich.console import Console +from rich.panel import Panel + +from pakdocling import __version__ +from pakdocling.models.schema import DocumentType +from pakdocling.pipeline import DocumentPipeline + +app = typer.Typer( + name="pakdocling", + help="Pakistani Document Intelligence Library - Structured Data Extraction CLI", + add_completion=False, +) + +console = Console() + + +@app.command() +def extract( + image_path: Path = typer.Argument( + ..., + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Path to Pakistani document image (JPG, PNG, WebP, TIFF)", + ), + doc_type: str = typer.Option( + "auto", + "--doc-type", + "-t", + help="Document type: auto, cnic, matric, intermediate, degree", + ), + output: Optional[Path] = typer.Option( + None, + "--output", + "-o", + help="Path to output JSON file to save extraction results", + ), + pretty: bool = typer.Option( + True, + "--pretty/--compact", + help="Format output JSON with pretty printing", + ), + preprocess: bool = typer.Option( + True, + "--preprocess/--no-preprocess", + help="Enable OpenCV deskewing and image enhancement", + ), +) -> None: + """Extract structured data from a Pakistani document image.""" + console.print(f"[bold blue]Processing document:[/bold blue] {image_path}") + + try: + pipeline = DocumentPipeline() + result = pipeline.extract( + image_source=str(image_path), + doc_type=doc_type, + do_preprocess=preprocess, + ) + + json_data = result.model_dump_json(indent=2 if pretty else None) + + if output: + output.write_text(json_data, encoding="utf-8") + console.print(f"[bold green]✓ Structured JSON saved to:[/bold green] {output}") + else: + typer.echo(json_data) + + except Exception as e: + console.print(f"[bold red]Extraction Error:[/bold red] {e}") + raise typer.Exit(code=1) from e -app = typer.Typer() @app.command() -def hello() -> None: - typer.echo("pakdocling is installed") +def info() -> None: + """Display pakdocling installation details and supported document models.""" + info_panel = Panel.fit( + f"[bold cyan]Pakistani Document Intelligence Library (pakdocling)[/bold cyan]\n" + f"[bold white]Version:[/bold white] {__version__}\n\n" + f"[bold yellow]Supported Documents (v1):[/bold yellow]\n" + f" • CNIC (Computerized National Identity Card - Old Green & New Blue Smart Cards)\n" + f" • Matriculation Certificate (BISE 10th Grade / SSC)\n" + f" • Intermediate Certificate (BISE 12th Grade / HSSC / FSc / ICS)\n" + f" • University Degree & Transcript (HEI Degrees & Transcripts)\n\n" + f"[bold green]Core Technology Stack:[/bold green]\n" + f" • EasyOCR & OpenCV Image Preprocessing\n" + f" • Pydantic Schema Validation & Typer CLI", + title="Pakdocling Info", + border_style="cyan", + ) + console.print(info_panel) + if __name__ == "__main__": app() \ No newline at end of file diff --git a/pakdocling/extractors/__init__.py b/pakdocling/extractors/__init__.py new file mode 100644 index 0000000..33084b1 --- /dev/null +++ b/pakdocling/extractors/__init__.py @@ -0,0 +1,15 @@ +"""Pakdocling extractor modules.""" + +from pakdocling.extractors.base import BaseExtractor +from pakdocling.extractors.cnic import CNICExtractor +from pakdocling.extractors.degree import DegreeExtractor +from pakdocling.extractors.intermediate import IntermediateExtractor +from pakdocling.extractors.matric import MatricExtractor + +__all__ = [ + "BaseExtractor", + "CNICExtractor", + "MatricExtractor", + "IntermediateExtractor", + "DegreeExtractor", +] diff --git a/pakdocling/extractors/base.py b/pakdocling/extractors/base.py new file mode 100644 index 0000000..76d4f40 --- /dev/null +++ b/pakdocling/extractors/base.py @@ -0,0 +1,20 @@ +"""Base Extractor interface for document parsers.""" + +from abc import ABC, abstractmethod +from typing import Any +from pydantic import BaseModel +from pakdocling.ocr.engine import OCRResultItem + + +class BaseExtractor(ABC): + """Abstract base class for all document field extractors.""" + + @abstractmethod + def extract(self, items: list[OCRResultItem], raw_text: str) -> BaseModel: + """Extract structured fields from OCR result items and raw text string.""" + pass + + @abstractmethod + def supports_raw_text(self, raw_text: str) -> bool: + """Return True if raw_text matches document characteristics.""" + pass diff --git a/pakdocling/extractors/cnic.py b/pakdocling/extractors/cnic.py new file mode 100644 index 0000000..ca85320 --- /dev/null +++ b/pakdocling/extractors/cnic.py @@ -0,0 +1,182 @@ +"""CNIC extractor for old green and new blue Smart Card formats.""" + +import re +from typing import Union +from pakdocling.extractors.base import BaseExtractor +from pakdocling.models.schema import CNICData, CNICVariant, Gender +from pakdocling.ocr.engine import OCRResultItem + + +class CNICExtractor(BaseExtractor): + """Extractor for Pakistani Computerized National Identity Cards (CNIC).""" + + CNIC_REGEX = re.compile(r"\b(\d{5})[-\s]?(\d{7})[-\s]?(\d{1})\b") + DATE_REGEX = re.compile(r"\b(\d{2})[./-](\d{2})[./-](\d{4})\b") + + def supports_raw_text(self, raw_text: str) -> bool: + """Check if text contains CNIC indicators or 13-digit CNIC pattern.""" + if self.CNIC_REGEX.search(raw_text): + return True + keywords = ["identity card", "cnic", "national identity", "nadra", "pakistan"] + lower = raw_text.lower() + return sum(1 for kw in keywords if kw in lower) >= 2 + + def _extract_cnic_number(self, text: str) -> Union[str, None]: + match = self.CNIC_REGEX.search(text) + if match: + return f"{match.group(1)}-{match.group(2)}-{match.group(3)}" + return None + + def _infer_gender(self, cnic_number: Union[str, None], text: str) -> Gender: + lower = text.lower() + if "female" in lower or "gender f" in lower or "gender: f" in lower: + return Gender.FEMALE + if "male" in lower or "gender m" in lower or "gender: m" in lower: + return Gender.MALE + + if cnic_number: + digits = cnic_number.replace("-", "").strip() + if len(digits) == 13 and digits[-1].isdigit(): + last_digit = int(digits[-1]) + return Gender.MALE if last_digit % 2 != 0 else Gender.FEMALE + + return Gender.UNKNOWN + + def _detect_variant(self, text: str) -> CNICVariant: + lower = text.lower() + if any(kw in lower for kw in ["smart", "nicop", "chip", "identity card"]): + return CNICVariant.NEW_BLUE + if any(kw in lower for kw in ["identity", "pakistan", "green"]): + return CNICVariant.OLD_GREEN + return CNICVariant.NEW_BLUE + + def _extract_dates(self, lines: list[str]) -> tuple[Union[str, None], Union[str, None], Union[str, None]]: + dob: Union[str, None] = None + issue: Union[str, None] = None + expiry: Union[str, None] = None + + all_dates: list[str] = [] + for line in lines: + for m in self.DATE_REGEX.finditer(line): + all_dates.append(f"{m.group(1)}.{m.group(2)}.{m.group(3)}") + + for i, line in enumerate(lines): + line_lower = line.lower() + m = self.DATE_REGEX.search(line) + found_date = f"{m.group(1)}.{m.group(2)}.{m.group(3)}" if m else None + + if "birth" in line_lower or "dob" in line_lower: + if found_date: + dob = found_date + elif i + 1 < len(lines): + next_m = self.DATE_REGEX.search(lines[i + 1]) + if next_m: + dob = f"{next_m.group(1)}.{next_m.group(2)}.{next_m.group(3)}" + + elif "issue" in line_lower: + if found_date: + issue = found_date + elif i + 1 < len(lines): + next_m = self.DATE_REGEX.search(lines[i + 1]) + if next_m: + issue = f"{next_m.group(1)}.{next_m.group(2)}.{next_m.group(3)}" + + elif "expiry" in line_lower: + if "lifetime" in line_lower: + expiry = "Lifetime" + elif found_date: + expiry = found_date + elif i + 1 < len(lines): + next_m = self.DATE_REGEX.search(lines[i + 1]) + if next_m: + expiry = f"{next_m.group(1)}.{next_m.group(2)}.{next_m.group(3)}" + + # Fallback date assignments if keywords missed + if not dob and len(all_dates) >= 1: + dob = all_dates[0] + if not issue and len(all_dates) >= 2: + issue = all_dates[1] + if not expiry and len(all_dates) >= 3: + expiry = all_dates[2] + + if not expiry and "lifetime" in "\n".join(lines).lower(): + expiry = "Lifetime" + + return dob, issue, expiry + + def _extract_names( + self, lines: list[str] + ) -> tuple[Union[str, None], Union[str, None], Union[str, None], Union[str, None]]: + name: Union[str, None] = None + father_name: Union[str, None] = None + husband_name: Union[str, None] = None + country: Union[str, None] = "Pakistan" + + for i, line in enumerate(lines): + line_clean = line.strip() + line_lower = line_clean.lower() + + if "father name" in line_lower or "father's name" in line_lower: + parts = line_clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + if val and not father_name: + father_name = val + + elif "husband name" in line_lower or "husband's name" in line_lower: + parts = line_clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + if val and not husband_name: + husband_name = val + + elif line_lower.startswith("name") or "name:" in line_lower: + # Exclude father name or husband name lines + if not any(k in line_lower for k in ["father", "husband"]): + parts = line_clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + if val and not name: + name = val + + elif "country of stay" in line_lower: + parts = line_clean.split(":", 1) + if len(parts) > 1 and parts[1].strip(): + country = parts[1].strip() + + return name, father_name, husband_name, country + + def extract(self, items: list[OCRResultItem], raw_text: str) -> CNICData: + lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + + cnic_num = self._extract_cnic_number(raw_text) + variant = self._detect_variant(raw_text) + gender = self._infer_gender(cnic_num, raw_text) + dob, issue, expiry = self._extract_dates(lines) + name, father_name, husband_name, country = self._extract_names(lines) + + # Confidence calculation + found_fields = sum( + 1 + for f in [cnic_num, name, father_name or husband_name, dob, issue, expiry] + if f is not None + ) + confidence = round(found_fields / 6.0, 2) + + return CNICData( + cnic_number=cnic_num, + variant=variant, + full_name=name, + father_name=father_name, + husband_name=husband_name, + gender=gender, + country_of_stay=country, + date_of_birth=dob, + date_of_issue=issue, + date_of_expiry=expiry, + confidence=confidence, + raw_text=raw_text, + ) diff --git a/pakdocling/extractors/degree.py b/pakdocling/extractors/degree.py new file mode 100644 index 0000000..538ccab --- /dev/null +++ b/pakdocling/extractors/degree.py @@ -0,0 +1,174 @@ +"""University Degree and Transcript Extractor for Pakistani Higher Education Institutes.""" + +import re +from typing import Union +from pakdocling.extractors.base import BaseExtractor +from pakdocling.models.schema import UniversityDegreeData +from pakdocling.ocr.engine import OCRResultItem + + +class DegreeExtractor(BaseExtractor): + """Extractor for Pakistani University Degrees, Diplomas, and Transcripts.""" + + CGPA_REGEX = re.compile( + r"\b(?:cgpa|gpa|cumulativ[e\s]+gpa)[.:\s]*([0-3]\.\d{1,2}|4\.00?)\b", re.IGNORECASE + ) + CGPA_FRACTION_REGEX = re.compile( + r"\b([0-3]\.\d{1,2}|4\.00?)\s*[/]\s*(4\.00?|5\.00?)\b" + ) + REG_REGEX = re.compile( + r"\b(?:registration|reg|roll)\s*(?:no|num|#)?[.:\s]*([a-zA-Z0-9/\-]+)\b", re.IGNORECASE + ) + YEAR_REGEX = re.compile(r"\b(19\d{2}|20\d{2})\b") + DEGREE_TITLE_REGEX = re.compile( + r"\b(bachelor\s+of\s+[a-zA-Z\s]+|master\s+of\s+[a-zA-Z\s]+|doctor\s+of\s+[a-zA-Z\s]+|bs\s+[a-zA-Z\s]+|ms\s+[a-zA-Z\s]+|m\.?phil\s+[a-zA-Z\s]+|ph\.?d\s+[a-zA-Z\s]+)\b", + re.IGNORECASE, + ) + + def _extract_degree_title(self, text: str) -> tuple[Union[str, None], Union[str, None]]: + for line in text.splitlines(): + line_clean = line.strip() + m = self.DEGREE_TITLE_REGEX.search(line_clean) + if m: + full_title = m.group(1).strip() + # Stop if title captured trailing keywords + full_title = re.split(r"\s+(graduation|year|cgpa|date|roll|reg|marks)\b", full_title, flags=re.IGNORECASE)[0].strip() + major: Union[str, None] = None + if " in " in full_title.lower(): + parts = re.split(r"\s+in\s+", full_title, flags=re.IGNORECASE) + major = parts[1].strip() + return full_title, major + return None, None + + UNIVERSITIES = [ + "National University of Sciences and Technology", + "NUST", + "FAST National University", + "NUCES", + "Quaid-i-Azam University", + "QAU", + "Lahore University of Management Sciences", + "LUMS", + "University of the Punjab", + "UET Lahore", + "UET Peshawar", + "UET Taxila", + "COMSATS University", + "Aga Khan University", + "GIKI", + "Ghulam Ishaq Khan Institute", + "Institute of Business Administration", + "IBA Karachi", + "Allama Iqbal Open University", + "AIOU", + "Air University", + "Bahria University", + "National University of Modern Languages", + "NUML", + "PIEAS", + "NED University", + "Dow University of Health Sciences", + ] + + def supports_raw_text(self, raw_text: str) -> bool: + lower = raw_text.lower() + if any( + kw in lower + for kw in [ + "degree", + "transcript", + "bachelor", + "master", + "cgpa", + "university", + "conferred upon", + ] + ): + return True + return any(uni.lower() in lower for uni in self.UNIVERSITIES) + + def _extract_university(self, text: str) -> Union[str, None]: + for uni in self.UNIVERSITIES: + if uni.lower() in text.lower(): + return uni + + m = re.search(r"\b(university\s+of\s+[a-zA-Z\s]+|institute\s+of\s+[a-zA-Z\s]+)\b", text, re.IGNORECASE) + if m: + return m.group(1).strip() + return None + + def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, None]]: + student_name: Union[str, None] = None + father_name: Union[str, None] = None + + for i, line in enumerate(lines): + clean = line.strip() + lower = clean.lower() + + if any(kw in lower for kw in ["conferred upon", "awarded to", "certified that", "name:"]): + parts = clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + val = re.sub(r"^(conferred upon|awarded to|certified that|mr\.|ms\.|miss)\s+", "", val, flags=re.IGNORECASE).strip() + if val and not student_name: + student_name = val + + if any(kw in lower for kw in ["son of", "daughter of", "father name", "father's name"]): + parts = clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + val = re.sub(r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE).strip() + if val and not father_name: + father_name = val + + return student_name, father_name + + def _extract_cgpa(self, text: str) -> tuple[Union[float, None], Union[float, None]]: + frac = self.CGPA_FRACTION_REGEX.search(text) + if frac: + return float(frac.group(1)), float(frac.group(2)) + + m = self.CGPA_REGEX.search(text) + if m: + return float(m.group(1)), 4.0 + + return None, 4.0 + + def extract(self, items: list[OCRResultItem], raw_text: str) -> UniversityDegreeData: + lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + + uni_name = self._extract_university(raw_text) + student_name, father_name = self._extract_names(lines) + cgpa, max_cgpa = self._extract_cgpa(raw_text) + degree_title, major = self._extract_degree_title(raw_text) + + reg_match = self.REG_REGEX.search(raw_text) + reg_num = reg_match.group(1) if reg_match else None + + years = self.YEAR_REGEX.findall(raw_text) + grad_year = int(years[-1]) if years else None + + found_fields = sum( + 1 + for f in [student_name, uni_name, degree_title, cgpa or grad_year, reg_num] + if f is not None + ) + confidence = round(found_fields / 5.0, 2) + + return UniversityDegreeData( + student_name=student_name, + father_name=father_name, + roll_number=reg_num, + registration_number=reg_num, + degree_title=degree_title, + major=major, + university_name=uni_name, + graduation_year=grad_year, + award_date=str(grad_year) if grad_year else None, + cgpa=cgpa, + max_cgpa=max_cgpa if max_cgpa else 4.0, + confidence=confidence, + raw_text=raw_text, + ) diff --git a/pakdocling/extractors/intermediate.py b/pakdocling/extractors/intermediate.py new file mode 100644 index 0000000..d41a20b --- /dev/null +++ b/pakdocling/extractors/intermediate.py @@ -0,0 +1,193 @@ +"""Intermediate Certificate Extractor for BISE 12th grade / HSSC certificates.""" + +import re +from typing import Union +from pakdocling.extractors.base import BaseExtractor +from pakdocling.models.schema import IntermediateCertificateData +from pakdocling.ocr.engine import OCRResultItem + + +class IntermediateExtractor(BaseExtractor): + """Extractor for Pakistani BISE Intermediate (HSSC 12th Grade / FA / FSc / ICS / ICom) Certificates.""" + + ROLL_REGEX = re.compile(r"\b(?:roll\s*no|rollno|roll\s*#)[.:\s]*([0-9]{5,8})\b", re.IGNORECASE) + REG_REGEX = re.compile( + r"\b(?:registration|reg)\s*(?:no|num|#)?[.:\s]*([a-zA-Z0-9/\-]+)\b", re.IGNORECASE + ) + YEAR_REGEX = re.compile(r"\b(19\d{2}|20\d{2})\b") + MARKS_FRACTION_REGEX = re.compile(r"\b(\d{3,4})\s*[/]\s*(\d{3,4})\b") + OBTAINED_REGEX = re.compile( + r"\b(?:marks\s*obtained|obtained\s*marks|marks)[.:\s]*(\d{3,4})\b", re.IGNORECASE + ) + TOTAL_REGEX = re.compile(r"\b(?:out\s*of|total\s*marks|total)[.:\s]*(\d{3,4})\b", re.IGNORECASE) + GRADE_REGEX = re.compile( + r"\b(?:grade|division)[.:\s]*([a-fA-F]\+?|1st|2nd|3rd)", re.IGNORECASE + ) + + BOARDS = [ + "BISE Lahore", + "BISE Rawalpindi", + "BISE Multan", + "BISE Faisalabad", + "BISE Gujranwala", + "BISE Sargodha", + "BISE Sahiwal", + "BISE Bahawalpur", + "BISE D.G. Khan", + "BISE Karachi", + "BISE Hyderabad", + "BISE Sukkur", + "BISE Larkana", + "BISE Mirpurkhas", + "BISE Peshawar", + "BISE Swat", + "BISE Abbottabad", + "BISE Bannu", + "BISE Mardan", + "BISE Kohat", + "BISE Quetta", + "BISE Mirpur", + "Federal Board", + "FBISE", + ] + + def supports_raw_text(self, raw_text: str) -> bool: + lower = raw_text.lower() + if any( + kw in lower + for kw in [ + "intermediate", + "higher secondary", + "hssc", + "f.sc", + "fsc", + "ics", + "i.com", + "f.a", + ] + ): + return True + return False + + def _extract_board(self, text: str) -> Union[str, None]: + for board in self.BOARDS: + if board.lower() in text.lower(): + return board + + m = re.search( + r"board\s+of\s+intermediate\s+(?:and|&)\s+secondary\s+education[,\s]+([a-zA-Z\s]+)", + text, + re.IGNORECASE, + ) + if m: + city = m.group(1).strip().split()[0] + return f"BISE {city.capitalize()}" + return None + + def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, None]]: + student_name: Union[str, None] = None + father_name: Union[str, None] = None + + for i, line in enumerate(lines): + clean = line.strip() + lower = clean.lower() + + if any(kw in lower for kw in ["certified that", "student name", "candidate name", "name:"]): + parts = clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + val = re.sub(r"^(certified that|mr\.|ms\.|miss|syed|syeda)\s+", "", val, flags=re.IGNORECASE).strip() + if val and not student_name: + student_name = val + + if any(kw in lower for kw in ["son of", "daughter of", "father name", "father's name"]): + parts = clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + val = re.sub(r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE).strip() + if val and not father_name: + father_name = val + + return student_name, father_name + + def _extract_group(self, text: str) -> Union[str, None]: + lower = text.lower() + if "pre-engineering" in lower or "pre engineering" in lower: + return "Pre-Engineering" + if "pre-medical" in lower or "pre medical" in lower: + return "Pre-Medical" + if "ics" in lower or "computer science" in lower: + return "ICS (Computer Science)" + if "commerce" in lower or "i.com" in lower or "icom" in lower: + return "Commerce" + if "general science" in lower: + return "General Science" + if "humanities" in lower or "arts" in lower or "f.a" in lower: + return "Humanities" + return None + + def extract(self, items: list[OCRResultItem], raw_text: str) -> IntermediateCertificateData: + lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + + board = self._extract_board(raw_text) + student_name, father_name = self._extract_names(lines) + group = self._extract_group(raw_text) + + roll_match = self.ROLL_REGEX.search(raw_text) + roll_num = roll_match.group(1) if roll_match else None + + reg_match = self.REG_REGEX.search(raw_text) + reg_num = reg_match.group(1) if reg_match else None + + year_match = self.YEAR_REGEX.search(raw_text) + passing_year = int(year_match.group(1)) if year_match else None + + obtained_marks: Union[float, None] = None + total_marks: Union[float, None] = None + + frac_match = self.MARKS_FRACTION_REGEX.search(raw_text) + if frac_match: + obtained_marks = float(frac_match.group(1)) + total_marks = float(frac_match.group(2)) + else: + obt_m = self.OBTAINED_REGEX.search(raw_text) + if obt_m: + obtained_marks = float(obt_m.group(1)) + tot_m = self.TOTAL_REGEX.search(raw_text) + if tot_m: + total_marks = float(tot_m.group(1)) + + if not total_marks and obtained_marks: + total_marks = 1100.0 # Standard total marks for HSSC in Pakistan + + percentage: Union[float, None] = None + if obtained_marks is not None and total_marks: + percentage = round((obtained_marks / total_marks) * 100.0, 2) + + grade_match = self.GRADE_REGEX.search(raw_text) + grade = grade_match.group(1).upper() if grade_match else None + + found_fields = sum( + 1 + for f in [roll_num, student_name, father_name, board, passing_year, obtained_marks] + if f is not None + ) + confidence = round(found_fields / 6.0, 2) + + return IntermediateCertificateData( + roll_number=roll_num, + registration_number=reg_num, + student_name=student_name, + father_name=father_name, + board=board, + passing_year=passing_year, + total_marks=total_marks, + obtained_marks=obtained_marks, + percentage=percentage, + grade=grade, + group=group, + confidence=confidence, + raw_text=raw_text, + ) diff --git a/pakdocling/extractors/matric.py b/pakdocling/extractors/matric.py new file mode 100644 index 0000000..36d0fbf --- /dev/null +++ b/pakdocling/extractors/matric.py @@ -0,0 +1,175 @@ +"""Matriculation Certificate Extractor for BISE 10th grade certificates.""" + +import re +from typing import Union +from pakdocling.extractors.base import BaseExtractor +from pakdocling.models.schema import MatricCertificateData +from pakdocling.ocr.engine import OCRResultItem + + +class MatricExtractor(BaseExtractor): + """Extractor for Pakistani BISE Matriculation (SSC 10th Grade) Certificates.""" + + ROLL_REGEX = re.compile(r"\b(?:roll\s*no|rollno|roll\s*#)[.:\s]*([0-9]{5,8})\b", re.IGNORECASE) + REG_REGEX = re.compile( + r"\b(?:registration|reg)\s*(?:no|num|#)?[.:\s]*([a-zA-Z0-9/\-]+)\b", re.IGNORECASE + ) + YEAR_REGEX = re.compile(r"\b(19\d{2}|20\d{2})\b") + MARKS_FRACTION_REGEX = re.compile(r"\b(\d{3,4})\s*[/]\s*(\d{3,4})\b") + OBTAINED_REGEX = re.compile( + r"\b(?:marks\s*obtained|obtained\s*marks|marks)[.:\s]*(\d{3,4})\b", re.IGNORECASE + ) + TOTAL_REGEX = re.compile(r"\b(?:out\s*of|total\s*marks|total)[.:\s]*(\d{3,4})\b", re.IGNORECASE) + GRADE_REGEX = re.compile( + r"\b(?:grade|division)[.:\s]*([a-fA-F]\+?|1st|2nd|3rd)", re.IGNORECASE + ) + + BOARDS = [ + "BISE Lahore", + "BISE Rawalpindi", + "BISE Multan", + "BISE Faisalabad", + "BISE Gujranwala", + "BISE Sargodha", + "BISE Sahiwal", + "BISE Bahawalpur", + "BISE D.G. Khan", + "BISE Karachi", + "BISE Hyderabad", + "BISE Sukkur", + "BISE Larkana", + "BISE Mirpurkhas", + "BISE Peshawar", + "BISE Swat", + "BISE Abbottabad", + "BISE Bannu", + "BISE Mardan", + "BISE Kohat", + "BISE Quetta", + "BISE Mirpur", + "Federal Board", + "FBISE", + ] + + def supports_raw_text(self, raw_text: str) -> bool: + lower = raw_text.lower() + if "matric" in lower or "secondary school certificate" in lower or "ssc" in lower: + return True + return any(board.lower() in lower for board in self.BOARDS) + + def _extract_board(self, text: str) -> Union[str, None]: + for board in self.BOARDS: + if board.lower() in text.lower(): + return board + + m = re.search( + r"board\s+of\s+intermediate\s+(?:and|&)\s+secondary\s+education[,\s]+([a-zA-Z\s]+)", + text, + re.IGNORECASE, + ) + if m: + city = m.group(1).strip().split()[0] + return f"BISE {city.capitalize()}" + return None + + def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, None]]: + student_name: Union[str, None] = None + father_name: Union[str, None] = None + + for i, line in enumerate(lines): + clean = line.strip() + lower = clean.lower() + + if any(kw in lower for kw in ["certified that", "student name", "candidate name", "name:"]): + parts = clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + val = re.sub(r"^(certified that|mr\.|ms\.|miss|syed|syeda)\s+", "", val, flags=re.IGNORECASE).strip() + if val and not student_name: + student_name = val + + if any(kw in lower for kw in ["son of", "daughter of", "father name", "father's name"]): + parts = clean.split(":", 1) + val = parts[1].strip() if len(parts) > 1 else "" + if not val and i + 1 < len(lines): + val = lines[i + 1].strip() + val = re.sub(r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE).strip() + if val and not father_name: + father_name = val + + return student_name, father_name + + def _extract_group(self, text: str) -> Union[str, None]: + lower = text.lower() + if "science" in lower: + return "Science" + if "humanities" in lower or "arts" in lower: + return "Humanities" + if "general" in lower: + return "General" + return None + + def extract(self, items: list[OCRResultItem], raw_text: str) -> MatricCertificateData: + lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + + board = self._extract_board(raw_text) + student_name, father_name = self._extract_names(lines) + group = self._extract_group(raw_text) + + roll_match = self.ROLL_REGEX.search(raw_text) + roll_num = roll_match.group(1) if roll_match else None + + reg_match = self.REG_REGEX.search(raw_text) + reg_num = reg_match.group(1) if reg_match else None + + year_match = self.YEAR_REGEX.search(raw_text) + passing_year = int(year_match.group(1)) if year_match else None + + obtained_marks: Union[float, None] = None + total_marks: Union[float, None] = None + + frac_match = self.MARKS_FRACTION_REGEX.search(raw_text) + if frac_match: + obtained_marks = float(frac_match.group(1)) + total_marks = float(frac_match.group(2)) + else: + obt_m = self.OBTAINED_REGEX.search(raw_text) + if obt_m: + obtained_marks = float(obt_m.group(1)) + tot_m = self.TOTAL_REGEX.search(raw_text) + if tot_m: + total_marks = float(tot_m.group(1)) + + if not total_marks and obtained_marks: + total_marks = 1100.0 # Standard total marks in Pakistani Matric certificates + + percentage: Union[float, None] = None + if obtained_marks is not None and total_marks: + percentage = round((obtained_marks / total_marks) * 100.0, 2) + + grade_match = self.GRADE_REGEX.search(raw_text) + grade = grade_match.group(1).upper() if grade_match else None + + found_fields = sum( + 1 + for f in [roll_num, student_name, father_name, board, passing_year, obtained_marks] + if f is not None + ) + confidence = round(found_fields / 6.0, 2) + + return MatricCertificateData( + roll_number=roll_num, + registration_number=reg_num, + student_name=student_name, + father_name=father_name, + board=board, + passing_year=passing_year, + total_marks=total_marks, + obtained_marks=obtained_marks, + percentage=percentage, + grade=grade, + group=group, + confidence=confidence, + raw_text=raw_text, + ) diff --git a/pakdocling/models/__init__.py b/pakdocling/models/__init__.py new file mode 100644 index 0000000..1059ccd --- /dev/null +++ b/pakdocling/models/__init__.py @@ -0,0 +1,25 @@ +"""Pakdocling models package.""" + +from pakdocling.models.schema import ( + CNICData, + CNICVariant, + DocumentType, + ExtractedDocumentData, + ExtractionResult, + Gender, + IntermediateCertificateData, + MatricCertificateData, + UniversityDegreeData, +) + +__all__ = [ + "DocumentType", + "CNICVariant", + "Gender", + "CNICData", + "MatricCertificateData", + "IntermediateCertificateData", + "UniversityDegreeData", + "ExtractedDocumentData", + "ExtractionResult", +] diff --git a/pakdocling/models/schema.py b/pakdocling/models/schema.py new file mode 100644 index 0000000..6d07bf5 --- /dev/null +++ b/pakdocling/models/schema.py @@ -0,0 +1,144 @@ +"""Schema definitions for Pakistani Document Intelligence Library.""" + +from enum import Enum +from typing import Any, Union +from pydantic import BaseModel, Field + + +class DocumentType(str, Enum): + """Supported document types in pakdocling.""" + + CNIC = "cnic" + MATRIC = "matric" + INTERMEDIATE = "intermediate" + DEGREE = "degree" + AUTO = "auto" + + +class CNICVariant(str, Enum): + """CNIC physical document format variants.""" + + OLD_GREEN = "old_green" + NEW_BLUE = "new_blue" + UNKNOWN = "unknown" + + +class Gender(str, Enum): + """Gender classification.""" + + MALE = "male" + FEMALE = "female" + OTHER = "other" + UNKNOWN = "unknown" + + +class CNICData(BaseModel): + """Structured data model for Pakistani Computerized National Identity Card (CNIC).""" + + cnic_number: Union[str, None] = Field( + default=None, + description="13-digit CNIC number in standard XXXXX-XXXXXXX-X format", + ) + variant: CNICVariant = Field( + default=CNICVariant.UNKNOWN, description="CNIC physical card format (old green / new blue smart)" + ) + full_name: Union[str, None] = Field(default=None, description="Cardholder full name") + father_name: Union[str, None] = Field(default=None, description="Father's full name") + husband_name: Union[str, None] = Field(default=None, description="Husband's full name if applicable") + gender: Gender = Field(default=Gender.UNKNOWN, description="Gender of cardholder") + country_of_stay: Union[str, None] = Field(default=None, description="Country of stay / residence") + date_of_birth: Union[str, None] = Field(default=None, description="Date of birth (DD.MM.YYYY format)") + date_of_issue: Union[str, None] = Field(default=None, description="Date of issue (DD.MM.YYYY format)") + date_of_expiry: Union[str, None] = Field( + default=None, description="Date of expiry (DD.MM.YYYY or Lifetime)" + ) + confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") + raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") + + +class MatricCertificateData(BaseModel): + """Structured data model for Matriculation (SSC / 10th Grade) Certificate.""" + + roll_number: Union[str, None] = Field(default=None, description="Candidate Examination Roll Number") + registration_number: Union[str, None] = Field(default=None, description="Board Registration Number") + student_name: Union[str, None] = Field(default=None, description="Student Full Name") + father_name: Union[str, None] = Field(default=None, description="Father Full Name") + board: Union[str, None] = Field( + default=None, description="Board of Intermediate and Secondary Education (e.g. BISE Lahore)" + ) + passing_year: Union[int, None] = Field(default=None, description="Year of passing / examination") + total_marks: Union[float, None] = Field(default=None, description="Total maximum marks") + obtained_marks: Union[float, None] = Field(default=None, description="Total marks obtained") + percentage: Union[float, None] = Field(default=None, description="Calculated or stated percentage") + grade: Union[str, None] = Field(default=None, description="Assigned grade (e.g. A+, A, B, C)") + group: Union[str, None] = Field(default=None, description="Study group (e.g. Science, Humanities)") + confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") + raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") + + +class IntermediateCertificateData(BaseModel): + """Structured data model for Intermediate (HSSC / 12th Grade) Certificate.""" + + roll_number: Union[str, None] = Field(default=None, description="Candidate Examination Roll Number") + registration_number: Union[str, None] = Field(default=None, description="Board Registration Number") + student_name: Union[str, None] = Field(default=None, description="Student Full Name") + father_name: Union[str, None] = Field(default=None, description="Father Full Name") + board: Union[str, None] = Field( + default=None, description="Board of Intermediate and Secondary Education (e.g. BISE Rawalpindi)" + ) + passing_year: Union[int, None] = Field(default=None, description="Year of passing / examination") + total_marks: Union[float, None] = Field(default=None, description="Total maximum marks") + obtained_marks: Union[float, None] = Field(default=None, description="Total marks obtained") + percentage: Union[float, None] = Field(default=None, description="Calculated or stated percentage") + grade: Union[str, None] = Field(default=None, description="Assigned grade (e.g. A+, A, B, C)") + group: Union[str, None] = Field( + default=None, description="Study group (e.g. Pre-Engineering, Pre-Medical, ICS, General Science)" + ) + confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") + raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") + + +class UniversityDegreeData(BaseModel): + """Structured data model for Higher Education University Degree or Transcript.""" + + student_name: Union[str, None] = Field(default=None, description="Student Full Name") + father_name: Union[str, None] = Field(default=None, description="Father Full Name") + roll_number: Union[str, None] = Field(default=None, description="Student Roll Number / Registration ID") + registration_number: Union[str, None] = Field(default=None, description="University Registration Number") + degree_title: Union[str, None] = Field( + default=None, description="Degree Award Title (e.g. Bachelor of Science in Computer Science)" + ) + major: Union[str, None] = Field(default=None, description="Major / Discipline") + university_name: Union[str, None] = Field(default=None, description="Issuing University Name") + graduation_year: Union[int, None] = Field(default=None, description="Graduation / Completion Year") + award_date: Union[str, None] = Field(default=None, description="Official degree conferral date") + cgpa: Union[float, None] = Field(default=None, description="Cumulative Grade Point Average (CGPA)") + max_cgpa: Union[float, None] = Field(default=4.0, description="Maximum scale CGPA") + total_marks: Union[float, None] = Field(default=None, description="Total Marks if applicable") + obtained_marks: Union[float, None] = Field(default=None, description="Obtained Marks if applicable") + confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") + raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") + + +ExtractedDocumentData = Union[ + CNICData, + MatricCertificateData, + IntermediateCertificateData, + UniversityDegreeData, +] + + +class ExtractionResult(BaseModel): + """Wrapper result for all extraction pipelines.""" + + document_type: DocumentType = Field(description="Identified document type") + success: bool = Field(description="Indicates whether extraction succeeded") + data: Union[ + CNICData, + MatricCertificateData, + IntermediateCertificateData, + UniversityDegreeData, + dict[str, Any], + ] = Field(description="Extracted structured data model or dictionary") + errors: list[str] = Field(default_factory=list, description="List of warnings or extraction errors") + processing_time_ms: float = Field(default=0.0, description="Processing time in milliseconds") diff --git a/pakdocling/ocr/__init__.py b/pakdocling/ocr/__init__.py new file mode 100644 index 0000000..01040eb --- /dev/null +++ b/pakdocling/ocr/__init__.py @@ -0,0 +1,10 @@ +"""Pakdocling OCR engine package.""" + +from pakdocling.ocr.engine import BaseOCREngine, EasyOCREngine, MockOCREngine, OCRResultItem + +__all__ = [ + "OCRResultItem", + "BaseOCREngine", + "EasyOCREngine", + "MockOCREngine", +] diff --git a/pakdocling/ocr/engine.py b/pakdocling/ocr/engine.py new file mode 100644 index 0000000..2830162 --- /dev/null +++ b/pakdocling/ocr/engine.py @@ -0,0 +1,113 @@ +"""OCR Engine interface and implementations for EasyOCR and Mock engines.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, Union +import numpy as np + + +@dataclass +class OCRResultItem: + """Individual OCR bounding box item with extracted text and confidence.""" + + text: str + bbox: Union[list[list[float]], None] = None + confidence: float = 1.0 + + +class BaseOCREngine(ABC): + """Abstract base interface for OCR engines.""" + + @abstractmethod + def extract_text( + self, image: Union[str, bytes, np.ndarray] + ) -> tuple[list[OCRResultItem], str]: + """Perform OCR extraction on input image. + + Returns: + A tuple of (list of OCRResultItem, raw concatenated text). + """ + pass + + +class EasyOCREngine(BaseOCREngine): + """EasyOCR implementation with lazy loading to defer PyTorch/EasyOCR initialization.""" + + def __init__(self, languages: Union[list[str], None] = None, gpu: bool = False) -> None: + self.languages = languages or ["en"] + self.gpu = gpu + self._reader: Any = None + + def _get_reader(self) -> Any: + if self._reader is None: + try: + import easyocr # type: ignore[import-untyped] + + self._reader = easyocr.Reader(self.languages, gpu=self.gpu) + except ImportError as e: + raise ImportError( + "EasyOCR is not installed. Please install easyocr via `pip install easyocr`." + ) from e + return self._reader + + def extract_text( + self, image: Union[str, bytes, np.ndarray] + ) -> tuple[list[OCRResultItem], str]: + from pakdocling.preprocessing.image import ImagePreprocessor + + preprocessed = ImagePreprocessor.load_image(image) + reader = self._get_reader() + + # Read text from image array + raw_results = reader.readtext(preprocessed) + + items: list[OCRResultItem] = [] + text_lines: list[str] = [] + + for bbox, text, prob in raw_results: + cleaned_text = str(text).strip() + if cleaned_text: + items.append( + OCRResultItem( + text=cleaned_text, + bbox=[[float(pt[0]), float(pt[1])] for pt in bbox], + confidence=float(prob), + ) + ) + text_lines.append(cleaned_text) + + full_raw_text = "\n".join(text_lines) + return items, full_raw_text + + +class MockOCREngine(BaseOCREngine): + """Mock OCR engine for fast unit testing and offline deterministic tests.""" + + def __init__( + self, + mock_items: Union[list[OCRResultItem], None] = None, + mock_text: Union[str, None] = None, + ) -> None: + self.mock_items = mock_items or [] + self.mock_text = mock_text + + def set_mock_data( + self, + mock_items: Union[list[OCRResultItem], None] = None, + mock_text: Union[str, None] = None, + ) -> None: + if mock_items is not None: + self.mock_items = mock_items + if mock_text is not None: + self.mock_text = mock_text + + def extract_text( + self, image: Union[str, bytes, np.ndarray] + ) -> tuple[list[OCRResultItem], str]: + if self.mock_text and not self.mock_items: + lines = [line.strip() for line in self.mock_text.splitlines() if line.strip()] + items = [OCRResultItem(text=line, confidence=0.99) for line in lines] + return items, self.mock_text + + raw_text = self.mock_text or "\n".join([item.text for item in self.mock_items]) + return self.mock_items, raw_text diff --git a/pakdocling/pipeline.py b/pakdocling/pipeline.py new file mode 100644 index 0000000..a8049e6 --- /dev/null +++ b/pakdocling/pipeline.py @@ -0,0 +1,121 @@ +"""Document Intelligence Pipeline orchestrator.""" + +import time +from typing import Union +from PIL import Image +import numpy as np + +from pakdocling.extractors.base import BaseExtractor +from pakdocling.extractors.cnic import CNICExtractor +from pakdocling.extractors.degree import DegreeExtractor +from pakdocling.extractors.intermediate import IntermediateExtractor +from pakdocling.extractors.matric import MatricExtractor +from pakdocling.models.schema import ( + CNICData, + DocumentType, + ExtractionResult, + IntermediateCertificateData, + MatricCertificateData, + UniversityDegreeData, +) +from pakdocling.ocr.engine import BaseOCREngine, EasyOCREngine +from pakdocling.preprocessing.image import ImagePreprocessor + + +class DocumentPipeline: + """Unified Document Intelligence Pipeline for Pakistani documents.""" + + def __init__( + self, + ocr_engine: Union[BaseOCREngine, None] = None, + preprocessor: Union[ImagePreprocessor, None] = None, + ) -> None: + self.ocr_engine = ocr_engine or EasyOCREngine() + self.preprocessor = preprocessor or ImagePreprocessor() + + self.extractors: dict[DocumentType, BaseExtractor] = { + DocumentType.CNIC: CNICExtractor(), + DocumentType.MATRIC: MatricExtractor(), + DocumentType.INTERMEDIATE: IntermediateExtractor(), + DocumentType.DEGREE: DegreeExtractor(), + } + + def detect_document_type(self, raw_text: str) -> DocumentType: + """Classify document type automatically based on raw text contents.""" + for doc_type, extractor in self.extractors.items(): + if extractor.supports_raw_text(raw_text): + return doc_type + return DocumentType.CNIC # Default fallback if unknown + + def extract( + self, + image_source: Union[str, bytes, np.ndarray, Image.Image], + doc_type: Union[DocumentType, str] = DocumentType.AUTO, + do_preprocess: bool = True, + ) -> ExtractionResult: + """Extract structured JSON fields from document image. + + Args: + image_source: Image file path, raw bytes, OpenCV ndarray, or PIL Image. + doc_type: Document type ('cnic', 'matric', 'intermediate', 'degree', or 'auto'). + do_preprocess: Apply OpenCV deskewing and contrast enhancement prior to OCR. + + Returns: + ExtractionResult object containing structured data model and metadata. + """ + start_time = time.time() + errors: list[str] = [] + + try: + # Preprocess image + if do_preprocess: + prep_result = self.preprocessor.preprocess(image_source) + target_image = prep_result["processed_gray"] + else: + target_image = self.preprocessor.load_image(image_source) + + # Perform OCR + ocr_items, raw_text = self.ocr_engine.extract_text(target_image) + + # Determine document type + if isinstance(doc_type, str): + target_doc_type = DocumentType(doc_type.lower()) + else: + target_doc_type = doc_type + + if target_doc_type == DocumentType.AUTO: + target_doc_type = self.detect_document_type(raw_text) + + extractor = self.extractors.get(target_doc_type, CNICExtractor()) + extracted_data = extractor.extract(ocr_items, raw_text) + + processing_time = round((time.time() - start_time) * 1000.0, 2) + + return ExtractionResult( + document_type=target_doc_type, + success=True, + data=extracted_data, + errors=errors, + processing_time_ms=processing_time, + ) + + except Exception as e: + processing_time = round((time.time() - start_time) * 1000.0, 2) + errors.append(str(e)) + return ExtractionResult( + document_type=DocumentType.AUTO, + success=False, + data=CNICData(), + errors=errors, + processing_time_ms=processing_time, + ) + + +def extract_document( + image_source: Union[str, bytes, np.ndarray, Image.Image], + doc_type: Union[DocumentType, str] = DocumentType.AUTO, + ocr_engine: Union[BaseOCREngine, None] = None, +) -> ExtractionResult: + """Convenience functional interface for pakdocling extraction.""" + pipeline = DocumentPipeline(ocr_engine=ocr_engine) + return pipeline.extract(image_source=image_source, doc_type=doc_type) diff --git a/pakdocling/preprocessing/__init__.py b/pakdocling/preprocessing/__init__.py new file mode 100644 index 0000000..caa3083 --- /dev/null +++ b/pakdocling/preprocessing/__init__.py @@ -0,0 +1,5 @@ +"""Pakdocling image preprocessing package.""" + +from pakdocling.preprocessing.image import ImagePreprocessor + +__all__ = ["ImagePreprocessor"] diff --git a/pakdocling/preprocessing/image.py b/pakdocling/preprocessing/image.py new file mode 100644 index 0000000..3d63adc --- /dev/null +++ b/pakdocling/preprocessing/image.py @@ -0,0 +1,134 @@ +"""Image preprocessing utilities using OpenCV and NumPy for Pakistani document analysis.""" + +import os +from typing import Any, Union +import cv2 +import numpy as np +from PIL import Image + + +class ImagePreprocessor: + """Preprocesses document images for improved OCR text recognition accuracy.""" + + @staticmethod + def load_image(source: Union[str, bytes, np.ndarray, Image.Image]) -> np.ndarray: + """Load image from path, bytes, PIL Image, or numpy array into OpenCV BGR array.""" + if isinstance(source, np.ndarray): + if len(source.shape) == 2: + return cv2.cvtColor(source, cv2.COLOR_GRAY2BGR) + return source + + if isinstance(source, str): + if not os.path.exists(source): + raise FileNotFoundError(f"Image path does not exist: {source}") + img = cv2.imread(source) + if img is None: + raise ValueError(f"Failed to decode image file: {source}") + return img + + if isinstance(source, bytes): + nparr = np.frombuffer(source, np.uint8) + img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + if img is None: + raise ValueError("Failed to decode image from bytes") + return img + + if isinstance(source, Image.Image): + rgb = np.array(source.convert("RGB")) + return cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) + + raise TypeError(f"Unsupported image source type: {type(source)}") + + @staticmethod + def to_grayscale(img: np.ndarray) -> np.ndarray: + """Convert BGR image to grayscale.""" + if len(img.shape) == 2: + return img + return cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + @staticmethod + def enhance_contrast(gray_img: np.ndarray, clip_limit: float = 2.0, tile_grid_size: tuple[int, int] = (8, 8)) -> np.ndarray: + """Apply Contrast Limited Adaptive Histogram Equalization (CLAHE).""" + clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=tile_grid_size) + return clahe.apply(gray_img) + + @staticmethod + def denoise(gray_img: np.ndarray, kernel_size: int = 3) -> np.ndarray: + """Denoise grayscale image using Gaussian blur.""" + if kernel_size % 2 == 0: + kernel_size += 1 + return cv2.GaussianBlur(gray_img, (kernel_size, kernel_size), 0) + + @staticmethod + def adaptive_threshold(gray_img: np.ndarray) -> np.ndarray: + """Apply adaptive thresholding for binarization.""" + return cv2.adaptiveThreshold( + gray_img, + 255, + cv2.ADAPTIVE_THRESH_GAUSSIAN_C, + cv2.THRESH_BINARY, + 11, + 2, + ) + + @staticmethod + def deskew(img: np.ndarray) -> np.ndarray: + """Detect and correct document skew angle.""" + gray = ImagePreprocessor.to_grayscale(img) if len(img.shape) == 3 else img + blur = cv2.GaussianBlur(gray, (5, 5), 0) + thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] + + # Find coordinates of all white pixels + coords = np.column_stack(np.where(thresh > 0)) + if len(coords) < 10: + return img + + angle = cv2.minAreaRect(coords)[-1] + if angle < -45: + angle = -(90 + angle) + else: + angle = -angle + + # If angle is minor, don't over-correct + if abs(angle) < 0.5 or abs(angle) > 45.0: + return img + + (h, w) = img.shape[:2] + center = (w // 2, h // 2) + m = cv2.getRotationMatrix2D(center, angle, 1.0) + rotated = cv2.warpAffine(img, m, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) + return rotated + + def preprocess( + self, + source: Union[str, bytes, np.ndarray, Image.Image], + do_deskew: bool = True, + do_contrast: bool = True, + do_denoise: bool = True, + do_threshold: bool = False, + ) -> dict[str, Any]: + """Full image preprocessing pipeline. + + Returns dict containing processed grayscale image, processed color image, and metadata. + """ + color_img = self.load_image(source) + if do_deskew: + color_img = self.deskew(color_img) + + gray = self.to_grayscale(color_img) + + if do_contrast: + gray = self.enhance_contrast(gray) + + if do_denoise: + gray = self.denoise(gray) + + final_img = self.adaptive_threshold(gray) if do_threshold else gray + + return { + "color_image": color_img, + "processed_gray": final_img, + "width": color_img.shape[1], + "height": color_img.shape[0], + "channels": color_img.shape[2] if len(color_img.shape) == 3 else 1, + } diff --git a/pyproject.toml b/pyproject.toml index 978f2e2..4d19120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,12 @@ classifiers = [ ] dependencies = [ "pydantic>=2.0", + "opencv-python-headless>=4.8.0", + "numpy>=1.24.0", + "pillow>=10.0.0", + "easyocr>=1.7.0", + "regex>=2023.0.0", + "typer>=0.12.0", ] [project.optional-dependencies] @@ -33,8 +39,6 @@ dev = [ "pytest-cov>=5.0", "ruff>=0.5", "mypy>=1.10", - "pydantic>=2.0", - "typer>=0.12", ] [project.scripts] diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..236659a --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,13 @@ +"""Unit tests for Typer CLI commands.""" + +from typer.testing import CliRunner +from pakdocling.cli import app + +runner = CliRunner() + + +def test_cli_info_command() -> None: + result = runner.invoke(app, ["info"]) + assert result.exit_code == 0 + assert "Pakistani Document Intelligence Library" in result.stdout + assert "Version:" in result.stdout diff --git a/tests/test_cnic_extractor.py b/tests/test_cnic_extractor.py new file mode 100644 index 0000000..4ea840e --- /dev/null +++ b/tests/test_cnic_extractor.py @@ -0,0 +1,58 @@ +"""Unit tests for CNIC Extractor.""" + +from pakdocling.extractors.cnic import CNICExtractor +from pakdocling.models.schema import CNICVariant, Gender + + +def test_cnic_extractor_male_new_blue() -> None: + extractor = CNICExtractor() + mock_text = """ + NATIONAL IDENTITY CARD + ISLAMIC REPUBLIC OF PAKISTAN + Name: Muhammad Bilal Khan + Father Name: Tariq Mehmood Khan + Gender: M + Country of Stay: Pakistan + Identity Number: 35202-9876543-1 + Date of Birth: 15.08.1995 + Date of Issue: 10.01.2020 + Date of Expiry: 10.01.2030 + """ + + data = extractor.extract(items=[], raw_text=mock_text) + + assert data.cnic_number == "35202-9876543-1" + assert data.full_name == "Muhammad Bilal Khan" + assert data.father_name == "Tariq Mehmood Khan" + assert data.gender == Gender.MALE + assert data.date_of_birth == "15.08.1995" + assert data.date_of_issue == "10.01.2020" + assert data.date_of_expiry == "10.01.2030" + assert data.confidence > 0.8 + + +def test_cnic_extractor_female_lifetime() -> None: + extractor = CNICExtractor() + mock_text = """ + PAKISTAN NATIONAL IDENTITY CARD + Name: Fatima Zahra + Husband Name: Hassan Raza + Identity Number: 61101-1234567-2 + Date of Birth: 01.01.1960 + Date of Issue: 05.05.2015 + Date of Expiry: Lifetime + """ + + data = extractor.extract(items=[], raw_text=mock_text) + + assert data.cnic_number == "61101-1234567-2" + assert data.full_name == "Fatima Zahra" + assert data.husband_name == "Hassan Raza" + assert data.gender == Gender.FEMALE + assert data.date_of_expiry == "Lifetime" + + +def test_cnic_supports_raw_text() -> None: + extractor = CNICExtractor() + assert extractor.supports_raw_text("35201-1234567-3") is True + assert extractor.supports_raw_text("Matriculation Certificate BISE Lahore") is False diff --git a/tests/test_educational_extractors.py b/tests/test_educational_extractors.py new file mode 100644 index 0000000..9875c7b --- /dev/null +++ b/tests/test_educational_extractors.py @@ -0,0 +1,88 @@ +"""Unit tests for Matric, Intermediate, and Degree extractors.""" + +from pakdocling.extractors.degree import DegreeExtractor +from pakdocling.extractors.intermediate import IntermediateExtractor +from pakdocling.extractors.matric import MatricExtractor + + +def test_matric_extractor() -> None: + extractor = MatricExtractor() + mock_text = """ + BOARD OF INTERMEDIATE AND SECONDARY EDUCATION LAHORE + SECONDARY SCHOOL CERTIFICATE (MATRICULATION) + ANNUAL EXAMINATION 2020 + Roll No: 154321 + Registration No: 2018-LHR-9876 + Certified that: Ahmed Ali + Son of: Kamran Ali + Group: Science + Marks Obtained: 980 / 1100 + Grade: A+ + """ + + data = extractor.extract(items=[], raw_text=mock_text) + + assert data.roll_number == "154321" + assert data.registration_number == "2018-LHR-9876" + assert data.student_name == "Ahmed Ali" + assert data.father_name == "Kamran Ali" + assert data.board == "BISE Lahore" + assert data.passing_year == 2020 + assert data.obtained_marks == 980.0 + assert data.total_marks == 1100.0 + assert data.percentage == 89.09 + assert data.grade == "A+" + assert data.group == "Science" + + +def test_intermediate_extractor() -> None: + extractor = IntermediateExtractor() + mock_text = """ + BOARD OF INTERMEDIATE AND SECONDARY EDUCATION RAWALPINDI + HIGHER SECONDARY SCHOOL CERTIFICATE (HSSC) + ANNUAL EXAMINATION 2022 + Roll No: 654321 + Reg No: 2020-RWP-4321 + Name: Sara Khan + Daughter of: Shahbaz Khan + Group: Pre-Medical + Marks: 995 + Out of: 1100 + Grade: A+ + """ + + data = extractor.extract(items=[], raw_text=mock_text) + + assert data.roll_number == "654321" + assert data.student_name == "Sara Khan" + assert data.father_name == "Shahbaz Khan" + assert data.board == "BISE Rawalpindi" + assert data.passing_year == 2022 + assert data.obtained_marks == 995.0 + assert data.group == "Pre-Medical" + + +def test_degree_extractor() -> None: + extractor = DegreeExtractor() + mock_text = """ + NATIONAL UNIVERSITY OF SCIENCES AND TECHNOLOGY (NUST) + ISLAMABAD, PAKISTAN + It is certified that: Zainab Shah + Daughter of: Anwar Shah + Registration No: NUST-2019-BSCS-0042 + having fulfilled all academic requirements is hereby awarded the degree of + Bachelor of Science in Software Engineering + Graduation Year: 2023 + CGPA: 3.82 / 4.00 + """ + + data = extractor.extract(items=[], raw_text=mock_text) + + assert data.student_name == "Zainab Shah" + assert data.father_name == "Anwar Shah" + assert data.university_name == "National University of Sciences and Technology" + assert data.degree_title == "Bachelor of Science in Software Engineering" + assert data.major == "Software Engineering" + assert data.registration_number == "NUST-2019-BSCS-0042" + assert data.cgpa == 3.82 + assert data.graduation_year == 2023 diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..51aa549 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,71 @@ +"""Unit tests for pakdocling Pydantic models.""" + +from pakdocling.models import ( + CNICData, + CNICVariant, + DocumentType, + ExtractionResult, + Gender, + IntermediateCertificateData, + MatricCertificateData, + UniversityDegreeData, +) + + +def test_cnic_model_defaults() -> None: + data = CNICData(cnic_number="35202-1234567-1", full_name="Muhammad Ali") + assert data.cnic_number == "35202-1234567-1" + assert data.full_name == "Muhammad Ali" + assert data.variant == CNICVariant.UNKNOWN + assert data.gender == Gender.UNKNOWN + + +def test_matric_model() -> None: + data = MatricCertificateData( + roll_number="123456", + student_name="Ayesha Khan", + board="BISE Lahore", + obtained_marks=950.0, + total_marks=1100.0, + percentage=86.36, + ) + assert data.roll_number == "123456" + assert data.percentage == 86.36 + assert data.board == "BISE Lahore" + + +def test_intermediate_model() -> None: + data = IntermediateCertificateData( + roll_number="654321", + student_name="Zaid Ahmed", + group="Pre-Engineering", + grade="A+", + ) + assert data.roll_number == "654321" + assert data.group == "Pre-Engineering" + assert data.grade == "A+" + + +def test_university_degree_model() -> None: + data = UniversityDegreeData( + student_name="Hamza Tariq", + degree_title="Bachelor of Science in Computer Science", + university_name="NUST", + cgpa=3.85, + ) + assert data.student_name == "Hamza Tariq" + assert data.cgpa == 3.85 + assert data.max_cgpa == 4.0 + + +def test_extraction_result_wrapper() -> None: + cnic = CNICData(cnic_number="61101-1234567-2") + result = ExtractionResult( + document_type=DocumentType.CNIC, + success=True, + data=cnic, + processing_time_ms=45.2, + ) + assert result.document_type == DocumentType.CNIC + assert result.success is True + assert result.data.cnic_number == "61101-1234567-2" # type: ignore[union-attr] diff --git a/tests/test_ocr.py b/tests/test_ocr.py new file mode 100644 index 0000000..14f4c93 --- /dev/null +++ b/tests/test_ocr.py @@ -0,0 +1,32 @@ +"""Unit tests for OCR engines interface and MockOCREngine.""" + +import numpy as np +from pakdocling.ocr import MockOCREngine, OCRResultItem + + +def test_mock_ocr_engine_with_text() -> None: + mock_text = "PAKISTAN NATIONAL IDENTITY CARD\nName: Usman Ali\nCNIC: 35201-1234567-3" + engine = MockOCREngine(mock_text=mock_text) + + dummy_img = np.zeros((10, 10, 3), dtype=np.uint8) + items, raw_text = engine.extract_text(dummy_img) + + assert raw_text == mock_text + assert len(items) == 3 + assert items[0].text == "PAKISTAN NATIONAL IDENTITY CARD" + assert items[2].text == "CNIC: 35201-1234567-3" + + +def test_mock_ocr_engine_with_items() -> None: + items_in = [ + OCRResultItem(text="BISE Lahore", confidence=0.99), + OCRResultItem(text="Roll No: 123456", confidence=0.95), + ] + engine = MockOCREngine(mock_items=items_in) + + dummy_img = np.zeros((10, 10, 3), dtype=np.uint8) + items_out, raw_text = engine.extract_text(dummy_img) + + assert len(items_out) == 2 + assert "BISE Lahore" in raw_text + assert "123456" in raw_text diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..ea91e5d --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,32 @@ +"""Unit tests for DocumentPipeline with MockOCREngine.""" + +import numpy as np +from pakdocling.models import DocumentType +from pakdocling.ocr import MockOCREngine +from pakdocling.pipeline import DocumentPipeline, extract_document + + +def test_pipeline_cnic_extraction() -> None: + mock_text = "PAKISTAN NATIONAL IDENTITY CARD\nName: Ali Raza\nCNIC: 35202-1234567-1\nDate of Birth: 01.01.1990" + mock_ocr = MockOCREngine(mock_text=mock_text) + + pipeline = DocumentPipeline(ocr_engine=mock_ocr) + + dummy_img = np.zeros((100, 100, 3), dtype=np.uint8) + result = pipeline.extract(dummy_img, doc_type=DocumentType.AUTO, do_preprocess=False) + + assert result.success is True + assert result.document_type == DocumentType.CNIC + assert result.data.cnic_number == "35202-1234567-1" # type: ignore[union-attr] + + +def test_pipeline_matric_auto_detection() -> None: + mock_text = "BISE Lahore\nSECONDARY SCHOOL CERTIFICATE\nRoll No: 123456\nName: Usman Ali" + mock_ocr = MockOCREngine(mock_text=mock_text) + + dummy_img = np.zeros((50, 50, 3), dtype=np.uint8) + result = extract_document(dummy_img, doc_type="auto", ocr_engine=mock_ocr) + + assert result.success is True + assert result.document_type == DocumentType.MATRIC + assert result.data.roll_number == "123456" # type: ignore[union-attr] diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py deleted file mode 100644 index 518b76c..0000000 --- a/tests/test_placeholder.py +++ /dev/null @@ -1,3 +0,0 @@ -def test_placeholder() -> None: - """Remove this once real tests are added.""" - assert True diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py new file mode 100644 index 0000000..e71158a --- /dev/null +++ b/tests/test_preprocessing.py @@ -0,0 +1,43 @@ +"""Unit tests for OpenCV image preprocessing module.""" + +import numpy as np +import pytest +from PIL import Image +from pakdocling.preprocessing import ImagePreprocessor + + +def test_load_image_numpy() -> None: + arr = np.zeros((100, 100, 3), dtype=np.uint8) + loaded = ImagePreprocessor.load_image(arr) + assert loaded.shape == (100, 100, 3) + + +def test_load_image_pil() -> None: + img = Image.new("RGB", (50, 50), color="white") + loaded = ImagePreprocessor.load_image(img) + assert loaded.shape == (50, 50, 3) + + +def test_grayscale_and_enhancement() -> None: + preprocessor = ImagePreprocessor() + arr = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8) + + gray = preprocessor.to_grayscale(arr) + assert gray.shape == (100, 100) + + enhanced = preprocessor.enhance_contrast(gray) + assert enhanced.shape == (100, 100) + + denoised = preprocessor.denoise(gray) + assert denoised.shape == (100, 100) + + +def test_preprocess_pipeline() -> None: + preprocessor = ImagePreprocessor() + arr = np.ones((120, 120, 3), dtype=np.uint8) * 200 + + result = preprocessor.preprocess(arr) + assert "color_image" in result + assert "processed_gray" in result + assert result["width"] == 120 + assert result["height"] == 120 From 329551e92a499b370a7a2ffdcdfa4257491b92f1 Mon Sep 17 00:00:00 2001 From: Illiyin Date: Tue, 11 Aug 2026 22:53:44 +0500 Subject: [PATCH 2/6] style: apply ruff format and fix line length limits --- README.md | 8 +- pakdocling/cli.py | 6 +- pakdocling/extractors/base.py | 3 +- pakdocling/extractors/cnic.py | 11 ++- pakdocling/extractors/degree.py | 36 ++++++--- pakdocling/extractors/intermediate.py | 25 ++++-- pakdocling/extractors/matric.py | 23 ++++-- pakdocling/models/schema.py | 105 +++++++++++++++++++------- pakdocling/ocr/engine.py | 15 ++-- pakdocling/pipeline.py | 6 +- pakdocling/preprocessing/image.py | 9 ++- tests/test_cli.py | 1 + tests/test_cnic_extractor.py | 2 +- tests/test_ocr.py | 1 + tests/test_pipeline.py | 8 +- tests/test_preprocessing.py | 2 +- 16 files changed, 181 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 99f87cf..6f01abe 100644 --- a/README.md +++ b/README.md @@ -79,18 +79,20 @@ print(result.model_dump_json(indent=2)) ```python from pakdocling import DocumentPipeline, MockOCREngine -mock_ocr = MockOCREngine(mock_text=""" +mock_ocr = MockOCREngine( + mock_text=""" NATIONAL UNIVERSITY OF SCIENCES AND TECHNOLOGY (NUST) Certified that Zainab Shah Registration No NUST-2019-BSCS-0042 is awarded Bachelor of Science in Software Engineering CGPA: 3.85 / 4.00 Graduation Year: 2023 -""") +""" +) pipeline = DocumentPipeline(ocr_engine=mock_ocr) result = pipeline.extract("dummy.png", doc_type="degree") print(result.data.degree_title) # "Bachelor of Science in Software Engineering" -print(result.data.cgpa) # 3.85 +print(result.data.cgpa) # 3.85 ``` --- diff --git a/pakdocling/cli.py b/pakdocling/cli.py index 70f30f1..34330cd 100644 --- a/pakdocling/cli.py +++ b/pakdocling/cli.py @@ -1,15 +1,13 @@ """Command Line Interface for pakdocling using Typer.""" -import json from pathlib import Path from typing import Optional + import typer -from rich import print as rprint from rich.console import Console from rich.panel import Panel from pakdocling import __version__ -from pakdocling.models.schema import DocumentType from pakdocling.pipeline import DocumentPipeline app = typer.Typer( @@ -99,4 +97,4 @@ def info() -> None: if __name__ == "__main__": - app() \ No newline at end of file + app() diff --git a/pakdocling/extractors/base.py b/pakdocling/extractors/base.py index 76d4f40..d2b3cda 100644 --- a/pakdocling/extractors/base.py +++ b/pakdocling/extractors/base.py @@ -1,8 +1,9 @@ """Base Extractor interface for document parsers.""" from abc import ABC, abstractmethod -from typing import Any + from pydantic import BaseModel + from pakdocling.ocr.engine import OCRResultItem diff --git a/pakdocling/extractors/cnic.py b/pakdocling/extractors/cnic.py index ca85320..fac87e4 100644 --- a/pakdocling/extractors/cnic.py +++ b/pakdocling/extractors/cnic.py @@ -2,6 +2,7 @@ import re from typing import Union + from pakdocling.extractors.base import BaseExtractor from pakdocling.models.schema import CNICData, CNICVariant, Gender from pakdocling.ocr.engine import OCRResultItem @@ -50,7 +51,9 @@ def _detect_variant(self, text: str) -> CNICVariant: return CNICVariant.OLD_GREEN return CNICVariant.NEW_BLUE - def _extract_dates(self, lines: list[str]) -> tuple[Union[str, None], Union[str, None], Union[str, None]]: + def _extract_dates( + self, lines: list[str] + ) -> tuple[Union[str, None], Union[str, None], Union[str, None]]: dob: Union[str, None] = None issue: Union[str, None] = None expiry: Union[str, None] = None @@ -150,7 +153,11 @@ def _extract_names( return name, father_name, husband_name, country def extract(self, items: list[OCRResultItem], raw_text: str) -> CNICData: - lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + lines = ( + [item.text for item in items] + if items + else [line.strip() for line in raw_text.splitlines() if line.strip()] + ) cnic_num = self._extract_cnic_number(raw_text) variant = self._detect_variant(raw_text) diff --git a/pakdocling/extractors/degree.py b/pakdocling/extractors/degree.py index 538ccab..36acd55 100644 --- a/pakdocling/extractors/degree.py +++ b/pakdocling/extractors/degree.py @@ -2,6 +2,7 @@ import re from typing import Union + from pakdocling.extractors.base import BaseExtractor from pakdocling.models.schema import UniversityDegreeData from pakdocling.ocr.engine import OCRResultItem @@ -13,9 +14,7 @@ class DegreeExtractor(BaseExtractor): CGPA_REGEX = re.compile( r"\b(?:cgpa|gpa|cumulativ[e\s]+gpa)[.:\s]*([0-3]\.\d{1,2}|4\.00?)\b", re.IGNORECASE ) - CGPA_FRACTION_REGEX = re.compile( - r"\b([0-3]\.\d{1,2}|4\.00?)\s*[/]\s*(4\.00?|5\.00?)\b" - ) + CGPA_FRACTION_REGEX = re.compile(r"\b([0-3]\.\d{1,2}|4\.00?)\s*[/]\s*(4\.00?|5\.00?)\b") REG_REGEX = re.compile( r"\b(?:registration|reg|roll)\s*(?:no|num|#)?[.:\s]*([a-zA-Z0-9/\-]+)\b", re.IGNORECASE ) @@ -32,7 +31,11 @@ def _extract_degree_title(self, text: str) -> tuple[Union[str, None], Union[str, if m: full_title = m.group(1).strip() # Stop if title captured trailing keywords - full_title = re.split(r"\s+(graduation|year|cgpa|date|roll|reg|marks)\b", full_title, flags=re.IGNORECASE)[0].strip() + full_title = re.split( + r"\s+(graduation|year|cgpa|date|roll|reg|marks)\b", + full_title, + flags=re.IGNORECASE, + )[0].strip() major: Union[str, None] = None if " in " in full_title.lower(): parts = re.split(r"\s+in\s+", full_title, flags=re.IGNORECASE) @@ -92,7 +95,9 @@ def _extract_university(self, text: str) -> Union[str, None]: if uni.lower() in text.lower(): return uni - m = re.search(r"\b(university\s+of\s+[a-zA-Z\s]+|institute\s+of\s+[a-zA-Z\s]+)\b", text, re.IGNORECASE) + m = re.search( + r"\b(university\s+of\s+[a-zA-Z\s]+|institute\s+of\s+[a-zA-Z\s]+)\b", text, re.IGNORECASE + ) if m: return m.group(1).strip() return None @@ -105,12 +110,19 @@ def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, clean = line.strip() lower = clean.lower() - if any(kw in lower for kw in ["conferred upon", "awarded to", "certified that", "name:"]): + if any( + kw in lower for kw in ["conferred upon", "awarded to", "certified that", "name:"] + ): parts = clean.split(":", 1) val = parts[1].strip() if len(parts) > 1 else "" if not val and i + 1 < len(lines): val = lines[i + 1].strip() - val = re.sub(r"^(conferred upon|awarded to|certified that|mr\.|ms\.|miss)\s+", "", val, flags=re.IGNORECASE).strip() + val = re.sub( + r"^(conferred upon|awarded to|certified that|mr\.|ms\.|miss)\s+", + "", + val, + flags=re.IGNORECASE, + ).strip() if val and not student_name: student_name = val @@ -119,7 +131,9 @@ def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, val = parts[1].strip() if len(parts) > 1 else "" if not val and i + 1 < len(lines): val = lines[i + 1].strip() - val = re.sub(r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE).strip() + val = re.sub( + r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE + ).strip() if val and not father_name: father_name = val @@ -137,7 +151,11 @@ def _extract_cgpa(self, text: str) -> tuple[Union[float, None], Union[float, Non return None, 4.0 def extract(self, items: list[OCRResultItem], raw_text: str) -> UniversityDegreeData: - lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + lines = ( + [item.text for item in items] + if items + else [line.strip() for line in raw_text.splitlines() if line.strip()] + ) uni_name = self._extract_university(raw_text) student_name, father_name = self._extract_names(lines) diff --git a/pakdocling/extractors/intermediate.py b/pakdocling/extractors/intermediate.py index d41a20b..524645d 100644 --- a/pakdocling/extractors/intermediate.py +++ b/pakdocling/extractors/intermediate.py @@ -2,13 +2,14 @@ import re from typing import Union + from pakdocling.extractors.base import BaseExtractor from pakdocling.models.schema import IntermediateCertificateData from pakdocling.ocr.engine import OCRResultItem class IntermediateExtractor(BaseExtractor): - """Extractor for Pakistani BISE Intermediate (HSSC 12th Grade / FA / FSc / ICS / ICom) Certificates.""" + """Extractor for Pakistani BISE Intermediate (HSSC 12th Grade) Certificates.""" ROLL_REGEX = re.compile(r"\b(?:roll\s*no|rollno|roll\s*#)[.:\s]*([0-9]{5,8})\b", re.IGNORECASE) REG_REGEX = re.compile( @@ -20,9 +21,7 @@ class IntermediateExtractor(BaseExtractor): r"\b(?:marks\s*obtained|obtained\s*marks|marks)[.:\s]*(\d{3,4})\b", re.IGNORECASE ) TOTAL_REGEX = re.compile(r"\b(?:out\s*of|total\s*marks|total)[.:\s]*(\d{3,4})\b", re.IGNORECASE) - GRADE_REGEX = re.compile( - r"\b(?:grade|division)[.:\s]*([a-fA-F]\+?|1st|2nd|3rd)", re.IGNORECASE - ) + GRADE_REGEX = re.compile(r"\b(?:grade|division)[.:\s]*([a-fA-F]\+?|1st|2nd|3rd)", re.IGNORECASE) BOARDS = [ "BISE Lahore", @@ -92,12 +91,16 @@ def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, clean = line.strip() lower = clean.lower() - if any(kw in lower for kw in ["certified that", "student name", "candidate name", "name:"]): + if any( + kw in lower for kw in ["certified that", "student name", "candidate name", "name:"] + ): parts = clean.split(":", 1) val = parts[1].strip() if len(parts) > 1 else "" if not val and i + 1 < len(lines): val = lines[i + 1].strip() - val = re.sub(r"^(certified that|mr\.|ms\.|miss|syed|syeda)\s+", "", val, flags=re.IGNORECASE).strip() + val = re.sub( + r"^(certified that|mr\.|ms\.|miss|syed|syeda)\s+", "", val, flags=re.IGNORECASE + ).strip() if val and not student_name: student_name = val @@ -106,7 +109,9 @@ def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, val = parts[1].strip() if len(parts) > 1 else "" if not val and i + 1 < len(lines): val = lines[i + 1].strip() - val = re.sub(r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE).strip() + val = re.sub( + r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE + ).strip() if val and not father_name: father_name = val @@ -129,7 +134,11 @@ def _extract_group(self, text: str) -> Union[str, None]: return None def extract(self, items: list[OCRResultItem], raw_text: str) -> IntermediateCertificateData: - lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + lines = ( + [item.text for item in items] + if items + else [line.strip() for line in raw_text.splitlines() if line.strip()] + ) board = self._extract_board(raw_text) student_name, father_name = self._extract_names(lines) diff --git a/pakdocling/extractors/matric.py b/pakdocling/extractors/matric.py index 36d0fbf..6387dcd 100644 --- a/pakdocling/extractors/matric.py +++ b/pakdocling/extractors/matric.py @@ -2,6 +2,7 @@ import re from typing import Union + from pakdocling.extractors.base import BaseExtractor from pakdocling.models.schema import MatricCertificateData from pakdocling.ocr.engine import OCRResultItem @@ -20,9 +21,7 @@ class MatricExtractor(BaseExtractor): r"\b(?:marks\s*obtained|obtained\s*marks|marks)[.:\s]*(\d{3,4})\b", re.IGNORECASE ) TOTAL_REGEX = re.compile(r"\b(?:out\s*of|total\s*marks|total)[.:\s]*(\d{3,4})\b", re.IGNORECASE) - GRADE_REGEX = re.compile( - r"\b(?:grade|division)[.:\s]*([a-fA-F]\+?|1st|2nd|3rd)", re.IGNORECASE - ) + GRADE_REGEX = re.compile(r"\b(?:grade|division)[.:\s]*([a-fA-F]\+?|1st|2nd|3rd)", re.IGNORECASE) BOARDS = [ "BISE Lahore", @@ -80,12 +79,16 @@ def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, clean = line.strip() lower = clean.lower() - if any(kw in lower for kw in ["certified that", "student name", "candidate name", "name:"]): + if any( + kw in lower for kw in ["certified that", "student name", "candidate name", "name:"] + ): parts = clean.split(":", 1) val = parts[1].strip() if len(parts) > 1 else "" if not val and i + 1 < len(lines): val = lines[i + 1].strip() - val = re.sub(r"^(certified that|mr\.|ms\.|miss|syed|syeda)\s+", "", val, flags=re.IGNORECASE).strip() + val = re.sub( + r"^(certified that|mr\.|ms\.|miss|syed|syeda)\s+", "", val, flags=re.IGNORECASE + ).strip() if val and not student_name: student_name = val @@ -94,7 +97,9 @@ def _extract_names(self, lines: list[str]) -> tuple[Union[str, None], Union[str, val = parts[1].strip() if len(parts) > 1 else "" if not val and i + 1 < len(lines): val = lines[i + 1].strip() - val = re.sub(r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE).strip() + val = re.sub( + r"^(son of|daughter of|s/o|d/o|mr\.)\s+", "", val, flags=re.IGNORECASE + ).strip() if val and not father_name: father_name = val @@ -111,7 +116,11 @@ def _extract_group(self, text: str) -> Union[str, None]: return None def extract(self, items: list[OCRResultItem], raw_text: str) -> MatricCertificateData: - lines = [item.text for item in items] if items else [line.strip() for line in raw_text.splitlines() if line.strip()] + lines = ( + [item.text for item in items] + if items + else [line.strip() for line in raw_text.splitlines() if line.strip()] + ) board = self._extract_board(raw_text) student_name, father_name = self._extract_names(lines) diff --git a/pakdocling/models/schema.py b/pakdocling/models/schema.py index 6d07bf5..5bd5b99 100644 --- a/pakdocling/models/schema.py +++ b/pakdocling/models/schema.py @@ -2,6 +2,7 @@ from enum import Enum from typing import Any, Union + from pydantic import BaseModel, Field @@ -40,61 +41,96 @@ class CNICData(BaseModel): description="13-digit CNIC number in standard XXXXX-XXXXXXX-X format", ) variant: CNICVariant = Field( - default=CNICVariant.UNKNOWN, description="CNIC physical card format (old green / new blue smart)" + default=CNICVariant.UNKNOWN, + description="CNIC physical card format (old green / new blue smart)", ) full_name: Union[str, None] = Field(default=None, description="Cardholder full name") father_name: Union[str, None] = Field(default=None, description="Father's full name") - husband_name: Union[str, None] = Field(default=None, description="Husband's full name if applicable") + husband_name: Union[str, None] = Field( + default=None, description="Husband's full name if applicable" + ) gender: Gender = Field(default=Gender.UNKNOWN, description="Gender of cardholder") - country_of_stay: Union[str, None] = Field(default=None, description="Country of stay / residence") - date_of_birth: Union[str, None] = Field(default=None, description="Date of birth (DD.MM.YYYY format)") - date_of_issue: Union[str, None] = Field(default=None, description="Date of issue (DD.MM.YYYY format)") + country_of_stay: Union[str, None] = Field( + default=None, description="Country of stay / residence" + ) + date_of_birth: Union[str, None] = Field( + default=None, description="Date of birth (DD.MM.YYYY format)" + ) + date_of_issue: Union[str, None] = Field( + default=None, description="Date of issue (DD.MM.YYYY format)" + ) date_of_expiry: Union[str, None] = Field( default=None, description="Date of expiry (DD.MM.YYYY or Lifetime)" ) - confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") + confidence: float = Field( + default=0.0, description="Overall field extraction confidence score (0-1)" + ) raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") class MatricCertificateData(BaseModel): """Structured data model for Matriculation (SSC / 10th Grade) Certificate.""" - roll_number: Union[str, None] = Field(default=None, description="Candidate Examination Roll Number") - registration_number: Union[str, None] = Field(default=None, description="Board Registration Number") + roll_number: Union[str, None] = Field( + default=None, description="Candidate Examination Roll Number" + ) + registration_number: Union[str, None] = Field( + default=None, description="Board Registration Number" + ) student_name: Union[str, None] = Field(default=None, description="Student Full Name") father_name: Union[str, None] = Field(default=None, description="Father Full Name") board: Union[str, None] = Field( default=None, description="Board of Intermediate and Secondary Education (e.g. BISE Lahore)" ) - passing_year: Union[int, None] = Field(default=None, description="Year of passing / examination") + passing_year: Union[int, None] = Field( + default=None, description="Year of passing / examination" + ) total_marks: Union[float, None] = Field(default=None, description="Total maximum marks") obtained_marks: Union[float, None] = Field(default=None, description="Total marks obtained") - percentage: Union[float, None] = Field(default=None, description="Calculated or stated percentage") + percentage: Union[float, None] = Field( + default=None, description="Calculated or stated percentage" + ) grade: Union[str, None] = Field(default=None, description="Assigned grade (e.g. A+, A, B, C)") - group: Union[str, None] = Field(default=None, description="Study group (e.g. Science, Humanities)") - confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") + group: Union[str, None] = Field( + default=None, description="Study group (e.g. Science, Humanities)" + ) + confidence: float = Field( + default=0.0, description="Overall field extraction confidence score (0-1)" + ) raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") class IntermediateCertificateData(BaseModel): """Structured data model for Intermediate (HSSC / 12th Grade) Certificate.""" - roll_number: Union[str, None] = Field(default=None, description="Candidate Examination Roll Number") - registration_number: Union[str, None] = Field(default=None, description="Board Registration Number") + roll_number: Union[str, None] = Field( + default=None, description="Candidate Examination Roll Number" + ) + registration_number: Union[str, None] = Field( + default=None, description="Board Registration Number" + ) student_name: Union[str, None] = Field(default=None, description="Student Full Name") father_name: Union[str, None] = Field(default=None, description="Father Full Name") board: Union[str, None] = Field( - default=None, description="Board of Intermediate and Secondary Education (e.g. BISE Rawalpindi)" + default=None, + description="Board of Intermediate and Secondary Education (e.g. BISE Rawalpindi)", + ) + passing_year: Union[int, None] = Field( + default=None, description="Year of passing / examination" ) - passing_year: Union[int, None] = Field(default=None, description="Year of passing / examination") total_marks: Union[float, None] = Field(default=None, description="Total maximum marks") obtained_marks: Union[float, None] = Field(default=None, description="Total marks obtained") - percentage: Union[float, None] = Field(default=None, description="Calculated or stated percentage") + percentage: Union[float, None] = Field( + default=None, description="Calculated or stated percentage" + ) grade: Union[str, None] = Field(default=None, description="Assigned grade (e.g. A+, A, B, C)") group: Union[str, None] = Field( - default=None, description="Study group (e.g. Pre-Engineering, Pre-Medical, ICS, General Science)" + default=None, + description="Study group (e.g. Pre-Engineering, Pre-Medical, ICS, General Science)", + ) + confidence: float = Field( + default=0.0, description="Overall field extraction confidence score (0-1)" ) - confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") @@ -103,20 +139,33 @@ class UniversityDegreeData(BaseModel): student_name: Union[str, None] = Field(default=None, description="Student Full Name") father_name: Union[str, None] = Field(default=None, description="Father Full Name") - roll_number: Union[str, None] = Field(default=None, description="Student Roll Number / Registration ID") - registration_number: Union[str, None] = Field(default=None, description="University Registration Number") + roll_number: Union[str, None] = Field( + default=None, description="Student Roll Number / Registration ID" + ) + registration_number: Union[str, None] = Field( + default=None, description="University Registration Number" + ) degree_title: Union[str, None] = Field( - default=None, description="Degree Award Title (e.g. Bachelor of Science in Computer Science)" + default=None, + description="Degree Award Title (e.g. Bachelor of Science in Computer Science)", ) major: Union[str, None] = Field(default=None, description="Major / Discipline") university_name: Union[str, None] = Field(default=None, description="Issuing University Name") - graduation_year: Union[int, None] = Field(default=None, description="Graduation / Completion Year") + graduation_year: Union[int, None] = Field( + default=None, description="Graduation / Completion Year" + ) award_date: Union[str, None] = Field(default=None, description="Official degree conferral date") - cgpa: Union[float, None] = Field(default=None, description="Cumulative Grade Point Average (CGPA)") + cgpa: Union[float, None] = Field( + default=None, description="Cumulative Grade Point Average (CGPA)" + ) max_cgpa: Union[float, None] = Field(default=4.0, description="Maximum scale CGPA") total_marks: Union[float, None] = Field(default=None, description="Total Marks if applicable") - obtained_marks: Union[float, None] = Field(default=None, description="Obtained Marks if applicable") - confidence: float = Field(default=0.0, description="Overall field extraction confidence score (0-1)") + obtained_marks: Union[float, None] = Field( + default=None, description="Obtained Marks if applicable" + ) + confidence: float = Field( + default=0.0, description="Overall field extraction confidence score (0-1)" + ) raw_text: Union[str, None] = Field(default=None, description="Raw extracted OCR text lines") @@ -140,5 +189,7 @@ class ExtractionResult(BaseModel): UniversityDegreeData, dict[str, Any], ] = Field(description="Extracted structured data model or dictionary") - errors: list[str] = Field(default_factory=list, description="List of warnings or extraction errors") + errors: list[str] = Field( + default_factory=list, description="List of warnings or extraction errors" + ) processing_time_ms: float = Field(default=0.0, description="Processing time in milliseconds") diff --git a/pakdocling/ocr/engine.py b/pakdocling/ocr/engine.py index 2830162..6eda734 100644 --- a/pakdocling/ocr/engine.py +++ b/pakdocling/ocr/engine.py @@ -1,8 +1,9 @@ """OCR Engine interface and implementations for EasyOCR and Mock engines.""" from abc import ABC, abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, Union + import numpy as np @@ -19,9 +20,7 @@ class BaseOCREngine(ABC): """Abstract base interface for OCR engines.""" @abstractmethod - def extract_text( - self, image: Union[str, bytes, np.ndarray] - ) -> tuple[list[OCRResultItem], str]: + def extract_text(self, image: Union[str, bytes, np.ndarray]) -> tuple[list[OCRResultItem], str]: """Perform OCR extraction on input image. Returns: @@ -50,9 +49,7 @@ def _get_reader(self) -> Any: ) from e return self._reader - def extract_text( - self, image: Union[str, bytes, np.ndarray] - ) -> tuple[list[OCRResultItem], str]: + def extract_text(self, image: Union[str, bytes, np.ndarray]) -> tuple[list[OCRResultItem], str]: from pakdocling.preprocessing.image import ImagePreprocessor preprocessed = ImagePreprocessor.load_image(image) @@ -101,9 +98,7 @@ def set_mock_data( if mock_text is not None: self.mock_text = mock_text - def extract_text( - self, image: Union[str, bytes, np.ndarray] - ) -> tuple[list[OCRResultItem], str]: + def extract_text(self, image: Union[str, bytes, np.ndarray]) -> tuple[list[OCRResultItem], str]: if self.mock_text and not self.mock_items: lines = [line.strip() for line in self.mock_text.splitlines() if line.strip()] items = [OCRResultItem(text=line, confidence=0.99) for line in lines] diff --git a/pakdocling/pipeline.py b/pakdocling/pipeline.py index a8049e6..5483c23 100644 --- a/pakdocling/pipeline.py +++ b/pakdocling/pipeline.py @@ -2,8 +2,9 @@ import time from typing import Union -from PIL import Image + import numpy as np +from PIL import Image from pakdocling.extractors.base import BaseExtractor from pakdocling.extractors.cnic import CNICExtractor @@ -14,9 +15,6 @@ CNICData, DocumentType, ExtractionResult, - IntermediateCertificateData, - MatricCertificateData, - UniversityDegreeData, ) from pakdocling.ocr.engine import BaseOCREngine, EasyOCREngine from pakdocling.preprocessing.image import ImagePreprocessor diff --git a/pakdocling/preprocessing/image.py b/pakdocling/preprocessing/image.py index 3d63adc..0468d4d 100644 --- a/pakdocling/preprocessing/image.py +++ b/pakdocling/preprocessing/image.py @@ -2,6 +2,7 @@ import os from typing import Any, Union + import cv2 import numpy as np from PIL import Image @@ -47,7 +48,9 @@ def to_grayscale(img: np.ndarray) -> np.ndarray: return cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) @staticmethod - def enhance_contrast(gray_img: np.ndarray, clip_limit: float = 2.0, tile_grid_size: tuple[int, int] = (8, 8)) -> np.ndarray: + def enhance_contrast( + gray_img: np.ndarray, clip_limit: float = 2.0, tile_grid_size: tuple[int, int] = (8, 8) + ) -> np.ndarray: """Apply Contrast Limited Adaptive Histogram Equalization (CLAHE).""" clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=tile_grid_size) return clahe.apply(gray_img) @@ -96,7 +99,9 @@ def deskew(img: np.ndarray) -> np.ndarray: (h, w) = img.shape[:2] center = (w // 2, h // 2) m = cv2.getRotationMatrix2D(center, angle, 1.0) - rotated = cv2.warpAffine(img, m, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) + rotated = cv2.warpAffine( + img, m, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE + ) return rotated def preprocess( diff --git a/tests/test_cli.py b/tests/test_cli.py index 236659a..7be6532 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ """Unit tests for Typer CLI commands.""" from typer.testing import CliRunner + from pakdocling.cli import app runner = CliRunner() diff --git a/tests/test_cnic_extractor.py b/tests/test_cnic_extractor.py index 4ea840e..6d3b690 100644 --- a/tests/test_cnic_extractor.py +++ b/tests/test_cnic_extractor.py @@ -1,7 +1,7 @@ """Unit tests for CNIC Extractor.""" from pakdocling.extractors.cnic import CNICExtractor -from pakdocling.models.schema import CNICVariant, Gender +from pakdocling.models.schema import Gender def test_cnic_extractor_male_new_blue() -> None: diff --git a/tests/test_ocr.py b/tests/test_ocr.py index 14f4c93..03f9265 100644 --- a/tests/test_ocr.py +++ b/tests/test_ocr.py @@ -1,6 +1,7 @@ """Unit tests for OCR engines interface and MockOCREngine.""" import numpy as np + from pakdocling.ocr import MockOCREngine, OCRResultItem diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ea91e5d..fedde0e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,13 +1,19 @@ """Unit tests for DocumentPipeline with MockOCREngine.""" import numpy as np + from pakdocling.models import DocumentType from pakdocling.ocr import MockOCREngine from pakdocling.pipeline import DocumentPipeline, extract_document def test_pipeline_cnic_extraction() -> None: - mock_text = "PAKISTAN NATIONAL IDENTITY CARD\nName: Ali Raza\nCNIC: 35202-1234567-1\nDate of Birth: 01.01.1990" + mock_text = ( + "PAKISTAN NATIONAL IDENTITY CARD\n" + "Name: Ali Raza\n" + "CNIC: 35202-1234567-1\n" + "Date of Birth: 01.01.1990" + ) mock_ocr = MockOCREngine(mock_text=mock_text) pipeline = DocumentPipeline(ocr_engine=mock_ocr) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index e71158a..db74fb9 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -1,8 +1,8 @@ """Unit tests for OpenCV image preprocessing module.""" import numpy as np -import pytest from PIL import Image + from pakdocling.preprocessing import ImagePreprocessor From 867082d685acfacecee515a22d0f5018db27b735 Mon Sep 17 00:00:00 2001 From: Illiyin Date: Tue, 11 Aug 2026 22:55:43 +0500 Subject: [PATCH 3/6] style: fix ruff formatting, line length limits, and mypy type checks --- pakdocling/extractors/cnic.py | 8 ++++++-- pakdocling/ocr/engine.py | 2 +- pakdocling/pipeline.py | 7 ++++--- pakdocling/preprocessing/image.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pakdocling/extractors/cnic.py b/pakdocling/extractors/cnic.py index fac87e4..ce23ae5 100644 --- a/pakdocling/extractors/cnic.py +++ b/pakdocling/extractors/cnic.py @@ -65,8 +65,12 @@ def _extract_dates( for i, line in enumerate(lines): line_lower = line.lower() - m = self.DATE_REGEX.search(line) - found_date = f"{m.group(1)}.{m.group(2)}.{m.group(3)}" if m else None + search_match = self.DATE_REGEX.search(line) + found_date = ( + f"{search_match.group(1)}.{search_match.group(2)}.{search_match.group(3)}" + if search_match + else None + ) if "birth" in line_lower or "dob" in line_lower: if found_date: diff --git a/pakdocling/ocr/engine.py b/pakdocling/ocr/engine.py index 6eda734..5eccf13 100644 --- a/pakdocling/ocr/engine.py +++ b/pakdocling/ocr/engine.py @@ -40,7 +40,7 @@ def __init__(self, languages: Union[list[str], None] = None, gpu: bool = False) def _get_reader(self) -> Any: if self._reader is None: try: - import easyocr # type: ignore[import-untyped] + import easyocr # type: ignore[import-untyped,import-not-found] self._reader = easyocr.Reader(self.languages, gpu=self.gpu) except ImportError as e: diff --git a/pakdocling/pipeline.py b/pakdocling/pipeline.py index 5483c23..cee324f 100644 --- a/pakdocling/pipeline.py +++ b/pakdocling/pipeline.py @@ -1,10 +1,10 @@ """Document Intelligence Pipeline orchestrator.""" import time -from typing import Union +from typing import Union, cast import numpy as np -from PIL import Image +from PIL import Image # type: ignore[import-untyped] from pakdocling.extractors.base import BaseExtractor from pakdocling.extractors.cnic import CNICExtractor @@ -14,6 +14,7 @@ from pakdocling.models.schema import ( CNICData, DocumentType, + ExtractedDocumentData, ExtractionResult, ) from pakdocling.ocr.engine import BaseOCREngine, EasyOCREngine @@ -85,7 +86,7 @@ def extract( target_doc_type = self.detect_document_type(raw_text) extractor = self.extractors.get(target_doc_type, CNICExtractor()) - extracted_data = extractor.extract(ocr_items, raw_text) + extracted_data = cast(ExtractedDocumentData, extractor.extract(ocr_items, raw_text)) processing_time = round((time.time() - start_time) * 1000.0, 2) diff --git a/pakdocling/preprocessing/image.py b/pakdocling/preprocessing/image.py index 0468d4d..75ef7ad 100644 --- a/pakdocling/preprocessing/image.py +++ b/pakdocling/preprocessing/image.py @@ -5,7 +5,7 @@ import cv2 import numpy as np -from PIL import Image +from PIL import Image # type: ignore[import-untyped] class ImagePreprocessor: From b5a05444e9e403be92b460327631aab46e2c5a1e Mon Sep 17 00:00:00 2001 From: Illiyin Date: Wed, 12 Aug 2026 01:15:16 +0500 Subject: [PATCH 4/6] fix(ci): update mypy config in pyproject.toml to ignore missing third-party imports and auto-detect runner python version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4d19120..e8e8862 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,10 @@ line-length = 100 select = ["E", "F", "I", "W"] [tool.mypy] -python_version = "3.10" warn_return_any = true warn_unused_configs = true disallow_untyped_defs = true +ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests"] From 13d490c3e342fd262597e1dcd54f0a3d8a08fb18 Mon Sep 17 00:00:00 2001 From: Illiyin Date: Wed, 12 Aug 2026 01:28:33 +0500 Subject: [PATCH 5/6] feat: align public API with Docling paradigms (DocumentConverter, convert, ConversionResult) --- README.md | 49 +++++++++++++++------------ pakdocling/__init__.py | 8 ++++- pakdocling/cli.py | 63 +++++++++++++++++++++++++++++------ pakdocling/models/__init__.py | 2 ++ pakdocling/models/schema.py | 31 +++++++++++++++-- pakdocling/pipeline.py | 52 ++++++++++++++++++----------- tests/test_pipeline.py | 24 ++++++++----- 7 files changed, 167 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 6f01abe..5104680 100644 --- a/README.md +++ b/README.md @@ -42,42 +42,48 @@ pip install -e ".[dev]" --- -## 💻 Python API Usage +## 💻 Python API Usage (Docling-Aligned) -### 1. Extracting Data from a CNIC Image +### 1. Converting a Document Image with `DocumentConverter` ```python -from pakdocling import extract_document +from pakdocling import DocumentConverter -# Auto-detect or specify doc_type='cnic' -result = extract_document("path/to/cnic_card.jpg", doc_type="cnic") +# Initialize converter +converter = DocumentConverter() + +# Convert CNIC or educational document image +result = converter.convert("path/to/cnic_card.jpg", doc_type="cnic") if result.success: - cnic = result.data + cnic = result.document # Pydantic model (CNICData) print(f"CNIC Number: {cnic.cnic_number}") print(f"Name: {cnic.full_name}") print(f"Father Name: {cnic.father_name}") print(f"Gender: {cnic.gender}") print(f"Date of Birth: {cnic.date_of_birth}") print(f"Card Variant: {cnic.variant}") + +# Docling-style export methods +json_output = result.export_to_json(indent=2) +dict_output = result.export_to_dict() ``` -### 2. Outputting Validated JSON +### 2. Functional Conversion Helper `convert()` ```python -from pakdocling import DocumentPipeline, DocumentType +from pakdocling import convert, DocumentType -pipeline = DocumentPipeline() -result = pipeline.extract("matric_certificate.png", doc_type=DocumentType.MATRIC) +result = convert("matric_certificate.png", doc_type=DocumentType.MATRIC) -# Export Pydantic model to JSON -print(result.model_dump_json(indent=2)) +# Export conversion result directly to formatted JSON +print(result.export_to_json(indent=2)) ``` ### 3. Offline & Fast Testing with `MockOCREngine` ```python -from pakdocling import DocumentPipeline, MockOCREngine +from pakdocling import DocumentConverter, MockOCREngine mock_ocr = MockOCREngine( mock_text=""" @@ -89,10 +95,11 @@ Graduation Year: 2023 """ ) -pipeline = DocumentPipeline(ocr_engine=mock_ocr) -result = pipeline.extract("dummy.png", doc_type="degree") -print(result.data.degree_title) # "Bachelor of Science in Software Engineering" -print(result.data.cgpa) # 3.85 +converter = DocumentConverter(ocr_engine=mock_ocr) +result = converter.convert("dummy.png", doc_type="degree") + +print(result.document.degree_title) # "Bachelor of Science in Software Engineering" +print(result.document.cgpa) # 3.85 ``` --- @@ -105,11 +112,11 @@ print(result.data.cgpa) # 3.85 # Check version & supported document schemas pakdocling info -# Extract data from an image file and print formatted JSON -pakdocling extract sample_cnic.jpg --doc-type cnic +# Convert document image and print formatted JSON (Docling API) +pakdocling convert sample_cnic.jpg --doc-type cnic -# Extract and save JSON to a file -pakdocling extract degree_transcript.png --doc-type auto -o output.json +# Convert and save JSON output to file +pakdocling convert degree_transcript.png --doc-type auto -o output.json ``` --- diff --git a/pakdocling/__init__.py b/pakdocling/__init__.py index 7f9ef9f..85aaecb 100644 --- a/pakdocling/__init__.py +++ b/pakdocling/__init__.py @@ -10,7 +10,9 @@ from pakdocling.models import ( CNICData, CNICVariant, + ConversionResult, DocumentType, + ExtractedDocumentData, ExtractionResult, Gender, IntermediateCertificateData, @@ -18,13 +20,16 @@ UniversityDegreeData, ) from pakdocling.ocr import BaseOCREngine, EasyOCREngine, MockOCREngine, OCRResultItem -from pakdocling.pipeline import DocumentPipeline, extract_document +from pakdocling.pipeline import DocumentConverter, DocumentPipeline, convert, extract_document from pakdocling.preprocessing import ImagePreprocessor __version__ = "0.0.1" __all__ = [ "__version__", + "DocumentConverter", + "ConversionResult", + "convert", "DocumentType", "CNICVariant", "Gender", @@ -32,6 +37,7 @@ "MatricCertificateData", "IntermediateCertificateData", "UniversityDegreeData", + "ExtractedDocumentData", "ExtractionResult", "BaseExtractor", "CNICExtractor", diff --git a/pakdocling/cli.py b/pakdocling/cli.py index 34330cd..e0dab9a 100644 --- a/pakdocling/cli.py +++ b/pakdocling/cli.py @@ -8,11 +8,11 @@ from rich.panel import Panel from pakdocling import __version__ -from pakdocling.pipeline import DocumentPipeline +from pakdocling.pipeline import DocumentConverter app = typer.Typer( name="pakdocling", - help="Pakistani Document Intelligence Library - Structured Data Extraction CLI", + help="Pakistani Document Intelligence Library - Docling-aligned CLI", add_completion=False, ) @@ -20,7 +20,7 @@ @app.command() -def extract( +def convert( image_path: Path = typer.Argument( ..., exists=True, @@ -52,18 +52,18 @@ def extract( help="Enable OpenCV deskewing and image enhancement", ), ) -> None: - """Extract structured data from a Pakistani document image.""" - console.print(f"[bold blue]Processing document:[/bold blue] {image_path}") + """Convert document image into structured JSON format (Docling API).""" + console.print(f"[bold blue]Converting document:[/bold blue] {image_path}") try: - pipeline = DocumentPipeline() - result = pipeline.extract( - image_source=str(image_path), + converter = DocumentConverter() + result = converter.convert( + source=str(image_path), doc_type=doc_type, do_preprocess=preprocess, ) - json_data = result.model_dump_json(indent=2 if pretty else None) + json_data = result.export_to_json(indent=2 if pretty else None) if output: output.write_text(json_data, encoding="utf-8") @@ -72,10 +72,53 @@ def extract( typer.echo(json_data) except Exception as e: - console.print(f"[bold red]Extraction Error:[/bold red] {e}") + console.print(f"[bold red]Conversion Error:[/bold red] {e}") raise typer.Exit(code=1) from e +@app.command() +def extract( + image_path: Path = typer.Argument( + ..., + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Path to Pakistani document image (JPG, PNG, WebP, TIFF)", + ), + doc_type: str = typer.Option( + "auto", + "--doc-type", + "-t", + help="Document type: auto, cnic, matric, intermediate, degree", + ), + output: Optional[Path] = typer.Option( + None, + "--output", + "-o", + help="Path to output JSON file to save extraction results", + ), + pretty: bool = typer.Option( + True, + "--pretty/--compact", + help="Format output JSON with pretty printing", + ), + preprocess: bool = typer.Option( + True, + "--preprocess/--no-preprocess", + help="Enable OpenCV deskewing and image enhancement", + ), +) -> None: + """Alias command for convert.""" + convert( + image_path=image_path, + doc_type=doc_type, + output=output, + pretty=pretty, + preprocess=preprocess, + ) + + @app.command() def info() -> None: """Display pakdocling installation details and supported document models.""" diff --git a/pakdocling/models/__init__.py b/pakdocling/models/__init__.py index 1059ccd..4cc60f7 100644 --- a/pakdocling/models/__init__.py +++ b/pakdocling/models/__init__.py @@ -3,6 +3,7 @@ from pakdocling.models.schema import ( CNICData, CNICVariant, + ConversionResult, DocumentType, ExtractedDocumentData, ExtractionResult, @@ -21,5 +22,6 @@ "IntermediateCertificateData", "UniversityDegreeData", "ExtractedDocumentData", + "ConversionResult", "ExtractionResult", ] diff --git a/pakdocling/models/schema.py b/pakdocling/models/schema.py index 5bd5b99..e119311 100644 --- a/pakdocling/models/schema.py +++ b/pakdocling/models/schema.py @@ -177,8 +177,8 @@ class UniversityDegreeData(BaseModel): ] -class ExtractionResult(BaseModel): - """Wrapper result for all extraction pipelines.""" +class ConversionResult(BaseModel): + """Wrapper result for all pakdocling conversions (Docling-aligned format).""" document_type: DocumentType = Field(description="Identified document type") success: bool = Field(description="Indicates whether extraction succeeded") @@ -188,8 +188,33 @@ class ExtractionResult(BaseModel): IntermediateCertificateData, UniversityDegreeData, dict[str, Any], - ] = Field(description="Extracted structured data model or dictionary") + ] = Field(description="Extracted structured document model or dictionary") errors: list[str] = Field( default_factory=list, description="List of warnings or extraction errors" ) processing_time_ms: float = Field(default=0.0, description="Processing time in milliseconds") + + @property + def document( + self, + ) -> Union[ + CNICData, + MatricCertificateData, + IntermediateCertificateData, + UniversityDegreeData, + dict[str, Any], + ]: + """Alias property matching Docling's .document attribute access.""" + return self.data + + def export_to_json(self, indent: Union[int, None] = None) -> str: + """Export conversion result to JSON string (Docling format).""" + return self.model_dump_json(indent=indent) + + def export_to_dict(self) -> dict[str, Any]: + """Export conversion result to Python dictionary (Docling format).""" + return self.model_dump() + + +# Backward compatibility alias +ExtractionResult = ConversionResult diff --git a/pakdocling/pipeline.py b/pakdocling/pipeline.py index cee324f..f97f454 100644 --- a/pakdocling/pipeline.py +++ b/pakdocling/pipeline.py @@ -13,16 +13,16 @@ from pakdocling.extractors.matric import MatricExtractor from pakdocling.models.schema import ( CNICData, + ConversionResult, DocumentType, ExtractedDocumentData, - ExtractionResult, ) from pakdocling.ocr.engine import BaseOCREngine, EasyOCREngine from pakdocling.preprocessing.image import ImagePreprocessor -class DocumentPipeline: - """Unified Document Intelligence Pipeline for Pakistani documents.""" +class DocumentConverter: + """Docling-aligned Document Converter for Pakistani documents.""" def __init__( self, @@ -46,21 +46,21 @@ def detect_document_type(self, raw_text: str) -> DocumentType: return doc_type return DocumentType.CNIC # Default fallback if unknown - def extract( + def convert( self, - image_source: Union[str, bytes, np.ndarray, Image.Image], + source: Union[str, bytes, np.ndarray, Image.Image], doc_type: Union[DocumentType, str] = DocumentType.AUTO, do_preprocess: bool = True, - ) -> ExtractionResult: - """Extract structured JSON fields from document image. + ) -> ConversionResult: + """Convert document image into a structured ConversionResult (Docling-aligned format). Args: - image_source: Image file path, raw bytes, OpenCV ndarray, or PIL Image. + source: Image file path, raw bytes, OpenCV ndarray, or PIL Image. doc_type: Document type ('cnic', 'matric', 'intermediate', 'degree', or 'auto'). do_preprocess: Apply OpenCV deskewing and contrast enhancement prior to OCR. Returns: - ExtractionResult object containing structured data model and metadata. + ConversionResult object containing structured data model and metadata. """ start_time = time.time() errors: list[str] = [] @@ -68,10 +68,10 @@ def extract( try: # Preprocess image if do_preprocess: - prep_result = self.preprocessor.preprocess(image_source) + prep_result = self.preprocessor.preprocess(source) target_image = prep_result["processed_gray"] else: - target_image = self.preprocessor.load_image(image_source) + target_image = self.preprocessor.load_image(source) # Perform OCR ocr_items, raw_text = self.ocr_engine.extract_text(target_image) @@ -90,7 +90,7 @@ def extract( processing_time = round((time.time() - start_time) * 1000.0, 2) - return ExtractionResult( + return ConversionResult( document_type=target_doc_type, success=True, data=extracted_data, @@ -101,7 +101,7 @@ def extract( except Exception as e: processing_time = round((time.time() - start_time) * 1000.0, 2) errors.append(str(e)) - return ExtractionResult( + return ConversionResult( document_type=DocumentType.AUTO, success=False, data=CNICData(), @@ -109,12 +109,26 @@ def extract( processing_time_ms=processing_time, ) + def extract( + self, + image_source: Union[str, bytes, np.ndarray, Image.Image], + doc_type: Union[DocumentType, str] = DocumentType.AUTO, + do_preprocess: bool = True, + ) -> ConversionResult: + """Alias for convert() method.""" + return self.convert(source=image_source, doc_type=doc_type, do_preprocess=do_preprocess) + -def extract_document( - image_source: Union[str, bytes, np.ndarray, Image.Image], +def convert( + source: Union[str, bytes, np.ndarray, Image.Image], doc_type: Union[DocumentType, str] = DocumentType.AUTO, ocr_engine: Union[BaseOCREngine, None] = None, -) -> ExtractionResult: - """Convenience functional interface for pakdocling extraction.""" - pipeline = DocumentPipeline(ocr_engine=ocr_engine) - return pipeline.extract(image_source=image_source, doc_type=doc_type) +) -> ConversionResult: + """Convenience Docling-aligned functional interface for document conversion.""" + converter = DocumentConverter(ocr_engine=ocr_engine) + return converter.convert(source=source, doc_type=doc_type) + + +# Backward compatibility aliases +DocumentPipeline = DocumentConverter +extract_document = convert diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index fedde0e..8d6a88e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,13 +1,13 @@ -"""Unit tests for DocumentPipeline with MockOCREngine.""" +"""Unit tests for DocumentConverter and convert with MockOCREngine.""" import numpy as np +from pakdocling import DocumentConverter, convert from pakdocling.models import DocumentType from pakdocling.ocr import MockOCREngine -from pakdocling.pipeline import DocumentPipeline, extract_document -def test_pipeline_cnic_extraction() -> None: +def test_converter_cnic_conversion() -> None: mock_text = ( "PAKISTAN NATIONAL IDENTITY CARD\n" "Name: Ali Raza\n" @@ -16,23 +16,31 @@ def test_pipeline_cnic_extraction() -> None: ) mock_ocr = MockOCREngine(mock_text=mock_text) - pipeline = DocumentPipeline(ocr_engine=mock_ocr) + converter = DocumentConverter(ocr_engine=mock_ocr) dummy_img = np.zeros((100, 100, 3), dtype=np.uint8) - result = pipeline.extract(dummy_img, doc_type=DocumentType.AUTO, do_preprocess=False) + result = converter.convert(dummy_img, doc_type=DocumentType.AUTO, do_preprocess=False) assert result.success is True assert result.document_type == DocumentType.CNIC + assert result.document.cnic_number == "35202-1234567-1" # type: ignore[union-attr] assert result.data.cnic_number == "35202-1234567-1" # type: ignore[union-attr] + # Test export methods + json_export = result.export_to_json(indent=2) + assert "35202-1234567-1" in json_export -def test_pipeline_matric_auto_detection() -> None: + dict_export = result.export_to_dict() + assert dict_export["document_type"] == DocumentType.CNIC + + +def test_convert_functional_interface() -> None: mock_text = "BISE Lahore\nSECONDARY SCHOOL CERTIFICATE\nRoll No: 123456\nName: Usman Ali" mock_ocr = MockOCREngine(mock_text=mock_text) dummy_img = np.zeros((50, 50, 3), dtype=np.uint8) - result = extract_document(dummy_img, doc_type="auto", ocr_engine=mock_ocr) + result = convert(dummy_img, doc_type="auto", ocr_engine=mock_ocr) assert result.success is True assert result.document_type == DocumentType.MATRIC - assert result.data.roll_number == "123456" # type: ignore[union-attr] + assert result.document.roll_number == "123456" # type: ignore[union-attr] From 19bbc85f270d4e99929c0074f5d7ea829d504ca6 Mon Sep 17 00:00:00 2001 From: Illiyin Date: Wed, 12 Aug 2026 01:42:29 +0500 Subject: [PATCH 6/6] feat: integrate fastNlMeansDenoising, urdu+english language defaults, and bounding box annotation --- pakdocling/ocr/engine.py | 15 +++++++++++---- pakdocling/preprocessing/image.py | 27 +++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/pakdocling/ocr/engine.py b/pakdocling/ocr/engine.py index 5eccf13..180bcda 100644 --- a/pakdocling/ocr/engine.py +++ b/pakdocling/ocr/engine.py @@ -32,9 +32,15 @@ def extract_text(self, image: Union[str, bytes, np.ndarray]) -> tuple[list[OCRRe class EasyOCREngine(BaseOCREngine): """EasyOCR implementation with lazy loading to defer PyTorch/EasyOCR initialization.""" - def __init__(self, languages: Union[list[str], None] = None, gpu: bool = False) -> None: - self.languages = languages or ["en"] + def __init__( + self, + languages: Union[list[str], None] = None, + gpu: bool = False, + min_confidence: float = 0.2, + ) -> None: + self.languages = languages or ["ur", "en"] self.gpu = gpu + self.min_confidence = min_confidence self._reader: Any = None def _get_reader(self) -> Any: @@ -63,12 +69,13 @@ def extract_text(self, image: Union[str, bytes, np.ndarray]) -> tuple[list[OCRRe for bbox, text, prob in raw_results: cleaned_text = str(text).strip() - if cleaned_text: + confidence = float(prob) + if cleaned_text and confidence >= self.min_confidence: items.append( OCRResultItem( text=cleaned_text, bbox=[[float(pt[0]), float(pt[1])] for pt in bbox], - confidence=float(prob), + confidence=confidence, ) ) text_lines.append(cleaned_text) diff --git a/pakdocling/preprocessing/image.py b/pakdocling/preprocessing/image.py index 75ef7ad..de81e93 100644 --- a/pakdocling/preprocessing/image.py +++ b/pakdocling/preprocessing/image.py @@ -56,12 +56,31 @@ def enhance_contrast( return clahe.apply(gray_img) @staticmethod - def denoise(gray_img: np.ndarray, kernel_size: int = 3) -> np.ndarray: - """Denoise grayscale image using Gaussian blur.""" - if kernel_size % 2 == 0: - kernel_size += 1 + def denoise(gray_img: np.ndarray, h: float = 10.0, method: str = "fast_nl_means") -> np.ndarray: + """Denoise grayscale image while preserving edges (fastNlMeansDenoising / GaussianBlur).""" + if method == "fast_nl_means": + return cv2.fastNlMeansDenoising(gray_img, h=int(h)) + kernel_size = int(h) if int(h) % 2 != 0 else int(h) + 1 return cv2.GaussianBlur(gray_img, (kernel_size, kernel_size), 0) + @staticmethod + def annotate_boxes( + img: np.ndarray, + ocr_items: list[Any], + color: tuple[int, int, int] = (0, 255, 0), + thickness: int = 2, + ) -> np.ndarray: + """Draw green bounding box rectangles around detected text regions.""" + annotated = img.copy() + for item in ocr_items: + bbox = getattr(item, "bbox", None) + conf = getattr(item, "confidence", 1.0) + if bbox and len(bbox) >= 4 and conf >= 0.2: + tl = (int(bbox[0][0]), int(bbox[0][1])) + br = (int(bbox[2][0]), int(bbox[2][1])) + cv2.rectangle(annotated, tl, br, color, thickness) + return annotated + @staticmethod def adaptive_threshold(gray_img: np.ndarray) -> np.ndarray: """Apply adaptive thresholding for binarization."""