Opendataloader PDF

OpenDataLoader PDF — AI-Ready Document Parsing

Overview

Parse PDF documents into clean, structured data optimized for AI consumption. Extract text with layout preservation, tables as structured JSON, images with captions, and rich metadata. Ideal for RAG pipelines, document analysis, and data extraction workflows.

Instructions

Step 1: Choose Your Parsing Strategy

Step 2: Set Up the Python Pipeline

pip install pdfplumber pymupdf camelot-py[cv] Pillow
# For OCR: pip install pytesseract easyocr

Step 3: Extract Text with Layout Awareness

import pdfplumber

def extract_text_structured(pdf_path):
    """Extract text preserving document structure."""
    pages = []
    with pdfplumber.open(pdf_path) as pdf:
        for i, page in enumerate(pdf.pages):
            text = page.extract_text(layout=True)
            words = page.extract_words(keep_blank_chars=True, extra_attrs=['fontname', 'size'])
            headers = [w for w in words if w['size'] > 14]
            pages.append({
                'page': i + 1, 'text': text,
                'headers': [h['text'] for h in headers],
                'word_count': len(words)
            })
    return pages

Step 4: Extract Tables as Structured Data

def extract_tables(pdf_path):
    """Extract tables as list of dicts."""
    results = []
    with pdfplumber.open(pdf_path) as pdf:
        for i, page in enumerate(pdf.pages):
            tables = page.extract_tables({"vertical_strategy": "text",
                "horizontal_strategy": "text", "snap_tolerance": 5})
            for j, table in enumerate(tables):
                if not table or len(table) < 2:
                    continue
                headers = [str(h).strip() for h in table[0]]
                rows = []
                for row in table[1:]:
                    row_dict = {}
                    for k, cell in enumerate(row):
                        key = headers[k] if k < len(headers) else f'col_{k}'
                        row_dict[key] = str(cell).strip() if cell else ''
                    rows.append(row_dict)
                results.append({'page': i+1, 'table_index': j, 'headers': headers,
                                'rows': rows, 'row_count': len(rows)})
    return results

Step 5: Extract Images and Metadata

import fitz  # PyMuPDF

def extract_images(pdf_path, output_dir='./images'):
    """Extract embedded images from PDF."""
    import os
    os.makedirs(output_dir, exist_ok=True)
    doc = fitz.open(pdf_path)
    images = []
    for page_num in range(len(doc)):
        page = doc[page_num]
        for img_idx, img in enumerate(page.get_images(full=True)):
            base_image = doc.extract_image(img[0])
            filename = f'page{page_num+1}_img{img_idx+1}.{base_image["ext"]}'
            filepath = os.path.join(output_dir, filename)
            with open(filepath, 'wb') as f:
                f.write(base_image['image'])
            images.append({'page': page_num+1, 'file': filepath,
                           'format': base_image['ext'],
                           'width': base_image.get('width'),
                           'height': base_image.get('height')})
    return images

def extract_metadata(pdf_path):
    """Extract PDF metadata."""
    doc = fitz.open(pdf_path)
    meta = doc.metadata
    return {'title': meta.get('title', ''), 'author': meta.get('author', ''),
            'pages': len(doc), 'encrypted': doc.is_encrypted}

Step 6: Build RAG-Ready Chunks

def chunk_for_rag(pages, chunk_size=500, overlap=50):
    """Split pages into overlapping chunks for RAG."""
    chunks = []
    for page in pages:
        text = page['text']
        if not text:
            continue
        words = text.split()
        for i in range(0, len(words), chunk_size - overlap):
            chunk_words = words[i:i + chunk_size]
            if len(chunk_words) < 20:
                continue
            chunks.append({'text': ' '.join(chunk_words), 'page': page['page'],
                           'chunk_index': len(chunks), 'word_count': len(chunk_words)})
    return chunks

Step 7: Full Pipeline — PDF to AI-Ready JSON

import json

def pdf_to_ai_ready(pdf_path, output_path=None):
    """Complete pipeline: PDF to structured AI-ready data."""
    result = {
        'source': pdf_path,
        'metadata': extract_metadata(pdf_path),
        'pages': extract_text_structured(pdf_path),
        'tables': extract_tables(pdf_path),
        'images': extract_images(pdf_path),
    }
    result['chunks'] = chunk_for_rag(result['pages'])
    result['stats'] = {
        'total_pages': len(result['pages']),
        'total_tables': len(result['tables']),
        'total_images': len(result['images']),
        'total_chunks': len(result['chunks']),
    }
    if output_path:
        with open(output_path, 'w') as f:
            json.dump(result, f, indent=2, default=str)
    return result

Step 8: Handle Scanned PDFs with OCR

import pytesseract
from PIL import Image

def ocr_pdf(pdf_path):
    """OCR scanned PDF pages."""
    doc = fitz.open(pdf_path)
    pages = []
    for i in range(len(doc)):
        pix = doc[i].get_pixmap(dpi=300)
        img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
        text = pytesseract.image_to_string(img)
        pages.append({'page': i + 1, 'text': text, 'method': 'ocr'})
    return pages

Examples

Example 1: Extract Data from a Quarterly Financial Report

A finance team processes a 48-page quarterly report PDF to feed into their analysis pipeline:

result = pdf_to_ai_ready('Q4-2025-Annual-Report-Acme-Corp.pdf', 'acme_q4.json')
print(result['stats'])
# {'total_pages': 48, 'total_tables': 12, 'total_images': 7, 'total_chunks': 34}

# Extract the revenue table from page 8
revenue_table = [t for t in result['tables'] if t['page'] == 8][0]
print(revenue_table['headers'])
# ['Quarter', 'Revenue ($M)', 'Growth (%)', 'Operating Margin']
print(revenue_table['rows'][0])
# {'Quarter': 'Q4 2025', 'Revenue ($M)': '847.3', 'Growth (%)': '12.4', 'Operating Margin': '23.1%'}

# Feed chunks into RAG system
for chunk in result['chunks']:
    embed_and_store(chunk['text'], metadata={'page': chunk['page'], 'source': 'acme_q4'})

Example 2: Batch Process Legal Contracts for Clause Extraction

A legal team processes a directory of scanned contract PDFs to identify key clauses:

import os

contract_dir = './contracts/vendor-agreements/'
for filename in os.listdir(contract_dir):
    if not filename.endswith('.pdf'):
        continue
    pdf_path = os.path.join(contract_dir, filename)

    # Try text extraction first, fall back to OCR for scanned docs
    result = pdf_to_ai_ready(pdf_path)
    total_text = sum(len(p['text'] or '') for p in result['pages'])
    if total_text < 100:  # likely scanned
        result['pages'] = ocr_pdf(pdf_path)
        result['chunks'] = chunk_for_rag(result['pages'])

    print(f"{filename}: {result['stats']['total_pages']} pages, "
          f"{result['stats']['total_chunks']} chunks, "
          f"{result['stats']['total_tables']} tables")
    # Output: "vendor-agreement-globaltech-2025.pdf: 24 pages, 18 chunks, 3 tables"

    # Save structured output for downstream AI analysis
    pdf_to_ai_ready(pdf_path, pdf_path.replace('.pdf', '.json'))

Guidelines

• Always check font encoding — some PDFs produce garbled text; try PyMuPDF if pdfplumber fails
• Use Camelot for bordered tables — pdfplumber works better for borderless tables
• Process large PDFs page-by-page — stream results to disk to avoid memory issues
• Vision LLM fallback — for complex layouts, send page screenshots to Claude or GPT-4o as images
• Validate extracted data — spot-check tables and text against the original PDF before using in production
• Handle encrypted PDFs — check doc.is_encrypted and prompt for password before extraction

References

• pdfplumber — detailed PDF text and table extraction
• PyMuPDF — fast PDF processing with image extraction
• Camelot — accurate table extraction from PDFs