Loading and chunking

from pathlib import Path

from pypdf import PdfReader


def load_pdf(path: Path) -> list[dict]:
    reader = PdfReader(path)
    pages = []
    for number, page in enumerate(reader.pages, start=1):
        text = page.extract_text() or ""
        pages.append({"source": path.name, "page": number, "text": text})
    return pages

import re


def clean(text: str) -> str:
    text = re.sub(r"Page \d+ of \d+", " ", text)          # page footers
    text = re.sub(r"CONFIDENTIAL - INTERNAL USE", " ", text)  # repeated banners
    text = re.sub(r"-\n(\w)", r"\1", text)                # re-join hyphenated line breaks
    text = re.sub(r"[ \t]+", " ", text)                   # collapse spaces, keep newlines
    text = re.sub(r"\n{3,}", "\n\n", text)                # at most one blank line
    return text.strip()


raw = "Refund Policy\n\n\n\nAnnual plans are refun-\ndable within 30 days.   Page 3 of 9\nCONFIDENTIAL - INTERNAL USE"
print(clean(raw))

def fixed_chunks(text: str, size: int = 12, overlap: int = 3) -> list[str]:
    words = text.split()
    step = size - overlap
    out = []
    for start in range(0, len(words), step):
        out.append(" ".join(words[start:start + size]))
        if start + size >= len(words):
            break
    return out


text = ("Backups run every night at 2 AM. Files are kept for 90 days. "
        "Restores can be started from the web dashboard or the desktop app. "
        "Large restores are delivered as a download link by email.")
for i, chunk in enumerate(fixed_chunks(text), 1):
    print(i, chunk)

import re

SEPARATORS = ["\n\n", "\n", ". ", " "]


def recursive_split(text: str, max_chars: int = 120, seps=SEPARATORS) -> list[str]:
    if len(text) <= max_chars:
        return [text.strip()] if text.strip() else []
    sep, *rest = seps
    parts = text.split(sep)
    chunks, current = [], ""
    for part in parts:
        candidate = f"{current}{sep}{part}" if current else part
        if len(candidate) <= max_chars:
            current = candidate
            continue
        if current:
            chunks.append(current.strip())
        if len(part) > max_chars and rest:
            chunks.extend(recursive_split(part, max_chars, rest))
            current = ""
        else:
            current = part
    if current.strip():
        chunks.append(current.strip())
    return chunks


doc = """Refund policy

Annual plans can be refunded within 30 days of purchase. Monthly plans are not refunded. Refunds go back to the original card.

Backups

Backups run nightly. Files are kept for 90 days."""

for c in recursive_split(doc):
    print(f"[{len(c):>3}] {c!r}")

def markdown_sections(md: str) -> list[dict]:
    sections, path, buffer = [], [], []

    def flush():
        body = " ".join(line.strip() for line in buffer if line.strip())
        if body:
            sections.append({"heading": " > ".join(path), "text": f"{' > '.join(path)}: {body}"})
        buffer.clear()

    for line in md.splitlines():
        if line.startswith("#"):
            flush()
            level = len(line) - len(line.lstrip("#"))
            path[:] = path[: level - 1] + [line.lstrip("#").strip()]
        else:
            buffer.append(line)
    flush()
    return sections


md = """# Admin guide
## Users
Invite users from Settings > Team.
## Security
### Two-factor authentication
Admins must enable 2FA within 7 days.
"""
for s in markdown_sections(md):
    print(s["text"])

import hashlib


def make_chunk_records(source: str, chunks: list[str], **meta) -> list[dict]:
    records = []
    for i, text in enumerate(chunks):
        records.append({
            "id": f"{source}#{i}",
            "source": source,
            "chunk_index": i,
            "text": text,
            "hash": hashlib.sha256(text.encode()).hexdigest()[:12],
            **meta,
        })
    return records


recs = make_chunk_records("admin-guide.md", ["Invite users from Settings > Team."],
                          doc_type="guide", access="all-staff", updated_at="2026-10-28")
print(recs[0])