Files and paths

from pathlib import Path

p = Path("data") / "raw" / "report.final.txt"

print(p)               # the full path
print(p.name)          # file name with extension
print(p.stem)          # file name without the last extension
print(p.suffix)        # last extension
print(p.suffixes)      # all extensions
print(p.parent)        # containing folder
print(p.parts)         # every segment
print(p.with_suffix(".md"))

from pathlib import Path

print("cwd:", Path.cwd())

SCRIPT_DIR = Path(__file__).resolve().parent     # folder containing this file
PROJECT_ROOT = SCRIPT_DIR.parent                 # one level up
DATA_DIR = PROJECT_ROOT / "data"

print("data dir:", DATA_DIR)

from pathlib import Path

out_dir = Path("output") / "chunks"
print(out_dir.exists())

out_dir.mkdir(parents=True, exist_ok=True)   # create it and any missing parents
print(out_dir.exists(), out_dir.is_dir(), out_dir.is_file())

from pathlib import Path

note = Path("notes") / "intro.txt"
note.parent.mkdir(parents=True, exist_ok=True)

note.write_text("Retrieval needs clean chunks.\nChunks need good boundaries.\n", encoding="utf-8")

text = note.read_text(encoding="utf-8")
print(text)
print(len(text.splitlines()), "lines,", len(text), "characters")

from pathlib import Path

log = Path("run.log")

with log.open("w", encoding="utf-8") as f:
    f.write("started\n")

for step in ["embed", "index", "query"]:
    with log.open("a", encoding="utf-8") as f:
        f.write(f"done: {step}\n")

print(log.read_text(encoding="utf-8"))

from pathlib import Path

path = Path("corpus.txt")
path.write_text("first line\n\nsecond line\nthird line\n", encoding="utf-8")

non_empty = 0
with path.open(encoding="utf-8") as f:
    for line_no, line in enumerate(f, start=1):
        line = line.rstrip("\n")
        if not line:
            continue
        non_empty += 1
        print(line_no, line)

print("non-empty lines:", non_empty)

from pathlib import Path

root = Path("kb")
for rel in ["intro.md", "guides/rag.md", "guides/agents.md", "guides/notes.txt"]:
    p = root / rel
    p.parent.mkdir(parents=True, exist_ok=True)
    p.write_text(f"# {p.stem}\n", encoding="utf-8")

print(sorted(p.name for p in root.glob("*.md")))
print(sorted(p.as_posix() for p in root.rglob("*.md")))
print(sorted(p.name for p in root.iterdir()))

from pathlib import Path

for p in sorted(Path("kb").rglob("*.md")):
    print(f"{p.as_posix():<22} {p.stat().st_size:>4} bytes")

import csv
from pathlib import Path

rows = [
    {"question": "What is RAG?", "answer": "Retrieval, then generation", "score": 0.9},
    {"question": "Define chunking", "answer": "Splitting text, with overlap", "score": 0.7},
]

path = Path("eval.csv")
with path.open("w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=["question", "answer", "score"])
    writer.writeheader()
    writer.writerows(rows)

print(path.read_text(encoding="utf-8"))

with path.open(newline="", encoding="utf-8") as f:
    for row in csv.DictReader(f):
        print(row["question"], "->", float(row["score"]))

from pathlib import Path

path = Path("legacy.txt")
path.write_bytes("café ".encode("utf-8") + b"\xff\xfe broken")

try:
    path.read_text(encoding="utf-8")
except UnicodeDecodeError as err:
    print("strict read failed:", err.reason)

print(path.read_text(encoding="utf-8", errors="replace"))

from pathlib import Path


def chunk_words(text: str, size: int = 8, overlap: int = 2) -> list[str]:
    if overlap >= size:
        raise ValueError("overlap must be smaller than size")
    words = text.split()
    step = size - overlap
    chunks = []
    for start in range(0, len(words), step):
        chunks.append(" ".join(words[start:start + size]))
        if start + size >= len(words):
            break
    return chunks


doc = Path("doc.txt")
doc.write_text(
    "Retrieval augmented generation first searches a knowledge base for "
    "relevant passages and then asks the language model to answer using "
    "only those passages as grounding context.",
    encoding="utf-8",
)

for i, chunk in enumerate(chunk_words(doc.read_text(encoding="utf-8")), start=1):
    print(f"[{i}] {chunk}")

import json
from pathlib import Path


def write_json_atomic(path: Path, data) -> None:
    tmp = path.with_suffix(path.suffix + ".tmp")
    tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
    tmp.replace(path)          # atomic rename over the destination


target = Path("index_meta.json")
write_json_atomic(target, {"chunks": 5, "model": "text-embedding-3-small"})
print(target.read_text(encoding="utf-8"))