tawasuyu/brahman

1#!/usr/bin/env python3
2"""Indexador de docs/ de gioser-web → Qdrant.
3
4Recorre crates/apps/gioser-web/docs/, parsea YAML frontmatter,
5trocea cada documento en fragmentos de párrafo, pide embeddings al
6servicio agnóstico y hace upsert a Qdrant.
7
8Uso:
9    python scripts/index-gioser-docs.py                          # usa defaults
10    python scripts/index-gioser-docs.py --rebuild                # recrea colección
11    python scripts/index-gioser-docs.py --docs ./docs --rebuild  # docs custom
12"""
13
14from __future__ import annotations
15
16import argparse
17import hashlib
18import os
19import re
20import sys
21import uuid
22from dataclasses import dataclass
23from pathlib import Path
24
25import httpx
26import yaml
27from qdrant_client import QdrantClient
28from qdrant_client.http import models as qm
29
30
31DEFAULT_DOCS = Path(__file__).resolve().parent.parent / "crates/apps/gioser-web/docs"
32DEFAULT_QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
33DEFAULT_EMBED_URL = os.getenv("EMBEDDINGS_URL", "http://localhost:8001")
34DEFAULT_COLLECTION = os.getenv("QDRANT_COLLECTION", "gioser")
35
36VALID_CAMINOS = {"logos", "uku", "kay", "nomos", "aire", "fuego", "tierra", "agua", "cuerpo", "sombra", "cosmos", "practica", "olvido"}
37FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?\n)---\s*\n(.*)$", re.DOTALL)
38
39
40@dataclass
41class Chunk:
42    doc_id: str
43    chunk_index: int
44    text: str
45    title: str
46    camino: str
47    tags: list[str]
48
49
50def parse_md(path: Path) -> tuple[dict, str]:
51    raw = path.read_text(encoding="utf-8")
52    m = FRONTMATTER_RE.match(raw)
53    if m:
54        meta = yaml.safe_load(m.group(1)) or {}
55        body = m.group(2)
56    else:
57        meta = {}
58        body = raw
59    return meta, body
60
61
62def chunk_body(body: str, min_chars: int = 200, max_chars: int = 900) -> list[str]:
63    """Fragmenta por párrafos respetando un mínimo y un máximo."""
64    paragraphs = [p.strip() for p in re.split(r"\n\s*\n", body) if p.strip()]
65    chunks: list[str] = []
66    buf = ""
67    for p in paragraphs:
68        candidate = f"{buf}\n\n{p}".strip() if buf else p
69        if len(candidate) >= max_chars:
70            if buf:
71                chunks.append(buf)
72            buf = p
73        else:
74            buf = candidate
75            if len(buf) >= min_chars:
76                chunks.append(buf)
77                buf = ""
78    if buf:
79        if chunks and len(buf) < min_chars:
80            chunks[-1] = f"{chunks[-1]}\n\n{buf}"
81        else:
82            chunks.append(buf)
83    return chunks
84
85
86def discover_chunks(docs_dir: Path) -> list[Chunk]:
87    out: list[Chunk] = []
88    for path in sorted(docs_dir.rglob("*.md")):
89        meta, body = parse_md(path)
90        camino = (meta.get("camino") or path.stem).lower()
91        if camino not in VALID_CAMINOS:
92            print(f"  ⚠ saltando {path}: camino '{camino}' inválido", file=sys.stderr)
93            continue
94        title = meta.get("title") or path.stem.replace("-", " ").title()
95        tags = list(meta.get("tags") or [])
96        doc_id = meta.get("id") or hashlib.sha1(str(path).encode()).hexdigest()[:12]
97        for i, chunk in enumerate(chunk_body(body)):
98            out.append(
99                Chunk(
100                    doc_id=doc_id,
101                    chunk_index=i,
102                    text=chunk,
103                    title=title if i == 0 else f"{title} · §{i + 1}",
104                    camino=camino,
105                    tags=tags,
106                )
107            )
108    return out
109
110
111def embed_batches(http: httpx.Client, embed_url: str, texts: list[str], batch: int = 32) -> list[list[float]]:
112    out: list[list[float]] = []
113    for i in range(0, len(texts), batch):
114        chunk = texts[i : i + batch]
115        r = http.post(
116            f"{embed_url}/embed",
117            json={"texts": chunk, "kind": "passage", "normalize": True},
118            timeout=120.0,
119        )
120        r.raise_for_status()
121        out.extend(r.json()["vectors"])
122    return out
123
124
125def ensure_collection(qdrant: QdrantClient, name: str, dim: int, rebuild: bool):
126    existing = {c.name for c in qdrant.get_collections().collections}
127    if name in existing and rebuild:
128        qdrant.delete_collection(name)
129        existing.discard(name)
130    if name not in existing:
131        qdrant.create_collection(
132            collection_name=name,
133            vectors_config=qm.VectorParams(size=dim, distance=qm.Distance.COSINE),
134        )
135
136
137def main():
138    ap = argparse.ArgumentParser(description="Indexa docs/ de gioser-web en Qdrant")
139    ap.add_argument("--docs", default=str(DEFAULT_DOCS))
140    ap.add_argument("--qdrant", default=DEFAULT_QDRANT_URL)
141    ap.add_argument("--embed", default=DEFAULT_EMBED_URL)
142    ap.add_argument("--collection", default=DEFAULT_COLLECTION)
143    ap.add_argument("--rebuild", action="store_true", help="borra y recrea la colección")
144    args = ap.parse_args()
145
146    docs_dir = Path(args.docs)
147    if not docs_dir.is_dir():
148        sys.exit(f"docs no existe: {docs_dir}")
149
150    chunks = discover_chunks(docs_dir)
151    if not chunks:
152        sys.exit("no se encontraron docs para indexar")
153    print(f"→ {len(chunks)} fragmentos descubiertos")
154
155    with httpx.Client() as http:
156        health = http.get(f"{args.embed}/health", timeout=10.0).json()
157        dim = int(health["dim"])
158        print(f"→ embeddings: {health['model']} (dim={dim})")
159
160        qdrant = QdrantClient(url=args.qdrant)
161        ensure_collection(qdrant, args.collection, dim, rebuild=args.rebuild)
162
163        vectors = embed_batches(http, args.embed, [c.text for c in chunks])
164
165    points = [
166        qm.PointStruct(
167            id=str(uuid.uuid5(uuid.NAMESPACE_URL, f"{c.doc_id}:{c.chunk_index}")),
168            vector=v,
169            payload={
170                "doc_id": c.doc_id,
171                "chunk_index": c.chunk_index,
172                "title": c.title,
173                "text": c.text,
174                "camino": c.camino,
175                "tags": c.tags,
176                "source": "gioser-web",
177            },
178        )
179        for c, v in zip(chunks, vectors)
180    ]
181    qdrant.upsert(collection_name=args.collection, points=points)
182    print(f"✓ {len(points)} puntos en colección '{args.collection}'")
183
184    if args.rebuild:
185        print("  ✔ colección recreada")
186
187
188if __name__ == "__main__":
189    main()