Update main.py

main.py

@@ -1,30 +1,29 @@
 # -*- coding: utf-8 -*-
 """
-HF Space - main.py de substitution pour tests Qdrant / indexation minimale
+HF Space - main.py de substitution pour tests Qdrant / indexation minimale (robuste)
 
 Endpoints:
 - GET  /                      → redirige vers UI_PATH (défaut: /ui)
 - GET  /ui (UI_PATH)          → UI Gradio
 - GET  /health                → healthcheck
 - GET  /api                   → infos service
+- GET  /debug/env             → aperçu config (sans secrets)
 - POST /wipe?project_id=XXX   → supprime la collection Qdrant
 - POST /index                 → lance un job d'indexation
 - GET  /status/{job_id}       → état + logs du job
 - GET  /collections/{proj}/count → count points dans Qdrant
 - POST /query                 → recherche sémantique
 
-ENV attendues :
+ENV:
 - QDRANT_URL, QDRANT_API_KEY (requis pour upsert)
 - COLLECTION_PREFIX (défaut "proj_")
 - EMB_PROVIDER ("hf" par défaut, "dummy" sinon)
 - HF_EMBED_MODEL (défaut "BAAI/bge-m3")
 - HUGGINGFACEHUB_API_TOKEN (si EMB_PROVIDER=hf)
+- EMB_FALLBACK_TO_DUMMY (true/false) → si vrai, bascule dummy si HF indisponible
 - LOG_LEVEL (défaut DEBUG)
 - PORT (fourni par HF, défaut 7860)
 - UI_PATH (défaut "/ui")
-
-Dépendances suggérées :
-fastapi>=0.111, uvicorn>=0.30, httpx>=0.27, pydantic>=2.7, gradio>=4.43, numpy>=2.0
 """
 
 from __future__ import annotations
@@ -63,13 +62,17 @@ EMB_PROVIDER = os.getenv("EMB_PROVIDER", "hf").lower()  # "hf" | "dummy"
 HF_EMBED_MODEL = os.getenv("HF_EMBED_MODEL", "BAAI/bge-m3")
 HF_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN", "")
 
+EMB_FALLBACK_TO_DUMMY = os.getenv("EMB_FALLBACK_TO_DUMMY", "false").lower() in ("1","true","yes","on")
+
 UI_PATH = os.getenv("UI_PATH", "/ui")  # UI montée ici par défaut
 
 if not QDRANT_URL or not QDRANT_API_KEY:
     LOG.warning("QDRANT_URL / QDRANT_API_KEY non fournis : l'upsert échouera.")
 
 if EMB_PROVIDER == "hf" and not HF_TOKEN:
-    LOG.warning("EMB_PROVIDER=hf sans HUGGINGFACEHUB_API_TOKEN. Utilise EMB_PROVIDER=dummy pour tester sans token.")
+    LOG.warning("EMB_PROVIDER=hf sans HUGGINGFACEHUB_API_TOKEN. "
+                "→ soit définis le token, soit mets EMB_PROVIDER=dummy, "
+                "soit active EMB_FALLBACK_TO_DUMMY=true.")
 
 # ------------------------------------------------------------------------------
 # Schémas Pydantic
@@ -154,6 +157,9 @@ def chunk_text(text: str, chunk_size: int, overlap: int) -> List[Tuple[int, int,
             i = j
     return res
 
+# ------------------------------------------------------------------------------
+# Qdrant helpers
+# ------------------------------------------------------------------------------
 async def ensure_collection(client: httpx.AsyncClient, coll: str, vector_size: int) -> None:
     """Crée la collection Qdrant (distance=Cosine), ou la recrée si dim mismatch."""
     url = f"{QDRANT_URL}/collections/{coll}"
@@ -214,7 +220,9 @@ async def embed_hf(client: httpx.AsyncClient, texts: List[str], model: str = HF_
     payload = {"inputs": texts, "options": {"wait_for_model": True}}
     r = await client.post(url, headers=headers, json=payload, timeout=120)
     if r.status_code != 200:
-        raise HTTPException(status_code=502, detail=f"HF Inference error: {r.text}")
+        detail = r.text
+        LOG.error(f"HF Inference error {r.status_code}: {detail[:400]}")
+        raise HTTPException(status_code=502, detail=f"HF Inference error {r.status_code}: {detail}")
     data = r.json()
     embeddings: List[List[float]] = []
     if isinstance(data, list):
@@ -237,89 +245,107 @@ def embed_dummy(texts: List[str], dim: int = 128) -> List[List[float]]:
     return out
 
 async def embed_texts(client: httpx.AsyncClient, texts: List[str]) -> List[List[float]]:
+    # Fallback optionnel si HF indisponible
     if EMB_PROVIDER == "hf":
-        return await embed_hf(client, texts)
+        try:
+            return await embed_hf(client, texts)
+        except Exception as e:
+            if EMB_FALLBACK_TO_DUMMY:
+                LOG.warning(f"Fallback embeddings → dummy (cause: {e})")
+                return embed_dummy(texts, dim=128)
+            raise
     return embed_dummy(texts, dim=128)
 
 # ------------------------------------------------------------------------------
-# Pipeline d'indexation
+# Pipeline d'indexation (robuste)
 # ------------------------------------------------------------------------------
 async def run_index_job(job: JobState, req: IndexRequest) -> None:
-    job.stage = "embedding"
-    job.total_files = len(req.files)
-    job.log(f"Index start project={req.project_id} files={len(req.files)} chunk_size={req.chunk_size} overlap={req.overlap} batch_size={req.batch_size} store_text={req.store_text}")
-
-    # Dédup global par hash du texte de fichier
-    file_hashes = [hash8(f.text) for f in req.files]
-    uniq = len(set(file_hashes))
-    if uniq != len(file_hashes):
-        job.log(f"Attention: {len(file_hashes)-uniq} fichier(s) ont un texte identique (hash dupliqué).")
-
-    # Chunking
-    records: List[Dict[str, Any]] = []
-    for f in req.files:
-        chunks = chunk_text(f.text, req.chunk_size, req.overlap)
-        if not chunks:
-            job.log(f"{f.path}: 0 chunk (trop court ou vide)")
-        for idx, (start, end, ch) in enumerate(chunks):
-            payload = {"path": f.path, "chunk": idx, "start": start, "end": end}
-            if req.store_text:
-                payload["text"] = ch
-            records.append({"payload": payload, "raw": ch})
-    job.total_chunks = len(records)
-    job.log(f"Total chunks = {job.total_chunks}")
-
-    if job.total_chunks == 0:
+    try:
+        job.stage = "embedding"
+        job.total_files = len(req.files)
+        job.log(
+            f"Index start project={req.project_id} files={len(req.files)} "
+            f"chunk_size={req.chunk_size} overlap={req.overlap} batch_size={req.batch_size} store_text={req.store_text} "
+            f"provider={EMB_PROVIDER} model={HF_EMBED_MODEL}"
+        )
+
+        # Dédup global par hash du texte de fichier
+        file_hashes = [hash8(f.text) for f in req.files]
+        uniq = len(set(file_hashes))
+        if uniq != len(file_hashes):
+            job.log(f"Attention: {len(file_hashes)-uniq} fichier(s) ont un texte identique (hash dupliqué).")
+
+        # Chunking
+        records: List[Dict[str, Any]] = []
+        for f in req.files:
+            chunks = chunk_text(f.text, req.chunk_size, req.overlap)
+            if not chunks:
+                job.log(f"{f.path}: 0 chunk (trop court ou vide)")
+            for idx, (start, end, ch) in enumerate(chunks):
+                payload = {"path": f.path, "chunk": idx, "start": start, "end": end}
+                if req.store_text:
+                    payload["text"] = ch
+                records.append({"payload": payload, "raw": ch})
+        job.total_chunks = len(records)
+        job.log(f"Total chunks = {job.total_chunks}")
+
+        if job.total_chunks == 0:
+            job.stage = "failed"
+            job.errors.append("Aucun chunk à indexer.")
+            job.finished_at = time.time()
+            return
+
+        async with httpx.AsyncClient(timeout=120) as client:
+            # Warmup dim
+            warmup_vec = (await embed_texts(client, [records[0]["raw"]]))[0]
+            vec_dim = len(warmup_vec)
+            job.log(f"Warmup embeddings dim={vec_dim}")
+
+            # Collection Qdrant
+            coll = f"{COLLECTION_PREFIX}{req.project_id}"
+            await ensure_collection(client, coll, vector_size=vec_dim)
+            job.log(f"Collection prête: {coll} (dim={vec_dim})")
+
+            job.stage = "upserting"
+            batch_points: List[Dict[str, Any]] = []
+
+            async def flush_batch():
+                nonlocal batch_points
+                if not batch_points:
+                    return 0
+                added = await qdrant_upsert(client, coll, batch_points)
+                job.upserted += added
+                job.log(f"+{added} points upsert (total={job.upserted})")
+                batch_points = []
+                return added
+
+            EMB_BATCH = max(8, min(64, req.batch_size * 2))
+            i = 0
+            while i < len(records):
+                sub = records[i : i + EMB_BATCH]
+                texts = [r["raw"] for r in sub]
+                vecs = await embed_texts(client, texts)
+                if len(vecs) != len(sub):
+                    raise HTTPException(status_code=500, detail="Embedding batch size mismatch")
+                job.embedded += len(vecs)
+
+                for r, v in zip(sub, vecs):
+                    point = {"id": str(uuid.uuid4()), "vector": v, "payload": r["payload"]}
+                    batch_points.append(point)
+                    if len(batch_points) >= req.batch_size:
+                        await flush_batch()
+                i += EMB_BATCH
+
+            await flush_batch()
+
+        job.stage = "done"
+        job.finished_at = time.time()
+        job.log("Index job terminé.")
+    except Exception as e:
         job.stage = "failed"
-        job.errors.append("Aucun chunk à indexer.")
+        job.errors.append(str(e))
         job.finished_at = time.time()
-        return
-
-    async with httpx.AsyncClient(timeout=120) as client:
-        # Warmup dim
-        warmup_vec = (await embed_texts(client, [records[0]["raw"]]))[0]
-        vec_dim = len(warmup_vec)
-        job.log(f"Warmup embeddings dim={vec_dim} provider={EMB_PROVIDER}")
-
-        # Collection Qdrant
-        coll = f"{COLLECTION_PREFIX}{req.project_id}"
-        await ensure_collection(client, coll, vector_size=vec_dim)
-
-        job.stage = "upserting"
-        batch_points: List[Dict[str, Any]] = []
-
-        async def flush_batch():
-            nonlocal batch_points
-            if not batch_points:
-                return 0
-            added = await qdrant_upsert(client, coll, batch_points)
-            job.upserted += added
-            job.log(f"+{added} points upsert (total={job.upserted})")
-            batch_points = []
-            return added
-
-        EMB_BATCH = max(8, min(64, req.batch_size * 2))
-        i = 0
-        while i < len(records):
-            sub = records[i : i + EMB_BATCH]
-            texts = [r["raw"] for r in sub]
-            vecs = await embed_texts(client, texts)
-            if len(vecs) != len(sub):
-                raise HTTPException(status_code=500, detail="Embedding batch size mismatch")
-            job.embedded += len(vecs)
-
-            for r, v in zip(sub, vecs):
-                point = {"id": str(uuid.uuid4()), "vector": v, "payload": r["payload"]}
-                batch_points.append(point)
-                if len(batch_points) >= req.batch_size:
-                    await flush_batch()
-            i += EMB_BATCH
-
-        await flush_batch()
-
-    job.stage = "done"
-    job.finished_at = time.time()
-    job.log("Index job terminé.")
+        job.log(f"❌ Exception: {e}")
 
 # ------------------------------------------------------------------------------
 # FastAPI app + endpoints
@@ -338,9 +364,29 @@ async def health():
 
 @fastapi_app.get("/api")
 async def api_info():
-    return {"ok": True, "service": "remote-indexer-min", "qdrant": bool(QDRANT_URL), "emb_provider": EMB_PROVIDER, "ui_path": UI_PATH}
-
-# Redirige "/" → UI_PATH (ex.: /ui). Ça évite tout conflit avec la route racine.
+    return {
+        "ok": True,
+        "service": "remote-indexer-min",
+        "qdrant": bool(QDRANT_URL),
+        "emb_provider": EMB_PROVIDER,
+        "hf_model": HF_EMBED_MODEL,
+        "ui_path": UI_PATH,
+        "fallback_to_dummy": EMB_FALLBACK_TO_DUMMY,
+    }
+
+@fastapi_app.get("/debug/env")
+async def debug_env():
+    return {
+        "qdrant_url_set": bool(QDRANT_URL),
+        "qdrant_key_set": bool(QDRANT_API_KEY),
+        "emb_provider": EMB_PROVIDER,
+        "hf_model": HF_EMBED_MODEL,
+        "hf_token_set": bool(HF_TOKEN),
+        "fallback_to_dummy": EMB_FALLBACK_TO_DUMMY,
+        "collection_prefix": COLLECTION_PREFIX,
+    }
+
+# Redirige "/" → UI_PATH (ex.: /ui).
 @fastapi_app.get("/")
 async def root_redirect():
     return RedirectResponse(url=UI_PATH, status_code=307)
@@ -474,8 +520,9 @@ with gr.Blocks(title="Remote Indexer - Minimal Test", analytics_enabled=False) a
     gr.Markdown("## 🔬 Remote Indexer — Tests sans console\n"
                 "Wipe → Index 2 docs → Status → Count → Query\n"
                 f"- **Embeddings**: `{EMB_PROVIDER}` (model: `{HF_EMBED_MODEL}`)\n"
-                f"- **Qdrant**: `{'OK' if QDRANT_URL else 'ABSENT'}`\n"
-                "Astuce: si pas de token HF, mets `EMB_PROVIDER=dummy`.")
+                f"- **Token HF présent**: `{'oui' if bool(HF_TOKEN) else 'non'}` — "
+                f"**Fallback dummy**: `{'on' if EMB_FALLBACK_TO_DUMMY else 'off'}`\n"
+                f"- **Qdrant**: `{'OK' if QDRANT_URL else 'ABSENT'}`")
     with gr.Row():
         project_tb = gr.Textbox(label="Project ID", value="DEEPWEB")
         jobid_tb = gr.Textbox(label="Job ID (pour Status)", value="", interactive=True)