SSE for specifications

app.py

@@ -74,7 +74,7 @@ def get_folder_name(working_group: str):
         code, num = m.groups()
         return (code, int(num))
     else:
-        raise ValueError("Format inattendu")
+        raise ValueError("Unattended format")
 
 @app.get("/get_meetings/{working_group}")
 def get_meetings(working_group: str):
@@ -98,7 +98,7 @@ def index_tdocs_wg_progress(req: IndexTDoc):
         for content in tdoc_indexer.process_workgroup(folder, url):
             yield content
         tdoc_indexer.save_indexer()
-        yield "event: end\ndata: Indexation terminé !\n\n"
+        yield "event: end\ndata: Indexation ended successfully !\n\n"
     return StreamingResponse(generate_events(), media_type="text/event-stream")
 
 @app.post("/index_tdocs/meeting")
@@ -118,7 +118,7 @@ def index_tdocs_meeting_progress(req: IndexTDoc):
             tdoc_indexer.process_meeting(meet, url)
             yield f"event: progress\ndata: {i+1}\n\n"
         tdoc_indexer.save_indexer()
-        yield "event: end\ndata: Indexation terminé !\n\n"
+        yield "event: end\ndata: Indexation ended successfully !\n\n"
     return StreamingResponse(generate_events(), media_type="text/event-stream")
 
 
@@ -128,20 +128,38 @@ def index_all_tdocs_progress():
         for content in tdoc_indexer.index_all_tdocs():
             yield content
         tdoc_indexer.save_indexer()
-        yield "event: end\ndata: Indexation terminé !\n\n"
+        yield "event: end\ndata: Indexation ended successfully !\n\n"
     return StreamingResponse(generate_events(), media_type="text/event-stream")
         
 
 @app.post("/index_specs/3gpp")
 def index_3gpp_specs_progress():
-    spec_3gpp_indexer.run()
-    spec_3gpp_indexer.save()
-    spec_3gpp_indexer.create_bm25_index()
-    return {"status": "OK"}
+    def generate_events():
+        for content in spec_3gpp_indexer.run():
+            yield content
+        yield "event: info\ndata: Saving index ...\n\n"
+        yield "event: get-maximum\ndata: 1\n\n"
+        yield "event: progress\ndata: 1\n\n"
+        spec_3gpp_indexer.save()
+        yield "event: info\ndata: Creating BM25 models ...\n\n"
+        yield "event: get-maximum\ndata: 1\n\n"
+        yield "event: progress\ndata: 1\n\n"
+        spec_3gpp_indexer.create_bm25_index()
+        yield "event: end\ndata: Indexation ended successfully !\n\n"
+    return StreamingResponse(generate_events(), media_type="text/event-stream")
 
 @app.post("/index_specs/etsi")
 def index_etsi_specs_progress():
-    spec_etsi_indexer.run()
-    spec_etsi_indexer.save()
-    spec_etsi_indexer.create_bm25_index()
-    return {"status": "OK"}
+    def generate_events():
+        for content in spec_etsi_indexer.run():
+            yield content
+        yield "event: info\ndata: Saving index ...\n\n"
+        yield "event: get-maximum\ndata: 1\n\n"
+        yield "event: progress\ndata: 1\n\n"
+        spec_etsi_indexer.save()
+        yield "event: info\ndata: Creating BM25 models ...\n\n"
+        yield "event: get-maximum\ndata: 1\n\n"
+        yield "event: progress\ndata: 1\n\n"
+        spec_etsi_indexer.create_bm25_index()
+        yield "event: end\ndata: Indexation ended successfully !\n\n"
+    return StreamingResponse(generate_events(), media_type="text/event-stream")

classes.py

@@ -538,12 +538,19 @@ class Spec3GPPIndexer:
 
     def run(self):
         print("Fetching specification tables from 3GPP...")
+        yield "event: info\ndata: Indexing 3GPP specs ...\n\n"
         specifications = self.fetch_spec_table()
         self.total_count = len(specifications)
         print(f"Processing {self.total_count} specs with {self.max_workers} threads...")
         with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
             futures = [executor.submit(self.process_specification, spec) for spec in specifications]
-            for f in concurrent.futures.as_completed(futures):
+            total = len(futures)
+            done_count = 0
+            yield f"event: get-maximum\ndata: {total}\n\n"
+
+            for future in concurrent.futures.as_completed(futures):
+                done_count += 1
+                yield f"event: progress\ndata: {done_count}\n\n"
                 if self.STOP_EVENT.is_set():
                     break
         print("All specs processed.")
@@ -776,13 +783,20 @@ class SpecETSIIndexer:
     
     def run(self):
         print("Démarrage indexation ETSI…")
+        yield "event: info\ndata: Indexing ETSI specs ...\n\n"
         specifications = self.df.to_dict(orient="records")
         self.total_count = len(specifications)
         print(f"Traitement de {self.total_count} specs avec {self.max_workers} threads...\n")
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
             futures = [executor.submit(self.process_specification, spec) for spec in specifications]
-            for f in concurrent.futures.as_completed(futures):
+            total = len(futures)
+            done_count = 0
+            yield f"event: get-maximum\ndata: {total}\n\n"
+
+            for future in concurrent.futures.as_completed(futures):
+                done_count += 1
+                yield f"event: progress\ndata: {done_count}\n\n"
                 if self.STOP_EVENT.is_set():
                     break
 

static/script.js

@@ -198,7 +198,7 @@ document.getElementById('tdocs-btn').addEventListener('click', async () => {
                     break;
                 case "end":
                     console.log("[end]", event.data);
-                    document.getElementById('infoText').textContent = "Indexation successful"
+                    document.getElementById('infoText').textContent = event.data;
                     break;
                 default:
                     if (event.data) {
@@ -211,30 +211,122 @@ document.getElementById('tdocs-btn').addEventListener('click', async () => {
     enableButtons()
 });
 
-document.getElementById('spec-3gpp-btn').addEventListener('click', () => {
+document.getElementById('spec-3gpp-btn').addEventListener('click', async () => {
     disableButtons()
-    fetch("/index_specs/3gpp", {
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json"
+    const response = await fetch("/index_specs/3gpp", {method: "POST", headers: {
+                    "Content-Type": "application/json",
+                    "Accept": "text/event-stream",
+                }});
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder("utf-8");
+    let buffer = "";
+
+    while (true) {
+        const {
+            done,
+            value
+        } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, {
+            stream: true
+        });
+
+        let events = buffer.split("\n\n"); // séparer les événements
+        buffer = events.pop(); // garder le reste pour le prochain chunk
+
+        for (const rawEvent of events) {
+            const event = {};
+            rawEvent.split("\n").forEach((line) => {
+                const [key, ...rest] = line.split(":");
+                if (key) event[key.trim()] = rest.join(":").trim();
+            });
+
+            // dispatch selon event.event ou par défaut message
+            switch (event.event) {
+                case "progress":
+                    console.log("[progress]", event.data);
+                    progress.value = event.data;
+                    break;
+                case "get-maximum":
+                    console.log("[new-max]", event.data);
+                    progress.max = event.data
+                    break;
+                case "info":
+                    console.log("[info]", event.data);
+                    document.getElementById('infoText').textContent = event.data
+                    break;
+                case "end":
+                    console.log("[end]", event.data);
+                    document.getElementById('infoText').textContent = event.data;
+                    break;
+                default:
+                    if (event.data) {
+                        console.log("[message]", event.data);
+                    };
             }
-        })
-        .then(resp => resp.text())
-        .then(data => {
-            enableButtons()
-        })
+        }
+    }
+
+    enableButtons()
 });
 
-document.getElementById('spec-etsi-btn').addEventListener('click', () => {
+document.getElementById('spec-etsi-btn').addEventListener('click', async () => {
     disableButtons()
-    fetch("/index_specs/etsi", {
-            method: "POST",
-            headers: {
-                "Content-Type": "application/json"
+    const response = await fetch("/index_specs/etsi", {method: "POST", headers: {
+                    "Content-Type": "application/json",
+                    "Accept": "text/event-stream",
+                }});
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder("utf-8");
+    let buffer = "";
+
+    while (true) {
+        const {
+            done,
+            value
+        } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, {
+            stream: true
+        });
+
+        let events = buffer.split("\n\n"); // séparer les événements
+        buffer = events.pop(); // garder le reste pour le prochain chunk
+
+        for (const rawEvent of events) {
+            const event = {};
+            rawEvent.split("\n").forEach((line) => {
+                const [key, ...rest] = line.split(":");
+                if (key) event[key.trim()] = rest.join(":").trim();
+            });
+
+            // dispatch selon event.event ou par défaut message
+            switch (event.event) {
+                case "progress":
+                    console.log("[progress]", event.data);
+                    progress.value = event.data;
+                    break;
+                case "get-maximum":
+                    console.log("[new-max]", event.data);
+                    progress.max = event.data
+                    break;
+                case "info":
+                    console.log("[info]", event.data);
+                    document.getElementById('infoText').textContent = event.data
+                    break;
+                case "end":
+                    console.log("[end]", event.data);
+                    document.getElementById('infoText').textContent = event.data;
+                    break;
+                default:
+                    if (event.data) {
+                        console.log("[message]", event.data);
+                    };
             }
-        })
-        .then(resp => resp.text())
-        .then(data => {
-            enableButtons()
-        })
+        }
+    }
+
+    enableButtons()
 });