WIP

app.py

@@ -1,3 +1,4 @@
+from typing import Literal
 from bs4 import BeautifulSoup
 import warnings
 import io
@@ -14,7 +15,7 @@ from nltk.stem import WordNetLemmatizer
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import json
 import traceback
-from fastapi import FastAPI, BackgroundTasks, HTTPException
+from fastapi import FastAPI, BackgroundTasks, HTTPException, Request
 from fastapi.staticfiles import StaticFiles
 from schemas import *
 from fastapi.middleware.cors import CORSMiddleware
@@ -253,6 +254,8 @@ def get_meetings(req: MeetingsRequest):
 
     return MeetingsResponse(meetings=dict(zip(all_meetings, meeting_folders)))
 
+# ============================================================================================================================================
+
 
 @app.post("/get_dataframe", response_model=DataResponse)
 def get_change_request_dataframe(req: DataRequest):
@@ -288,6 +291,8 @@ def get_change_request_dataframe(req: DataRequest):
     df = filtered_df.fillna("")
     return DataResponse(data=df[["TDoc", "Title", "Type", "TDoc Status", "Agenda item description", "URL"]].to_dict(orient="records"))
 
+# ==================================================================================================================================
+
 
 @app.post("/download_tdocs")
 def download_tdocs(req: DownloadRequest):
@@ -337,6 +342,8 @@ def download_tdocs(req: DownloadRequest):
         media_type="application/zip"
     )
 
+# ========================================================================================================================
+
 
 @app.post("/generate_requirements", response_model=RequirementsResponse)
 async def gen_reqs(req: RequirementsRequest, background_tasks: BackgroundTasks):
@@ -411,6 +418,81 @@ async def gen_reqs(req: RequirementsRequest, background_tasks: BackgroundTasks):
                 background_tasks.add_task(asyncio.sleep, 60)
     return RequirementsResponse(requirements=all_requirements)
 
+# ======================================================================================================================================================================================
+
+SUBPROCESS_SEMAPHORE = asyncio.Semaphore(32)
+
+
+class ProgressUpdate(BaseModel):
+    """Defines the structure of a single SSE message."""
+    status: Literal["progress", "complete"]
+    data: dict
+    total_docs: int
+    processed_docs: int
+
+
+@app.post("/generate_requirements/v2")
+async def gen_reqs(req: RequirementsRequest, con: Request):
+    """Extract requirements from the specified TDocs using a LLM and returns SSE events about the progress of ongoing operations"""
+
+    documents = req.documents
+    n_docs = len(documents)
+
+    logging.info("Generating requirements for documents: {}".format(
+        [doc.document for doc in documents]))
+
+    def prompt(doc_id, full):
+        return f"Here's the document whose ID is {doc_id} : {full}\n\nExtract all requirements and group them by context, returning a list of objects where each object includes a document ID, a concise description of the context where the requirements apply (not a chapter title or copied text), and a list of associated requirements; always return the result as a list, even if only one context is found. Remove the errors"
+
+    async def _process_document(doc) -> list[DocRequirements]:
+        doc_id = doc.document
+        url = doc.url
+
+        # convert the docx to txt for use
+        try:
+            full = "\n".join(docx_to_txt(doc_id, url))
+        except Exception as e:
+            traceback.print_exception(e)
+            return [DocRequirements(document=doc_id, context="Error LLM", requirements=[])]
+
+        try:
+            model_used = "gemini-v2"
+            resp_ai = await llm_router.acompletion(
+                model=model_used,
+                messages=[
+                    {"role": "user", "content": prompt(doc_id, full)}],
+                response_format=RequirementsResponse
+            )
+            return RequirementsResponse.model_validate_json(resp_ai.choices[0].message.content).requirements
+        except Exception as e:
+            return [DocRequirements(document=doc_id, context="Error LLM", requirements=[])]
+
+    # futures for all processed documents
+    process_futures = [_process_document(doc) for doc in documents]
+
+    # lambda to print progress
+    def progress_update(x): return f"data: {x.model_dump_json()}\n\n"
+
+    # async generator that generates  the SSE events for progress
+    async def _stream_generator(docs: list[asyncio.Future]):
+        items = []
+        n_processed = 0
+
+        yield progress_update(ProgressUpdate(status="progress", data={}, total_docs=n_docs, processed_docs=0))
+
+        for doc in asyncio.as_completed(docs):
+            result = await doc
+            items.extend(result)
+            n_processed += 1
+            yield progress_update(ProgressUpdate(status="progress", data={}, total_docs=n_docs, processed_docs=n_processed))
+
+            final_response = RequirementsResponse(requirements=items)
+
+        yield progress_update(ProgressUpdate(status="complete", data=final_response.model_dump(), total_docs=n_docs, processed_docs=n_processed))
+
+    return StreamingResponse(_stream_generator(process_futures), media_type="text/event-stream")
+# =======================================================================================================================================================================================
+
 
 @app.post("/get_reqs_from_query", response_model=ReqSearchResponse)
 def find_requirements_from_problem_description(req: ReqSearchRequest):

index.html

@@ -11,58 +11,66 @@
 
 <body class="bg-gray-100 min-h-screen">
     <!-- Loading Overlay -->
-    <div id="loading-overlay" class="fixed inset-0 bg-black opacity-50 flex items-center justify-center z-50 hidden">
+    <div id="loading-overlay" class="fixed inset-0 bg-black/50 flex items-center justify-center z-50 hidden">
         <div class="bg-white p-6 rounded-lg shadow-lg text-center">
-            <div id="loading-bar" class="w-64 h-4 bg-gray-200 rounded-full mb-4">
+            <!-- <div id="loading-bar" class="w-64 h-4 bg-gray-200 rounded-full mb-4">
                 <div class="h-full bg-blue-500 rounded-full animate-pulse"></div>
-            </div>
+            </div> -->
+
+            <span class="loading loading-spinner loading-xl"></span>
             <p id="progress-text" class="text-gray-700">Chargement en cours...</p>
         </div>
     </div>
 
     <div class="container mx-auto p-6">
         <h1 class="text-3xl font-bold text-center mb-8">Requirements Extractor</h1>
+        <div id="selection-container" class="mb-6">
+            <div class="grid grid-cols-1 md:grid-cols-2 gap-4 mb-4">
+                <!-- Working Group -->
+                <div>
+                    <label for="working-group-select" class="block text-sm font-medium text-gray-700 mb-2">Working
+                        Group</label>
+                    <select id="working-group-select" class="w-full p-2 border border-gray-300 rounded-md">
+                        <option value="" selected>Select a working group</option>
+                        <option value="SA1">SA1</option>
+                        <option value="SA2">SA2</option>
+                        <option value="SA3">SA3</option>
+                        <option value="SA4">SA4</option>
+                        <option value="SA5">SA5</option>
+                        <option value="SA6">SA6</option>
+                        <option value="CT1">CT1</option>
+                        <option value="CT2">CT2</option>
+                        <option value="CT3">CT3</option>
+                        <option value="CT4">CT4</option>
+                        <option value="CT5">CT5</option>
+                        <option value="CT6">CT6</option>
+                    </select>
+                </div>
 
-        <!-- Working Group Selection -->
-        <div id="working-group-container" class="mb-6">
-            <label for="working-group-select" class="block text-sm font-medium text-gray-700 mb-2">Working Group</label>
-            <select id="working-group-select" class="w-full p-2 border border-gray-300 rounded-md">
-                <option value="">Select a working group</option>
-                <option value="SA1">SA1</option>
-                <option value="SA2">SA2</option>
-                <option value="SA3">SA3</option>
-                <option value="SA4">SA4</option>
-                <option value="SA5">SA5</option>
-                <option value="SA6">SA6</option>
-                <option value="CT1">CT1</option>
-                <option value="CT2">CT2</option>
-                <option value="CT3">CT3</option>
-                <option value="CT4">CT4</option>
-                <option value="CT5">CT5</option>
-                <option value="CT6">CT6</option>
-            </select>
-            <button id="get-meetings-btn" class="mt-2 px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600">
-                Get Meetings
-            </button>
-        </div>
+                <!-- Meeting -->
+                <div>
+                    <label for="meeting-select" class="block text-sm font-medium text-gray-700 mb-2">Meeting</label>
+                    <select id="meeting-select" class="w-full p-2 border border-gray-300 rounded-md">
+                        <option value="">Select a meeting</option>
+                    </select>
+                </div>
+            </div>
 
-        <!-- Meeting Selection -->
-        <div id="meeting-container" class="mb-6 hidden">
-            <label for="meeting-select" class="block text-sm font-medium text-gray-700 mb-2">Meeting</label>
-            <select id="meeting-select" class="w-full p-2 border border-gray-300 rounded-md">
-                <option value="">Select a meeting</option>
-            </select>
-            <button id="get-tdocs-btn" class="mt-2 px-4 py-2 bg-green-500 text-white rounded-md hover:bg-green-600">
-                Get TDocs
-            </button>
+            <!-- Buttons -->
+            <div class="flex flex-col sm:flex-row gap-2">
+                <!-- <button id="get-meetings-btn" class="px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600">
+                    Get Meetings
+                </button> -->
+                <button id="get-tdocs-btn" class="px-4 py-2 bg-green-500 text-white rounded-md hover:bg-green-600">
+                    Get TDocs
+                </button>
+            </div>
         </div>
 
-
-
         <!-- Add an horizontal separation -->
         <hr>
         <!-- Tab list for subsections -->
-        <div role="tablist" class="tabs tabs-border" id="tab-container">
+        <div role="tablist" class="tabs tabs-border tabs-xl" id="tab-container">
             <a role="tab" class="tab tab-active" id="doc-table-tab" onclick="switchTab('doc-table-tab')">📝
                 Documents</a>
             <a role="tab" class="tab tab-disabled" id="requirements-tab" onclick="switchTab('requirements-tab')">
@@ -242,8 +250,7 @@
                         <label class="label">
                             <span class="label-text text-base-content">Number of categories</span>
                         </label>
-                        <input type="checkbox" class="toggle toggle-primary" id="auto-detect-toggle"
-                            onchange="toggleAutoDetect(this)" />
+                        <input type="checkbox" class="toggle toggle-primary" id="auto-detect-toggle" />
                         <div class="text-xs mt-1 text-base-content">Manual | Auto detect</div>
                     </div>
 
@@ -260,9 +267,11 @@
                     </div>
                     <div class="justify-end pl-5">
                         <!--Copy reqs button-->
-                        <button class="btn btn-square" id="copy-reqs-btn" aria-label="Copy">
-                            📋
-                        </button>
+                        <div class="tooltip" data-tip="Copy selected requirements to clipboard">
+                            <button class="btn btn-square" id="copy-reqs-btn" aria-label="Copy">
+                                📋
+                            </button>
+                        </div>
                     </div>
                 </div>
                 <div id="categorized-requirements-list"></div>
@@ -272,7 +281,7 @@
             <div id="solutions-action-buttons-container" class="mb-6 hidden">
                 <div class="flex flex-wrap gap-2 justify-center items-center">
                     <div class="join">
-                        <input id="category-count" type="number" class="input join-item w-24" min="1" value="3" />
+                        <input id="solution-gen-nsteps" type="number" class="input join-item w-24" min="1" value="3" />
                         <button id="get-solutions-btn"
                             class="btn join-item px-4 py-2 bg-indigo-500 text-white rounded-md hover:bg-indigo-600">
                             Run multiple steps
@@ -304,6 +313,7 @@
             </div>
         </div>
 
+        <script src="/static/sse.js"></script>
         <script src="/static/script.js"></script>
 </body>
 

static/script.js

@@ -1,3 +1,4 @@
+
 // ==================================== Variables globales ========================================
 let requirements = [];
 
@@ -584,13 +585,27 @@ async function extractRequirements() {
     toggleElementsEnabled(['extract-requirements-btn'], false);
 
     try {
-        const response = await fetch('/generate_requirements', {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({ documents: selectedData })
+        const response = await postWithSSE('/generate_requirements/v2', { documents: selectedData }, {
+            onMessage: (msg) => {
+                console.log("SSE message:");
+                console.log(msg);
+
+                showLoadingOverlay(`Extraction des requirements en cours... (${msg.processed_docs}/${msg.total_docs})`);
+            },
+            onError: (err) => {
+                console.error(`Error while fetching requirements: ${err}`);
+                throw err;
+            }
         });
 
-        const data = await response.json();
+
+    //     const response = await fetch('/generate_requirements/', {
+    //         method: 'POST',
+    //         headers: { 'Content-Type': 'application/json' },
+    //         body: req
+    //     });
+
+        const data = response.data; // data in the SSE message contains the requirements response
         requirements = data.requirements;
         let req_id = 0;
         data.requirements.forEach(obj => {
@@ -604,6 +619,7 @@ async function extractRequirements() {
                 req_id++;
             })
         })
+        
         displayRequirements(requirements);
 
         toggleContainersVisibility(['requirements-container', 'query-requirements-container'], true);
@@ -773,7 +789,7 @@ function copySelectedRequirementsAsMarkdown() {
     selected.categories.forEach(category => {
         lines.push(`### ${category.title}`);
         category.requirements.forEach(req => {
-            lines.push(`- ${req.requirement}`);
+            lines.push(`- ${req.requirement} (${req.document})`);
         });
         lines.push(''); // Add an empty line after each category
     });
@@ -782,7 +798,7 @@ function copySelectedRequirementsAsMarkdown() {
 
     navigator.clipboard.writeText(markdownText).then(() => {
         console.log("Markdown copied to clipboard.");
-        alert("Selected requirements copied to clipboard"); 
+        alert("Selected requirements copied to clipboard");
     }).catch(err => {
         console.error("Failed to copy markdown:", err);
     });
@@ -1282,7 +1298,10 @@ async function workflow(steps = 1) {
 
 document.addEventListener('DOMContentLoaded', function () {
     // Événements des boutons principaux
-    document.getElementById('get-meetings-btn').addEventListener('click', getMeetings);
+    // document.getElementById('get-meetings-btn').addEventListener('click', getMeetings);
+    document.getElementById('working-group-select').addEventListener('change', (ev) => {
+        getMeetings();
+    });
     document.getElementById('get-tdocs-btn').addEventListener('click', getTDocs);
     document.getElementById('download-tdocs-btn').addEventListener('click', downloadTDocs);
     document.getElementById('extract-requirements-btn').addEventListener('click', extractRequirements);
@@ -1297,7 +1316,8 @@ document.addEventListener('DOMContentLoaded', function () {
 
     // Événements pour les boutons de solutions (à implémenter plus tard)
     document.getElementById('get-solutions-btn').addEventListener('click', () => {
-        alert('Fonctionnalité à implémenter');
+        const n_steps = document.getElementById('solution-gen-nsteps').value;
+        workflow(n_steps);
     });
     document.getElementById('get-solutions-step-btn').addEventListener('click', () => {
         workflow();

static/sse.js

@@ -0,0 +1,80 @@
+// sse-fetch.js
+
+/**
+ * Performs a POST request and handles the response as a Server-Sent Events (SSE) stream.
+ * The standard EventSource API does not support POST requests, so we use fetch.
+ *
+ * @param {string} url The URL to send the POST request to.
+ * @param {object} body The JSON body for the POST request.
+ * @param {object} callbacks An object containing callback functions.
+ * @param {(data: object) => void} callbacks.onMessage A function called for each message received.
+ * @param {(error: Error) => void} callbacks.onError A function called if an error occurs.
+ */
+async function postWithSSE(url, body, callbacks) {
+  const { onMessage, onError } = callbacks;
+
+  try {
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Accept': 'text/event-stream' // Politely ask for an event stream
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    if (!response.body) {
+      throw new Error('Response body is null.');
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+
+    while (true) {
+      const { value, done } = await reader.read();
+
+      // Decode the chunk of data and add it to our buffer.
+      // The `stream: true` option is important for multi-byte characters.
+      const chunk = decoder.decode(value, { stream: true });
+      buffer += chunk;
+
+      // SSE messages are separated by double newlines (`\n\n`).
+      // A single chunk from the stream might contain multiple messages or a partial message.
+      // We process all complete messages in the buffer.
+      let boundary;
+      while ((boundary = buffer.indexOf('\n\n')) !== -1) {
+        const messageString = buffer.substring(0, boundary);
+        buffer = buffer.substring(boundary + 2); // Remove the processed message from the buffer
+
+        // Skip empty keep-alive messages
+        if (messageString.trim() === '') {
+          continue;
+        }
+
+        // SSE "data:" lines. Your server only uses `data:`.
+        // We remove the "data: " prefix to get the JSON payload.
+        if (messageString.startsWith('data:')) {
+          const jsonData = messageString.substring('data: '.length);
+          try {
+            const parsedData = JSON.parse(jsonData);
+            if (parsedData.status === "complete")
+              return parsedData;
+            else
+              onMessage(parsedData);
+          } catch (e) {
+            console.error("Failed to parse JSON from SSE message:", jsonData, e);
+            // Optionally call the onError callback for parsing errors
+            if (onError) onError(new Error("Failed to parse JSON from SSE message."));
+          }
+        }
+      }
+    }
+  } catch (error) {
+    throw error;
+  }
+}