Prepare for splitting API routes into their own routers

api/docs.py

@@ -0,0 +1,4 @@
+from fastapi.routing import APIRouter
+
+# API router for requirement extraction from docs / doc list retrieval / download
+router = APIRouter()

app.py

@@ -30,7 +30,8 @@ from aiolimiter import AsyncLimiter
 load_dotenv()
 
 logging.basicConfig(
-    level=logging.DEBUG if (os.environ.get("DEBUG_LOG", "0") == "1") else logging.INFO,
+    level=logging.DEBUG if (os.environ.get(
+        "DEBUG_LOG", "0") == "1") else logging.INFO,
     format='[%(asctime)s][%(levelname)s][%(filename)s:%(lineno)d]: %(message)s',
     datefmt='%Y-%m-%d %H:%M:%S'
 )
@@ -43,7 +44,6 @@ nltk.download('wordnet')
 warnings.filterwarnings("ignore")
 
 app = FastAPI(title="Requirements Extractor")
-app.mount("/static", StaticFiles(directory="static"), name="static")
 app.add_middleware(CORSMiddleware, allow_credentials=True, allow_headers=[
                    "*"], allow_methods=["*"], allow_origins=["*"])
 
@@ -54,7 +54,7 @@ llm_router = Router(model_list=[
         {
             "model": "gemini/gemini-2.0-flash",
             "api_key": os.environ.get("GEMINI"),
-            "max_retries": 10,
+            "max_retries": 5,
             "rpm": 15,
             "allowed_fails": 1,
             "cooldown": 30,
@@ -66,17 +66,13 @@ llm_router = Router(model_list=[
         {
             "model": "gemini/gemini-2.5-flash",
             "api_key": os.environ.get("GEMINI"),
-            "max_retries": 10,
+            "max_retries": 5,
             "rpm": 10,
             "allowed_fails": 1,
             "cooldown": 30,
         }
     }], fallbacks=[{"gemini-v2": ["gemini-v1"]}], num_retries=10, retry_after=30)
 
-limiter_mapping = {
-    model["model_name"]: AsyncLimiter(model["litellm_params"]["rpm"], 60)
-    for model in llm_router.model_list
-}
 lemmatizer = WordNetLemmatizer()
 
 NSMAP = {
@@ -217,10 +213,7 @@ def docx_to_txt(doc_id: str, url: str):
     return txt_data
 
 
-@app.get("/")
-def render_page():
-    return FileResponse("index.html")
-
+# ============================================= Doc routes =========================================================
 
 @app.post("/get_meetings", response_model=MeetingsResponse)
 def get_meetings(req: MeetingsRequest):
@@ -282,7 +275,7 @@ def get_change_request_dataframe(req: DataRequest):
     soup = BeautifulSoup(resp.text, "html.parser")
     files = [item.get_text() for item in soup.select("tr td a")
              if item.get_text().endswith(".xlsx")]
-    
+
     if files == []:
         raise HTTPException(status_code=404, detail="No XLSX has been found")
 
@@ -377,33 +370,19 @@ async def gen_reqs(req: RequirementsRequest, background_tasks: BackgroundTasks):
             return RequirementsResponse(requirements=[DocRequirements(document=doc_id, context="Error LLM", requirements=[])]).requirements
 
         try:
-            model_used = "gemini-v2"  # À adapter si fallback activé
-            async with limiter_mapping[model_used]:
-                resp_ai = await llm_router.acompletion(
-                    model=model_used,
-                    messages=[
-                        {"role": "user", "content": prompt(doc_id, full)}],
-                    response_format=RequirementsResponse
-                )
+            resp_ai = await llm_router.acompletion(
+                model="gemini-v2",
+                messages=[
+                    {"role": "user", "content": prompt(doc_id, full)}],
+                response_format=RequirementsResponse
+            )
+
             return RequirementsResponse.model_validate_json(resp_ai.choices[0].message.content).requirements
+
         except Exception as e:
-            if "rate limit" in str(e).lower():
-                try:
-                    model_used = "gemini-v2"  # À adapter si fallback activé
-                    async with limiter_mapping[model_used]:
-                        resp_ai = await llm_router.acompletion(
-                            model=model_used,
-                            messages=[
-                                {"role": "user", "content": prompt(doc_id, full)}],
-                            response_format=RequirementsResponse
-                        )
-                    return RequirementsResponse.model_validate_json(resp_ai.choices[0].message.content).requirements
-                except Exception as fallback_e:
-                    traceback.print_exception(fallback_e)
-                    return RequirementsResponse(requirements=[DocRequirements(document=doc_id, context="Error LLM", requirements=[])]).requirements
-            else:
-                traceback.print_exception(e)
-                return RequirementsResponse(requirements=[DocRequirements(document=doc_id, context="Error LLM", requirements=[])]).requirements
+            logging.error(
+                f"Failed to process document {doc_id}", e, stack_info=True)
+            return RequirementsResponse(requirements=[DocRequirements(document=doc_id, context="Error LLM", requirements=[])]).requirements
 
     async def process_batch(batch):
         results = await asyncio.gather(*(process_document(doc) for doc in batch))
@@ -448,6 +427,9 @@ async def gen_reqs(req: RequirementsRequest, con: Request):
     logging.info("Generating requirements for documents: {}".format(
         [doc.document for doc in documents]))
 
+    # limit max concurrency of LLM requests to prevent a huge pile of errors because of small rate limits
+    concurrency_sema = asyncio.Semaphore(4)
+
     def prompt(doc_id, full):
         return f"Here's the document whose ID is {doc_id} : {full}\n\nExtract all requirements and group them by context, returning a list of objects where each object includes a document ID, a concise description of the context where the requirements apply (not a chapter title or copied text), and a list of associated requirements; always return the result as a list, even if only one context is found. Remove the errors"
 
@@ -463,6 +445,8 @@ async def gen_reqs(req: RequirementsRequest, con: Request):
             return [DocRequirements(document=doc_id, context="Error LLM", requirements=[])]
 
         try:
+            await concurrency_sema.acquire()
+
             model_used = "gemini-v2"
             resp_ai = await llm_router.acompletion(
                 model=model_used,
@@ -473,6 +457,8 @@ async def gen_reqs(req: RequirementsRequest, con: Request):
             return RequirementsResponse.model_validate_json(resp_ai.choices[0].message.content).requirements
         except Exception as e:
             return [DocRequirements(document=doc_id, context="Error LLM", requirements=[])]
+        finally:
+            concurrency_sema.release()
 
     # futures for all processed documents
     process_futures = [_process_document(doc) for doc in documents]
@@ -525,3 +511,6 @@ def find_requirements_from_problem_description(req: ReqSearchRequest):
             status_code=500, detail="LLM error : Generated a wrong index, please try again.")
 
     return ReqSearchResponse(requirements=[requirements[i] for i in out_llm])
+
+
+app.mount("/", StaticFiles(directory="static", html=True), name="static")

static/index.html

@@ -0,0 +1,341 @@
+<!DOCTYPE html>
+<html lang="fr">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Requirements Extractor</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/marked/16.0.0/lib/marked.umd.min.js"
+        integrity="sha512-ygzQGrI/8Bfkm9ToUkBEuSMrapUZcHUys05feZh4ScVrKCYEXJsCBYNeVWZ0ghpH+n3Sl7OYlRZ/1ko01pYUCQ=="
+        crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <link href="https://cdn.jsdelivr.net/npm/daisyui@5" rel="stylesheet" type="text/css" />
+    <script src="https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4"></script>
+</head>
+
+<body class="bg-gray-100 min-h-screen">
+    <!-- Loading Overlay -->
+    <div id="loading-overlay" class="fixed inset-0 bg-black/50 flex items-center justify-center z-50 hidden">
+        <div class="bg-white p-6 rounded-lg shadow-lg text-center">
+            <span class="loading loading-spinner loading-xl"></span>
+            <p id="progress-text" class="text-gray-700">Chargement en cours...</p>
+        </div>
+    </div>
+
+    <div class="container mx-auto p-6">
+        <h1 class="text-3xl font-bold text-center mb-8">Requirements Extractor</h1>
+        <div id="selection-container" class="mb-6">
+            <div class="grid grid-cols-1 md:grid-cols-2 gap-4 mb-4">
+                <!-- Working Group -->
+                <div>
+                    <label for="working-group-select" class="block text-sm font-medium text-gray-700 mb-2">Working
+                        Group</label>
+                    <select id="working-group-select" class="w-full p-2 border border-gray-300 rounded-md">
+                        <option value="" selected>Select a working group</option>
+                        <option value="SA1">SA1</option>
+                        <option value="SA2">SA2</option>
+                        <option value="SA3">SA3</option>
+                        <option value="SA4">SA4</option>
+                        <option value="SA5">SA5</option>
+                        <option value="SA6">SA6</option>
+                        <option value="CT1">CT1</option>
+                        <option value="CT2">CT2</option>
+                        <option value="CT3">CT3</option>
+                        <option value="CT4">CT4</option>
+                        <option value="CT5">CT5</option>
+                        <option value="CT6">CT6</option>
+                        <option value="RAN1">RAN1</option>
+                        <option value="RAN2">RAN2</option>
+                    </select>
+                </div>
+
+                <!-- Meeting -->
+                <div>
+                    <label for="meeting-select" class="block text-sm font-medium text-gray-700 mb-2">Meeting</label>
+                    <select id="meeting-select" class="w-full p-2 border border-gray-300 rounded-md">
+                        <option value="">Select a meeting</option>
+                    </select>
+                </div>
+            </div>
+
+            <!-- Buttons -->
+            <div class="flex flex-col sm:flex-row gap-2">
+                <!-- <button id="get-meetings-btn" class="px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600">
+                    Get Meetings
+                </button> -->
+                <button id="get-tdocs-btn" class="px-4 py-2 bg-green-500 text-white rounded-md hover:bg-green-600">
+                    Get TDocs
+                </button>
+            </div>
+        </div>
+
+        <!-- Add an horizontal separation -->
+        <hr>
+        <!-- Tab list for subsections -->
+        <div role="tablist" class="tabs tabs-border tabs-xl" id="tab-container">
+            <a role="tab" class="tab tab-active" id="doc-table-tab" onclick="switchTab('doc-table-tab')">📝
+                Documents</a>
+            <a role="tab" class="tab tab-disabled" id="requirements-tab" onclick="switchTab('requirements-tab')">
+                <div class="flex items-center gap-1">
+                    <div class="badge badge-neutral badge-outline badge-xs" id="requirements-tab-badge">0</div>
+                    <span>Requirements</span>
+                </div>
+            </a>
+            <a role="tab" class="tab tab-disabled" id="solutions-tab" onclick="switchTab('solutions-tab')">Group and
+                Solve</a>
+            <a role="tab" class="tab tab-disabled" id="query-tab" onclick="switchTab('query-tab')">🔎 Find relevant
+                requirements</a>
+        </div>
+
+        <div id="doc-table-tab-contents" class="hidden">
+            <!-- Filters -->
+            <div id="filters-container" class="mb-6 hidden pt-10">
+                <div class="grid grid-cols-1 md:grid-cols-3 gap-4">
+                    <!-- Type Filter Dropdown -->
+                    <div class="dropdown dropdown-bottom dropdown-center w-full mb-4">
+                        <div tabindex="0" role="button" class="btn w-full justify-between" id="doc-type-filter-btn">
+                            <span id="doc-type-filter-label">Type</span>
+                            <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 ml-2" fill="none" viewBox="0 0 24 24"
+                                stroke="currentColor">
+                                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                                    d="M19 9l-7 7-7-7" />
+                            </svg>
+                        </div>
+                        <ul tabindex="0"
+                            class="dropdown-content z-[1] menu p-2 shadow bg-base-100 rounded-box w-full min-w-[200px] max-h-60 overflow-y-auto"
+                            id="doc-type-filter-menu">
+                            <!-- Rempli par JS -->
+                        </ul>
+                    </div>
+
+                    <!-- Status Filter Dropdown -->
+                    <div class="dropdown dropdown-bottom dropdown-center w-full mb-4">
+                        <div tabindex="0" role="button" class="btn w-full justify-between" id="status-dropdown-btn">
+                            <span id="status-filter-label">Status (Tous)</span>
+                            <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 ml-2" fill="none" viewBox="0 0 24 24"
+                                stroke="currentColor">
+                                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                                    d="M19 9l-7 7-7-7" />
+                            </svg>
+                        </div>
+                        <ul tabindex="0"
+                            class="dropdown-content z-[1] p-2 shadow bg-base-100 rounded-box w-full max-h-60 overflow-y-auto">
+                            <li class="border-b pb-2 mb-2">
+                                <label class="flex items-center gap-2 cursor-pointer">
+                                    <input type="checkbox" class="status-checkbox" value="all" checked>
+                                    <span class="font-semibold">Tous</span>
+                                </label>
+                            </li>
+                            <div id="status-options" class="flex flex-col gap-1"></div>
+                        </ul>
+                    </div>
+
+                    <!-- Agenda Item Filter Dropdown -->
+                    <div class="dropdown dropdown-bottom dropdown-center w-full mb-4">
+                        <div tabindex="0" role="button" class="btn w-full justify-between" id="agenda-dropdown-btn">
+                            <span id="agenda-filter-label">Agenda Item (Tous)</span>
+                            <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 ml-2" fill="none" viewBox="0 0 24 24"
+                                stroke="currentColor">
+                                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                                    d="M19 9l-7 7-7-7" />
+                            </svg>
+                        </div>
+                        <ul tabindex="0"
+                            class="dropdown-content z-[1] p-2 shadow bg-base-100 rounded-box w-full max-h-60 overflow-y-auto">
+                            <li class="border-b pb-2 mb-2">
+                                <label class="flex items-center gap-2 cursor-pointer">
+                                    <input type="checkbox" class="agenda-checkbox" value="all" checked>
+                                    <span class="font-semibold">Tous</span>
+                                </label>
+                            </li>
+                            <div id="agenda-options" class="flex flex-col gap-1"></div>
+                        </ul>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Data Table Informations -->
+            <div class="flex justify-between items-center mb-2 pt-5" id="data-table-info-container">
+                <!-- Left side: buttons -->
+                <div class="flex gap-2 items-center">
+                    <div class="tooltip" data-tip="Extract requirements from selected documents">
+                        <button id="extract-requirements-btn"
+                            class="bg-orange-300 text-white text-sm rounded px-3 py-1 shadow hover:bg-orange-600">
+                            <svg class="w-6 h-6 text-gray-800 dark:text-white" aria-hidden="true"
+                                xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none"
+                                viewBox="0 0 24 24">
+                                <path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round"
+                                    stroke-width="2"
+                                    d="M9 8h6m-6 4h6m-6 4h6M6 3v18l2-2 2 2 2-2 2 2 2-2 2 2V3l-2 2-2-2-2 2-2-2-2 2-2-2Z" />
+                            </svg>Extract Requirements
+                        </button>
+                    </div>
+                    <button id="download-tdocs-btn" class="text-sm rounded px-3 py-1 shadow cursor-pointer">
+                        📦 Download Selected TDocs
+                    </button>
+                </div>
+
+                <!-- Right side: document counts -->
+                <div class="flex gap-2 items-center">
+                    <span id="displayed-count" class="text-sm text-gray-700 bg-white rounded px-3 py-1 shadow">
+                        0 total documents
+                    </span>
+                    <span id="selected-count" class="text-sm text-blue-700 bg-blue-50 rounded px-3 py-1 shadow">
+                        0 selected documents
+                    </span>
+                </div>
+            </div>
+
+
+            <!-- Data Table -->
+            <div id="data-table-container" class="mb-6">
+                <table id="data-table" class="w-full bg-white rounded-lg shadow overflow-hidden">
+                    <thead class="bg-gray-50">
+                        <tr>
+                            <th class="px-4 py-2 text-left">
+                                <input type="checkbox" id="select-all-checkbox">
+                            </th>
+                            <th class="px-4 py-2 text-left">TDoc</th>
+                            <th class="px-4 py-2 text-left">Title</th>
+                            <th class="px-4 py-2 text-left">Type</th>
+                            <th class="px-4 py-2 text-left">Status</th>
+                            <th class="px-4 py-2 text-left">Agenda Item</th>
+                            <th class="px-4 py-2 text-left">URL</th>
+                        </tr>
+                    </thead>
+                    <tbody></tbody>
+                </table>
+            </div>
+        </div>
+
+        <div id="requirements-tab-contents" class="hidden pt-10">
+            <!-- Requirement list container -->
+            <div id="requirements-container" class="mb-6">
+                <div class="flex">
+                    <h2 class="text-2xl font-bold mb-4">Extracted requirement list</h2>
+                    <div class="justify-end pl-5">
+                        <!--Copy ALL reqs button-->
+                        <div class="tooltip" data-tip="Copy ALL requirements to clipboard">
+                            <button class="btn btn-square" id="copy-all-reqs-btn" aria-label="Copy">
+                                📋
+                            </button>
+                        </div>
+                    </div>
+                </div>
+                <div id="requirements-list"></div>
+            </div>
+        </div>
+
+
+        <!-- Query Requirements -->
+        <div id="query-tab-contents" class="hidden pt-10">
+            <div id="query-requirements-container" class="mb-6">
+                <h2 class="text-2xl font-bold mb-4">Find relevant requirements for a query</h2>
+                <div class="flex gap-2">
+                    <input type="text" id="query-input" class="flex-1 p-2 border border-gray-300 rounded-md"
+                        placeholder="Enter your query...">
+                    <button id="search-requirements-btn"
+                        class="px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600">
+                        Search
+                    </button>
+                </div>
+                <div id="query-results" class="mt-4"></div>
+            </div>
+        </div>
+
+        <!-- Solution tab -->
+        <div id="solutions-tab-contents" class="hidden pt-10">
+            <!--Header de catégorisation des requirements-->
+            <div class="w-full mx-auto mt-4 p-4 bg-base-100 rounded-box shadow-lg border border-base-200">
+                <!-- Séléction du nb de catégories / mode auto-->
+                <div class="flex flex-wrap justify-begin items-center gap-6">
+                    <!-- Number input -->
+                    <div class="form-control flex flex-col items-center justify-center">
+                        <label class="label" for="category-count">
+                            <span class="label-text text-base-content"># Categories</span>
+                        </label>
+                        <input id="category-count" type="number" class="input input-bordered w-24" min="1" value="3" />
+                    </div>
+
+                    <!-- Auto detect toggle -->
+                    <div class="form-control flex flex-col items-center justify-center">
+                        <label class="label">
+                            <span class="label-text text-base-content">Number of categories</span>
+                        </label>
+                        <input type="checkbox" class="toggle toggle-primary" id="auto-detect-toggle" checked="true" />
+                        <div class="text-xs mt-1 text-base-content">Manual | Auto detect</div>
+                    </div>
+
+                    <!-- Action Button -->
+                    <button id="categorize-requirements-btn" class="btn btn-primary btn-l self-center">
+                        Categorize
+                    </button>
+                </div>
+                <div class="flex flex-wrap justify-end items-center gap-6">
+                    <div class="tooltip" data-tip="Use Insight Finder's API to generate solutions">
+                        <label class="label">
+                            <span class="label-text text-base-content">Use insight finder solver</span>
+                        </label>
+                        <input type="checkbox" class="toggle toggle-primary" id="use-insight-finder-solver"
+                            checked="false" />
+                    </div>
+                </div>
+            </div>
+            <div id="categorized-requirements-container pt-10" class="mb-6">
+                <div class="flex mb-4 pt-10">
+                    <div class="justify-begin">
+                        <h2 class="text-2xl font-bold">Requirements categories list</h2>
+                    </div>
+                    <div class="justify-end pl-5">
+                        <!--Copy reqs button-->
+                        <div class="tooltip" data-tip="Copy selected requirements to clipboard">
+                            <button class="btn btn-square" id="copy-reqs-btn" aria-label="Copy">
+                                📋
+                            </button>
+                        </div>
+                    </div>
+                </div>
+                <div id="categorized-requirements-list"></div>
+            </div>
+
+            <!-- Solutions Action Buttons -->
+            <div id="solutions-action-buttons-container" class="mb-6 hidden">
+                <div class="flex flex-wrap gap-2 justify-center items-center">
+                    <div class="join">
+                        <input id="solution-gen-nsteps" type="number" class="input join-item w-24" min="1" value="3" />
+                        <button id="get-solutions-btn"
+                            class="btn join-item px-4 py-2 bg-indigo-500 text-white rounded-md hover:bg-indigo-600">
+                            Run multiple steps
+                        </button>
+                    </div>
+                    <button id="get-solutions-step-btn"
+                        class="px-4 py-2 bg-pink-500 text-white rounded-md hover:bg-pink-600">
+                        Get Solutions (Step-by-step)
+                    </button>
+                    <div class="dropdown dropdown-center">
+                        <div id="additional-gen-instr-btn" tabindex="0" role="button" class="btn m-1">🎨 Additional
+                            generation constraints</div>
+                        <div tabindex="0" class="dropdown-content card card-sm bg-base-100 z-1 w-256 shadow-md">
+                            <div class="card-body">
+                                <h3>Add additional constraints to follow when generating the solutions</h3>
+                                <textarea id="additional-gen-instr"
+                                    class="textarea textarea-bordered w-full"></textarea>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+
+            </div>
+
+            <!-- Solutions Container -->
+            <div id="solutions-container" class="mb-6">
+                <h2 class="text-2xl font-bold mb-4">Solutions</h2>
+                <div id="solutions-list"></div>
+            </div>
+        </div>
+
+        <script src="js/sse.js"></script>
+        <script src="js/ui-utils.js"></script>
+        <script src="js/script.js"></script>
+</body>
+
+</html>