Update app.py

app.py

@@ -106,7 +106,8 @@ echo $! > "server/server.pid"
         subprocess.Popen(["bash", str(start_script)])
         st.success("Le serveur llama.cpp est en cours de démarrage...")
         time.sleep(5)
-        if check_server_status():
+        check_server_status()
+        if st.session_state.server_status == "Actif":
             st.success("Serveur llama.cpp démarré avec succès!")
         else:
             st.error("Le serveur n'a pas pu démarrer. Vérifiez les logs dans le dossier logs/.")
@@ -134,7 +135,8 @@ fi
         subprocess.run(["bash", str(stop_script)])
         st.success("Le serveur llama.cpp est en cours d'arrêt...")
         time.sleep(2)
-        if not check_server_status():
+        check_server_status()
+        if st.session_state.server_status == "Inactif":
             st.success("Serveur llama.cpp arrêté avec succès!")
         else:
             st.warning("Le serveur n'a pas pu être arrêté correctement.")
@@ -281,15 +283,17 @@ def check_api_keys():
         'GITHUB_API_TOKEN': os.getenv('GITHUB_API_TOKEN'),
         'HUGGINGFACE_API_TOKEN': os.getenv('HUGGINGFACE_API_TOKEN'),
         'NVD_API_KEY': os.getenv('NVD_API_KEY'),
-        'STACK_EXCHANGE_API_KEY': os.getenv('STACK_EXCHANGE_API_KEY')
+        'STACK_EXCHANGE_API_KEY': os.getenv('STACK_EXCHANGE_API_KEY'),
     }
     
     valid_keys = {k: v for k, v in keys.items() if v and v != f'your_{k.lower()}_here'}
     
-    config.USE_API_KEYS = len(valid_keys) > 0
-    if not config.USE_API_KEYS:
+    if len(valid_keys) == 0:
         logger.warning("Aucune clé d'API valide trouvée. Le bot fonctionnera en mode dégradé avec des pauses plus longues.")
     else:
+        missing = set(keys.keys()) - set(valid_keys.keys())
+        if missing:
+            logger.warning(f"Clés d'API manquantes ou non configurées: {', '.join(missing)}")
         logger.info(f"Clés d'API valides trouvées pour: {', '.join(valid_keys.keys())}.")
         
     return valid_keys
@@ -297,7 +301,7 @@ def check_api_keys():
 def make_request(url, headers=None, params=None, is_api_call=True):
     config.REQUEST_COUNT += 1
     
-    pause_factor = 1 if config.USE_API_KEYS else 2
+    pause_factor = 1 if len(check_api_keys()) > 0 else 2
     
     if config.REQUEST_COUNT >= config.MAX_REQUESTS_BEFORE_PAUSE:
         pause_time = random.uniform(config.MIN_PAUSE * pause_factor, config.MAX_PAUSE * pause_factor)
@@ -368,70 +372,71 @@ def save_qa_pair(question, answer, category, subcategory, source, attack_signatu
         st.session_state.total_qa_pairs += 1
         st.session_state.qa_data.append(qa_data)
         
-        logger.info(f"Paire Q/R sauvegardée: {filename} (Total: {st.session_state.total_qa_pairs})")
-        st.session_state.logs.append(f"Sauvegardé: {filename}")
+        log_message = f"Paire Q/R sauvegardée: {filename} (Total: {st.session_state.total_qa_pairs})"
+        logger.info(log_message)
+        st.session_state.logs.append(log_message)
     except Exception as e:
         logger.error(f"Erreur lors de la sauvegarde du fichier {filename}: {str(e)}")
+        st.session_state.logs.append(f"Erreur: Impossible de sauvegarder {filename}")
 
-def collect_kaggle_data(queries, num_pages, results_per_page):
+
+def collect_kaggle_data(queries):
     logger.info("Début de la collecte des données Kaggle...")
     
-    if not os.getenv('KAGGLE_USERNAME') or not os.getenv('KAGGLE_KEY'):
-        logger.warning("Clés Kaggle non configurées. La collecte Kaggle est ignorée.")
-        st.session_state.logs.append("ATTENTION: Clés Kaggle non configurées. Collecte ignorée.")
-        return
-        
-    os.environ['KAGGLE_USERNAME'] = os.getenv('KAGGLE_USERNAME')
-    os.environ['KAGGLE_KEY'] = os.getenv('KAGGLE_KEY')
-    import kaggle
-    kaggle.api.authenticate()
-    
-    search_queries = queries.split('\n') if queries else ["cybersecurity", "vulnerability"]
-    
-    if ia_enricher.available and st.session_state.enable_enrichment:
-        adaptive_queries = ia_enricher.generate_adaptive_queries("Initial data keywords: " + ", ".join(search_queries))
-        search_queries.extend(adaptive_queries)
-    
-    for query in list(set(search_queries)):
-        logger.info(f"Recherche de datasets Kaggle pour: {query}")
-        try:
-            # Kaggle API ne supporte pas la pagination et "results_per_page"
-            datasets = kaggle.api.dataset_list(search=query, max_results=results_per_page)
-            if not datasets:
-                logger.info(f"Aucun dataset trouvé pour la requête '{query}'.")
-                continue
+    try:
+        if not os.getenv('KAGGLE_USERNAME') or not os.getenv('KAGGLE_KEY'):
+            logger.warning("Clés Kaggle non configurées. La collecte Kaggle est ignorée.")
+            st.session_state.logs.append("ATTENTION: Clés Kaggle non configurées. Collecte Kaggle ignorée.")
+            return
             
-            for dataset in datasets:
-                dataset_ref = dataset.ref
-                if ia_enricher.available and st.session_state.enable_enrichment:
-                    is_relevant, _, _, relevance_score = ia_enricher.analyze_content_relevance(dataset.title + " " + dataset.subtitle)
-                    if not is_relevant or relevance_score < st.session_state.min_relevance:
-                        logger.info(f"Dataset non pertinent ({relevance_score}%): {dataset_ref}. Ignoré.")
-                        continue
-                
-                logger.info(f"Traitement du dataset: {dataset_ref}")
-                download_dir = Path("data") / "security" / "kaggle" / dataset_ref.replace('/', '_')
-                shutil.rmtree(download_dir, ignore_errors=True) # S'assurer que le dossier est vide
-                download_dir.mkdir(parents=True, exist_ok=True)
-                kaggle.api.dataset_download_files(dataset_ref, path=download_dir, unzip=True)
-                
-                for file_path in download_dir.glob('*'):
-                    if file_path.is_file() and file_path.suffix.lower() in ['.json', '.csv', '.txt']:
-                        try:
-                            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
-                                file_content = f.read()[:5000]
-                            is_relevant, signatures, security_tags, _ = ia_enricher.analyze_content_relevance(file_content)
-                            if is_relevant:
-                                save_qa_pair(
-                                    question=f"Quelles informations de sécurité contient le fichier {file_path.name} du dataset '{dataset.title}'?",
-                                    answer=file_content, category="security", subcategory="vulnerability",
-                                    source=f"kaggle_{dataset_ref}", attack_signatures=signatures, tags=security_tags
-                                )
-                        except Exception as e:
-                            logger.error(f"Erreur lors du traitement du fichier {file_path}: {str(e)}")
-                time.sleep(random.uniform(2, 4))
-        except Exception as e:
-            logger.error(f"Erreur lors de la collecte des données Kaggle pour {query}: {str(e)}")
+        os.environ['KAGGLE_USERNAME'] = os.getenv('KAGGLE_USERNAME')
+        os.environ['KAGGLE_KEY'] = os.getenv('KAGGLE_KEY')
+        import kaggle
+        kaggle.api.authenticate()
+        
+        search_queries = queries.split('\n') if queries else ["cybersecurity", "vulnerability"]
+        
+        if ia_enricher.available and st.session_state.enable_enrichment:
+            adaptive_queries = ia_enricher.generate_adaptive_queries("Initial data keywords: " + ", ".join(search_queries))
+            search_queries.extend(adaptive_queries)
+        
+        for query in list(set(search_queries)):
+            logger.info(f"Recherche de datasets Kaggle pour: {query}")
+            try:
+                datasets = kaggle.api.dataset_list(search=query, max_results=5)
+                for dataset in datasets:
+                    dataset_ref = dataset.ref
+                    if ia_enricher.available and st.session_state.enable_enrichment:
+                        is_relevant, _, _, relevance_score = ia_enricher.analyze_content_relevance(dataset.title + " " + dataset.subtitle)
+                        if not is_relevant or relevance_score < st.session_state.min_relevance:
+                            logger.info(f"Dataset non pertinent ({relevance_score}%): {dataset_ref}. Ignoré.")
+                            continue
+                    
+                    logger.info(f"Traitement du dataset: {dataset_ref}")
+                    download_dir = Path("data") / "security" / "kaggle" / dataset_ref.replace('/', '_')
+                    shutil.rmtree(download_dir, ignore_errors=True)
+                    download_dir.mkdir(parents=True, exist_ok=True)
+                    kaggle.api.dataset_download_files(dataset_ref, path=download_dir, unzip=True)
+                    
+                    for file_path in download_dir.glob('*'):
+                        if file_path.is_file() and file_path.suffix.lower() in ['.json', '.csv', '.txt']:
+                            try:
+                                with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                                    file_content = f.read()[:5000]
+                                is_relevant, signatures, security_tags, _ = ia_enricher.analyze_content_relevance(file_content)
+                                if is_relevant:
+                                    save_qa_pair(
+                                        question=f"Quelles informations de sécurité contient le fichier {file_path.name} du dataset '{dataset.title}'?",
+                                        answer=file_content, category="security", subcategory="vulnerability",
+                                        source=f"kaggle_{dataset_ref}", attack_signatures=signatures, tags=security_tags
+                                    )
+                            except Exception as e:
+                                logger.error(f"Erreur lors du traitement du fichier {file_path}: {str(e)}")
+                    time.sleep(random.uniform(2, 4))
+            except Exception as e:
+                logger.error(f"Erreur lors de la collecte des données Kaggle pour {query}: {str(e)}")
+    except Exception as e:
+        logger.error(f"Erreur inattendue dans collect_kaggle_data: {str(e)}")
     logger.info("Collecte des données Kaggle terminée.")
 
 def collect_github_data(queries, num_pages, results_per_page):
@@ -444,7 +449,7 @@ def collect_github_data(queries, num_pages, results_per_page):
         headers["Authorization"] = f"token {github_token}"
     else:
         logger.warning("Clé GitHub non configurée. La collecte GitHub est ignorée.")
-        st.session_state.logs.append("ATTENTION: Clé GitHub non configurée. Collecte ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé GitHub non configurée. Collecte GitHub ignorée.")
         return
     
     search_queries = queries.split('\n') if queries else ["topic:devsecops", "topic:security"]
@@ -453,59 +458,64 @@ def collect_github_data(queries, num_pages, results_per_page):
         logger.info(f"Recherche de repositories pour: '{query}' sur {num_pages} page(s)")
         
         for page_number in range(1, num_pages + 1):
-            logger.info(f"Consultation de la page {page_number}...")
-            search_url = f"{base_url}/search/repositories"
-            
-            params = {
-                "q": query,
-                "sort": "stars",
-                "per_page": results_per_page,
-                "page": page_number
-            }
-            
-            response = make_request(search_url, headers=headers, params=params)
-            if not response:
-                break
-            
-            data = response.json()
-            items = data.get("items", [])
-            
-            if not items:
-                logger.info(f"Fin des résultats pour cette requête (page {page_number}).")
-                break
-            
-            for repo in items:
-                repo_name = repo["full_name"].replace("/", "_")
-                logger.info(f"Traitement du repository: {repo['full_name']}")
+            try:
+                logger.info(f"Consultation de la page {page_number}...")
+                st.session_state.logs.append(f"GitHub: page {page_number} pour '{query}'")
+                search_url = f"{base_url}/search/repositories"
                 
-                issues_url = f"{base_url}/repos/{repo['full_name']}/issues"
-                issues_params = {"state": "closed", "labels": "security,bug,vulnerability", "per_page": 10}
-                issues_response = make_request(issues_url, headers=headers, params=issues_params)
+                params = {
+                    "q": query,
+                    "sort": "stars",
+                    "per_page": results_per_page,
+                    "page": page_number
+                }
                 
-                if issues_response:
-                    issues_data = issues_response.json()
-                    for issue in issues_data:
-                        if "pull_request" in issue: continue
-                        question = issue.get("title", "")
-                        body = clean_html(issue.get("body", ""))
-                        if not question or not body or len(body) < 50: continue
-                        
-                        comments_url = issue.get("comments_url")
-                        comments_response = make_request(comments_url, headers=headers)
-                        answer_parts = []
-                        if comments_response:
-                            comments_data = comments_response.json()
-                            for comment in comments_data:
-                                comment_body = clean_html(comment.get("body", ""))
-                                if comment_body: answer_parts.append(comment_body)
-                        
-                        if answer_parts:
-                            answer = "\n\n".join(answer_parts)
-                            save_qa_pair(
-                                question=f"{question}: {body}", answer=answer, category="devsecops",
-                                subcategory="github", source=f"github_{repo_name}"
-                            )
-                time.sleep(random.uniform(1, 3))
+                response = make_request(search_url, headers=headers, params=params)
+                if not response:
+                    break
+                
+                data = response.json()
+                items = data.get("items", [])
+                
+                if not items:
+                    logger.info(f"Fin des résultats pour cette requête (page {page_number}).")
+                    break
+                
+                for repo in items:
+                    repo_name = repo["full_name"].replace("/", "_")
+                    logger.info(f"Traitement du repository: {repo['full_name']}")
+                    
+                    issues_url = f"{base_url}/repos/{repo['full_name']}/issues"
+                    issues_params = {"state": "closed", "labels": "security,bug,vulnerability", "per_page": 10}
+                    issues_response = make_request(issues_url, headers=headers, params=issues_params)
+                    
+                    if issues_response:
+                        issues_data = issues_response.json()
+                        for issue in issues_data:
+                            if "pull_request" in issue: continue
+                            question = issue.get("title", "")
+                            body = clean_html(issue.get("body", ""))
+                            if not question or not body or len(body) < 50: continue
+                            
+                            comments_url = issue.get("comments_url")
+                            comments_response = make_request(comments_url, headers=headers)
+                            answer_parts = []
+                            if comments_response:
+                                comments_data = comments_response.json()
+                                for comment in comments_data:
+                                    comment_body = clean_html(comment.get("body", ""))
+                                    if comment_body: answer_parts.append(comment_body)
+                            
+                            if answer_parts:
+                                answer = "\n\n".join(answer_parts)
+                                save_qa_pair(
+                                    question=f"{question}: {body}", answer=answer, category="devsecops",
+                                    subcategory="github", source=f"github_{repo_name}"
+                                )
+                    time.sleep(random.uniform(1, 3))
+            except Exception as e:
+                logger.error(f"Erreur lors de la collecte GitHub pour la page {page_number}: {str(e)}")
+                st.session_state.logs.append(f"Erreur GitHub: {str(e)}")
     logger.info("Collecte des données GitHub terminée.")
 
 def collect_huggingface_data(queries, num_pages, results_per_page):
@@ -518,50 +528,52 @@ def collect_huggingface_data(queries, num_pages, results_per_page):
         headers["Authorization"] = f"Bearer {hf_token}"
     else:
         logger.warning("Clé Hugging Face non configurée. La collecte Hugging Face est ignorée.")
-        st.session_state.logs.append("ATTENTION: Clé Hugging Face non configurée. Collecte ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé Hugging Face non configurée. Collecte Hugging Face ignorée.")
         return
     
     search_queries = queries.split('\n') if queries else ["security", "devsecops"]
     for query in search_queries:
         logger.info(f"Recherche de datasets pour: {query}")
         
-        # Hugging Face API ne supporte pas la pagination par page_number, mais par 'limit' et 'offset'
-        # On va simuler la pagination en ajustant l'offset
         for page_number in range(num_pages):
-            offset = page_number * results_per_page
-            search_url = f"{base_url}/datasets"
-            params = {"search": query, "limit": results_per_page, "offset": offset}
-            
-            response = make_request(search_url, headers=headers, params=params)
-            if not response: continue
-            
-            data = response.json()
-            if not data:
-                logger.info(f"Fin des résultats pour la requête '{query}'.")
-                break
+            try:
+                offset = page_number * results_per_page
+                search_url = f"{base_url}/datasets"
+                params = {"search": query, "limit": results_per_page, "offset": offset}
                 
-            for dataset in data:
-                dataset_id = dataset["id"].replace("/", "_")
-                logger.info(f"Traitement du dataset: {dataset['id']}")
-                dataset_url = f"{base_url}/datasets/{dataset['id']}"
-                dataset_response = make_request(dataset_url, headers=headers)
+                response = make_request(search_url, headers=headers, params=params)
+                if not response: continue
                 
-                if dataset_response:
-                    dataset_data = dataset_response.json()
-                    description = clean_html(dataset_data.get("description", ""))
-                    if not description or len(description) < 100: continue
-                    tags = dataset_data.get("tags", [])
-                    tags_text = ", ".join(tags) if tags else "No tags"
-                    answer = f"Dataset: {dataset_data.get('id', '')}\nDownloads: {dataset_data.get('downloads', 0)}\nTags: {tags_text}\n\n{description}"
+                data = response.json()
+                if not data:
+                    logger.info(f"Fin des résultats pour la requête '{query}'.")
+                    break
                     
-                    save_qa_pair(
-                        question=f"What is the {dataset_data.get('id', '')} dataset about?", answer=answer,
-                        category="security", subcategory="dataset", source=f"huggingface_{dataset_id}", tags=tags
-                    )
-                time.sleep(random.uniform(1, 3))
+                for dataset in data:
+                    dataset_id = dataset["id"].replace("/", "_")
+                    logger.info(f"Traitement du dataset: {dataset['id']}")
+                    dataset_url = f"{base_url}/datasets/{dataset['id']}"
+                    dataset_response = make_request(dataset_url, headers=headers)
+                    
+                    if dataset_response:
+                        dataset_data = dataset_response.json()
+                        description = clean_html(dataset_data.get("description", ""))
+                        if not description or len(description) < 100: continue
+                        tags = dataset_data.get("tags", [])
+                        tags_text = ", ".join(tags) if tags else "No tags"
+                        answer = f"Dataset: {dataset_data.get('id', '')}\nDownloads: {dataset_data.get('downloads', 0)}\nTags: {tags_text}\n\n{description}"
+                        
+                        save_qa_pair(
+                            question=f"What is the {dataset_data.get('id', '')} dataset about?", answer=answer,
+                            category="security", subcategory="dataset", source=f"huggingface_{dataset_id}", tags=tags
+                        )
+                    time.sleep(random.uniform(1, 3))
+            except Exception as e:
+                logger.error(f"Erreur lors de la collecte Hugging Face: {str(e)}")
+                st.session_state.logs.append(f"Erreur Hugging Face: {str(e)}")
     logger.info("Collecte des données Hugging Face terminée.")
 
-def collect_nvd_data(queries, num_pages, results_per_page):
+def collect_nvd_data(num_pages, results_per_page):
     logger.info("Début de la collecte des données NVD...")
     base_url = "https://services.nvd.nist.gov/rest/json/cves/2.0"
     headers = {"Accept": "application/json"}
@@ -571,46 +583,51 @@ def collect_nvd_data(queries, num_pages, results_per_page):
         headers["apiKey"] = nvd_key
     else:
         logger.warning("Clé NVD non configurée. La collecte NVD est ignorée.")
-        st.session_state.logs.append("ATTENTION: Clé NVD non configurée. Collecte ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé NVD non configurée. Collecte NVD ignorée.")
         return
     
     for page in range(num_pages):
-        start_index = page * results_per_page
-        logger.info(f"Consultation de la page NVD, index de départ: {start_index}")
-        params = {"resultsPerPage": results_per_page, "startIndex": start_index}
-        response = make_request(base_url, headers=headers, params=params)
-        
-        if not response:
-            logger.warning("Impossible de récupérer les données du NVD. Arrêt de la collecte NVD.")
-            break
-            
-        data = response.json()
-        vulnerabilities = data.get("vulnerabilities", [])
-        if not vulnerabilities:
-            logger.info("Fin des résultats pour la collecte NVD.")
-            break
-        
-        logger.info(f"Traitement de {len(vulnerabilities)} vulnérabilités...")
-        
-        for vuln in vulnerabilities:
-            cve_data = vuln.get("cve", {})
-            cve_id = cve_data.get("id", "")
-            descriptions = cve_data.get("descriptions", [])
-            description = next((desc.get("value", "") for desc in descriptions if desc.get("lang") == "en"), "")
-            if not description or len(description) < 50: continue
+        try:
+            start_index = page * results_per_page
+            logger.info(f"Consultation de la page NVD, index de départ: {start_index}")
+            st.session_state.logs.append(f"NVD: page {page + 1}")
+            params = {"resultsPerPage": results_per_page, "startIndex": start_index}
+            response = make_request(base_url, headers=headers, params=params)
             
-            cvss_v3 = cve_data.get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {})
-            severity = cvss_v3.get("baseSeverity", "UNKNOWN")
-            score = cvss_v3.get("baseScore", 0)
-            references = [ref.get("url", "") for ref in cve_data.get("references", [])]
+            if not response:
+                logger.warning("Impossible de récupérer les données du NVD. Arrêt de la collecte NVD.")
+                break
+                
+            data = response.json()
+            vulnerabilities = data.get("vulnerabilities", [])
+            if not vulnerabilities:
+                logger.info("Fin des résultats pour la collecte NVD.")
+                break
             
-            answer = f"CVE ID: {cve_id}\nSeverity: {severity}\nCVSS Score: {score}\nReferences: {', '.join(references[:5])}\n\nDescription: {description}"
+            logger.info(f"Traitement de {len(vulnerabilities)} vulnérabilités...")
             
-            save_qa_pair(
-                question=f"What is the vulnerability {cve_id}?", answer=answer,
-                category="security", subcategory="vulnerability", source=f"nvd_{cve_id}"
-            )
-        time.sleep(random.uniform(1, 3))
+            for vuln in vulnerabilities:
+                cve_data = vuln.get("cve", {})
+                cve_id = cve_data.get("id", "")
+                descriptions = cve_data.get("descriptions", [])
+                description = next((desc.get("value", "") for desc in descriptions if desc.get("lang") == "en"), "")
+                if not description or len(description) < 50: continue
+                
+                cvss_v3 = cve_data.get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {})
+                severity = cvss_v3.get("baseSeverity", "UNKNOWN")
+                score = cvss_v3.get("baseScore", 0)
+                references = [ref.get("url", "") for ref in cve_data.get("references", [])]
+                
+                answer = f"CVE ID: {cve_id}\nSeverity: {severity}\nCVSS Score: {score}\nReferences: {', '.join(references[:5])}\n\nDescription: {description}"
+                
+                save_qa_pair(
+                    question=f"What is the vulnerability {cve_id}?", answer=answer,
+                    category="security", subcategory="vulnerability", source=f"nvd_{cve_id}"
+                )
+            time.sleep(random.uniform(1, 3))
+        except Exception as e:
+            logger.error(f"Erreur lors de la collecte NVD: {str(e)}")
+            st.session_state.logs.append(f"Erreur NVD: {str(e)}")
     logger.info("Collecte des données NVD terminée.")
 
 def collect_stack_exchange_data(queries, num_pages, results_per_page):
@@ -623,7 +640,7 @@ def collect_stack_exchange_data(queries, num_pages, results_per_page):
         params_base["key"] = se_key
     else:
         logger.warning("Clé Stack Exchange non configurée. La collecte est ignorée.")
-        st.session_state.logs.append("ATTENTION: Clé Stack Exchange non configurée. Collecte ignorée.")
+        st.session_state.logs.append("ATTENTION: Clé Stack Exchange non configurée. Collecte Stack Exchange ignorée.")
         return
     
     sites = [
@@ -645,43 +662,97 @@ def collect_stack_exchange_data(queries, num_pages, results_per_page):
         
         for tag in list(set(tags)):
             logger.info(f"Recherche de questions avec le tag: '{tag}'")
-            questions_url = f"{base_url}/questions"
             
             for page_number in range(1, num_pages + 1):
-                params = {**params_base, "site": site, "tagged": tag, "page": page_number}
-                
-                response = make_request(questions_url, params=params)
-                if not response: continue
-                
-                questions_data = response.json()
-                items = questions_data.get("items", [])
-                
-                if not items:
-                    logger.info(f"Fin des résultats pour le tag '{tag}' à la page {page_number}.")
-                    break
+                try:
+                    questions_url = f"{base_url}/questions"
+                    params = {**params_base, "site": site, "tagged": tag, "page": page_number}
                     
-                for question in items:
-                    question_id = question.get("question_id")
-                    title = question.get("title", "")
-                    body = clean_html(question.get("body", ""))
-                    if not body or len(body) < 50: continue
+                    response = make_request(questions_url, params=params)
+                    if not response: continue
                     
-                    answers_url = f"{base_url}/questions/{question_id}/answers"
-                    answers_params = {**params_base, "site": site}
-                    answers_response = make_request(answers_url, params=answers_params)
-                    answer_body = ""
-                    if answers_response and answers_response.json().get("items"):
-                        answer_body = clean_html(answers_response.json()["items"][0].get("body", ""))
+                    questions_data = response.json()
+                    items = questions_data.get("items", [])
                     
-                    if answer_body:
-                        save_qa_pair(
-                            question=title, answer=answer_body, category=category,
-                            subcategory=subcategory, source=f"{site}_{question_id}", tags=question.get("tags", [])
-                        )
-                    time.sleep(random.uniform(1, 3))
+                    if not items:
+                        logger.info(f"Fin des résultats pour le tag '{tag}' à la page {page_number}.")
+                        break
+                        
+                    for question in items:
+                        question_id = question.get("question_id")
+                        title = question.get("title", "")
+                        body = clean_html(question.get("body", ""))
+                        if not body or len(body) < 50: continue
+                        
+                        answers_url = f"{base_url}/questions/{question_id}/answers"
+                        answers_params = {**params_base, "site": site}
+                        answers_response = make_request(answers_url, params=answers_params)
+                        answer_body = ""
+                        if answers_response and answers_response.json().get("items"):
+                            answer_body = clean_html(answers_response.json()["items"][0].get("body", ""))
+                        
+                        if answer_body:
+                            save_qa_pair(
+                                question=title, answer=answer_body, category=category,
+                                subcategory=subcategory, source=f"{site}_{question_id}", tags=question.get("tags", [])
+                            )
+                        time.sleep(random.uniform(1, 3))
+                except Exception as e:
+                    logger.error(f"Erreur lors de la collecte Stack Exchange: {str(e)}")
+                    st.session_state.logs.append(f"Erreur Stack Exchange: {str(e)}")
     logger.info("Collecte des données Stack Exchange terminée.")
 
-def run_data_collection(sources, queries, num_pages, results_per_page):
+def collect_web_data(url):
+    logger.info(f"Début de la collecte des données pour l'URL: {url}")
+    
+    try:
+        st.session_state.logs.append(f"Début de la collecte pour {url}...")
+        response = make_request(url)
+        if not response or response.status_code != 200:
+            logger.error(f"Impossible de récupérer le contenu de l'URL: {url}")
+            st.session_state.logs.append(f"Erreur: Impossible de récupérer l'URL {url}")
+            return
+
+        soup = BeautifulSoup(response.text, 'html.parser')
+        
+        for script in soup(["script", "style", "header", "footer", "nav"]):
+            script.extract()
+        
+        raw_text = soup.get_text()
+        clean_text = clean_html(raw_text)
+        
+        if not clean_text or len(clean_text) < 100:
+            logger.warning(f"Contenu de l'URL trop court ou vide: {url}")
+            st.session_state.logs.append("Avertissement: Contenu de l'URL trop court ou vide.")
+            return
+
+        is_relevant, signatures, security_tags, _ = ia_enricher.analyze_content_relevance(clean_text)
+        
+        if is_relevant:
+            title = soup.title.string if soup.title else os.path.basename(url)
+            question = f"What security information is on the page '{title}'?"
+            answer = clean_text
+            
+            save_qa_pair(
+                question=question,
+                answer=answer,
+                category="security",
+                subcategory="web-scraping",
+                source=f"web_{re.sub(r'[^a-zA-Z0-9]+', '', url)[:30]}",
+                attack_signatures=signatures,
+                tags=security_tags
+            )
+        else:
+            logger.info(f"Contenu de l'URL non pertinent pour DevSecOps: {url}")
+            st.session_state.logs.append(f"Contenu de l'URL non pertinent pour DevSecOps.")
+
+    except Exception as e:
+        logger.error(f"Erreur lors du scraping de l'URL {url}: {str(e)}")
+        st.session_state.logs.append(f"Erreur lors du scraping de l'URL {url}")
+    
+    logger.info("Collecte des données web terminée.")
+
+def run_data_collection(sources, queries, web_url, num_pages, results_per_page):
     st.session_state.bot_status = "En cours d'exécution"
     st.session_state.logs = []
     
@@ -689,51 +760,63 @@ def run_data_collection(sources, queries, num_pages, results_per_page):
     
     progress_bar = st.progress(0)
     status_text = st.empty()
+    log_container = st.empty()
     
     enabled_sources = [s for s, enabled in sources.items() if enabled]
+    
+    if "Web Scraping" in enabled_sources and web_url:
+        enabled_sources.remove("Web Scraping")
+        enabled_sources.insert(0, "Web Scraping")
+        
     total_sources = len(enabled_sources)
     completed_sources = 0
     
     for source_name in enabled_sources:
-        if source_name == "Kaggle" and 'KAGGLE_USERNAME' not in os.environ:
-            logger.warning("Clés Kaggle non définies dans les variables d'environnement. Saut de la collecte Kaggle.")
-            continue
-        if source_name == "GitHub" and not valid_keys.get('GITHUB_API_TOKEN'):
-            logger.warning("Clé GitHub non définie. Saut de la collecte GitHub.")
-            continue
-        if source_name == "Hugging Face" and not valid_keys.get('HUGGINGFACE_API_TOKEN'):
-            logger.warning("Clé Hugging Face non définie. Saut de la collecte Hugging Face.")
-            continue
-        if source_name == "NVD" and not valid_keys.get('NVD_API_KEY'):
-            logger.warning("Clé NVD non définie. Saut de la collecte NVD.")
-            continue
-        if source_name == "Stack Exchange" and not valid_keys.get('STACK_EXCHANGE_API_KEY'):
-            logger.warning("Clé Stack Exchange non définie. Saut de la collecte Stack Exchange.")
-            continue
-            
         status_text.text(f"Collecte des données de {source_name}...")
-        try:
-            if source_name == "Kaggle":
-                collect_kaggle_data(queries.get("Kaggle", ""), num_pages, results_per_page)
-            elif source_name == "GitHub":
+        log_container.text("Logs en temps réel:\n" + "\n".join(st.session_state.logs))
+        
+        if source_name == "Kaggle":
+            collect_kaggle_data(queries.get("Kaggle", ""))
+        elif source_name == "GitHub":
+            if 'GITHUB_API_TOKEN' in valid_keys:
                 collect_github_data(queries.get("GitHub", ""), num_pages, results_per_page)
-            elif source_name == "Hugging Face":
+            else:
+                logger.warning("Clé GitHub non définie. Saut de la collecte GitHub.")
+                st.session_state.logs.append("ATTENTION: Clé GitHub non définie. Collecte GitHub ignorée.")
+        elif source_name == "Hugging Face":
+            if 'HUGGINGFACE_API_TOKEN' in valid_keys:
                 collect_huggingface_data(queries.get("Hugging Face", ""), num_pages, results_per_page)
-            elif source_name == "NVD":
-                collect_nvd_data(queries.get("NVD", ""), num_pages, results_per_page)
-            elif source_name == "Stack Exchange":
+            else:
+                logger.warning("Clé Hugging Face non définie. Saut de la collecte Hugging Face.")
+                st.session_state.logs.append("ATTENTION: Clé Hugging Face non définie. Collecte Hugging Face ignorée.")
+        elif source_name == "NVD":
+            if 'NVD_API_KEY' in valid_keys:
+                collect_nvd_data(num_pages, results_per_page)
+            else:
+                logger.warning("Clé NVD non définie. Saut de la collecte NVD.")
+                st.session_state.logs.append("ATTENTION: Clé NVD non définie. Collecte NVD ignorée.")
+        elif source_name == "Stack Exchange":
+            if 'STACK_EXCHANGE_API_KEY' in valid_keys:
                 collect_stack_exchange_data(queries.get("Stack Exchange", ""), num_pages, results_per_page)
-        except Exception as e:
-            logger.error(f"Erreur fatale lors de la collecte de {source_name}: {str(e)}")
-        
+            else:
+                logger.warning("Clé Stack Exchange non définie. Saut de la collecte Stack Exchange.")
+                st.session_state.logs.append("ATTENTION: Clé Stack Exchange non définie. Collecte Stack Exchange ignorée.")
+        elif source_name == "Web Scraping":
+            if web_url:
+                collect_web_data(web_url)
+            else:
+                st.session_state.logs.append("ATTENTION: URL de scraping non fournie. Collecte ignorée.")
+
         completed_sources += 1
         progress_bar.progress(completed_sources / total_sources)
-    
+        log_container.text("Logs en temps réel:\n" + "\n".join(st.session_state.logs))
+        time.sleep(1) # Pause pour la mise à jour visuelle
+
     st.session_state.bot_status = "Arrêté"
     st.info("Collecte des données terminée!")
     progress_bar.empty()
     status_text.empty()
-    
+    log_container.text("Logs en temps réel:\n" + "\n".join(st.session_state.logs))
     st.rerun()
 
 def main():
@@ -773,22 +856,27 @@ def main():
         st.header("Lancer la collecte")
         
         st.subheader("Sources de données")
-        sources_columns = st.columns(5)
+        sources_columns = st.columns(6)
         sources = {
             "GitHub": sources_columns[0].checkbox("GitHub", value=True),
-            "Kaggle": sources_columns[1].checkbox("Kaggle", value=True),
-            "Hugging Face": sources_columns[2].checkbox("Hugging Face", value=True),
-            "NVD": sources_columns[3].checkbox("NVD", value=True),
-            "Stack Exchange": sources_columns[4].checkbox("Stack Exchange", value=True),
+            "Hugging Face": sources_columns[1].checkbox("Hugging Face", value=True),
+            "NVD": sources_columns[2].checkbox("NVD", value=True),
+            "Stack Exchange": sources_columns[3].checkbox("Stack Exchange", value=True),
+            "Kaggle": sources_columns[4].checkbox("Kaggle", value=True),
+            "Web Scraping": sources_columns[5].checkbox("Web Scraping", value=True)
         }
+
+        web_url = st.text_input("URL à scraper (optionnel)", help="Entrez une URL pour extraire les données de sécurité.")
+
+        st.info("En cliquant sur 'Lancer la collecte', vous reconnaissez que vous disposez des droits légaux de scraping et de manipulation des données du site fourni, et nous déclinons toute responsabilité.")
         
         st.subheader("Requêtes de recherche")
         queries = {}
         queries["GitHub"] = st.text_area("Requêtes GitHub (une par ligne)", "topic:devsecops\ntopic:security\nvulnerability")
         queries["Kaggle"] = st.text_area("Requêtes Kaggle (une par ligne)", "cybersecurity\nvulnerability dataset\npenetration testing")
         queries["Hugging Face"] = st.text_area("Requêtes Hugging Face (une par ligne)", "security dataset\nvulnerability\nlanguage model security")
-        queries["NVD"] = ""
         queries["Stack Exchange"] = st.text_area("Tags Stack Exchange (un par ligne)", "devsecops\nsecurity\nvulnerability")
+        queries["NVD"] = ""
         
         st.markdown("---")
         
@@ -797,7 +885,7 @@ def main():
                 st.session_state.logs = []
                 st.session_state.qa_data = []
                 st.session_state.total_qa_pairs = 0
-                run_data_collection(sources, queries, num_pages, results_per_page)
+                run_data_collection(sources, queries, web_url, num_pages, results_per_page)
         else:
             st.warning("La collecte est en cours. Veuillez attendre qu'elle se termine.")
             if st.button("Forcer l'arrêt", use_container_width=True, type="secondary"):
@@ -862,12 +950,14 @@ def main():
         with llm_col1:
             if st.button("Démarrer le serveur LLM", type="primary", use_container_width=True):
                 start_llm_server()
+                st.rerun()
             if st.button("Vérifier le statut du serveur", use_container_width=True):
                 check_server_status()
                 st.rerun()
         with llm_col2:
             if st.button("Arrêter le serveur LLM", type="secondary", use_container_width=True):
                 stop_llm_server()
+                st.rerun()
         
         st.markdown("---")