|
|
|
|
|
import os |
|
|
import sys |
|
|
import time |
|
|
import random |
|
|
import logging |
|
|
import requests |
|
|
import json |
|
|
import re |
|
|
import subprocess |
|
|
import shutil |
|
|
from datetime import datetime |
|
|
from pathlib import Path |
|
|
import streamlit as st |
|
|
import pandas as pd |
|
|
import plotly.express as px |
|
|
import plotly.graph_objects as go |
|
|
from bs4 import BeautifulSoup |
|
|
import html2text |
|
|
|
|
|
|
|
|
from config import app_config as config |
|
|
|
|
|
|
|
|
st.set_page_config( |
|
|
page_title="DevSecOps Data Bot", |
|
|
layout="wide", |
|
|
initial_sidebar_state="expanded" |
|
|
) |
|
|
|
|
|
|
|
|
config.init_session_state() |
|
|
|
|
|
|
|
|
h = html2text.HTML2Text() |
|
|
h.ignore_links = True |
|
|
|
|
|
|
|
|
def setup_logging(): |
|
|
log_dir = Path("logs") |
|
|
log_dir.mkdir(exist_ok=True) |
|
|
log_file = log_dir / f"data_collector_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" |
|
|
logging.basicConfig( |
|
|
level=logging.INFO, |
|
|
format='%(asctime)s - %(levelname)s - %(message)s', |
|
|
handlers=[ |
|
|
logging.FileHandler(log_file), |
|
|
logging.StreamHandler(sys.stdout) |
|
|
] |
|
|
) |
|
|
return logging.getLogger(__name__) |
|
|
|
|
|
logger = setup_logging() |
|
|
|
|
|
|
|
|
def check_server_status(): |
|
|
try: |
|
|
response = requests.get(config.LLM_SERVER_URL.replace("/completion", "/health"), timeout=2) |
|
|
if response.status_code == 200: |
|
|
st.session_state.server_status = "Actif" |
|
|
return True |
|
|
else: |
|
|
st.session_state.server_status = "Inactif" |
|
|
return False |
|
|
except requests.exceptions.RequestException: |
|
|
st.session_state.server_status = "Inactif" |
|
|
return False |
|
|
|
|
|
def start_llm_server(): |
|
|
if check_server_status(): |
|
|
st.info("Le serveur llama.cpp est déjà en cours d'exécution.") |
|
|
return |
|
|
|
|
|
model_path = Path("models/qwen2.5-coder-1.5b-q8_0.gguf") |
|
|
if not model_path.exists(): |
|
|
st.error("Le modèle GGUF n'existe pas. Veuillez le placer dans le dossier models/.") |
|
|
return |
|
|
|
|
|
llama_server = Path("llama.cpp/build/bin/llama-server") |
|
|
if not llama_server.exists(): |
|
|
st.error("llama.cpp n'est pas compilé. Veuillez compiler llama.cpp d'abord.") |
|
|
return |
|
|
|
|
|
start_script = Path("server/start_server.sh") |
|
|
if not start_script.exists(): |
|
|
with open(start_script, 'w') as f: |
|
|
f.write(f"""#!/bin/bash |
|
|
MODEL_PATH="{str(model_path)}" |
|
|
if [ ! -f "$MODEL_PATH" ]; then |
|
|
echo "Le modèle GGUF est introuvable à: $MODEL_PATH" |
|
|
exit 1 |
|
|
fi |
|
|
"{str(llama_server)}" \\ |
|
|
-m "$MODEL_PATH" \\ |
|
|
--port 8080 \\ |
|
|
--host 0.0.0.0 \\ |
|
|
-c 4096 \\ |
|
|
-ngl 999 \\ |
|
|
--threads 8 \\ |
|
|
> "logs/llama_server.log" 2>&1 & |
|
|
echo $! > "server/server.pid" |
|
|
""") |
|
|
os.chmod(start_script, 0o755) |
|
|
|
|
|
try: |
|
|
subprocess.Popen(["bash", str(start_script)]) |
|
|
st.success("Le serveur llama.cpp est en cours de démarrage...") |
|
|
time.sleep(5) |
|
|
check_server_status() |
|
|
if st.session_state.server_status == "Actif": |
|
|
st.success("Serveur llama.cpp démarré avec succès!") |
|
|
else: |
|
|
st.error("Le serveur n'a pas pu démarrer. Vérifiez les logs dans le dossier logs/.") |
|
|
except Exception as e: |
|
|
st.error(f"Erreur lors du démarrage du serveur: {str(e)}") |
|
|
|
|
|
def stop_llm_server(): |
|
|
stop_script = Path("server/stop_server.sh") |
|
|
if not stop_script.exists(): |
|
|
with open(stop_script, 'w') as f: |
|
|
f.write("""#!/bin/bash |
|
|
PID_FILE="server/server.pid" |
|
|
if [ -f "$PID_FILE" ]; then |
|
|
PID=$(cat "$PID_FILE") |
|
|
kill $PID |
|
|
rm "$PID_FILE" |
|
|
echo "Serveur llama.cpp arrêté." |
|
|
else |
|
|
echo "Aucun PID de serveur trouvé." |
|
|
fi |
|
|
""") |
|
|
os.chmod(stop_script, 0o755) |
|
|
|
|
|
try: |
|
|
subprocess.run(["bash", str(stop_script)]) |
|
|
st.success("Le serveur llama.cpp est en cours d'arrêt...") |
|
|
time.sleep(2) |
|
|
check_server_status() |
|
|
if st.session_state.server_status == "Inactif": |
|
|
st.success("Serveur llama.cpp arrêté avec succès!") |
|
|
else: |
|
|
st.warning("Le serveur n'a pas pu être arrêté correctement.") |
|
|
except Exception as e: |
|
|
st.error(f"Erreur lors de l'arrêt du serveur: {str(e)}") |
|
|
|
|
|
def load_prompts(): |
|
|
prompts_file = Path("config/prompts.json") |
|
|
if not prompts_file.exists(): |
|
|
default_prompts = { |
|
|
"enrich_qa": { |
|
|
"system": "Tu es un expert DevSecOps. Améliore cette paire question/réponse en y ajoutant des tags, des signatures d'attaques potentielles, et en structurant les informations. Réponds uniquement avec un objet JSON.", |
|
|
"prompt_template": "Question originale: {question}\nRéponse originale: {answer}\nContexte: {context}\n\nFournis une version améliorée sous forme de JSON:\n{{\n \"question\": \"question améliorée\",\n \"answer\": \"réponse améliorée\",\n \"tags\": [\"tag1\", \"tag2\"],\n \"attack_signatures\": [\"signature1\", \"signature2\"]\n}}" |
|
|
}, |
|
|
"analyze_relevance": { |
|
|
"system": "Analyse ce contenu et détermine s'il est pertinent pour DevSecOps. Si pertinent, extrais les signatures d'attaques connues. Réponds uniquement avec un objet JSON.", |
|
|
"prompt_template": "Contenu: {content}...\n\nRéponds sous forme de JSON:\n{{\n \"relevant\": true,\n \"attack_signatures\": [\"signature1\", \"signature2\"],\n \"security_tags\": [\"tag1\", \"tag2\"],\n \"it_relevance_score\": 0-100\n}}" |
|
|
}, |
|
|
"generate_queries": { |
|
|
"system": "Analyse les données actuelles et génère 5 nouvelles requêtes de recherche pour trouver plus de contenu DevSecOps pertinent, en particulier des signatures d'attaques et vulnérabilités. Réponds uniquement avec un objet JSON.", |
|
|
"prompt_template": "Données actuelles: {current_data}...\n\nRéponds sous forme de JSON:\n{{\n \"queries\": [\"query1\", \"query2\", \"query3\", \"query4\", \"query5\"]\n}}" |
|
|
} |
|
|
} |
|
|
with open(prompts_file, 'w') as f: |
|
|
json.dump(default_prompts, f, indent=2) |
|
|
|
|
|
with open(prompts_file, 'r', encoding='utf-8') as f: |
|
|
return json.load(f) |
|
|
|
|
|
PROMPTS = load_prompts() |
|
|
|
|
|
class IAEnricher: |
|
|
def __init__(self): |
|
|
self.server_url = config.LLM_SERVER_URL |
|
|
self.available = check_server_status() |
|
|
if self.available: |
|
|
logger.info("Serveur llama.cpp détecté et prêt.") |
|
|
else: |
|
|
logger.warning("Serveur llama.cpp non disponible. Les fonctionnalités d'IA seront désactivées.") |
|
|
|
|
|
def _get_qwen_response(self, prompt, **kwargs): |
|
|
if not self.available: |
|
|
return None |
|
|
|
|
|
payload = { |
|
|
"prompt": prompt, |
|
|
"n_predict": kwargs.get('n_predict', 512), |
|
|
"temperature": kwargs.get('temperature', 0.7), |
|
|
"stop": ["<|im_end|>", "</s>"] |
|
|
} |
|
|
|
|
|
try: |
|
|
response = requests.post(self.server_url, json=payload, timeout=60) |
|
|
if response.status_code == 200: |
|
|
return response.json().get('content', '') |
|
|
else: |
|
|
logger.error(f"Erreur du serveur LLM: {response.status_code} - {response.text}") |
|
|
return None |
|
|
except requests.exceptions.RequestException as e: |
|
|
logger.error(f"Erreur de connexion au serveur LLM: {str(e)}") |
|
|
return None |
|
|
|
|
|
def enrich_qa_pair(self, question, answer, context=""): |
|
|
if not self.available or not st.session_state.enable_enrichment: |
|
|
return question, answer, [], [] |
|
|
|
|
|
prompt_template = PROMPTS.get("enrich_qa", {}).get("prompt_template", "") |
|
|
system_prompt = PROMPTS.get("enrich_qa", {}).get("system", "") |
|
|
|
|
|
full_prompt = f"{system_prompt}\n\n{prompt_template.format(question=question, answer=answer, context=context[:500])}" |
|
|
response_text = self._get_qwen_response(full_prompt, n_predict=1024) |
|
|
|
|
|
if response_text: |
|
|
try: |
|
|
start = response_text.find('{') |
|
|
end = response_text.rfind('}') |
|
|
if start != -1 and end != -1: |
|
|
json_str = response_text[start:end+1] |
|
|
enriched_data = json.loads(json_str) |
|
|
return ( |
|
|
enriched_data.get('question', question), |
|
|
enriched_data.get('answer', answer), |
|
|
enriched_data.get('tags', []), |
|
|
enriched_data.get('attack_signatures', []) |
|
|
) |
|
|
except json.JSONDecodeError as e: |
|
|
logger.warning(f"Erreur de décodage JSON de la réponse IA: {e}") |
|
|
|
|
|
return question, answer, [], [] |
|
|
|
|
|
def analyze_content_relevance(self, content): |
|
|
if not self.available or not st.session_state.enable_enrichment: |
|
|
return True, [], [], 50 |
|
|
|
|
|
prompt_template = PROMPTS.get("analyze_relevance", {}).get("prompt_template", "") |
|
|
system_prompt = PROMPTS.get("analyze_relevance", {}).get("system", "") |
|
|
|
|
|
full_prompt = f"{system_prompt}\n\n{prompt_template.format(content=content[:1500])}" |
|
|
response_text = self._get_qwen_response(full_prompt, n_predict=256, temperature=st.session_state.temperature) |
|
|
|
|
|
if response_text: |
|
|
try: |
|
|
start = response_text.find('{') |
|
|
end = response_text.rfind('}') |
|
|
if start != -1 and end != -1: |
|
|
json_str = response_text[start:end+1] |
|
|
analysis = json.loads(json_str) |
|
|
return ( |
|
|
analysis.get('relevant', True), |
|
|
analysis.get('attack_signatures', []), |
|
|
analysis.get('security_tags', []), |
|
|
analysis.get('it_relevance_score', 50) |
|
|
) |
|
|
except json.JSONDecodeError as e: |
|
|
logger.warning(f"Erreur de décodage JSON de la réponse IA: {e}") |
|
|
return True, [], [], 50 |
|
|
|
|
|
def generate_adaptive_queries(self, current_data): |
|
|
if not self.available or not st.session_state.enable_enrichment: |
|
|
return [] |
|
|
|
|
|
prompt_template = PROMPTS.get("generate_queries", {}).get("prompt_template", "") |
|
|
system_prompt = PROMPTS.get("generate_queries", {}).get("system", "") |
|
|
|
|
|
full_prompt = f"{system_prompt}\n\n{prompt_template.format(current_data=current_data[:1000])}" |
|
|
response_text = self._get_qwen_response(full_prompt, n_predict=st.session_state.n_predict) |
|
|
|
|
|
if response_text: |
|
|
try: |
|
|
start = response_text.find('{') |
|
|
end = response_text.rfind('}') |
|
|
if start != -1 and end != -1: |
|
|
json_str = response_text[start:end+1] |
|
|
queries_data = json.loads(json_str) |
|
|
return queries_data.get('queries', []) |
|
|
except json.JSONDecodeError as e: |
|
|
logger.warning(f"Erreur de décodage JSON de la réponse IA: {e}") |
|
|
return [] |
|
|
|
|
|
ia_enricher = IAEnricher() |
|
|
|
|
|
def check_api_keys(): |
|
|
keys = { |
|
|
'GITHUB_API_TOKEN': os.getenv('GITHUB_API_TOKEN'), |
|
|
'HUGGINGFACE_API_TOKEN': os.getenv('HUGGINGFACE_API_TOKEN'), |
|
|
'NVD_API_KEY': os.getenv('NVD_API_KEY'), |
|
|
'STACK_EXCHANGE_API_KEY': os.getenv('STACK_EXCHANGE_API_KEY'), |
|
|
} |
|
|
|
|
|
valid_keys = {k: v for k, v in keys.items() if v and v != f'your_{k.lower()}_here'} |
|
|
|
|
|
if len(valid_keys) == 0: |
|
|
logger.warning("Aucune clé d'API valide trouvée. Le bot fonctionnera en mode dégradé avec des pauses plus longues.") |
|
|
else: |
|
|
missing = set(keys.keys()) - set(valid_keys.keys()) |
|
|
if missing: |
|
|
logger.warning(f"Clés d'API manquantes ou non configurées: {', '.join(missing)}") |
|
|
logger.info(f"Clés d'API valides trouvées pour: {', '.join(valid_keys.keys())}.") |
|
|
|
|
|
return valid_keys |
|
|
|
|
|
def make_request(url, headers=None, params=None, is_api_call=True): |
|
|
config.REQUEST_COUNT += 1 |
|
|
|
|
|
pause_factor = 1 if len(check_api_keys()) > 0 else 2 |
|
|
|
|
|
if config.REQUEST_COUNT >= config.MAX_REQUESTS_BEFORE_PAUSE: |
|
|
pause_time = random.uniform(config.MIN_PAUSE * pause_factor, config.MAX_PAUSE * pause_factor) |
|
|
logger.info(f"Pause de {pause_time:.2f} secondes après {config.REQUEST_COUNT} requêtes...") |
|
|
time.sleep(pause_time) |
|
|
config.REQUEST_COUNT = 0 |
|
|
|
|
|
try: |
|
|
response = requests.get(url, headers=headers, params=params, timeout=30) |
|
|
|
|
|
if response.status_code == 200: |
|
|
return response |
|
|
elif response.status_code in [401, 403]: |
|
|
logger.warning(f"Accès non autorisé à {url}. Vérifiez vos clés d'API.") |
|
|
return None |
|
|
elif response.status_code == 429: |
|
|
retry_after = int(response.headers.get('Retry-After', 10)) |
|
|
logger.warning(f"Limite de débit atteinte. Pause de {retry_after} secondes...") |
|
|
time.sleep(retry_after) |
|
|
return make_request(url, headers, params, is_api_call) |
|
|
else: |
|
|
logger.warning(f"Statut HTTP {response.status_code} pour {url}") |
|
|
return None |
|
|
except requests.exceptions.RequestException as e: |
|
|
logger.error(f"Erreur lors de la requête à {url}: {str(e)}") |
|
|
return None |
|
|
|
|
|
def clean_html(html_content): |
|
|
if not html_content: |
|
|
return "" |
|
|
text = h.handle(html_content) |
|
|
text = re.sub(r'\s+', ' ', text).strip() |
|
|
return text |
|
|
|
|
|
def save_qa_pair(question, answer, category, subcategory, source, attack_signatures=None, tags=None): |
|
|
if ia_enricher.available and st.session_state.enable_enrichment: |
|
|
enriched_question, enriched_answer, enriched_tags, enriched_signatures = ia_enricher.enrich_qa_pair( |
|
|
question, answer, f"{category}/{subcategory}" |
|
|
) |
|
|
|
|
|
question = enriched_question |
|
|
answer = enriched_answer |
|
|
tags = list(set((tags or []) + enriched_tags)) |
|
|
attack_signatures = list(set((attack_signatures or []) + enriched_signatures)) |
|
|
|
|
|
save_dir = Path("data") / category / "qa" |
|
|
save_dir.mkdir(parents=True, exist_ok=True) |
|
|
|
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") |
|
|
filename = f"{subcategory}_{source}_{st.session_state.total_qa_pairs}_{timestamp}.json" |
|
|
filename = re.sub(r'[^\w\s-]', '', filename).replace(' ', '_') |
|
|
|
|
|
qa_data = { |
|
|
"question": question, |
|
|
"answer": answer, |
|
|
"category": category, |
|
|
"subcategory": subcategory, |
|
|
"source": source, |
|
|
"timestamp": timestamp, |
|
|
"attack_signatures": attack_signatures or [], |
|
|
"tags": tags or [] |
|
|
} |
|
|
|
|
|
try: |
|
|
with open(save_dir / filename, "w", encoding="utf-8") as f: |
|
|
json.dump(qa_data, f, indent=2, ensure_ascii=False) |
|
|
|
|
|
st.session_state.total_qa_pairs += 1 |
|
|
st.session_state.qa_data.append(qa_data) |
|
|
|
|
|
log_message = f"Paire Q/R sauvegardée: {filename} (Total: {st.session_state.total_qa_pairs})" |
|
|
logger.info(log_message) |
|
|
st.session_state.logs.append(log_message) |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors de la sauvegarde du fichier {filename}: {str(e)}") |
|
|
st.session_state.logs.append(f"Erreur: Impossible de sauvegarder {filename}") |
|
|
|
|
|
|
|
|
def collect_kaggle_data(queries): |
|
|
logger.info("Début de la collecte des données Kaggle...") |
|
|
|
|
|
try: |
|
|
if not os.getenv('KAGGLE_USERNAME') or not os.getenv('KAGGLE_KEY'): |
|
|
logger.warning("Clés Kaggle non configurées. La collecte Kaggle est ignorée.") |
|
|
st.session_state.logs.append("ATTENTION: Clés Kaggle non configurées. Collecte Kaggle ignorée.") |
|
|
return |
|
|
|
|
|
os.environ['KAGGLE_USERNAME'] = os.getenv('KAGGLE_USERNAME') |
|
|
os.environ['KAGGLE_KEY'] = os.getenv('KAGGLE_KEY') |
|
|
import kaggle |
|
|
kaggle.api.authenticate() |
|
|
|
|
|
search_queries = queries.split('\n') if queries else ["cybersecurity", "vulnerability"] |
|
|
|
|
|
if ia_enricher.available and st.session_state.enable_enrichment: |
|
|
adaptive_queries = ia_enricher.generate_adaptive_queries("Initial data keywords: " + ", ".join(search_queries)) |
|
|
search_queries.extend(adaptive_queries) |
|
|
|
|
|
for query in list(set(search_queries)): |
|
|
logger.info(f"Recherche de datasets Kaggle pour: {query}") |
|
|
try: |
|
|
datasets = kaggle.api.dataset_list(search=query, max_results=5) |
|
|
for dataset in datasets: |
|
|
dataset_ref = dataset.ref |
|
|
if ia_enricher.available and st.session_state.enable_enrichment: |
|
|
is_relevant, _, _, relevance_score = ia_enricher.analyze_content_relevance(dataset.title + " " + dataset.subtitle) |
|
|
if not is_relevant or relevance_score < st.session_state.min_relevance: |
|
|
logger.info(f"Dataset non pertinent ({relevance_score}%): {dataset_ref}. Ignoré.") |
|
|
continue |
|
|
|
|
|
logger.info(f"Traitement du dataset: {dataset_ref}") |
|
|
download_dir = Path("data") / "security" / "kaggle" / dataset_ref.replace('/', '_') |
|
|
shutil.rmtree(download_dir, ignore_errors=True) |
|
|
download_dir.mkdir(parents=True, exist_ok=True) |
|
|
kaggle.api.dataset_download_files(dataset_ref, path=download_dir, unzip=True) |
|
|
|
|
|
for file_path in download_dir.glob('*'): |
|
|
if file_path.is_file() and file_path.suffix.lower() in ['.json', '.csv', '.txt']: |
|
|
try: |
|
|
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: |
|
|
file_content = f.read()[:5000] |
|
|
is_relevant, signatures, security_tags, _ = ia_enricher.analyze_content_relevance(file_content) |
|
|
if is_relevant: |
|
|
save_qa_pair( |
|
|
question=f"Quelles informations de sécurité contient le fichier {file_path.name} du dataset '{dataset.title}'?", |
|
|
answer=file_content, category="security", subcategory="vulnerability", |
|
|
source=f"kaggle_{dataset_ref}", attack_signatures=signatures, tags=security_tags |
|
|
) |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors du traitement du fichier {file_path}: {str(e)}") |
|
|
time.sleep(random.uniform(2, 4)) |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors de la collecte des données Kaggle pour {query}: {str(e)}") |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur inattendue dans collect_kaggle_data: {str(e)}") |
|
|
logger.info("Collecte des données Kaggle terminée.") |
|
|
|
|
|
def collect_github_data(queries, num_pages, results_per_page): |
|
|
logger.info("Début de la collecte des données GitHub...") |
|
|
base_url = "https://api.github.com" |
|
|
headers = {"Accept": "application/vnd.github.v3+json"} |
|
|
|
|
|
github_token = os.getenv('GITHUB_API_TOKEN') |
|
|
if github_token: |
|
|
headers["Authorization"] = f"token {github_token}" |
|
|
else: |
|
|
logger.warning("Clé GitHub non configurée. La collecte GitHub est ignorée.") |
|
|
st.session_state.logs.append("ATTENTION: Clé GitHub non configurée. Collecte GitHub ignorée.") |
|
|
return |
|
|
|
|
|
search_queries = queries.split('\n') if queries else ["topic:devsecops", "topic:security"] |
|
|
|
|
|
for query in search_queries: |
|
|
logger.info(f"Recherche de repositories pour: '{query}' sur {num_pages} page(s)") |
|
|
|
|
|
for page_number in range(1, num_pages + 1): |
|
|
try: |
|
|
logger.info(f"Consultation de la page {page_number}...") |
|
|
st.session_state.logs.append(f"GitHub: page {page_number} pour '{query}'") |
|
|
search_url = f"{base_url}/search/repositories" |
|
|
|
|
|
params = { |
|
|
"q": query, |
|
|
"sort": "stars", |
|
|
"per_page": results_per_page, |
|
|
"page": page_number |
|
|
} |
|
|
|
|
|
response = make_request(search_url, headers=headers, params=params) |
|
|
if not response: |
|
|
break |
|
|
|
|
|
data = response.json() |
|
|
items = data.get("items", []) |
|
|
|
|
|
if not items: |
|
|
logger.info(f"Fin des résultats pour cette requête (page {page_number}).") |
|
|
break |
|
|
|
|
|
for repo in items: |
|
|
repo_name = repo["full_name"].replace("/", "_") |
|
|
logger.info(f"Traitement du repository: {repo['full_name']}") |
|
|
|
|
|
issues_url = f"{base_url}/repos/{repo['full_name']}/issues" |
|
|
issues_params = {"state": "closed", "labels": "security,bug,vulnerability", "per_page": 10} |
|
|
issues_response = make_request(issues_url, headers=headers, params=issues_params) |
|
|
|
|
|
if issues_response: |
|
|
issues_data = issues_response.json() |
|
|
for issue in issues_data: |
|
|
if "pull_request" in issue: continue |
|
|
question = issue.get("title", "") |
|
|
body = clean_html(issue.get("body", "")) |
|
|
if not question or not body or len(body) < 50: continue |
|
|
|
|
|
comments_url = issue.get("comments_url") |
|
|
comments_response = make_request(comments_url, headers=headers) |
|
|
answer_parts = [] |
|
|
if comments_response: |
|
|
comments_data = comments_response.json() |
|
|
for comment in comments_data: |
|
|
comment_body = clean_html(comment.get("body", "")) |
|
|
if comment_body: answer_parts.append(comment_body) |
|
|
|
|
|
if answer_parts: |
|
|
answer = "\n\n".join(answer_parts) |
|
|
save_qa_pair( |
|
|
question=f"{question}: {body}", answer=answer, category="devsecops", |
|
|
subcategory="github", source=f"github_{repo_name}" |
|
|
) |
|
|
time.sleep(random.uniform(1, 3)) |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors de la collecte GitHub pour la page {page_number}: {str(e)}") |
|
|
st.session_state.logs.append(f"Erreur GitHub: {str(e)}") |
|
|
logger.info("Collecte des données GitHub terminée.") |
|
|
|
|
|
def collect_huggingface_data(queries, num_pages, results_per_page): |
|
|
logger.info("Début de la collecte des données Hugging Face...") |
|
|
base_url = "https://huggingface.co/api" |
|
|
headers = {"Accept": "application/json"} |
|
|
|
|
|
hf_token = os.getenv('HUGGINGFACE_API_TOKEN') |
|
|
if hf_token: |
|
|
headers["Authorization"] = f"Bearer {hf_token}" |
|
|
else: |
|
|
logger.warning("Clé Hugging Face non configurée. La collecte Hugging Face est ignorée.") |
|
|
st.session_state.logs.append("ATTENTION: Clé Hugging Face non configurée. Collecte Hugging Face ignorée.") |
|
|
return |
|
|
|
|
|
search_queries = queries.split('\n') if queries else ["security", "devsecops"] |
|
|
for query in search_queries: |
|
|
logger.info(f"Recherche de datasets pour: {query}") |
|
|
|
|
|
for page_number in range(num_pages): |
|
|
try: |
|
|
offset = page_number * results_per_page |
|
|
search_url = f"{base_url}/datasets" |
|
|
params = {"search": query, "limit": results_per_page, "offset": offset} |
|
|
|
|
|
response = make_request(search_url, headers=headers, params=params) |
|
|
if not response: continue |
|
|
|
|
|
data = response.json() |
|
|
if not data: |
|
|
logger.info(f"Fin des résultats pour la requête '{query}'.") |
|
|
break |
|
|
|
|
|
for dataset in data: |
|
|
dataset_id = dataset["id"].replace("/", "_") |
|
|
logger.info(f"Traitement du dataset: {dataset['id']}") |
|
|
dataset_url = f"{base_url}/datasets/{dataset['id']}" |
|
|
dataset_response = make_request(dataset_url, headers=headers) |
|
|
|
|
|
if dataset_response: |
|
|
dataset_data = dataset_response.json() |
|
|
description = clean_html(dataset_data.get("description", "")) |
|
|
if not description or len(description) < 100: continue |
|
|
tags = dataset_data.get("tags", []) |
|
|
tags_text = ", ".join(tags) if tags else "No tags" |
|
|
answer = f"Dataset: {dataset_data.get('id', '')}\nDownloads: {dataset_data.get('downloads', 0)}\nTags: {tags_text}\n\n{description}" |
|
|
|
|
|
save_qa_pair( |
|
|
question=f"What is the {dataset_data.get('id', '')} dataset about?", answer=answer, |
|
|
category="security", subcategory="dataset", source=f"huggingface_{dataset_id}", tags=tags |
|
|
) |
|
|
time.sleep(random.uniform(1, 3)) |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors de la collecte Hugging Face: {str(e)}") |
|
|
st.session_state.logs.append(f"Erreur Hugging Face: {str(e)}") |
|
|
logger.info("Collecte des données Hugging Face terminée.") |
|
|
|
|
|
def collect_nvd_data(num_pages, results_per_page): |
|
|
logger.info("Début de la collecte des données NVD...") |
|
|
base_url = "https://services.nvd.nist.gov/rest/json/cves/2.0" |
|
|
headers = {"Accept": "application/json"} |
|
|
|
|
|
nvd_key = os.getenv('NVD_API_KEY') |
|
|
if nvd_key: |
|
|
headers["apiKey"] = nvd_key |
|
|
else: |
|
|
logger.warning("Clé NVD non configurée. La collecte NVD est ignorée.") |
|
|
st.session_state.logs.append("ATTENTION: Clé NVD non configurée. Collecte NVD ignorée.") |
|
|
return |
|
|
|
|
|
for page in range(num_pages): |
|
|
try: |
|
|
start_index = page * results_per_page |
|
|
logger.info(f"Consultation de la page NVD, index de départ: {start_index}") |
|
|
st.session_state.logs.append(f"NVD: page {page + 1}") |
|
|
params = {"resultsPerPage": results_per_page, "startIndex": start_index} |
|
|
response = make_request(base_url, headers=headers, params=params) |
|
|
|
|
|
if not response: |
|
|
logger.warning("Impossible de récupérer les données du NVD. Arrêt de la collecte NVD.") |
|
|
break |
|
|
|
|
|
data = response.json() |
|
|
vulnerabilities = data.get("vulnerabilities", []) |
|
|
if not vulnerabilities: |
|
|
logger.info("Fin des résultats pour la collecte NVD.") |
|
|
break |
|
|
|
|
|
logger.info(f"Traitement de {len(vulnerabilities)} vulnérabilités...") |
|
|
|
|
|
for vuln in vulnerabilities: |
|
|
cve_data = vuln.get("cve", {}) |
|
|
cve_id = cve_data.get("id", "") |
|
|
descriptions = cve_data.get("descriptions", []) |
|
|
description = next((desc.get("value", "") for desc in descriptions if desc.get("lang") == "en"), "") |
|
|
if not description or len(description) < 50: continue |
|
|
|
|
|
cvss_v3 = cve_data.get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {}) |
|
|
severity = cvss_v3.get("baseSeverity", "UNKNOWN") |
|
|
score = cvss_v3.get("baseScore", 0) |
|
|
references = [ref.get("url", "") for ref in cve_data.get("references", [])] |
|
|
|
|
|
answer = f"CVE ID: {cve_id}\nSeverity: {severity}\nCVSS Score: {score}\nReferences: {', '.join(references[:5])}\n\nDescription: {description}" |
|
|
|
|
|
save_qa_pair( |
|
|
question=f"What is the vulnerability {cve_id}?", answer=answer, |
|
|
category="security", subcategory="vulnerability", source=f"nvd_{cve_id}" |
|
|
) |
|
|
time.sleep(random.uniform(1, 3)) |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors de la collecte NVD: {str(e)}") |
|
|
st.session_state.logs.append(f"Erreur NVD: {str(e)}") |
|
|
logger.info("Collecte des données NVD terminée.") |
|
|
|
|
|
def collect_stack_exchange_data(queries, num_pages, results_per_page): |
|
|
logger.info("Début de la collecte des données Stack Exchange...") |
|
|
base_url = "https://api.stackexchange.com/2.3" |
|
|
params_base = {"pagesize": results_per_page, "order": "desc", "sort": "votes", "filter": "withbody"} |
|
|
|
|
|
se_key = os.getenv('STACK_EXCHANGE_API_KEY') |
|
|
if se_key: |
|
|
params_base["key"] = se_key |
|
|
else: |
|
|
logger.warning("Clé Stack Exchange non configurée. La collecte est ignorée.") |
|
|
st.session_state.logs.append("ATTENTION: Clé Stack Exchange non configurée. Collecte Stack Exchange ignorée.") |
|
|
return |
|
|
|
|
|
sites = [ |
|
|
{"site": "security", "category": "security", "subcategory": "security"}, |
|
|
{"site": "devops", "category": "devsecops", "subcategory": "devops"} |
|
|
] |
|
|
|
|
|
tags_by_site = { |
|
|
"security": ["security", "vulnerability"], |
|
|
"devops": ["devops", "ci-cd"] |
|
|
} |
|
|
|
|
|
for site_config in sites: |
|
|
site = site_config["site"] |
|
|
category = site_config["category"] |
|
|
subcategory = site_config["subcategory"] |
|
|
logger.info(f"Collecte des données du site: {site}") |
|
|
tags = tags_by_site.get(site, []) + (queries.split('\n') if queries else []) |
|
|
|
|
|
for tag in list(set(tags)): |
|
|
logger.info(f"Recherche de questions avec le tag: '{tag}'") |
|
|
|
|
|
for page_number in range(1, num_pages + 1): |
|
|
try: |
|
|
questions_url = f"{base_url}/questions" |
|
|
params = {**params_base, "site": site, "tagged": tag, "page": page_number} |
|
|
|
|
|
response = make_request(questions_url, params=params) |
|
|
if not response: continue |
|
|
|
|
|
questions_data = response.json() |
|
|
items = questions_data.get("items", []) |
|
|
|
|
|
if not items: |
|
|
logger.info(f"Fin des résultats pour le tag '{tag}' à la page {page_number}.") |
|
|
break |
|
|
|
|
|
for question in items: |
|
|
question_id = question.get("question_id") |
|
|
title = question.get("title", "") |
|
|
body = clean_html(question.get("body", "")) |
|
|
if not body or len(body) < 50: continue |
|
|
|
|
|
answers_url = f"{base_url}/questions/{question_id}/answers" |
|
|
answers_params = {**params_base, "site": site} |
|
|
answers_response = make_request(answers_url, params=answers_params) |
|
|
answer_body = "" |
|
|
if answers_response and answers_response.json().get("items"): |
|
|
answer_body = clean_html(answers_response.json()["items"][0].get("body", "")) |
|
|
|
|
|
if answer_body: |
|
|
save_qa_pair( |
|
|
question=title, answer=answer_body, category=category, |
|
|
subcategory=subcategory, source=f"{site}_{question_id}", tags=question.get("tags", []) |
|
|
) |
|
|
time.sleep(random.uniform(1, 3)) |
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors de la collecte Stack Exchange: {str(e)}") |
|
|
st.session_state.logs.append(f"Erreur Stack Exchange: {str(e)}") |
|
|
logger.info("Collecte des données Stack Exchange terminée.") |
|
|
|
|
|
def collect_web_data(url): |
|
|
logger.info(f"Début de la collecte des données pour l'URL: {url}") |
|
|
|
|
|
try: |
|
|
st.session_state.logs.append(f"Début de la collecte pour {url}...") |
|
|
response = make_request(url) |
|
|
if not response or response.status_code != 200: |
|
|
logger.error(f"Impossible de récupérer le contenu de l'URL: {url}") |
|
|
st.session_state.logs.append(f"Erreur: Impossible de récupérer l'URL {url}") |
|
|
return |
|
|
|
|
|
soup = BeautifulSoup(response.text, 'html.parser') |
|
|
|
|
|
for script in soup(["script", "style", "header", "footer", "nav"]): |
|
|
script.extract() |
|
|
|
|
|
raw_text = soup.get_text() |
|
|
clean_text = clean_html(raw_text) |
|
|
|
|
|
if not clean_text or len(clean_text) < 100: |
|
|
logger.warning(f"Contenu de l'URL trop court ou vide: {url}") |
|
|
st.session_state.logs.append("Avertissement: Contenu de l'URL trop court ou vide.") |
|
|
return |
|
|
|
|
|
is_relevant, signatures, security_tags, _ = ia_enricher.analyze_content_relevance(clean_text) |
|
|
|
|
|
if is_relevant: |
|
|
title = soup.title.string if soup.title else os.path.basename(url) |
|
|
question = f"What security information is on the page '{title}'?" |
|
|
answer = clean_text |
|
|
|
|
|
save_qa_pair( |
|
|
question=question, |
|
|
answer=answer, |
|
|
category="security", |
|
|
subcategory="web-scraping", |
|
|
source=f"web_{re.sub(r'[^a-zA-Z0-9]+', '', url)[:30]}", |
|
|
attack_signatures=signatures, |
|
|
tags=security_tags |
|
|
) |
|
|
else: |
|
|
logger.info(f"Contenu de l'URL non pertinent pour DevSecOps: {url}") |
|
|
st.session_state.logs.append(f"Contenu de l'URL non pertinent pour DevSecOps.") |
|
|
|
|
|
except Exception as e: |
|
|
logger.error(f"Erreur lors du scraping de l'URL {url}: {str(e)}") |
|
|
st.session_state.logs.append(f"Erreur lors du scraping de l'URL {url}") |
|
|
|
|
|
logger.info("Collecte des données web terminée.") |
|
|
|
|
|
def run_data_collection(sources, queries, web_url, num_pages, results_per_page): |
|
|
st.session_state.bot_status = "En cours d'exécution" |
|
|
st.session_state.logs = [] |
|
|
|
|
|
valid_keys = check_api_keys() |
|
|
|
|
|
progress_bar = st.progress(0) |
|
|
status_text = st.empty() |
|
|
log_container = st.empty() |
|
|
|
|
|
enabled_sources = [s for s, enabled in sources.items() if enabled] |
|
|
|
|
|
if "Web Scraping" in enabled_sources and web_url: |
|
|
enabled_sources.remove("Web Scraping") |
|
|
enabled_sources.insert(0, "Web Scraping") |
|
|
|
|
|
total_sources = len(enabled_sources) |
|
|
completed_sources = 0 |
|
|
|
|
|
for source_name in enabled_sources: |
|
|
status_text.text(f"Collecte des données de {source_name}...") |
|
|
log_container.text("Logs en temps réel:\n" + "\n".join(st.session_state.logs)) |
|
|
|
|
|
if source_name == "Kaggle": |
|
|
collect_kaggle_data(queries.get("Kaggle", "")) |
|
|
elif source_name == "GitHub": |
|
|
if 'GITHUB_API_TOKEN' in valid_keys: |
|
|
collect_github_data(queries.get("GitHub", ""), num_pages, results_per_page) |
|
|
else: |
|
|
logger.warning("Clé GitHub non définie. Saut de la collecte GitHub.") |
|
|
st.session_state.logs.append("ATTENTION: Clé GitHub non définie. Collecte GitHub ignorée.") |
|
|
elif source_name == "Hugging Face": |
|
|
if 'HUGGINGFACE_API_TOKEN' in valid_keys: |
|
|
collect_huggingface_data(queries.get("Hugging Face", ""), num_pages, results_per_page) |
|
|
else: |
|
|
logger.warning("Clé Hugging Face non définie. Saut de la collecte Hugging Face.") |
|
|
st.session_state.logs.append("ATTENTION: Clé Hugging Face non définie. Collecte Hugging Face ignorée.") |
|
|
elif source_name == "NVD": |
|
|
if 'NVD_API_KEY' in valid_keys: |
|
|
collect_nvd_data(num_pages, results_per_page) |
|
|
else: |
|
|
logger.warning("Clé NVD non définie. Saut de la collecte NVD.") |
|
|
st.session_state.logs.append("ATTENTION: Clé NVD non définie. Collecte NVD ignorée.") |
|
|
elif source_name == "Stack Exchange": |
|
|
if 'STACK_EXCHANGE_API_KEY' in valid_keys: |
|
|
collect_stack_exchange_data(queries.get("Stack Exchange", ""), num_pages, results_per_page) |
|
|
else: |
|
|
logger.warning("Clé Stack Exchange non définie. Saut de la collecte Stack Exchange.") |
|
|
st.session_state.logs.append("ATTENTION: Clé Stack Exchange non définie. Collecte Stack Exchange ignorée.") |
|
|
elif source_name == "Web Scraping": |
|
|
if web_url: |
|
|
collect_web_data(web_url) |
|
|
else: |
|
|
st.session_state.logs.append("ATTENTION: URL de scraping non fournie. Collecte ignorée.") |
|
|
|
|
|
completed_sources += 1 |
|
|
progress_bar.progress(completed_sources / total_sources) |
|
|
log_container.text("Logs en temps réel:\n" + "\n".join(st.session_state.logs)) |
|
|
time.sleep(1) |
|
|
|
|
|
st.session_state.bot_status = "Arrêté" |
|
|
st.info("Collecte des données terminée!") |
|
|
progress_bar.empty() |
|
|
status_text.empty() |
|
|
log_container.text("Logs en temps réel:\n" + "\n".join(st.session_state.logs)) |
|
|
st.rerun() |
|
|
|
|
|
def main(): |
|
|
st.title("DevSecOps Data Bot") |
|
|
st.markdown(""" |
|
|
Ce bot est conçu pour collecter des données de diverses sources (GitHub, Kaggle, Hugging Face, NVD, Stack Exchange) |
|
|
afin de construire un jeu de données de questions/réponses DevSecOps. |
|
|
""") |
|
|
|
|
|
tabs = st.tabs(["Bot", "Statistiques & Données", "Configuration"]) |
|
|
|
|
|
with tabs[0]: |
|
|
st.header("État du bot") |
|
|
col1, col2, col3 = st.columns(3) |
|
|
with col1: |
|
|
st.metric("Statut", st.session_state.bot_status) |
|
|
with col2: |
|
|
st.metric("Paires Q/R", st.session_state.total_qa_pairs) |
|
|
with col3: |
|
|
st.metric("Statut du serveur LLM", st.session_state.server_status) |
|
|
|
|
|
st.markdown("---") |
|
|
|
|
|
st.header("Paramètres de la collecte") |
|
|
col1, col2 = st.columns(2) |
|
|
num_pages = col1.slider( |
|
|
"Nombre de pages à consulter par source", |
|
|
min_value=1, max_value=20, value=5, |
|
|
help="Le bot consultera jusqu'à X pages de résultats pour chaque source." |
|
|
) |
|
|
results_per_page = col2.slider( |
|
|
"Nombre de résultats par page", |
|
|
min_value=10, max_value=100, value=20, |
|
|
help="Le bot demandera jusqu'à Y résultats pour chaque page consultée." |
|
|
) |
|
|
|
|
|
st.header("Lancer la collecte") |
|
|
|
|
|
st.subheader("Sources de données") |
|
|
sources_columns = st.columns(6) |
|
|
sources = { |
|
|
"GitHub": sources_columns[0].checkbox("GitHub", value=True), |
|
|
"Hugging Face": sources_columns[1].checkbox("Hugging Face", value=True), |
|
|
"NVD": sources_columns[2].checkbox("NVD", value=True), |
|
|
"Stack Exchange": sources_columns[3].checkbox("Stack Exchange", value=True), |
|
|
"Kaggle": sources_columns[4].checkbox("Kaggle", value=True), |
|
|
"Web Scraping": sources_columns[5].checkbox("Web Scraping", value=True) |
|
|
} |
|
|
|
|
|
web_url = st.text_input("URL à scraper (optionnel)", help="Entrez une URL pour extraire les données de sécurité.") |
|
|
|
|
|
st.info("En cliquant sur 'Lancer la collecte', vous reconnaissez que vous disposez des droits légaux de scraping et de manipulation des données du site fourni, et nous déclinons toute responsabilité.") |
|
|
|
|
|
st.subheader("Requêtes de recherche") |
|
|
queries = {} |
|
|
queries["GitHub"] = st.text_area("Requêtes GitHub (une par ligne)", "topic:devsecops\ntopic:security\nvulnerability") |
|
|
queries["Kaggle"] = st.text_area("Requêtes Kaggle (une par ligne)", "cybersecurity\nvulnerability dataset\npenetration testing") |
|
|
queries["Hugging Face"] = st.text_area("Requêtes Hugging Face (une par ligne)", "security dataset\nvulnerability\nlanguage model security") |
|
|
queries["Stack Exchange"] = st.text_area("Tags Stack Exchange (un par ligne)", "devsecops\nsecurity\nvulnerability") |
|
|
queries["NVD"] = "" |
|
|
|
|
|
st.markdown("---") |
|
|
|
|
|
if st.session_state.bot_status == "Arrêté": |
|
|
if st.button("Lancer la collecte", use_container_width=True, type="primary"): |
|
|
st.session_state.logs = [] |
|
|
st.session_state.qa_data = [] |
|
|
st.session_state.total_qa_pairs = 0 |
|
|
run_data_collection(sources, queries, web_url, num_pages, results_per_page) |
|
|
else: |
|
|
st.warning("La collecte est en cours. Veuillez attendre qu'elle se termine.") |
|
|
if st.button("Forcer l'arrêt", use_container_width=True, type="secondary"): |
|
|
st.session_state.bot_status = "Arrêté" |
|
|
st.info("La collecte a été arrêtée manuellement.") |
|
|
|
|
|
st.markdown("---") |
|
|
st.subheader("Logs d'exécution") |
|
|
log_container = st.container(border=True) |
|
|
with log_container: |
|
|
for log in st.session_state.logs: |
|
|
st.text(log) |
|
|
|
|
|
with tabs[1]: |
|
|
st.header("Statistiques") |
|
|
if st.session_state.qa_data: |
|
|
df = pd.DataFrame(st.session_state.qa_data) |
|
|
|
|
|
st.subheader("Aperçu des données") |
|
|
st.dataframe(df, use_container_width=True) |
|
|
|
|
|
st.subheader("Répartition par source") |
|
|
source_counts = df['source'].apply(lambda x: x.split('_')[0]).value_counts().reset_index() |
|
|
source_counts.columns = ['Source', 'Nombre'] |
|
|
fig_source = px.bar(source_counts, x='Source', y='Nombre', title="Nombre de paires Q/R par source") |
|
|
st.plotly_chart(fig_source, use_container_width=True) |
|
|
|
|
|
st.subheader("Répartition par catégorie") |
|
|
category_counts = df['category'].value_counts().reset_index() |
|
|
category_counts.columns = ['Catégorie', 'Nombre'] |
|
|
fig_cat = px.pie(category_counts, names='Catégorie', values='Nombre', title="Répartition par catégorie") |
|
|
st.plotly_chart(fig_cat, use_container_width=True) |
|
|
|
|
|
st.subheader("Téléchargement des données") |
|
|
col1, col2 = st.columns(2) |
|
|
with col1: |
|
|
json_data = json.dumps(st.session_state.qa_data, indent=2) |
|
|
st.download_button( |
|
|
label="Télécharger JSON", |
|
|
data=json_data, |
|
|
file_name=f"devsecops_data_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json", |
|
|
mime="application/json", |
|
|
use_container_width=True |
|
|
) |
|
|
with col2: |
|
|
csv_data = df.to_csv(index=False) |
|
|
st.download_button( |
|
|
label="Télécharger CSV", |
|
|
data=csv_data, |
|
|
file_name=f"devsecops_data_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", |
|
|
mime="text/csv", |
|
|
use_container_width=True |
|
|
) |
|
|
else: |
|
|
st.info("Aucune donnée à afficher. Lancez d'abord la collecte de données.") |
|
|
|
|
|
with tabs[2]: |
|
|
st.header("Configuration Avancée") |
|
|
st.subheader("Paramètres du serveur LLM") |
|
|
|
|
|
llm_col1, llm_col2 = st.columns(2) |
|
|
with llm_col1: |
|
|
if st.button("Démarrer le serveur LLM", type="primary", use_container_width=True): |
|
|
start_llm_server() |
|
|
st.rerun() |
|
|
if st.button("Vérifier le statut du serveur", use_container_width=True): |
|
|
check_server_status() |
|
|
st.rerun() |
|
|
with llm_col2: |
|
|
if st.button("Arrêter le serveur LLM", type="secondary", use_container_width=True): |
|
|
stop_llm_server() |
|
|
st.rerun() |
|
|
|
|
|
st.markdown("---") |
|
|
|
|
|
st.subheader("Paramètres d'enrichissement IA") |
|
|
st.session_state.enable_enrichment = st.checkbox("Activer l'enrichissement IA", value=st.session_state.enable_enrichment, help="Utilise le LLM pour améliorer les paires Q/R.") |
|
|
st.session_state.min_relevance = st.slider("Score de pertinence minimum", 0, 100, st.session_state.min_relevance, help="Les contenus en dessous de ce score ne seront pas traités.") |
|
|
st.session_state.temperature = st.slider("Température de l'IA", 0.0, 1.0, st.session_state.temperature, help="Contrôle la créativité de l'IA. 0.0 = plus déterministe, 1.0 = plus créatif.") |
|
|
st.session_state.n_predict = st.slider("Nombre de tokens", 128, 1024, st.session_state.n_predict, help="Nombre maximum de tokens à générer par l'IA.") |
|
|
|
|
|
if __name__ == "__main__": |
|
|
main() |
