Commit
·
8b7e088
1
Parent(s):
b9b7087
fix
Browse files- app.py +12 -15
- cognitive_mapping_probe/concepts.py +4 -15
app.py
CHANGED
|
@@ -3,7 +3,7 @@ import pandas as pd
|
|
| 3 |
import traceback
|
| 4 |
import gc
|
| 5 |
import torch
|
| 6 |
-
import json
|
| 7 |
|
| 8 |
from cognitive_mapping_probe.orchestrator_seismograph import run_seismic_analysis
|
| 9 |
from cognitive_mapping_probe.auto_experiment import run_auto_suite, get_curated_experiments
|
|
@@ -21,19 +21,18 @@ def cleanup_memory():
|
|
| 21 |
dbg("Memory cleanup complete.")
|
| 22 |
|
| 23 |
def run_single_analysis_display(*args, progress=gr.Progress(track_tqdm=True)):
|
| 24 |
-
"""Wrapper für ein einzelnes manuelles Experiment."""
|
| 25 |
try:
|
| 26 |
results = run_seismic_analysis(*args, progress_callback=progress)
|
| 27 |
stats, deltas = results.get("stats", {}), results.get("state_deltas", [])
|
| 28 |
df = pd.DataFrame({"Internal Step": range(len(deltas)), "State Change (Delta)": deltas})
|
| 29 |
stats_md = f"### Statistical Signature\n- **Mean Delta:** {stats.get('mean_delta', 0):.4f}\n- **Std Dev Delta:** {stats.get('std_delta', 0):.4f}\n- **Max Delta:** {stats.get('max_delta', 0):.4f}\n"
|
| 30 |
-
|
| 31 |
-
# Stelle sicher, dass das Ergebnis für die JSON-Komponente serialisierbar ist
|
| 32 |
serializable_results = json.dumps(results, indent=2, default=str)
|
| 33 |
-
|
| 34 |
return f"{results.get('verdict', 'Error')}\n\n{stats_md}", df, serializable_results
|
| 35 |
except Exception:
|
| 36 |
-
|
|
|
|
|
|
|
| 37 |
finally:
|
| 38 |
cleanup_memory()
|
| 39 |
|
|
@@ -44,22 +43,20 @@ PLOT_PARAMS = {
|
|
| 44 |
}
|
| 45 |
|
| 46 |
def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
|
| 47 |
-
"""
|
|
|
|
|
|
|
| 48 |
try:
|
| 49 |
summary_df, plot_df, all_results = run_auto_suite(model_id, int(num_steps), int(seed), experiment_name, progress)
|
| 50 |
-
|
| 51 |
-
dbg("Plot DataFrame Head for Auto-Suite:\n", plot_df.head())
|
| 52 |
-
|
| 53 |
new_plot = gr.LinePlot(value=plot_df, **PLOT_PARAMS)
|
| 54 |
-
|
| 55 |
-
# KORREKTUR: Serialisiere das Ergebnis-Dictionary explizit zu einem JSON-String,
|
| 56 |
-
# bevor es an die `gr.JSON`-Komponente zurückgegeben wird.
|
| 57 |
serializable_results = json.dumps(all_results, indent=2, default=str)
|
| 58 |
-
|
| 59 |
return summary_df, new_plot, serializable_results
|
| 60 |
except Exception:
|
|
|
|
|
|
|
| 61 |
empty_plot = gr.LinePlot(value=pd.DataFrame(), **PLOT_PARAMS)
|
| 62 |
-
|
|
|
|
| 63 |
finally:
|
| 64 |
cleanup_memory()
|
| 65 |
|
|
|
|
| 3 |
import traceback
|
| 4 |
import gc
|
| 5 |
import torch
|
| 6 |
+
import json
|
| 7 |
|
| 8 |
from cognitive_mapping_probe.orchestrator_seismograph import run_seismic_analysis
|
| 9 |
from cognitive_mapping_probe.auto_experiment import run_auto_suite, get_curated_experiments
|
|
|
|
| 21 |
dbg("Memory cleanup complete.")
|
| 22 |
|
| 23 |
def run_single_analysis_display(*args, progress=gr.Progress(track_tqdm=True)):
|
| 24 |
+
"""Wrapper für ein einzelnes manuelles Experiment mit robuster Fehlerbehandlung."""
|
| 25 |
try:
|
| 26 |
results = run_seismic_analysis(*args, progress_callback=progress)
|
| 27 |
stats, deltas = results.get("stats", {}), results.get("state_deltas", [])
|
| 28 |
df = pd.DataFrame({"Internal Step": range(len(deltas)), "State Change (Delta)": deltas})
|
| 29 |
stats_md = f"### Statistical Signature\n- **Mean Delta:** {stats.get('mean_delta', 0):.4f}\n- **Std Dev Delta:** {stats.get('std_delta', 0):.4f}\n- **Max Delta:** {stats.get('max_delta', 0):.4f}\n"
|
|
|
|
|
|
|
| 30 |
serializable_results = json.dumps(results, indent=2, default=str)
|
|
|
|
| 31 |
return f"{results.get('verdict', 'Error')}\n\n{stats_md}", df, serializable_results
|
| 32 |
except Exception:
|
| 33 |
+
# Im Fehlerfall, gib für jede Komponente einen leeren, aber typ-korrekten Wert zurück.
|
| 34 |
+
error_message = f"### ❌ Analysis Failed\n```\n{traceback.format_exc()}\n```"
|
| 35 |
+
return error_message, pd.DataFrame(), "{}"
|
| 36 |
finally:
|
| 37 |
cleanup_memory()
|
| 38 |
|
|
|
|
| 43 |
}
|
| 44 |
|
| 45 |
def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
|
| 46 |
+
"""
|
| 47 |
+
Wrapper für die automatisierte Experiment-Suite mit robuster Fehlerbehandlung.
|
| 48 |
+
"""
|
| 49 |
try:
|
| 50 |
summary_df, plot_df, all_results = run_auto_suite(model_id, int(num_steps), int(seed), experiment_name, progress)
|
|
|
|
|
|
|
|
|
|
| 51 |
new_plot = gr.LinePlot(value=plot_df, **PLOT_PARAMS)
|
|
|
|
|
|
|
|
|
|
| 52 |
serializable_results = json.dumps(all_results, indent=2, default=str)
|
|
|
|
| 53 |
return summary_df, new_plot, serializable_results
|
| 54 |
except Exception:
|
| 55 |
+
# KORREKTUR: Gib für jede Komponente einen typ-korrekten, leeren Wert zurück.
|
| 56 |
+
# Insbesondere für die JSON-Komponente einen leeren JSON-String.
|
| 57 |
empty_plot = gr.LinePlot(value=pd.DataFrame(), **PLOT_PARAMS)
|
| 58 |
+
error_string_for_json = json.dumps({"error": traceback.format_exc()}, indent=2)
|
| 59 |
+
return pd.DataFrame([{"Error": "Experiment failed. See Raw JSON."}]), empty_plot, error_string_for_json
|
| 60 |
finally:
|
| 61 |
cleanup_memory()
|
| 62 |
|
cognitive_mapping_probe/concepts.py
CHANGED
|
@@ -5,19 +5,17 @@ from tqdm import tqdm
|
|
| 5 |
from .llm_iface import LLM
|
| 6 |
from .utils import dbg
|
| 7 |
|
| 8 |
-
# Eine Liste neutraler Wörter zur Berechnung der Baseline-Aktivierung.
|
| 9 |
BASELINE_WORDS = [
|
| 10 |
"thing", "place", "idea", "person", "object", "time", "way", "day", "man", "world",
|
| 11 |
"life", "hand", "part", "child", "eye", "woman", "fact", "group", "case", "point"
|
| 12 |
]
|
| 13 |
|
| 14 |
-
# REFAKTORISIERUNG: Diese Funktion wird auf Modulebene verschoben, um sie testbar zu machen.
|
| 15 |
-
# Sie ist nun keine lokale Funktion innerhalb von `get_concept_vector` mehr.
|
| 16 |
@torch.no_grad()
|
| 17 |
def _get_last_token_hidden_state(llm: LLM, prompt: str) -> torch.Tensor:
|
| 18 |
"""Hilfsfunktion, um den Hidden State des letzten Tokens eines Prompts zu erhalten."""
|
| 19 |
inputs = llm.tokenizer(prompt, return_tensors="pt").to(llm.model.device)
|
| 20 |
with torch.no_grad():
|
|
|
|
| 21 |
outputs = llm.model(**inputs, output_hidden_states=True)
|
| 22 |
last_hidden_state = outputs.hidden_states[-1][0, -1, :].cpu()
|
| 23 |
assert last_hidden_state.shape == (llm.config.hidden_size,), \
|
|
@@ -26,28 +24,19 @@ def _get_last_token_hidden_state(llm: LLM, prompt: str) -> torch.Tensor:
|
|
| 26 |
|
| 27 |
@torch.no_grad()
|
| 28 |
def get_concept_vector(llm: LLM, concept: str, baseline_words: List[str] = BASELINE_WORDS) -> torch.Tensor:
|
| 29 |
-
"""
|
| 30 |
-
Extrahiert einen Konzeptvektor mittels der kontrastiven Methode.
|
| 31 |
-
"""
|
| 32 |
dbg(f"Extracting contrastive concept vector for '{concept}'...")
|
| 33 |
-
|
| 34 |
prompt_template = "Here is a sentence about the concept of {}."
|
| 35 |
-
|
| 36 |
dbg(f" - Getting activation for '{concept}'")
|
| 37 |
target_hs = _get_last_token_hidden_state(llm, prompt_template.format(concept))
|
| 38 |
-
|
| 39 |
baseline_hss = []
|
| 40 |
for word in tqdm(baseline_words, desc=f" - Calculating baseline for '{concept}'", leave=False, bar_format="{l_bar}{bar:10}{r_bar}"):
|
| 41 |
baseline_hss.append(_get_last_token_hidden_state(llm, prompt_template.format(word)))
|
| 42 |
-
|
| 43 |
-
assert all(hs.shape == target_hs.shape for hs in baseline_hss), "Shape mismatch in baseline hidden states."
|
| 44 |
-
|
| 45 |
mean_baseline_hs = torch.stack(baseline_hss).mean(dim=0)
|
| 46 |
dbg(f" - Mean baseline vector computed with norm {torch.norm(mean_baseline_hs).item():.2f}")
|
| 47 |
-
|
| 48 |
concept_vector = target_hs - mean_baseline_hs
|
| 49 |
norm = torch.norm(concept_vector).item()
|
| 50 |
dbg(f"Concept vector for '{concept}' extracted with norm {norm:.2f}.")
|
| 51 |
-
|
| 52 |
-
assert torch.isfinite(concept_vector).all(), "Concept vector contains NaN or Inf values."
|
| 53 |
return concept_vector
|
|
|
|
| 5 |
from .llm_iface import LLM
|
| 6 |
from .utils import dbg
|
| 7 |
|
|
|
|
| 8 |
BASELINE_WORDS = [
|
| 9 |
"thing", "place", "idea", "person", "object", "time", "way", "day", "man", "world",
|
| 10 |
"life", "hand", "part", "child", "eye", "woman", "fact", "group", "case", "point"
|
| 11 |
]
|
| 12 |
|
|
|
|
|
|
|
| 13 |
@torch.no_grad()
|
| 14 |
def _get_last_token_hidden_state(llm: LLM, prompt: str) -> torch.Tensor:
|
| 15 |
"""Hilfsfunktion, um den Hidden State des letzten Tokens eines Prompts zu erhalten."""
|
| 16 |
inputs = llm.tokenizer(prompt, return_tensors="pt").to(llm.model.device)
|
| 17 |
with torch.no_grad():
|
| 18 |
+
# WAHRSCHEINLICHE FEHLERQUELLE: Sicherstellen, dass hier wirklich `llm` steht.
|
| 19 |
outputs = llm.model(**inputs, output_hidden_states=True)
|
| 20 |
last_hidden_state = outputs.hidden_states[-1][0, -1, :].cpu()
|
| 21 |
assert last_hidden_state.shape == (llm.config.hidden_size,), \
|
|
|
|
| 24 |
|
| 25 |
@torch.no_grad()
|
| 26 |
def get_concept_vector(llm: LLM, concept: str, baseline_words: List[str] = BASELINE_WORDS) -> torch.Tensor:
|
| 27 |
+
"""Extrahiert einen Konzeptvektor mittels der kontrastiven Methode."""
|
|
|
|
|
|
|
| 28 |
dbg(f"Extracting contrastive concept vector for '{concept}'...")
|
|
|
|
| 29 |
prompt_template = "Here is a sentence about the concept of {}."
|
|
|
|
| 30 |
dbg(f" - Getting activation for '{concept}'")
|
| 31 |
target_hs = _get_last_token_hidden_state(llm, prompt_template.format(concept))
|
|
|
|
| 32 |
baseline_hss = []
|
| 33 |
for word in tqdm(baseline_words, desc=f" - Calculating baseline for '{concept}'", leave=False, bar_format="{l_bar}{bar:10}{r_bar}"):
|
| 34 |
baseline_hss.append(_get_last_token_hidden_state(llm, prompt_template.format(word)))
|
| 35 |
+
assert all(hs.shape == target_hs.shape for hs in baseline_hss)
|
|
|
|
|
|
|
| 36 |
mean_baseline_hs = torch.stack(baseline_hss).mean(dim=0)
|
| 37 |
dbg(f" - Mean baseline vector computed with norm {torch.norm(mean_baseline_hs).item():.2f}")
|
|
|
|
| 38 |
concept_vector = target_hs - mean_baseline_hs
|
| 39 |
norm = torch.norm(concept_vector).item()
|
| 40 |
dbg(f"Concept vector for '{concept}' extracted with norm {norm:.2f}.")
|
| 41 |
+
assert torch.isfinite(concept_vector).all()
|
|
|
|
| 42 |
return concept_vector
|