neuralworm commited on
Commit
8b7e088
·
1 Parent(s): b9b7087
Files changed (2) hide show
  1. app.py +12 -15
  2. cognitive_mapping_probe/concepts.py +4 -15
app.py CHANGED
@@ -3,7 +3,7 @@ import pandas as pd
3
  import traceback
4
  import gc
5
  import torch
6
- import json # Importiere das json-Modul
7
 
8
  from cognitive_mapping_probe.orchestrator_seismograph import run_seismic_analysis
9
  from cognitive_mapping_probe.auto_experiment import run_auto_suite, get_curated_experiments
@@ -21,19 +21,18 @@ def cleanup_memory():
21
  dbg("Memory cleanup complete.")
22
 
23
  def run_single_analysis_display(*args, progress=gr.Progress(track_tqdm=True)):
24
- """Wrapper für ein einzelnes manuelles Experiment."""
25
  try:
26
  results = run_seismic_analysis(*args, progress_callback=progress)
27
  stats, deltas = results.get("stats", {}), results.get("state_deltas", [])
28
  df = pd.DataFrame({"Internal Step": range(len(deltas)), "State Change (Delta)": deltas})
29
  stats_md = f"### Statistical Signature\n- **Mean Delta:** {stats.get('mean_delta', 0):.4f}\n- **Std Dev Delta:** {stats.get('std_delta', 0):.4f}\n- **Max Delta:** {stats.get('max_delta', 0):.4f}\n"
30
-
31
- # Stelle sicher, dass das Ergebnis für die JSON-Komponente serialisierbar ist
32
  serializable_results = json.dumps(results, indent=2, default=str)
33
-
34
  return f"{results.get('verdict', 'Error')}\n\n{stats_md}", df, serializable_results
35
  except Exception:
36
- return f"### Analysis Failed\n```\n{traceback.format_exc()}\n```", pd.DataFrame(), "{}"
 
 
37
  finally:
38
  cleanup_memory()
39
 
@@ -44,22 +43,20 @@ PLOT_PARAMS = {
44
  }
45
 
46
  def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
47
- """Wrapper für die automatisierte Experiment-Suite mit korrekter JSON-Serialisierung."""
 
 
48
  try:
49
  summary_df, plot_df, all_results = run_auto_suite(model_id, int(num_steps), int(seed), experiment_name, progress)
50
-
51
- dbg("Plot DataFrame Head for Auto-Suite:\n", plot_df.head())
52
-
53
  new_plot = gr.LinePlot(value=plot_df, **PLOT_PARAMS)
54
-
55
- # KORREKTUR: Serialisiere das Ergebnis-Dictionary explizit zu einem JSON-String,
56
- # bevor es an die `gr.JSON`-Komponente zurückgegeben wird.
57
  serializable_results = json.dumps(all_results, indent=2, default=str)
58
-
59
  return summary_df, new_plot, serializable_results
60
  except Exception:
 
 
61
  empty_plot = gr.LinePlot(value=pd.DataFrame(), **PLOT_PARAMS)
62
- return pd.DataFrame(), empty_plot, f"### ❌ Auto-Experiment Failed\n```\n{traceback.format_exc()}\n```"
 
63
  finally:
64
  cleanup_memory()
65
 
 
3
  import traceback
4
  import gc
5
  import torch
6
+ import json
7
 
8
  from cognitive_mapping_probe.orchestrator_seismograph import run_seismic_analysis
9
  from cognitive_mapping_probe.auto_experiment import run_auto_suite, get_curated_experiments
 
21
  dbg("Memory cleanup complete.")
22
 
23
  def run_single_analysis_display(*args, progress=gr.Progress(track_tqdm=True)):
24
+ """Wrapper für ein einzelnes manuelles Experiment mit robuster Fehlerbehandlung."""
25
  try:
26
  results = run_seismic_analysis(*args, progress_callback=progress)
27
  stats, deltas = results.get("stats", {}), results.get("state_deltas", [])
28
  df = pd.DataFrame({"Internal Step": range(len(deltas)), "State Change (Delta)": deltas})
29
  stats_md = f"### Statistical Signature\n- **Mean Delta:** {stats.get('mean_delta', 0):.4f}\n- **Std Dev Delta:** {stats.get('std_delta', 0):.4f}\n- **Max Delta:** {stats.get('max_delta', 0):.4f}\n"
 
 
30
  serializable_results = json.dumps(results, indent=2, default=str)
 
31
  return f"{results.get('verdict', 'Error')}\n\n{stats_md}", df, serializable_results
32
  except Exception:
33
+ # Im Fehlerfall, gib für jede Komponente einen leeren, aber typ-korrekten Wert zurück.
34
+ error_message = f"### ❌ Analysis Failed\n```\n{traceback.format_exc()}\n```"
35
+ return error_message, pd.DataFrame(), "{}"
36
  finally:
37
  cleanup_memory()
38
 
 
43
  }
44
 
45
  def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
46
+ """
47
+ Wrapper für die automatisierte Experiment-Suite mit robuster Fehlerbehandlung.
48
+ """
49
  try:
50
  summary_df, plot_df, all_results = run_auto_suite(model_id, int(num_steps), int(seed), experiment_name, progress)
 
 
 
51
  new_plot = gr.LinePlot(value=plot_df, **PLOT_PARAMS)
 
 
 
52
  serializable_results = json.dumps(all_results, indent=2, default=str)
 
53
  return summary_df, new_plot, serializable_results
54
  except Exception:
55
+ # KORREKTUR: Gib für jede Komponente einen typ-korrekten, leeren Wert zurück.
56
+ # Insbesondere für die JSON-Komponente einen leeren JSON-String.
57
  empty_plot = gr.LinePlot(value=pd.DataFrame(), **PLOT_PARAMS)
58
+ error_string_for_json = json.dumps({"error": traceback.format_exc()}, indent=2)
59
+ return pd.DataFrame([{"Error": "Experiment failed. See Raw JSON."}]), empty_plot, error_string_for_json
60
  finally:
61
  cleanup_memory()
62
 
cognitive_mapping_probe/concepts.py CHANGED
@@ -5,19 +5,17 @@ from tqdm import tqdm
5
  from .llm_iface import LLM
6
  from .utils import dbg
7
 
8
- # Eine Liste neutraler Wörter zur Berechnung der Baseline-Aktivierung.
9
  BASELINE_WORDS = [
10
  "thing", "place", "idea", "person", "object", "time", "way", "day", "man", "world",
11
  "life", "hand", "part", "child", "eye", "woman", "fact", "group", "case", "point"
12
  ]
13
 
14
- # REFAKTORISIERUNG: Diese Funktion wird auf Modulebene verschoben, um sie testbar zu machen.
15
- # Sie ist nun keine lokale Funktion innerhalb von `get_concept_vector` mehr.
16
  @torch.no_grad()
17
  def _get_last_token_hidden_state(llm: LLM, prompt: str) -> torch.Tensor:
18
  """Hilfsfunktion, um den Hidden State des letzten Tokens eines Prompts zu erhalten."""
19
  inputs = llm.tokenizer(prompt, return_tensors="pt").to(llm.model.device)
20
  with torch.no_grad():
 
21
  outputs = llm.model(**inputs, output_hidden_states=True)
22
  last_hidden_state = outputs.hidden_states[-1][0, -1, :].cpu()
23
  assert last_hidden_state.shape == (llm.config.hidden_size,), \
@@ -26,28 +24,19 @@ def _get_last_token_hidden_state(llm: LLM, prompt: str) -> torch.Tensor:
26
 
27
  @torch.no_grad()
28
  def get_concept_vector(llm: LLM, concept: str, baseline_words: List[str] = BASELINE_WORDS) -> torch.Tensor:
29
- """
30
- Extrahiert einen Konzeptvektor mittels der kontrastiven Methode.
31
- """
32
  dbg(f"Extracting contrastive concept vector for '{concept}'...")
33
-
34
  prompt_template = "Here is a sentence about the concept of {}."
35
-
36
  dbg(f" - Getting activation for '{concept}'")
37
  target_hs = _get_last_token_hidden_state(llm, prompt_template.format(concept))
38
-
39
  baseline_hss = []
40
  for word in tqdm(baseline_words, desc=f" - Calculating baseline for '{concept}'", leave=False, bar_format="{l_bar}{bar:10}{r_bar}"):
41
  baseline_hss.append(_get_last_token_hidden_state(llm, prompt_template.format(word)))
42
-
43
- assert all(hs.shape == target_hs.shape for hs in baseline_hss), "Shape mismatch in baseline hidden states."
44
-
45
  mean_baseline_hs = torch.stack(baseline_hss).mean(dim=0)
46
  dbg(f" - Mean baseline vector computed with norm {torch.norm(mean_baseline_hs).item():.2f}")
47
-
48
  concept_vector = target_hs - mean_baseline_hs
49
  norm = torch.norm(concept_vector).item()
50
  dbg(f"Concept vector for '{concept}' extracted with norm {norm:.2f}.")
51
-
52
- assert torch.isfinite(concept_vector).all(), "Concept vector contains NaN or Inf values."
53
  return concept_vector
 
5
  from .llm_iface import LLM
6
  from .utils import dbg
7
 
 
8
  BASELINE_WORDS = [
9
  "thing", "place", "idea", "person", "object", "time", "way", "day", "man", "world",
10
  "life", "hand", "part", "child", "eye", "woman", "fact", "group", "case", "point"
11
  ]
12
 
 
 
13
  @torch.no_grad()
14
  def _get_last_token_hidden_state(llm: LLM, prompt: str) -> torch.Tensor:
15
  """Hilfsfunktion, um den Hidden State des letzten Tokens eines Prompts zu erhalten."""
16
  inputs = llm.tokenizer(prompt, return_tensors="pt").to(llm.model.device)
17
  with torch.no_grad():
18
+ # WAHRSCHEINLICHE FEHLERQUELLE: Sicherstellen, dass hier wirklich `llm` steht.
19
  outputs = llm.model(**inputs, output_hidden_states=True)
20
  last_hidden_state = outputs.hidden_states[-1][0, -1, :].cpu()
21
  assert last_hidden_state.shape == (llm.config.hidden_size,), \
 
24
 
25
  @torch.no_grad()
26
  def get_concept_vector(llm: LLM, concept: str, baseline_words: List[str] = BASELINE_WORDS) -> torch.Tensor:
27
+ """Extrahiert einen Konzeptvektor mittels der kontrastiven Methode."""
 
 
28
  dbg(f"Extracting contrastive concept vector for '{concept}'...")
 
29
  prompt_template = "Here is a sentence about the concept of {}."
 
30
  dbg(f" - Getting activation for '{concept}'")
31
  target_hs = _get_last_token_hidden_state(llm, prompt_template.format(concept))
 
32
  baseline_hss = []
33
  for word in tqdm(baseline_words, desc=f" - Calculating baseline for '{concept}'", leave=False, bar_format="{l_bar}{bar:10}{r_bar}"):
34
  baseline_hss.append(_get_last_token_hidden_state(llm, prompt_template.format(word)))
35
+ assert all(hs.shape == target_hs.shape for hs in baseline_hss)
 
 
36
  mean_baseline_hs = torch.stack(baseline_hss).mean(dim=0)
37
  dbg(f" - Mean baseline vector computed with norm {torch.norm(mean_baseline_hs).item():.2f}")
 
38
  concept_vector = target_hs - mean_baseline_hs
39
  norm = torch.norm(concept_vector).item()
40
  dbg(f"Concept vector for '{concept}' extracted with norm {norm:.2f}.")
41
+ assert torch.isfinite(concept_vector).all()
 
42
  return concept_vector