Commit
·
8a082d7
1
Parent(s):
094008d
fix plots?
Browse files
app.py
CHANGED
|
@@ -23,12 +23,6 @@ def run_single_analysis_display(*args, progress=gr.Progress(track_tqdm=True)):
|
|
| 23 |
finally:
|
| 24 |
cleanup_memory()
|
| 25 |
|
| 26 |
-
PLOT_PARAMS_DEFAULT = {
|
| 27 |
-
"x": "Step", "y": "Value", "color": "Metric",
|
| 28 |
-
"title": "Comparative Cognitive Dynamics", "color_legend_title": "Metrics",
|
| 29 |
-
"color_legend_position": "bottom", "show_label": True, "height": 400, "interactive": True
|
| 30 |
-
}
|
| 31 |
-
|
| 32 |
def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
|
| 33 |
"""Wrapper, der die speziellen Plots für die verschiedenen Experimente handhaben kann."""
|
| 34 |
try:
|
|
@@ -36,21 +30,28 @@ def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=
|
|
| 36 |
|
| 37 |
dataframe_component = gr.DataFrame(label="Comparative Statistical Signature", value=summary_df, wrap=True, row_count=(len(summary_df), "dynamic"))
|
| 38 |
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
if experiment_name == "ACT Titration (Point of No Return)":
|
| 41 |
plot_params.update({
|
| 42 |
"x": "Patch Step", "y": "Post-Patch Mean Delta", "color": None,
|
| 43 |
"title": "Attractor Capture Time (ACT) - Phase Transition", "mark": "line",
|
|
|
|
| 44 |
})
|
| 45 |
-
plot_params.pop("color_legend_title", None)
|
| 46 |
elif experiment_name == "Mechanistic Probe (Attention Entropies)":
|
| 47 |
plot_params.update({
|
| 48 |
"x": "Step", "y": "Value", "color": "Metric",
|
| 49 |
"title": "Mechanistic Analysis: State Delta vs. Attention Entropy",
|
|
|
|
| 50 |
})
|
| 51 |
-
else:
|
| 52 |
plot_params.update({
|
| 53 |
-
"y": "Delta", "color": "Experiment",
|
|
|
|
| 54 |
})
|
| 55 |
|
| 56 |
new_plot = gr.LinePlot(value=plot_df, **plot_params)
|
|
@@ -109,7 +110,8 @@ with gr.Blocks(theme=theme, title="Cognitive Seismograph 2.3") as demo:
|
|
| 109 |
|
| 110 |
with gr.Column(scale=2):
|
| 111 |
gr.Markdown("### Suite Results Summary")
|
| 112 |
-
|
|
|
|
| 113 |
auto_summary_df = gr.DataFrame(label="Comparative Statistical Signature", wrap=True)
|
| 114 |
with gr.Accordion("Raw JSON for all runs", open=False):
|
| 115 |
auto_raw_json = gr.JSON()
|
|
|
|
| 23 |
finally:
|
| 24 |
cleanup_memory()
|
| 25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
|
| 27 |
"""Wrapper, der die speziellen Plots für die verschiedenen Experimente handhaben kann."""
|
| 28 |
try:
|
|
|
|
| 30 |
|
| 31 |
dataframe_component = gr.DataFrame(label="Comparative Statistical Signature", value=summary_df, wrap=True, row_count=(len(summary_df), "dynamic"))
|
| 32 |
|
| 33 |
+
# FINALE KORREKTUR: Robuste Plot-Parameter-Logik
|
| 34 |
+
plot_params = {
|
| 35 |
+
"title": "Comparative Cognitive Dynamics",
|
| 36 |
+
"color_legend_position": "bottom", "show_label": True, "height": 400, "interactive": True
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
if experiment_name == "ACT Titration (Point of No Return)":
|
| 40 |
plot_params.update({
|
| 41 |
"x": "Patch Step", "y": "Post-Patch Mean Delta", "color": None,
|
| 42 |
"title": "Attractor Capture Time (ACT) - Phase Transition", "mark": "line",
|
| 43 |
+
"color_legend_title": None,
|
| 44 |
})
|
|
|
|
| 45 |
elif experiment_name == "Mechanistic Probe (Attention Entropies)":
|
| 46 |
plot_params.update({
|
| 47 |
"x": "Step", "y": "Value", "color": "Metric",
|
| 48 |
"title": "Mechanistic Analysis: State Delta vs. Attention Entropy",
|
| 49 |
+
"color_legend_title": "Metric",
|
| 50 |
})
|
| 51 |
+
else: # Default für alle anderen Multi-Lauf-Experimente
|
| 52 |
plot_params.update({
|
| 53 |
+
"x": "Step", "y": "Delta", "color": "Experiment",
|
| 54 |
+
"color_legend_title": "Experiment Runs",
|
| 55 |
})
|
| 56 |
|
| 57 |
new_plot = gr.LinePlot(value=plot_df, **plot_params)
|
|
|
|
| 110 |
|
| 111 |
with gr.Column(scale=2):
|
| 112 |
gr.Markdown("### Suite Results Summary")
|
| 113 |
+
# Initialisiere den Plot mit den Default-Parametern
|
| 114 |
+
auto_plot_output = gr.LinePlot(x="Step", y="Delta", color="Experiment", title="Comparative Cognitive Dynamics")
|
| 115 |
auto_summary_df = gr.DataFrame(label="Comparative Statistical Signature", wrap=True)
|
| 116 |
with gr.Accordion("Raw JSON for all runs", open=False):
|
| 117 |
auto_raw_json = gr.JSON()
|
cognitive_mapping_probe/auto_experiment.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
import pandas as pd
|
|
|
|
| 2 |
from typing import Dict, List, Tuple
|
| 3 |
|
| 4 |
from .llm_iface import get_or_load_model, release_model
|
|
@@ -104,7 +105,7 @@ def run_auto_suite(
|
|
| 104 |
raise ValueError(f"Experiment protocol '{experiment_name}' not found.")
|
| 105 |
|
| 106 |
all_results, summary_data, plot_data_frames = {}, [], []
|
| 107 |
-
llm = None
|
| 108 |
|
| 109 |
try:
|
| 110 |
if experiment_name == "Sequential Intervention (Self-Analysis -> Deletion)":
|
|
@@ -163,29 +164,30 @@ def run_auto_suite(
|
|
| 163 |
"Step": range(min_len), "State Delta": deltas[:min_len], "Attention Entropy": entropies[:min_len]
|
| 164 |
})
|
| 165 |
|
| 166 |
-
|
| 167 |
-
summary_data.append(summary_df_single) # Append DataFrame to list
|
| 168 |
plot_df = df.melt(id_vars=['Step'], value_vars=['State Delta', 'Attention Entropy'], var_name='Metric', value_name='Value')
|
|
|
|
| 169 |
|
| 170 |
-
|
| 171 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
|
| 173 |
-
else: # Handles
|
| 174 |
for i, run_spec in enumerate(protocol):
|
| 175 |
label = run_spec["label"]
|
| 176 |
current_probe_type = run_spec.get("probe_type", "seismic")
|
| 177 |
dbg(f"--- Running Auto-Experiment: '{label}' ({i+1}/{len(protocol)}) ---")
|
| 178 |
|
| 179 |
results = {}
|
| 180 |
-
if current_probe_type == "
|
| 181 |
-
results = run_act_titration_probe(
|
| 182 |
-
model_id=model_id, source_prompt_type=run_spec["source_prompt_type"],
|
| 183 |
-
dest_prompt_type=run_spec["dest_prompt_type"], patch_steps=run_spec["patch_steps"],
|
| 184 |
-
seed=seed, num_steps=num_steps, progress_callback=progress_callback,
|
| 185 |
-
)
|
| 186 |
-
summary_data.extend(results.get("titration_data", []))
|
| 187 |
-
|
| 188 |
-
elif current_probe_type == "causal_surgery":
|
| 189 |
results = run_causal_surgery_probe(
|
| 190 |
model_id=model_id, source_prompt_type=run_spec["source_prompt_type"],
|
| 191 |
dest_prompt_type=run_spec["dest_prompt_type"], patch_step=run_spec["patch_step"],
|
|
@@ -200,7 +202,6 @@ def run_auto_suite(
|
|
| 200 |
"Introspective Report": results.get("introspective_report", "N/A"),
|
| 201 |
"Patch Info": f"Source: {patch_info.get('source_prompt')}, Reset KV: {patch_info.get('kv_cache_reset')}"
|
| 202 |
})
|
| 203 |
-
|
| 204 |
elif current_probe_type == "triangulation":
|
| 205 |
results = run_triangulation_probe(
|
| 206 |
model_id=model_id, prompt_type=run_spec["prompt_type"], seed=seed, num_steps=num_steps,
|
|
@@ -213,7 +214,6 @@ def run_auto_suite(
|
|
| 213 |
"Std Dev Delta": stats.get("std_delta"), "Max Delta": stats.get("max_delta"),
|
| 214 |
"Introspective Report": results.get("introspective_report", "N/A")
|
| 215 |
})
|
| 216 |
-
|
| 217 |
else: # seismic
|
| 218 |
results = run_seismic_analysis(
|
| 219 |
model_id=model_id, prompt_type=run_spec["prompt_type"], seed=seed, num_steps=num_steps,
|
|
|
|
| 1 |
import pandas as pd
|
| 2 |
+
import gc
|
| 3 |
from typing import Dict, List, Tuple
|
| 4 |
|
| 5 |
from .llm_iface import get_or_load_model, release_model
|
|
|
|
| 105 |
raise ValueError(f"Experiment protocol '{experiment_name}' not found.")
|
| 106 |
|
| 107 |
all_results, summary_data, plot_data_frames = {}, [], []
|
| 108 |
+
llm = None
|
| 109 |
|
| 110 |
try:
|
| 111 |
if experiment_name == "Sequential Intervention (Self-Analysis -> Deletion)":
|
|
|
|
| 164 |
"Step": range(min_len), "State Delta": deltas[:min_len], "Attention Entropy": entropies[:min_len]
|
| 165 |
})
|
| 166 |
|
| 167 |
+
summary_df = df.drop(columns='Step').agg(['mean', 'std', 'max']).reset_index().rename(columns={'index':'Statistic'})
|
|
|
|
| 168 |
plot_df = df.melt(id_vars=['Step'], value_vars=['State Delta', 'Attention Entropy'], var_name='Metric', value_name='Value')
|
| 169 |
+
return summary_df, plot_df, all_results
|
| 170 |
|
| 171 |
+
elif probe_type == "act_titration":
|
| 172 |
+
run_spec = protocol[0]
|
| 173 |
+
label = run_spec["label"]
|
| 174 |
+
dbg(f"--- Running ACT Titration Experiment: '{label}' ---")
|
| 175 |
+
results = run_act_titration_probe(
|
| 176 |
+
model_id=model_id, source_prompt_type=run_spec["source_prompt_type"],
|
| 177 |
+
dest_prompt_type=run_spec["dest_prompt_type"], patch_steps=run_spec["patch_steps"],
|
| 178 |
+
seed=seed, num_steps=num_steps, progress_callback=progress_callback,
|
| 179 |
+
)
|
| 180 |
+
all_results[label] = results
|
| 181 |
+
summary_data.extend(results.get("titration_data", []))
|
| 182 |
|
| 183 |
+
else: # Handles seismic, triangulation, causal_surgery
|
| 184 |
for i, run_spec in enumerate(protocol):
|
| 185 |
label = run_spec["label"]
|
| 186 |
current_probe_type = run_spec.get("probe_type", "seismic")
|
| 187 |
dbg(f"--- Running Auto-Experiment: '{label}' ({i+1}/{len(protocol)}) ---")
|
| 188 |
|
| 189 |
results = {}
|
| 190 |
+
if current_probe_type == "causal_surgery":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
results = run_causal_surgery_probe(
|
| 192 |
model_id=model_id, source_prompt_type=run_spec["source_prompt_type"],
|
| 193 |
dest_prompt_type=run_spec["dest_prompt_type"], patch_step=run_spec["patch_step"],
|
|
|
|
| 202 |
"Introspective Report": results.get("introspective_report", "N/A"),
|
| 203 |
"Patch Info": f"Source: {patch_info.get('source_prompt')}, Reset KV: {patch_info.get('kv_cache_reset')}"
|
| 204 |
})
|
|
|
|
| 205 |
elif current_probe_type == "triangulation":
|
| 206 |
results = run_triangulation_probe(
|
| 207 |
model_id=model_id, prompt_type=run_spec["prompt_type"], seed=seed, num_steps=num_steps,
|
|
|
|
| 214 |
"Std Dev Delta": stats.get("std_delta"), "Max Delta": stats.get("max_delta"),
|
| 215 |
"Introspective Report": results.get("introspective_report", "N/A")
|
| 216 |
})
|
|
|
|
| 217 |
else: # seismic
|
| 218 |
results = run_seismic_analysis(
|
| 219 |
model_id=model_id, prompt_type=run_spec["prompt_type"], seed=seed, num_steps=num_steps,
|
docs/12B-results-Mechanistic Probe (Attention Entropies).png
ADDED
|
Git LFS Details
|
docs/12B-results-Mechanistic Probe (Attention Entropies).txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
mean 148.70083333333332 2.694883015950521 std 33.019324548173515 0.34383212147542636 max 3103.4762369791666665
|
| 2 |
+
|