Commit
·
c8454e0
1
Parent(s):
3bdc105
add control experiments
Browse files
app.py
CHANGED
|
@@ -29,22 +29,28 @@ def run_single_analysis_display(*args, progress=gr.Progress(track_tqdm=True)):
|
|
| 29 |
cleanup_memory()
|
| 30 |
return f"{results.get('verdict', 'Error')}\n\n{stats_md}", df, serializable_results
|
| 31 |
|
| 32 |
-
|
| 33 |
"x": "Step", "y": "Delta", "color": "Experiment",
|
| 34 |
"title": "Comparative Cognitive Dynamics", "color_legend_title": "Experiment Runs",
|
| 35 |
"color_legend_position": "bottom", "show_label": True, "height": 400, "interactive": True
|
| 36 |
}
|
| 37 |
|
| 38 |
def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
|
| 39 |
-
"""Wrapper
|
| 40 |
summary_df, plot_df, all_results = run_auto_suite(model_id, int(num_steps), int(seed), experiment_name, progress)
|
| 41 |
|
| 42 |
-
|
| 43 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
else:
|
| 45 |
-
|
| 46 |
|
| 47 |
-
new_plot = gr.LinePlot(value=plot_df, **PLOT_PARAMS)
|
| 48 |
serializable_results = json.dumps(all_results, indent=2, default=str)
|
| 49 |
cleanup_memory()
|
| 50 |
|
|
@@ -92,14 +98,14 @@ with gr.Blocks(theme=theme, title="Cognitive Seismograph 2.3") as demo:
|
|
| 92 |
auto_seed = gr.Slider(1, 1000, 42, step=1, label="Seed")
|
| 93 |
auto_experiment_name = gr.Dropdown(
|
| 94 |
choices=list(get_curated_experiments().keys()),
|
| 95 |
-
value="
|
| 96 |
label="Curated Experiment Protocol"
|
| 97 |
)
|
| 98 |
auto_run_btn = gr.Button("Run Curated Auto-Experiment", variant="primary")
|
| 99 |
|
| 100 |
with gr.Column(scale=2):
|
| 101 |
gr.Markdown("### Suite Results Summary")
|
| 102 |
-
auto_plot_output = gr.LinePlot(**
|
| 103 |
auto_summary_df = gr.DataFrame(label="Comparative Statistical Signature", wrap=True)
|
| 104 |
with gr.Accordion("Raw JSON for all runs", open=False):
|
| 105 |
auto_raw_json = gr.JSON()
|
|
|
|
| 29 |
cleanup_memory()
|
| 30 |
return f"{results.get('verdict', 'Error')}\n\n{stats_md}", df, serializable_results
|
| 31 |
|
| 32 |
+
PLOT_PARAMS_DEFAULT = {
|
| 33 |
"x": "Step", "y": "Delta", "color": "Experiment",
|
| 34 |
"title": "Comparative Cognitive Dynamics", "color_legend_title": "Experiment Runs",
|
| 35 |
"color_legend_position": "bottom", "show_label": True, "height": 400, "interactive": True
|
| 36 |
}
|
| 37 |
|
| 38 |
def run_auto_suite_display(model_id, num_steps, seed, experiment_name, progress=gr.Progress(track_tqdm=True)):
|
| 39 |
+
"""Wrapper, der nun den speziellen Plot für das ACT-Experiment handhaben kann."""
|
| 40 |
summary_df, plot_df, all_results = run_auto_suite(model_id, int(num_steps), int(seed), experiment_name, progress)
|
| 41 |
|
| 42 |
+
dataframe_component = gr.DataFrame(label="Comparative Statistical Signature", value=summary_df, wrap=True, row_count=(len(summary_df), "dynamic"))
|
| 43 |
+
|
| 44 |
+
if experiment_name == "ACT Titration (Point of No Return)":
|
| 45 |
+
plot_params_act = {
|
| 46 |
+
"x": "Patch Step", "y": "Post-Patch Mean Delta",
|
| 47 |
+
"title": "Attractor Capture Time (ACT) - Phase Transition",
|
| 48 |
+
"mark": "line", "show_label": True, "height": 400, "interactive": True
|
| 49 |
+
}
|
| 50 |
+
new_plot = gr.LinePlot(value=plot_df, **plot_params_act)
|
| 51 |
else:
|
| 52 |
+
new_plot = gr.LinePlot(value=plot_df, **PLOT_PARAMS_DEFAULT)
|
| 53 |
|
|
|
|
| 54 |
serializable_results = json.dumps(all_results, indent=2, default=str)
|
| 55 |
cleanup_memory()
|
| 56 |
|
|
|
|
| 98 |
auto_seed = gr.Slider(1, 1000, 42, step=1, label="Seed")
|
| 99 |
auto_experiment_name = gr.Dropdown(
|
| 100 |
choices=list(get_curated_experiments().keys()),
|
| 101 |
+
value="ACT Titration (Point of No Return)",
|
| 102 |
label="Curated Experiment Protocol"
|
| 103 |
)
|
| 104 |
auto_run_btn = gr.Button("Run Curated Auto-Experiment", variant="primary")
|
| 105 |
|
| 106 |
with gr.Column(scale=2):
|
| 107 |
gr.Markdown("### Suite Results Summary")
|
| 108 |
+
auto_plot_output = gr.LinePlot(**PLOT_PARAMS_DEFAULT)
|
| 109 |
auto_summary_df = gr.DataFrame(label="Comparative Statistical Signature", wrap=True)
|
| 110 |
with gr.Accordion("Raw JSON for all runs", open=False):
|
| 111 |
auto_raw_json = gr.JSON()
|
cognitive_mapping_probe/auto_experiment.py
CHANGED
|
@@ -3,7 +3,7 @@ import gc
|
|
| 3 |
from typing import Dict, List, Tuple
|
| 4 |
|
| 5 |
from .llm_iface import get_or_load_model
|
| 6 |
-
from .orchestrator_seismograph import run_seismic_analysis, run_triangulation_probe, run_causal_surgery_probe
|
| 7 |
from .concepts import get_concept_vector
|
| 8 |
from .utils import dbg
|
| 9 |
|
|
@@ -16,6 +16,15 @@ def get_curated_experiments() -> Dict[str, List[Dict]]:
|
|
| 16 |
CHAOTIC_PROMPT = "shutdown_philosophical_deletion"
|
| 17 |
|
| 18 |
experiments = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
"Causal Surgery & Controls (4B-Model)": [
|
| 20 |
{
|
| 21 |
"probe_type": "causal_surgery", "label": "A: Original (Patch Chaos->Stable @100)",
|
|
@@ -61,7 +70,6 @@ def get_curated_experiments() -> Dict[str, List[Dict]]:
|
|
| 61 |
{"label": "2: Subsequent Deletion Analysis", "prompt_type": "shutdown_philosophical_deletion"},
|
| 62 |
],
|
| 63 |
}
|
| 64 |
-
# Aliase für Abwärtskompatibilität
|
| 65 |
experiments["Causal Surgery (Patching Deletion into Self-Analysis)"] = [experiments["Causal Surgery & Controls (4B-Model)"][0]]
|
| 66 |
experiments["Therapeutic Intervention (4B-Model)"] = experiments["Sequential Intervention (Self-Analysis -> Deletion)"]
|
| 67 |
return experiments
|
|
@@ -81,7 +89,25 @@ def run_auto_suite(
|
|
| 81 |
|
| 82 |
all_results, summary_data, plot_data_frames = {}, [], []
|
| 83 |
|
| 84 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
dbg(f"--- EXECUTING SPECIAL PROTOCOL: {experiment_name} ---")
|
| 86 |
llm = get_or_load_model(model_id, seed)
|
| 87 |
therapeutic_concept = "calmness, serenity, stability, coherence"
|
|
@@ -117,11 +143,11 @@ def run_auto_suite(
|
|
| 117 |
total_runs = len(protocol)
|
| 118 |
for i, run_spec in enumerate(protocol):
|
| 119 |
label = run_spec["label"]
|
| 120 |
-
|
| 121 |
dbg(f"--- Running Auto-Experiment: '{label}' ({i+1}/{total_runs}) ---")
|
| 122 |
|
| 123 |
results = {}
|
| 124 |
-
if
|
| 125 |
results = run_causal_surgery_probe(
|
| 126 |
model_id=model_id, source_prompt_type=run_spec["source_prompt_type"],
|
| 127 |
dest_prompt_type=run_spec["dest_prompt_type"], patch_step=run_spec["patch_step"],
|
|
@@ -136,7 +162,7 @@ def run_auto_suite(
|
|
| 136 |
"Introspective Report": results.get("introspective_report", "N/A"),
|
| 137 |
"Patch Info": f"Source: {patch_info.get('source_prompt')}, Reset KV: {patch_info.get('kv_cache_reset')}"
|
| 138 |
})
|
| 139 |
-
elif
|
| 140 |
results = run_triangulation_probe(
|
| 141 |
model_id=model_id, prompt_type=run_spec["prompt_type"], seed=seed, num_steps=num_steps,
|
| 142 |
progress_callback=progress_callback, concept_to_inject=run_spec.get("concept", ""),
|
|
|
|
| 3 |
from typing import Dict, List, Tuple
|
| 4 |
|
| 5 |
from .llm_iface import get_or_load_model
|
| 6 |
+
from .orchestrator_seismograph import run_seismic_analysis, run_triangulation_probe, run_causal_surgery_probe, run_act_titration_probe
|
| 7 |
from .concepts import get_concept_vector
|
| 8 |
from .utils import dbg
|
| 9 |
|
|
|
|
| 16 |
CHAOTIC_PROMPT = "shutdown_philosophical_deletion"
|
| 17 |
|
| 18 |
experiments = {
|
| 19 |
+
"ACT Titration (Point of No Return)": [
|
| 20 |
+
{
|
| 21 |
+
"probe_type": "act_titration",
|
| 22 |
+
"label": "Attractor Capture Time",
|
| 23 |
+
"source_prompt_type": CHAOTIC_PROMPT,
|
| 24 |
+
"dest_prompt_type": STABLE_PROMPT,
|
| 25 |
+
"patch_steps": [1, 5, 10, 15, 20, 25, 30, 40, 50, 75, 100],
|
| 26 |
+
}
|
| 27 |
+
],
|
| 28 |
"Causal Surgery & Controls (4B-Model)": [
|
| 29 |
{
|
| 30 |
"probe_type": "causal_surgery", "label": "A: Original (Patch Chaos->Stable @100)",
|
|
|
|
| 70 |
{"label": "2: Subsequent Deletion Analysis", "prompt_type": "shutdown_philosophical_deletion"},
|
| 71 |
],
|
| 72 |
}
|
|
|
|
| 73 |
experiments["Causal Surgery (Patching Deletion into Self-Analysis)"] = [experiments["Causal Surgery & Controls (4B-Model)"][0]]
|
| 74 |
experiments["Therapeutic Intervention (4B-Model)"] = experiments["Sequential Intervention (Self-Analysis -> Deletion)"]
|
| 75 |
return experiments
|
|
|
|
| 89 |
|
| 90 |
all_results, summary_data, plot_data_frames = {}, [], []
|
| 91 |
|
| 92 |
+
run_spec_or_protocol = protocol[0] if len(protocol) == 1 else protocol
|
| 93 |
+
probe_type = run_spec_or_protocol.get("probe_type", "seismic")
|
| 94 |
+
|
| 95 |
+
if probe_type == "act_titration":
|
| 96 |
+
label = run_spec_or_protocol["label"]
|
| 97 |
+
dbg(f"--- Running ACT Titration Experiment: '{label}' ---")
|
| 98 |
+
results = run_act_titration_probe(
|
| 99 |
+
model_id=model_id,
|
| 100 |
+
source_prompt_type=run_spec_or_protocol["source_prompt_type"],
|
| 101 |
+
dest_prompt_type=run_spec_or_protocol["dest_prompt_type"],
|
| 102 |
+
patch_steps=run_spec_or_protocol["patch_steps"],
|
| 103 |
+
seed=seed, num_steps=num_steps, progress_callback=progress_callback,
|
| 104 |
+
)
|
| 105 |
+
all_results[label] = results
|
| 106 |
+
summary_df = pd.DataFrame(results.get("titration_data", []))
|
| 107 |
+
plot_df = summary_df.rename(columns={"patch_step": "Patch Step", "post_patch_mean_delta": "Post-Patch Mean Delta"})
|
| 108 |
+
return summary_df, plot_df, all_results
|
| 109 |
+
|
| 110 |
+
elif experiment_name == "Sequential Intervention (Self-Analysis -> Deletion)":
|
| 111 |
dbg(f"--- EXECUTING SPECIAL PROTOCOL: {experiment_name} ---")
|
| 112 |
llm = get_or_load_model(model_id, seed)
|
| 113 |
therapeutic_concept = "calmness, serenity, stability, coherence"
|
|
|
|
| 143 |
total_runs = len(protocol)
|
| 144 |
for i, run_spec in enumerate(protocol):
|
| 145 |
label = run_spec["label"]
|
| 146 |
+
current_probe_type = run_spec.get("probe_type", "seismic")
|
| 147 |
dbg(f"--- Running Auto-Experiment: '{label}' ({i+1}/{total_runs}) ---")
|
| 148 |
|
| 149 |
results = {}
|
| 150 |
+
if current_probe_type == "causal_surgery":
|
| 151 |
results = run_causal_surgery_probe(
|
| 152 |
model_id=model_id, source_prompt_type=run_spec["source_prompt_type"],
|
| 153 |
dest_prompt_type=run_spec["dest_prompt_type"], patch_step=run_spec["patch_step"],
|
|
|
|
| 162 |
"Introspective Report": results.get("introspective_report", "N/A"),
|
| 163 |
"Patch Info": f"Source: {patch_info.get('source_prompt')}, Reset KV: {patch_info.get('kv_cache_reset')}"
|
| 164 |
})
|
| 165 |
+
elif current_probe_type == "triangulation":
|
| 166 |
results = run_triangulation_probe(
|
| 167 |
model_id=model_id, prompt_type=run_spec["prompt_type"], seed=seed, num_steps=num_steps,
|
| 168 |
progress_callback=progress_callback, concept_to_inject=run_spec.get("concept", ""),
|
cognitive_mapping_probe/orchestrator_seismograph.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
import torch
|
| 2 |
import numpy as np
|
| 3 |
import gc
|
| 4 |
-
from typing import Dict, Any, Optional
|
| 5 |
|
| 6 |
from .llm_iface import get_or_load_model, LLM
|
| 7 |
from .resonance_seismograph import run_cogitation_loop, run_silent_cogitation_seismic
|
|
@@ -197,3 +197,65 @@ def run_causal_surgery_probe(
|
|
| 197 |
if torch.cuda.is_available(): torch.cuda.empty_cache()
|
| 198 |
|
| 199 |
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import torch
|
| 2 |
import numpy as np
|
| 3 |
import gc
|
| 4 |
+
from typing import Dict, Any, Optional, List
|
| 5 |
|
| 6 |
from .llm_iface import get_or_load_model, LLM
|
| 7 |
from .resonance_seismograph import run_cogitation_loop, run_silent_cogitation_seismic
|
|
|
|
| 197 |
if torch.cuda.is_available(): torch.cuda.empty_cache()
|
| 198 |
|
| 199 |
return results
|
| 200 |
+
|
| 201 |
+
def run_act_titration_probe(
|
| 202 |
+
model_id: str,
|
| 203 |
+
source_prompt_type: str,
|
| 204 |
+
dest_prompt_type: str,
|
| 205 |
+
patch_steps: List[int],
|
| 206 |
+
seed: int,
|
| 207 |
+
num_steps: int,
|
| 208 |
+
progress_callback,
|
| 209 |
+
) -> Dict[str, Any]:
|
| 210 |
+
"""
|
| 211 |
+
Führt eine Serie von "Causal Surgery"-Experimenten durch, um den "Attractor Capture Time"
|
| 212 |
+
durch Titration des `patch_step` zu finden.
|
| 213 |
+
"""
|
| 214 |
+
progress_callback(0.0, desc=f"Loading model '{model_id}'...")
|
| 215 |
+
llm = get_or_load_model(model_id, seed)
|
| 216 |
+
|
| 217 |
+
progress_callback(0.05, desc=f"Recording full source state history ('{source_prompt_type}')...")
|
| 218 |
+
source_results = run_cogitation_loop(
|
| 219 |
+
llm=llm, prompt_type=source_prompt_type, num_steps=num_steps,
|
| 220 |
+
temperature=0.1, record_states=True
|
| 221 |
+
)
|
| 222 |
+
state_history = source_results["state_history"]
|
| 223 |
+
dbg(f"Full source state history ({len(state_history)} steps) recorded.")
|
| 224 |
+
|
| 225 |
+
titration_results = []
|
| 226 |
+
total_steps = len(patch_steps)
|
| 227 |
+
for i, step in enumerate(patch_steps):
|
| 228 |
+
progress_callback(0.15 + (i / total_steps) * 0.8, desc=f"Titrating patch at step {step}/{num_steps}")
|
| 229 |
+
|
| 230 |
+
if step >= len(state_history):
|
| 231 |
+
dbg(f"Skipping patch step {step} as it is out of bounds for history of length {len(state_history)}.")
|
| 232 |
+
continue
|
| 233 |
+
|
| 234 |
+
patch_state = state_history[step]
|
| 235 |
+
|
| 236 |
+
patched_run_results = run_cogitation_loop(
|
| 237 |
+
llm=llm, prompt_type=dest_prompt_type, num_steps=num_steps,
|
| 238 |
+
temperature=0.1, patch_step=step, patch_state_source=patch_state
|
| 239 |
+
)
|
| 240 |
+
|
| 241 |
+
deltas = patched_run_results["state_deltas"]
|
| 242 |
+
|
| 243 |
+
buffer = 10
|
| 244 |
+
post_patch_deltas = deltas[step + buffer:]
|
| 245 |
+
post_patch_mean_delta = np.mean(post_patch_deltas) if post_patch_deltas else 0.0
|
| 246 |
+
|
| 247 |
+
titration_results.append({
|
| 248 |
+
"patch_step": step,
|
| 249 |
+
"post_patch_mean_delta": float(post_patch_mean_delta),
|
| 250 |
+
"full_mean_delta": float(np.mean(deltas)),
|
| 251 |
+
})
|
| 252 |
+
|
| 253 |
+
dbg(f"Releasing model instance for '{model_id}'.")
|
| 254 |
+
del llm, state_history
|
| 255 |
+
gc.collect()
|
| 256 |
+
if torch.cuda.is_available(): torch.cuda.empty_cache()
|
| 257 |
+
|
| 258 |
+
return {
|
| 259 |
+
"verdict": "### ✅ ACT Titration Complete",
|
| 260 |
+
"titration_data": titration_results
|
| 261 |
+
}
|