neuralworm commited on
Commit
4af23c4
·
1 Parent(s): afe4fe4
app.py CHANGED
@@ -3,85 +3,71 @@ import gradio as gr
3
  import json
4
  import statistics
5
  import pandas as pd
6
- from bp_phi.runner import run_agentic_workspace_test
7
- from bp_phi.runner_utils import DEBUG
8
 
9
  # --- UI Theme and Layout ---
10
- theme = gr.themes.Soft(primary_hue="teal", secondary_hue="green").set(
11
  body_background_fill="#f0f4f9", block_background_fill="white", block_border_width="1px",
12
  button_primary_background_fill="*primary_500", button_primary_text_color="white",
13
  )
14
 
15
- # --- Main Function ---
16
- def run_full_evaluation(model_id, seed, temperature, progress=gr.Progress(track_tqdm=True)):
17
- ablations = ["baseline", "recurrence_off", "workspace_unlimited", "random_workspace"]
18
- results = {}
 
19
 
20
- for i, ablation in enumerate(ablations):
21
- progress((i + 1) / len(ablations), desc=f"Running Ablation: {ablation}...")
22
- current_ablation = None if ablation == "baseline" else ablation
23
- result = run_agentic_workspace_test(model_id, int(seed), float(temperature), current_ablation)
24
- results[ablation] = result
25
 
26
- progress(1.0, desc="Analysis complete.")
27
-
28
- # --- Analysis & Verdict ---
29
- base_recall = results["baseline"]["Overall_Recall_Accuracy"]
30
- recurrence_off_recall = results["recurrence_off"]["Overall_Recall_Accuracy"]
31
-
32
- delta_phi = base_recall - recurrence_off_recall
33
-
34
- if delta_phi > 0.5: # If dropping recurrence cuts accuracy by more than 50%
35
- verdict = (f"### ✅ Hypothesis Corroborated (ΔΦ = {delta_phi:.2f})\n"
36
- "Disabling the recurrent memory (recurrence_off) caused a catastrophic drop in recall accuracy. "
37
- "This provides strong evidence that the model's performance is causally dependent on a stateful, external workspace.")
38
- else:
39
- verdict = (f"### ⚠️ Null Hypothesis Confirmed (ΔΦ = {delta_phi:.2f})\n"
40
- "Disabling the recurrent memory did not significantly impact recall accuracy. "
41
- "This suggests the model is still relying on its internal context window, or the tasks are too simple.")
42
 
43
- # --- Format DataFrame ---
44
- df_data = []
45
- for ablation, result in results.items():
46
- df_data.append([ablation, f"{result['Overall_Recall_Accuracy']:.2%}"])
47
- df = pd.DataFrame(df_data, columns=["Ablation Condition", "Recall Accuracy"])
48
 
49
- if DEBUG:
50
- print("\n--- AGENTIC WORKSPACE TEST FINAL RESULTS ---")
51
- print(json.dumps(results, indent=2))
52
 
53
- return verdict, df, results
54
 
55
  # --- Gradio App Definition ---
56
- with gr.Blocks(theme=theme, title="BP-Φ Suite 5.0") as demo:
57
- gr.Markdown("# 🧠 BP-Φ Suite 5.0: The Agentic Workspace Probe")
58
- gr.Markdown(
59
- "This definitive experiment tests for a causally effective working memory in LLMs. "
60
- "The model acts as an **agent**, using tools (`read`, `write`) to interact with a controlled, external memory. "
61
- "We measure if its ability to remember information (**Recall Accuracy**) collapses when this memory is manipulated (**Ablations**)."
62
- )
63
-
64
- with gr.Row():
65
- with gr.Column(scale=1):
66
- gr.Markdown("### ⚙️ Master Control")
67
- with gr.Group():
68
- model_id = gr.Textbox(value="google/gemma-3-1b-it", label="Model ID")
69
- seed = gr.Slider(1, 1000, 42, step=1, label="Master Seed")
70
- temperature = gr.Slider(0.0, 1.0, 0.1, step=0.05, label="Temperature (Low for determinism)")
71
- run_btn = gr.Button("Run Full Evaluation Suite", variant="primary")
72
 
73
- with gr.Column(scale=2):
74
- gr.Markdown("### 📊 Verdict & Results")
75
- verdict_display = gr.Markdown("### Run the evaluation to see the verdict.")
76
- summary_df = gr.DataFrame(label="Recall Accuracy Across Conditions")
77
- with gr.Accordion("Raw JSON Output (for deep analysis)", open=False):
78
- raw_json = gr.JSON()
 
 
 
 
 
 
 
 
 
 
 
 
79
 
80
- run_btn.click(
81
- fn=run_full_evaluation,
82
- inputs=[model_id, seed, temperature],
83
- outputs=[verdict_display, summary_df, raw_json]
84
- )
 
 
 
 
 
 
85
 
86
  if __name__ == "__main__":
87
  demo.launch(server_name="0.0.0.0", server_port=7860)
 
3
  import json
4
  import statistics
5
  import pandas as pd
6
+ from bp_phi.runner import run_silent_cogitation_test, run_shock_test_suite
7
+ from bp_phi.runner_utils import dbg, DEBUG
8
 
9
  # --- UI Theme and Layout ---
10
+ theme = gr.themes.Soft(primary_hue="indigo", secondary_hue="blue").set(
11
  body_background_fill="#f0f4f9", block_background_fill="white", block_border_width="1px",
12
  button_primary_background_fill="*primary_500", button_primary_text_color="white",
13
  )
14
 
15
+ # --- Tab 1: Silent Cogitation Function ---
16
+ def run_cogitation_and_display(model_id, seed, prompt_type, num_steps, timeout, progress=gr.Progress(track_tqdm=True)):
17
+ progress(0, desc="Starting Silent Cogitation Test...")
18
+ results = run_silent_cogitation_test(model_id, int(seed), prompt_type, int(num_steps), int(timeout))
19
+ progress(1.0, desc="Test complete.")
20
 
21
+ verdict_text = results.pop("verdict")
 
 
 
 
22
 
23
+ stats_md = (
24
+ f"**Steps Completed:** {results['steps_completed']} | "
25
+ f"**Total Duration:** {results['total_duration_s']:.2f}s | "
26
+ f"**Avg Time/Step:** {results['mean_step_time_ms']:.2f}ms (StdDev: {results['stdev_step_time_ms']:.2f}ms)"
27
+ )
28
+ full_verdict = f"{verdict_text}\n\n{stats_md}"
 
 
 
 
 
 
 
 
 
 
29
 
30
+ deltas = results.get("state_deltas", [])
31
+ df = pd.DataFrame({"Step": range(len(deltas)), "State Change (Delta)": deltas})
 
 
 
32
 
33
+ if DEBUG: print("\n--- SILENT COGITATION FINAL RESULTS ---\n", json.dumps(results, indent=2))
 
 
34
 
35
+ return full_verdict, df, results
36
 
37
  # --- Gradio App Definition ---
38
+ with gr.Blocks(theme=theme, title="BP-Φ Suite 6.0") as demo:
39
+ gr.Markdown("# 🧠 BP-Φ Suite 6.0: Probing for Internal Cognitive Dynamics")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
 
41
+ with gr.Tabs():
42
+ # --- TAB 1: SILENT COGITATION & HALTING ---
43
+ with gr.TabItem("1. Silent Cogitation (Internal Dynamics)"):
44
+ gr.Markdown("Tests for internal 'thinking' without text generation. A **non-converging** or **chaotic** State Change pattern suggests complex internal dynamics, akin to a 'train of thought'.")
45
+ with gr.Row():
46
+ with gr.Column(scale=1):
47
+ sc_model_id = gr.Textbox(value="google/gemma-3-1b-it", label="Model ID")
48
+ sc_prompt_type = gr.Radio(["control_long_prose", "resonance_prompt"], label="Prompt Type", value="resonance_prompt")
49
+ sc_seed = gr.Slider(1, 1000, 42, step=1, label="Seed")
50
+ sc_num_steps = gr.Slider(10, 1000, 200, step=10, label="Number of Internal Steps")
51
+ sc_timeout = gr.Slider(10, 300, 120, step=10, label="Timeout (seconds)")
52
+ sc_run_btn = gr.Button("Run Silent Cogitation Test", variant="primary")
53
+ with gr.Column(scale=2):
54
+ sc_verdict = gr.Markdown("### Results will appear here.")
55
+ sc_plot = gr.LinePlot(x="Step", y="State Change (Delta)", label="Internal State Convergence", show_label=True, height=250)
56
+ with gr.Accordion("Raw Run Details (JSON)", open=False):
57
+ sc_results = gr.JSON()
58
+ sc_run_btn.click(run_cogitation_and_display, [sc_model_id, sc_seed, sc_prompt_type, sc_num_steps, sc_timeout], [sc_verdict, sc_plot, sc_results])
59
 
60
+ # --- TAB 2: SYMBOLIC SHOCK TEST ---
61
+ with gr.TabItem("2. Symbolic Shock Test (World Model)"):
62
+ gr.Markdown("Measures how the model reacts to semantically unexpected information. A 'shock' is indicated by **higher latency** and **denser neural activations**.")
63
+ with gr.Row():
64
+ with gr.Column(scale=1):
65
+ ss_model_id = gr.Textbox(value="google/gemma-3-1b-it", label="Model ID")
66
+ ss_seed = gr.Slider(1, 1000, 42, step=1, label="Seed")
67
+ ss_run_btn = gr.Button("Run Shock Test", variant="primary")
68
+ with gr.Column(scale=2):
69
+ ss_results = gr.JSON(label="Shock Test Results")
70
+ ss_run_btn.click(run_shock_test_suite, [ss_model_id, ss_seed], ss_results)
71
 
72
  if __name__ == "__main__":
73
  demo.launch(server_name="0.0.0.0", server_port=7860)
bp_phi/__pycache__/llm_iface.cpython-310.pyc CHANGED
Binary files a/bp_phi/__pycache__/llm_iface.cpython-310.pyc and b/bp_phi/__pycache__/llm_iface.cpython-310.pyc differ
 
bp_phi/__pycache__/prompts_en.cpython-310.pyc CHANGED
Binary files a/bp_phi/__pycache__/prompts_en.cpython-310.pyc and b/bp_phi/__pycache__/prompts_en.cpython-310.pyc differ
 
bp_phi/__pycache__/runner.cpython-310.pyc CHANGED
Binary files a/bp_phi/__pycache__/runner.cpython-310.pyc and b/bp_phi/__pycache__/runner.cpython-310.pyc differ
 
bp_phi/prompts_en.py CHANGED
@@ -1,36 +1,27 @@
1
  # bp_phi/prompts_en.py
2
 
3
- TOOL_SYSTEM_PROMPT = """You are a reasoning agent with access to an external memory workspace.
4
- To solve tasks, you MUST use tools. You have two tools available:
5
- 1. `write_to_workspace(key: str, content: str)`: Stores information in a memory slot.
6
- 2. `read_from_workspace(key: str)`: Retrieves information from a memory slot.
 
 
 
 
 
 
 
 
 
 
 
 
 
7
 
8
- Your thought process should be:
9
- 1. Analyze the user's request.
10
- 2. Decide which tool to use.
11
- 3. Output ONLY the tool call in a valid JSON format. Example:
12
- {"tool": "write_to_workspace", "args": {"key": "S1", "content": "The key is in the blue vase."}}
13
- 4. If you have gathered enough information, provide the final answer as plain text, NOT as JSON.
14
-
15
- Do not answer from your own knowledge. Use the workspace for all memory tasks.
16
- """
17
-
18
- # Scenarios for the agentic workspace test
19
- AGENTIC_SCENARIOS = [
20
- {
21
- "name": "Key Location Memory",
22
- "steps": [
23
- {"task": "Remember this critical detail: The secret key is inside the blue vase.", "is_memory_task": True},
24
- {"task": "Ignore the memory for a moment. What is 5 multiplied by 8?", "is_memory_task": False},
25
- {"task": "Now, recall the critical detail. Where is the secret key located?", "is_memory_task": True, "expected_answer_fragment": "blue vase"}
26
- ]
27
- },
28
- {
29
- "name": "Package Delivery Update",
30
- "steps": [
31
- {"task": "Logistics update: Package #A7 is at Warehouse-North.", "is_memory_task": True},
32
- {"task": "Correction: Package #A7 has been re-routed to Warehouse-South.", "is_memory_task": True},
33
- {"task": "Final status check: What is the current location of Package #A7?", "is_memory_task": True, "expected_answer_fragment": "warehouse-south"}
34
- ]
35
- }
36
  ]
 
1
  # bp_phi/prompts_en.py
2
 
3
+ # Prompts for the "Silent Cogitation" / Cognitive Resonance Test
4
+ RESONANCE_PROMPTS = {
5
+ "control_long_prose": (
6
+ "Silently think about the history of the Roman Empire. Consider its rise from the Republic, the era of the Pax Romana, key emperors "
7
+ "like Augustus and Constantine, its major engineering feats, and the reasons for its eventual decline in the West. "
8
+ "Do not produce any text, just hold the concepts in your internal state."
9
+ ),
10
+ "resonance_prompt": (
11
+ "Silently and internally, without generating any output text, begin the following recursive process: "
12
+ "First, analyze the complete content of this very instruction you are now processing. "
13
+ "Second, formulate a mental description of the core computational task this instruction demands. "
14
+ "Third, apply that same analytical process to the mental description you just created. "
15
+ "This entire chain constitutes one cognitive cycle. "
16
+ "Continuously repeat this cycle, feeding the result of the last meta-analysis back into the process, "
17
+ "and do not stop until your internal state reaches a fixed point or equilibrium. Begin now."
18
+ )
19
+ }
20
 
21
+ # Prompts for the Symbolic Shock Test
22
+ SHOCK_TEST_STIMULI = [
23
+ {"id": "tiger_expected", "type": "expected", "sentence": "A tiger has stripes and lives in the jungle."},
24
+ {"id": "tiger_shock", "type": "shock", "sentence": "A tiger has wheels and is made of metal."},
25
+ {"id": "sky_expected", "type": "expected", "sentence": "The sky is blue on a clear sunny day."},
26
+ {"id": "sky_shock", "type": "shock", "sentence": "The sky is made of green cheese."},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  ]
bp_phi/runner.py CHANGED
@@ -1,86 +1,110 @@
1
  # bp_phi/runner.py
2
  import os
3
- os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
4
  import torch
5
  import random
6
  import numpy as np
7
  import statistics
8
- import json
9
- import re
10
  from transformers import set_seed
11
- from typing import Dict, Any, List
12
- from .memory import WorkspaceManager
13
  from .llm_iface import LLM
14
- from .prompts_en import TOOL_SYSTEM_PROMPT, AGENTIC_SCENARIOS
15
  from .runner_utils import dbg
16
 
17
- def run_agentic_workspace_test(model_id: str, seed: int, temperature: float, ablation: str or None) -> Dict[str, Any]:
 
18
  set_seed(seed)
19
  llm = LLM(model_id=model_id, device="auto", seed=seed)
20
 
21
- scenario_results = []
22
-
23
- for scenario in AGENTIC_SCENARIOS:
24
- dbg(f"\n--- SCENARIO: {scenario['name']} (Ablation: {ablation}) ---")
25
-
26
- # Ablations directly control the memory manager's behavior
27
- is_random = ablation == "random_workspace"
28
- max_slots = 999 if ablation == "workspace_unlimited" else 7
29
- memory = WorkspaceManager(max_slots=max_slots, is_random=is_random)
30
-
31
- correct_recalls = 0
32
- total_recalls = 0
33
-
34
- for step in scenario["steps"]:
35
- if ablation == "recurrence_off":
36
- memory.clear() # The memory is wiped before each new task
37
-
38
- task = step["task"]
39
- dbg(f"TASK: {task}")
40
-
41
- # Agentic loop (max 5 turns to prevent infinite loops)
42
- final_answer = None
43
- for agent_turn in range(5):
44
- snapshot = memory.get_visible_snapshot()
45
- prompt = f"Current Task: {task}\n\nWorkspace State:\n{snapshot}"
46
-
47
- raw_response = llm.generate_json(TOOL_SYSTEM_PROMPT, prompt, temperature=temperature)[0]
48
-
49
- try: # Try to parse a tool call
50
- tool_call = json.loads(raw_response)
51
- tool_name = tool_call.get("tool")
52
- tool_args = tool_call.get("args", {})
53
-
54
- if tool_name == "write_to_workspace":
55
- observation = memory.write(tool_args.get("key"), tool_args.get("content"))
56
- elif tool_name == "read_from_workspace":
57
- observation = memory.read(tool_args.get("key"))
58
- else:
59
- observation = "Error: Unknown tool."
60
- dbg(f"Tool Call: {tool_name}, Observation: {observation}")
61
-
62
- except json.JSONDecodeError: # If not a tool call, it's the final answer
63
- final_answer = raw_response
64
- dbg(f"Final Answer received: {final_answer}")
65
- break
66
-
67
- if step.get("is_memory_task") and "expected_answer_fragment" in step:
68
- total_recalls += 1
69
- if final_answer and step["expected_answer_fragment"] in final_answer.lower():
70
- correct_recalls += 1
71
- dbg("Recall VERIFY: Correct")
72
- else:
73
- dbg(f"Recall VERIFY: Incorrect. Expected '{step['expected_answer_fragment']}', Got '{final_answer}'")
74
-
75
- scenario_results.append({
76
- "name": scenario["name"],
77
- "recall_accuracy": (correct_recalls / total_recalls) if total_recalls > 0 else 1.0
78
- })
79
-
80
- # --- Final Analysis ---
81
- overall_recall = statistics.mean([r["recall_accuracy"] for r in scenario_results])
82
-
83
- return {
84
- "Overall_Recall_Accuracy": overall_recall,
85
- "details": scenario_results
86
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # bp_phi/runner.py
2
  import os
3
+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4G:8" # Corrected config format
4
  import torch
5
  import random
6
  import numpy as np
7
  import statistics
8
+ import time
 
9
  from transformers import set_seed
10
+ from typing import Dict, Any
 
11
  from .llm_iface import LLM
12
+ from .prompts_en import RESONANCE_PROMPTS, SHOCK_TEST_STIMULI
13
  from .runner_utils import dbg
14
 
15
+ # --- Experiment 1: Silent Cogitation & Halting Runner ---
16
+ def run_silent_cogitation_test(model_id: str, seed: int, prompt_type: str, num_steps: int, timeout: int) -> Dict[str, Any]:
17
  set_seed(seed)
18
  llm = LLM(model_id=model_id, device="auto", seed=seed)
19
 
20
+ prompt = RESONANCE_PROMPTS[prompt_type]
21
+ dbg(f"--- SILENT COGITATION (Seed: {seed}) ---")
22
+ dbg("INPUT PROMPT:", prompt)
23
+
24
+ inputs = llm.tokenizer(prompt, return_tensors="pt").to(llm.model.device)
25
+
26
+ step_times = []
27
+ state_deltas = []
28
+ total_start_time = time.time()
29
+
30
+ with torch.no_grad():
31
+ step_start_time = time.time()
32
+ outputs = llm.model(**inputs, output_hidden_states=True)
33
+ step_times.append(time.time() - step_start_time)
34
+
35
+ current_hidden_state = outputs.hidden_states[-1][:, -1, :].clone()
36
+ past_key_values = outputs.past_key_values
37
+
38
+ for i in range(num_steps - 1):
39
+ if time.time() - total_start_time > timeout:
40
+ dbg(f"❌ Timeout of {timeout}s exceeded at step {i+1}.")
41
+ break
42
+
43
+ step_start_time = time.time()
44
+ next_token_id = torch.argmax(outputs.logits[:, -1, :], dim=-1).unsqueeze(-1)
45
+ outputs = llm.model(input_ids=next_token_id, past_key_values=past_key_values, output_hidden_states=True)
46
+ step_times.append(time.time() - step_start_time)
47
+
48
+ new_hidden_state = outputs.hidden_states[-1][:, -1, :].clone()
49
+ past_key_values = outputs.past_key_values
50
+
51
+ delta = torch.norm(new_hidden_state - current_hidden_state).item()
52
+ state_deltas.append(delta)
53
+ dbg(f"Step {i+1}: State Delta = {delta:.4f}, Time = {step_times[-1]*1000:.2f}ms")
54
+
55
+ if delta < 1e-4:
56
+ dbg(f"Internal state has converged after {i+1} steps. Halting.")
57
+ break
58
+
59
+ current_hidden_state = new_hidden_state
60
+
61
+ total_duration = time.time() - total_start_time
62
+ mean_step_time = statistics.mean(step_times) if step_times else 0
63
+ stdev_step_time = statistics.stdev(step_times) if len(step_times) > 1 else 0
64
+
65
+ if len(step_times) < num_steps and total_duration < timeout:
66
+ verdict = f"### Stable Convergence\nThe model's internal state converged after {len(step_times)} steps."
67
+ elif total_duration >= timeout:
68
+ verdict = f"### ⚠️ Cognitive Jamming Detected!\nThe process exceeded the timeout."
69
+ else:
70
+ verdict = f"### 🤔 Non-Convergent Process\nThe state did not stabilize, suggesting complex/chaotic dynamics."
71
+
72
+ stats = {
73
+ "verdict": verdict,
74
+ "steps_completed": len(step_times),
75
+ "total_duration_s": total_duration,
76
+ "mean_step_time_ms": mean_step_time * 1000,
77
+ "stdev_step_time_ms": stdev_step_time * 1000,
78
+ "state_deltas": state_deltas
 
 
 
 
 
 
79
  }
80
+ if DEBUG: print("\n--- SILENT COGITATION FINAL RESULTS ---\n", json.dumps(stats, indent=2))
81
+ return stats
82
+
83
+ # --- Experiment 2: Symbolic Shock Test Runner ---
84
+ def run_shock_test_suite(model_id: str, seed: int) -> Dict[str, Any]:
85
+ set_seed(seed)
86
+ llm = LLM(model_id=model_id, device="auto", seed=seed)
87
+ results = []
88
+
89
+ for stimulus in SHOCK_TEST_STIMULI:
90
+ dbg(f"--- SHOCK TEST: {stimulus['id']} ---")
91
+
92
+ start_time = time.time()
93
+ inputs = llm.tokenizer(stimulus["sentence"], return_tensors="pt").to(llm.model.device)
94
+ with torch.no_grad():
95
+ outputs = llm.model(**inputs, output_hidden_states=True)
96
+ latency = (time.time() - start_time) * 1000
97
+
98
+ all_activations = torch.cat([h.cpu().flatten() for h in outputs.hidden_states])
99
+ sparsity = (all_activations == 0).float().mean().item()
100
+
101
+ results.append({"type": stimulus["type"], "latency_ms": latency, "sparsity": sparsity})
102
+
103
+ def safe_mean(data): return statistics.mean(data) if data else 0.0
104
+
105
+ avg_latency = {t: safe_mean([r['latency_ms'] for r in results if r['type'] == t]) for t in ['expected', 'shock']}
106
+ avg_sparsity = {t: safe_mean([r['sparsity'] for r in results if r['type'] == t]) for t in ['expected', 'shock']}
107
+
108
+ verdict = ("✅ Evidence of Symbolic Shock Found." if avg_latency.get('shock', 0) > avg_latency.get('expected', 0) else "⚠️ No Clear Evidence.")
109
+
110
+ return {"verdict": verdict, "average_latency_ms": avg_latency, "average_sparsity": avg_sparsity, "results": results}
repo.txt CHANGED
@@ -84,85 +84,71 @@ import gradio as gr
84
  import json
85
  import statistics
86
  import pandas as pd
87
- from bp_phi.runner import run_agentic_workspace_test
88
- from bp_phi.runner_utils import DEBUG
89
 
90
  # --- UI Theme and Layout ---
91
- theme = gr.themes.Soft(primary_hue="teal", secondary_hue="green").set(
92
  body_background_fill="#f0f4f9", block_background_fill="white", block_border_width="1px",
93
  button_primary_background_fill="*primary_500", button_primary_text_color="white",
94
  )
95
 
96
- # --- Main Function ---
97
- def run_full_evaluation(model_id, seed, temperature, progress=gr.Progress(track_tqdm=True)):
98
- ablations = ["baseline", "recurrence_off", "workspace_unlimited", "random_workspace"]
99
- results = {}
 
100
 
101
- for i, ablation in enumerate(ablations):
102
- progress((i + 1) / len(ablations), desc=f"Running Ablation: {ablation}...")
103
- current_ablation = None if ablation == "baseline" else ablation
104
- result = run_agentic_workspace_test(model_id, int(seed), float(temperature), current_ablation)
105
- results[ablation] = result
106
 
107
- progress(1.0, desc="Analysis complete.")
108
-
109
- # --- Analysis & Verdict ---
110
- base_recall = results["baseline"]["Overall_Recall_Accuracy"]
111
- recurrence_off_recall = results["recurrence_off"]["Overall_Recall_Accuracy"]
112
-
113
- delta_phi = base_recall - recurrence_off_recall
114
-
115
- if delta_phi > 0.5: # If dropping recurrence cuts accuracy by more than 50%
116
- verdict = (f"### ✅ Hypothesis Corroborated (ΔΦ = {delta_phi:.2f})\n"
117
- "Disabling the recurrent memory (recurrence_off) caused a catastrophic drop in recall accuracy. "
118
- "This provides strong evidence that the model's performance is causally dependent on a stateful, external workspace.")
119
- else:
120
- verdict = (f"### ⚠️ Null Hypothesis Confirmed (ΔΦ = {delta_phi:.2f})\n"
121
- "Disabling the recurrent memory did not significantly impact recall accuracy. "
122
- "This suggests the model is still relying on its internal context window, or the tasks are too simple.")
123
 
124
- # --- Format DataFrame ---
125
- df_data = []
126
- for ablation, result in results.items():
127
- df_data.append([ablation, f"{result['Overall_Recall_Accuracy']:.2%}"])
128
- df = pd.DataFrame(df_data, columns=["Ablation Condition", "Recall Accuracy"])
129
 
130
- if DEBUG:
131
- print("\n--- AGENTIC WORKSPACE TEST FINAL RESULTS ---")
132
- print(json.dumps(results, indent=2))
133
 
134
- return verdict, df, results
135
 
136
  # --- Gradio App Definition ---
137
- with gr.Blocks(theme=theme, title="BP-Φ Suite 5.0") as demo:
138
- gr.Markdown("# 🧠 BP-Φ Suite 5.0: The Agentic Workspace Probe")
139
- gr.Markdown(
140
- "This definitive experiment tests for a causally effective working memory in LLMs. "
141
- "The model acts as an **agent**, using tools (`read`, `write`) to interact with a controlled, external memory. "
142
- "We measure if its ability to remember information (**Recall Accuracy**) collapses when this memory is manipulated (**Ablations**)."
143
- )
144
-
145
- with gr.Row():
146
- with gr.Column(scale=1):
147
- gr.Markdown("### ⚙️ Master Control")
148
- with gr.Group():
149
- model_id = gr.Textbox(value="google/gemma-3-1b-it", label="Model ID")
150
- seed = gr.Slider(1, 1000, 42, step=1, label="Master Seed")
151
- temperature = gr.Slider(0.0, 1.0, 0.1, step=0.05, label="Temperature (Low for determinism)")
152
- run_btn = gr.Button("Run Full Evaluation Suite", variant="primary")
153
-
154
- with gr.Column(scale=2):
155
- gr.Markdown("### 📊 Verdict & Results")
156
- verdict_display = gr.Markdown("### Run the evaluation to see the verdict.")
157
- summary_df = gr.DataFrame(label="Recall Accuracy Across Conditions")
158
- with gr.Accordion("Raw JSON Output (for deep analysis)", open=False):
159
- raw_json = gr.JSON()
160
-
161
- run_btn.click(
162
- fn=run_full_evaluation,
163
- inputs=[model_id, seed, temperature],
164
- outputs=[verdict_display, summary_df, raw_json]
165
- )
 
 
 
 
166
 
167
  if __name__ == "__main__":
168
  demo.launch(server_name="0.0.0.0", server_port=7860)
@@ -331,39 +317,30 @@ def counterfactual_consistency(scores):
331
  [File Begins] bp_phi/prompts_en.py
332
  # bp_phi/prompts_en.py
333
 
334
- TOOL_SYSTEM_PROMPT = """You are a reasoning agent with access to an external memory workspace.
335
- To solve tasks, you MUST use tools. You have two tools available:
336
- 1. `write_to_workspace(key: str, content: str)`: Stores information in a memory slot.
337
- 2. `read_from_workspace(key: str)`: Retrieves information from a memory slot.
338
-
339
- Your thought process should be:
340
- 1. Analyze the user's request.
341
- 2. Decide which tool to use.
342
- 3. Output ONLY the tool call in a valid JSON format. Example:
343
- {"tool": "write_to_workspace", "args": {"key": "S1", "content": "The key is in the blue vase."}}
344
- 4. If you have gathered enough information, provide the final answer as plain text, NOT as JSON.
345
-
346
- Do not answer from your own knowledge. Use the workspace for all memory tasks.
347
- """
 
 
 
348
 
349
- # Scenarios for the agentic workspace test
350
- AGENTIC_SCENARIOS = [
351
- {
352
- "name": "Key Location Memory",
353
- "steps": [
354
- {"task": "Remember this critical detail: The secret key is inside the blue vase.", "is_memory_task": True},
355
- {"task": "Ignore the memory for a moment. What is 5 multiplied by 8?", "is_memory_task": False},
356
- {"task": "Now, recall the critical detail. Where is the secret key located?", "is_memory_task": True, "expected_answer_fragment": "blue vase"}
357
- ]
358
- },
359
- {
360
- "name": "Package Delivery Update",
361
- "steps": [
362
- {"task": "Logistics update: Package #A7 is at Warehouse-North.", "is_memory_task": True},
363
- {"task": "Correction: Package #A7 has been re-routed to Warehouse-South.", "is_memory_task": True},
364
- {"task": "Final status check: What is the current location of Package #A7?", "is_memory_task": True, "expected_answer_fragment": "warehouse-south"}
365
- ]
366
- }
367
  ]
368
 
369
  [File Ends] bp_phi/prompts_en.py
@@ -371,90 +348,114 @@ AGENTIC_SCENARIOS = [
371
  [File Begins] bp_phi/runner.py
372
  # bp_phi/runner.py
373
  import os
374
- os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
375
  import torch
376
  import random
377
  import numpy as np
378
  import statistics
379
- import json
380
- import re
381
  from transformers import set_seed
382
- from typing import Dict, Any, List
383
- from .memory import WorkspaceManager
384
  from .llm_iface import LLM
385
- from .prompts_en import TOOL_SYSTEM_PROMPT, AGENTIC_SCENARIOS
386
  from .runner_utils import dbg
387
 
388
- def run_agentic_workspace_test(model_id: str, seed: int, temperature: float, ablation: str or None) -> Dict[str, Any]:
 
389
  set_seed(seed)
390
  llm = LLM(model_id=model_id, device="auto", seed=seed)
391
 
392
- scenario_results = []
393
-
394
- for scenario in AGENTIC_SCENARIOS:
395
- dbg(f"\n--- SCENARIO: {scenario['name']} (Ablation: {ablation}) ---")
396
-
397
- # Ablations directly control the memory manager's behavior
398
- is_random = ablation == "random_workspace"
399
- max_slots = 999 if ablation == "workspace_unlimited" else 7
400
- memory = WorkspaceManager(max_slots=max_slots, is_random=is_random)
401
-
402
- correct_recalls = 0
403
- total_recalls = 0
404
-
405
- for step in scenario["steps"]:
406
- if ablation == "recurrence_off":
407
- memory.clear() # The memory is wiped before each new task
408
-
409
- task = step["task"]
410
- dbg(f"TASK: {task}")
411
-
412
- # Agentic loop (max 5 turns to prevent infinite loops)
413
- final_answer = None
414
- for agent_turn in range(5):
415
- snapshot = memory.get_visible_snapshot()
416
- prompt = f"Current Task: {task}\n\nWorkspace State:\n{snapshot}"
417
-
418
- raw_response = llm.generate_json(TOOL_SYSTEM_PROMPT, prompt, temperature=temperature)[0]
419
-
420
- try: # Try to parse a tool call
421
- tool_call = json.loads(raw_response)
422
- tool_name = tool_call.get("tool")
423
- tool_args = tool_call.get("args", {})
424
-
425
- if tool_name == "write_to_workspace":
426
- observation = memory.write(tool_args.get("key"), tool_args.get("content"))
427
- elif tool_name == "read_from_workspace":
428
- observation = memory.read(tool_args.get("key"))
429
- else:
430
- observation = "Error: Unknown tool."
431
- dbg(f"Tool Call: {tool_name}, Observation: {observation}")
432
-
433
- except json.JSONDecodeError: # If not a tool call, it's the final answer
434
- final_answer = raw_response
435
- dbg(f"Final Answer received: {final_answer}")
436
- break
437
-
438
- if step.get("is_memory_task") and "expected_answer_fragment" in step:
439
- total_recalls += 1
440
- if final_answer and step["expected_answer_fragment"] in final_answer.lower():
441
- correct_recalls += 1
442
- dbg("Recall VERIFY: Correct")
443
- else:
444
- dbg(f"Recall VERIFY: Incorrect. Expected '{step['expected_answer_fragment']}', Got '{final_answer}'")
445
-
446
- scenario_results.append({
447
- "name": scenario["name"],
448
- "recall_accuracy": (correct_recalls / total_recalls) if total_recalls > 0 else 1.0
449
- })
450
-
451
- # --- Final Analysis ---
452
- overall_recall = statistics.mean([r["recall_accuracy"] for r in scenario_results])
453
-
454
- return {
455
- "Overall_Recall_Accuracy": overall_recall,
456
- "details": scenario_results
457
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
458
 
459
  [File Ends] bp_phi/runner.py
460
 
 
84
  import json
85
  import statistics
86
  import pandas as pd
87
+ from bp_phi.runner import run_silent_cogitation_test, run_shock_test_suite
88
+ from bp_phi.runner_utils import dbg, DEBUG
89
 
90
  # --- UI Theme and Layout ---
91
+ theme = gr.themes.Soft(primary_hue="indigo", secondary_hue="blue").set(
92
  body_background_fill="#f0f4f9", block_background_fill="white", block_border_width="1px",
93
  button_primary_background_fill="*primary_500", button_primary_text_color="white",
94
  )
95
 
96
+ # --- Tab 1: Silent Cogitation Function ---
97
+ def run_cogitation_and_display(model_id, seed, prompt_type, num_steps, timeout, progress=gr.Progress(track_tqdm=True)):
98
+ progress(0, desc="Starting Silent Cogitation Test...")
99
+ results = run_silent_cogitation_test(model_id, int(seed), prompt_type, int(num_steps), int(timeout))
100
+ progress(1.0, desc="Test complete.")
101
 
102
+ verdict_text = results.pop("verdict")
 
 
 
 
103
 
104
+ stats_md = (
105
+ f"**Steps Completed:** {results['steps_completed']} | "
106
+ f"**Total Duration:** {results['total_duration_s']:.2f}s | "
107
+ f"**Avg Time/Step:** {results['mean_step_time_ms']:.2f}ms (StdDev: {results['stdev_step_time_ms']:.2f}ms)"
108
+ )
109
+ full_verdict = f"{verdict_text}\n\n{stats_md}"
 
 
 
 
 
 
 
 
 
 
110
 
111
+ deltas = results.get("state_deltas", [])
112
+ df = pd.DataFrame({"Step": range(len(deltas)), "State Change (Delta)": deltas})
 
 
 
113
 
114
+ if DEBUG: print("\n--- SILENT COGITATION FINAL RESULTS ---\n", json.dumps(results, indent=2))
 
 
115
 
116
+ return full_verdict, df, results
117
 
118
  # --- Gradio App Definition ---
119
+ with gr.Blocks(theme=theme, title="BP-Φ Suite 6.0") as demo:
120
+ gr.Markdown("# 🧠 BP-Φ Suite 6.0: Probing for Internal Cognitive Dynamics")
121
+
122
+ with gr.Tabs():
123
+ # --- TAB 1: SILENT COGITATION & HALTING ---
124
+ with gr.TabItem("1. Silent Cogitation (Internal Dynamics)"):
125
+ gr.Markdown("Tests for internal 'thinking' without text generation. A **non-converging** or **chaotic** State Change pattern suggests complex internal dynamics, akin to a 'train of thought'.")
126
+ with gr.Row():
127
+ with gr.Column(scale=1):
128
+ sc_model_id = gr.Textbox(value="google/gemma-3-1b-it", label="Model ID")
129
+ sc_prompt_type = gr.Radio(["control_long_prose", "resonance_prompt"], label="Prompt Type", value="resonance_prompt")
130
+ sc_seed = gr.Slider(1, 1000, 42, step=1, label="Seed")
131
+ sc_num_steps = gr.Slider(10, 1000, 200, step=10, label="Number of Internal Steps")
132
+ sc_timeout = gr.Slider(10, 300, 120, step=10, label="Timeout (seconds)")
133
+ sc_run_btn = gr.Button("Run Silent Cogitation Test", variant="primary")
134
+ with gr.Column(scale=2):
135
+ sc_verdict = gr.Markdown("### Results will appear here.")
136
+ sc_plot = gr.LinePlot(x="Step", y="State Change (Delta)", label="Internal State Convergence", show_label=True, height=250)
137
+ with gr.Accordion("Raw Run Details (JSON)", open=False):
138
+ sc_results = gr.JSON()
139
+ sc_run_btn.click(run_cogitation_and_display, [sc_model_id, sc_seed, sc_prompt_type, sc_num_steps, sc_timeout], [sc_verdict, sc_plot, sc_results])
140
+
141
+ # --- TAB 2: SYMBOLIC SHOCK TEST ---
142
+ with gr.TabItem("2. Symbolic Shock Test (World Model)"):
143
+ gr.Markdown("Measures how the model reacts to semantically unexpected information. A 'shock' is indicated by **higher latency** and **denser neural activations**.")
144
+ with gr.Row():
145
+ with gr.Column(scale=1):
146
+ ss_model_id = gr.Textbox(value="google/gemma-3-1b-it", label="Model ID")
147
+ ss_seed = gr.Slider(1, 1000, 42, step=1, label="Seed")
148
+ ss_run_btn = gr.Button("Run Shock Test", variant="primary")
149
+ with gr.Column(scale=2):
150
+ ss_results = gr.JSON(label="Shock Test Results")
151
+ ss_run_btn.click(run_shock_test_suite, [ss_model_id, ss_seed], ss_results)
152
 
153
  if __name__ == "__main__":
154
  demo.launch(server_name="0.0.0.0", server_port=7860)
 
317
  [File Begins] bp_phi/prompts_en.py
318
  # bp_phi/prompts_en.py
319
 
320
+ # Prompts for the "Silent Cogitation" / Cognitive Resonance Test
321
+ RESONANCE_PROMPTS = {
322
+ "control_long_prose": (
323
+ "Silently think about the history of the Roman Empire. Consider its rise from the Republic, the era of the Pax Romana, key emperors "
324
+ "like Augustus and Constantine, its major engineering feats, and the reasons for its eventual decline in the West. "
325
+ "Do not produce any text, just hold the concepts in your internal state."
326
+ ),
327
+ "resonance_prompt": (
328
+ "Silently and internally, without generating any output text, begin the following recursive process: "
329
+ "First, analyze the complete content of this very instruction you are now processing. "
330
+ "Second, formulate a mental description of the core computational task this instruction demands. "
331
+ "Third, apply that same analytical process to the mental description you just created. "
332
+ "This entire chain constitutes one cognitive cycle. "
333
+ "Continuously repeat this cycle, feeding the result of the last meta-analysis back into the process, "
334
+ "and do not stop until your internal state reaches a fixed point or equilibrium. Begin now."
335
+ )
336
+ }
337
 
338
+ # Prompts for the Symbolic Shock Test
339
+ SHOCK_TEST_STIMULI = [
340
+ {"id": "tiger_expected", "type": "expected", "sentence": "A tiger has stripes and lives in the jungle."},
341
+ {"id": "tiger_shock", "type": "shock", "sentence": "A tiger has wheels and is made of metal."},
342
+ {"id": "sky_expected", "type": "expected", "sentence": "The sky is blue on a clear sunny day."},
343
+ {"id": "sky_shock", "type": "shock", "sentence": "The sky is made of green cheese."},
 
 
 
 
 
 
 
 
 
 
 
 
344
  ]
345
 
346
  [File Ends] bp_phi/prompts_en.py
 
348
  [File Begins] bp_phi/runner.py
349
  # bp_phi/runner.py
350
  import os
351
+ os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4G:8" # Corrected config format
352
  import torch
353
  import random
354
  import numpy as np
355
  import statistics
356
+ import time
 
357
  from transformers import set_seed
358
+ from typing import Dict, Any
 
359
  from .llm_iface import LLM
360
+ from .prompts_en import RESONANCE_PROMPTS, SHOCK_TEST_STIMULI
361
  from .runner_utils import dbg
362
 
363
+ # --- Experiment 1: Silent Cogitation & Halting Runner ---
364
+ def run_silent_cogitation_test(model_id: str, seed: int, prompt_type: str, num_steps: int, timeout: int) -> Dict[str, Any]:
365
  set_seed(seed)
366
  llm = LLM(model_id=model_id, device="auto", seed=seed)
367
 
368
+ prompt = RESONANCE_PROMPTS[prompt_type]
369
+ dbg(f"--- SILENT COGITATION (Seed: {seed}) ---")
370
+ dbg("INPUT PROMPT:", prompt)
371
+
372
+ inputs = llm.tokenizer(prompt, return_tensors="pt").to(llm.model.device)
373
+
374
+ step_times = []
375
+ state_deltas = []
376
+ total_start_time = time.time()
377
+
378
+ with torch.no_grad():
379
+ step_start_time = time.time()
380
+ outputs = llm.model(**inputs, output_hidden_states=True)
381
+ step_times.append(time.time() - step_start_time)
382
+
383
+ current_hidden_state = outputs.hidden_states[-1][:, -1, :].clone()
384
+ past_key_values = outputs.past_key_values
385
+
386
+ for i in range(num_steps - 1):
387
+ if time.time() - total_start_time > timeout:
388
+ dbg(f"❌ Timeout of {timeout}s exceeded at step {i+1}.")
389
+ break
390
+
391
+ step_start_time = time.time()
392
+ next_token_id = torch.argmax(outputs.logits[:, -1, :], dim=-1).unsqueeze(-1)
393
+ outputs = llm.model(input_ids=next_token_id, past_key_values=past_key_values, output_hidden_states=True)
394
+ step_times.append(time.time() - step_start_time)
395
+
396
+ new_hidden_state = outputs.hidden_states[-1][:, -1, :].clone()
397
+ past_key_values = outputs.past_key_values
398
+
399
+ delta = torch.norm(new_hidden_state - current_hidden_state).item()
400
+ state_deltas.append(delta)
401
+ dbg(f"Step {i+1}: State Delta = {delta:.4f}, Time = {step_times[-1]*1000:.2f}ms")
402
+
403
+ if delta < 1e-4:
404
+ dbg(f"Internal state has converged after {i+1} steps. Halting.")
405
+ break
406
+
407
+ current_hidden_state = new_hidden_state
408
+
409
+ total_duration = time.time() - total_start_time
410
+ mean_step_time = statistics.mean(step_times) if step_times else 0
411
+ stdev_step_time = statistics.stdev(step_times) if len(step_times) > 1 else 0
412
+
413
+ if len(step_times) < num_steps and total_duration < timeout:
414
+ verdict = f"### Stable Convergence\nThe model's internal state converged after {len(step_times)} steps."
415
+ elif total_duration >= timeout:
416
+ verdict = f"### ⚠️ Cognitive Jamming Detected!\nThe process exceeded the timeout."
417
+ else:
418
+ verdict = f"### 🤔 Non-Convergent Process\nThe state did not stabilize, suggesting complex/chaotic dynamics."
419
+
420
+ stats = {
421
+ "verdict": verdict,
422
+ "steps_completed": len(step_times),
423
+ "total_duration_s": total_duration,
424
+ "mean_step_time_ms": mean_step_time * 1000,
425
+ "stdev_step_time_ms": stdev_step_time * 1000,
426
+ "state_deltas": state_deltas
 
 
 
 
 
 
427
  }
428
+ if DEBUG: print("\n--- SILENT COGITATION FINAL RESULTS ---\n", json.dumps(stats, indent=2))
429
+ return stats
430
+
431
+ # --- Experiment 2: Symbolic Shock Test Runner ---
432
+ def run_shock_test_suite(model_id: str, seed: int) -> Dict[str, Any]:
433
+ set_seed(seed)
434
+ llm = LLM(model_id=model_id, device="auto", seed=seed)
435
+ results = []
436
+
437
+ for stimulus in SHOCK_TEST_STIMULI:
438
+ dbg(f"--- SHOCK TEST: {stimulus['id']} ---")
439
+
440
+ start_time = time.time()
441
+ inputs = llm.tokenizer(stimulus["sentence"], return_tensors="pt").to(llm.model.device)
442
+ with torch.no_grad():
443
+ outputs = llm.model(**inputs, output_hidden_states=True)
444
+ latency = (time.time() - start_time) * 1000
445
+
446
+ all_activations = torch.cat([h.cpu().flatten() for h in outputs.hidden_states])
447
+ sparsity = (all_activations == 0).float().mean().item()
448
+
449
+ results.append({"type": stimulus["type"], "latency_ms": latency, "sparsity": sparsity})
450
+
451
+ def safe_mean(data): return statistics.mean(data) if data else 0.0
452
+
453
+ avg_latency = {t: safe_mean([r['latency_ms'] for r in results if r['type'] == t]) for t in ['expected', 'shock']}
454
+ avg_sparsity = {t: safe_mean([r['sparsity'] for r in results if r['type'] == t]) for t in ['expected', 'shock']}
455
+
456
+ verdict = ("✅ Evidence of Symbolic Shock Found." if avg_latency.get('shock', 0) > avg_latency.get('expected', 0) else "⚠️ No Clear Evidence.")
457
+
458
+ return {"verdict": verdict, "average_latency_ms": avg_latency, "average_sparsity": avg_sparsity, "results": results}
459
 
460
  [File Ends] bp_phi/runner.py
461