Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -11,10 +11,10 @@ from model import SWCKModel, SeedParser, EntropyEstimator # Assuming model.py is
|
|
| 11 |
# --- Vocabulary and Tokenizer Setup ---
|
| 12 |
PAD_TOKEN_STR = "<pad>"; SOS_TOKEN_STR = "<sos>"; EOS_TOKEN_STR = "<eos>"; UNK_TOKEN_STR = "<unk>"
|
| 13 |
PAD_TOKEN = 0; SOS_TOKEN = 1; EOS_TOKEN = 2; UNK_TOKEN = 3
|
| 14 |
-
SEQ_LEN_APP = 64
|
| 15 |
|
| 16 |
# --- Model Configuration ---
|
| 17 |
-
VOCAB_SIZE_APP = 189
|
| 18 |
D_MODEL_APP = 64
|
| 19 |
N_HEADS_APP = 2
|
| 20 |
D_FF_APP = 128
|
|
@@ -38,7 +38,7 @@ This is a stream of consciousness, a digital mindscape.
|
|
| 38 |
The target is not just prediction, but a form of self-understanding, however metaphorical.
|
| 39 |
Let the adaptive blocks find their balance. Let the entropy guide the wiring.
|
| 40 |
A painter paints. A scientist explores. A writer writes. The machine... becomes.
|
| 41 |
-
"""
|
| 42 |
|
| 43 |
# Global model variables
|
| 44 |
swck_model_global = None
|
|
@@ -48,14 +48,13 @@ idx_to_word_global = None
|
|
| 48 |
device_global = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 49 |
model_load_status_global = "Model not loaded."
|
| 50 |
|
| 51 |
-
CHECKPOINT_FILENAME = "swck_model_conceptual_app.pth.tar"
|
| 52 |
|
| 53 |
-
# Loss Weights (should match train.py for consistency if loading that checkpoint)
|
| 54 |
MAIN_LOSS_WEIGHT_APP = 1.0
|
| 55 |
BLOCK_TARGET_ENTROPY_LOSS_WEIGHT_APP = 0.02
|
| 56 |
OVERALL_OUTPUT_ENTROPY_REG_WEIGHT_APP = 0.01
|
| 57 |
GATE_SPARSITY_LOSS_WEIGHT_APP = 0.001
|
| 58 |
-
WIRING_PHASE_EPOCHS_APP = 1
|
| 59 |
|
| 60 |
|
| 61 |
def build_vocab_from_corpus_text_app(corpus_text):
|
|
@@ -94,12 +93,11 @@ def initialize_or_load_model_app():
|
|
| 94 |
}
|
| 95 |
|
| 96 |
swck_model_global = SWCKModel(**model_args).to(device_global)
|
| 97 |
-
|
| 98 |
-
swck_model_global.debug_prints_enabled = True
|
| 99 |
if hasattr(swck_model_global, 'seed_parser'): swck_model_global.seed_parser.debug_prints_enabled = True
|
| 100 |
for i,block in enumerate(swck_model_global.adaptive_blocks):
|
| 101 |
-
block.debug_prints_enabled = True
|
| 102 |
-
print(f"App: Debug prints enabled for AdaptiveBlock {i}")
|
| 103 |
|
| 104 |
|
| 105 |
if os.path.exists(CHECKPOINT_FILENAME):
|
|
@@ -108,27 +106,29 @@ def initialize_or_load_model_app():
|
|
| 108 |
checkpoint = torch.load(CHECKPOINT_FILENAME, map_location=device_global)
|
| 109 |
swck_model_global.load_state_dict(checkpoint['model_state_dict'])
|
| 110 |
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
if 'optimizer_state_dict' in checkpoint: # Load optimizer state if you want to continue training
|
| 114 |
optimizer_global.load_state_dict(checkpoint['optimizer_state_dict'])
|
| 115 |
|
| 116 |
-
|
| 117 |
-
if 'word_to_idx' in checkpoint: # Overwrite with checkpoint vocab if present
|
| 118 |
loaded_w2i = checkpoint['word_to_idx']
|
| 119 |
-
|
|
|
|
| 120 |
word_to_idx_global = loaded_w2i
|
| 121 |
idx_to_word_global = {v: k for k,v in loaded_w2i.items()}
|
| 122 |
-
|
|
|
|
| 123 |
else:
|
| 124 |
-
print("App: Checkpoint vocab
|
|
|
|
|
|
|
|
|
|
| 125 |
|
| 126 |
model_load_status_global = f"Model loaded successfully from {CHECKPOINT_FILENAME}."
|
| 127 |
print(model_load_status_global)
|
| 128 |
except Exception as e:
|
| 129 |
print(f"App: Error loading model from checkpoint: {e}. Initializing new model.")
|
| 130 |
-
|
| 131 |
-
swck_model_global = SWCKModel(**model_args).to(device_global)
|
| 132 |
optimizer_global = optim.AdamW(swck_model_global.parameters(), lr=0.001)
|
| 133 |
model_load_status_global = "Error loading checkpoint. Using new (untrained) model."
|
| 134 |
else:
|
|
@@ -136,11 +136,10 @@ def initialize_or_load_model_app():
|
|
| 136 |
optimizer_global = optim.AdamW(swck_model_global.parameters(), lr=0.001)
|
| 137 |
model_load_status_global = "Initialized a new (untrained) model."
|
| 138 |
|
| 139 |
-
swck_model_global.eval()
|
| 140 |
return model_load_status_global
|
| 141 |
|
| 142 |
|
| 143 |
-
# --- Dataset for in-app training ---
|
| 144 |
class AppSWCKDataset(Dataset):
|
| 145 |
def __init__(self, text_corpus_str, w2i_map, seq_len, sos_id, eos_id, pad_id):
|
| 146 |
tokens = re.sub(r'\s+', ' ', text_corpus_str.lower()).strip().split()
|
|
@@ -149,9 +148,11 @@ class AppSWCKDataset(Dataset):
|
|
| 149 |
self.seq_len = seq_len
|
| 150 |
self.sos_id, self.eos_id, self.pad_id = sos_id, eos_id, pad_id
|
| 151 |
self.samples = []
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
|
|
|
|
|
|
| 155 |
self.samples.append((input_seq, target_seq))
|
| 156 |
print(f"AppSWCKDataset: Created {len(self.samples)} training samples for in-app training.")
|
| 157 |
|
|
@@ -166,7 +167,6 @@ def app_swck_collate_fn(batch):
|
|
| 166 |
padded_tgt = nn.utils.rnn.pad_sequence(tgt_list, batch_first=True, padding_value=PAD_TOKEN)
|
| 167 |
return padded_src, padded_tgt
|
| 168 |
|
| 169 |
-
# --- In-app Training Function (Simplified) ---
|
| 170 |
def run_short_training_session(num_epochs_app, batch_size_app, learning_rate_app, progress=gr.Progress(track_tqdm=True)):
|
| 171 |
global swck_model_global, optimizer_global, word_to_idx_global, model_load_status_global
|
| 172 |
|
|
@@ -176,56 +176,80 @@ def run_short_training_session(num_epochs_app, batch_size_app, learning_rate_app
|
|
| 176 |
print("\n--- App: Starting Short Training Session ---")
|
| 177 |
progress(0, desc="Preparing training data...")
|
| 178 |
|
| 179 |
-
# Use the extended text for training
|
| 180 |
training_corpus = SEED_PHRASE_APP + " " + EXTENDED_TEXT_FOR_TRAINING_APP
|
| 181 |
app_dataset = AppSWCKDataset(training_corpus, word_to_idx_global, SEQ_LEN_APP, SOS_TOKEN, EOS_TOKEN, PAD_TOKEN)
|
| 182 |
if not app_dataset.samples:
|
| 183 |
return "App Training Error: No samples created from the corpus."
|
| 184 |
|
| 185 |
-
app_dataloader = DataLoader(app_dataset, batch_size=batch_size_app, shuffle=True, collate_fn=app_swck_collate_fn)
|
| 186 |
|
| 187 |
-
# Re-initialize optimizer or update LR
|
| 188 |
if optimizer_global is None:
|
| 189 |
optimizer_global = optim.AdamW(swck_model_global.parameters(), lr=learning_rate_app)
|
| 190 |
-
else:
|
| 191 |
for param_group in optimizer_global.param_groups:
|
| 192 |
param_group['lr'] = learning_rate_app
|
| 193 |
|
| 194 |
criterion_main_app = nn.CrossEntropyLoss(ignore_index=PAD_TOKEN)
|
| 195 |
|
| 196 |
-
training_log_output = ""
|
| 197 |
-
swck_model_global.train()
|
| 198 |
|
| 199 |
-
for epoch in progress.tqdm(range(num_epochs_app), desc="Training Epochs"):
|
| 200 |
-
swck_model_global.set_wiring_phase(epoch < WIRING_PHASE_EPOCHS_APP)
|
| 201 |
epoch_loss = 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
for batch_idx, (src_batch, tgt_batch) in enumerate(app_dataloader):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
src_batch, tgt_batch = src_batch.to(device_global), tgt_batch.to(device_global)
|
| 204 |
-
decoder_input_tokens = src_batch
|
| 205 |
-
gold_standard_for_loss = tgt_batch
|
|
|
|
| 206 |
src_key_padding_mask = (decoder_input_tokens == PAD_TOKEN)
|
| 207 |
|
| 208 |
optimizer_global.zero_grad()
|
| 209 |
logits, entropy_report = swck_model_global(decoder_input_tokens, src_key_padding_mask=src_key_padding_mask)
|
| 210 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
block_entropy_loss = torch.tensor(0.0, device=device_global)
|
| 213 |
if entropy_report["block_output_entropies"]:
|
| 214 |
-
for i,
|
| 215 |
-
|
| 216 |
-
block_entropy_loss += F.mse_loss(
|
| 217 |
-
if entropy_report["block_output_entropies"]:
|
| 218 |
block_entropy_loss = block_entropy_loss / len(entropy_report["block_output_entropies"])
|
| 219 |
|
| 220 |
overall_entropy_loss = entropy_report["overall_output_entropy"]
|
| 221 |
gate_sparsity_loss = torch.tensor(0.0, device=device_global)
|
| 222 |
if entropy_report["block_gate_weights"]:
|
| 223 |
-
for
|
| 224 |
-
gate_sparsity_loss += torch.mean(
|
| 225 |
-
if entropy_report["block_gate_weights"]:
|
| 226 |
gate_sparsity_loss = - (gate_sparsity_loss / len(entropy_report["block_gate_weights"]))
|
| 227 |
|
| 228 |
-
|
| 229 |
combined_loss = (MAIN_LOSS_WEIGHT_APP * main_loss +
|
| 230 |
BLOCK_TARGET_ENTROPY_LOSS_WEIGHT_APP * block_entropy_loss +
|
| 231 |
OVERALL_OUTPUT_ENTROPY_REG_WEIGHT_APP * overall_entropy_loss +
|
|
@@ -236,33 +260,38 @@ def run_short_training_session(num_epochs_app, batch_size_app, learning_rate_app
|
|
| 236 |
optimizer_global.step()
|
| 237 |
epoch_loss += combined_loss.item()
|
| 238 |
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
print(log_line)
|
| 242 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
|
| 244 |
-
avg_epoch_loss = epoch_loss / len(app_dataloader)
|
| 245 |
epoch_summary = f"Epoch {epoch+1}/{num_epochs_app} - Avg Loss: {avg_epoch_loss:.4f}\n"
|
| 246 |
print(epoch_summary)
|
| 247 |
training_log_output += epoch_summary
|
| 248 |
-
# progress.update() # Not needed with track_tqdm
|
| 249 |
-
|
| 250 |
-
swck_model_global.eval() # Set back to eval mode
|
| 251 |
|
| 252 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
try:
|
| 254 |
torch.save({
|
| 255 |
'model_state_dict': swck_model_global.state_dict(),
|
| 256 |
-
'optimizer_state_dict': optimizer_global.state_dict(),
|
| 257 |
'word_to_idx': word_to_idx_global,
|
| 258 |
'idx_to_word': idx_to_word_global,
|
| 259 |
-
|
| 260 |
-
'model_hyperparameters': { # Example of saving model construction args
|
| 261 |
'vocab_size': VOCAB_SIZE_APP, 'd_model': D_MODEL_APP, 'n_heads': N_HEADS_APP,
|
| 262 |
'd_ff': D_FF_APP, 'num_adaptive_blocks': NUM_ADAPTIVE_BLOCKS_APP, 'dropout': DROPOUT_APP
|
| 263 |
}
|
| 264 |
}, CHECKPOINT_FILENAME)
|
| 265 |
-
save_msg = f"Training finished. Model checkpoint saved to {CHECKPOINT_FILENAME} in Space."
|
| 266 |
print(save_msg)
|
| 267 |
training_log_output += save_msg
|
| 268 |
model_load_status_global = f"Model trained in-app & saved. Last status: {save_msg}"
|
|
@@ -274,14 +303,16 @@ def run_short_training_session(num_epochs_app, batch_size_app, learning_rate_app
|
|
| 274 |
|
| 275 |
return training_log_output
|
| 276 |
|
| 277 |
-
# --- Text Generation Function (adapted from train.py) ---
|
| 278 |
def generate_text_for_app(prompt_str, max_len_gen, temperature_gen):
|
| 279 |
-
global model_load_status_global
|
| 280 |
if swck_model_global is None or word_to_idx_global is None or idx_to_word_global is None:
|
| 281 |
return "Model not loaded. Please check server logs or try training.", "Model not available."
|
| 282 |
|
| 283 |
swck_model_global.eval()
|
| 284 |
swck_model_global.set_wiring_phase(False)
|
|
|
|
|
|
|
|
|
|
| 285 |
|
| 286 |
print(f"App: Generating for prompt: '{prompt_str}', max_len: {max_len_gen}, temp: {temperature_gen}")
|
| 287 |
|
|
@@ -290,8 +321,12 @@ def generate_text_for_app(prompt_str, max_len_gen, temperature_gen):
|
|
| 290 |
debug_info_lines = [f"Prompt tokens: {generated_ids_app}"]
|
| 291 |
|
| 292 |
with torch.no_grad():
|
| 293 |
-
for i in range(max_len_gen):
|
| 294 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
input_tensor = torch.tensor([current_context_ids], dtype=torch.long).to(device_global)
|
| 296 |
padding_mask = (input_tensor == PAD_TOKEN)
|
| 297 |
|
|
@@ -302,9 +337,9 @@ def generate_text_for_app(prompt_str, max_len_gen, temperature_gen):
|
|
| 302 |
next_token_id = torch.argmax(next_token_logits).item()
|
| 303 |
else:
|
| 304 |
probs = F.softmax(next_token_logits / temperature_gen, dim=-1)
|
| 305 |
-
if probs.isnan().any() or probs.isinf().any() or torch.sum(probs).item() < 1e-9 :
|
| 306 |
print(f"Warning: Invalid probabilities at step {i}. Using uniform.")
|
| 307 |
-
probs = torch.ones_like(next_token_logits) / next_token_logits.size(-1)
|
| 308 |
next_token_id = torch.multinomial(probs, 1).item()
|
| 309 |
|
| 310 |
if next_token_id == EOS_TOKEN:
|
|
@@ -315,12 +350,15 @@ def generate_text_for_app(prompt_str, max_len_gen, temperature_gen):
|
|
| 315 |
if i < 10 :
|
| 316 |
current_word = idx_to_word_global.get(next_token_id, UNK_TOKEN_STR)
|
| 317 |
overall_ent = entropy_report_infer['overall_output_entropy'].item()
|
| 318 |
-
if entropy_report_infer['block_output_entropies']
|
| 319 |
b0_ent = entropy_report_infer['block_output_entropies'][0].item()
|
| 320 |
-
|
| 321 |
-
|
|
|
|
|
|
|
|
|
|
| 322 |
else:
|
| 323 |
-
debug_info_lines.append(f"Gen {i+1}: '{current_word}', OvrlEnt={overall_ent:.3f}, No block entropy report.")
|
| 324 |
|
| 325 |
|
| 326 |
generated_text_list = [idx_to_word_global.get(idx, UNK_TOKEN_STR) for idx in generated_ids_app[1:]]
|
|
@@ -331,12 +369,14 @@ def generate_text_for_app(prompt_str, max_len_gen, temperature_gen):
|
|
| 331 |
final_text = re.sub(r'\s+', ' ', final_text).strip()
|
| 332 |
|
| 333 |
debug_output_str = "\n".join(debug_info_lines)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 334 |
return final_text, debug_output_str
|
| 335 |
|
| 336 |
# --- Gradio Interface ---
|
| 337 |
-
# Load model on app startup
|
| 338 |
-
initial_load_status = initialize_or_load_model_app()
|
| 339 |
-
|
| 340 |
|
| 341 |
with gr.Blocks(title="SWCK Conceptual Demo") as demo:
|
| 342 |
gr.Markdown(f"""
|
|
@@ -364,12 +404,18 @@ with gr.Blocks(title="SWCK Conceptual Demo") as demo:
|
|
| 364 |
with gr.Row():
|
| 365 |
train_epochs_slider = gr.Slider(minimum=1, maximum=5, value=1, step=1, label="Number of Training Epochs")
|
| 366 |
train_batch_size_slider = gr.Slider(minimum=1, maximum=8, value=2, step=1, label="Training Batch Size")
|
| 367 |
-
|
|
|
|
| 368 |
|
| 369 |
start_training_button = gr.Button("Start Short Training Session")
|
| 370 |
-
training_status_output = gr.Textbox(label="Training Log / Status:", lines=10, interactive=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 371 |
|
| 372 |
-
# Define actions
|
| 373 |
generate_button.click(
|
| 374 |
fn=generate_text_for_app,
|
| 375 |
inputs=[prompt_input, max_len_slider, temp_slider],
|
|
@@ -380,12 +426,11 @@ with gr.Blocks(title="SWCK Conceptual Demo") as demo:
|
|
| 380 |
fn=run_short_training_session,
|
| 381 |
inputs=[train_epochs_slider, train_batch_size_slider, train_lr_slider],
|
| 382 |
outputs=[training_status_output]
|
| 383 |
-
).then(fn=
|
| 384 |
-
|
| 385 |
-
# For simplicity, the training function itself prints to console and returns a string.
|
| 386 |
-
# A more robust status update would use gr.HTML or JS.
|
| 387 |
|
| 388 |
if __name__ == "__main__":
|
| 389 |
-
#
|
| 390 |
-
# On Spaces,
|
| 391 |
-
|
|
|
|
|
|
| 11 |
# --- Vocabulary and Tokenizer Setup ---
|
| 12 |
PAD_TOKEN_STR = "<pad>"; SOS_TOKEN_STR = "<sos>"; EOS_TOKEN_STR = "<eos>"; UNK_TOKEN_STR = "<unk>"
|
| 13 |
PAD_TOKEN = 0; SOS_TOKEN = 1; EOS_TOKEN = 2; UNK_TOKEN = 3
|
| 14 |
+
SEQ_LEN_APP = 64
|
| 15 |
|
| 16 |
# --- Model Configuration ---
|
| 17 |
+
VOCAB_SIZE_APP = 189
|
| 18 |
D_MODEL_APP = 64
|
| 19 |
N_HEADS_APP = 2
|
| 20 |
D_FF_APP = 128
|
|
|
|
| 38 |
The target is not just prediction, but a form of self-understanding, however metaphorical.
|
| 39 |
Let the adaptive blocks find their balance. Let the entropy guide the wiring.
|
| 40 |
A painter paints. A scientist explores. A writer writes. The machine... becomes.
|
| 41 |
+
"""
|
| 42 |
|
| 43 |
# Global model variables
|
| 44 |
swck_model_global = None
|
|
|
|
| 48 |
device_global = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 49 |
model_load_status_global = "Model not loaded."
|
| 50 |
|
| 51 |
+
CHECKPOINT_FILENAME = "swck_model_conceptual_app.pth.tar"
|
| 52 |
|
|
|
|
| 53 |
MAIN_LOSS_WEIGHT_APP = 1.0
|
| 54 |
BLOCK_TARGET_ENTROPY_LOSS_WEIGHT_APP = 0.02
|
| 55 |
OVERALL_OUTPUT_ENTROPY_REG_WEIGHT_APP = 0.01
|
| 56 |
GATE_SPARSITY_LOSS_WEIGHT_APP = 0.001
|
| 57 |
+
WIRING_PHASE_EPOCHS_APP = 1
|
| 58 |
|
| 59 |
|
| 60 |
def build_vocab_from_corpus_text_app(corpus_text):
|
|
|
|
| 93 |
}
|
| 94 |
|
| 95 |
swck_model_global = SWCKModel(**model_args).to(device_global)
|
| 96 |
+
swck_model_global.debug_prints_enabled = True # Top-level model debug
|
|
|
|
| 97 |
if hasattr(swck_model_global, 'seed_parser'): swck_model_global.seed_parser.debug_prints_enabled = True
|
| 98 |
for i,block in enumerate(swck_model_global.adaptive_blocks):
|
| 99 |
+
block.debug_prints_enabled = True # Block-level debug
|
| 100 |
+
# print(f"App: Debug prints explicitly enabled for AdaptiveBlock {i}")
|
| 101 |
|
| 102 |
|
| 103 |
if os.path.exists(CHECKPOINT_FILENAME):
|
|
|
|
| 106 |
checkpoint = torch.load(CHECKPOINT_FILENAME, map_location=device_global)
|
| 107 |
swck_model_global.load_state_dict(checkpoint['model_state_dict'])
|
| 108 |
|
| 109 |
+
optimizer_global = optim.AdamW(swck_model_global.parameters(), lr=0.001)
|
| 110 |
+
if 'optimizer_state_dict' in checkpoint:
|
|
|
|
| 111 |
optimizer_global.load_state_dict(checkpoint['optimizer_state_dict'])
|
| 112 |
|
| 113 |
+
if 'word_to_idx' in checkpoint:
|
|
|
|
| 114 |
loaded_w2i = checkpoint['word_to_idx']
|
| 115 |
+
# Basic check, could be more robust
|
| 116 |
+
if isinstance(loaded_w2i, dict) and len(loaded_w2i) > 4:
|
| 117 |
word_to_idx_global = loaded_w2i
|
| 118 |
idx_to_word_global = {v: k for k,v in loaded_w2i.items()}
|
| 119 |
+
VOCAB_SIZE_APP = len(word_to_idx_global) # Ensure vocab size reflects loaded
|
| 120 |
+
print(f"App: Overwrote vocab with checkpoint's vocab. New size: {VOCAB_SIZE_APP}")
|
| 121 |
else:
|
| 122 |
+
print("App: Checkpoint vocab seems invalid, using app's rebuilt vocab.")
|
| 123 |
+
else:
|
| 124 |
+
print("App: word_to_idx not in checkpoint, using app's rebuilt vocab.")
|
| 125 |
+
|
| 126 |
|
| 127 |
model_load_status_global = f"Model loaded successfully from {CHECKPOINT_FILENAME}."
|
| 128 |
print(model_load_status_global)
|
| 129 |
except Exception as e:
|
| 130 |
print(f"App: Error loading model from checkpoint: {e}. Initializing new model.")
|
| 131 |
+
swck_model_global = SWCKModel(**model_args).to(device_global) # Re-init
|
|
|
|
| 132 |
optimizer_global = optim.AdamW(swck_model_global.parameters(), lr=0.001)
|
| 133 |
model_load_status_global = "Error loading checkpoint. Using new (untrained) model."
|
| 134 |
else:
|
|
|
|
| 136 |
optimizer_global = optim.AdamW(swck_model_global.parameters(), lr=0.001)
|
| 137 |
model_load_status_global = "Initialized a new (untrained) model."
|
| 138 |
|
| 139 |
+
swck_model_global.eval()
|
| 140 |
return model_load_status_global
|
| 141 |
|
| 142 |
|
|
|
|
| 143 |
class AppSWCKDataset(Dataset):
|
| 144 |
def __init__(self, text_corpus_str, w2i_map, seq_len, sos_id, eos_id, pad_id):
|
| 145 |
tokens = re.sub(r'\s+', ' ', text_corpus_str.lower()).strip().split()
|
|
|
|
| 148 |
self.seq_len = seq_len
|
| 149 |
self.sos_id, self.eos_id, self.pad_id = sos_id, eos_id, pad_id
|
| 150 |
self.samples = []
|
| 151 |
+
# Create overlapping sequences for language modeling
|
| 152 |
+
# Ensure target is seq_len for consistency with input to model.
|
| 153 |
+
for i in range(len(token_ids) - seq_len -1): # -1 to ensure target has full seq_len
|
| 154 |
+
input_seq = [self.sos_id] + token_ids[i : i + seq_len] # length seq_len + 1
|
| 155 |
+
target_seq = token_ids[i + 1 : i + seq_len + 1] + [self.eos_id] # length seq_len + 1
|
| 156 |
self.samples.append((input_seq, target_seq))
|
| 157 |
print(f"AppSWCKDataset: Created {len(self.samples)} training samples for in-app training.")
|
| 158 |
|
|
|
|
| 167 |
padded_tgt = nn.utils.rnn.pad_sequence(tgt_list, batch_first=True, padding_value=PAD_TOKEN)
|
| 168 |
return padded_src, padded_tgt
|
| 169 |
|
|
|
|
| 170 |
def run_short_training_session(num_epochs_app, batch_size_app, learning_rate_app, progress=gr.Progress(track_tqdm=True)):
|
| 171 |
global swck_model_global, optimizer_global, word_to_idx_global, model_load_status_global
|
| 172 |
|
|
|
|
| 176 |
print("\n--- App: Starting Short Training Session ---")
|
| 177 |
progress(0, desc="Preparing training data...")
|
| 178 |
|
|
|
|
| 179 |
training_corpus = SEED_PHRASE_APP + " " + EXTENDED_TEXT_FOR_TRAINING_APP
|
| 180 |
app_dataset = AppSWCKDataset(training_corpus, word_to_idx_global, SEQ_LEN_APP, SOS_TOKEN, EOS_TOKEN, PAD_TOKEN)
|
| 181 |
if not app_dataset.samples:
|
| 182 |
return "App Training Error: No samples created from the corpus."
|
| 183 |
|
| 184 |
+
app_dataloader = DataLoader(app_dataset, batch_size=int(batch_size_app), shuffle=True, collate_fn=app_swck_collate_fn)
|
| 185 |
|
|
|
|
| 186 |
if optimizer_global is None:
|
| 187 |
optimizer_global = optim.AdamW(swck_model_global.parameters(), lr=learning_rate_app)
|
| 188 |
+
else:
|
| 189 |
for param_group in optimizer_global.param_groups:
|
| 190 |
param_group['lr'] = learning_rate_app
|
| 191 |
|
| 192 |
criterion_main_app = nn.CrossEntropyLoss(ignore_index=PAD_TOKEN)
|
| 193 |
|
| 194 |
+
training_log_output = f"Starting training for {num_epochs_app} epochs...\n"
|
| 195 |
+
swck_model_global.train()
|
| 196 |
|
| 197 |
+
for epoch in progress.tqdm(range(int(num_epochs_app)), desc="Training Epochs"):
|
| 198 |
+
swck_model_global.set_wiring_phase(epoch < WIRING_PHASE_EPOCHS_APP)
|
| 199 |
epoch_loss = 0.0
|
| 200 |
+
|
| 201 |
+
# Enable debug for first batch of first epoch
|
| 202 |
+
first_batch_debug = (epoch == 0)
|
| 203 |
+
|
| 204 |
for batch_idx, (src_batch, tgt_batch) in enumerate(app_dataloader):
|
| 205 |
+
if first_batch_debug and batch_idx == 0:
|
| 206 |
+
swck_model_global.debug_prints_enabled = True
|
| 207 |
+
for blk in swck_model_global.adaptive_blocks: blk.debug_prints_enabled = True
|
| 208 |
+
elif not (first_batch_debug and batch_idx == 0) : # Disable after first batch for speed
|
| 209 |
+
swck_model_global.debug_prints_enabled = False
|
| 210 |
+
for blk in swck_model_global.adaptive_blocks: blk.debug_prints_enabled = False
|
| 211 |
+
|
| 212 |
+
|
| 213 |
src_batch, tgt_batch = src_batch.to(device_global), tgt_batch.to(device_global)
|
| 214 |
+
decoder_input_tokens = src_batch[:, :-1] # Remove EOS from input
|
| 215 |
+
gold_standard_for_loss = tgt_batch[:, 1:] # Remove SOS from target
|
| 216 |
+
|
| 217 |
src_key_padding_mask = (decoder_input_tokens == PAD_TOKEN)
|
| 218 |
|
| 219 |
optimizer_global.zero_grad()
|
| 220 |
logits, entropy_report = swck_model_global(decoder_input_tokens, src_key_padding_mask=src_key_padding_mask)
|
| 221 |
+
|
| 222 |
+
# Ensure logits and gold_standard_for_loss are aligned for CrossEntropyLoss
|
| 223 |
+
# Logits: (B, S_len_in, VocabSize)
|
| 224 |
+
# Gold: (B, S_len_target)
|
| 225 |
+
# If S_len_in == S_len_target, it's fine.
|
| 226 |
+
if logits.size(1) != gold_standard_for_loss.size(1):
|
| 227 |
+
# This can happen if seq len handling differs slightly, adjust shorter one
|
| 228 |
+
min_len = min(logits.size(1), gold_standard_for_loss.size(1))
|
| 229 |
+
logits_for_loss = logits[:, :min_len, :].contiguous()
|
| 230 |
+
gold_for_loss_aligned = gold_standard_for_loss[:, :min_len].contiguous()
|
| 231 |
+
else:
|
| 232 |
+
logits_for_loss = logits
|
| 233 |
+
gold_for_loss_aligned = gold_standard_for_loss
|
| 234 |
+
|
| 235 |
+
main_loss = criterion_main_app(logits_for_loss.view(-1, logits_for_loss.size(-1)), gold_for_loss_aligned.view(-1))
|
| 236 |
|
| 237 |
block_entropy_loss = torch.tensor(0.0, device=device_global)
|
| 238 |
if entropy_report["block_output_entropies"]:
|
| 239 |
+
for i, block_entropy_tensor in enumerate(entropy_report["block_output_entropies"]):
|
| 240 |
+
target_entropy_val = swck_model_global.seed_parser.get_block_config(i)["target_entropy"]
|
| 241 |
+
block_entropy_loss += F.mse_loss(block_entropy_tensor, torch.tensor(target_entropy_val, device=device_global))
|
| 242 |
+
if entropy_report["block_output_entropies"]: # Avoid division by zero
|
| 243 |
block_entropy_loss = block_entropy_loss / len(entropy_report["block_output_entropies"])
|
| 244 |
|
| 245 |
overall_entropy_loss = entropy_report["overall_output_entropy"]
|
| 246 |
gate_sparsity_loss = torch.tensor(0.0, device=device_global)
|
| 247 |
if entropy_report["block_gate_weights"]:
|
| 248 |
+
for gates_softmax_tensor in entropy_report["block_gate_weights"]:
|
| 249 |
+
gate_sparsity_loss += torch.mean(gates_softmax_tensor * torch.log(gates_softmax_tensor + 1e-9))
|
| 250 |
+
if entropy_report["block_gate_weights"]: # Avoid division by zero
|
| 251 |
gate_sparsity_loss = - (gate_sparsity_loss / len(entropy_report["block_gate_weights"]))
|
| 252 |
|
|
|
|
| 253 |
combined_loss = (MAIN_LOSS_WEIGHT_APP * main_loss +
|
| 254 |
BLOCK_TARGET_ENTROPY_LOSS_WEIGHT_APP * block_entropy_loss +
|
| 255 |
OVERALL_OUTPUT_ENTROPY_REG_WEIGHT_APP * overall_entropy_loss +
|
|
|
|
| 260 |
optimizer_global.step()
|
| 261 |
epoch_loss += combined_loss.item()
|
| 262 |
|
| 263 |
+
log_line = f" Epoch {epoch+1}, Batch {batch_idx+1}/{len(app_dataloader)}, Loss: {combined_loss.item():.4f}"
|
| 264 |
+
if batch_idx % max(1, len(app_dataloader)//2) == 0 or batch_idx == len(app_dataloader)-1 : # Log less frequently to UI
|
| 265 |
+
print(log_line)
|
| 266 |
+
training_log_output += log_line + "\n"
|
| 267 |
+
|
| 268 |
+
# Disable debug prints after the very first batch of the first epoch
|
| 269 |
+
swck_model_global.debug_prints_enabled = False
|
| 270 |
+
for blk in swck_model_global.adaptive_blocks: blk.debug_prints_enabled = False
|
| 271 |
+
|
| 272 |
|
| 273 |
+
avg_epoch_loss = epoch_loss / len(app_dataloader) if len(app_dataloader) > 0 else epoch_loss
|
| 274 |
epoch_summary = f"Epoch {epoch+1}/{num_epochs_app} - Avg Loss: {avg_epoch_loss:.4f}\n"
|
| 275 |
print(epoch_summary)
|
| 276 |
training_log_output += epoch_summary
|
|
|
|
|
|
|
|
|
|
| 277 |
|
| 278 |
+
# Ensure debug prints are off after training session
|
| 279 |
+
swck_model_global.debug_prints_enabled = False
|
| 280 |
+
for blk in swck_model_global.adaptive_blocks: blk.debug_prints_enabled = False
|
| 281 |
+
swck_model_global.eval()
|
| 282 |
+
|
| 283 |
try:
|
| 284 |
torch.save({
|
| 285 |
'model_state_dict': swck_model_global.state_dict(),
|
| 286 |
+
'optimizer_state_dict': optimizer_global.state_dict(),
|
| 287 |
'word_to_idx': word_to_idx_global,
|
| 288 |
'idx_to_word': idx_to_word_global,
|
| 289 |
+
'model_hyperparameters': {
|
|
|
|
| 290 |
'vocab_size': VOCAB_SIZE_APP, 'd_model': D_MODEL_APP, 'n_heads': N_HEADS_APP,
|
| 291 |
'd_ff': D_FF_APP, 'num_adaptive_blocks': NUM_ADAPTIVE_BLOCKS_APP, 'dropout': DROPOUT_APP
|
| 292 |
}
|
| 293 |
}, CHECKPOINT_FILENAME)
|
| 294 |
+
save_msg = f"Training finished. Model checkpoint saved to {CHECKPOINT_FILENAME} in Space's ephemeral storage."
|
| 295 |
print(save_msg)
|
| 296 |
training_log_output += save_msg
|
| 297 |
model_load_status_global = f"Model trained in-app & saved. Last status: {save_msg}"
|
|
|
|
| 303 |
|
| 304 |
return training_log_output
|
| 305 |
|
|
|
|
| 306 |
def generate_text_for_app(prompt_str, max_len_gen, temperature_gen):
|
| 307 |
+
global model_load_status_global
|
| 308 |
if swck_model_global is None or word_to_idx_global is None or idx_to_word_global is None:
|
| 309 |
return "Model not loaded. Please check server logs or try training.", "Model not available."
|
| 310 |
|
| 311 |
swck_model_global.eval()
|
| 312 |
swck_model_global.set_wiring_phase(False)
|
| 313 |
+
# Temporarily enable debug for generation if needed, then disable
|
| 314 |
+
# swck_model_global.debug_prints_enabled = True # For generation debug
|
| 315 |
+
# for blk in swck_model_global.adaptive_blocks: blk.debug_prints_enabled = True
|
| 316 |
|
| 317 |
print(f"App: Generating for prompt: '{prompt_str}', max_len: {max_len_gen}, temp: {temperature_gen}")
|
| 318 |
|
|
|
|
| 321 |
debug_info_lines = [f"Prompt tokens: {generated_ids_app}"]
|
| 322 |
|
| 323 |
with torch.no_grad():
|
| 324 |
+
for i in range(int(max_len_gen)): # Ensure max_len_gen is int
|
| 325 |
+
# Context windowing for input_tensor
|
| 326 |
+
# Take up to SEQ_LEN_APP tokens from the end of generated_ids_app
|
| 327 |
+
context_start_idx = max(0, len(generated_ids_app) - SEQ_LEN_APP)
|
| 328 |
+
current_context_ids = generated_ids_app[context_start_idx:]
|
| 329 |
+
|
| 330 |
input_tensor = torch.tensor([current_context_ids], dtype=torch.long).to(device_global)
|
| 331 |
padding_mask = (input_tensor == PAD_TOKEN)
|
| 332 |
|
|
|
|
| 337 |
next_token_id = torch.argmax(next_token_logits).item()
|
| 338 |
else:
|
| 339 |
probs = F.softmax(next_token_logits / temperature_gen, dim=-1)
|
| 340 |
+
if probs.isnan().any() or probs.isinf().any() or torch.sum(probs).item() < 1e-9 :
|
| 341 |
print(f"Warning: Invalid probabilities at step {i}. Using uniform.")
|
| 342 |
+
probs = torch.ones_like(next_token_logits) / next_token_logits.size(-1)
|
| 343 |
next_token_id = torch.multinomial(probs, 1).item()
|
| 344 |
|
| 345 |
if next_token_id == EOS_TOKEN:
|
|
|
|
| 350 |
if i < 10 :
|
| 351 |
current_word = idx_to_word_global.get(next_token_id, UNK_TOKEN_STR)
|
| 352 |
overall_ent = entropy_report_infer['overall_output_entropy'].item()
|
| 353 |
+
if entropy_report_infer['block_output_entropies'] and len(entropy_report_infer['block_output_entropies']) > 0:
|
| 354 |
b0_ent = entropy_report_infer['block_output_entropies'][0].item()
|
| 355 |
+
if entropy_report_infer['block_gate_weights'] and len(entropy_report_infer['block_gate_weights']) > 0:
|
| 356 |
+
b0_gates_str = ", ".join([f"{g.item():.2f}" for g in entropy_report_infer['block_gate_weights'][0]])
|
| 357 |
+
debug_info_lines.append(f"Gen {i+1}: '{current_word}', OvrlEnt={overall_ent:.3f}, B0Ent={b0_ent:.3f}, B0Gates=[{b0_gates_str}]")
|
| 358 |
+
else:
|
| 359 |
+
debug_info_lines.append(f"Gen {i+1}: '{current_word}', OvrlEnt={overall_ent:.3f}, B0Ent={b0_ent:.3f}, No B0 gates.")
|
| 360 |
else:
|
| 361 |
+
debug_info_lines.append(f"Gen {i+1}: '{current_word}', OvrlEnt={overall_ent:.3f}, No block entropy/gate report.")
|
| 362 |
|
| 363 |
|
| 364 |
generated_text_list = [idx_to_word_global.get(idx, UNK_TOKEN_STR) for idx in generated_ids_app[1:]]
|
|
|
|
| 369 |
final_text = re.sub(r'\s+', ' ', final_text).strip()
|
| 370 |
|
| 371 |
debug_output_str = "\n".join(debug_info_lines)
|
| 372 |
+
|
| 373 |
+
# Disable debug prints after generation
|
| 374 |
+
# swck_model_global.debug_prints_enabled = False
|
| 375 |
+
# for blk in swck_model_global.adaptive_blocks: blk.debug_prints_enabled = False
|
| 376 |
return final_text, debug_output_str
|
| 377 |
|
| 378 |
# --- Gradio Interface ---
|
| 379 |
+
initial_load_status = initialize_or_load_model_app() # Load model on app startup
|
|
|
|
|
|
|
| 380 |
|
| 381 |
with gr.Blocks(title="SWCK Conceptual Demo") as demo:
|
| 382 |
gr.Markdown(f"""
|
|
|
|
| 404 |
with gr.Row():
|
| 405 |
train_epochs_slider = gr.Slider(minimum=1, maximum=5, value=1, step=1, label="Number of Training Epochs")
|
| 406 |
train_batch_size_slider = gr.Slider(minimum=1, maximum=8, value=2, step=1, label="Training Batch Size")
|
| 407 |
+
# REMOVED format="%.1e"
|
| 408 |
+
train_lr_slider = gr.Slider(minimum=1e-5, maximum=1e-3, value=5e-4, step=1e-5, label="Learning Rate")
|
| 409 |
|
| 410 |
start_training_button = gr.Button("Start Short Training Session")
|
| 411 |
+
training_status_output = gr.Textbox(label="Training Log / Status:", lines=10, interactive=False,show_label=True )
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
model_status_md = gr.Markdown(value=f"**Model Status:** {model_load_status_global}")
|
| 415 |
+
|
| 416 |
+
def update_status_text(): # Helper to refresh status after training
|
| 417 |
+
return f"**Model Status:** {model_load_status_global}"
|
| 418 |
|
|
|
|
| 419 |
generate_button.click(
|
| 420 |
fn=generate_text_for_app,
|
| 421 |
inputs=[prompt_input, max_len_slider, temp_slider],
|
|
|
|
| 426 |
fn=run_short_training_session,
|
| 427 |
inputs=[train_epochs_slider, train_batch_size_slider, train_lr_slider],
|
| 428 |
outputs=[training_status_output]
|
| 429 |
+
).then(fn=update_status_text, inputs=None, outputs=model_status_md)
|
| 430 |
+
|
|
|
|
|
|
|
| 431 |
|
| 432 |
if __name__ == "__main__":
|
| 433 |
+
# The Gradio app launch options (like debug=True) are for local execution.
|
| 434 |
+
# On Hugging Face Spaces, these are typically controlled by the environment.
|
| 435 |
+
# The `print()` statements will go to the Space's console logs.
|
| 436 |
+
demo.launch(debug=True)
|