aagamjtdev commited on
Commit
c08f202
·
1 Parent(s): 1636abd
Files changed (1) hide show
  1. app.py +0 -998
app.py CHANGED
@@ -1,1001 +1,3 @@
1
- # import gradio as gr
2
- # import subprocess
3
- # import os
4
- # import sys
5
- # from datetime import datetime
6
- #
7
- # # The name of your existing training script
8
- # TRAINING_SCRIPT = "LayoutLM_Train_Passage.py"
9
- #
10
- # # --- CORRECTED MODEL PATH BASED ON LayoutLM_Train_Passage.py ---
11
- # MODEL_OUTPUT_DIR = "checkpoints"
12
- # MODEL_FILE_NAME = "layoutlmv3_crf_passage.pth"
13
- # MODEL_FILE_PATH = os.path.join(MODEL_OUTPUT_DIR, MODEL_FILE_NAME)
14
- #
15
- #
16
- # # ----------------------------------------------------------------
17
- #
18
- #
19
- # def train_model(dataset_file: gr.File, batch_size: int, epochs: int, lr: float, max_len: int, progress=gr.Progress()):
20
- # """
21
- # Handles the Gradio submission and executes the training script using subprocess.
22
- # """
23
- #
24
- # # 1. Setup: Create output directory if it doesn't exist
25
- # os.makedirs(MODEL_OUTPUT_DIR, exist_ok=True)
26
- #
27
- # # 2. File Handling: Use the temporary path of the uploaded file
28
- # # if dataset_file is None or not dataset_file.path.endswith(".json"):
29
- # # return "❌ ERROR: Please upload a valid Label Studio JSON file.", None
30
- #
31
- # input_path = dataset_file.path
32
- #
33
- # progress(0.1, desc="Starting LayoutLMv3 Training...")
34
- #
35
- # log_output = f"--- Training Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
36
- #
37
- # # 3. Construct the subprocess command
38
- # command = [
39
- # sys.executable,
40
- # TRAINING_SCRIPT,
41
- # "--mode", "train",
42
- # "--input", input_path,
43
- # "--batch_size", str(batch_size),
44
- # "--epochs", str(epochs),
45
- # "--lr", str(lr),
46
- # "--max_len", str(max_len)
47
- # ]
48
- #
49
- # log_output += f"Executing command: {' '.join(command)}\n\n"
50
- #
51
- # try:
52
- # # 4. Run the training script and capture output
53
- # process = subprocess.Popen(
54
- # command,
55
- # stdout=subprocess.PIPE,
56
- # stderr=subprocess.STDOUT,
57
- # text=True,
58
- # bufsize=1
59
- # )
60
- #
61
- # # Stream logs in real-time
62
- # for line in iter(process.stdout.readline, ""):
63
- # log_output += line
64
- # yield log_output, None # Send partial log to Gradio output
65
- #
66
- # process.stdout.close()
67
- # return_code = process.wait()
68
- #
69
- # # 5. Check for successful completion
70
- # if return_code == 0:
71
- # log_output += "\n✅ TRAINING COMPLETE! Model saved."
72
- #
73
- # # 6. Prepare download links based on script's saved path
74
- # model_exists = os.path.exists(MODEL_FILE_PATH)
75
- #
76
- # if model_exists:
77
- # log_output += f"\nModel path: {MODEL_FILE_PATH}"
78
- # # Return final log, and the file path for Gradio's download component
79
- # return log_output, MODEL_FILE_PATH
80
- # else:
81
- # log_output += f"\n⚠️ WARNING: Training completed, but model file not found at expected path ({MODEL_FILE_PATH})."
82
- # return log_output, None
83
- # else:
84
- # log_output += f"\n\n❌ TRAINING FAILED with return code {return_code}. Check logs above."
85
- # return log_output, None
86
- #
87
- # except FileNotFoundError:
88
- # return f"❌ ERROR: The training script '{TRAINING_SCRIPT}' was not found. Ensure it is in the root directory of your Space.", None
89
- # except Exception as e:
90
- # return f"❌ An unexpected error occurred: {e}", None
91
- #
92
- #
93
- # # --- Gradio Interface Setup (using Blocks for a nicer layout) ---
94
- # with gr.Blocks(title="LayoutLMv3 Fine-Tuning App") as demo:
95
- # gr.Markdown("# 🚀 LayoutLMv3 Fine-Tuning on Hugging Face Spaces")
96
- # gr.Markdown(
97
- # """
98
- # Upload your Label Studio JSON file, set your hyperparameters, and click **Train Model** to fine-tune the LayoutLMv3 model using your script.
99
- #
100
- # **Note:** The trained model is saved in the **`checkpoints/`** folder as **`layoutlmv3_crf_passage.pth`**.
101
- # """
102
- # )
103
- #
104
- # with gr.Row():
105
- # with gr.Column(scale=1):
106
- # file_input = gr.File(
107
- # label="1. Upload Label Studio JSON Dataset"
108
- # )
109
- #
110
- # gr.Markdown("---")
111
- # gr.Markdown("### ⚙️ Training Parameters")
112
- #
113
- # batch_size_input = gr.Slider(
114
- # minimum=1, maximum=32, step=1, value=4, label="Batch Size (--batch_size)"
115
- # )
116
- # epochs_input = gr.Slider(
117
- # minimum=1, maximum=20, step=1, value=5, label="Epochs (--epochs)"
118
- # )
119
- # lr_input = gr.Number(
120
- # value=5e-5, label="Learning Rate (--lr)"
121
- # )
122
- # max_len_input = gr.Number(
123
- # value=512, label="Max Sequence Length (--max_len)"
124
- # )
125
- #
126
- # with gr.Column(scale=2):
127
- # train_button = gr.Button("🔥 Train Model", variant="primary")
128
- #
129
- # log_output = gr.Textbox(
130
- # label="Training Log Output",
131
- # lines=20,
132
- # autoscroll=True,
133
- # placeholder="Click 'Train Model' to start and see real-time logs..."
134
- # )
135
- #
136
- # gr.Markdown("---")
137
- # gr.Markdown(f"### 🎉 Trained Model Output (Saved to `{MODEL_OUTPUT_DIR}/`)")
138
- #
139
- # # Only providing the download link for the saved .pth model file
140
- # model_download = gr.File(label=f"Trained Model File ({MODEL_FILE_NAME})", interactive=False)
141
- #
142
- # # Define the action when the button is clicked
143
- # train_button.click(
144
- # fn=train_model,
145
- # inputs=[file_input, batch_size_input, epochs_input, lr_input, max_len_input],
146
- # outputs=[log_output, model_download]
147
- # )
148
- #
149
- # if __name__ == "__main__":
150
- # demo.launch(server_port=7860, server_name="0.0.0.0")
151
-
152
-
153
- # import gradio as gr
154
- # import subprocess
155
- # import os
156
- # import sys
157
- # from datetime import datetime
158
- #
159
- # # The name of your existing training script
160
- # TRAINING_SCRIPT = "LayoutLM_Train_Passage.py"
161
- #
162
- # # --- CORRECTED MODEL PATH BASED ON LayoutLM_Train_Passage.py ---
163
- # MODEL_OUTPUT_DIR = "checkpoints"
164
- # MODEL_FILE_NAME = "layoutlmv3_crf_passage.pth"
165
- # MODEL_FILE_PATH = os.path.join(MODEL_OUTPUT_DIR, MODEL_FILE_NAME)
166
- #
167
- #
168
- # # ----------------------------------------------------------------
169
- #
170
- #
171
- # def train_model(dataset_file: gr.File, batch_size: int, epochs: int, lr: float, max_len: int, progress=gr.Progress()):
172
- # """
173
- # Handles the Gradio submission and executes the training script using subprocess.
174
- # """
175
- #
176
- # # 1. Setup: Create output directory if it doesn't exist
177
- # os.makedirs(MODEL_OUTPUT_DIR, exist_ok=True)
178
- #
179
- # # 2. File Handling: Use the temporary path of the uploaded file
180
- # if dataset_file is None:
181
- # yield "❌ ERROR: Please upload a file.", None
182
- # return
183
- #
184
- # # FIX: Gradio returns the path in the .name attribute, not .path
185
- # input_path = dataset_file.name
186
- #
187
- # if not input_path.lower().endswith(".json"):
188
- # yield "❌ ERROR: Please upload a valid Label Studio JSON file (.json).", None
189
- # return
190
- #
191
- # progress(0.1, desc="Starting LayoutLMv3 Training...")
192
- #
193
- # log_output = f"--- Training Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
194
- #
195
- # # 3. Construct the subprocess command
196
- # command = [
197
- # sys.executable,
198
- # TRAINING_SCRIPT,
199
- # "--mode", "train",
200
- # "--input", input_path,
201
- # "--batch_size", str(batch_size),
202
- # "--epochs", str(epochs),
203
- # "--lr", str(lr),
204
- # "--max_len", str(max_len)
205
- # ]
206
- #
207
- # log_output += f"Executing command: {' '.join(command)}\n\n"
208
- # yield log_output, None # Yield the command to the log output
209
- #
210
- # try:
211
- # # 4. Run the training script and capture output
212
- # process = subprocess.Popen(
213
- # command,
214
- # stdout=subprocess.PIPE,
215
- # stderr=subprocess.STDOUT,
216
- # text=True,
217
- # bufsize=1
218
- # )
219
- #
220
- # # Stream logs in real-time
221
- # for line in iter(process.stdout.readline, ""):
222
- # log_output += line
223
- # yield log_output, None # Send partial log to Gradio output
224
- #
225
- # process.stdout.close()
226
- # return_code = process.wait()
227
- #
228
- # # 5. Check for successful completion
229
- # if return_code == 0:
230
- # log_output += "\n✅ TRAINING COMPLETE! Model saved."
231
- #
232
- # # 6. Prepare download links based on script's saved path
233
- # model_exists = os.path.exists(MODEL_FILE_PATH)
234
- #
235
- # if model_exists:
236
- # log_output += f"\nModel path: {MODEL_FILE_PATH}"
237
- # # Return final log, and the file path for Gradio's download component
238
- # return log_output, MODEL_FILE_PATH
239
- # else:
240
- # log_output += f"\n⚠️ WARNING: Training completed, but model file not found at expected path ({MODEL_FILE_PATH})."
241
- # return log_output, None
242
- # else:
243
- # log_output += f"\n\n❌ TRAINING FAILED with return code {return_code}. Check logs above."
244
- # return log_output, None
245
- #
246
- # except FileNotFoundError:
247
- # return f"❌ ERROR: The training script '{TRAINING_SCRIPT}' was not found. Ensure it is in the root directory of your Space.", None
248
- # except Exception as e:
249
- # return f"❌ An unexpected error occurred: {e}", None
250
- #
251
- #
252
- # # --- Gradio Interface Setup (using Blocks for a nicer layout) ---
253
- # with gr.Blocks(title="LayoutLMv3 Fine-Tuning App") as demo:
254
- # gr.Markdown("# 🚀 LayoutLMv3 Fine-Tuning on Hugging Face Spaces")
255
- # gr.Markdown(
256
- # """
257
- # Upload your Label Studio JSON file, set your hyperparameters, and click **Train Model** to fine-tune the LayoutLMv3 model using your script.
258
- #
259
- # **Note:** The trained model is saved in the **`checkpoints/`** folder as **`layoutlmv3_crf_passage.pth`**.
260
- # """
261
- # )
262
- #
263
- # with gr.Row():
264
- # with gr.Column(scale=1):
265
- # file_input = gr.File(
266
- # label="1. Upload Label Studio JSON Dataset"
267
- # )
268
- #
269
- # gr.Markdown("---")
270
- # gr.Markdown("### ⚙️ Training Parameters")
271
- #
272
- # batch_size_input = gr.Slider(
273
- # minimum=1, maximum=32, step=1, value=4, label="Batch Size (--batch_size)"
274
- # )
275
- # epochs_input = gr.Slider(
276
- # minimum=1, maximum=20, step=1, value=5, label="Epochs (--epochs)"
277
- # )
278
- # lr_input = gr.Number(
279
- # value=5e-5, label="Learning Rate (--lr)"
280
- # )
281
- # max_len_input = gr.Number(
282
- # value=512, label="Max Sequence Length (--max_len)"
283
- # )
284
- #
285
- # with gr.Column(scale=2):
286
- # train_button = gr.Button("🔥 Train Model", variant="primary")
287
- #
288
- # log_output = gr.Textbox(
289
- # label="Training Log Output",
290
- # lines=20,
291
- # autoscroll=True,
292
- # placeholder="Click 'Train Model' to start and see real-time logs..."
293
- # )
294
- #
295
- # gr.Markdown("---")
296
- # gr.Markdown(f"### 🎉 Trained Model Output (Saved to `{MODEL_OUTPUT_DIR}/`)")
297
- #
298
- # # Only providing the download link for the saved .pth model file
299
- # model_download = gr.File(label=f"Trained Model File ({MODEL_FILE_NAME})", interactive=False)
300
- #
301
- # # Define the action when the button is clicked
302
- # train_button.click(
303
- # fn=train_model,
304
- # inputs=[file_input, batch_size_input, epochs_input, lr_input, max_len_input],
305
- # outputs=[log_output, model_download]
306
- # )
307
- #
308
- # if __name__ == "__main__":
309
- # # Removed server_port and server_name as they are often unnecessary
310
- # # and sometimes cause issues in managed Space environments.
311
- # demo.launch()
312
-
313
- #
314
- # import gradio as gr
315
- # import subprocess
316
- # import os
317
- # import sys
318
- # from datetime import datetime
319
- #
320
- # # FIX: Update the script name to the correct one you uploaded
321
- # TRAINING_SCRIPT = "HF_LayoutLM_with_Passage.py"
322
- #
323
- # # --- CORRECTED MODEL PATH BASED ON YOUR SCRIPT ---
324
- # MODEL_OUTPUT_DIR = "checkpoints"
325
- # MODEL_FILE_NAME = "layoutlmv3_crf_passage.pth"
326
- # MODEL_FILE_PATH = os.path.join(MODEL_OUTPUT_DIR, MODEL_FILE_NAME)
327
- #
328
- #
329
- # # ----------------------------------------------------------------
330
- #
331
- #
332
- # def train_model(dataset_file: gr.File, batch_size: int, epochs: int, lr: float, max_len: int, progress=gr.Progress()):
333
- # """
334
- # Handles the Gradio submission and executes the training script using subprocess.
335
- # """
336
- #
337
- # # 1. Setup: Create output directory if it doesn't exist
338
- # os.makedirs(MODEL_OUTPUT_DIR, exist_ok=True)
339
- #
340
- # # 2. File Handling: Use the temporary path of the uploaded file
341
- # if dataset_file is None:
342
- # yield "❌ ERROR: Please upload a file.", None
343
- # return
344
- #
345
- # # Using .name (Corrected in previous steps)
346
- # input_path = dataset_file.name
347
- #
348
- # if not input_path.lower().endswith(".json"):
349
- # yield "❌ ERROR: Please upload a valid Label Studio JSON file (.json).", None
350
- # return
351
- #
352
- # progress(0.1, desc="Starting LayoutLMv3 Training...")
353
- #
354
- # log_output = f"--- Training Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
355
- #
356
- # # 3. Construct the subprocess command
357
- # command = [
358
- # sys.executable,
359
- # # Now uses the corrected TRAINING_SCRIPT variable
360
- # TRAINING_SCRIPT,
361
- # "--mode", "train",
362
- # "--input", input_path,
363
- # "--batch_size", str(batch_size),
364
- # "--epochs", str(epochs),
365
- # "--lr", str(lr),
366
- # "--max_len", str(max_len)
367
- # ]
368
- #
369
- # log_output += f"Executing command: {' '.join(command)}\n\n"
370
- # yield log_output, None # Yield the command to the log output
371
- #
372
- # try:
373
- # # 4. Run the training script and capture output
374
- # process = subprocess.Popen(
375
- # command,
376
- # stdout=subprocess.PIPE,
377
- # stderr=subprocess.STDOUT,
378
- # text=True,
379
- # bufsize=1
380
- # )
381
- #
382
- # # Stream logs in real-time
383
- # for line in iter(process.stdout.readline, ""):
384
- # log_output += line
385
- # yield log_output, None # Send partial log to Gradio output
386
- #
387
- # process.stdout.close()
388
- # return_code = process.wait()
389
- #
390
- # # 5. Check for successful completion
391
- # if return_code == 0:
392
- # log_output += "\n✅ TRAINING COMPLETE! Model saved."
393
- #
394
- # # 6. Prepare download links based on script's saved path
395
- # model_exists = os.path.exists(MODEL_FILE_PATH)
396
- #
397
- # if model_exists:
398
- # log_output += f"\nModel path: {MODEL_FILE_PATH}"
399
- # # Return final log, and the file path for Gradio's download component
400
- # return log_output, MODEL_FILE_PATH
401
- # else:
402
- # log_output += f"\n⚠️ WARNING: Training completed, but model file not found at expected path ({MODEL_FILE_PATH})."
403
- # return log_output, None
404
- # else:
405
- # log_output += f"\n\n❌ TRAINING FAILED with return code {return_code}. Check logs above."
406
- # return log_output, None
407
- #
408
- # except FileNotFoundError:
409
- # return f"❌ ERROR: The training script '{TRAINING_SCRIPT}' was not found. Ensure it is in the root directory of your Space.", None
410
- # except Exception as e:
411
- # return f"❌ An unexpected error occurred: {e}", None
412
- #
413
- #
414
- # # --- Gradio Interface Setup (using Blocks for a nicer layout) ---
415
- # with gr.Blocks(title="LayoutLMv3 Fine-Tuning App") as demo:
416
- # gr.Markdown("# 🚀 LayoutLMv3 Fine-Tuning on Hugging Face Spaces")
417
- # gr.Markdown(
418
- # """
419
- # Upload your Label Studio JSON file, set your hyperparameters, and click **Train Model** to fine-tune the LayoutLMv3 model using your script.
420
- #
421
- # **Note:** The trained model is saved in the **`checkpoints/`** folder as **`layoutlmv3_crf_passage.pth`**.
422
- # """
423
- # )
424
- #
425
- # with gr.Row():
426
- # with gr.Column(scale=1):
427
- # file_input = gr.File(
428
- # label="1. Upload Label Studio JSON Dataset"
429
- # )
430
- #
431
- # gr.Markdown("---")
432
- # gr.Markdown("### ⚙️ Training Parameters")
433
- #
434
- # batch_size_input = gr.Slider(
435
- # minimum=1, maximum=32, step=1, value=4, label="Batch Size (--batch_size)"
436
- # )
437
- # epochs_input = gr.Slider(
438
- # minimum=1, maximum=20, step=1, value=5, label="Epochs (--epochs)"
439
- # )
440
- # lr_input = gr.Number(
441
- # value=5e-5, label="Learning Rate (--lr)"
442
- # )
443
- # max_len_input = gr.Number(
444
- # value=512, label="Max Sequence Length (--max_len)"
445
- # )
446
- #
447
- # with gr.Column(scale=2):
448
- # train_button = gr.Button("🔥 Train Model", variant="primary")
449
- #
450
- # log_output = gr.Textbox(
451
- # label="Training Log Output",
452
- # lines=20,
453
- # autoscroll=True,
454
- # placeholder="Click 'Train Model' to start and see real-time logs..."
455
- # )
456
- #
457
- # gr.Markdown("---")
458
- # gr.Markdown(f"### 🎉 Trained Model Output (Saved to `{MODEL_OUTPUT_DIR}/`)")
459
- #
460
- # # Only providing the download link for the saved .pth model file
461
- # model_download = gr.File(label=f"Trained Model File ({MODEL_FILE_NAME})", interactive=False)
462
- #
463
- # # Define the action when the button is clicked
464
- # train_button.click(
465
- # fn=train_model,
466
- # inputs=[file_input, batch_size_input, epochs_input, lr_input, max_len_input],
467
- # outputs=[log_output, model_download]
468
- # )
469
- #
470
- # if __name__ == "__main__":
471
- # demo.launch()
472
-
473
- #
474
- # import gradio as gr
475
- # import subprocess
476
- # import os
477
- # import sys
478
- # from datetime import datetime
479
- # import shutil
480
- #
481
- # # FIX: Update the script name to the correct one you uploaded
482
- # TRAINING_SCRIPT = "HF_LayoutLM_with_Passage.py"
483
- #
484
- # # --- CORRECTED MODEL PATH BASED ON YOUR SCRIPT ---
485
- # MODEL_OUTPUT_DIR = "checkpoints"
486
- # MODEL_FILE_NAME = "layoutlmv3_crf_passage.pth"
487
- # MODEL_FILE_PATH = os.path.join(MODEL_OUTPUT_DIR, MODEL_FILE_NAME)
488
- #
489
- #
490
- # # ----------------------------------------------------------------
491
- #
492
- #
493
- # def train_model(dataset_file: gr.File, batch_size: int, epochs: int, lr: float, max_len: int, progress=gr.Progress()):
494
- # """
495
- # Handles the Gradio submission and executes the training script using subprocess.
496
- # """
497
- #
498
- # # 1. Setup: Create output directory if it doesn't exist
499
- # os.makedirs(MODEL_OUTPUT_DIR, exist_ok=True)
500
- #
501
- # # 2. File Handling: Use the temporary path of the uploaded file
502
- # if dataset_file is None:
503
- # return "❌ ERROR: Please upload a file.", None
504
- #
505
- # # Using .name (Corrected in previous steps)
506
- # input_path = dataset_file.name
507
- #
508
- # if not input_path.lower().endswith(".json"):
509
- # return "❌ ERROR: Please upload a valid Label Studio JSON file (.json).", None
510
- #
511
- # progress(0.1, desc="Starting LayoutLMv3 Training...")
512
- #
513
- # log_output = f"--- Training Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
514
- #
515
- # # 3. Construct the subprocess command
516
- # command = [
517
- # sys.executable,
518
- # TRAINING_SCRIPT,
519
- # "--mode", "train",
520
- # "--input", input_path,
521
- # "--batch_size", str(batch_size),
522
- # "--epochs", str(epochs),
523
- # "--lr", str(lr),
524
- # "--max_len", str(max_len)
525
- # ]
526
- #
527
- # log_output += f"Executing command: {' '.join(command)}\n\n"
528
- #
529
- # try:
530
- # # 4. Run the training script and capture output
531
- # process = subprocess.Popen(
532
- # command,
533
- # stdout=subprocess.PIPE,
534
- # stderr=subprocess.STDOUT,
535
- # text=True,
536
- # bufsize=1
537
- # )
538
- #
539
- # # Stream logs in real-time
540
- # for line in iter(process.stdout.readline, ""):
541
- # log_output += line
542
- # # Print to console as well for debugging
543
- # print(line, end='')
544
- #
545
- # process.stdout.close()
546
- # return_code = process.wait()
547
- #
548
- # # 5. Check for successful completion
549
- # if return_code == 0:
550
- # log_output += "\n✅ TRAINING COMPLETE! Model saved."
551
- # print("\n✅ TRAINING COMPLETE! Model saved.")
552
- #
553
- # # 6. Verify model file exists
554
- # if os.path.exists(MODEL_FILE_PATH):
555
- # file_size = os.path.getsize(MODEL_FILE_PATH) / (1024 * 1024) # Size in MB
556
- # log_output += f"\n📦 Model file: {MODEL_FILE_PATH}"
557
- # log_output += f"\n📊 Model size: {file_size:.2f} MB"
558
- # log_output += f"\n⬇️ Click the download button below to save your model!"
559
- #
560
- # print(f"\n✅ Model exists at: {MODEL_FILE_PATH} ({file_size:.2f} MB)")
561
- #
562
- # # Create a copy in the root directory for easier access
563
- # root_copy = MODEL_FILE_NAME
564
- # try:
565
- # shutil.copy2(MODEL_FILE_PATH, root_copy)
566
- # log_output += f"\n📋 Copy created: {root_copy}"
567
- # print(f"✅ Created copy at: {root_copy}")
568
- # except Exception as e:
569
- # log_output += f"\n⚠️ Could not create root copy: {e}"
570
- # root_copy = MODEL_FILE_PATH
571
- #
572
- # # Return the full absolute path to ensure Gradio can find it
573
- # absolute_path = os.path.abspath(root_copy)
574
- # log_output += f"\n🔗 Download path: {absolute_path}"
575
- #
576
- # return log_output, absolute_path
577
- # else:
578
- # log_output += f"\n⚠️ WARNING: Training completed, but model file not found at expected path ({MODEL_FILE_PATH})."
579
- # log_output += f"\n🔍 Checking directory contents..."
580
- #
581
- # # List files in checkpoints directory for debugging
582
- # if os.path.exists(MODEL_OUTPUT_DIR):
583
- # files = os.listdir(MODEL_OUTPUT_DIR)
584
- # log_output += f"\n📁 Files in {MODEL_OUTPUT_DIR}: {files}"
585
- # else:
586
- # log_output += f"\n❌ Directory {MODEL_OUTPUT_DIR} does not exist!"
587
- #
588
- # return log_output, None
589
- # else:
590
- # log_output += f"\n\n❌ TRAINING FAILED with return code {return_code}. Check logs above."
591
- # return log_output, None
592
- #
593
- # except FileNotFoundError:
594
- # error_msg = f"❌ ERROR: The training script '{TRAINING_SCRIPT}' was not found. Ensure it is in the root directory of your Space."
595
- # print(error_msg)
596
- # return error_msg, None
597
- # except Exception as e:
598
- # error_msg = f"❌ An unexpected error occurred: {e}"
599
- # print(error_msg)
600
- # import traceback
601
- # print(traceback.format_exc())
602
- # return error_msg, None
603
- #
604
- #
605
- # # --- Gradio Interface Setup (using Blocks for a nicer layout) ---
606
- # with gr.Blocks(title="LayoutLMv3 Fine-Tuning App", theme=gr.themes.Soft()) as demo:
607
- # gr.Markdown("# 🚀 LayoutLMv3 Fine-Tuning on Hugging Face Spaces")
608
- # gr.Markdown(
609
- # """
610
- # Upload your Label Studio JSON file, set your hyperparameters, and click **Train Model** to fine-tune the LayoutLMv3 model.
611
- #
612
- # **⚠️ IMPORTANT - Free Tier Users:**
613
- # - **Download your model IMMEDIATELY** after training completes!
614
- # - The model file is **temporary** and will be deleted when the Space restarts.
615
- # - The download button will appear below once training is complete.
616
- # - Model is saved as: **`layoutlmv3_crf_passage.pth`**
617
- #
618
- # **⏱️ Timeout Note:** Training may timeout on free tier. Consider reducing epochs or batch size for faster training.
619
- # """
620
- # )
621
- #
622
- # with gr.Row():
623
- # with gr.Column(scale=1):
624
- # gr.Markdown("### 📁 Dataset Upload")
625
- # file_input = gr.File(
626
- # label="Upload Label Studio JSON Dataset",
627
- # file_types=[".json"]
628
- # )
629
- #
630
- # gr.Markdown("---")
631
- # gr.Markdown("### ⚙️ Training Parameters")
632
- #
633
- # batch_size_input = gr.Slider(
634
- # minimum=1, maximum=16, step=1, value=4,
635
- # label="Batch Size",
636
- # info="Smaller = less memory, slower training"
637
- # )
638
- # epochs_input = gr.Slider(
639
- # minimum=1, maximum=10, step=1, value=3,
640
- # label="Epochs",
641
- # info="Fewer epochs = faster training (recommended: 3-5)"
642
- # )
643
- # lr_input = gr.Number(
644
- # value=5e-5, label="Learning Rate",
645
- # info="Default: 5e-5"
646
- # )
647
- # max_len_input = gr.Slider(
648
- # minimum=128, maximum=512, step=128, value=512,
649
- # label="Max Sequence Length",
650
- # info="Shorter = faster training, less memory"
651
- # )
652
- #
653
- # train_button = gr.Button("🔥 Start Training", variant="primary", size="lg")
654
- #
655
- # with gr.Column(scale=2):
656
- # gr.Markdown("### 📊 Training Progress")
657
- #
658
- # log_output = gr.Textbox(
659
- # label="Training Logs",
660
- # lines=25,
661
- # max_lines=30,
662
- # autoscroll=True,
663
- # show_copy_button=True,
664
- # placeholder="Click 'Start Training' to begin...\n\nLogs will appear here in real-time."
665
- # )
666
- #
667
- # gr.Markdown("### ⬇️ Download Trained Model")
668
- #
669
- # model_download = gr.File(
670
- # label="Trained Model File (layoutlmv3_crf_passage.pth)",
671
- # interactive=False,
672
- # visible=True
673
- # )
674
- #
675
- # gr.Markdown(
676
- # """
677
- # **📥 Download Instructions:**
678
- # 1. Wait for training to complete (✅ appears in logs)
679
- # 2. Click the download button/icon that appears above
680
- # 3. Save the `.pth` file to your local machine
681
- # 4. **Do this immediately** - file is temporary!
682
- #
683
- # **🔧 Troubleshooting:**
684
- # - If download button doesn't appear, check the logs for errors
685
- # - Try reducing epochs or batch size if timeout occurs
686
- # - Ensure your JSON file is properly formatted
687
- # """
688
- # )
689
- #
690
- # # Define the action when the button is clicked
691
- # train_button.click(
692
- # fn=train_model,
693
- # inputs=[file_input, batch_size_input, epochs_input, lr_input, max_len_input],
694
- # outputs=[log_output, model_download],
695
- # api_name="train"
696
- # )
697
- #
698
- # # Add example info
699
- # gr.Markdown(
700
- # """
701
- # ---
702
- # ### 📖 About
703
- # This Space fine-tunes LayoutLMv3 with CRF for document understanding tasks including:
704
- # - Questions, Options, Answers
705
- # - Section Headings
706
- # - Passages
707
- #
708
- # **Model Details:** LayoutLMv3-base + CRF layer for sequence labeling
709
- # """
710
- # )
711
- #
712
- # if __name__ == "__main__":
713
- # demo.launch()
714
-
715
-
716
- # import gradio as gr
717
- # import subprocess
718
- # import os
719
- # import sys
720
- # from datetime import datetime
721
- # import shutil
722
- #
723
- # # FIX: Update the script name to the correct one you uploaded
724
- # TRAINING_SCRIPT = "HF_LayoutLM_with_Passage.py"
725
- #
726
- # # --- CORRECTED MODEL PATH BASED ON YOUR SCRIPT ---
727
- # MODEL_OUTPUT_DIR = "checkpoints"
728
- # MODEL_FILE_NAME = "layoutlmv3_crf_passage.pth"
729
- # MODEL_FILE_PATH = os.path.join(MODEL_OUTPUT_DIR, MODEL_FILE_NAME)
730
- #
731
- #
732
- # # ----------------------------------------------------------------
733
- #
734
- #
735
- # def train_model(dataset_file: gr.File, batch_size: int, epochs: int, lr: float, max_len: int, progress=gr.Progress()):
736
- # """
737
- # Handles the Gradio submission and executes the training script using subprocess.
738
- # """
739
- #
740
- # # 1. Setup: Create output directory if it doesn't exist
741
- # os.makedirs(MODEL_OUTPUT_DIR, exist_ok=True)
742
- #
743
- # # 2. File Handling: Use the temporary path of the uploaded file
744
- # if dataset_file is None:
745
- # return "❌ ERROR: Please upload a file.", None, gr.Button(visible=False)
746
- #
747
- # # Using .name (Corrected in previous steps)
748
- # input_path = dataset_file.name
749
- #
750
- # if not input_path.lower().endswith(".json"):
751
- # return "❌ ERROR: Please upload a valid Label Studio JSON file (.json).", None, gr.Button(visible=False)
752
- #
753
- # progress(0.1, desc="Starting LayoutLMv3 Training...")
754
- #
755
- # log_output = f"--- Training Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
756
- #
757
- # # 3. Construct the subprocess command
758
- # command = [
759
- # sys.executable,
760
- # TRAINING_SCRIPT,
761
- # "--mode", "train",
762
- # "--input", input_path,
763
- # "--batch_size", str(batch_size),
764
- # "--epochs", str(epochs),
765
- # "--lr", str(lr),
766
- # "--max_len", str(max_len)
767
- # ]
768
- #
769
- # log_output += f"Executing command: {' '.join(command)}\n\n"
770
- #
771
- # try:
772
- # # 4. Run the training script and capture output
773
- # process = subprocess.Popen(
774
- # command,
775
- # stdout=subprocess.PIPE,
776
- # stderr=subprocess.STDOUT,
777
- # text=True,
778
- # bufsize=1
779
- # )
780
- #
781
- # # Stream logs in real-time
782
- # for line in iter(process.stdout.readline, ""):
783
- # log_output += line
784
- # # Print to console as well for debugging
785
- # print(line, end='')
786
- #
787
- # process.stdout.close()
788
- # return_code = process.wait()
789
- #
790
- # # 5. Check for successful completion
791
- # if return_code == 0:
792
- # log_output += "\n✅ TRAINING COMPLETE! Model saved."
793
- # print("\n✅ TRAINING COMPLETE! Model saved.")
794
- #
795
- # # 6. Verify model file exists
796
- # if os.path.exists(MODEL_FILE_PATH):
797
- # file_size = os.path.getsize(MODEL_FILE_PATH) / (1024 * 1024) # Size in MB
798
- # log_output += f"\n📦 Model file: {MODEL_FILE_PATH}"
799
- # log_output += f"\n📊 Model size: {file_size:.2f} MB"
800
- #
801
- # print(f"\n✅ Model exists at: {MODEL_FILE_PATH} ({file_size:.2f} MB)")
802
- #
803
- # # Create a copy in the root directory with timestamp for uniqueness
804
- # timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
805
- # download_filename = f"layoutlmv3_trained_{timestamp}.pth"
806
- #
807
- # try:
808
- # shutil.copy2(MODEL_FILE_PATH, download_filename)
809
- # log_output += f"\n📋 Download file created: {download_filename}"
810
- # print(f"✅ Created download file: {download_filename}")
811
- # except Exception as e:
812
- # log_output += f"\n⚠️ Could not create download file: {e}"
813
- # download_filename = MODEL_FILE_PATH
814
- #
815
- # # Return the path and make download button visible
816
- # log_output += f"\n\n🎉 SUCCESS! Click the 'Download Model' button below to save your model."
817
- # log_output += f"\n⚠️ IMPORTANT: Download NOW - file will be deleted when Space restarts!"
818
- #
819
- # return log_output, download_filename, gr.Button(visible=True)
820
- # else:
821
- # log_output += f"\n⚠️ WARNING: Training completed, but model file not found at expected path ({MODEL_FILE_PATH})."
822
- # log_output += f"\n🔍 Checking directory contents..."
823
- #
824
- # # List files in checkpoints directory for debugging
825
- # if os.path.exists(MODEL_OUTPUT_DIR):
826
- # files = os.listdir(MODEL_OUTPUT_DIR)
827
- # log_output += f"\n📁 Files in {MODEL_OUTPUT_DIR}: {files}"
828
- # else:
829
- # log_output += f"\n❌ Directory {MODEL_OUTPUT_DIR} does not exist!"
830
- #
831
- # return log_output, None, gr.Button(visible=False)
832
- # else:
833
- # log_output += f"\n\n❌ TRAINING FAILED with return code {return_code}. Check logs above."
834
- # return log_output, None, gr.Button(visible=False)
835
- #
836
- # except FileNotFoundError:
837
- # error_msg = f"❌ ERROR: The training script '{TRAINING_SCRIPT}' was not found. Ensure it is in the root directory of your Space."
838
- # print(error_msg)
839
- # return error_msg, None, gr.Button(visible=False)
840
- # except Exception as e:
841
- # error_msg = f"❌ An unexpected error occurred: {e}"
842
- # print(error_msg)
843
- # import traceback
844
- # print(traceback.format_exc())
845
- # return error_msg, None, gr.Button(visible=False)
846
- #
847
- #
848
- # def download_model():
849
- # """
850
- # Returns the model file for download.
851
- # """
852
- # if os.path.exists(MODEL_FILE_PATH):
853
- # return MODEL_FILE_PATH
854
- # else:
855
- # # Check for any .pth files in current directory
856
- # pth_files = [f for f in os.listdir('.') if f.endswith('.pth')]
857
- # if pth_files:
858
- # return pth_files[0]
859
- #
860
- # # Check checkpoints directory
861
- # if os.path.exists(MODEL_OUTPUT_DIR):
862
- # pth_files = [os.path.join(MODEL_OUTPUT_DIR, f) for f in os.listdir(MODEL_OUTPUT_DIR) if f.endswith('.pth')]
863
- # if pth_files:
864
- # return pth_files[0]
865
- #
866
- # return None
867
- #
868
- #
869
- # # --- Gradio Interface Setup (using Blocks for a nicer layout) ---
870
- # with gr.Blocks(title="LayoutLMv3 Fine-Tuning App", theme=gr.themes.Soft()) as demo:
871
- # gr.Markdown("# 🚀 LayoutLMv3 Fine-Tuning on Hugging Face Spaces")
872
- # gr.Markdown(
873
- # """
874
- # Upload your Label Studio JSON file, set your hyperparameters, and click **Train Model** to fine-tune the LayoutLMv3 model.
875
- #
876
- # **⚠️ IMPORTANT - Free Tier Users:**
877
- # - **Download your model IMMEDIATELY** after training completes!
878
- # - The model file is **temporary** and will be deleted when the Space restarts.
879
- # - A download button will appear below once training is complete.
880
- #
881
- # **⏱️ Timeout Note:** Training may timeout on free tier. Consider reducing epochs or batch size for faster training.
882
- # """
883
- # )
884
- #
885
- # with gr.Row():
886
- # with gr.Column(scale=1):
887
- # gr.Markdown("### 📁 Dataset Upload")
888
- # file_input = gr.File(
889
- # label="Upload Label Studio JSON Dataset",
890
- # file_types=[".json"]
891
- # )
892
- #
893
- # gr.Markdown("---")
894
- # gr.Markdown("### ⚙️ Training Parameters")
895
- #
896
- # batch_size_input = gr.Slider(
897
- # minimum=1, maximum=16, step=1, value=4,
898
- # label="Batch Size",
899
- # info="Smaller = less memory, slower training"
900
- # )
901
- # epochs_input = gr.Slider(
902
- # minimum=1, maximum=10, step=1, value=3,
903
- # label="Epochs",
904
- # info="Fewer epochs = faster training (recommended: 3-5)"
905
- # )
906
- # lr_input = gr.Number(
907
- # value=5e-5, label="Learning Rate",
908
- # info="Default: 5e-5"
909
- # )
910
- # max_len_input = gr.Slider(
911
- # minimum=128, maximum=512, step=128, value=512,
912
- # label="Max Sequence Length",
913
- # info="Shorter = faster training, less memory"
914
- # )
915
- #
916
- # train_button = gr.Button("🔥 Start Training", variant="primary", size="lg")
917
- #
918
- # with gr.Column(scale=2):
919
- # gr.Markdown("### 📊 Training Progress")
920
- #
921
- # log_output = gr.Textbox(
922
- # label="Training Logs",
923
- # lines=25,
924
- # max_lines=30,
925
- # autoscroll=True,
926
- # show_copy_button=True,
927
- # placeholder="Click 'Start Training' to begin...\n\nLogs will appear here in real-time."
928
- # )
929
- #
930
- # gr.Markdown("### ⬇️ Download Trained Model")
931
- #
932
- # # Hidden state to store the file path
933
- # model_path_state = gr.State(value=None)
934
- #
935
- # # Download button (initially hidden)
936
- # download_btn = gr.Button(
937
- # "📥 Download Model (.pth file)",
938
- # variant="primary",
939
- # size="lg",
940
- # visible=False
941
- # )
942
- #
943
- # # File output for download
944
- # model_download = gr.File(
945
- # label="Your trained model will appear here",
946
- # interactive=False,
947
- # visible=True
948
- # )
949
- #
950
- # gr.Markdown(
951
- # """
952
- # **📥 Download Instructions:**
953
- # 1. Wait for training to complete (✅ appears in logs)
954
- # 2. Click the **"Download Model"** button above
955
- # 3. Save the `.pth` file to your local machine
956
- # 4. **Do this immediately** - file is temporary!
957
- #
958
- # **🔧 Troubleshooting:**
959
- # - If download button doesn't appear, check the logs for errors
960
- # - Try reducing epochs or batch size if timeout occurs
961
- # - Ensure your JSON file is properly formatted
962
- # """
963
- # )
964
- #
965
- # # Define the training action
966
- # train_button.click(
967
- # fn=train_model,
968
- # inputs=[file_input, batch_size_input, epochs_input, lr_input, max_len_input],
969
- # outputs=[log_output, model_path_state, download_btn],
970
- # api_name="train"
971
- # )
972
- #
973
- # # Define the download action
974
- # download_btn.click(
975
- # fn=lambda path: path,
976
- # inputs=[model_path_state],
977
- # outputs=[model_download]
978
- # )
979
- #
980
- # # Add example info
981
- # gr.Markdown(
982
- # """
983
- # ---
984
- # ### 📖 About
985
- # This Space fine-tunes LayoutLMv3 with CRF for document understanding tasks including:
986
- # - Questions, Options, Answers
987
- # - Section Headings
988
- # - Passages
989
- #
990
- # **Model Details:** LayoutLMv3-base + CRF layer for sequence labeling
991
- # """
992
- # )
993
- #
994
- # if __name__ == "__main__":
995
- # demo.launch()
996
-
997
-
998
-
999
 
1000
  import gradio as gr
1001
  import subprocess
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
 
2
  import gradio as gr
3
  import subprocess