Refactor generation functions to utilize a safe wrapper for HF pipeline calls, improving error handling and stability.

app.py

@@ -91,6 +91,18 @@ def get_textgen_pipeline():
         )
     return _gen_pipe
 
+def _safe_hf_generate(pipe, prompt: str, **gen_kwargs):
+    """Call HF generate pipeline with best-effort fallbacks to avoid TorchDynamo/Inductor issues."""
+    try:
+        return pipe(prompt, **gen_kwargs)
+    except Exception:
+        # Best-effort: disable dynamo via env and retry once
+        try:
+            os.environ["TORCHDYNAMO_DISABLE"] = "1"
+        except Exception:
+            pass
+        return pipe(prompt, **gen_kwargs)
+
 
 def set_current_model_id(new_model_id: str) -> str:
     global current_model_id, _gen_pipe
@@ -348,31 +360,17 @@ def generate_recording_agent_reply(chat_history: List[Tuple[str, str]]) -> str:
         import torch._dynamo as _dynamo  # type: ignore
     except Exception:
         _dynamo = None
-    if _dynamo is not None:
-        _dynamo.config.suppress_errors = True  # best-effort safe fallback
-    if hasattr(torch, "_dynamo"):
-        with torch._dynamo.disable():  # type: ignore[attr-defined]
-            gen = pipe(
-                prompt,
-                max_new_tokens=96,
-                temperature=0.7,
-                do_sample=True,
-                top_p=0.9,
-                top_k=50,
-                pad_token_id=tokenizer.eos_token_id,
-                return_full_text=False,
-            )
-    else:
-        gen = pipe(
-            prompt,
-            max_new_tokens=96,
-            temperature=0.7,
-            do_sample=True,
-            top_p=0.9,
-            top_k=50,
-            pad_token_id=tokenizer.eos_token_id,
-            return_full_text=False,
-        )
+    gen = _safe_hf_generate(
+        pipe,
+        prompt,
+        max_new_tokens=96,
+        temperature=0.7,
+        do_sample=True,
+        top_p=0.9,
+        top_k=50,
+        pad_token_id=tokenizer.eos_token_id,
+        return_full_text=False,
+    )
     reply = gen[0]["generated_text"].strip()
     # Ensure it's a single concise question/sentence
     if len(reply) > 300:
@@ -409,27 +407,15 @@ def scoring_agent_infer(chat_history: List[Tuple[str, str]], features: Dict[str,
         import torch._dynamo as _dynamo  # type: ignore
     except Exception:
         _dynamo = None
-    if _dynamo is not None:
-        _dynamo.config.suppress_errors = True
-    if hasattr(torch, "_dynamo"):
-        with torch._dynamo.disable():  # type: ignore[attr-defined]
-            gen = pipe(
-                prompt,
-                max_new_tokens=256,
-                temperature=0.0,
-                do_sample=False,
-                pad_token_id=tokenizer.eos_token_id,
-                return_full_text=False,
-            )
-    else:
-        gen = pipe(
-            prompt,
-            max_new_tokens=256,
-            temperature=0.0,
-            do_sample=False,
-            pad_token_id=tokenizer.eos_token_id,
-            return_full_text=False,
-        )
+    gen = _safe_hf_generate(
+        pipe,
+        prompt,
+        max_new_tokens=256,
+        temperature=0.0,
+        do_sample=False,
+        pad_token_id=tokenizer.eos_token_id,
+        return_full_text=False,
+    )
     out_text = gen[0]["generated_text"]
     parsed = safe_json_extract(out_text)