Update app.py

app.py

@@ -386,13 +386,16 @@ def generate_code_from_pseudo(pseudo_code, max_length, temperature, top_k, top_p
                     if generated is None:
                         raise ValueError("Tokenizer decode returned None")
                     
-                    # Clean up common GPT-2 artifacts
+                    # Clean up common GPT-2 artifacts - more aggressive cleaning
                     generated = generated.replace('Ġ', ' ').replace('▁', ' ')  # Handle different space tokens
                     generated = ' '.join(generated.split())  # Normalize whitespace
                     
+                    # Additional cleaning for common BPE artifacts
+                    generated = generated.replace('<0x0A>', '\n').replace('<0x20>', ' ')
+                    
                     # Check for gibberish (too many special characters)
-                    special_ratio = sum(1 for c in generated if not c.isalnum() and c not in ' \n\t.,;()[]{}+-*/=<>!&|^~%#@?:\'\"') / max(len(generated), 1)
-                    if special_ratio > 0.5:  # More than 50% special chars = likely gibberish
+                    special_ratio = sum(1 for c in generated if not c.isalnum() and c not in ' \n\t.,;()[]{}+-*/=<>!&|^~%#@?:\'\"\\') / max(len(generated), 1)
+                    if special_ratio > 0.7:  # More than 70% special chars = likely gibberish
                         raise ValueError("Decoded output appears to be gibberish")
                         
                 except Exception as decode_error:
@@ -407,8 +410,8 @@ def generate_code_from_pseudo(pseudo_code, max_length, temperature, top_k, top_p
                         generated = ' '.join(generated.split())
                         
                         # Check for gibberish again
-                        special_ratio = sum(1 for c in generated if not c.isalnum() and c not in ' \n\t.,;()[]{}+-*/=<>!&|^~%#@?:\'\"') / max(len(generated), 1)
-                        if special_ratio > 0.5:
+                        special_ratio = sum(1 for c in generated if not c.isalnum() and c not in ' \n\t.,;()[]{}+-*/=<>!&|^~%#@?:\'\"\\') / max(len(generated), 1)
+                        if special_ratio > 0.7:
                             raise ValueError("Decoded output still appears to be gibberish")
                             
                     except Exception as decode_error2:
@@ -509,8 +512,11 @@ def generate_code_from_pseudo(pseudo_code, max_length, temperature, top_k, top_p
                 
                 # Final validation: ensure code is meaningful
                 try:
-                    # Check if code contains at least some alphanumeric characters
-                    if not any(c.isalnum() for c in code):
+                    # Check if code contains at least some alphanumeric characters or code keywords
+                    has_alnum = any(c.isalnum() for c in code)
+                    has_code_indicators = any(keyword in code.lower() for keyword in ['def ', 'class ', 'import ', 'if ', 'for ', 'while ', 'return ', 'print(', 'bool', 'int', 'str', 'list'])
+                    
+                    if not has_alnum and not has_code_indicators:
                         code = f"# Generated sequence {i+1} contains no readable content"
                     elif len(code) < 5:  # Too short to be meaningful
                         code = f"# Generated sequence {i+1} too short: {code}"