Update app.py

app.py

@@ -377,32 +377,104 @@ def generate_code_from_pseudo(pseudo_code, max_length, temperature, top_k, top_p
                     generated_codes.append(f"# Generation {i+1} failed: No valid tokens")
                     continue
                 
-                # Decode with skip_special_tokens=True for cleaner output
+                # Decode with comprehensive error handling
                 try:
+                    # First attempt: decode with skip_special_tokens=False
                     generated = loaded_tokenizer.decode(valid_tokens, skip_special_tokens=False)
+                    
+                    # Check if decode returned None or contains None
+                    if generated is None:
+                        raise ValueError("Tokenizer decode returned None")
+                    
+                    # Check for None in the string (shouldn't happen but be safe)
+                    if 'None' in str(generated) or '\x00' in str(generated):
+                        raise ValueError("Decoded string contains invalid characters")
+                        
                 except Exception as decode_error:
-                    # Fallback: try with skip_special_tokens=True
+                    # Second attempt: decode with skip_special_tokens=True
                     try:
                         generated = loaded_tokenizer.decode(valid_tokens, skip_special_tokens=True)
+                        if generated is None:
+                            raise ValueError("Tokenizer decode (skip_special) returned None")
+                        if 'None' in str(generated) or '\x00' in str(generated):
+                            raise ValueError("Decoded string contains invalid characters")
                     except Exception as decode_error2:
-                        # Last resort: convert tokens to string manually
-                        generated = f"# Decode failed: {str(decode_error2)}"
+                        # Third attempt: manual token-to-string conversion
+                        try:
+                            # Convert tokens to string manually using vocab
+                            if hasattr(loaded_tokenizer, 'get_vocab'):
+                                vocab = loaded_tokenizer.get_vocab()
+                                inv_vocab = {v: k for k, v in vocab.items()}
+                                
+                                # Convert tokens to strings, skip unknown tokens
+                                token_strings = []
+                                for token_id in valid_tokens:
+                                    if token_id in inv_vocab:
+                                        token_str = inv_vocab[token_id]
+                                        # Skip special tokens that might cause issues
+                                        if token_str not in ['<pad>', '<unk>', '<mask>', '<s>', '</s>', '<PAD>', '<SEP>', '<CODE>', '<PSEUDO>']:
+                                            token_strings.append(token_str)
+                                
+                                generated = ''.join(token_strings)
+                                
+                                if not generated or generated.isspace():
+                                    raise ValueError("Manual conversion produced empty string")
+                            else:
+                                raise ValueError("Tokenizer has no get_vocab method")
+                                
+                        except Exception as manual_error:
+                            # Final fallback: create a safe representation
+                            generated = f"# Decode failed: {str(decode_error)}\n# Manual conversion failed: {str(manual_error)}\n# Raw tokens: {valid_tokens[:10]}..."
+                
+                # Final safety check: ensure we have a string
+                if not isinstance(generated, str):
+                    generated = str(generated) if generated is not None else "# Decode returned non-string object"
                 
                 # Handle None result from decode
                 if generated is None:
                     generated = f"# Generation {i+1}: Decode returned None"
                 
-                # Extract code part
-                if '<CODE>' in generated:
-                    code = generated.split('<CODE>')[-1].strip()
-                    # Remove special tokens
-                    code = code.replace('<PAD>', '').replace('<SEP>', '').replace('</s>', '').replace('<s>', '').strip()
-                else:
-                    code = generated.strip()
+                # Extract code part with safety checks
+                try:
+                    if '<CODE>' in generated:
+                        code_parts = generated.split('<CODE>')
+                        if len(code_parts) > 1:
+                            code = code_parts[-1].strip()
+                        else:
+                            code = generated.strip()
+                    else:
+                        code = generated.strip()
+                    
+                    # Remove special tokens safely
+                    special_tokens = ['<PAD>', '<SEP>', '</s>', '<s>', '<unk>', '<mask>', '<|endoftext|>']
+                    for token in special_tokens:
+                        code = code.replace(token, '')
+                    
+                    # Clean up extra whitespace
+                    code = ' '.join(code.split())
+                    
+                    # Ensure we have some content
+                    if not code or code.isspace():
+                        code = f"# Generated sequence {i+1} was empty after cleaning"
+                        
+                except Exception as extract_error:
+                    code = f"# Error extracting code from sequence {i+1}: {str(extract_error)}"
                 
-                # Ensure we have some content
-                if not code or code.isspace():
-                    code = f"# Generated sequence {i+1} was empty"
+                # Final validation: ensure code is meaningful
+                try:
+                    # Check if code contains at least some alphanumeric characters
+                    if not any(c.isalnum() for c in code):
+                        code = f"# Generated sequence {i+1} contains no readable content"
+                    elif len(code) < 5:  # Too short to be meaningful
+                        code = f"# Generated sequence {i+1} too short: {code}"
+                    elif code.count('#') > len(code) * 0.8:  # Mostly error messages
+                        code = f"# Generated sequence {i+1} mostly errors: {code[:50]}..."
+                    else:
+                        # Looks good, keep as is
+                        pass
+                        
+                except Exception as validation_error:
+                    code = f"# Validation error for sequence {i+1}: {str(validation_error)}"
                 
                 generated_codes.append(code)
                 
@@ -413,7 +485,16 @@ def generate_code_from_pseudo(pseudo_code, max_length, temperature, top_k, top_p
         
         # Ensure we have at least one result
         if not generated_codes:
-            generated_codes = ["# No valid generations produced"]
+            generated_codes = ["# No valid generations produced - check model and tokenizer compatibility"]
+        
+        # Log generation summary for debugging
+        valid_generations = [code for code in generated_codes if not code.startswith('#')]
+        error_generations = [code for code in generated_codes if code.startswith('#')]
+        
+        if error_generations:
+            print(f"Generation completed: {len(valid_generations)} valid, {len(error_generations)} errors")
+            for error in error_generations[:3]:  # Log first 3 errors
+                print(f"  Error: {error[:100]}...")
         
         # Use the first generated code as primary output
         primary_code = generated_codes[0] if generated_codes else "# No code generated"