Fix bug with safe suffix removal (#34)
Browse files- Safe suffix removal (507cf490804a20826b1defcf7f4dbf9af57a0f78)
Co-authored-by: Junkun Chen <shtpgshus@users.noreply.huggingface.co>
    	
        sample_finetune_speech.py
    CHANGED
    
    | @@ -284,7 +284,7 @@ def evaluate( | |
| 284 | 
             
                        for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
         | 
| 285 | 
             
                    ]
         | 
| 286 | 
             
                    all_generated_texts.extend(generated_text)
         | 
| 287 | 
            -
                    labels = [processor.decode(_label_ids[_label_ids != 0]). | 
| 288 | 
             
                    all_labels.extend(labels)
         | 
| 289 |  | 
| 290 | 
             
                all_generated_texts = gather_object(all_generated_texts)
         | 
|  | |
| 284 | 
             
                        for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
         | 
| 285 | 
             
                    ]
         | 
| 286 | 
             
                    all_generated_texts.extend(generated_text)
         | 
| 287 | 
            +
                    labels = [processor.decode(_label_ids[_label_ids != 0]).removesuffix(ANSWER_SUFFIX) for _label_ids in inputs["labels"]]
         | 
| 288 | 
             
                    all_labels.extend(labels)
         | 
| 289 |  | 
| 290 | 
             
                all_generated_texts = gather_object(all_generated_texts)
         | 

 
		