Gustavo de Rosa
		
	commited on
		
		
					Commit 
							
							·
						
						492fc70
	
1
								Parent(s):
							
							b15fb48
								
chore(root): Changes pad_token to <|dummy_85|>.
Browse files- config.json +1 -1
 - generation_config.json +1 -1
 - special_tokens_map.json +1 -1
 - tokenizer_config.json +1 -1
 
    	
        config.json
    CHANGED
    
    | 
         @@ -19,7 +19,7 @@ 
     | 
|
| 19 | 
         
             
              "num_hidden_layers": 40,
         
     | 
| 20 | 
         
             
              "num_key_value_heads": 10,
         
     | 
| 21 | 
         
             
              "original_max_position_embeddings": 16384,
         
     | 
| 22 | 
         
            -
              "pad_token_id":  
     | 
| 23 | 
         
             
              "resid_pdrop": 0.0,
         
     | 
| 24 | 
         
             
              "rms_norm_eps": 1e-05,
         
     | 
| 25 | 
         
             
              "rope_scaling": null,
         
     | 
| 
         | 
|
| 19 | 
         
             
              "num_hidden_layers": 40,
         
     | 
| 20 | 
         
             
              "num_key_value_heads": 10,
         
     | 
| 21 | 
         
             
              "original_max_position_embeddings": 16384,
         
     | 
| 22 | 
         
            +
              "pad_token_id": 100349,
         
     | 
| 23 | 
         
             
              "resid_pdrop": 0.0,
         
     | 
| 24 | 
         
             
              "rms_norm_eps": 1e-05,
         
     | 
| 25 | 
         
             
              "rope_scaling": null,
         
     | 
    	
        generation_config.json
    CHANGED
    
    | 
         @@ -2,6 +2,6 @@ 
     | 
|
| 2 | 
         
             
                "_from_model_config": true,
         
     | 
| 3 | 
         
             
                "bos_token_id": 100257,
         
     | 
| 4 | 
         
             
                "eos_token_id": 100265,
         
     | 
| 5 | 
         
            -
                "pad_token_id":  
     | 
| 6 | 
         
             
                "transformers_version": "4.47.0"
         
     | 
| 7 | 
         
             
            }
         
     | 
| 
         | 
|
| 2 | 
         
             
                "_from_model_config": true,
         
     | 
| 3 | 
         
             
                "bos_token_id": 100257,
         
     | 
| 4 | 
         
             
                "eos_token_id": 100265,
         
     | 
| 5 | 
         
            +
                "pad_token_id": 100349,
         
     | 
| 6 | 
         
             
                "transformers_version": "4.47.0"
         
     | 
| 7 | 
         
             
            }
         
     | 
    	
        special_tokens_map.json
    CHANGED
    
    | 
         @@ -1,5 +1,5 @@ 
     | 
|
| 1 | 
         
             
            {
         
     | 
| 2 | 
         
             
              "bos_token": "<|endoftext|>",
         
     | 
| 3 | 
         
             
              "eos_token": "<|im_end|>",
         
     | 
| 4 | 
         
            -
              "pad_token": "<| 
     | 
| 5 | 
         
             
            }
         
     | 
| 
         | 
|
| 1 | 
         
             
            {
         
     | 
| 2 | 
         
             
              "bos_token": "<|endoftext|>",
         
     | 
| 3 | 
         
             
              "eos_token": "<|im_end|>",
         
     | 
| 4 | 
         
            +
              "pad_token": "<|dummy_85|>"
         
     | 
| 5 | 
         
             
            }
         
     | 
    	
        tokenizer_config.json
    CHANGED
    
    | 
         @@ -775,6 +775,6 @@ 
     | 
|
| 775 | 
         
             
              "clean_up_tokenization_spaces": false,
         
     | 
| 776 | 
         
             
              "eos_token": "<|im_end|>",
         
     | 
| 777 | 
         
             
              "model_max_length": 16384,
         
     | 
| 778 | 
         
            -
              "pad_token": "<| 
     | 
| 779 | 
         
             
              "tokenizer_class": "GPT2Tokenizer"
         
     | 
| 780 | 
         
             
            }
         
     | 
| 
         | 
|
| 775 | 
         
             
              "clean_up_tokenization_spaces": false,
         
     | 
| 776 | 
         
             
              "eos_token": "<|im_end|>",
         
     | 
| 777 | 
         
             
              "model_max_length": 16384,
         
     | 
| 778 | 
         
            +
              "pad_token": "<|dummy_85|>",
         
     | 
| 779 | 
         
             
              "tokenizer_class": "GPT2Tokenizer"
         
     | 
| 780 | 
         
             
            }
         
     |