Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes.  
							See raw diff
- config.json +70 -0
- configuration_deepseek.py +210 -0
- model-00001-of-000163.safetensors +3 -0
- model-00002-of-000163.safetensors +3 -0
- model-00003-of-000163.safetensors +3 -0
- model-00004-of-000163.safetensors +3 -0
- model-00005-of-000163.safetensors +3 -0
- model-00006-of-000163.safetensors +3 -0
- model-00007-of-000163.safetensors +3 -0
- model-00008-of-000163.safetensors +3 -0
- model-00009-of-000163.safetensors +3 -0
- model-00010-of-000163.safetensors +3 -0
- model-00011-of-000163.safetensors +3 -0
- model-00012-of-000163.safetensors +3 -0
- model-00013-of-000163.safetensors +3 -0
- model-00014-of-000163.safetensors +3 -0
- model-00015-of-000163.safetensors +3 -0
- model-00017-of-000163.safetensors +3 -0
- model-00018-of-000163.safetensors +3 -0
- model-00019-of-000163.safetensors +3 -0
- model-00020-of-000163.safetensors +3 -0
- model-00021-of-000163.safetensors +3 -0
- model-00022-of-000163.safetensors +3 -0
- model-00023-of-000163.safetensors +3 -0
- model-00024-of-000163.safetensors +3 -0
- model-00025-of-000163.safetensors +3 -0
- model-00026-of-000163.safetensors +3 -0
- model-00028-of-000163.safetensors +3 -0
- model-00030-of-000163.safetensors +3 -0
- model-00031-of-000163.safetensors +3 -0
- model-00032-of-000163.safetensors +3 -0
- model-00033-of-000163.safetensors +3 -0
- model-00034-of-000163.safetensors +3 -0
- model-00035-of-000163.safetensors +3 -0
- model-00036-of-000163.safetensors +3 -0
- model-00037-of-000163.safetensors +3 -0
- model-00039-of-000163.safetensors +3 -0
- model-00040-of-000163.safetensors +3 -0
- model-00041-of-000163.safetensors +3 -0
- model-00042-of-000163.safetensors +3 -0
- model-00043-of-000163.safetensors +3 -0
- model-00044-of-000163.safetensors +3 -0
- model-00045-of-000163.safetensors +3 -0
- model-00046-of-000163.safetensors +3 -0
- model-00047-of-000163.safetensors +3 -0
- model-00048-of-000163.safetensors +3 -0
- model-00049-of-000163.safetensors +3 -0
- model-00050-of-000163.safetensors +3 -0
- model-00051-of-000163.safetensors +3 -0
- model-00052-of-000163.safetensors +3 -0
    	
        config.json
    ADDED
    
    | @@ -0,0 +1,70 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            {
         | 
| 2 | 
            +
              "architectures": [
         | 
| 3 | 
            +
                "DeepseekV3ForCausalLM"
         | 
| 4 | 
            +
              ],
         | 
| 5 | 
            +
              "attention_bias": false,
         | 
| 6 | 
            +
              "attention_dropout": 0.0,
         | 
| 7 | 
            +
              "auto_map": {
         | 
| 8 | 
            +
                "AutoConfig": "configuration_deepseek.DeepseekV3Config",
         | 
| 9 | 
            +
                "AutoModel": "modeling_deepseek.DeepseekV3Model",
         | 
| 10 | 
            +
                "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
         | 
| 11 | 
            +
              },
         | 
| 12 | 
            +
              "aux_loss_alpha": 0.001,
         | 
| 13 | 
            +
              "bos_token_id": 0,
         | 
| 14 | 
            +
              "eos_token_id": 1,
         | 
| 15 | 
            +
              "ep_size": 1,
         | 
| 16 | 
            +
              "first_k_dense_replace": 3,
         | 
| 17 | 
            +
              "hidden_act": "silu",
         | 
| 18 | 
            +
              "hidden_size": 7168,
         | 
| 19 | 
            +
              "initializer_range": 0.02,
         | 
| 20 | 
            +
              "intermediate_size": 18432,
         | 
| 21 | 
            +
              "kv_lora_rank": 512,
         | 
| 22 | 
            +
              "max_position_embeddings": 163840,
         | 
| 23 | 
            +
              "model_type": "deepseek_v3",
         | 
| 24 | 
            +
              "moe_intermediate_size": 2048,
         | 
| 25 | 
            +
              "moe_layer_freq": 1,
         | 
| 26 | 
            +
              "n_group": 8,
         | 
| 27 | 
            +
              "n_routed_experts": 256,
         | 
| 28 | 
            +
              "n_shared_experts": 1,
         | 
| 29 | 
            +
              "norm_topk_prob": true,
         | 
| 30 | 
            +
              "num_attention_heads": 128,
         | 
| 31 | 
            +
              "num_experts_per_tok": 8,
         | 
| 32 | 
            +
              "num_hidden_layers": 61,
         | 
| 33 | 
            +
              "num_key_value_heads": 128,
         | 
| 34 | 
            +
              "num_nextn_predict_layers": 1,
         | 
| 35 | 
            +
              "pretraining_tp": 1,
         | 
| 36 | 
            +
              "q_lora_rank": 1536,
         | 
| 37 | 
            +
              "qk_nope_head_dim": 128,
         | 
| 38 | 
            +
              "qk_rope_head_dim": 64,
         | 
| 39 | 
            +
              "quantization_config": {
         | 
| 40 | 
            +
                "activation_scheme": "dynamic",
         | 
| 41 | 
            +
                "fmt": "e4m3",
         | 
| 42 | 
            +
                "quant_method": "fp8",
         | 
| 43 | 
            +
                "weight_block_size": [
         | 
| 44 | 
            +
                  128,
         | 
| 45 | 
            +
                  128
         | 
| 46 | 
            +
                ]
         | 
| 47 | 
            +
              },
         | 
| 48 | 
            +
              "rms_norm_eps": 1e-06,
         | 
| 49 | 
            +
              "rope_scaling": {
         | 
| 50 | 
            +
                "beta_fast": 32,
         | 
| 51 | 
            +
                "beta_slow": 1,
         | 
| 52 | 
            +
                "factor": 40,
         | 
| 53 | 
            +
                "mscale": 1.0,
         | 
| 54 | 
            +
                "mscale_all_dim": 1.0,
         | 
| 55 | 
            +
                "original_max_position_embeddings": 4096,
         | 
| 56 | 
            +
                "type": "yarn"
         | 
| 57 | 
            +
              },
         | 
| 58 | 
            +
              "rope_theta": 10000,
         | 
| 59 | 
            +
              "routed_scaling_factor": 2.5,
         | 
| 60 | 
            +
              "scoring_func": "sigmoid",
         | 
| 61 | 
            +
              "seq_aux": true,
         | 
| 62 | 
            +
              "tie_word_embeddings": false,
         | 
| 63 | 
            +
              "topk_group": 4,
         | 
| 64 | 
            +
              "topk_method": "noaux_tc",
         | 
| 65 | 
            +
              "torch_dtype": "bfloat16",
         | 
| 66 | 
            +
              "transformers_version": "4.46.3",
         | 
| 67 | 
            +
              "use_cache": true,
         | 
| 68 | 
            +
              "v_head_dim": 128,
         | 
| 69 | 
            +
              "vocab_size": 129280
         | 
| 70 | 
            +
            }
         | 
    	
        configuration_deepseek.py
    ADDED
    
    | @@ -0,0 +1,210 @@ | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            from transformers.configuration_utils import PretrainedConfig
         | 
| 2 | 
            +
            from transformers.utils import logging
         | 
| 3 | 
            +
             | 
| 4 | 
            +
            logger = logging.get_logger(__name__)
         | 
| 5 | 
            +
             | 
| 6 | 
            +
            DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
         | 
| 7 | 
            +
            class DeepseekV3Config(PretrainedConfig):
         | 
| 8 | 
            +
                r"""
         | 
| 9 | 
            +
                This is the configuration class to store the configuration of a [`DeepseekV3Model`]. It is used to instantiate an DeepSeek
         | 
| 10 | 
            +
                model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
         | 
| 11 | 
            +
                defaults will yield a similar configuration to that of the DeepSeek-V3.
         | 
| 12 | 
            +
             | 
| 13 | 
            +
                Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
         | 
| 14 | 
            +
                documentation from [`PretrainedConfig`] for more information.
         | 
| 15 | 
            +
             | 
| 16 | 
            +
             | 
| 17 | 
            +
                Args:
         | 
| 18 | 
            +
                    vocab_size (`int`, *optional*, defaults to 129280):
         | 
| 19 | 
            +
                        Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
         | 
| 20 | 
            +
                        `inputs_ids` passed when calling [`DeepseekV3Model`]
         | 
| 21 | 
            +
                    hidden_size (`int`, *optional*, defaults to 4096):
         | 
| 22 | 
            +
                        Dimension of the hidden representations.
         | 
| 23 | 
            +
                    intermediate_size (`int`, *optional*, defaults to 11008):
         | 
| 24 | 
            +
                        Dimension of the MLP representations.
         | 
| 25 | 
            +
                    moe_intermediate_size (`int`, *optional*, defaults to 1407):
         | 
| 26 | 
            +
                        Dimension of the MoE representations.
         | 
| 27 | 
            +
                    num_hidden_layers (`int`, *optional*, defaults to 32):
         | 
| 28 | 
            +
                        Number of hidden layers in the Transformer decoder.
         | 
| 29 | 
            +
                    num_nextn_predict_layers (`int`, *optional*, defaults to 1):
         | 
| 30 | 
            +
                        Number of nextn predict layers in the DeepSeekV3 Model.
         | 
| 31 | 
            +
                    num_attention_heads (`int`, *optional*, defaults to 32):
         | 
| 32 | 
            +
                        Number of attention heads for each attention layer in the Transformer decoder.
         | 
| 33 | 
            +
                    n_shared_experts (`int`, *optional*, defaults to None):
         | 
| 34 | 
            +
                        Number of shared experts, None means dense model.
         | 
| 35 | 
            +
                    n_routed_experts (`int`, *optional*, defaults to None):
         | 
| 36 | 
            +
                        Number of routed experts, None means dense model.
         | 
| 37 | 
            +
                    routed_scaling_factor (`float`, *optional*, defaults to 1.0):
         | 
| 38 | 
            +
                        Scaling factor or routed experts.
         | 
| 39 | 
            +
                    topk_method (`str`, *optional*, defaults to `gready`):
         | 
| 40 | 
            +
                        Topk method used in routed gate.
         | 
| 41 | 
            +
                    n_group (`int`, *optional*, defaults to None):
         | 
| 42 | 
            +
                        Number of groups for routed experts.
         | 
| 43 | 
            +
                    topk_group (`int`, *optional*, defaults to None):
         | 
| 44 | 
            +
                        Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
         | 
| 45 | 
            +
                    num_experts_per_tok (`int`, *optional*, defaults to None):
         | 
| 46 | 
            +
                        Number of selected experts, None means dense model.
         | 
| 47 | 
            +
                    moe_layer_freq (`int`, *optional*, defaults to 1):
         | 
| 48 | 
            +
                        The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
         | 
| 49 | 
            +
                    first_k_dense_replace (`int`, *optional*, defaults to 0):
         | 
| 50 | 
            +
                        Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
         | 
| 51 | 
            +
                                                                        \--k dense layers--/
         | 
| 52 | 
            +
                    norm_topk_prob (`bool`, *optional*, defaults to False):
         | 
| 53 | 
            +
                        Whether to normalize the weights of the routed experts.
         | 
| 54 | 
            +
                    scoring_func (`str`, *optional*, defaults to 'softmax'):
         | 
| 55 | 
            +
                        Method of computing expert weights.
         | 
| 56 | 
            +
                    aux_loss_alpha (`float`, *optional*, defaults to 0.001):
         | 
| 57 | 
            +
                        Auxiliary loss weight coefficient.
         | 
| 58 | 
            +
                    seq_aux = (`bool`, *optional*, defaults to True):
         | 
| 59 | 
            +
                        Whether to compute the auxiliary loss for each individual sample.
         | 
| 60 | 
            +
                    num_key_value_heads (`int`, *optional*):
         | 
| 61 | 
            +
                        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
         | 
| 62 | 
            +
                        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
         | 
| 63 | 
            +
                        `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
         | 
| 64 | 
            +
                        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
         | 
| 65 | 
            +
                        by meanpooling all the original heads within that group. For more details checkout [this
         | 
| 66 | 
            +
                        paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
         | 
| 67 | 
            +
                        `num_attention_heads`.
         | 
| 68 | 
            +
                    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
         | 
| 69 | 
            +
                        The non-linear activation function (function or string) in the decoder.
         | 
| 70 | 
            +
                    max_position_embeddings (`int`, *optional*, defaults to 2048):
         | 
| 71 | 
            +
                        The maximum sequence length that this model might ever be used with.
         | 
| 72 | 
            +
                    initializer_range (`float`, *optional*, defaults to 0.02):
         | 
| 73 | 
            +
                        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
         | 
| 74 | 
            +
                    rms_norm_eps (`float`, *optional*, defaults to 1e-06):
         | 
| 75 | 
            +
                        The epsilon used by the rms normalization layers.
         | 
| 76 | 
            +
                    use_cache (`bool`, *optional*, defaults to `True`):
         | 
| 77 | 
            +
                        Whether or not the model should return the last key/values attentions (not used by all models). Only
         | 
| 78 | 
            +
                        relevant if `config.is_decoder=True`.
         | 
| 79 | 
            +
                    pad_token_id (`int`, *optional*):
         | 
| 80 | 
            +
                        Padding token id.
         | 
| 81 | 
            +
                    bos_token_id (`int`, *optional*, defaults to 1):
         | 
| 82 | 
            +
                        Beginning of stream token id.
         | 
| 83 | 
            +
                    eos_token_id (`int`, *optional*, defaults to 2):
         | 
| 84 | 
            +
                        End of stream token id.
         | 
| 85 | 
            +
                    pretraining_tp (`int`, *optional*, defaults to 1):
         | 
| 86 | 
            +
                        Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
         | 
| 87 | 
            +
                        document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
         | 
| 88 | 
            +
                        necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
         | 
| 89 | 
            +
                        issue](https://github.com/pytorch/pytorch/issues/76232).
         | 
| 90 | 
            +
                    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
         | 
| 91 | 
            +
                        Whether to tie weight embeddings
         | 
| 92 | 
            +
                    rope_theta (`float`, *optional*, defaults to 10000.0):
         | 
| 93 | 
            +
                        The base period of the RoPE embeddings.
         | 
| 94 | 
            +
                    rope_scaling (`Dict`, *optional*):
         | 
| 95 | 
            +
                        Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
         | 
| 96 | 
            +
                        strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
         | 
| 97 | 
            +
                        `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
         | 
| 98 | 
            +
                        `max_position_embeddings` to the expected new maximum.
         | 
| 99 | 
            +
                    attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
         | 
| 100 | 
            +
                        Whether to use a bias in the query, key, value and output projection layers during self-attention.
         | 
| 101 | 
            +
                    attention_dropout (`float`, *optional*, defaults to 0.0):
         | 
| 102 | 
            +
                        The dropout ratio for the attention probabilities.
         | 
| 103 | 
            +
             | 
| 104 | 
            +
                ```python
         | 
| 105 | 
            +
                >>> from transformers import DeepseekV3Model, DeepseekV3Config
         | 
| 106 | 
            +
             | 
| 107 | 
            +
                >>> # Initializing a Deepseek-V3 style configuration
         | 
| 108 | 
            +
                >>> configuration = DeepseekV3Config()
         | 
| 109 | 
            +
             | 
| 110 | 
            +
                >>> # Accessing the model configuration
         | 
| 111 | 
            +
                >>> configuration = model.config
         | 
| 112 | 
            +
                ```"""
         | 
| 113 | 
            +
             | 
| 114 | 
            +
                model_type = "deepseek_v3"
         | 
| 115 | 
            +
                keys_to_ignore_at_inference = ["past_key_values"]
         | 
| 116 | 
            +
             | 
| 117 | 
            +
                def __init__(
         | 
| 118 | 
            +
                    self,
         | 
| 119 | 
            +
                    vocab_size=129280,
         | 
| 120 | 
            +
                    hidden_size=7168,
         | 
| 121 | 
            +
                    intermediate_size=18432,
         | 
| 122 | 
            +
                    moe_intermediate_size = 2048,
         | 
| 123 | 
            +
                    num_hidden_layers=61,
         | 
| 124 | 
            +
                    num_nextn_predict_layers=1,
         | 
| 125 | 
            +
                    num_attention_heads=128,
         | 
| 126 | 
            +
                    num_key_value_heads=128,
         | 
| 127 | 
            +
                    n_shared_experts = 1,
         | 
| 128 | 
            +
                    n_routed_experts = 256,
         | 
| 129 | 
            +
                    ep_size = 1,
         | 
| 130 | 
            +
                    routed_scaling_factor = 2.5,
         | 
| 131 | 
            +
                    kv_lora_rank = 512,
         | 
| 132 | 
            +
                    q_lora_rank = 1536,
         | 
| 133 | 
            +
                    qk_rope_head_dim = 64,
         | 
| 134 | 
            +
                    v_head_dim = 128,
         | 
| 135 | 
            +
                    qk_nope_head_dim = 128,
         | 
| 136 | 
            +
                    topk_method = 'noaux_tc',
         | 
| 137 | 
            +
                    n_group = 8,
         | 
| 138 | 
            +
                    topk_group = 4,
         | 
| 139 | 
            +
                    num_experts_per_tok = 8,
         | 
| 140 | 
            +
                    moe_layer_freq = 1,
         | 
| 141 | 
            +
                    first_k_dense_replace = 3,
         | 
| 142 | 
            +
                    norm_topk_prob = True,
         | 
| 143 | 
            +
                    scoring_func = 'sigmoid',
         | 
| 144 | 
            +
                    aux_loss_alpha = 0.001,
         | 
| 145 | 
            +
                    seq_aux = True,
         | 
| 146 | 
            +
                    hidden_act="silu",
         | 
| 147 | 
            +
                    max_position_embeddings=4096,
         | 
| 148 | 
            +
                    initializer_range=0.02,
         | 
| 149 | 
            +
                    rms_norm_eps=1e-6,
         | 
| 150 | 
            +
                    use_cache=True,
         | 
| 151 | 
            +
                    pad_token_id=None,
         | 
| 152 | 
            +
                    bos_token_id=0,
         | 
| 153 | 
            +
                    eos_token_id=1,
         | 
| 154 | 
            +
                    pretraining_tp=1,
         | 
| 155 | 
            +
                    tie_word_embeddings=False,
         | 
| 156 | 
            +
                    rope_theta=10000.0,
         | 
| 157 | 
            +
                    rope_scaling=None,
         | 
| 158 | 
            +
                    attention_bias=False,
         | 
| 159 | 
            +
                    attention_dropout=0.0,
         | 
| 160 | 
            +
                    **kwargs,
         | 
| 161 | 
            +
                ):
         | 
| 162 | 
            +
                    self.vocab_size = vocab_size
         | 
| 163 | 
            +
                    self.max_position_embeddings = max_position_embeddings
         | 
| 164 | 
            +
                    self.hidden_size = hidden_size
         | 
| 165 | 
            +
                    self.intermediate_size = intermediate_size
         | 
| 166 | 
            +
                    self.moe_intermediate_size = moe_intermediate_size
         | 
| 167 | 
            +
                    self.num_hidden_layers = num_hidden_layers
         | 
| 168 | 
            +
                    self.num_nextn_predict_layers = num_nextn_predict_layers
         | 
| 169 | 
            +
                    self.num_attention_heads = num_attention_heads
         | 
| 170 | 
            +
                    self.n_shared_experts = n_shared_experts
         | 
| 171 | 
            +
                    self.n_routed_experts = n_routed_experts
         | 
| 172 | 
            +
                    self.ep_size = ep_size
         | 
| 173 | 
            +
                    self.routed_scaling_factor = routed_scaling_factor
         | 
| 174 | 
            +
                    self.kv_lora_rank = kv_lora_rank
         | 
| 175 | 
            +
                    self.q_lora_rank = q_lora_rank
         | 
| 176 | 
            +
                    self.qk_rope_head_dim = qk_rope_head_dim
         | 
| 177 | 
            +
                    self.v_head_dim = v_head_dim
         | 
| 178 | 
            +
                    self.qk_nope_head_dim = qk_nope_head_dim
         | 
| 179 | 
            +
                    self.topk_method = topk_method
         | 
| 180 | 
            +
                    self.n_group = n_group
         | 
| 181 | 
            +
                    self.topk_group = topk_group
         | 
| 182 | 
            +
                    self.num_experts_per_tok = num_experts_per_tok
         | 
| 183 | 
            +
                    self.moe_layer_freq = moe_layer_freq
         | 
| 184 | 
            +
                    self.first_k_dense_replace = first_k_dense_replace
         | 
| 185 | 
            +
                    self.norm_topk_prob = norm_topk_prob
         | 
| 186 | 
            +
                    self.scoring_func = scoring_func
         | 
| 187 | 
            +
                    self.aux_loss_alpha = aux_loss_alpha
         | 
| 188 | 
            +
                    self.seq_aux = seq_aux
         | 
| 189 | 
            +
                    # for backward compatibility
         | 
| 190 | 
            +
                    if num_key_value_heads is None:
         | 
| 191 | 
            +
                        num_key_value_heads = num_attention_heads
         | 
| 192 | 
            +
             | 
| 193 | 
            +
                    self.num_key_value_heads = num_key_value_heads
         | 
| 194 | 
            +
                    self.hidden_act = hidden_act
         | 
| 195 | 
            +
                    self.initializer_range = initializer_range
         | 
| 196 | 
            +
                    self.rms_norm_eps = rms_norm_eps
         | 
| 197 | 
            +
                    self.pretraining_tp = pretraining_tp
         | 
| 198 | 
            +
                    self.use_cache = use_cache
         | 
| 199 | 
            +
                    self.rope_theta = rope_theta
         | 
| 200 | 
            +
                    self.rope_scaling = rope_scaling
         | 
| 201 | 
            +
                    self.attention_bias = attention_bias
         | 
| 202 | 
            +
                    self.attention_dropout = attention_dropout
         | 
| 203 | 
            +
             | 
| 204 | 
            +
                    super().__init__(
         | 
| 205 | 
            +
                        pad_token_id=pad_token_id,
         | 
| 206 | 
            +
                        bos_token_id=bos_token_id,
         | 
| 207 | 
            +
                        eos_token_id=eos_token_id,
         | 
| 208 | 
            +
                        tie_word_embeddings=tie_word_embeddings,
         | 
| 209 | 
            +
                        **kwargs,
         | 
| 210 | 
            +
                    )
         | 
    	
        model-00001-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:673331cdfd7495679af5350feb865812f17a8cc569c077871a9dcd30342d5319
         | 
| 3 | 
            +
            size 5234138288
         | 
    	
        model-00002-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ece9d76e6478958a6f8037ef31215a7bd68181212206e1a70c19f46b12cdcf35
         | 
| 3 | 
            +
            size 4302381728
         | 
    	
        model-00003-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:e09f80c00b0d13ed787dd863579b49d144a1dea12c2913b44c682f84e4c9326d
         | 
| 3 | 
            +
            size 4302382136
         | 
    	
        model-00004-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:98f3721a54910781f375872b94551a3115f0bbab5fa02d2927188b5b09225bcb
         | 
| 3 | 
            +
            size 4302347768
         | 
    	
        model-00005-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:2f14cea6b84f478c9a74f882e3a81cd9631434d52eb5a28846a9022955598f37
         | 
| 3 | 
            +
            size 4302381912
         | 
    	
        model-00006-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:158cb3e3c31f8c26de0034365f80b8d0c0af7a925e33ba456163d02974ccc560
         | 
| 3 | 
            +
            size 4372071352
         | 
    	
        model-00007-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:6b0f244617edf612f7c15d2bf4ef853ccb0ceeb0ef92118f95d14f8272823808
         | 
| 3 | 
            +
            size 4306077848
         | 
    	
        model-00008-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:5bb9af11421c7e46cd5fff833f47c138a1a02a1c65ff713ee5c8f53f85f08f3b
         | 
| 3 | 
            +
            size 4302382112
         | 
    	
        model-00009-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:0f2f59e1478d96e1fb05c39ae1c7d845735691d1628bed66879efe33eaa790b9
         | 
| 3 | 
            +
            size 4302347960
         | 
    	
        model-00010-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:98d74b467f787f60a313567c022946298f111acaef1047888bf36bc56730a5b9
         | 
| 3 | 
            +
            size 4302381720
         | 
    	
        model-00011-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:28d322ff4839f888fc2d2194cef827b27272c6ffcfbe34497e7931f65916b126
         | 
| 3 | 
            +
            size 4302382136
         | 
    	
        model-00012-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:493b8fc87ad8700c3c7bff412d593867c20afba172dd268eed59974254b50db0
         | 
| 3 | 
            +
            size 1321583272
         | 
    	
        model-00013-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:7ccfefbeeff9a9382b56db78343fe10543583ca54f54007baddd634462a1d163
         | 
| 3 | 
            +
            size 4302314992
         | 
    	
        model-00014-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:960ecda16ab550c256c4766f2fa085cbff0e623a8820762924e33efb47b99645
         | 
| 3 | 
            +
            size 4302382088
         | 
    	
        model-00015-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:1415e26d8c90477d66f3e46d65a8760dd511ce66d8fb47371abf7cbacc2b5ab3
         | 
| 3 | 
            +
            size 4302347992
         | 
    	
        model-00017-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:361d32d33d9da1875c1eb9f2057515fb2d153c73bb688228ebcc07a41a2ae536
         | 
| 3 | 
            +
            size 4302382136
         | 
    	
        model-00018-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:109a371099e27f7373f59e931778db120236dbf1a7a1e47e32cbb670ae64eaee
         | 
| 3 | 
            +
            size 4302347800
         | 
    	
        model-00019-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:1caa71f022ab304d867c4d557e474250072b9cf236352ab81faca5ecec74303a
         | 
| 3 | 
            +
            size 4302381880
         | 
    	
        model-00020-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:308aa6a9c8172541aad334a491126e47c5b28905111a67c6fc24018f95f9dbe2
         | 
| 3 | 
            +
            size 4302382136
         | 
    	
        model-00021-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:8b6cd69f262bc921808bc38acfeaa064b1ea09a0a6a61b317695ec3677993c36
         | 
| 3 | 
            +
            size 4302348176
         | 
    	
        model-00022-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:7d3d32018b9b1893fefc26bf1e675ead3981d9c490d7a1a3d79ed3d4631146c7
         | 
| 3 | 
            +
            size 4302382656
         | 
    	
        model-00023-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:fe2bd75abd5672f554137bfc5418c7f35798edafedac29a77317f230a6aedf73
         | 
| 3 | 
            +
            size 4302348584
         | 
    	
        model-00024-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:073bb278d95c71cbf97229a4df0ff472c0ee14124cda15b9fe0668bdfc5c4014
         | 
| 3 | 
            +
            size 4302382264
         | 
    	
        model-00025-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:a4babb21e1758223657f77fe9b5ca4b7a13ac1be0b49c6b265317ca6cbd269ed
         | 
| 3 | 
            +
            size 4302382720
         | 
    	
        model-00026-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:5dbeaf9629bdd34c512afc280bde3f6ff121be3f681348bec823c4f12fac129d
         | 
| 3 | 
            +
            size 4302348392
         | 
    	
        model-00028-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:4fdbae4fab1874cbbde83d5a99ea2c3178d68442d5747d56ab6b122b51df92dd
         | 
| 3 | 
            +
            size 4302382720
         | 
    	
        model-00030-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:15ab069d233b7488ef92d051f50b0d32902b917dd242ea2e88897b6d3cb4a30a
         | 
| 3 | 
            +
            size 4302382640
         | 
    	
        model-00031-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:c37003878178e291abc2c4fc6688ea3cdd9baee2b34c73225c3c8fdbb3892479
         | 
| 3 | 
            +
            size 4302348600
         | 
    	
        model-00032-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:f9a9a8578c7836965c34933452771d729ebe5834b69127192a7b6e57db8bb4d1
         | 
| 3 | 
            +
            size 4302382248
         | 
    	
        model-00033-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:3b77ef50f467201af7c7fce8b60f1cfecff27f649811c0b005de617f9502d958
         | 
| 3 | 
            +
            size 4302382720
         | 
    	
        model-00034-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:7c9395b7e7f8e4c448f85d04c8e615bc3eb9e8701347835102aa43563ec4b470
         | 
| 3 | 
            +
            size 1747416576
         | 
    	
        model-00035-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:27a57644595016dbef35b5e819403ca26c0f0b3af25088475b3214d70d5a8dca
         | 
| 3 | 
            +
            size 4302315568
         | 
    	
        model-00036-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:a7d0a465271e80f9b482b337034c7b9ecf68fae02dfc9481233a907e6ff1c848
         | 
| 3 | 
            +
            size 4302382672
         | 
    	
        model-00037-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:fc3dbb381a9927cef08f0a6f7dfda21cec689de84b8ca7ebf665cc9827522a6a
         | 
| 3 | 
            +
            size 4302348568
         | 
    	
        model-00039-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:da48130dea9cf4f188d6071ddb0ea7835a34799ffe7d5e9670e81c4c983ec862
         | 
| 3 | 
            +
            size 4302382720
         | 
    	
        model-00040-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:dbd7ee0c7f3f62e97635d572fa0f236f0c680e3d76eae7e23aec07273b88c7a8
         | 
| 3 | 
            +
            size 4302348376
         | 
    	
        model-00041-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:72cbb3929c3e01370b046066852c2f97649f696011e60e7d9c8ae5c4a09f7ed7
         | 
| 3 | 
            +
            size 4302382472
         | 
    	
        model-00042-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:6b28fc75705b80909ab523ac8e9026f1661b2348575202004ea8b7f3db725300
         | 
| 3 | 
            +
            size 4302382720
         | 
    	
        model-00043-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ea3989e5a6fb006259534b6182150bbc4980f648717f90b986f45b7b0531cdea
         | 
| 3 | 
            +
            size 4302348184
         | 
    	
        model-00044-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:9fbfbb238a18e36c2fc30965ec3cf2590f4bc99950f082bceca72cc6f95a1a56
         | 
| 3 | 
            +
            size 4302382656
         | 
    	
        model-00045-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:f76be0b6ca28a62f8cbea4ae278b8e537a61c91762b1bc6b15b018e2535a97af
         | 
| 3 | 
            +
            size 4302348584
         | 
    	
        model-00046-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:fd2710070e58fbedc55ef55a49855c03d184055d7e0bf368348e99848c334401
         | 
| 3 | 
            +
            size 4302382264
         | 
    	
        model-00047-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:5675bcd0811712df0cf15669dc74bed52d1b7586de67ea0ce210f46d70a90d07
         | 
| 3 | 
            +
            size 4302382720
         | 
    	
        model-00048-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:78c9eb927b6940accec807f4d7673bd411f22251a86fb20641d1659bb98c5758
         | 
| 3 | 
            +
            size 4302348392
         | 
    	
        model-00049-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:ccd64d0bb0c85d97d15733733e73ee831ba296466ddebe2bd3b11314e5585654
         | 
| 3 | 
            +
            size 4302382448
         | 
    	
        model-00050-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:f3171f7b8b588becc39671ef6f74176e8fa50ac42f0d08defbc88e3f91c3c161
         | 
| 3 | 
            +
            size 4302382720
         | 
    	
        model-00051-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:892d1f4ea037b6130050d9585164aff5af820154207979bcffd4c53e3bf95c6e
         | 
| 3 | 
            +
            size 4302348200
         | 
    	
        model-00052-of-000163.safetensors
    ADDED
    
    | @@ -0,0 +1,3 @@ | |
|  | |
|  | |
|  | 
|  | |
| 1 | 
            +
            version https://git-lfs.github.com/spec/v1
         | 
| 2 | 
            +
            oid sha256:3d72b7c1c3bb2e575e9c250e965df44882f51e899d4183fa8dcdf8d526586cd6
         | 
| 3 | 
            +
            size 4302382640
         | 
