Upload folder using huggingface_hub

README.md

@@ -1,5 +1,6 @@
 ---
 thumbnail: "https://assets-global.website-files.com/646b351987a8d8ce158d1940/64ec9e96b4334c0e1ac41504_Logo%20with%20white%20text.svg"
+base_model: ORIGINAL_REPO_NAME
 metrics:
 - memory_disk
 - memory_inference
@@ -30,7 +31,7 @@ tags:
 - Contact us and tell us which model to compress next [here](https://www.pruna.ai/contact).
 - Request access to easily compress your *own* AI models [here](https://z0halsaff74.typeform.com/pruna-access?typeform-source=www.pruna.ai).
 - Read the documentations to know more [here](https://pruna-ai-pruna.readthedocs-hosted.com/en/latest/)
-- Join Pruna AI community on Discord [here](https://discord.gg/rskEr4BZJx) to share feedback/suggestions or get help.
+- Join Pruna AI community on Discord [here](https://discord.gg/CP4VSgck) to share feedback/suggestions or get help.
 
 ## Results
 
@@ -39,7 +40,7 @@ tags:
 **Frequently Asked Questions**
 - ***How does the compression work?*** The model is compressed with llm-int8.
 - ***How does the model quality change?*** The quality of the model output might vary compared to the base model.
-- ***How is the model efficiency evaluated?*** These results were obtained on NVIDIA A100-PCIE-40GB with configuration described in `model/smash_config.json` and are obtained after a hardware warmup. The smashed model is directly compared to the original base model. Efficiency results may vary in other settings (e.g. other hardware, image size, batch size, ...). We recommend to directly run them in the use-case conditions to know if the smashed model can benefit you.
+- ***How is the model efficiency evaluated?*** These results were obtained with configuration described in `model/smash_config.json` and are obtained after a hardware warmup. The smashed model is directly compared to the original base model. Efficiency results may vary in other settings (e.g. other hardware, image size, batch size, ...). We recommend to directly run them in the use-case conditions to know if the smashed model can benefit you.
 - ***What is the model format?*** We use safetensors.
 - ***What calibration data has been used?*** If needed by the compression method, we used WikiText as the calibration data.
 - ***What is the naming convention for Pruna Huggingface models?*** We take the original model name and append "turbo", "tiny", or "green" if the smashed model has a measured inference speed, inference memory, or inference energy consumption which is less than 90% of the original base model.
@@ -51,7 +52,7 @@ tags:
 
 You can run the smashed model with these steps:
 
-0. Check requirements from the original repo deepseek-ai/deepseek-coder-6.7b-instruct installed. In particular, check python, cuda, and transformers versions.
+0. Check requirements from the original repo ORIGINAL_REPO_NAME installed. In particular, check python, cuda, and transformers versions.
 1. Make sure that you have installed quantization related packages.
     ```bash
     pip install transformers accelerate bitsandbytes>0.37.0
@@ -59,15 +60,15 @@ You can run the smashed model with these steps:
 2. Load & run the model.
     ```python 
    from transformers import AutoModelForCausalLM, AutoTokenizer
+   
 
-    model = AutoModelForCausalLM.from_pretrained("PrunaAI/deepseek-ai-deepseek-coder-6.7b-instruct-bnb-8bit-smashed",
-                                                 trust_remote_code=True, device_map='auto')
-    tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-6.7b-instruct")
+   model = AutoModelForCausalLM.from_pretrained("PrunaAI/deepseek-ai-deepseek-coder-6.7b-instruct-bnb-8bit-smashed", trust_remote_code=True, device_map='auto')
+   tokenizer = AutoTokenizer.from_pretrained("ORIGINAL_REPO_NAME")
     
-    input_ids = tokenizer("What is the color of prunes?,", return_tensors='pt').to(model.device)["input_ids"]
+   input_ids = tokenizer("What is the color of prunes?,", return_tensors='pt').to(model.device)["input_ids"]
     
-    outputs = model.generate(input_ids, max_new_tokens=216)
-    tokenizer.decode(outputs[0])
+   outputs = model.generate(input_ids, max_new_tokens=216)
+   tokenizer.decode(outputs[0])
     ```
 
 ## Configurations
@@ -76,7 +77,7 @@ The configuration info are in `smash_config.json`.
 
 ## Credits & License
 
-The license of the smashed model follows the license of the original model. Please check the license of the original model deepseek-ai/deepseek-coder-6.7b-instruct before using this model which provided the base model. The license  of the `pruna-engine` is [here](https://pypi.org/project/pruna-engine/) on Pypi.
+The license of the smashed model follows the license of the original model. Please check the license of the original model ORIGINAL_REPO_NAME before using this model which provided the base model. The license  of the `pruna-engine` is [here](https://pypi.org/project/pruna-engine/) on Pypi.
 
 ## Want to compress other models?
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/tmp/tmpngcl1iwp",
+  "_name_or_path": "/tmp/models/tmpqd9n0q63y95mbh0r",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -7,18 +7,23 @@
   "attention_dropout": 0.0,
   "bos_token_id": 32013,
   "eos_token_id": 32021,
+  "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 11008,
   "max_position_embeddings": 16384,
+  "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 32,
   "pretraining_tp": 1,
   "quantization_config": {
+    "_load_in_4bit": false,
+    "_load_in_8bit": true,
     "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "fp4",
     "bnb_4bit_use_double_quant": false,
     "llm_int8_enable_fp32_cpu_offload": false,
@@ -34,12 +39,13 @@
   "rms_norm_eps": 1e-06,
   "rope_scaling": {
     "factor": 4.0,
+    "rope_type": "linear",
     "type": "linear"
   },
   "rope_theta": 100000,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
-  "transformers_version": "4.37.1",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
   "use_cache": true,
   "vocab_size": 32256
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 32013,
   "eos_token_id": 32021,
-  "transformers_version": "4.37.1"
+  "transformers_version": "4.48.2"
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32beb7aec8302ee92bb644654835afd260210827370d23e65a81f8ff617c312d
-size 4990371688
+oid sha256:0a419eaa5d29e10c2b563e1aac76974e8172397a4f3f40cdb5a283fb3565b4fe
+size 4990390389

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc797e6c2e36f48489870e4413fcf2682c16eb39a68ec069c4c40c7b5d026a43
-size 2020143664
+oid sha256:afdc73249db0bc771722097326e32064589a32cb06298ca2c5a9a9f0717229be
+size 2020150371

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 7010459648
+    "total_size": 7010459872
   },
   "weight_map": {
     "lm_head.weight": "model-00002-of-00002.safetensors",
@@ -8,515 +8,739 @@
     "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.23.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.23.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.23.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.23.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.23.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.23.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.23.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.25.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.25.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.25.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.26.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.26.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.26.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.27.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.27.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.27.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.28.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.28.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.28.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.29.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.29.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.29.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.down_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.gate_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.up_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.k_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.o_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.q_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.v_proj.SCB": "model-00002-of-00002.safetensors",
     "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight_format": "model-00002-of-00002.safetensors",
     "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.down_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.gate_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.up_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.k_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.o_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.q_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.v_proj.SCB": "model-00001-of-00002.safetensors",
     "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight_format": "model-00001-of-00002.safetensors",
     "model.norm.weight": "model-00002-of-00002.safetensors"
   }
 }

smash_config.json

@@ -1,27 +1,23 @@
 {
-    "api_key": null,
-    "verify_url": "http://johnrachwan.pythonanywhere.com",
-    "smash_config": {
-        "pruners": "None",
-        "factorizers": "None",
-        "quantizers": "['llm-int8']",
-        "compilers": "None",
-        "task": "text_text_generation",
-        "device": "cuda",
-        "cache_dir": "/ceph/hdd/staff/charpent/.cache/modelsl8ullxi1",
-        "batch_size": 1,
-        "model_name": "deepseek-ai/deepseek-coder-6.7b-instruct",
-        "pruning_ratio": 0.0,
-        "n_quantization_bits": 8,
-        "output_deviation": 0.005,
-        "max_batch_size": 1,
-        "qtype_weight": "torch.qint8",
-        "qtype_activation": "torch.quint8",
-        "qobserver": "<class 'torch.ao.quantization.observer.MinMaxObserver'>",
-        "qscheme": "torch.per_tensor_symmetric",
-        "qconfig": "x86",
-        "group_size": 128,
-        "damp_percent": 0.1,
-        "save_load_fn": "bitsandbytes"
-    }
+    "batchers": null,
+    "cachers": null,
+    "compilers": null,
+    "distillers": null,
+    "pruners": null,
+    "quantizers": "llm-int8",
+    "recoverers": null,
+    "quant_llm-int8_compute_dtype": "bfloat16",
+    "quant_llm-int8_double_quant": false,
+    "quant_llm-int8_enable_fp32_cpu_offload": false,
+    "quant_llm-int8_has_fp16_weight": false,
+    "quant_llm-int8_quant_type": "fp4",
+    "quant_llm-int8_threshold": 6.0,
+    "quant_llm-int8_weight_bits": 8,
+    "max_batch_size": 1,
+    "device": "cuda",
+    "cache_dir": "/tmp/models/tmpqd9n0q63",
+    "task": "",
+    "save_load_fn": "llm-int8",
+    "save_load_fn_args": {},
+    "api_key": null
 }