update to fp32 weights

config.json

@@ -186,7 +186,7 @@
     }
   },
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float32",
   "transformers_version": "4.52.3",
   "use_cache": true,
   "vit_config": {

model-00001-of-00004.safetensors → model-00001-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba6a7678f86a232a70a801fba317c3dfe8b754003e71861f54173f0bc53831e0
-size 4878581216
+oid sha256:44d1112e516614a010331bba82eeeac9d6039e2f15b1c3c58f2a01b9e536c3b5
+size 4978520816

model-00002-of-00004.safetensors → model-00002-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4a4a1c7e20477448b5ac6a2f3b6e502dd127fb43019d6095256f071f3f27a06
-size 4932745864
+oid sha256:500c53be3ecef1e9e6a02211d2322096affca2dce6ec4e0413f9c1059a18430d
+size 4778633920

model-00003-of-00004.safetensors → model-00003-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f5f9708d386c65a2e3b3d788ffdc360e5e305fdb9ac10f797b2f54e2c7f8f10
-size 4994552920
+oid sha256:314ccb5eee07eb898b0f7fc7b0e79e20a8c917acf0caa34a63379ab2e73a4401
+size 4661160168

model-00004-of-00004.safetensors → model-00004-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b0357c2b2be3790f125c6b2de468689cf857ee61f3d7183842e603eb64529aa
-size 1433042592
+oid sha256:cc6678709f4720f62d175f755fd92fc3a71528712eb6d7bdf7822c82ea0f7e96
+size 4661160192

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e5268ef2052de977b771942380ddafd7dd746ba3a1f3353483a1c6a20fa5fe5
+size 4661160192

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7273b2173b8c25346ec9071d2b3b8c489ce977bce9115ec19cf6b1027493a5b1
+size 4997750712

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32beeed0e17cfaf1fb2f83b39c28ba8044a6ef41a7a865b4d7e3f0751fbfa2c4
+size 3739371680

model.yaml

@@ -0,0 +1,257 @@
+model_name: molmo
+llm:
+  d_model: 3584
+  n_heads: 28
+  n_kv_heads: 4
+  head_dim: null
+  qkv_bias: true
+  clip_qkv: null
+  n_layers: 28
+  mlp_ratio: 4
+  mlp_hidden_size: 37888
+  activation_type: swiglu
+  block_type: sequential
+  rope: true
+  rope_full_precision: true
+  rope_theta: 1000000.0
+  rope_type: default
+  rope_factor: null
+  rope_high_freq_factor: null
+  rope_low_freq_factor: null
+  rope_original_max_position_embeddings: null
+  attention_type: sdpa
+  float32_attention: true
+  attention_dropout: 0.0
+  attention_layer_norm: false
+  attention_layer_norm_type: olmo
+  residual_dropout: 0.1
+  response_residual_dropout: 0.0
+  layer_norm_type: rms
+  layer_norm_with_affine: true
+  layer_norm_eps: 1.0e-06
+  attention_layer_norm_with_affine: true
+  max_sequence_length: 4096
+  max_position_embeddings: null
+  include_bias: false
+  bias_for_layer_norm: null
+  norm_after: false
+  moe_num_experts: 8
+  moe_top_k: 2
+  moe_mlp_impl: sparse
+  moe_log_expert_assignment: false
+  moe_shared_expert: false
+  moe_lbl_in_fp32: false
+  moe_interleave: false
+  moe_loss_weight: 0.1
+  moe_zloss_weight: null
+  moe_dropless: true
+  moe_capacity_factor: 1.25
+  embedding_dropout: 0.0
+  scale_logits: false
+  vocab_size: 152064
+  additional_vocab_size: 128
+  weight_tying: false
+  embedding_size: 152064
+  use_position_ids: true
+  tokenizer:
+    identifier: Qwen/Qwen2.5-7B
+    tokenizer_dir: null
+    depth_tokens: true
+  init_path: gs://mm-olmo/pretrained_llms/qwen2.5-7b.pt
+  init_incremental: null
+  new_embedding_init_range: 0.02
+  initializer_range: 0.02
+  normalize_input_embeds: false
+  activation_checkpoint: whole_layer
+  compile: blocks
+  fix_pad_tokenizer: false
+  resize_vocab: false
+  init_std: 0.02
+  init_fn: normal
+  init_cutoff_factor: null
+vision_backbone:
+  vit:
+    image_model_type: siglip
+    image_default_input_size:
+    - 378
+    - 378
+    image_patch_size: 14
+    image_pos_patch_size: 14
+    image_emb_dim: 1152
+    image_num_heads: 16
+    image_num_key_value_heads: 16
+    image_num_layers: 27
+    image_head_dim: 72
+    image_mlp_dim: 4304
+    image_mlp_activations: gelu_pytorch_tanh
+    image_dropout_rate: 0.0
+    image_num_pos: 729
+    image_norm_eps: 1.0e-06
+    attention_dropout: 0.0
+    residual_dropout: 0.0
+    initializer_range: 0.02
+    float32_attention: true
+    attention_type: sdpa
+    activation_checkpointing: true
+    init_path: gs://mm-olmo/pretrained_image_encoders/siglip2-so400m-14-384.pt
+    resize_mode: siglip
+    pad_value: 0.0
+    normalize: siglip
+  image_pooling_2d: attention_meanq
+  pooling_attention_mask: false
+  image_projector: mlp
+  image_padding_embed: null
+  vit_layers:
+  - -3
+  - -9
+  skip_unused_layers: true
+  image_feature_dropout: 0.0
+  connector_activation_checkpointing: true
+  compile_vit: blocks
+data_formatter:
+  prompt_templates: uber_model
+  message_format: role
+  system_prompt: demo_or_style
+  always_start_with_space: false
+  default_inference_len: 65
+  select_answer: best
+  debug: false
+  image_last: false
+  format_message_list: null
+  p_one_message: 0.0
+mm_preprocessor:
+  crop_mode: overlap-and-resize-c2
+  max_crops: 8
+  max_images: 2
+  max_multi_image_crops: 8
+  pooling_w: 2
+  pooling_h: 2
+  overlap_margins:
+  - 4
+  - 4
+  use_col_tokens: true
+  loss_token_weighting: root_subsegments
+  legacy_image_mask: false
+  max_answer_len: null
+  img_aug: true
+bi_directional_attn: null
+lora_enable: true
+lora_rank: 32
+lora_alpha: 16
+lora_dropout: 0.0
+lora_bias: none
+n_action_bins: 256
+norm_stats:
+  libero_object_no_noops_modified:
+    action:
+      mean:
+      - 0.07096529006958008
+      - 0.13498851656913757
+      - -0.04601382836699486
+      - 0.00123520044144243
+      - 0.006998839322477579
+      - -0.015027612447738647
+      - 0.46428999304771423
+      std:
+      - 0.2681235373020172
+      - 0.43846824765205383
+      - 0.4474974274635315
+      - 0.024446550756692886
+      - 0.049355510622262955
+      - 0.042107198387384415
+      - 0.49879148602485657
+      max:
+      - 0.9375
+      - 0.8919642567634583
+      - 0.9375
+      - 0.17678570747375488
+      - 0.35035714507102966
+      - 0.1810714304447174
+      - 1.0
+      min:
+      - -0.8839285969734192
+      - -0.9375
+      - -0.9375
+      - -0.15000000596046448
+      - -0.29035714268684387
+      - -0.32892856001853943
+      - 0.0
+      q01:
+      - -0.5383928418159485
+      - -0.8758928775787354
+      - -0.9375
+      - -0.06964285671710968
+      - -0.11678571254014969
+      - -0.15964286029338837
+      - 0.0
+      q99:
+      - 0.8464285731315613
+      - 0.84375
+      - 0.9375
+      - 0.08142857253551483
+      - 0.14892856776714325
+      - 0.0867857113480568
+      - 1.0
+    proprio:
+      mean:
+      - -0.02999030612409115
+      - -0.007947085425257683
+      - 0.20293472707271576
+      - 3.1086409091949463
+      - -0.21404768526554108
+      - -0.11307074874639511
+      - 0.0
+      - 0.029380427673459053
+      - -0.030556727200746536
+      std:
+      - 0.06694897264242172
+      - 0.17608462274074554
+      - 0.07807064801454544
+      - 0.0868484303355217
+      - 0.33540457487106323
+      - 0.20728276669979095
+      - 0.0
+      - 0.00956575945019722
+      - 0.009197483770549297
+      max:
+      - 0.14580604434013367
+      - 0.33216384053230286
+      - 0.3857804834842682
+      - 3.4003844261169434
+      - 0.7954911589622498
+      - 0.6642207503318787
+      - 0.0
+      - 0.04104341194033623
+      - -0.00018117300351150334
+      min:
+      - -0.1765444278717041
+      - -0.29457300901412964
+      - 0.008128180168569088
+      - 2.2890501022338867
+      - -1.883241891860962
+      - -1.0600427389144897
+      - 0.0
+      - 0.0006495157140307128
+      - -0.041782498359680176
+      q01:
+      - -0.14911890715360643
+      - -0.25978428691625594
+      - 0.009925739830359817
+      - 2.7545341420173646
+      - -1.3996034812927245
+      - -0.6867720144987106
+      - 0.0
+      - 0.008197814421728254
+      - -0.04015838988125324
+      q99:
+      - 0.09063626825809479
+      - 0.29066365867853167
+      - 0.3370887073874472
+      - 3.2611824750900267
+      - 0.32092821151018125
+      - 0.4037663781642913
+      - 0.0
+      - 0.039891827926039694
+      - -0.009106044843792932
+    num_transitions: 66984
+    num_trajectories: 454