update config

stanford-crfm · Sep 18, 2024 · 7d0f3c0 · 7d0f3c0
1 parent f27def8
commit 7d0f3c0
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/config/olmo_7b_sft_convhf.yaml b/config/olmo_7b_sft_convhf.yaml
@@ -0,0 +1,12 @@
+model:  # 7B class model
+  type: llama
+  seq_len: 2048
+  hidden_dim: 4096
+  intermediate_dim: 11008
+  num_layers: 32
+  num_heads: 32
+  num_kv_heads: 32
+  use_flash_attention: True
+  use_bias: false
+  use_layer_norm_weight: false
+  flash_attention_block_size: 1024