From 7d0f3c0e0fc5d1d7b65c810ab252be58ae1dd76a Mon Sep 17 00:00:00 2001
From: Ahmed Ahmed <ahmedah@stanford.edu>
Date: Wed, 18 Sep 2024 14:23:36 -0700
Subject: [PATCH] update config

---
 config/olmo_7b_sft_convhf.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 config/olmo_7b_sft_convhf.yaml

diff --git a/config/olmo_7b_sft_convhf.yaml b/config/olmo_7b_sft_convhf.yaml
new file mode 100644
index 000000000..fceee87ce
--- /dev/null
+++ b/config/olmo_7b_sft_convhf.yaml
@@ -0,0 +1,12 @@
+model:  # 7B class model
+  type: llama
+  seq_len: 2048
+  hidden_dim: 4096
+  intermediate_dim: 11008
+  num_layers: 32
+  num_heads: 32
+  num_kv_heads: 32
+  use_flash_attention: True
+  use_bias: false
+  use_layer_norm_weight: false
+  flash_attention_block_size: 1024