Add further aligntment

make dropout % and window size match
ReaLLMASIC · Sep 17, 2024 · 57be408 · 57be408
1 parent de054e7
commit 57be408
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/gpt_conf.py b/gpt_conf.py
@@ -7,7 +7,7 @@ class GPTConfig:
  vocab_size: int = 50304 # GPT-2 vocab_size of 50257, padded up to nearest multiple of 64 for efficiency
  n_layer: int = 6
  n_head: int = 6
- n_kv_group: int = 6
+ n_kv_group: int = None
  n_embd: int = 384
 
  # Steering Vectors
@@ -38,8 +38,8 @@ class GPTConfig:
  export_scale_matrices_npz: str = None
  export_scale_matrices_each_eval: bool = False
 
- dropout: float = 0.0
- window_size: int = 128
+ dropout: float = 0.2
+ window_size: int = None
  gate: bool = False
  use_moe: bool = False
  moe_layer_freq: int = 2