improve lora and merge

mesolitica · Dec 27, 2024 · ee40705 · ee40705
1 parent beaa0ce
commit ee40705
Show file tree

Hide file tree

Showing 10 changed files with 2,978 additions and 1 deletion.
diff --git a/session/llama3/lora-128-8b.sh b/session/llama3/lora-128-8b.sh
@@ -1,7 +1,8 @@
 WANDB_PROJECT=unsloth-Meta-Llama-3.1-8B-Instruct-lora-128-embedding-8k-multipack \
+HF_HOME="/workspace/cache" \
 TORCH_DISTRIBUTED_DEBUG="info" \
 torchrun --nproc_per_node 4 \
--m run-instruction-lora-liger-flex \
+-m run-instruction-lora-embedding-multipack \
 --deepspeed ds_config_zero3.json \
 --model_name_or_path unsloth/Meta-Llama-3.1-8B-Instruct \
 --per_device_train_batch_size 3 \