rename data process files and add together multiturn data preprocess

sotopia-lab · Nov 6, 2023 · 5b5720c · 5b5720c
1 parent 5248438
commit 5b5720c
Show file tree

Hide file tree

Showing 9 changed files with 19 additions and 711 deletions.
diff --git a/data_process/data/data_process.py b/data_process/data/data_process.py
diff --git a/data_process/dummyfile b/data_process/dummyfile
diff --git a/...fastchat_data/fastchat_data_preprocess.py → data_process/fastchat_data_preprocess.py b/...fastchat_data/fastchat_data_preprocess.py → data_process/fastchat_data_preprocess.py
diff --git a/...mpletion_data/together_data_preprocess.py → ...ata_process/completion_data_preprocess.py b/...mpletion_data/together_data_preprocess.py → ...ata_process/completion_data_preprocess.py
diff --git a/together_ai_ft/data_process/multiturn_data_preprocess.py b/together_ai_ft/data_process/multiturn_data_preprocess.py
@@ -0,0 +1,19 @@
+import json
+import os
+
+sotopia_data_dir = "/Users/pamela/Documents/capstone/sotopia-ft-data/GPT4-4_Redis_Easy_No_Filter"
+
+together_data_template = """<s>[INST] {user_msg} [/INST] {assistant_msg} </s>"""
+
+lines = []
+for file in os.listdir(sotopia_data_dir):
+    with open(os.path.join(sotopia_data_dir, file), 'r') as f:
+        file_dict = json.load(f)
+        text = together_data_template.format(
+            user_msg=file_dict["prompt"], assistant_msg=file_dict["result"])
+        lines.append(text)
+
+with open("together-ft-gpt4-gpt4-easy-no-filter.jsonl", 'w') as f:
+    for line in lines:
+        f.write(json.dumps({"text": line}))
+        f.write('\n')
diff --git a/together_ai_ft/together_ai_data.jsonl b/together_ai_ft/together_ai_data.jsonl