code release

VITA-Group · Apr 1, 2024 · 2851a73 · 2851a73
1 parent 8a61e4e
commit 2851a73
Show file tree

Hide file tree

Showing 12 changed files with 47 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -16,18 +16,48 @@ Konstantinos N. Plataniotis, and Zhangyang Wang
 
 As show in figure above, we introduce **Comp**ositional **4D** Scene Generation. Previous work concentrate on object-centric 4D objects. In comparison, our work extends the boundaries to the demanding task of constructing compositional 4D scenes. We integrate GPT-4 to decompose the scene and design proper trajectories, resulting in larger-scale movements and more realistic object interactions.
 
+## Representative Results
 
 
 ## Static Assets
 
 We release our pre-generated static assets in `data/` directory. During training we keep the static 3D Gaussians fixed and only optimize the deformation modules.
 
-## Custom Prompts
-
-## Training
-
-## Testing
+## Setup
+```bash
+conda env create -f environment.yml
+conda activate Comp4D
+pip install -r requirements.txt
+
+# 3D Gaussian Splatting modules, skip if you already installed them
+# a modified gaussian splatting (+ depth, alpha rendering)
+git clone --recursive https://github.com/ashawkey/diff-gaussian-rasterization
+pip install ./diff-gaussian-rasterization
+pip install ./simple-knn
+```
+
+## Example Case Training
+#### Prompt Case
+"a butterfly flies towards the flower"
+
+#### Compositional Scene training
+```
+python train_comp.py --configs arguments/comp_butterfly_flower_zs.py -e butterflyflower_exp --image_weight_override 0.02 --nn_weight 1000 --with_reg --cfg_override 100.0 --loss_dx_weight_override 0.005
+```
+
+## Rendering
+```
+python render_comp_video.py --skip_train --configs arguments/comp_butterfly_flower_zs.py --skip_test --model_path output_demo/date/butterflyflower_exp_date/ --iteration 3000
+```
 
 ## Citation
 
 If you find this repository/work helpful in your research, please consider citing the paper and starring the repo ⭐.
+```
+@article{xu2024comp4d,
+  title={Comp4D: LLM-Guided Compositional 4D Scene Generation},
+  author={Xu, Dejia and Liang, Hanwen and Bhatt, Neel P and Hu, Hezhen and Liang, Hanxue and Plataniotis, Konstantinos N and Wang, Zhangyang},
+  journal={arXiv preprint arXiv:2403.16993},
+  year={2024}
+}
+```
diff --git a/arguments/comp_butterfly_flower_vc.py b/arguments/comp_butterfly_flower_vc.py
@@ -1,5 +1,5 @@
 OptimizationParams = dict(
-    prompt='a butterfly flies around the flower',
+    prompt='a butterfly flies towards the flower',
     # first one is static
     obj_prompt = [
         'a flower',
@@ -40,8 +40,8 @@
     name="rose",
     rife=False,
     cloud_path = [
-        './data/8w/a_flower.ply',
-        './data/8w/a_butterfly.ply',
+        './data/a_flower.ply',
+        './data/a_butterfly.ply',
     ]
 )
 

diff --git a/arguments/comp_butterfly_flower_zs.py b/arguments/comp_butterfly_flower_zs.py
@@ -40,8 +40,8 @@
     name="rose",
     rife=False,
     cloud_path = [
-        './data/8w/a_flower.ply',
-        './data/8w/a_butterfly.ply',
+        './data/a_flower.ply',
+        './data/a_butterfly.ply',
     ]
 )
 

diff --git a/arguments/comp_fish_rock_vc.py b/arguments/comp_fish_rock_vc.py
@@ -40,8 +40,8 @@
     name="rose",
     rife=False,
     cloud_path = [
-        './data/8w/a_rock.ply',
-        './data/8w/a_yellow_fish.ply',
+        './data/a_rock.ply',
+        './data/a_yellow_fish.ply',
     ]
 )
 

diff --git a/arguments/comp_fish_rock_zs.py b/arguments/comp_fish_rock_zs.py
@@ -40,8 +40,8 @@
     name="rose",
     rife=False,
     cloud_path = [
-        './data/8w/a_rock.ply',
-        './data/8w/a_yellow_fish.ply',
+        './data/a_rock.ply',
+        './data/a_yellow_fish.ply',
     ]
 )
 

diff --git a/data/8w/a_butterfly.ply b/data/8w/a_butterfly.ply
diff --git a/data/8w/a_flower.ply b/data/8w/a_flower.ply
diff --git a/data/8w/a_rock.ply b/data/8w/a_rock.ply
diff --git a/data/8w/a_yellow_fish.ply → data/a_yellow_fish.ply b/data/8w/a_yellow_fish.ply → data/a_yellow_fish.ply
diff --git a/environment.yml b/environment.yml
@@ -1,4 +1,4 @@
-name: 4DGen
+name: Comp4D
 channels:
   - defaults
 dependencies:

diff --git a/mesh.py b/mesh.py
@@ -2,7 +2,7 @@
 import os
 
 src = '../4dfy/output/fourdfy_stage_2_low_vram/'
-dst = './data/8w/'
+dst = './data/'
 os.makedirs(dst, exist_ok=True)
 
 for obj_dir in os.listdir(src):

diff --git a/mesh2ply.sh b/mesh2ply.sh
@@ -1,6 +1 @@
-python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_baby_bunny@20240228-192002/save/it25000-export/model.obj data/8w/a_baby_bunny.ply
-python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_cat_walking@20240223-122044/save/it25000-export/model.obj data/8w/a_cat_walking.ply
-python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_stack_of_pancakes@20240223-175313/save/it25000-export/model.obj data/8w/a_stack_of_pancakes.ply
-python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_corgi@20240223-060738/save/it25000-export/model.obj data/8w/a_corgi.ply
-python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_standing_panda@20240221-125435/save/it25000-export/model.obj data/8w/a_standing_panda.ply
-python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_horse_running@20240226-135936/save/it25000-export/model.obj data/8w/a_horse_running.ply
+python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_baby_bunny@20240228-192002/save/it25000-export/model.obj data/a_baby_bunny.ply