diff --git a/README.md b/README.md index cc22851..1412360 100644 --- a/README.md +++ b/README.md @@ -16,18 +16,48 @@ Konstantinos N. Plataniotis, and Zhangyang Wang As show in figure above, we introduce **Comp**ositional **4D** Scene Generation. Previous work concentrate on object-centric 4D objects. In comparison, our work extends the boundaries to the demanding task of constructing compositional 4D scenes. We integrate GPT-4 to decompose the scene and design proper trajectories, resulting in larger-scale movements and more realistic object interactions. +## Representative Results ## Static Assets We release our pre-generated static assets in `data/` directory. During training we keep the static 3D Gaussians fixed and only optimize the deformation modules. -## Custom Prompts - -## Training - -## Testing +## Setup +```bash +conda env create -f environment.yml +conda activate Comp4D +pip install -r requirements.txt + +# 3D Gaussian Splatting modules, skip if you already installed them +# a modified gaussian splatting (+ depth, alpha rendering) +git clone --recursive https://github.com/ashawkey/diff-gaussian-rasterization +pip install ./diff-gaussian-rasterization +pip install ./simple-knn +``` + +## Example Case Training +#### Prompt Case +"a butterfly flies towards the flower" + +#### Compositional Scene training +``` +python train_comp.py --configs arguments/comp_butterfly_flower_zs.py -e butterflyflower_exp --image_weight_override 0.02 --nn_weight 1000 --with_reg --cfg_override 100.0 --loss_dx_weight_override 0.005 +``` + +## Rendering +``` +python render_comp_video.py --skip_train --configs arguments/comp_butterfly_flower_zs.py --skip_test --model_path output_demo/date/butterflyflower_exp_date/ --iteration 3000 +``` ## Citation If you find this repository/work helpful in your research, please consider citing the paper and starring the repo ⭐. +``` +@article{xu2024comp4d, + title={Comp4D: LLM-Guided Compositional 4D Scene Generation}, + author={Xu, Dejia and Liang, Hanwen and Bhatt, Neel P and Hu, Hezhen and Liang, Hanxue and Plataniotis, Konstantinos N and Wang, Zhangyang}, + journal={arXiv preprint arXiv:2403.16993}, + year={2024} +} +``` diff --git a/arguments/comp_butterfly_flower_vc.py b/arguments/comp_butterfly_flower_vc.py index 0e1af8a..841e03b 100644 --- a/arguments/comp_butterfly_flower_vc.py +++ b/arguments/comp_butterfly_flower_vc.py @@ -1,5 +1,5 @@ OptimizationParams = dict( - prompt='a butterfly flies around the flower', + prompt='a butterfly flies towards the flower', # first one is static obj_prompt = [ 'a flower', @@ -40,8 +40,8 @@ name="rose", rife=False, cloud_path = [ - './data/8w/a_flower.ply', - './data/8w/a_butterfly.ply', + './data/a_flower.ply', + './data/a_butterfly.ply', ] ) diff --git a/arguments/comp_butterfly_flower_zs.py b/arguments/comp_butterfly_flower_zs.py index e47fcf5..b56eee8 100644 --- a/arguments/comp_butterfly_flower_zs.py +++ b/arguments/comp_butterfly_flower_zs.py @@ -40,8 +40,8 @@ name="rose", rife=False, cloud_path = [ - './data/8w/a_flower.ply', - './data/8w/a_butterfly.ply', + './data/a_flower.ply', + './data/a_butterfly.ply', ] ) diff --git a/arguments/comp_fish_rock_vc.py b/arguments/comp_fish_rock_vc.py index bdd82f8..7cce903 100644 --- a/arguments/comp_fish_rock_vc.py +++ b/arguments/comp_fish_rock_vc.py @@ -40,8 +40,8 @@ name="rose", rife=False, cloud_path = [ - './data/8w/a_rock.ply', - './data/8w/a_yellow_fish.ply', + './data/a_rock.ply', + './data/a_yellow_fish.ply', ] ) diff --git a/arguments/comp_fish_rock_zs.py b/arguments/comp_fish_rock_zs.py index e879fad..772c672 100644 --- a/arguments/comp_fish_rock_zs.py +++ b/arguments/comp_fish_rock_zs.py @@ -40,8 +40,8 @@ name="rose", rife=False, cloud_path = [ - './data/8w/a_rock.ply', - './data/8w/a_yellow_fish.ply', + './data/a_rock.ply', + './data/a_yellow_fish.ply', ] ) diff --git a/data/8w/a_butterfly.ply b/data/8w/a_butterfly.ply deleted file mode 100644 index 3ab4031..0000000 Binary files a/data/8w/a_butterfly.ply and /dev/null differ diff --git a/data/8w/a_flower.ply b/data/8w/a_flower.ply deleted file mode 100644 index 92def12..0000000 Binary files a/data/8w/a_flower.ply and /dev/null differ diff --git a/data/8w/a_rock.ply b/data/8w/a_rock.ply deleted file mode 100644 index 0dc61d2..0000000 Binary files a/data/8w/a_rock.ply and /dev/null differ diff --git a/data/8w/a_yellow_fish.ply b/data/a_yellow_fish.ply similarity index 100% rename from data/8w/a_yellow_fish.ply rename to data/a_yellow_fish.ply diff --git a/environment.yml b/environment.yml index f20799b..455626a 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: 4DGen +name: Comp4D channels: - defaults dependencies: diff --git a/mesh.py b/mesh.py index 2809934..0768644 100644 --- a/mesh.py +++ b/mesh.py @@ -2,7 +2,7 @@ import os src = '../4dfy/output/fourdfy_stage_2_low_vram/' -dst = './data/8w/' +dst = './data/' os.makedirs(dst, exist_ok=True) for obj_dir in os.listdir(src): diff --git a/mesh2ply.sh b/mesh2ply.sh index a5658f2..2350b8c 100644 --- a/mesh2ply.sh +++ b/mesh2ply.sh @@ -1,6 +1 @@ -python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_baby_bunny@20240228-192002/save/it25000-export/model.obj data/8w/a_baby_bunny.ply -python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_cat_walking@20240223-122044/save/it25000-export/model.obj data/8w/a_cat_walking.ply -python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_stack_of_pancakes@20240223-175313/save/it25000-export/model.obj data/8w/a_stack_of_pancakes.ply -python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_corgi@20240223-060738/save/it25000-export/model.obj data/8w/a_corgi.ply -python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_standing_panda@20240221-125435/save/it25000-export/model.obj data/8w/a_standing_panda.ply -python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_horse_running@20240226-135936/save/it25000-export/model.obj data/8w/a_horse_running.ply +python mesh2ply_8w.py ../4dfy/output/fourdfy_stage_2_low_vram/a_baby_bunny@20240228-192002/save/it25000-export/model.obj data/a_baby_bunny.ply