depth mods

wangyida · Jul 10, 2024 · cdd1708 · cdd1708
1 parent 934c547
commit cdd1708
Show file tree

Hide file tree

Showing 11 changed files with 390 additions and 262 deletions.
diff --git a/configs/nerf-hmvs-wbg.yaml b/configs/nerf-hmvs-wbg.yaml
@@ -5,17 +5,23 @@ seed: 42
 dataset:
  name: hmvs
  root_dir: /home/yidaw/Documents/datasets/Buildings
+ pose_file: extrinsics.npy # Tile_+002_+002_AT.bin
+ pcd_file: pointcloud.npz # false # /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
  mesh_outpath: false
+ img_wh:
+ - 960
+ - 540
  img_downscale: 4 # specify training image size by either img_wh or img_downscale
- cam_downscale: 75 # if false, to rescale the cameras/pts positions automatically
+ cam_downscale: false # 75 # if false, to rescale the cameras/pts positions automatically
  repose: true # false
  up_est_method: no-change # if true, use estimated ground plane normal direction as up direction
- center_est_method: point # camera/point
+ center_est_method: camera # camera/point
  n_test_traj_steps: 16
  apply_mask: false
  apply_depth: false
  load_data_on_gpu: false
- max_imgs: 400
+ max_imgs: 1200
+ preprocess_only: false
 
 model:
  name: nerf
@@ -55,7 +61,7 @@ model:
  n_neurons: 64
  n_hidden_layers: 1
  max_imgs: ${dataset.max_imgs}
- use_appearance_embedding: false
+ use_appearance_embedding: true # false
  use_average_appearance_embedding: ture
  appearance_embedding_dim: 16
  use_distortion: false
@@ -73,7 +79,7 @@ model:
  n_neurons: 64
  n_hidden_layers: 2
  max_imgs: ${dataset.max_imgs}
- use_appearance_embedding: false
+ use_appearance_embedding: true # false
  use_average_appearance_embedding: ture
  appearance_embedding_dim: 16
  # background model configurations
@@ -112,7 +118,7 @@ model:
  n_hidden_layers: 2
  color_activation: sigmoid
  max_imgs: ${dataset.max_imgs}
- use_appearance_embedding: false
+ use_appearance_embedding: true # false
  use_average_appearance_embedding: ture
  appearance_embedding_dim: 8
 

diff --git a/configs/neuralangelo-hmvs-360.yaml b/configs/neuralangelo-hmvs-360.yaml
@@ -5,8 +5,8 @@ seed: 42
 dataset:
  name: hmvs
  root_dir: /home/yidaw/Documents/datasets/DJI/DJI_010
- pose_path: /home/yidaw/Documents/datasets/DJI/DJI_010/Mesh/ATOrigin.bin
- pcd_path: /home/yidaw/Documents/datasets/DJI/DJI_010/Mesh/TexturedModel/Tile_+000_+000_0/building_interest.ply
+ pose_file: Mesh/ATOrigin.bin
+ pcd_file: Mesh/TexturedModel/Tile_+000_+000_0/building_interest.ply
  mesh_exported_path: false
  img_downscale: 4 # better with more than 1000 x 1000 resolution
  cam_downscale: 17 # 2.5 # if false, to rescale the cameras/pts positions automatically

diff --git a/configs/neuralangelo-hmvs-eagle-wbg.yaml b/configs/neuralangelo-hmvs-eagle-wbg.yaml
@@ -5,13 +5,13 @@ seed: 42
 dataset:
  name: hmvs
  root_dir: /home/yidaw/Documents/datasets/Buildings
- pose_path: /home/yidaw/Documents/datasets/Buildings/Building/Tile_+002_+002_AT.bin
- pcd_path: /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply
+ pose_file: extrinsics.npy # Tile_+002_+002_AT.bin
+ pcd_file: pointcloud.npz # false # /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
  mesh_outpath: false
  img_wh:
  - 960
  - 540
- img_downscale: 2 # 4 # better with more than 1000 x 1000 resolution
+ img_downscale: 4 # better with more than 1000 x 1000 resolution
  cam_downscale: 200 # if false, to rescale the cameras/pts positions automatically
  repose: true # false
  up_est_method: ground # no-change/camera/ground/z-axis 
@@ -20,7 +20,8 @@ dataset:
  apply_mask: false
  apply_depth: false
  load_data_on_gpu: false
- max_imgs: 1600
+ max_imgs: 1200
+ preprocess_only: false
 
 model:
  name: neus
@@ -200,8 +201,8 @@ trainer:
  max_steps: 6e4
  log_every_n_steps: 500
  num_sanity_val_steps: 0
- val_check_interval: 2e4
+ val_check_interval: 3e4
  limit_train_batches: 1.0
- limit_val_batches: 6 # 3
+ limit_val_batches: 7 # 3
  enable_progress_bar: true
  precision: 16
diff --git a/configs/neuralangelo-hmvs-eagle.yaml b/configs/neuralangelo-hmvs-eagle.yaml
@@ -5,8 +5,8 @@ seed: 42
 dataset:
  name: hmvs
  root_dir: /home/yidaw/Documents/datasets/Buildings
- pose_path: /home/yidaw/Documents/datasets/Buildings/Building/Tile_+002_+002_AT.bin
- pcd_path: /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply
+ pose_file: Building/Tile_+002_+002_AT.bin
+ pcd_file: Building/building_interest.ply
  mesh_outpath: false
  img_downscale: 4 # better with more than 1000 x 1000 resolution
  cam_downscale: 200 # if false, to rescale the cameras/pts positions automatically

diff --git a/configs/neus-hmvs-360-wbg.yaml b/configs/neus-hmvs-360-wbg.yaml
@@ -5,8 +5,8 @@ seed: 42
 dataset:
  name: hmvs
  root_dir: /home/yidaw/Documents/datasets/Buildings
- pose_path: /home/yidaw/Documents/datasets/Buildings/Building/Tile_+002_+002_AT.bin
- pcd_path: /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
+ pose_file: Tile_+002_+002_AT.bin
+ pcd_file: Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
  mesh_exported_path: false
  img_downscale: 4 # better with more than 1000 x 1000 resolution
  cam_downscale: 17 # if false, to rescale the cameras/pts positions automatically

diff --git a/configs/neus-hmvs-eagle-wbg.yaml b/configs/neus-hmvs-eagle-wbg.yaml
@@ -5,8 +5,8 @@ seed: 42
 dataset:
  name: hmvs
  root_dir: /home/yidaw/Documents/datasets/Buildings
- pose_path: /home/yidaw/Documents/datasets/Buildings/Building/Tile_+002_+002_AT.bin
- pcd_path: false # /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
+ pose_file: extrinsics.npy # Tile_+002_+002_AT.bin
+ pcd_file: pointcloud.npz # false # /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
  mesh_outpath: false
  img_wh:
  - 960
@@ -20,7 +20,8 @@ dataset:
  apply_mask: false
  apply_depth: true
  load_data_on_gpu: false
- max_imgs: 1000
+ max_imgs: 1200
+ preprocess_only: false
 
 model:
  name: neus
@@ -198,8 +199,8 @@ trainer:
  max_steps: 6e4
  log_every_n_steps: 500
  num_sanity_val_steps: 0
- val_check_interval: 1e4 # 6e4
+ val_check_interval: 3e4 # 6e4
  limit_train_batches: 1.0
- limit_val_batches: 6
+ limit_val_batches: 7
  enable_progress_bar: true
  precision: 16
diff --git a/configs/neus-hmvs-eagle.yaml b/configs/neus-hmvs-eagle.yaml
@@ -5,7 +5,7 @@ seed: 42
 dataset:
  name: hmvs
  root_dir: /home/yidaw/Documents/datasets/Buildings
- pose_path: /home/yidaw/Documents/datasets/Buildings/Building/Tile_+002_+002_AT.bin
+ pose_file: extrinsics.npy # Tile_+002_+002_AT.bin
  pcd_path: /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
  mesh_outpath: false
  img_downscale: 4 # better with more than 1000 x 1000 resolution

diff --git a/datasets/colmap.py b/datasets/colmap.py
@@ -88,6 +88,7 @@ def setup(self, config, split):
  all_c2w, all_images, all_fg_masks, all_depths, all_depth_masks, all_vis_masks = [], [], [], [], [], []
 
  pts3d = read_points3d_binary(os.path.join(self.config.root_dir, 'sparse/0/points3D.bin'))
+ pts3d_rgb = np.array([pts3d[k].rgb for k in pts3d])
  pts3d = np.array([pts3d[k].xyz for k in pts3d])
 
  for i, d in enumerate(imdata.values()):
@@ -100,7 +101,7 @@ def setup(self, config, split):
  if not os.path.exists(img_path):
  os.makedirs(os.path.join(self.config.root_dir, f"images_{self.config.img_downscale}"), exist_ok=True)
  img_path = os.path.join(self.config.root_dir, 'images', d.name)
- img = Image.open(img_path)
+ img = Image.open(img_path.replace('JPG','jpg'))
  if img.size[0] != w or img.size[1] != h:
  img = img.resize(img_wh, Image.BICUBIC)
  img.save(os.path.join(self.config.root_dir, f"images_{self.config.img_downscale}", d.name))
@@ -135,14 +136,15 @@ def setup(self, config, split):
  import open3d as o3d
  # NOTE: save the point cloud using point cloud
  pcd = o3d.geometry.PointCloud()
- mesh_init_path = os.path.join(self.config.root_dir, 'sparse/0/points3D.obj')
+ mesh_init_path = os.path.join(self.config.root_dir, 'sparse/0/points3D_mesh.ply')
  if os.path.exists(mesh_init_path):
  mesh_o3d = o3d.t.geometry.TriangleMesh.from_legacy(o3d.io.read_triangle_mesh(mesh_init_path))
  else:
  mesh_dir = os.path.join(self.config.root_dir, 'meshes')
  os.makedirs(mesh_dir, exist_ok=True)
  if not os.path.exists(os.path.join(mesh_dir, 'layout_pcd_gt.ply')):
  pcd.points = o3d.utility.Vector3dVector(pts3d)
+ pcd.colors = o3d.utility.Vector3dVector(pts3d_rgb)
  else:
  pcd = o3d.io.read_point_cloud(os.path.join(mesh_dir, 'layout_pcd_gt.ply'))
  # pts3d = torch.from_numpy(pts3d).float()