add SP surfaces and rasterized depth

configs/neuralangelo-hmvs-eagle-wbg.yaml

@@ -8,12 +8,15 @@ dataset:
   pose_path: /home/yidaw/Documents/datasets/Buildings/Building/Tile_+002_+002_AT.bin
   pcd_path: /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply
   mesh_outpath: false
-  img_downscale: 4 # better with more than 1000 x 1000 resolution
+  img_wh:
+    - 960
+    - 540
+  img_downscale: 2 # 4 # better with more than 1000 x 1000 resolution
   cam_downscale: 200 # if false, to rescale the cameras/pts positions automatically
   repose: true # false
   up_est_method: ground # no-change/camera/ground/z-axis 
   center_est_method: point # camera/point
-  n_test_traj_steps: 60
+  n_test_traj_steps: 120
   apply_mask: false
   apply_depth: false
   load_data_on_gpu: false
@@ -46,7 +49,7 @@ model:
     grad_type: finite_difference
     isosurface:
       method: mc
-      resolution: 768 # 1536
+      resolution: 1024 # 1536
       chunk: 2097152 # 128**3
       threshold: 0.
     xyz_encoding_config:
@@ -66,7 +69,7 @@ model:
       output_activation: none
       n_neurons: 64
       n_hidden_layers: 1
-      sphere_init: false # true
+      sphere_init: true
       sphere_init_radius: 0.5
       weight_norm: true
     finite_difference_eps: progressive
@@ -140,7 +143,7 @@ system:
     lambda_rgb_l1: 1.
     lambda_rgb_cos: 0.
     lambda_mask: 0.0
-    lambda_depth_l1: [5e3, 1.0, 0.1, 8e4] # [5e3, 0.0, 1.0, 1e5]
+    lambda_depth_l1: [5e3, 1.0, 0.1, 6e4] # [5e3, 0.0, 1.0, 1e5]
     lambda_eikonal: 0.1
     lambda_adaptive: 1e-6
     lambda_curvature: 5e-4 # topology warmup
@@ -194,11 +197,11 @@ export:
   export_vertex_color: True
 
 trainer:
-  max_steps: 4e4 # 8e4
+  max_steps: 6e4
   log_every_n_steps: 500
   num_sanity_val_steps: 0
-  val_check_interval: 5e3
+  val_check_interval: 2e4
   limit_train_batches: 1.0
-  limit_val_batches: 3
+  limit_val_batches: 6 # 3
   enable_progress_bar: true
   precision: 16

configs/neus-blender-wbg.yaml

@@ -140,7 +140,7 @@ system:
     lambda_rgb_l1: 1.
     lambda_rgb_cos: 0.
     lambda_mask: 0.0
-    lambda_depth_l1: [5e3, 1.0, 0.0, 8e4] # 1.
+    lambda_depth_l1: [5e3, 1.0, 0.0, 6e4] # 1.
     lambda_eikonal: 0.1
     lambda_adaptive: 1e-6
     lambda_curvature: 0
@@ -150,7 +150,7 @@ system:
     lambda_distortion_bg: 0.0
     lambda_opaque: 1e-2
     lambda_entropy: 1e-4
-    lambda_surface_bias: 0.0 # [0, 0.0, 0.1, 8e4] # refer to D-NeuS
+    lambda_surface_bias: 0.0 # [0, 0.0, 0.1, 6e4] # refer to D-NeuS
   optimizer:
     name: AdamW
     args:
@@ -193,7 +193,7 @@ export:
   export_vertex_color: True  
 
 trainer:
-  max_steps: 8e4
+  max_steps: 6e4
   log_every_n_steps: 500
   num_sanity_val_steps: 0
   val_check_interval: 5e3 # 1e4

configs/neus-colmap.yaml

@@ -5,9 +5,11 @@ seed: 42
 dataset:
   name: colmap
   root_dir: ./load/unbounded360/garden
-  img_downscale: 2 # 4 # better with more than 1000 x 1000 resolution
+  mesh_outpath: false
+  img_downscale: 4 # better with more than 1000 x 1000 resolution
   cam_downscale: 2.5 # if false, to rescale the cameras/pts positions automatically
-  up_est_method: ground # z-axis # if true, use estimated ground plane normal direction as up direction
+  repose: true # false
+  up_est_method: camera # z-axis # if true, use estimated ground plane normal direction as up direction
   center_est_method: point # camera
   n_test_traj_steps: 60
   apply_mask: false
@@ -134,8 +136,8 @@ system:
     lambda_rgb_mse: 1.
     lambda_rgb_l1: 1.
     lambda_rgb_cos: 0.0
-    lambda_mask: 0.0
-    lambda_depth: 0.0
+    lambda_mask: 0.1 # 0.0
+    lambda_depth_l1: [5e3, 1.0, 0.1, 6e4] # [5e3, 0.0, 1.0, 1e5]
     lambda_eikonal: 0.1
     lambda_adaptive: 1e-6
     lambda_curvature: 0
@@ -188,10 +190,10 @@ export:
   export_vertex_color: True
 
 trainer:
-  max_steps: 1e5
+  max_steps: 6e4
   log_every_n_steps: 500
   num_sanity_val_steps: 0
-  val_check_interval: 5e4
+  val_check_interval: 2e4
   limit_train_batches: 1.0
   limit_val_batches: 4
   enable_progress_bar: true

configs/neus-hmvs-eagle-wbg.yaml

@@ -6,23 +6,26 @@ dataset:
   name: hmvs
   root_dir: /home/yidaw/Documents/datasets/Buildings
   pose_path: /home/yidaw/Documents/datasets/Buildings/Building/Tile_+002_+002_AT.bin
-  pcd_path: /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
+  pcd_path: false # /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
   mesh_outpath: false
-  img_downscale: 2 # 4 # better with more than 1000 x 1000 resolution
-  cam_downscale: 200 # if false, to rescale the cameras/pts positions automatically
-  repose: true # false
+  img_wh:
+    - 960
+    - 540
+  img_downscale: 4 # better with more than 1000 x 1000 resolution
+  cam_downscale: false # if false, to rescale the cameras/pts positions automatically
+  repose: true
   up_est_method: ground # no-change/camera/ground/z-axis 
   center_est_method: point # camera/point
-  n_test_traj_steps: 60
+  n_test_traj_steps: 120
   apply_mask: false
-  apply_depth: false
+  apply_depth: true
   load_data_on_gpu: false
-  max_imgs: 1600
+  max_imgs: 1000
 
 model:
   name: neus
   radius: 1.0
-  num_samples_per_ray: 1024 # 4096
+  num_samples_per_ray: 2048 # 4096
   train_num_rays: 256
   max_train_num_rays: 8192
   grid_prune: true
@@ -139,7 +142,7 @@ system:
     lambda_rgb_l1: 1.
     lambda_rgb_cos: 0.
     lambda_mask: 0.0
-    lambda_depth_l1: [5e3, 1.0, 0.1, 8e4] # [5e3, 0.0, 1.0, 1e5]
+    lambda_depth_l1: [5e3, 1.0, 0.1, 6e4] # [5e3, 0.0, 1.0, 6e4]
     lambda_eikonal: 0.1
     lambda_adaptive: 1e-6
     lambda_curvature: 0.0 # topology warmup
@@ -149,7 +152,7 @@ system:
     lambda_distortion_bg: 0.0
     lambda_opaque: 1e-2
     lambda_entropy: 1e-4
-    lambda_surface_bias: 0.0 # [5e3, 0.0, 0.1, 8e4] # refer to D-NeuS
+    lambda_surface_bias: 0. # [5e3, 0.0, 0.1, 6e4] # refer to D-NeuS
   optimizer:
     name: AdamW
     args:
@@ -192,11 +195,11 @@ export:
   export_vertex_color: True
 
 trainer:
-  max_steps: 4e4 # 8e4
+  max_steps: 6e4
   log_every_n_steps: 500
   num_sanity_val_steps: 0
-  val_check_interval: 5e3
+  val_check_interval: 1e4 # 6e4
   limit_train_batches: 1.0
-  limit_val_batches: 3
+  limit_val_batches: 6
   enable_progress_bar: true
   precision: 16

configs/neus-hmvs-eagle.yaml

@@ -9,7 +9,7 @@ dataset:
   pcd_path: /home/yidaw/Documents/datasets/Buildings/Building/building_interest.ply # Bridge/Tile_+004_+008_AT.ply
   mesh_outpath: false
   img_downscale: 4 # better with more than 1000 x 1000 resolution
-  cam_downscale: 200 # if false, to rescale the cameras/pts positions automatically
+  cam_downscale: 150 # 200 # if false, to rescale the cameras/pts positions automatically
   repose: true # false
   up_est_method: no-change # camera/ground/z-axis 
   center_est_method: point # camera/point
@@ -151,8 +151,8 @@ trainer:
   max_steps: 8e4
   log_every_n_steps: 500
   num_sanity_val_steps: 0
-  val_check_interval: 5e3 # 4e4
+  val_check_interval: 1e4
   limit_train_batches: 1.0
-  limit_val_batches: 6 # 2
+  limit_val_batches: 3
   enable_progress_bar: true
   precision: 16

datasets/__init__.py

@@ -24,16 +24,13 @@ def make(name, config):
             import R3DUtil
         else:
             print(
-                "In case you are deploying on", colored("STARMAP platform,",
+                "In case you are deploying on", colored("cloud,",
                         'yellow'), "compiled parsers such as",
                 colored("'R3DUtils'", 'yellow'), "or",
                 colored("'R3DParser'", 'yellow'),
-                "should be explicitly included.")
-            print(
-                "try -",
-                colored(
-                    "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/home/yidaw/Documents/buildboat/R3DParser/3rd/R3DLib/bin/",
-                    'blue'))
+                "should be explicitly included,")
+            print(colored("with", 'blue'),
+                    "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${HOME}/Documents/buildboat/R3DParser/3rd/R3DLib/bin/")
         from . import hmvs
     dataset = datasets[name](config)
     return dataset

datasets/blender.py

@@ -84,6 +84,8 @@ def setup(self, config, split):
         pts_clt = o3d.geometry.PointCloud()
         for i, frame in enumerate(meta['frames']):
             c2w_npy = np.array(frame['transform_matrix'])
+            # NOTE: only specific dataset, e.g. Baoru's medical images needs to convert
+            # c2w_npy[:3, 1:3] *= -1.  # COLMAP => OpenGL
             c2w = torch.from_numpy(c2w_npy[:3, :4])
             self.all_c2w.append(c2w)
 
@@ -95,6 +97,8 @@ def setup(self, config, split):
 
             # load the estimated or recorded depth map
             if "depth_path" in frame:
+                mesh_dir = os.path.join(self.config.root_dir, 'meshes')
+                os.makedirs(mesh_dir, exist_ok=True)
                 depth_path = os.path.join(self.config.root_dir, f"{frame['depth_path']}")
                 if depth_path.split('.')[-1] == 'tiff':
                     depth = Image.open(depth_path).convert('I')
@@ -125,10 +129,9 @@ def setup(self, config, split):
                     intrin_o3d = o3d.camera.PinholeCameraIntrinsic(
                         w, h, self.focal_x, self.focal_y, self.cx, self.cy)
                     # Open3D uses world-to-camera extrinsics
-                    c2w_npy[:3, 1:3] *= -1.  # COLMAP => OpenGL
                     pts_frm = o3d.geometry.PointCloud.create_from_depth_image(
                         depth_o3d, intrinsic=intrin_o3d, extrinsic=np.linalg.inv(c2w_npy), depth_scale=1)
-                    # o3d.io.write_point_cloud(f"./test/layout_depth_frame_{str(i + 1)}.ply", pts_frm)
+                    # o3d.io.write_point_cloud(os.path.join(mesh_dir, f"./layout_depth_frame_{str(i + 1)}.ply"), pts_frm)
                     pts_clt.points = (o3d.utility.Vector3dVector(
                         np.concatenate((np.array(pts_clt.points), np.array(pts_frm.points)),
                                        axis=0)))
@@ -184,8 +187,25 @@ def setup(self, config, split):
                 vis_mask = torch.ones_like(img[...,0], device=img.device)
             self.all_vis_masks.append(vis_mask)
 
-        # pts_clt = pts_clt.voxel_down_sample(voxel_size=0.01)
-        o3d.io.write_point_cloud("./test/layout_depth_clt.ply", pts_clt)
+        pts_clt = pts_clt.voxel_down_sample(voxel_size=0.5)
+        pts_clt.estimate_normals(
+                search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=1.0, max_nn=20))
+        o3d.io.write_point_cloud(os.path.join(mesh_dir, 'layout_depth_clt.ply'), pts_clt)
+        # Poisson surface on top of given GT points
+        mesh_poisson_path = os.path.join(mesh_dir, 'layout_mesh_ps.ply')
+        if not os.path.exists(mesh_poisson_path):
+            print(colored(
+                    'Extracting Poisson surface on top of given GT points',
+                    'blue'))
+            with o3d.utility.VerbosityContextManager(
+                    o3d.utility.VerbosityLevel.Debug) as cm:
+                # Poisson
+                mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
+                    pts_clt, depth=10, linear_fit=True)
+                densities = np.asarray(densities)
+                vertices_to_remove = densities < np.quantile(densities, 0.02)
+                mesh.remove_vertices_by_mask(vertices_to_remove)
+            o3d.io.write_triangle_mesh(mesh_poisson_path, mesh)
 
         self.all_c2w, self.all_images, self.all_fg_masks, self.all_depths, self.all_depth_masks, self.all_vis_masks = \
             torch.stack(self.all_c2w, dim=0).float().to(self.rank), \
@@ -196,7 +216,7 @@ def setup(self, config, split):
             torch.stack(self.all_vis_masks, dim=0).float()
 
         # translate
-        # self.all_c2w[...,3] -= self.all_c2w[...,3].mean(0)
+        self.all_c2w[...,3] -= self.all_c2w[...,3].mean(0)
 
         # rescale
         if 'cam_downscale' not in self.config: