May I ask about how to use cam_pos parameters?

[TikaToka]

Hello, team.

I am working with graspnet-1b and I want to make a point cloud using the graspnet w.r.t camera view. (x is frame width, y is frame height, and z is camera view )

However, As I purely apply all the information given in

1. CamK.npy for intrinsic
2. camera_poses[i] for i-th frame cam_extrinsic_pos

to make point cloud, the x, y, z, alignment is wrong

def create_point_cloud_from_rgbd_direct(rgb_image, depth_image, intrinsic_parameters, ex_cam_pos):
    # Convert OpenCV images to Open3D images
    rgb_image_o3d = o3d.geometry.Image(rgb_image)
    depth_image_o3d = o3d.geometry.Image(depth_image)

    # Create RGBD image
    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
        rgb_image_o3d, depth_image_o3d, 
        # depth_scale=1000.0,
        # depth_trunc=10.0,
        convert_rgb_to_intensity=False)

    # Create camera intrinsic
    intrinsic = o3d.camera.PinholeCameraIntrinsic(
        intrinsic_parameters['width'],
        intrinsic_parameters['height'],
        intrinsic_parameters['fx'],
        intrinsic_parameters['fy'],
        intrinsic_parameters['cx'],
        intrinsic_parameters['cy'])

    # Create point cloud
    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(
        rgbd_image, intrinsic, extrinsic=ex_cam_pos)

    return pcd

  # Build PointCloud
  intrinsic_parameters = {
      'width': width,
      'height': height,
      'fx': int_pos[0, 0],
      'fy': int_pos[1, 1],
      'cx': int_pos[0, 2],
      'cy': int_pos[1, 2],
  }

  pcd = create_point_cloud_from_rgbd_direct(rgb_img, depth_img, intrinsic_parameters, ex_cam_pos)

  camera_pos = ex_cam_pos[:3, 3]
  rotation_matrix = ex_cam_pos[:3, :3]

  # Create a visualizer
  vis = o3d.visualization.Visualizer()
  vis.create_window(width=intrinsic_parameters['width'], height=intrinsic_parameters['height'])

  # Add the point clouds and coordinate frame to the visualizer
  vis.add_geometry(pcd)

  # Create a view control
  view_control = vis.get_view_control()

  # Set the camera parameters
  camera_parameters = view_control.convert_to_pinhole_camera_parameters()
  camera_parameters.extrinsic = ex_cam_pos
  camera_parameters.intrinsic = o3d.camera.PinholeCameraIntrinsic(
      intrinsic_parameters['width'],
      intrinsic_parameters['height'],
      intrinsic_parameters['fx'],
      intrinsic_parameters['fy'],
      intrinsic_parameters['cx'],
      intrinsic_parameters['cy'])
  view_control.convert_from_pinhole_camera_parameters(camera_parameters)

  # Run the visualizer
  vis.run()
  vis.destroy_window()

original rgb

pointcloud

I have been working for few days, but I haven't figured out hot to solve it.

Sorry for beginners question