Skip to content

Commit

Permalink
use trash to remove RBD images asynchronously
Browse files Browse the repository at this point in the history
We should remove RBD images asynchronously to finish each reconiliation
iteration quickly. This commit supports asynchronous removal of the RBD
images by the following two steps:

1. Run `rbd trash mv` to move the RBD image to trash.
2. Run `ceph rbd task add trash remove` to add a task to remove an image
   from trash.

cf. https://github.com/ceph/ceph-csi/blob/d8a040719e0ad491ad144fe361b67e6bd2109710/docs/design/proposals/rbd-snap-clone.md#rbd-cli-commands-to-delete-a-snapshot

Signed-off-by: Ryotaro Banno <ryotaro.banno@gmail.com>
  • Loading branch information
ushitora-anqou committed Dec 12, 2024
1 parent 9bfeb7f commit 10cec2b
Showing 4 changed files with 43 additions and 2 deletions.
2 changes: 2 additions & 0 deletions internal/ceph/ceph.go
Original file line number Diff line number Diff line change
@@ -43,6 +43,8 @@ type CephCmd interface {
RBDInfo(pool, image string) (*RBDImageInfo, error)
RBDLs(pool string) ([]string, error)
RBDRm(pool, image string) error
RBDTrashMv(pool, image string) error
CephRBDTaskAddTrashRemove(pool, image string) error
RBDSnapCreate(pool, image, snap string) error
RBDSnapLs(pool, image string) ([]RBDSnapshot, error)
RBDSnapRm(pool, image, snap string) error
18 changes: 18 additions & 0 deletions internal/ceph/rbd.go
Original file line number Diff line number Diff line change
@@ -62,6 +62,24 @@ func (c *cephCmdImpl) RBDRm(pool, image string) error {
return nil
}

// RBDTrashMv removes an RBD image asynchronously.
func (c *cephCmdImpl) RBDTrashMv(pool, image string) error {
_, err := c.command.execute("rbd", "trash", "mv", fmt.Sprintf("%s/%s", pool, image))
if err != nil {
return fmt.Errorf("failed to move RBD image to trash: %w", err)
}
return nil
}

// CephRBDTaskTrashRemove adds a task to remove the image from trash.
func (c *cephCmdImpl) CephRBDTaskAddTrashRemove(pool, imageID string) error {
_, err := c.command.execute("ceph", "rbd", "task", "add", "trash", "remove", fmt.Sprintf("%s/%s", pool, imageID))
if err != nil {
return fmt.Errorf("failed to add task to remove the image from trash: %w", err)
}
return nil
}

// RBDSnapCreate creates an RBD snapshot.
func (c *cephCmdImpl) RBDSnapCreate(pool, image, snap string) error {
_, err := c.command.execute("rbd", "snap", "create", fmt.Sprintf("%s/%s@%s", pool, image, snap))
8 changes: 8 additions & 0 deletions internal/controller/internal/testutil/fake_rbd.go
Original file line number Diff line number Diff line change
@@ -38,6 +38,14 @@ func (f *fakeRBD) RBDRm(pool, image string) error {
return nil
}

func (f *fakeRBD) RBDTrashMv(pool, image string) error {
return nil
}

func (f *fakeRBD) CephRBDTaskAddTrashRemove(pool, image string) error {
return nil
}

func (f *fakeRBD) RBDSnapCreate(pool, image, snap string) error {
key := pool + "/" + image

17 changes: 15 additions & 2 deletions internal/controller/persistentvolume_controller.go
Original file line number Diff line number Diff line change
@@ -132,12 +132,25 @@ func (r *PersistentVolumeReconciler) removeRBDImage(ctx context.Context, pv *cor

images, err := r.ceph.RBDLs(pool)
if err != nil {
return fmt.Errorf("failed to list RBD images: %v", err)
return fmt.Errorf("failed to list RBD images: %w", err)
}

if !slices.Contains(images, image) {
return nil
}

return r.ceph.RBDRm(pool, image)
imageInfo, err := r.ceph.RBDInfo(pool, image)
if err != nil {
return fmt.Errorf("failed to get info about the RBD image: %s/%s: %w", pool, image, err)
}

if err := r.ceph.RBDTrashMv(pool, image); err != nil {
return fmt.Errorf("failed to move the RBD image to trash: %s/%s: %w", pool, image, err)
}

if err := r.ceph.CephRBDTaskAddTrashRemove(pool, imageInfo.ID); err != nil {
return fmt.Errorf("failed to add task to remove the RBD image from trash: %s/%s: %w", pool, image, err)
}

return nil
}

0 comments on commit 10cec2b

Please sign in to comment.