io_uring: Support Compare command for verification

Added 'verify_mode' option to io_uring_cmd with --cmd_type=nvme to
support data compare verification with Compare commands rather than Read
commands.  This patch newly added IO_U_F_VER_IN_DEV io_u flag to
represent that verification should be done in device side, not the host
side to skip the actual verification phase in verify_io_u().

Signed-off-by: Minwoo Im <minwoo.im.dev@gmail.com>
Link: https://lore.kernel.org/r/20240926010802.27131-1-minwoo.im.dev@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

engines/io_uring.c

@@ -41,6 +41,11 @@ enum uring_cmd_write_mode {
  FIO_URING_CMD_WMODE_VERIFY,
 };
 
+enum uring_cmd_verify_mode {
+ FIO_URING_CMD_VMODE_READ = 1,
+ FIO_URING_CMD_VMODE_COMPARE,
+};
+
 struct io_sq_ring {
  unsigned *head;
  unsigned *tail;
@@ -99,6 +104,7 @@ struct ioring_options {
  unsigned int readfua;
  unsigned int writefua;
  unsigned int write_mode;
+ unsigned int verify_mode;
  struct cmdprio_options cmdprio_options;
  unsigned int fixedbufs;
  unsigned int registerfiles;
@@ -194,6 +200,26 @@ static struct fio_option options[] = {
  .category = FIO_OPT_C_ENGINE,
  .group = FIO_OPT_G_IOURING,
  },
+ {
+ .name = "verify_mode",
+ .lname = "Do verify based on the configured command (e.g., Read or Compare command)",
+ .type = FIO_OPT_STR,
+ .off1 = offsetof(struct ioring_options, verify_mode),
+ .help = "Issue Read or Compare command in the verification phase",
+ .def = "read",
+ .posval = {
+ { .ival = "read",
+ .oval = FIO_URING_CMD_VMODE_READ,
+ .help = "Issue Read commands in the verification phase"
+ },
+ { .ival = "compare",
+ .oval = FIO_URING_CMD_VMODE_COMPARE,
+ .help = "Issue Compare commands in the verification phase"
+ },
+ },
+ .category = FIO_OPT_C_ENGINE,
+ .group = FIO_OPT_G_IOURING,
+ },
  {
  .name = "fixedbufs",
  .lname = "Fixed (pre-mapped) IO buffers",
@@ -443,6 +469,7 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
  struct nvme_dsm *dsm;
  void *ptr = ld->dsm;
  unsigned int dsm_size;
+ uint8_t read_opcode = nvme_cmd_read;
 
  /* only supports nvme_uring_cmd */
  if (o->cmd_type != FIO_URING_CMD_NVME)
@@ -483,9 +510,21 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
  ptr += io_u->index * dsm_size;
  dsm = (struct nvme_dsm *)ptr;
 
+ /*
+ * If READ command belongs to the verification phase and the
+ * verify_mode=compare, convert READ to COMPARE command.
+ */
+ if (io_u->flags & IO_U_F_VER_LIST && io_u->ddir == DDIR_READ &&
+ o->verify_mode == FIO_URING_CMD_VMODE_COMPARE) {
+ populate_verify_io_u(td, io_u);
+ read_opcode = nvme_cmd_compare;
+ io_u_set(td, io_u, IO_U_F_VER_IN_DEV);
+ }
+
  return fio_nvme_uring_cmd_prep(cmd, io_u,
  o->nonvectored ? NULL : &ld->iovecs[io_u->index],
- dsm, ld->write_opcode, ld->cdw12_flags[io_u->ddir]);
+ dsm, read_opcode, ld->write_opcode,
+ ld->cdw12_flags[io_u->ddir]);
 }
 
 static struct io_u *fio_ioring_event(struct thread_data *td, int event)

engines/nvme.c

@@ -363,7 +363,8 @@ void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
  struct iovec *iov, struct nvme_dsm *dsm,
- uint8_t write_opcode, unsigned int cdw12_flags)
+ uint8_t read_opcode, uint8_t write_opcode,
+ unsigned int cdw12_flags)
 {
  struct nvme_data *data = FILE_ENG_DATA(io_u->file);
  __u64 slba;
@@ -373,7 +374,7 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 
  switch (io_u->ddir) {
  case DDIR_READ:
- cmd->opcode = nvme_cmd_read;
+ cmd->opcode = read_opcode;
  break;
  case DDIR_WRITE:
  cmd->opcode = write_opcode;

engines/nvme.h

@@ -77,6 +77,7 @@ enum nvme_io_opcode {
  nvme_cmd_write = 0x01,
  nvme_cmd_read = 0x02,
  nvme_cmd_write_uncor = 0x04,
+ nvme_cmd_compare = 0x05,
  nvme_cmd_write_zeroes = 0x08,
  nvme_cmd_dsm = 0x09,
  nvme_cmd_verify = 0x0c,
@@ -431,7 +432,8 @@ int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
  struct iovec *iov, struct nvme_dsm *dsm,
- uint8_t write_opcode, unsigned int cdw12_flags);
+ uint8_t read_opcode, uint8_t write_opcode,
+ unsigned int cdw12_flags);
 
 void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
  struct nvme_cmd_ext_io_opts *opts);

fio.1

@@ -2675,6 +2675,20 @@ Use Verify commands for write operations
 .RE
 .RE
 .TP
+.BI (io_uring_cmd)verify_mode \fR=\fPstr
+Specifies the type of command to be used in the verification phase. Defaults to 'read'.
+.RS
+.RS
+.TP
+.B read
+Use Read commands for data verification
+.TP
+.B compare
+Use Compare commands for data verification
+.TP
+.RE
+.RE
+.TP
 .BI (sg)sg_write_mode \fR=\fPstr
 Specify the type of write commands to issue. This option can take multiple
 values:

io_u.h

@@ -23,6 +23,7 @@ enum {
  IO_U_F_VER_LIST = 1 << 7,
  IO_U_F_PATTERN_DONE = 1 << 8,
  IO_U_F_DEVICE_ERROR = 1 << 9,
+ IO_U_F_VER_IN_DEV = 1 << 10, /* Verify data in device */
 };
 
 /*

verify.c

@@ -901,6 +901,13 @@ int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr)
  if (td_ioengine_flagged(td, FIO_FAKEIO))
  return 0;
 
+ /*
+ * If data has already been verified from the device, we can skip
+ * the actual verification phase here.
+ */
+ if (io_u->flags & IO_U_F_VER_IN_DEV)
+ return 0;
+
  if (io_u->flags & IO_U_F_TRIMMED) {
  ret = verify_trimmed_io_u(td, io_u);
  goto done;