io_uring: Support Compare command for verification

Added 'verify_mode' option to io_uring_cmd with --cmd_type=nvme to
support data compare verification with Compare commands rather than Read
commands.  This patch newly added IO_U_F_VER_IN_DEV io_u flag to
represent that verification should be done in device side, not the host
side to skip the actual verification phase in verify_io_u().

Signed-off-by: Minwoo Im <minwoo.im.dev@gmail.com>
Link: https://lore.kernel.org/r/20240926010802.27131-1-minwoo.im.dev@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

engines/io_uring.c

@@ -41,6 +41,11 @@ enum uring_cmd_write_mode {
 	FIO_URING_CMD_WMODE_VERIFY,
 };
 
+enum uring_cmd_verify_mode {
+	FIO_URING_CMD_VMODE_READ = 1,
+	FIO_URING_CMD_VMODE_COMPARE,
+};
+
 struct io_sq_ring {
 	unsigned *head;
 	unsigned *tail;
@@ -99,6 +104,7 @@ struct ioring_options {
 	unsigned int readfua;
 	unsigned int writefua;
 	unsigned int write_mode;
+	unsigned int verify_mode;
 	struct cmdprio_options cmdprio_options;
 	unsigned int fixedbufs;
 	unsigned int registerfiles;
@@ -194,6 +200,26 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group	= FIO_OPT_G_IOURING,
 	},
+	{
+		.name	= "verify_mode",
+		.lname	= "Do verify based on the configured command (e.g., Read or Compare command)",
+		.type	= FIO_OPT_STR,
+		.off1	= offsetof(struct ioring_options, verify_mode),
+		.help	= "Issue Read or Compare command in the verification phase",
+		.def	= "read",
+		.posval = {
+			  { .ival = "read",
+			    .oval = FIO_URING_CMD_VMODE_READ,
+			    .help = "Issue Read commands in the verification phase"
+			  },
+			  { .ival = "compare",
+			    .oval = FIO_URING_CMD_VMODE_COMPARE,
+			    .help = "Issue Compare commands in the verification phase"
+			  },
+		},
+		.category = FIO_OPT_C_ENGINE,
+		.group	= FIO_OPT_G_IOURING,
+	},
 	{
 		.name	= "fixedbufs",
 		.lname	= "Fixed (pre-mapped) IO buffers",
@@ -443,6 +469,7 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
 	struct nvme_dsm *dsm;
 	void *ptr = ld->dsm;
 	unsigned int dsm_size;
+	uint8_t read_opcode = nvme_cmd_read;
 
 	/* only supports nvme_uring_cmd */
 	if (o->cmd_type != FIO_URING_CMD_NVME)
@@ -483,9 +510,21 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
 	ptr += io_u->index * dsm_size;
 	dsm = (struct nvme_dsm *)ptr;
 
+	/*
+	 * If READ command belongs to the verification phase and the
+	 * verify_mode=compare, convert READ to COMPARE command.
+	 */
+	if (io_u->flags & IO_U_F_VER_LIST && io_u->ddir == DDIR_READ &&
+			o->verify_mode == FIO_URING_CMD_VMODE_COMPARE) {
+		populate_verify_io_u(td, io_u);
+		read_opcode = nvme_cmd_compare;
+		io_u_set(td, io_u, IO_U_F_VER_IN_DEV);
+	}
+
 	return fio_nvme_uring_cmd_prep(cmd, io_u,
 			o->nonvectored ? NULL : &ld->iovecs[io_u->index],
-			dsm, ld->write_opcode, ld->cdw12_flags[io_u->ddir]);
+			dsm, read_opcode, ld->write_opcode,
+			ld->cdw12_flags[io_u->ddir]);
 }
 
 static struct io_u *fio_ioring_event(struct thread_data *td, int event)

engines/nvme.c

@@ -363,7 +363,8 @@ void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 			    struct iovec *iov, struct nvme_dsm *dsm,
-			    uint8_t write_opcode, unsigned int cdw12_flags)
+			    uint8_t read_opcode, uint8_t write_opcode,
+			    unsigned int cdw12_flags)
 {
 	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
 	__u64 slba;
@@ -373,7 +374,7 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 
 	switch (io_u->ddir) {
 	case DDIR_READ:
-		cmd->opcode = nvme_cmd_read;
+		cmd->opcode = read_opcode;
 		break;
 	case DDIR_WRITE:
 		cmd->opcode = write_opcode;

engines/nvme.h

@@ -77,6 +77,7 @@ enum nvme_io_opcode {
 	nvme_cmd_write			= 0x01,
 	nvme_cmd_read			= 0x02,
 	nvme_cmd_write_uncor		= 0x04,
+	nvme_cmd_compare		= 0x05,
 	nvme_cmd_write_zeroes		= 0x08,
 	nvme_cmd_dsm			= 0x09,
 	nvme_cmd_verify			= 0x0c,
@@ -431,7 +432,8 @@ int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 			    struct iovec *iov, struct nvme_dsm *dsm,
-			    uint8_t write_opcode, unsigned int cdw12_flags);
+			    uint8_t read_opcode, uint8_t write_opcode,
+			    unsigned int cdw12_flags);
 
 void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 		      struct nvme_cmd_ext_io_opts *opts);

fio.1

@@ -2675,6 +2675,20 @@ Use Verify commands for write operations
 .RE
 .RE
 .TP
+.BI (io_uring_cmd)verify_mode \fR=\fPstr
+Specifies the type of command to be used in the verification phase. Defaults to 'read'.
+.RS
+.RS
+.TP
+.B read
+Use Read commands for data verification
+.TP
+.B compare
+Use Compare commands for data verification
+.TP
+.RE
+.RE
+.TP
 .BI (sg)sg_write_mode \fR=\fPstr
 Specify the type of write commands to issue. This option can take multiple
 values:

io_u.h

@@ -23,6 +23,7 @@ enum {
 	IO_U_F_VER_LIST		= 1 << 7,
 	IO_U_F_PATTERN_DONE	= 1 << 8,
 	IO_U_F_DEVICE_ERROR	= 1 << 9,
+	IO_U_F_VER_IN_DEV	= 1 << 10, /* Verify data in device */
 };
 
 /*

verify.c

@@ -901,6 +901,13 @@ int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr)
 	if (td_ioengine_flagged(td, FIO_FAKEIO))
 		return 0;
 
+	/*
+	 * If data has already been verified from the device, we can skip
+	 * the actual verification phase here.
+	 */
+	if (io_u->flags & IO_U_F_VER_IN_DEV)
+		return 0;
+
 	if (io_u->flags & IO_U_F_TRIMMED) {
 		ret = verify_trimmed_io_u(td, io_u);
 		goto done;