ioctl: get_log_page by nvme uring cmd

Use io_uring for fetching log pages.

This showed about a 10% performance improvement for some large log pages.

Signed-off-by: letli <[email protected]>

README.md

@@ -155,6 +155,7 @@ A few build options can be specified on the command line when invoking meson.
 | libdbus     | auto, enabled, [disabled] | Enables D-Bus dependent features (libnvme-mi: End point discovery), adds build dependency on libdbus |
 | json-c      | [auto], enabled, disabled | (recommended) Enables JSON-C dependend features (e.g. config.json parsing), adds build depdency on json-c |
 | keyutils    | [auto], enabled, disabled | Enables keyutils dependent features (e.g. authentication), adds build dependency on keyutils |
+| liburing    | [auto], enabled, disabled | Enables liburing dependent features (e.g. get log page by uring cmd), adds build depdency on liburing |
 
 See the full configuration options with
 

meson.build

@@ -68,6 +68,13 @@ else
 endif
 conf.set('CONFIG_JSONC', json_c_dep.found(), description: 'Is json-c required?')
 
+if get_option('liburing').disabled()
+    liburing_dep = dependency('', required: false)
+else
+    liburing_dep = dependency('liburing', version: '>=2.2', required: get_option('liburing'))
+endif
+conf.set('CONFIG_LIBURING', liburing_dep.found(), description: 'Is liburing available?')
+
 if get_option('openssl').disabled()
   openssl_dep = dependency('', required: false)
 else
@@ -311,6 +318,7 @@ if meson.version().version_compare('>=0.53.0')
         'keyutitls':         keyutils_dep.found(),
         'libdbus':           libdbus_dep.found(),
         'Python 3':          py3_dep.found(),
+        'liburing':          liburing_dep.found(),
     }
     summary(dep_dict, section: 'Dependencies')
     conf_dict = {

meson_options.txt

@@ -13,3 +13,4 @@ option('openssl', type : 'feature', value: 'auto', description : 'OpenSSL suppor
 option('libdbus', type : 'feature', value: 'disabled', description : 'libdbus support')
 option('json-c', type : 'feature', value: 'auto', description : 'JSON support')
 option('keyutils', type: 'feature', value: 'auto', description: 'keyutils support')
+option('liburing', type : 'feature', value: 'auto', description : 'liburing support')

src/meson.build

@@ -35,6 +35,7 @@ deps = [
     json_c_dep,
     openssl_dep,
     keyutils_dep,
+    liburing_dep,
 ]
 
 mi_deps = [

src/nvme/ioctl.c

@@ -8,6 +8,9 @@
  */
 #include <errno.h>
 #include <fcntl.h>
+#ifdef CONFIG_LIBURING
+#include <liburing.h>
+#endif
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -316,6 +319,100 @@ int nvme_get_log(struct nvme_get_log_args *args)
 	return nvme_submit_admin_passthru(args->fd, &cmd, args->result);
 }
 
+#ifdef CONFIG_LIBURING
+enum {
+	IO_URING_NOT_AVAILABLE,
+	IO_URING_AVAILABLE,
+} io_uring_kernel_support = IO_URING_NOT_AVAILABLE;
+
+/*
+ * gcc specific attribute, call automatically on the library loading.
+ * if IORING_OP_URING_CMD is not supported, fallback to ioctl interface.
+ */
+__attribute__((constructor))
+static void nvme_uring_cmd_probe()
+{
+	struct io_uring_probe *probe = io_uring_get_probe();
+	if (NULL != probe && io_uring_opcode_supported(probe, IORING_OP_URING_CMD))
+		io_uring_kernel_support = IO_URING_AVAILABLE;
+}
+
+static int nvme_uring_cmd_setup(struct io_uring *ring)
+{
+	return io_uring_queue_init(NVME_URING_ENTRIES, ring, IORING_SETUP_SQE128 | IORING_SETUP_CQE32);
+}
+
+static void nvme_uring_cmd_exit(struct io_uring *ring)
+{
+	io_uring_queue_exit(ring);
+}
+
+static int nvme_uring_cmd_admin_passthru_async(struct io_uring *ring, struct nvme_get_log_args *args)
+{
+	__u32 numd = (args->len >> 2) - 1;
+	__u16 numdu = numd >> 16, numdl = numd & 0xffff;
+
+	__u32 cdw10 = NVME_SET(args->lid, LOG_CDW10_LID) |
+			NVME_SET(args->lsp, LOG_CDW10_LSP) |
+			NVME_SET(!!args->rae, LOG_CDW10_RAE) |
+			NVME_SET(numdl, LOG_CDW10_NUMDL);
+	__u32 cdw11 = NVME_SET(numdu, LOG_CDW11_NUMDU) |
+			NVME_SET(args->lsi, LOG_CDW11_LSI);
+	__u32 cdw12 = args->lpo & 0xffffffff;
+	__u32 cdw13 = args->lpo >> 32;
+	__u32 cdw14 = NVME_SET(args->uuidx, LOG_CDW14_UUID) |
+			NVME_SET(!!args->ot, LOG_CDW14_OT) |
+			NVME_SET(args->csi, LOG_CDW14_CSI);
+
+	if (args->args_size < sizeof(struct nvme_get_log_args)) {
+		errno = EINVAL;
+		return -1;
+	}
+
+	struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
+	if (!sqe)
+		return -1;
+
+	struct nvme_uring_cmd *cmd = (void *)&sqe->cmd;
+
+	cmd->opcode        = nvme_admin_get_log_page,
+	cmd->nsid          = args->nsid,
+	cmd->addr          = (__u64)(uintptr_t)args->log,
+	cmd->data_len      = args->len,
+	cmd->cdw10         = cdw10,
+	cmd->cdw11         = cdw11,
+	cmd->cdw12         = cdw12,
+	cmd->cdw13         = cdw13,
+	cmd->cdw14         = cdw14,
+	cmd->timeout_ms    = args->timeout,
+
+	sqe->fd = args->fd;
+	sqe->opcode = IORING_OP_URING_CMD;
+	sqe->cmd_op = NVME_URING_CMD_ADMIN;
+
+	int ret = io_uring_submit(ring);
+	if (ret < 0) {
+		errno = -ret;
+		return -1;
+	}
+	return 0;
+}
+
+static int nvme_uring_cmd_wait_complete(struct io_uring *ring, int n)
+{
+	struct io_uring_cqe *cqe;
+	for (int i = 0; i < n; i++) {
+		int ret = io_uring_wait_cqe(ring, &cqe);
+		if (ret) {
+			errno = -ret;
+			return -1;
+		}
+		io_uring_cqe_seen(ring, cqe);
+	}
+	return n;
+}
+#endif
+
 int nvme_get_log_page(int fd, __u32 xfer_len, struct nvme_get_log_args *args)
 {
 	__u64 offset = 0, xfer, data_len = args->len;
@@ -326,6 +423,15 @@ int nvme_get_log_page(int fd, __u32 xfer_len, struct nvme_get_log_args *args)
 
 	args->fd = fd;
 
+#ifdef CONFIG_LIBURING
+	int n = 0;
+	struct io_uring ring;
+	if (io_uring_kernel_support == IO_URING_AVAILABLE) {
+		ret = nvme_uring_cmd_setup(&ring);
+		if (ret)
+			return -1;
+	}
+#endif
 	/*
 	 * 4k is the smallest possible transfer unit, so restricting to 4k
 	 * avoids having to check the MDTS value of the controller.
@@ -344,6 +450,16 @@ int nvme_get_log_page(int fd, __u32 xfer_len, struct nvme_get_log_args *args)
 		args->len = xfer;
 		args->log = ptr;
 		args->rae = offset + xfer < data_len || retain;
+#ifdef CONFIG_LIBURING
+		if (io_uring_kernel_support == IO_URING_AVAILABLE) {
+			if (n >= NVME_URING_ENTRIES) {
+				nvme_uring_cmd_wait_complete(&ring, n);
+				n = 0;
+			}
+			n += 1;
+			ret = nvme_uring_cmd_admin_passthru_async(&ring, args);
+		} else
+#endif
 		ret = nvme_get_log(args);
 		if (ret)
 			return ret;
@@ -352,6 +468,14 @@ int nvme_get_log_page(int fd, __u32 xfer_len, struct nvme_get_log_args *args)
 		ptr += xfer;
 	} while (offset < data_len);
 
+#ifdef CONFIG_LIBURING
+	if (io_uring_kernel_support == IO_URING_AVAILABLE) {
+		ret = nvme_uring_cmd_wait_complete(&ring, n);
+		nvme_uring_cmd_exit(&ring);
+		if (ret < 0)
+			return -1;
+	}
+#endif
 	return 0;
 }
 

src/nvme/ioctl.h

@@ -40,6 +40,11 @@
  */
 #define NVME_LOG_PAGE_PDU_SIZE 4096
 
+/*
+ * should not exceed CAP.MQES, 16 is rational for most ssd
+ */
+#define NVME_URING_ENTRIES 16
+
 /**
  * struct nvme_passthru_cmd - nvme passthrough command structure
  * @opcode:	Operation code, see &enum nvme_io_opcodes and &enum nvme_admin_opcodes
@@ -180,6 +185,8 @@ struct nvme_uring_cmd {
 /* io_uring async commands: */
 #define NVME_URING_CMD_IO	_IOWR('N', 0x80, struct nvme_uring_cmd)
 #define NVME_URING_CMD_IO_VEC	_IOWR('N', 0x81, struct nvme_uring_cmd)
+#define NVME_URING_CMD_ADMIN	_IOWR('N', 0x82, struct nvme_uring_cmd)
+#define NVME_URING_CMD_ADMIN_VEC _IOWR('N', 0x83, struct nvme_uring_cmd)
 
 #endif /* _UAPI_LINUX_NVME_IOCTL_H */