Updating based on PR Feedback(5)

1. Added new module parameter zfs_dio_enabled which allows for all reads
   and writes to pass through the ARC. This module parameter can be set
   to 0 by default in OpenZFS 2.3 release if necessary.
2. Updated ZTS direct tests to account for the new zfs_dio_enabled
   module parameter.
3. Updated libzfs.abi to account for changes.

Signed-off-by: Brian Atkinson <batkinson@lanl.gov>

lib/libzfs/libzfs.abi

@@ -3450,6 +3450,8 @@
     <typedef-decl name='zpool_prefetch_type_t' type-id='0299ab50' id='e55ff6bc'/>
     <qualified-type-def type-id='8e8d4be3' const='yes' id='693c3853'/>
     <pointer-type-def type-id='693c3853' size-in-bits='64' id='22cce67b'/>
+    <qualified-type-def type-id='9c313c2d' const='yes' id='c3b7ba7d'/>
+    <pointer-type-def type-id='c3b7ba7d' size-in-bits='64' id='713a56f5'/>
     <pointer-type-def type-id='a093cbb8' size-in-bits='64' id='b13f38c3'/>
     <pointer-type-def type-id='35acf840' size-in-bits='64' id='17f3480d'/>
     <function-decl name='zpool_props_refresh' mangled-name='zpool_props_refresh' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_props_refresh'>

man/man4/zfs.4

@@ -291,6 +291,14 @@ Default dnode block size as a power of 2.
 .It Sy zfs_default_ibs Ns = Ns Sy 17 Po 128 KiB Pc Pq int
 Default dnode indirect block size as a power of 2.
 .
+.It Sy zfs_dio_enabled Ns = Ns Sy 0 Ns | Ns 1 Pq int
+Enable Direct I/O.
+If this setting is 0, then all I/O requests will be directed through the ARC
+acting as though the dataset property
+.Sy direct
+was set to
+.Sy disabled .
+.
 .It Sy zfs_history_output_max Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
 When attempting to log an output nvlist of an ioctl in the on-disk history,
 the output will not be stored if it is larger than this size (in bytes).

module/zfs/zfs_vnops.c

@@ -74,6 +74,14 @@ int zfs_bclone_enabled = 1;
  */
 static int zfs_bclone_wait_dirty = 0;
 
+/*
+ * Enable Direct I/O. If this setting is 0, then all I/O requests will be
+ * directed through the ARC acting as though the dataset property direct was
+ * set to disabled.
+ */
+static int zfs_dio_enabled = 1;
+
+
 /*
  * Maximum bytes to read per chunk in zfs_read().
  */
@@ -209,6 +217,10 @@ zfs_check_direct_enabled(znode_t *zp, int ioflags, boolean_t *is_direct)
 	*is_direct = B_FALSE;
 	int error;
 
+	if (!zfs_dio_enabled) {
+		return (0);
+	}
+
 	if ((error = zfs_enter(zfsvfs, FTAG)) != 0)
 		return (error);
 
@@ -249,6 +261,11 @@ zfs_setup_direct(struct znode *zp, zfs_uio_t *uio, zfs_uio_rw_t rw,
 	int ioflag = *ioflagp;
 	int error = 0;
 
+	if (!zfs_dio_enabled) {
+		*ioflagp &= ~O_DIRECT;
+		return (0);
+	}
+
 	if (os->os_direct == ZFS_DIRECT_DISABLED && (ioflag & O_DIRECT)) {
 		error = EAGAIN;
 		goto out;
@@ -1803,3 +1820,6 @@ ZFS_MODULE_PARAM(zfs, zfs_, bclone_enabled, INT, ZMOD_RW,
 
 ZFS_MODULE_PARAM(zfs, zfs_, bclone_wait_dirty, INT, ZMOD_RW,
 	"Wait for dirty blocks when cloning");
+
+ZFS_MODULE_PARAM(zfs, zfs_, dio_enabled, INT, ZMOD_RW,
+	"Enable Direct I/O");

tests/zfs-tests/include/tunables.cfg

@@ -101,6 +101,7 @@ VOL_RECURSIVE			vol.recursive			UNSUPPORTED
 VOL_USE_BLK_MQ			UNSUPPORTED			zvol_use_blk_mq
 BCLONE_ENABLED			bclone_enabled			zfs_bclone_enabled
 BCLONE_WAIT_DIRTY		bclone_wait_dirty		zfs_bclone_wait_dirty
+DIO_ENABLED			dio_enabled			zfs_dio_enabled
 XATTR_COMPAT			xattr_compat			zfs_xattr_compat
 ZEVENT_LEN_MAX			zevent.len_max			zfs_zevent_len_max
 ZEVENT_RETAIN_MAX		zevent.retain_max		zfs_zevent_retain_max

tests/zfs-tests/tests/functional/direct/cleanup.ksh

@@ -28,4 +28,10 @@
 
 verify_runnable "global"
 
-default_cleanup
+default_cleanup_noexit
+
+if tunable_exists DIO_ENABLED ; then
+	log_must restore_tunable DIO_ENABLED
+fi
+
+log_pass

tests/zfs-tests/tests/functional/direct/setup.ksh

@@ -27,6 +27,11 @@
 . $STF_SUITE/include/libtest.shlib
 verify_runnable "global"
 
+if tunable_exists DIO_ENABLED ; then
+	log_must save_tunable DIO_ENABLED
+	log_must set_tunable32 DIO_ENABLED 1
+fi
+
 default_raidz_setup_noexit "$DISKS"
 log_must zfs set compression=off $TESTPOOL/$TESTFS
 log_pass