zvol: Enable zvol threading functionality on FreeBSD

Make zvol I/O requests processing asynchronous on FreeBSD side. Clone
zvol threading logic and required module parameters from Linux side.
Make zvol threadpool creation/destruction logic shared for both Linux
and FreeBSD. Use OS physio routines in async mode, in case if zvol is
exported in cdev mode. Disable volthreading zfs parameter on FreeBSD
side by default.

Signed-off-by: Fedor Uporov <[email protected]>

Author: fuporovvStack

include/sys/zvol_impl.h

@@ -60,6 +60,30 @@ typedef struct zvol_state {
 	boolean_t		zv_threading;	/* volthreading property */
 } zvol_state_t;
 
+/*
+ * zvol taskqs
+ */
+typedef struct zv_taskq {
+	uint_t tqs_cnt;
+	taskq_t **tqs_taskq;
+} zv_taskq_t;
+
+typedef struct zv_request_stack {
+	zvol_state_t	*zv;
+	struct bio	*bio;
+	struct request	*rq;
+} zv_request_t;
+
+typedef struct zv_request_task {
+	zv_request_t	zvr;
+	taskq_ent_t	ent;
+} zv_request_task_t;
+
+/*
+ * Switch taskq at multiple of 512 MB offset. This can be set to a lower value
+ * to utilize more threads for small files but may affect prefetch hits.
+ */
+#define	ZVOL_TASKQ_OFFSET_SHIFT 29
 
 extern krwlock_t zvol_state_lock;
 #define	ZVOL_HT_SIZE	1024
@@ -70,6 +94,8 @@ extern zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE];
 extern unsigned int zvol_volmode;
 extern unsigned int zvol_inhibit_dev;
 
+extern zv_taskq_t zvol_taskqs;
+
 /*
  * platform independent functions exported to platform code
  */
@@ -86,14 +112,16 @@ void zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset,
     uint64_t size, boolean_t commit);
 int zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
     struct lwb *lwb, zio_t *zio);
-int zvol_init_impl(void);
+int zvol_init_impl(uint32_t ncpus, uint32_t num_taskqs, uint32_t zvol_threads);
 void zvol_fini_impl(void);
 void zvol_wait_close(zvol_state_t *zv);
 int zvol_clone_range(zvol_state_handle_t *, uint64_t,
     zvol_state_handle_t *, uint64_t, uint64_t);
 void zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype,
     uint64_t off, uint64_t len, uint64_t blksz, const blkptr_t *bps,
     size_t nbps);
+zv_request_task_t *zv_request_task_create(zv_request_t zvr);
+void zv_request_task_free(zv_request_task_t *task);
 
 /*
  * platform dependent functions exported to platform independent code

module/os/freebsd/zfs/zvol_os.c

@@ -99,6 +99,7 @@
 #include <geom/geom.h>
 #include <sys/zvol.h>
 #include <sys/zvol_impl.h>
+#include <cityhash.h>
 
 #include "zfs_namecheck.h"
 
@@ -146,8 +147,19 @@ SYSCTL_NODE(_vfs_zfs, OID_AUTO, vol, CTLFLAG_RW, 0, "ZFS VOLUME");
 SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, mode, CTLFLAG_RWTUN, &zvol_volmode, 0,
 	"Expose as GEOM providers (1), device files (2) or neither");
 static boolean_t zpool_on_zvol = B_FALSE;
+static unsigned int zvol_threads = 0;
+static unsigned int zvol_num_taskqs = 0;
+static unsigned int zvol_request_sync = 1;
 SYSCTL_INT(_vfs_zfs_vol, OID_AUTO, recursive, CTLFLAG_RWTUN, &zpool_on_zvol, 0,
 	"Allow zpools to use zvols as vdevs (DANGEROUS)");
+SYSCTL_UINT(_vfs_zfs_vol, OID_AUTO, threads, CTLFLAG_RWTUN, &zvol_threads, 0,
+	"Number of threads for I/O requests. Set to 0 to use all active CPUs");
+SYSCTL_UINT(_vfs_zfs_vol, OID_AUTO, taskqs, CTLFLAG_RWTUN, &zvol_num_taskqs, 0,
+	"Number of zvol taskqs");
+SYSCTL_UINT(_vfs_zfs_vol, OID_AUTO, sync, CTLFLAG_RWTUN, &zvol_request_sync, 0,
+	"Synchronously handle bio requests");
+
+struct request {};
 
 /*
  * Toggle unmap functionality.
@@ -660,9 +672,10 @@ zvol_cdev_kqfilter(struct cdev *dev, struct knote *kn)
 }
 
 static void
-zvol_geom_bio_strategy(struct bio *bp)
+zvol_strategy_impl(zv_request_t *zvr)
 {
 	zvol_state_t *zv;
+	struct bio *bp;
 	uint64_t off, volsize;
 	size_t resid;
 	char *addr;
@@ -673,11 +686,8 @@ zvol_geom_bio_strategy(struct bio *bp)
 	boolean_t is_dumpified;
 	boolean_t commit;
 
-	if (bp->bio_to)
-		zv = bp->bio_to->private;
-	else
-		zv = bp->bio_dev->si_drv2;
-
+	bp = zvr->bio;
+	zv = zvr->zv;
 	if (zv == NULL) {
 		error = SET_ERROR(ENXIO);
 		goto out;
@@ -813,6 +823,58 @@ zvol_geom_bio_strategy(struct bio *bp)
 		biofinish(bp, NULL, error);
 }
 
+static void
+zvol_strategy_task(void *arg)
+{
+	zv_request_task_t *task = arg;
+
+	zvol_strategy_impl(&task->zvr);
+	zv_request_task_free(task);
+}
+
+static void
+zvol_geom_bio_strategy(struct bio *bp)
+{
+	zv_taskq_t *ztqs = &zvol_taskqs;
+	zv_request_task_t *task;
+	zvol_state_t *zv;
+	uint64_t taskq_hash;
+	uint32_t tq_idx;
+	int error;
+
+	if (bp->bio_to)
+		zv = bp->bio_to->private;
+	else
+		zv = bp->bio_dev->si_drv2;
+
+	if (zv == NULL) {
+		error = SET_ERROR(ENXIO);
+		if (bp->bio_to)
+			g_io_deliver(bp, error);
+		else
+			biofinish(bp, NULL, error);
+
+		return;
+	}
+
+	zv_request_t zvr = {
+		.zv = zv,
+		.bio = bp,
+	};
+
+	if (zvol_request_sync || zv->zv_threading == B_FALSE) {
+		zvol_strategy_impl(&zvr);
+		return;
+	}
+
+	taskq_hash = cityhash3((uintptr_t)zv, curcpu, bp->bio_offset >>
+	    ZVOL_TASKQ_OFFSET_SHIFT);
+	tq_idx = taskq_hash % ztqs->tqs_cnt;
+	task = zv_request_task_create(zvr);
+	taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx], zvol_strategy_task, task,
+	    0, &task->ent);
+}
+
 /*
  * Character device mode implementation
  */
@@ -839,6 +901,9 @@ zvol_cdev_read(struct cdev *dev, struct uio *uio_s, int ioflag)
 	    (zfs_uio_offset(&uio) < 0 || zfs_uio_offset(&uio) > volsize))
 		return (SET_ERROR(EIO));
 
+	if (!zvol_request_sync && !zv->zv_threading)
+		return (physread(dev, uio_s, ioflag));
+
 	rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER);
 	ssize_t start_resid = zfs_uio_resid(&uio);
 	lr = zfs_rangelock_enter(&zv->zv_rangelock, zfs_uio_offset(&uio),
@@ -886,6 +951,9 @@ zvol_cdev_write(struct cdev *dev, struct uio *uio_s, int ioflag)
 	    (zfs_uio_offset(&uio) < 0 || zfs_uio_offset(&uio) > volsize))
 		return (SET_ERROR(EIO));
 
+	if (!zvol_request_sync && !zv->zv_threading)
+		return (physwrite(dev, uio_s, ioflag));
+
 	ssize_t start_resid = zfs_uio_resid(&uio);
 	commit = (ioflag & IO_SYNC) ||
 	    (zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS);
@@ -1385,6 +1453,7 @@ zvol_os_create_minor(const char *name)
 	uint64_t volsize;
 	uint64_t volmode, hash;
 	int error;
+	uint64_t volthreading;
 	bool replayed_zil = B_FALSE;
 
 	ZFS_LOG(1, "Creating ZVOL %s...", name);
@@ -1479,6 +1548,13 @@ zvol_os_create_minor(const char *name)
 	zv->zv_volsize = volsize;
 	zv->zv_objset = os;
 
+	/* Default */
+	zv->zv_threading = B_FALSE;
+	error = dsl_prop_get_integer(name, "volthreading", &volthreading,
+	    NULL);
+	if (error == 0)
+		zv->zv_threading = volthreading;
+
 	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
 	error = dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
 	if (error)
@@ -1606,8 +1682,7 @@ zvol_busy(void)
 int
 zvol_init(void)
 {
-	zvol_init_impl();
-	return (0);
+	return (zvol_init_impl(mp_ncpus, zvol_num_taskqs, zvol_threads));
 }
 
 void

module/os/linux/zfs/zvol_os.c

@@ -117,45 +117,8 @@ struct zvol_state_os {
 	boolean_t use_blk_mq;
 };
 
-typedef struct zv_taskq {
-	uint_t tqs_cnt;
-	taskq_t **tqs_taskq;
-} zv_taskq_t;
-static zv_taskq_t zvol_taskqs;
 static struct ida zvol_ida;
 
-typedef struct zv_request_stack {
-	zvol_state_t	*zv;
-	struct bio	*bio;
-	struct request *rq;
-} zv_request_t;
-
-typedef struct zv_work {
-	struct request  *rq;
-	struct work_struct work;
-} zv_work_t;
-
-typedef struct zv_request_task {
-	zv_request_t zvr;
-	taskq_ent_t	ent;
-} zv_request_task_t;
-
-static zv_request_task_t *
-zv_request_task_create(zv_request_t zvr)
-{
-	zv_request_task_t *task;
-	task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP);
-	taskq_init_ent(&task->ent);
-	task->zvr = zvr;
-	return (task);
-}
-
-static void
-zv_request_task_free(zv_request_task_t *task)
-{
-	kmem_free(task, sizeof (*task));
-}
-
 /*
  * This is called when a new block multiqueue request comes in.  A request
  * contains one or more BIOs.
@@ -1793,59 +1756,17 @@ zvol_init(void)
 {
 	int error;
 
-	/*
-	 * zvol_threads is the module param the user passes in.
-	 *
-	 * zvol_actual_threads is what we use internally, since the user can
-	 * pass zvol_thread = 0 to mean "use all the CPUs" (the default).
-	 */
-	static unsigned int zvol_actual_threads;
+	ida_init(&zvol_ida);
 
-	if (zvol_threads == 0) {
-		/*
-		 * See dde9380a1 for why 32 was chosen here.  This should
-		 * probably be refined to be some multiple of the number
-		 * of CPUs.
-		 */
-		zvol_actual_threads = MAX(num_online_cpus(), 32);
-	} else {
-		zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024);
+	error = zvol_init_impl(num_online_cpus(), zvol_num_taskqs,
+	    zvol_threads);
+	if (error) {
+		printk(KERN_INFO "ZFS: zvol_init_impl() failed %d\n", error);
+		return (error);
 	}
 
-	/*
-	 * Use atleast 32 zvol_threads but for many core system,
-	 * prefer 6 threads per taskq, but no more taskqs
-	 * than threads in them on large systems.
-	 *
-	 *                 taskq   total
-	 * cpus    taskqs  threads threads
-	 * ------- ------- ------- -------
-	 * 1       1       32       32
-	 * 2       1       32       32
-	 * 4       1       32       32
-	 * 8       2       16       32
-	 * 16      3       11       33
-	 * 32      5       7        35
-	 * 64      8       8        64
-	 * 128     11      12       132
-	 * 256     16      16       256
-	 */
-	zv_taskq_t *ztqs = &zvol_taskqs;
-	uint_t num_tqs = MIN(num_online_cpus(), zvol_num_taskqs);
-	if (num_tqs == 0) {
-		num_tqs = 1 + num_online_cpus() / 6;
-		while (num_tqs * num_tqs > zvol_actual_threads)
-			num_tqs--;
-	}
-	uint_t per_tq_thread = zvol_actual_threads / num_tqs;
-	if (per_tq_thread * num_tqs < zvol_actual_threads)
-		per_tq_thread++;
-	ztqs->tqs_cnt = num_tqs;
-	ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP);
 	error = register_blkdev(zvol_major, ZVOL_DRIVER);
 	if (error) {
-		kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * sizeof (taskq_t *));
-		ztqs->tqs_taskq = NULL;
 		printk(KERN_INFO "ZFS: register_blkdev() failed %d\n", error);
 		return (error);
 	}
@@ -1864,47 +1785,15 @@ zvol_init(void)
 		    1024);
 	}
 
-	for (uint_t i = 0; i < num_tqs; i++) {
-		char name[32];
-		(void) snprintf(name, sizeof (name), "%s_tq-%u",
-		    ZVOL_DRIVER, i);
-		ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread,
-		    maxclsyspri, per_tq_thread, INT_MAX,
-		    TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
-		if (ztqs->tqs_taskq[i] == NULL) {
-			for (int j = i - 1; j >= 0; j--)
-				taskq_destroy(ztqs->tqs_taskq[j]);
-			unregister_blkdev(zvol_major, ZVOL_DRIVER);
-			kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
-			    sizeof (taskq_t *));
-			ztqs->tqs_taskq = NULL;
-			return (-ENOMEM);
-		}
-	}
-
-	zvol_init_impl();
-	ida_init(&zvol_ida);
 	return (0);
 }
 
 void
 zvol_fini(void)
 {
-	zv_taskq_t *ztqs = &zvol_taskqs;
-	zvol_fini_impl();
 	unregister_blkdev(zvol_major, ZVOL_DRIVER);
 
-	if (ztqs->tqs_taskq == NULL) {
-		ASSERT3U(ztqs->tqs_cnt, ==, 0);
-	} else {
-		for (uint_t i = 0; i < ztqs->tqs_cnt; i++) {
-			ASSERT3P(ztqs->tqs_taskq[i], !=, NULL);
-			taskq_destroy(ztqs->tqs_taskq[i]);
-		}
-		kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
-		    sizeof (taskq_t *));
-		ztqs->tqs_taskq = NULL;
-	}
+	zvol_fini_impl();
 
 	ida_destroy(&zvol_ida);
 }

module/zcommon/zfs_prop.c

@@ -643,7 +643,12 @@ zfs_prop_init(void)
 	    ZFS_TYPE_VOLUME, "512 to 128k, power of 2",	"VOLBLOCK", B_FALSE,
 	    sfeatures);
 	zprop_register_index(ZFS_PROP_VOLTHREADING, "volthreading",
-	    1, PROP_DEFAULT, ZFS_TYPE_VOLUME, "on | off", "zvol threading",
+#ifdef __FreeBSD__
+	    0,
+#else
+	    1,
+#endif
+	    PROP_DEFAULT, ZFS_TYPE_VOLUME, "on | off", "zvol threading",
 	    boolean_table, sfeatures);
 	zprop_register_number(ZFS_PROP_USEDSNAP, "usedbysnapshots", 0,
 	    PROP_READONLY, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "<size>",