zvol: Enable zvol threading functionality on FreeBSD

Make zvol I/O requests processing asynchronous on FreeBSD side in some
cases. Clone zvol threading logic and required module parameters from
Linux side. Make zvol threadpool creation/destruction logic shared for
both Linux and FreeBSD.
The IO requests are processed asynchronously in next cases:
- volmode=geom: if IO request passed thru zvol_geom_worker thread.
- volmode=cdev: if IO request passed thru struct cdevsw .d_strategy
routine, mean is AIO request.
In all other cases the IO requests are processed synchronously.

Signed-off-by: Fedor Uporov <[email protected]>

Author: fuporovvStack

include/sys/zvol_impl.h

@@ -60,6 +60,32 @@ typedef struct zvol_state {
 	boolean_t		zv_threading;	/* volthreading property */
 } zvol_state_t;
 
+/*
+ * zvol taskqs
+ */
+typedef struct zv_taskq {
+	uint_t tqs_cnt;
+	taskq_t **tqs_taskq;
+} zv_taskq_t;
+
+typedef struct zv_request_stack {
+	zvol_state_t	*zv;
+	struct bio	*bio;
+#ifdef __linux__
+	struct request	*rq;
+#endif
+} zv_request_t;
+
+typedef struct zv_request_task {
+	zv_request_t	zvr;
+	taskq_ent_t	ent;
+} zv_request_task_t;
+
+/*
+ * Switch taskq at multiple of 512 MB offset. This can be set to a lower value
+ * to utilize more threads for small files but may affect prefetch hits.
+ */
+#define	ZVOL_TASKQ_OFFSET_SHIFT 29
 
 extern krwlock_t zvol_state_lock;
 #define	ZVOL_HT_SIZE	1024
@@ -69,6 +95,10 @@ extern zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE];
 
 extern unsigned int zvol_volmode;
 extern unsigned int zvol_inhibit_dev;
+extern unsigned int zvol_threads;
+extern unsigned int zvol_num_taskqs;
+extern unsigned int zvol_request_sync;
+extern zv_taskq_t zvol_taskqs;
 
 /*
  * platform independent functions exported to platform code
@@ -86,14 +116,16 @@ void zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset,
     uint64_t size, boolean_t commit);
 int zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
     struct lwb *lwb, zio_t *zio);
-int zvol_init_impl(void);
+int zvol_init_impl(uint32_t ncpus, uint32_t num_taskqs, uint32_t zvol_threads);
 void zvol_fini_impl(void);
 void zvol_wait_close(zvol_state_t *zv);
 int zvol_clone_range(zvol_state_handle_t *, uint64_t,
     zvol_state_handle_t *, uint64_t, uint64_t);
 void zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype,
     uint64_t off, uint64_t len, uint64_t blksz, const blkptr_t *bps,
     size_t nbps);
+zv_request_task_t *zv_request_task_create(zv_request_t zvr);
+void zv_request_task_free(zv_request_task_t *task);
 
 /*
  * platform dependent functions exported to platform independent code

module/os/freebsd/zfs/zvol_os.c

@@ -99,6 +99,7 @@
 #include <geom/geom.h>
 #include <sys/zvol.h>
 #include <sys/zvol_impl.h>
+#include <cityhash.h>
 
 #include "zfs_namecheck.h"
 
@@ -169,7 +170,7 @@ static d_close_t	zvol_cdev_close;
 static d_ioctl_t	zvol_cdev_ioctl;
 static d_read_t		zvol_cdev_read;
 static d_write_t	zvol_cdev_write;
-static d_strategy_t	zvol_geom_bio_strategy;
+static d_strategy_t	zvol_bio_strategy;
 static d_kqfilter_t	zvol_cdev_kqfilter;
 
 static struct cdevsw zvol_cdevsw = {
@@ -181,7 +182,7 @@ static struct cdevsw zvol_cdevsw = {
 	.d_ioctl =	zvol_cdev_ioctl,
 	.d_read =	zvol_cdev_read,
 	.d_write =	zvol_cdev_write,
-	.d_strategy =	zvol_geom_bio_strategy,
+	.d_strategy =	zvol_bio_strategy,
 	.d_kqfilter =	zvol_cdev_kqfilter,
 };
 
@@ -211,7 +212,8 @@ static int zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace);
 static void zvol_geom_worker(void *arg);
 static void zvol_geom_bio_start(struct bio *bp);
 static int zvol_geom_bio_getattr(struct bio *bp);
-/* static d_strategy_t	zvol_geom_bio_strategy; (declared elsewhere) */
+/* static d_strategy_t	zvol_bio_strategy; (declared elsewhere) */
+static void zvol_geom_bio_strategy(struct bio *bp, boolean_t sync);
 
 /*
  * GEOM mode implementation
@@ -544,7 +546,7 @@ zvol_geom_worker(void *arg)
 			continue;
 		}
 		mtx_unlock(&zsg->zsg_queue_mtx);
-		zvol_geom_bio_strategy(bp);
+		zvol_geom_bio_strategy(bp, B_FALSE);
 	}
 }
 
@@ -576,7 +578,7 @@ zvol_geom_bio_start(struct bio *bp)
 		return;
 	}
 
-	zvol_geom_bio_strategy(bp);
+	zvol_geom_bio_strategy(bp, B_TRUE);
 }
 
 static int
@@ -660,9 +662,10 @@ zvol_cdev_kqfilter(struct cdev *dev, struct knote *kn)
 }
 
 static void
-zvol_geom_bio_strategy(struct bio *bp)
+zvol_strategy_impl(zv_request_t *zvr)
 {
 	zvol_state_t *zv;
+	struct bio *bp;
 	uint64_t off, volsize;
 	size_t resid;
 	char *addr;
@@ -673,11 +676,8 @@ zvol_geom_bio_strategy(struct bio *bp)
 	boolean_t is_dumpified;
 	boolean_t commit;
 
-	if (bp->bio_to)
-		zv = bp->bio_to->private;
-	else
-		zv = bp->bio_dev->si_drv2;
-
+	bp = zvr->bio;
+	zv = zvr->zv;
 	if (zv == NULL) {
 		error = SET_ERROR(ENXIO);
 		goto out;
@@ -813,6 +813,64 @@ zvol_geom_bio_strategy(struct bio *bp)
 		biofinish(bp, NULL, error);
 }
 
+static void
+zvol_strategy_task(void *arg)
+{
+	zv_request_task_t *task = arg;
+
+	zvol_strategy_impl(&task->zvr);
+	zv_request_task_free(task);
+}
+
+static void
+zvol_geom_bio_strategy(struct bio *bp, boolean_t sync)
+{
+	zv_taskq_t *ztqs = &zvol_taskqs;
+	zv_request_task_t *task;
+	zvol_state_t *zv;
+	uint64_t taskq_hash;
+	uint32_t tq_idx;
+	int error;
+
+	if (bp->bio_to)
+		zv = bp->bio_to->private;
+	else
+		zv = bp->bio_dev->si_drv2;
+
+	if (zv == NULL) {
+		error = SET_ERROR(ENXIO);
+		if (bp->bio_to)
+			g_io_deliver(bp, error);
+		else
+			biofinish(bp, NULL, error);
+
+		return;
+	}
+
+	zv_request_t zvr = {
+		.zv = zv,
+		.bio = bp,
+	};
+
+	if (sync || zvol_request_sync) {
+		zvol_strategy_impl(&zvr);
+		return;
+	}
+
+	taskq_hash = cityhash3((uintptr_t)zv, curcpu, bp->bio_offset >>
+	    ZVOL_TASKQ_OFFSET_SHIFT);
+	tq_idx = taskq_hash % ztqs->tqs_cnt;
+	task = zv_request_task_create(zvr);
+	taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx], zvol_strategy_task, task,
+	    0, &task->ent);
+}
+
+static void
+zvol_bio_strategy(struct bio *bp)
+{
+	zvol_geom_bio_strategy(bp, B_FALSE);
+}
+
 /*
  * Character device mode implementation
  */
@@ -1606,8 +1664,7 @@ zvol_busy(void)
 int
 zvol_init(void)
 {
-	zvol_init_impl();
-	return (0);
+	return (zvol_init_impl(mp_ncpus, zvol_num_taskqs, zvol_threads));
 }
 
 void