Re: [PATCH v2 5/5] virtio-scsi: introduce multiqueue support

[Michael S. Tsirkin]

On Tue, Dec 18, 2012 at 01:32:52PM +0100, Paolo Bonzini wrote:
> This patch adds queue steering to virtio-scsi.  When a target is sent
> multiple requests, we always drive them to the same queue so that FIFO
> processing order is kept.  However, if a target was idle, we can choose
> a queue arbitrarily.  In this case the queue is chosen according to the
> current VCPU, so the driver expects the number of request queues to be
> equal to the number of VCPUs.  This makes it easy and fast to select
> the queue, and also lets the driver optimize the IRQ affinity for the
> virtqueues (each virtqueue's affinity is set to the CPU that "owns"
> the queue).
> 
> The speedup comes from improving cache locality and giving CPU affinity
> to the virtqueues, which is why this scheme was selected.  Assuming that
> the thread that is sending requests to the device is I/O-bound, it is
> likely to be sleeping at the time the ISR is executed, and thus executing
> the ISR on the same processor that sent the requests is cheap.
> 
> However, the kernel will not execute the ISR on the "best" processor
> unless you explicitly set the affinity.  This is because in practice
> you will have many such I/O-bound processes and thus many otherwise
> idle processors.  Then the kernel will execute the ISR on a random
> processor, rather than the one that is sending requests to the device.
> 
> The alternative to per-CPU virtqueues is per-target virtqueues.  To
> achieve the same locality, we could dynamically choose the virtqueue's
> affinity based on the CPU of the last task that sent a request.  This
> is less appealing because we do not set the affinity directly---we only
> provide a hint to the irqbalanced running in userspace.  Dynamically
> changing the affinity only works if the userspace applies the hint
> fast enough.
> 
> Signed-off-by: Paolo Bonzini <pbonzini@xxxxxxxxxx>
> ---
> 	v1->v2: improved comments and commit messages, added memory barriers
> 
>  drivers/scsi/virtio_scsi.c |  234 +++++++++++++++++++++++++++++++++++++------
>  1 files changed, 201 insertions(+), 33 deletions(-)
> 
> diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
> index 4f6c6a3..ca9d29d 100644
> --- a/drivers/scsi/virtio_scsi.c
> +++ b/drivers/scsi/virtio_scsi.c
> @@ -26,6 +26,7 @@
>  
>  #define VIRTIO_SCSI_MEMPOOL_SZ 64
>  #define VIRTIO_SCSI_EVENT_LEN 8
> +#define VIRTIO_SCSI_VQ_BASE 2
>  
>  /* Command queue element */
>  struct virtio_scsi_cmd {
> @@ -57,24 +58,57 @@ struct virtio_scsi_vq {
>  	struct virtqueue *vq;
>  };
>  
> -/* Per-target queue state */
> +/*
> + * Per-target queue state.
> + *
> + * This struct holds the data needed by the queue steering policy.  When a
> + * target is sent multiple requests, we need to drive them to the same queue so
> + * that FIFO processing order is kept.  However, if a target was idle, we can
> + * choose a queue arbitrarily.  In this case the queue is chosen according to
> + * the current VCPU, so the driver expects the number of request queues to be
> + * equal to the number of VCPUs.  This makes it easy and fast to select the
> + * queue, and also lets the driver optimize the IRQ affinity for the virtqueues
> + * (each virtqueue's affinity is set to the CPU that "owns" the queue).
> + *
> + * An interesting effect of this policy is that only writes to req_vq need to
> + * take the tgt_lock.  Read can be done outside the lock because:
> + *
> + * - writes of req_vq only occur when atomic_inc_return(&tgt->reqs) returns 1.
> + *   In that case, no other CPU is reading req_vq: even if they were in
> + *   virtscsi_queuecommand_multi, they would be spinning on tgt_lock.
> + *
> + * - reads of req_vq only occur when the target is not idle (reqs != 0).
> + *   A CPU that enters virtscsi_queuecommand_multi will not modify req_vq.
> + *
> + * Similarly, decrements of reqs are never concurrent with writes of req_vq.
> + * Thus they can happen outside the tgt_lock, provided of course we make reqs
> + * an atomic_t.
> + */
>  struct virtio_scsi_target_state {
> -	/* Never held at the same time as vq_lock.  */
> +	/* This spinlock ever held at the same time as vq_lock.  */
>  	spinlock_t tgt_lock;
> +
> +	/* Count of outstanding requests.  */
> +	atomic_t reqs;
> +
> +	/* Currently active virtqueue for requests sent to this target.  */
> +	struct virtio_scsi_vq *req_vq;
>  };
>  
>  /* Driver instance state */
>  struct virtio_scsi {
>  	struct virtio_device *vdev;
>  
> -	struct virtio_scsi_vq ctrl_vq;
> -	struct virtio_scsi_vq event_vq;
> -	struct virtio_scsi_vq req_vq;
> -
>  	/* Get some buffers ready for event vq */
>  	struct virtio_scsi_event_node event_list[VIRTIO_SCSI_EVENT_LEN];
>  
>  	struct virtio_scsi_target_state *tgt;
> +
> +	u32 num_queues;
> +
> +	struct virtio_scsi_vq ctrl_vq;
> +	struct virtio_scsi_vq event_vq;
> +	struct virtio_scsi_vq req_vqs[];
>  };
>  
>  static struct kmem_cache *virtscsi_cmd_cache;
> @@ -109,6 +143,7 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
>  	struct virtio_scsi_cmd *cmd = buf;
>  	struct scsi_cmnd *sc = cmd->sc;
>  	struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;
> +	struct virtio_scsi_target_state *tgt = &vscsi->tgt[sc->device->id];
>  
>  	dev_dbg(&sc->device->sdev_gendev,
>  		"cmd %p response %u status %#02x sense_len %u\n",
> @@ -163,6 +198,8 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
>  
>  	mempool_free(cmd, virtscsi_cmd_pool);
>  	sc->scsi_done(sc);
> +
> +	atomic_dec(&tgt->reqs);
>  }
>  
>  static void virtscsi_vq_done(struct virtio_scsi *vscsi, struct virtqueue *vq,
> @@ -182,11 +219,45 @@ static void virtscsi_req_done(struct virtqueue *vq)
>  {
>  	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
>  	struct virtio_scsi *vscsi = shost_priv(sh);
> +	int index = vq->index - VIRTIO_SCSI_VQ_BASE;
> +	struct virtio_scsi_vq *req_vq = &vscsi->req_vqs[index];
>  	unsigned long flags;
>  
> -	spin_lock_irqsave(&vscsi->req_vq.vq_lock, flags);
> +	/*
> +	 * Read req_vq before decrementing the reqs field in
> +	 * virtscsi_complete_cmd.
> +	 *
> +	 * With barriers:
> +	 *
> +	 * 	CPU #0			virtscsi_queuecommand_multi (CPU #1)
> +	 * 	------------------------------------------------------------
> +	 * 	lock vq_lock
> +	 * 	read req_vq
> +	 * 	read reqs (reqs = 1)
> +	 * 	write reqs (reqs = 0)
> +	 * 				increment reqs (reqs = 1)
> +	 * 				write req_vq
> +	 *
> +	 * Possible reordering without barriers:
> +	 *
> +	 * 	CPU #0			virtscsi_queuecommand_multi (CPU #1)
> +	 * 	------------------------------------------------------------
> +	 * 	lock vq_lock
> +	 * 	read reqs (reqs = 1)
> +	 * 	write reqs (reqs = 0)
> +	 * 				increment reqs (reqs = 1)
> +	 * 				write req_vq
> +	 * 	read (wrong) req_vq
> +	 *
> +	 * We do not need a full smp_rmb, because req_vq is required to get
> +	 * to tgt->reqs: tgt is &vscsi->tgt[sc->device->id], where sc is stored
> +	 * in the virtqueue as the user token.
> +	 */
> +	smp_read_barrier_depends();
> +
> +	spin_lock_irqsave(&req_vq->vq_lock, flags);
>  	virtscsi_vq_done(vscsi, vq, virtscsi_complete_cmd);
> -	spin_unlock_irqrestore(&vscsi->req_vq.vq_lock, flags);
> +	spin_unlock_irqrestore(&req_vq->vq_lock, flags);
>  };
>  
>  static void virtscsi_complete_free(struct virtio_scsi *vscsi, void *buf)
> @@ -424,11 +495,12 @@ static int virtscsi_kick_cmd(struct virtio_scsi_target_state *tgt,
>  	return ret;
>  }
>  
> -static int virtscsi_queuecommand(struct Scsi_Host *sh, struct scsi_cmnd *sc)
> +static int virtscsi_queuecommand(struct virtio_scsi *vscsi,
> +				 struct virtio_scsi_target_state *tgt,
> +				 struct scsi_cmnd *sc)
>  {
> -	struct virtio_scsi *vscsi = shost_priv(sh);
> -	struct virtio_scsi_target_state *tgt = &vscsi->tgt[sc->device->id];
>  	struct virtio_scsi_cmd *cmd;
> +	struct virtio_scsi_vq *req_vq;
>  	int ret;
>  
>  	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
> @@ -461,7 +533,8 @@ static int virtscsi_queuecommand(struct Scsi_Host *sh, struct scsi_cmnd *sc)
>  	BUG_ON(sc->cmd_len > VIRTIO_SCSI_CDB_SIZE);
>  	memcpy(cmd->req.cmd.cdb, sc->cmnd, sc->cmd_len);
>  
> -	if (virtscsi_kick_cmd(tgt, &vscsi->req_vq, cmd,
> +	req_vq = ACCESS_ONCE(tgt->req_vq);

This ACCESS_ONCE without a barrier looks strange to me.
Can req_vq change? Needs a comment.

> +	if (virtscsi_kick_cmd(tgt, req_vq, cmd,
>  			      sizeof cmd->req.cmd, sizeof cmd->resp.cmd,
>  			      GFP_ATOMIC) == 0)
>  		ret = 0;
> @@ -472,6 +545,48 @@ out:
>  	return ret;
>  }
>  
> +static int virtscsi_queuecommand_single(struct Scsi_Host *sh,
> +					struct scsi_cmnd *sc)
> +{
> +	struct virtio_scsi *vscsi = shost_priv(sh);
> +	struct virtio_scsi_target_state *tgt = &vscsi->tgt[sc->device->id];
> +
> +	atomic_inc(&tgt->reqs);

And here we don't have barrier after atomic? Why? Needs a comment.

> +	return virtscsi_queuecommand(vscsi, tgt, sc);
> +}
> +
> +static int virtscsi_queuecommand_multi(struct Scsi_Host *sh,
> +				       struct scsi_cmnd *sc)
> +{
> +	struct virtio_scsi *vscsi = shost_priv(sh);
> +	struct virtio_scsi_target_state *tgt = &vscsi->tgt[sc->device->id];
> +	unsigned long flags;
> +	u32 queue_num;
> +
> +	/*
> +	 * Using an atomic_t for tgt->reqs lets the virtqueue handler
> +	 * decrement it without taking the spinlock.
> +	 *
> +	 * We still need a critical section to prevent concurrent submissions
> +	 * from picking two different req_vqs.
> +	 */
> +	spin_lock_irqsave(&tgt->tgt_lock, flags);
> +	if (atomic_inc_return(&tgt->reqs) == 1) {
> +		queue_num = smp_processor_id();
> +		while (unlikely(queue_num >= vscsi->num_queues))
> +			queue_num -= vscsi->num_queues;
> +
> +		/*
> +		 * Write reqs before writing req_vq, matching the
> +		 * smp_read_barrier_depends() in virtscsi_req_done.
> +		 */
> +		smp_wmb();
> +		tgt->req_vq = &vscsi->req_vqs[queue_num];
> +	}
> +	spin_unlock_irqrestore(&tgt->tgt_lock, flags);
> +	return virtscsi_queuecommand(vscsi, tgt, sc);
> +}
> +
>  static int virtscsi_tmf(struct virtio_scsi *vscsi, struct virtio_scsi_cmd *cmd)
>  {
>  	DECLARE_COMPLETION_ONSTACK(comp);
> @@ -541,12 +656,26 @@ static int virtscsi_abort(struct scsi_cmnd *sc)
>  	return virtscsi_tmf(vscsi, cmd);
>  }
>  
> -static struct scsi_host_template virtscsi_host_template = {
> +static struct scsi_host_template virtscsi_host_template_single = {
>  	.module = THIS_MODULE,
>  	.name = "Virtio SCSI HBA",
>  	.proc_name = "virtio_scsi",
> -	.queuecommand = virtscsi_queuecommand,
>  	.this_id = -1,
> +	.queuecommand = virtscsi_queuecommand_single,
> +	.eh_abort_handler = virtscsi_abort,
> +	.eh_device_reset_handler = virtscsi_device_reset,
> +
> +	.can_queue = 1024,
> +	.dma_boundary = UINT_MAX,
> +	.use_clustering = ENABLE_CLUSTERING,
> +};
> +
> +static struct scsi_host_template virtscsi_host_template_multi = {
> +	.module = THIS_MODULE,
> +	.name = "Virtio SCSI HBA",
> +	.proc_name = "virtio_scsi",
> +	.this_id = -1,
> +	.queuecommand = virtscsi_queuecommand_multi,
>  	.eh_abort_handler = virtscsi_abort,
>  	.eh_device_reset_handler = virtscsi_device_reset,
>  
> @@ -572,16 +701,27 @@ static struct scsi_host_template virtscsi_host_template = {
>  				  &__val, sizeof(__val)); \
>  	})
>  
> +
>  static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
> -			     struct virtqueue *vq)
> +			     struct virtqueue *vq, bool affinity)
>  {
>  	spin_lock_init(&virtscsi_vq->vq_lock);
>  	virtscsi_vq->vq = vq;
> +	if (affinity)
> +		virtqueue_set_affinity(vq, vq->index - VIRTIO_SCSI_VQ_BASE);

I've been thinking about how set_affinity
interacts with online/offline CPUs.
Any idea?


>  }
>  
> -static void virtscsi_init_tgt(struct virtio_scsi_target_state *tgt)
> +static void virtscsi_init_tgt(struct virtio_scsi *vscsi, int i)
>  {
> +	struct virtio_scsi_target_state *tgt = &vscsi->tgt[i];
>  	spin_lock_init(&tgt->tgt_lock);
> +	atomic_set(&tgt->reqs, 0);
> +
> +	/*
> +	 * The default is unused for multiqueue, but with a single queue
> +	 * or target we use it in virtscsi_queuecommand.
> +	 */
> +	tgt->req_vq = &vscsi->req_vqs[0];
>  }
>  
>  static void virtscsi_scan(struct virtio_device *vdev)
> @@ -609,28 +749,41 @@ static int virtscsi_init(struct virtio_device *vdev,
>  			 struct virtio_scsi *vscsi, int num_targets)
>  {
>  	int err;
> -	struct virtqueue *vqs[3];
>  	u32 i, sg_elems;
> +	u32 num_vqs;
> +	vq_callback_t **callbacks;
> +	const char **names;
> +	struct virtqueue **vqs;
>  
> -	vq_callback_t *callbacks[] = {
> -		virtscsi_ctrl_done,
> -		virtscsi_event_done,
> -		virtscsi_req_done
> -	};
> -	const char *names[] = {
> -		"control",
> -		"event",
> -		"request"
> -	};
> +	num_vqs = vscsi->num_queues + VIRTIO_SCSI_VQ_BASE;
> +	vqs = kmalloc(num_vqs * sizeof(struct virtqueue *), GFP_KERNEL);
> +	callbacks = kmalloc(num_vqs * sizeof(vq_callback_t *), GFP_KERNEL);
> +	names = kmalloc(num_vqs * sizeof(char *), GFP_KERNEL);
> +
> +	if (!callbacks || !vqs || !names) {
> +		err = -ENOMEM;
> +		goto out;
> +	}
> +
> +	callbacks[0] = virtscsi_ctrl_done;
> +	callbacks[1] = virtscsi_event_done;
> +	names[0] = "control";
> +	names[1] = "event";
> +	for (i = VIRTIO_SCSI_VQ_BASE; i < num_vqs; i++) {
> +		callbacks[i] = virtscsi_req_done;
> +		names[i] = "request";
> +	}
>  
>  	/* Discover virtqueues and write information to configuration.  */
> -	err = vdev->config->find_vqs(vdev, 3, vqs, callbacks, names);
> +	err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names);
>  	if (err)
>  		return err;
>  
> -	virtscsi_init_vq(&vscsi->ctrl_vq, vqs[0]);
> -	virtscsi_init_vq(&vscsi->event_vq, vqs[1]);
> -	virtscsi_init_vq(&vscsi->req_vq, vqs[2]);
> +	virtscsi_init_vq(&vscsi->ctrl_vq, vqs[0], false);
> +	virtscsi_init_vq(&vscsi->event_vq, vqs[1], false);
> +	for (i = VIRTIO_SCSI_VQ_BASE; i < num_vqs; i++)
> +		virtscsi_init_vq(&vscsi->req_vqs[i - VIRTIO_SCSI_VQ_BASE],
> +				 vqs[i], vscsi->num_queues > 1);

So affinity is true if >1 vq? I am guessing this is not
going to do the right thing unless you have at least
as many vqs as CPUs.

>  
>  	virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE);
>  	virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE);
> @@ -647,11 +800,14 @@ static int virtscsi_init(struct virtio_device *vdev,
>  		goto out;
>  	}
>  	for (i = 0; i < num_targets; i++)
> -		virtscsi_init_tgt(&vscsi->tgt[i]);
> +		virtscsi_init_tgt(vscsi, i);
>  
>  	err = 0;
>  
>  out:
> +	kfree(names);
> +	kfree(callbacks);
> +	kfree(vqs);
>  	if (err)
>  		virtscsi_remove_vqs(vdev);
>  	return err;
> @@ -664,11 +820,22 @@ static int __devinit virtscsi_probe(struct virtio_device *vdev)
>  	int err;
>  	u32 sg_elems, num_targets;
>  	u32 cmd_per_lun;
> +	u32 num_queues;
> +	struct scsi_host_template *hostt;
> +
> +	/* We need to know how many queues before we allocate.  */
> +	num_queues = virtscsi_config_get(vdev, num_queues) ?: 1;
>  
>  	/* Allocate memory and link the structs together.  */
>  	num_targets = virtscsi_config_get(vdev, max_target) + 1;
> -	shost = scsi_host_alloc(&virtscsi_host_template, sizeof(*vscsi));
>  
> +	if (num_queues == 1)
> +		hostt = &virtscsi_host_template_single;
> +	else
> +		hostt = &virtscsi_host_template_multi;
> +
> +	shost = scsi_host_alloc(hostt,
> +		sizeof(*vscsi) + sizeof(vscsi->req_vqs[0]) * num_queues);
>  	if (!shost)
>  		return -ENOMEM;
>  
> @@ -676,6 +843,7 @@ static int __devinit virtscsi_probe(struct virtio_device *vdev)
>  	shost->sg_tablesize = sg_elems;
>  	vscsi = shost_priv(shost);
>  	vscsi->vdev = vdev;
> +	vscsi->num_queues = num_queues;
>  	vdev->priv = shost;
>  
>  	err = virtscsi_init(vdev, vscsi, num_targets);
> -- 
> 1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/