OS-7321 QEMU occasionally forgets to send a NOTIFY_ON_EMPTY interrupt

Reviewed by: Rui Loura <rui.loura@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
Approved by: Robert Mustacchi <rm@joyent.com>

hw/virtio-net.c

@@ -199,7 +199,7 @@ static void virtio_net_rein_tick(void *opaque)
 		return;
 	}
 
-	ret = virtqueue_stalled(n->tx_vq);
+	ret = virtqueue_stalled(&n->vdev, n->tx_vq);
 	if (ret == 1) {
 		virtio_net_rein_event(n, REIN_INJECT, n->rein_timer_ticks);
 		virtio_net_rein_disable(n);

hw/virtio.c

@@ -904,18 +904,28 @@ int virtqueue_handled(VirtQueue *vq)
  * the guest disabled the queue and is waiting for an interrupt from the host to
  * go and enable it again. In fact, when in this state a little bit of libproc
  * magic gets us going again rather reliably.
- * 
+ *
  * Eventually the guest will go through and unmask interrupts saying that it
  * wants an injection. If we reach a point in time where the last seen available
  * index is equal to the available index ring and is equal to the used index
  * ring, then we'll go ahead and install the interupt.
  */
-int virtqueue_stalled(VirtQueue *vq)
+int virtqueue_stalled(VirtIODevice *vdev, VirtQueue *vq)
 {
 	smp_mb();
 
-	if (vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT)
-		return (0);
+	if (vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT) {
+		/*
+		 * The guest has not enabled interrupts on the available ring.
+		 *
+		 * If the notify-on-empty feature is enabled, the specification
+		 * says we should interrupt anyway when the available ring is
+		 * completely drained.  Otherwise, continue to wait.
+		 */
+		if (!(vdev->guest_features & (1 << VIRTIO_F_NOTIFY_ON_EMPTY))) {
+			return (0);
+		}
+	}
 
 	if (vring_used_flags(vq) & VRING_USED_F_NO_NOTIFY)
 		return (0);
@@ -924,11 +934,27 @@ int virtqueue_stalled(VirtQueue *vq)
 		return (0);
 
 	/* We could have also lost the interrupt the other way */
-	if (vq->last_avail_idx != vring_avail_idx(vq))
+	if (vq->last_avail_idx != vring_avail_idx(vq)) {
+		/*
+		 * There are still some descriptors in the available ring to
+		 * process; return and process the queue again.
+		 */
 		return (2);
+	}
 
-	if (vq->last_avail_idx != vring_used_idx(vq))
+	if (vq->last_avail_idx != vring_used_idx(vq)) {
+		/*
+		 * Both the available ring index and the used ring index begin
+		 * at zero.  The available ring index is incremented by the
+		 * guest for each frame sent, and the used ring index is
+		 * incremented by the host each time we return the memory to
+		 * the guest.
+		 *
+		 * If the values are not equal, we have not accepted and then
+		 * returned an equal number of descriptors.
+		 */
 		return (0);
+	}
 
 	/*
 	 * Interrupts are enabled and we're at a point in time where we would

hw/virtio.h

@@ -226,6 +226,6 @@ EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq);
 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq);
 void virtio_queue_notify_vq(VirtQueue *vq);
 void virtio_irq(VirtQueue *vq);
-int virtqueue_stalled(VirtQueue *vq);
+int virtqueue_stalled(VirtIODevice *vdev, VirtQueue *vq);
 int virtqueue_handled(VirtQueue *vq);
 #endif