diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1832ea7982a6..d0904ac0aa93 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -67,6 +67,11 @@ #define LAST_ADD_TIME_INVALID(vq) #endif +enum vq_layout { + VQ_LAYOUT_SPLIT = 0, + VQ_LAYOUT_PACKED, +}; + struct vring_desc_state_split { void *data; /* Data for callback. */ @@ -159,12 +164,29 @@ struct vring_virtqueue_packed { size_t event_size_in_bytes; }; +struct vring_virtqueue; + +struct virtqueue_ops { + int (*add)(struct vring_virtqueue *vq, struct scatterlist *sgs[], + unsigned int total_sg, unsigned int out_sgs, + unsigned int in_sgs, void *data, + void *ctx, bool premapped, gfp_t gfp); + void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx); + bool (*kick_prepare)(struct vring_virtqueue *vq); + void (*disable_cb)(struct vring_virtqueue *vq); + bool (*enable_cb_delayed)(struct vring_virtqueue *vq); + unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq); + bool (*poll)(const struct vring_virtqueue *vq, + unsigned int last_used_idx); + void *(*detach_unused_buf)(struct vring_virtqueue *vq); + bool (*more_used)(const struct vring_virtqueue *vq); + int (*resize)(struct vring_virtqueue *vq, u32 num); + void (*reset)(struct vring_virtqueue *vq); +}; + struct vring_virtqueue { struct virtqueue vq; - /* Is this a packed ring? */ - bool packed_ring; - /* Is DMA API used? */ bool use_map_api; @@ -180,6 +202,8 @@ struct vring_virtqueue { /* Host publishes avail event idx */ bool event; + enum vq_layout layout; + /* Head of free buffer list. */ unsigned int free_head; /* Number we've added since last sync. */ @@ -231,6 +255,12 @@ static void vring_free(struct virtqueue *_vq); #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq) + +static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq) +{ + return vq->layout == VQ_LAYOUT_PACKED; +} + static bool virtqueue_use_indirect(const struct vring_virtqueue *vq, unsigned int total_sg) { @@ -433,7 +463,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num) { vq->vq.num_free = num; - if (vq->packed_ring) + if (virtqueue_is_packed(vq)) vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); else vq->last_used_idx = 0; @@ -1121,6 +1151,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split, return 0; } +static const struct virtqueue_ops split_ops; + static struct virtqueue *__vring_new_virtqueue_split(unsigned int index, struct vring_virtqueue_split *vring_split, struct virtio_device *vdev, @@ -1138,7 +1170,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index, if (!vq) return NULL; - vq->packed_ring = false; + vq->layout = VQ_LAYOUT_SPLIT; vq->vq.callback = callback; vq->vq.vdev = vdev; vq->vq.name = name; @@ -2071,11 +2103,12 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq) /* we need to reset the desc.flags. For more, see is_used_desc_packed() */ memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes); - virtqueue_init(vq, vq->packed.vring.num); virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback); } +static const struct virtqueue_ops packed_ops; + static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index, struct vring_virtqueue_packed *vring_packed, struct virtio_device *vdev, @@ -2106,7 +2139,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index, #else vq->broken = false; #endif - vq->packed_ring = true; + vq->layout = VQ_LAYOUT_PACKED; vq->map = map; vq->use_map_api = vring_use_map_api(vdev); @@ -2194,6 +2227,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num) return -ENOMEM; } +static const struct virtqueue_ops split_ops = { + .add = virtqueue_add_split, + .get = virtqueue_get_buf_ctx_split, + .kick_prepare = virtqueue_kick_prepare_split, + .disable_cb = virtqueue_disable_cb_split, + .enable_cb_delayed = virtqueue_enable_cb_delayed_split, + .enable_cb_prepare = virtqueue_enable_cb_prepare_split, + .poll = virtqueue_poll_split, + .detach_unused_buf = virtqueue_detach_unused_buf_split, + .more_used = more_used_split, + .resize = virtqueue_resize_split, + .reset = virtqueue_reset_split, +}; + +static const struct virtqueue_ops packed_ops = { + .add = virtqueue_add_packed, + .get = virtqueue_get_buf_ctx_packed, + .kick_prepare = virtqueue_kick_prepare_packed, + .disable_cb = virtqueue_disable_cb_packed, + .enable_cb_delayed = virtqueue_enable_cb_delayed_packed, + .enable_cb_prepare = virtqueue_enable_cb_prepare_packed, + .poll = virtqueue_poll_packed, + .detach_unused_buf = virtqueue_detach_unused_buf_packed, + .more_used = more_used_packed, + .resize = virtqueue_resize_packed, + .reset = virtqueue_reset_packed, +}; + static int virtqueue_disable_and_recycle(struct virtqueue *_vq, void (*recycle)(struct virtqueue *vq, void *buf)) { @@ -2236,6 +2297,42 @@ static int virtqueue_enable_after_reset(struct virtqueue *_vq) * Generic functions and exported symbols. */ +#define VIRTQUEUE_CALL(vq, op, ...) \ + ({ \ + typeof(vq) __VIRTQUEUE_CALL_vq = (vq); \ + typeof(split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__)) ret; \ + \ + switch (__VIRTQUEUE_CALL_vq->layout) { \ + case VQ_LAYOUT_SPLIT: \ + ret = split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \ + break; \ + case VQ_LAYOUT_PACKED: \ + ret = packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__);\ + break; \ + default: \ + BUG(); \ + break; \ + } \ + ret; \ +}) + +#define VOID_VIRTQUEUE_CALL(vq, op, ...) \ + ({ \ + typeof(vq) __VIRTQUEUE_CALL_vq = (vq); \ + \ + switch (__VIRTQUEUE_CALL_vq->layout) { \ + case VQ_LAYOUT_SPLIT: \ + split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \ + break; \ + case VQ_LAYOUT_PACKED: \ + packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \ + break; \ + default: \ + BUG(); \ + break; \ + } \ +}) + static inline int virtqueue_add(struct virtqueue *_vq, struct scatterlist *sgs[], unsigned int total_sg, @@ -2248,10 +2345,9 @@ static inline int virtqueue_add(struct virtqueue *_vq, { struct vring_virtqueue *vq = to_vvq(_vq); - return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg, - out_sgs, in_sgs, data, ctx, premapped, gfp) : - virtqueue_add_split(vq, sgs, total_sg, - out_sgs, in_sgs, data, ctx, premapped, gfp); + return VIRTQUEUE_CALL(vq, add, sgs, total_sg, + out_sgs, in_sgs, data, + ctx, premapped, gfp); } /** @@ -2441,8 +2537,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) : - virtqueue_kick_prepare_split(vq); + return VIRTQUEUE_CALL(vq, kick_prepare); } EXPORT_SYMBOL_GPL(virtqueue_kick_prepare); @@ -2512,8 +2607,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len, { struct vring_virtqueue *vq = to_vvq(_vq); - return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) : - virtqueue_get_buf_ctx_split(vq, len, ctx); + return VIRTQUEUE_CALL(vq, get, len, ctx); } EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx); @@ -2535,10 +2629,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - if (vq->packed_ring) - virtqueue_disable_cb_packed(vq); - else - virtqueue_disable_cb_split(vq); + VOID_VIRTQUEUE_CALL(vq, disable_cb); } EXPORT_SYMBOL_GPL(virtqueue_disable_cb); @@ -2561,8 +2652,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq) if (vq->event_triggered) vq->event_triggered = false; - return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) : - virtqueue_enable_cb_prepare_split(vq); + return VIRTQUEUE_CALL(vq, enable_cb_prepare); } EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare); @@ -2583,8 +2673,8 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx) return false; virtio_mb(vq->weak_barriers); - return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) : - virtqueue_poll_split(vq, last_used_idx); + + return VIRTQUEUE_CALL(vq, poll, last_used_idx); } EXPORT_SYMBOL_GPL(virtqueue_poll); @@ -2627,8 +2717,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) if (vq->event_triggered) data_race(vq->event_triggered = false); - return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) : - virtqueue_enable_cb_delayed_split(vq); + return VIRTQUEUE_CALL(vq, enable_cb_delayed); } EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); @@ -2644,14 +2733,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) : - virtqueue_detach_unused_buf_split(vq); + return VIRTQUEUE_CALL(vq, detach_unused_buf); } EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf); static inline bool more_used(const struct vring_virtqueue *vq) { - return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq); + return VIRTQUEUE_CALL(vq, more_used); } /** @@ -2781,7 +2869,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, if (!num) return -EINVAL; - if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) + if (virtqueue_get_vring_size(_vq) == num) return 0; err = virtqueue_disable_and_recycle(_vq, recycle); @@ -2790,10 +2878,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num, if (recycle_done) recycle_done(_vq); - if (vq->packed_ring) - err = virtqueue_resize_packed(vq, num); - else - err = virtqueue_resize_split(vq, num); + err = VIRTQUEUE_CALL(vq, resize, num); err_reset = virtqueue_enable_after_reset(_vq); if (err_reset) @@ -2831,10 +2916,7 @@ int virtqueue_reset(struct virtqueue *_vq, if (recycle_done) recycle_done(_vq); - if (vq->packed_ring) - virtqueue_reset_packed(vq); - else - virtqueue_reset_split(vq); + VOID_VIRTQUEUE_CALL(vq, reset); return virtqueue_enable_after_reset(_vq); } @@ -2877,7 +2959,7 @@ static void vring_free(struct virtqueue *_vq) struct vring_virtqueue *vq = to_vvq(_vq); if (vq->we_own_ring) { - if (vq->packed_ring) { + if (virtqueue_is_packed(vq)) { vring_free_queue(vq->vq.vdev, vq->packed.ring_size_in_bytes, vq->packed.vring.desc, @@ -2906,7 +2988,7 @@ static void vring_free(struct virtqueue *_vq) vq->map); } } - if (!vq->packed_ring) { + if (!virtqueue_is_packed(vq)) { kfree(vq->split.desc_state); kfree(vq->split.desc_extra); } @@ -2931,7 +3013,7 @@ u32 vring_notification_data(struct virtqueue *_vq) struct vring_virtqueue *vq = to_vvq(_vq); u16 next; - if (vq->packed_ring) + if (virtqueue_is_packed(vq)) next = (vq->packed.next_avail_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) | vq->packed.avail_wrap_counter << @@ -2984,7 +3066,8 @@ unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq) const struct vring_virtqueue *vq = to_vvq(_vq); - return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; + return virtqueue_is_packed(vq) ? vq->packed.vring.num : + vq->split.vring.num; } EXPORT_SYMBOL_GPL(virtqueue_get_vring_size); @@ -3067,7 +3150,7 @@ dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq) BUG_ON(!vq->we_own_ring); - if (vq->packed_ring) + if (virtqueue_is_packed(vq)) return vq->packed.ring_dma_addr; return vq->split.queue_dma_addr; @@ -3080,7 +3163,7 @@ dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq) BUG_ON(!vq->we_own_ring); - if (vq->packed_ring) + if (virtqueue_is_packed(vq)) return vq->packed.driver_event_dma_addr; return vq->split.queue_dma_addr + @@ -3094,7 +3177,7 @@ dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq) BUG_ON(!vq->we_own_ring); - if (vq->packed_ring) + if (virtqueue_is_packed(vq)) return vq->packed.device_event_dma_addr; return vq->split.queue_dma_addr +