ublk user copy syscalls may be issued from any task, so they take a reference count on the struct ublk_io to check whether it is owned by the ublk server and prevent a concurrent UBLK_IO_COMMIT_AND_FETCH_REQ from completing the request. However, if the user copy syscall is issued on the io's daemon task, a concurrent UBLK_IO_COMMIT_AND_FETCH_REQ isn't possible, so the atomic reference count dance is unnecessary. Check for UBLK_IO_FLAG_OWNED_BY_SRV to ensure the request is dispatched to the sever and obtain the request from ublk_io's req field instead of looking it up on the tagset. Skip the reference count increment and decrement. Commit 8a8fe42d765b ("ublk: optimize UBLK_IO_REGISTER_IO_BUF on daemon task") made an analogous optimization for ublk zero copy buffer registration.
Signed-off-by: Caleb Sander Mateos csander@purestorage.com --- drivers/block/ublk_drv.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 042df4de9253..a0fbabd49feb 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -180,11 +180,11 @@ struct ublk_io { /* * The number of uses of this I/O by the ublk server * if user copy or zero copy are enabled: * - UBLK_REFCOUNT_INIT from dispatch to the server * until UBLK_IO_COMMIT_AND_FETCH_REQ - * - 1 for each inflight ublk_ch_{read,write}_iter() call + * - 1 for each inflight ublk_ch_{read,write}_iter() call not on task * - 1 for each io_uring registered buffer not registered on task * The I/O can only be completed once all references are dropped. * User copy and buffer registration operations are only permitted * if the reference count is nonzero. */ @@ -2644,10 +2644,11 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir) struct ublk_queue *ubq; struct request *req; struct ublk_io *io; unsigned data_len; bool is_integrity; + bool on_daemon; size_t buf_off; u16 tag, q_id; ssize_t ret;
if (!user_backed_iter(iter)) @@ -2670,13 +2671,24 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir)
if (tag >= ub->dev_info.queue_depth) return -EINVAL;
io = &ubq->ios[tag]; - req = __ublk_check_and_get_req(ub, q_id, tag, io); - if (!req) - return -EINVAL; + on_daemon = current == READ_ONCE(io->task); + if (on_daemon) { + /* On daemon, io can't be completed concurrently, so skip ref */ + if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) + return -EINVAL; + + req = io->req; + if (!ublk_rq_has_data(req)) + return -EINVAL; + } else { + req = __ublk_check_and_get_req(ub, q_id, tag, io); + if (!req) + return -EINVAL; + }
if (is_integrity) { struct blk_integrity *bi = &req->q->limits.integrity;
data_len = bio_integrity_bytes(bi, blk_rq_sectors(req)); @@ -2697,11 +2709,12 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir) ret = ublk_copy_user_integrity(req, buf_off, iter, dir); else ret = ublk_copy_user_pages(req, buf_off, iter, dir);
out: - ublk_put_req_ref(io, req); + if (!on_daemon) + ublk_put_req_ref(io, req); return ret; }
static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to) {