|
Packit |
d3489f |
/* SPDX-License-Identifier: MIT */
|
|
Packit |
d3489f |
#ifndef LIB_URING_H
|
|
Packit |
d3489f |
#define LIB_URING_H
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
#include <sys/socket.h>
|
|
Packit |
d3489f |
#include <sys/uio.h>
|
|
Packit |
d3489f |
#include <sys/stat.h>
|
|
Packit |
d3489f |
#include <errno.h>
|
|
Packit |
d3489f |
#include <signal.h>
|
|
Packit |
d3489f |
#include <stdbool.h>
|
|
Packit |
d3489f |
#include <inttypes.h>
|
|
Packit |
d3489f |
#include <time.h>
|
|
Packit |
d3489f |
#include <linux/swab.h>
|
|
Packit |
d3489f |
#include "liburing/compat.h"
|
|
Packit |
d3489f |
#include "liburing/io_uring.h"
|
|
Packit |
d3489f |
#include "liburing/barrier.h"
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
#ifdef __cplusplus
|
|
Packit |
d3489f |
extern "C" {
|
|
Packit |
d3489f |
#endif
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Library interface to io_uring
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
struct io_uring_sq {
|
|
Packit |
d3489f |
unsigned *khead;
|
|
Packit |
d3489f |
unsigned *ktail;
|
|
Packit |
d3489f |
unsigned *kring_mask;
|
|
Packit |
d3489f |
unsigned *kring_entries;
|
|
Packit |
d3489f |
unsigned *kflags;
|
|
Packit |
d3489f |
unsigned *kdropped;
|
|
Packit |
d3489f |
unsigned *array;
|
|
Packit |
d3489f |
struct io_uring_sqe *sqes;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
unsigned sqe_head;
|
|
Packit |
d3489f |
unsigned sqe_tail;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
size_t ring_sz;
|
|
Packit |
d3489f |
void *ring_ptr;
|
|
Packit |
d3489f |
};
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
struct io_uring_cq {
|
|
Packit |
d3489f |
unsigned *khead;
|
|
Packit |
d3489f |
unsigned *ktail;
|
|
Packit |
d3489f |
unsigned *kring_mask;
|
|
Packit |
d3489f |
unsigned *kring_entries;
|
|
Packit |
d3489f |
unsigned *kflags;
|
|
Packit |
d3489f |
unsigned *koverflow;
|
|
Packit |
d3489f |
struct io_uring_cqe *cqes;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
size_t ring_sz;
|
|
Packit |
d3489f |
void *ring_ptr;
|
|
Packit |
d3489f |
};
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
struct io_uring {
|
|
Packit |
d3489f |
struct io_uring_sq sq;
|
|
Packit |
d3489f |
struct io_uring_cq cq;
|
|
Packit |
d3489f |
unsigned flags;
|
|
Packit |
d3489f |
int ring_fd;
|
|
Packit |
d3489f |
};
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Library interface
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* return an allocated io_uring_probe structure, or NULL if probe fails (for
|
|
Packit |
d3489f |
* example, if it is not available). The caller is responsible for freeing it
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
extern struct io_uring_probe *io_uring_get_probe_ring(struct io_uring *ring);
|
|
Packit |
d3489f |
/* same as io_uring_get_probe_ring, but takes care of ring init and teardown */
|
|
Packit |
d3489f |
extern struct io_uring_probe *io_uring_get_probe(void);
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline int io_uring_opcode_supported(struct io_uring_probe *p, int op)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
if (op > p->last_op)
|
|
Packit |
d3489f |
return 0;
|
|
Packit |
d3489f |
return (p->ops[op].flags & IO_URING_OP_SUPPORTED) != 0;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
extern int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_params *p);
|
|
Packit |
d3489f |
extern int io_uring_queue_init(unsigned entries, struct io_uring *ring,
|
|
Packit |
d3489f |
unsigned flags);
|
|
Packit |
d3489f |
extern int io_uring_queue_mmap(int fd, struct io_uring_params *p,
|
|
Packit |
d3489f |
struct io_uring *ring);
|
|
Packit |
d3489f |
extern int io_uring_ring_dontfork(struct io_uring *ring);
|
|
Packit |
d3489f |
extern void io_uring_queue_exit(struct io_uring *ring);
|
|
Packit |
d3489f |
unsigned io_uring_peek_batch_cqe(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe **cqes, unsigned count);
|
|
Packit |
d3489f |
extern int io_uring_wait_cqes(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe **cqe_ptr, unsigned wait_nr,
|
|
Packit |
d3489f |
struct __kernel_timespec *ts, sigset_t *sigmask);
|
|
Packit |
d3489f |
extern int io_uring_wait_cqe_timeout(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe **cqe_ptr, struct __kernel_timespec *ts);
|
|
Packit |
d3489f |
extern int io_uring_submit(struct io_uring *ring);
|
|
Packit |
d3489f |
extern int io_uring_submit_and_wait(struct io_uring *ring, unsigned wait_nr);
|
|
Packit |
d3489f |
extern struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring);
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
extern int io_uring_register_buffers(struct io_uring *ring,
|
|
Packit |
d3489f |
const struct iovec *iovecs,
|
|
Packit |
d3489f |
unsigned nr_iovecs);
|
|
Packit |
d3489f |
extern int io_uring_unregister_buffers(struct io_uring *ring);
|
|
Packit |
d3489f |
extern int io_uring_register_files(struct io_uring *ring, const int *files,
|
|
Packit |
d3489f |
unsigned nr_files);
|
|
Packit |
d3489f |
extern int io_uring_unregister_files(struct io_uring *ring);
|
|
Packit |
d3489f |
extern int io_uring_register_files_update(struct io_uring *ring, unsigned off,
|
|
Packit |
d3489f |
int *files, unsigned nr_files);
|
|
Packit |
d3489f |
extern int io_uring_register_eventfd(struct io_uring *ring, int fd);
|
|
Packit |
d3489f |
extern int io_uring_register_eventfd_async(struct io_uring *ring, int fd);
|
|
Packit |
d3489f |
extern int io_uring_unregister_eventfd(struct io_uring *ring);
|
|
Packit |
d3489f |
extern int io_uring_register_probe(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_probe *p, unsigned nr);
|
|
Packit |
d3489f |
extern int io_uring_register_personality(struct io_uring *ring);
|
|
Packit |
d3489f |
extern int io_uring_unregister_personality(struct io_uring *ring, int id);
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Helper for the peek/wait single cqe functions. Exported because of that,
|
|
Packit |
d3489f |
* but probably shouldn't be used directly in an application.
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
extern int __io_uring_get_cqe(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe **cqe_ptr, unsigned submit,
|
|
Packit |
d3489f |
unsigned wait_nr, sigset_t *sigmask);
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
#define LIBURING_UDATA_TIMEOUT ((__u64) -1)
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
#define io_uring_for_each_cqe(ring, head, cqe) \
|
|
Packit |
d3489f |
/* \
|
|
Packit |
d3489f |
* io_uring_smp_load_acquire() enforces the order of tail \
|
|
Packit |
d3489f |
* and CQE reads. \
|
|
Packit |
d3489f |
*/ \
|
|
Packit |
d3489f |
for (head = *(ring)->cq.khead; \
|
|
Packit |
d3489f |
(cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \
|
|
Packit |
d3489f |
&(ring)->cq.cqes[head & (*(ring)->cq.kring_mask)] : NULL)); \
|
|
Packit |
d3489f |
head++) \
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Must be called after io_uring_for_each_cqe()
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
static inline void io_uring_cq_advance(struct io_uring *ring,
|
|
Packit |
d3489f |
unsigned nr)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
if (nr) {
|
|
Packit |
d3489f |
struct io_uring_cq *cq = &ring->cq;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Ensure that the kernel only sees the new value of the head
|
|
Packit |
d3489f |
* index after the CQEs have been read.
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
io_uring_smp_store_release(cq->khead, *cq->khead + nr);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Must be called after io_uring_{peek,wait}_cqe() after the cqe has
|
|
Packit |
d3489f |
* been processed by the application.
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
static inline void io_uring_cqe_seen(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe *cqe)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
if (cqe)
|
|
Packit |
d3489f |
io_uring_cq_advance(ring, 1);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Command prep helpers
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
static inline void io_uring_sqe_set_data(struct io_uring_sqe *sqe, void *data)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
sqe->user_data = (unsigned long) data;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void *io_uring_cqe_get_data(const struct io_uring_cqe *cqe)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
return (void *) (uintptr_t) cqe->user_data;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_sqe_set_flags(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
unsigned flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
sqe->flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_rw(int op, struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
const void *addr, unsigned len,
|
|
Packit |
d3489f |
__u64 offset)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
sqe->opcode = op;
|
|
Packit |
d3489f |
sqe->flags = 0;
|
|
Packit |
d3489f |
sqe->ioprio = 0;
|
|
Packit |
d3489f |
sqe->fd = fd;
|
|
Packit |
d3489f |
sqe->off = offset;
|
|
Packit |
d3489f |
sqe->addr = (unsigned long) addr;
|
|
Packit |
d3489f |
sqe->len = len;
|
|
Packit |
d3489f |
sqe->rw_flags = 0;
|
|
Packit |
d3489f |
sqe->user_data = 0;
|
|
Packit |
d3489f |
sqe->__pad2[0] = sqe->__pad2[1] = sqe->__pad2[2] = 0;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_splice(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
int fd_in, uint64_t off_in,
|
|
Packit |
d3489f |
int fd_out, uint64_t off_out,
|
|
Packit |
d3489f |
unsigned int nbytes,
|
|
Packit |
d3489f |
unsigned int splice_flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_SPLICE, sqe, fd_out, NULL, nbytes, off_out);
|
|
Packit |
d3489f |
sqe->splice_off_in = off_in;
|
|
Packit |
d3489f |
sqe->splice_fd_in = fd_in;
|
|
Packit |
d3489f |
sqe->splice_flags = splice_flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_readv(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
const struct iovec *iovecs,
|
|
Packit |
d3489f |
unsigned nr_vecs, off_t offset)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_READV, sqe, fd, iovecs, nr_vecs, offset);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
void *buf, unsigned nbytes,
|
|
Packit |
d3489f |
off_t offset, int buf_index)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_READ_FIXED, sqe, fd, buf, nbytes, offset);
|
|
Packit |
d3489f |
sqe->buf_index = buf_index;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
const struct iovec *iovecs,
|
|
Packit |
d3489f |
unsigned nr_vecs, off_t offset)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_WRITEV, sqe, fd, iovecs, nr_vecs, offset);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
const void *buf, unsigned nbytes,
|
|
Packit |
d3489f |
off_t offset, int buf_index)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_WRITE_FIXED, sqe, fd, buf, nbytes, offset);
|
|
Packit |
d3489f |
sqe->buf_index = buf_index;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
struct msghdr *msg, unsigned flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_RECVMSG, sqe, fd, msg, 1, 0);
|
|
Packit |
d3489f |
sqe->msg_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_sendmsg(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
const struct msghdr *msg, unsigned flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_SENDMSG, sqe, fd, msg, 1, 0);
|
|
Packit |
d3489f |
sqe->msg_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
unsigned poll_mask)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_POLL_ADD, sqe, fd, NULL, 0, 0);
|
|
Packit |
d3489f |
#if __BYTE_ORDER == __BIG_ENDIAN
|
|
Packit |
d3489f |
poll_mask = __swahw32(poll_mask);
|
|
Packit |
d3489f |
#endif
|
|
Packit |
d3489f |
sqe->poll32_events = poll_mask;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_poll_remove(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
void *user_data)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_POLL_REMOVE, sqe, -1, user_data, 0, 0);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_fsync(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
unsigned fsync_flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_FSYNC, sqe, fd, NULL, 0, 0);
|
|
Packit |
d3489f |
sqe->fsync_flags = fsync_flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_nop(struct io_uring_sqe *sqe)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_NOP, sqe, -1, NULL, 0, 0);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_timeout(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
struct __kernel_timespec *ts,
|
|
Packit |
d3489f |
unsigned count, unsigned flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_TIMEOUT, sqe, -1, ts, 1, count);
|
|
Packit |
d3489f |
sqe->timeout_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_timeout_remove(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
__u64 user_data, unsigned flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_TIMEOUT_REMOVE, sqe, -1,
|
|
Packit |
d3489f |
(void *)(unsigned long)user_data, 0, 0);
|
|
Packit |
d3489f |
sqe->timeout_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_accept(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
struct sockaddr *addr,
|
|
Packit |
d3489f |
socklen_t *addrlen, int flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_ACCEPT, sqe, fd, addr, 0,
|
|
Packit |
d3489f |
(__u64) (unsigned long) addrlen);
|
|
Packit |
d3489f |
sqe->accept_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_cancel(struct io_uring_sqe *sqe, void *user_data,
|
|
Packit |
d3489f |
int flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_ASYNC_CANCEL, sqe, -1, user_data, 0, 0);
|
|
Packit |
d3489f |
sqe->cancel_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_link_timeout(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
struct __kernel_timespec *ts,
|
|
Packit |
d3489f |
unsigned flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_LINK_TIMEOUT, sqe, -1, ts, 1, 0);
|
|
Packit |
d3489f |
sqe->timeout_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_connect(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
const struct sockaddr *addr,
|
|
Packit |
d3489f |
socklen_t addrlen)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_CONNECT, sqe, fd, addr, 0, addrlen);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_files_update(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
int *fds, unsigned nr_fds,
|
|
Packit |
d3489f |
int offset)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_FILES_UPDATE, sqe, -1, fds, nr_fds, offset);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_fallocate(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
int mode, off_t offset, off_t len)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_FALLOCATE, sqe, fd,
|
|
Packit |
d3489f |
(const uintptr_t *) (unsigned long) len, mode, offset);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_openat(struct io_uring_sqe *sqe, int dfd,
|
|
Packit |
d3489f |
const char *path, int flags, mode_t mode)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_OPENAT, sqe, dfd, path, mode, 0);
|
|
Packit |
d3489f |
sqe->open_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_close(struct io_uring_sqe *sqe, int fd)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_CLOSE, sqe, fd, NULL, 0, 0);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_read(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
void *buf, unsigned nbytes, off_t offset)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_READ, sqe, fd, buf, nbytes, offset);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_write(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
const void *buf, unsigned nbytes, off_t offset)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_WRITE, sqe, fd, buf, nbytes, offset);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
struct statx;
|
|
Packit |
d3489f |
static inline void io_uring_prep_statx(struct io_uring_sqe *sqe, int dfd,
|
|
Packit |
d3489f |
const char *path, int flags, unsigned mask,
|
|
Packit |
d3489f |
struct statx *statxbuf)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_STATX, sqe, dfd, path, mask,
|
|
Packit |
d3489f |
(__u64) (unsigned long) statxbuf);
|
|
Packit |
d3489f |
sqe->statx_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_fadvise(struct io_uring_sqe *sqe, int fd,
|
|
Packit |
d3489f |
off_t offset, off_t len, int advice)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_FADVISE, sqe, fd, NULL, len, offset);
|
|
Packit |
d3489f |
sqe->fadvise_advice = advice;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_madvise(struct io_uring_sqe *sqe, void *addr,
|
|
Packit |
d3489f |
off_t length, int advice)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_MADVISE, sqe, -1, addr, length, 0);
|
|
Packit |
d3489f |
sqe->fadvise_advice = advice;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,
|
|
Packit |
d3489f |
const void *buf, size_t len, int flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_SEND, sqe, sockfd, buf, len, 0);
|
|
Packit |
d3489f |
sqe->msg_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
|
|
Packit |
d3489f |
void *buf, size_t len, int flags)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_RECV, sqe, sockfd, buf, len, 0);
|
|
Packit |
d3489f |
sqe->msg_flags = flags;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
|
|
Packit |
d3489f |
const char *path, struct open_how *how)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_OPENAT2, sqe, dfd, path, sizeof(*how),
|
|
Packit |
d3489f |
(uint64_t) (uintptr_t) how);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
struct epoll_event;
|
|
Packit |
d3489f |
static inline void io_uring_prep_epoll_ctl(struct io_uring_sqe *sqe, int epfd,
|
|
Packit |
d3489f |
int fd, int op,
|
|
Packit |
d3489f |
struct epoll_event *ev)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_EPOLL_CTL, sqe, epfd, ev, op, fd);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_provide_buffers(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
void *addr, int len, int nr,
|
|
Packit |
d3489f |
int bgid, int bid)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_PROVIDE_BUFFERS, sqe, nr, addr, len, bid);
|
|
Packit |
d3489f |
sqe->buf_group = bgid;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline void io_uring_prep_remove_buffers(struct io_uring_sqe *sqe,
|
|
Packit |
d3489f |
int nr, int bgid)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
io_uring_prep_rw(IORING_OP_REMOVE_BUFFERS, sqe, nr, NULL, 0, 0);
|
|
Packit |
d3489f |
sqe->buf_group = bgid;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline unsigned io_uring_sq_ready(struct io_uring *ring)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
/* always use real head, to avoid losing sync for short submit */
|
|
Packit |
d3489f |
return ring->sq.sqe_tail - *ring->sq.khead;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline unsigned io_uring_sq_space_left(struct io_uring *ring)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
return *ring->sq.kring_entries - io_uring_sq_ready(ring);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline unsigned io_uring_cq_ready(struct io_uring *ring)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
return io_uring_smp_load_acquire(ring->cq.ktail) - *ring->cq.khead;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline bool io_uring_cq_eventfd_enabled(struct io_uring *ring)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
if (!ring->cq.kflags)
|
|
Packit |
d3489f |
return true;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
return !(*ring->cq.kflags & IORING_CQ_EVENTFD_DISABLED);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
static inline int io_uring_cq_eventfd_toggle(struct io_uring *ring,
|
|
Packit |
d3489f |
bool enabled)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
uint32_t flags;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
if (!!enabled == io_uring_cq_eventfd_enabled(ring))
|
|
Packit |
d3489f |
return 0;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
if (!ring->cq.kflags)
|
|
Packit |
d3489f |
return -EOPNOTSUPP;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
flags = *ring->cq.kflags;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
if (enabled)
|
|
Packit |
d3489f |
flags &= ~IORING_CQ_EVENTFD_DISABLED;
|
|
Packit |
d3489f |
else
|
|
Packit |
d3489f |
flags |= IORING_CQ_EVENTFD_DISABLED;
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
IO_URING_WRITE_ONCE(*ring->cq.kflags, flags);
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
return 0;
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Return an IO completion, waiting for 'wait_nr' completions if one isn't
|
|
Packit |
d3489f |
* readily available. Returns 0 with cqe_ptr filled in on success, -errno on
|
|
Packit |
d3489f |
* failure.
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
static inline int io_uring_wait_cqe_nr(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe **cqe_ptr,
|
|
Packit |
d3489f |
unsigned wait_nr)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
return __io_uring_get_cqe(ring, cqe_ptr, 0, wait_nr, NULL);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Return an IO completion, if one is readily available. Returns 0 with
|
|
Packit |
d3489f |
* cqe_ptr filled in on success, -errno on failure.
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
static inline int io_uring_peek_cqe(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe **cqe_ptr)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
return io_uring_wait_cqe_nr(ring, cqe_ptr, 0);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
/*
|
|
Packit |
d3489f |
* Return an IO completion, waiting for it if necessary. Returns 0 with
|
|
Packit |
d3489f |
* cqe_ptr filled in on success, -errno on failure.
|
|
Packit |
d3489f |
*/
|
|
Packit |
d3489f |
static inline int io_uring_wait_cqe(struct io_uring *ring,
|
|
Packit |
d3489f |
struct io_uring_cqe **cqe_ptr)
|
|
Packit |
d3489f |
{
|
|
Packit |
d3489f |
return io_uring_wait_cqe_nr(ring, cqe_ptr, 1);
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
#ifdef __cplusplus
|
|
Packit |
d3489f |
}
|
|
Packit |
d3489f |
#endif
|
|
Packit |
d3489f |
|
|
Packit |
d3489f |
#endif
|