/* SPDX-License-Identifier: MIT */

/*

 * Description: run various CQ ring overflow tests

 *

 */

#include <errno.h>

#include <stdio.h>

#include <unistd.h>

#include <stdlib.h>

#include <string.h>

#include <fcntl.h>

#include "liburing.h"

#define FILE_SIZE	(256 * 1024)

#define BS		4096

#define BUFFERS		(FILE_SIZE / BS)

static struct iovec *vecs;

static int create_buffers(void)

{

	int i;

	vecs = malloc(BUFFERS * sizeof(struct iovec));

	for (i = 0; i < BUFFERS; i++) {

		if (posix_memalign(&vecs[i].iov_base, BS, BS))

			return 1;

		vecs[i].iov_len = BS;

	}

	return 0;

}

static int create_file(const char *file)

{

	ssize_t ret;

	char *buf;

	int fd;

	buf = malloc(FILE_SIZE);

	memset(buf, 0xaa, FILE_SIZE);

	fd = open(file, O_WRONLY | O_CREAT, 0644);

	if (fd < 0) {

		perror("open file");

		return 1;

	}

	ret = write(fd, buf, FILE_SIZE);

	close(fd);

	return ret != FILE_SIZE;

}

#define ENTRIES	8

static int test_io(const char *file, unsigned long usecs, unsigned *drops, int fault)

{

	struct io_uring_sqe *sqe;

	struct io_uring_cqe *cqe;

	struct io_uring_params p;

	unsigned reaped, total;

	struct io_uring ring;

	int nodrop, i, fd, ret;

	fd = open(file, O_RDONLY | O_DIRECT);

	if (fd < 0) {

		perror("file open");

		goto err;

	}

	memset(&p, 0, sizeof(p));

	ret = io_uring_queue_init_params(ENTRIES, &ring, &p);

	if (ret) {

		fprintf(stderr, "ring create failed: %d\n", ret);

		goto err;

	}

	nodrop = 0;

	if (p.features & IORING_FEAT_NODROP)

		nodrop = 1;

	total = 0;

	for (i = 0; i < BUFFERS / 2; i++) {

		off_t offset;

		sqe = io_uring_get_sqe(&ring);

		if (!sqe) {

			fprintf(stderr, "sqe get failed\n");

			goto err;

		}

		offset = BS * (rand() % BUFFERS);

		if (fault && i == ENTRIES + 4)

			vecs[i].iov_base = NULL;

		io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset);

		ret = io_uring_submit(&ring);

		if (nodrop && ret == -EBUSY) {

			*drops = 1;

			total = i;

			break;

		} else if (ret != 1) {

			fprintf(stderr, "submit got %d, wanted %d\n", ret, 1);

			total = i;

			break;

		}

		total++;

	}

	if (*drops)

		goto reap_it;

	usleep(usecs);

	for (i = total; i < BUFFERS; i++) {

		off_t offset;

		sqe = io_uring_get_sqe(&ring);

		if (!sqe) {

			fprintf(stderr, "sqe get failed\n");

			goto err;

		}

		offset = BS * (rand() % BUFFERS);

		io_uring_prep_readv(sqe, fd, &vecs[i], 1, offset);

		ret = io_uring_submit(&ring);

		if (nodrop && ret == -EBUSY) {

			*drops = 1;

			break;

		} else if (ret != 1) {

			fprintf(stderr, "submit got %d, wanted %d\n", ret, 1);

			break;

		}

		total++;

	}

reap_it:

	reaped = 0;

	do {

		if (nodrop) {

			/* nodrop should never lose events */

			if (reaped == total)

				break;

		} else {

			if (reaped + *ring.cq.koverflow == total)

				break;

		}

		ret = io_uring_wait_cqe(&ring, &cqe);

		if (ret) {

			fprintf(stderr, "wait_cqe=%d\n", ret);

			goto err;

		}

		if (cqe->res != BS) {

			if (!(fault && cqe->res == -EFAULT)) {

				fprintf(stderr, "cqe res %d, wanted %d\n",

						cqe->res, BS);

				goto err;

			}

		}

		io_uring_cqe_seen(&ring, cqe);

		reaped++;

	} while (1);

	if (!io_uring_peek_cqe(&ring, &cqe)) {

		fprintf(stderr, "found unexpected completion\n");

		goto err;

	}

	if (!nodrop) {

		*drops = *ring.cq.koverflow;

	} else if (*ring.cq.koverflow) {

		fprintf(stderr, "Found %u overflows\n", *ring.cq.koverflow);

		goto err;

	}

	io_uring_queue_exit(&ring);

	close(fd);

	return 0;

err:

	if (fd != -1)

		close(fd);

	io_uring_queue_exit(&ring);

	return 1;

}

static int reap_events(struct io_uring *ring, unsigned nr_events, int do_wait)

{

	struct io_uring_cqe *cqe;

	int i, ret = 0, seq = 0;

	for (i = 0; i < nr_events; i++) {

		if (do_wait)

			ret = io_uring_wait_cqe(ring, &cqe);

		else

			ret = io_uring_peek_cqe(ring, &cqe);

		if (ret) {

			if (ret != -EAGAIN)

				fprintf(stderr, "cqe peek failed: %d\n", ret);

			break;

		}

		if (cqe->user_data != seq) {

			fprintf(stderr, "cqe sequence out-of-order\n");

			fprintf(stderr, "got %d, wanted %d\n", (int) cqe->user_data,

					seq);

			return -EINVAL;

		}

		seq++;

		io_uring_cqe_seen(ring, cqe);

	}

	return i ? i : ret;

}

/*

 * Setup ring with CQ_NODROP and check we get -EBUSY on trying to submit new IO

 * on an overflown ring, and that we get all the events (even overflows) when

 * we finally reap them.

 */

static int test_overflow_nodrop(void)

{

	struct __kernel_timespec ts;

	struct io_uring_sqe *sqe;

	struct io_uring_params p;

	struct io_uring ring;

	unsigned pending;

	int ret, i, j;

	memset(&p, 0, sizeof(p));

	ret = io_uring_queue_init_params(4, &ring, &p);

	if (ret) {

		fprintf(stderr, "io_uring_queue_init failed %d\n", ret);

		return 1;

	}

	if (!(p.features & IORING_FEAT_NODROP)) {

		fprintf(stdout, "FEAT_NODROP not supported, skipped\n");

		return 0;

	}

	ts.tv_sec = 0;

	ts.tv_nsec = 10000000;

	/* submit 4x4 SQEs, should overflow the ring by 8 */

	pending = 0;

	for (i = 0; i < 4; i++) {

		for (j = 0; j < 4; j++) {

			sqe = io_uring_get_sqe(&ring);

			if (!sqe) {

				fprintf(stderr, "get sqe failed\n");

				goto err;

			}

			io_uring_prep_timeout(sqe, &ts, -1U, 0);

			sqe->user_data = (i * 4) + j;

		}

		ret = io_uring_submit(&ring);

		if (ret <= 0) {

			if (ret == -EBUSY)

				break;

			fprintf(stderr, "sqe submit failed: %d, %d\n", ret, pending);

			goto err;

		}

		pending += ret;

	}

	/* wait for timers to fire */

	usleep(2 * 10000);

	/*

	 * We should have 16 pending CQEs now, 8 of them in the overflow list. Any

	 * attempt to queue more IO should return -EBUSY

	 */

	sqe = io_uring_get_sqe(&ring);

	if (!sqe) {

		fprintf(stderr, "get sqe failed\n");

		goto err;

	}

	io_uring_prep_nop(sqe);

	ret = io_uring_submit(&ring);

	if (ret != -EBUSY) {

		fprintf(stderr, "expected sqe submit busy: %d\n", ret);

		goto err;

	}

	/* reap the events we should have available */

	ret = reap_events(&ring, pending, 1);

	if (ret < 0) {

		fprintf(stderr, "ret=%d\n", ret);

		goto err;

	}

	if (*ring.cq.koverflow) {

		fprintf(stderr, "cq ring overflow %d, expected 0\n",

				*ring.cq.koverflow);

		goto err;

	}

	io_uring_queue_exit(&ring);

	return 0;

err:

	io_uring_queue_exit(&ring);

	return 1;

}

/*

 * Submit some NOPs and watch if the overflow is correct

 */

static int test_overflow(void)

{

	struct io_uring ring;

	struct io_uring_params p;

	struct io_uring_sqe *sqe;

	unsigned pending;

	int ret, i, j;

	memset(&p, 0, sizeof(p));

	ret = io_uring_queue_init_params(4, &ring, &p);

	if (ret) {

		fprintf(stderr, "io_uring_queue_init failed %d\n", ret);

		return 1;

	}

	/* submit 4x4 SQEs, should overflow the ring by 8 */

	pending = 0;

	for (i = 0; i < 4; i++) {

		for (j = 0; j < 4; j++) {

			sqe = io_uring_get_sqe(&ring);

			if (!sqe) {

				fprintf(stderr, "get sqe failed\n");

				goto err;

			}

			io_uring_prep_nop(sqe);

			sqe->user_data = (i * 4) + j;

		}

		ret = io_uring_submit(&ring);

		if (ret == 4) {

			pending += 4;

			continue;

		}

		if (p.features & IORING_FEAT_NODROP) {

			if (ret == -EBUSY)

				break;

		}

		fprintf(stderr, "sqe submit failed: %d\n", ret);

		goto err;

	}

	/* we should now have 8 completions ready */

	ret = reap_events(&ring, pending, 0);

	if (ret < 0)

		goto err;

	if (!(p.features & IORING_FEAT_NODROP)) {

		if (*ring.cq.koverflow != 8) {

			fprintf(stderr, "cq ring overflow %d, expected 8\n",

					*ring.cq.koverflow);

			goto err;

		}

	}

	io_uring_queue_exit(&ring);

	return 0;

err:

	io_uring_queue_exit(&ring);

	return 1;

}

/*

 * Test attempted submit with overflown cq ring that can't get flushed

 */

static int test_overflow_nodrop_submit_ebusy(void)

{

	struct __kernel_timespec ts;

	struct io_uring_sqe *sqe;

	struct io_uring_params p;

	struct io_uring ring;

	unsigned pending;

	int ret, i, j;

	memset(&p, 0, sizeof(p));

	ret = io_uring_queue_init_params(4, &ring, &p);

	if (ret) {

		fprintf(stderr, "io_uring_queue_init failed %d\n", ret);

		return 1;

	}

	if (!(p.features & IORING_FEAT_NODROP)) {

		fprintf(stdout, "FEAT_NODROP not supported, skipped\n");

		return 0;

	}

	ts.tv_sec = 1;

	ts.tv_nsec = 0;

	/* submit 4x4 SQEs, should overflow the ring by 8 */

	pending = 0;

	for (i = 0; i < 4; i++) {

		for (j = 0; j < 4; j++) {

			sqe = io_uring_get_sqe(&ring);

			if (!sqe) {

				fprintf(stderr, "get sqe failed\n");

				goto err;

			}

			io_uring_prep_timeout(sqe, &ts, -1U, 0);

			sqe->user_data = (i * 4) + j;

		}

		ret = io_uring_submit(&ring);

		if (ret <= 0) {

			fprintf(stderr, "sqe submit failed: %d, %d\n", ret, pending);

			goto err;

		}

		pending += ret;

	}

	/* wait for timers to fire */

	usleep(1100000);

	/*

	 * We should have 16 pending CQEs now, 8 of them in the overflow list. Any

	 * attempt to queue more IO should return -EBUSY

	 */

	sqe = io_uring_get_sqe(&ring);

	if (!sqe) {

		fprintf(stderr, "get sqe failed\n");

		goto err;

	}

	io_uring_prep_nop(sqe);

	ret = io_uring_submit(&ring);

	if (ret != -EBUSY) {

		fprintf(stderr, "expected sqe submit busy: %d\n", ret);

		goto err;

	}

	/*

	 * Now peek existing events so the CQ ring is empty, apart from the

	 * backlog

	 */

	ret = reap_events(&ring, pending, 0);

	if (ret < 0) {

		fprintf(stderr, "ret=%d\n", ret);

		goto err;

	} else if (ret < 8) {

		fprintf(stderr, "only found %d events, expected 8\n", ret);

		goto err;

	}

	/*

	 * We should now be able to submit our previous nop that's still

	 * in the sq ring, as the kernel can flush the existing backlog

	 * to the now empty CQ ring.

	 */

	ret = io_uring_submit(&ring);

	if (ret != 1) {

		fprintf(stderr, "submit got %d, expected 1\n", ret);

		goto err;

	}

	io_uring_queue_exit(&ring);

	return 0;

err:

	io_uring_queue_exit(&ring);

	return 1;

}

int main(int argc, char *argv[])

{

	unsigned iters, drops;

	unsigned long usecs;

	int ret;

	if (argc > 1)

		return 0;

	ret = test_overflow();

	if (ret) {

		printf("test_overflow failed\n");

		return ret;

	}

	ret = test_overflow_nodrop();

	if (ret) {

		printf("test_overflow_nodrop failed\n");

		return ret;

	}

	ret = test_overflow_nodrop_submit_ebusy();

	if (ret) {

		fprintf(stderr, "test_overflow_npdrop_submit_ebusy failed\n");

		return ret;

	}

	if (create_file(".basic-rw")) {

		fprintf(stderr, "file creation failed\n");

		goto err;

	}

	if (create_buffers()) {

		fprintf(stderr, "file creation failed\n");

		goto err;

	}

	iters = 0;

	usecs = 1000;

	do {

		drops = 0;

		if (test_io(".basic-rw", usecs, &drops, 0)) {

			fprintf(stderr, "test_io nofault failed\n");

			goto err;

		}

		if (drops)

			break;

		usecs = (usecs * 12) / 10;

		iters++;

	} while (iters < 40);

	if (test_io(".basic-rw", usecs, &drops, 0)) {

		fprintf(stderr, "test_io nofault failed\n");

		goto err;

	}

	if (test_io(".basic-rw", usecs, &drops, 1)) {

		fprintf(stderr, "test_io fault failed\n");

		goto err;

	}

	unlink(".basic-rw");

	return 0;

err:

	unlink(".basic-rw");

	return 1;

}