io/uring: basic Linux io_uring support

meson.build

@@ -334,6 +334,7 @@ endif
 subdir('src/util')
 subdir('src/time')
 subdir('src/io')
+subdir('src/io/uring')
 subdir('src/system')
 subdir('src/thread')
 subdir('src/net')

meson_options.txt

@@ -4,6 +4,7 @@ option('test', type: 'boolean', value: false, description: 'Build the unit tests
 
 option('syslog', type: 'feature', description: 'syslog support')
 option('inotify', type: 'boolean', value: true, description: 'inotify support (for automatic database update)')
+option('io_uring', type: 'feature', description: 'Linux io_uring support using liburing')
 
 option('daemon', type: 'boolean', value: true, description: 'enable daemonization')
 option('systemd', type: 'feature', description: 'systemd support')

src/io/uring/CancellableOperation.hxx

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2020 CM4all GmbH
+ * All rights reserved.
+ *
+ * author: Max Kellermann <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "Operation.hxx"
+
+#include <boost/intrusive/list_hook.hpp>
+
+#include <cassert>
+#include <utility>
+
+namespace Uring {
+
+class CancellableOperation
+	: public boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>>
+{
+	Operation *operation;
+
+public:
+	CancellableOperation(Operation &_operation) noexcept
+		:operation(&_operation)
+	{
+		assert(operation->cancellable == nullptr);
+		operation->cancellable = this;
+	}
+
+	~CancellableOperation() noexcept {
+		assert(operation == nullptr);
+	}
+
+	void Cancel(Operation &_operation) noexcept {
+		(void)_operation;
+		assert(operation == &_operation);
+
+		operation = nullptr;
+
+		// TODO: io_uring_prep_cancel()
+	}
+
+	void OnUringCompletion(int res) noexcept {
+		if (operation == nullptr)
+			return;
+
+		assert(operation->cancellable == this);
+		operation->cancellable = nullptr;
+
+		std::exchange(operation, nullptr)->OnUringCompletion(res);
+	}
+};
+
+} // namespace Uring

src/io/uring/Handler.hxx

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2020 CM4all GmbH
+ * All rights reserved.
+ *
+ * author: Max Kellermann <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <exception>
+
+struct statx;
+class UniqueFileDescriptor;
+
+namespace Uring {
+
+class OpenStatHandler {
+public:
+	virtual void OnOpenStat(UniqueFileDescriptor fd,
+				struct statx &st) noexcept = 0;
+	virtual void OnOpenStatError(std::exception_ptr e) noexcept = 0;
+};
+
+} // namespace Uring

src/io/uring/Operation.cxx

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2020 CM4all GmbH
+ * All rights reserved.
+ *
+ * author: Max Kellermann <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "Operation.hxx"
+#include "CancellableOperation.hxx"
+
+namespace Uring {
+
+void
+Operation::CancelUring() noexcept
+{
+	if (cancellable == nullptr)
+		return;
+
+	std::exchange(cancellable, nullptr)->Cancel(*this);
+}
+
+} // namespace Uring

src/io/uring/Operation.hxx

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 CM4all GmbH
+ * All rights reserved.
+ *
+ * author: Max Kellermann <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+namespace Uring {
+
+class CancellableOperation;
+
+class Operation {
+	friend class CancellableOperation;
+
+	CancellableOperation *cancellable = nullptr;
+
+public:
+	~Operation() noexcept {
+		CancelUring();
+	}
+
+	bool IsUringPending() const noexcept {
+		return cancellable != nullptr;
+	}
+
+	void CancelUring() noexcept;
+
+	virtual void OnUringCompletion(int res) noexcept = 0;
+};
+
+} // namespace Uring

src/io/uring/Queue.cxx

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2020 CM4all GmbH
+ * All rights reserved.
+ *
+ * author: Max Kellermann <[email protected]>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "Queue.hxx"
+#include "CancellableOperation.hxx"
+#include "Operation.hxx"
+#include "util/DeleteDisposer.hxx"
+
+#include <cassert>
+
+namespace Uring {
+
+Queue::Queue(unsigned entries, unsigned flags)
+	:ring(entries, flags)
+{
+}
+
+Queue::~Queue() noexcept
+{
+	operations.clear_and_dispose(DeleteDisposer{});
+}
+
+void
+Queue::AddPending(struct io_uring_sqe &sqe,
+		  Operation &operation) noexcept
+{
+	auto *c = new CancellableOperation(operation);
+	operations.push_back(*c);
+	io_uring_sqe_set_data(&sqe, c);
+}
+
+void
+Queue::DispatchOneCompletion(struct io_uring_cqe &cqe) noexcept
+{
+	void *data = io_uring_cqe_get_data(&cqe);
+	if (data != nullptr) {
+		auto *c = (CancellableOperation *)data;
+		c->OnUringCompletion(cqe.res);
+		operations.erase_and_dispose(operations.iterator_to(*c),
+					     DeleteDisposer{});
+	}
+
+	ring.SeenCompletion(cqe);
+}
+
+bool
+Queue::DispatchOneCompletion()
+{
+	auto *cqe = ring.PeekCompletion();
+	if (cqe == nullptr)
+		return false;
+
+	DispatchOneCompletion(*cqe);
+	return true;
+}
+
+bool
+Queue::WaitDispatchOneCompletion()
+{
+	auto *cqe = ring.WaitCompletion();
+	if (cqe == nullptr)
+		return false;
+
+	DispatchOneCompletion(*cqe);
+	return true;
+}
+
+} // namespace Uring