diff --git a/coroio/backends/uring.cpp b/coroio/backends/uring.cpp index bb7a14c..55a647c 100644 --- a/coroio/backends/uring.cpp +++ b/coroio/backends/uring.cpp @@ -3,6 +3,7 @@ #include #include +#include #ifdef HAVE_URING @@ -12,6 +13,29 @@ std::unique_ptr TUring::CreateNotifier() { return std::make_unique>(*this); } +namespace { + +constexpr int BufferGroup = 0; + +// The user_data of a buffered read: odd, unlike a TOperation address. +uint64_t Tag(int fd, uint32_t gen) { + return (static_cast(gen) << 32) | (static_cast(fd) << 1) | 1; +} + +bool IsTag(uint64_t data) { + return data & 1; +} + +int TagFd(uint64_t data) { + return static_cast(static_cast(data) >> 1); +} + +uint32_t TagGen(uint64_t data) { + return static_cast(data >> 32); +} + +} // namespace + // The kernel sizes the hash that cancels by user_data search from the CQ: 16 buckets for the // default 512 entries, so such a cancel walked ~1/32 of the pending operations; 8192 gives 256, the most. TUring::TUring(int queueSize) @@ -56,7 +80,13 @@ TUring::TUring(int queueSize, int completionQueueSize) } TUring::~TUring() { + Buffered_.clear(); // the leases not taken go back to the ring io_uring_submit(&Ring_); // best effort: closes prepared by a Wait() that threw + if (BufRing_) { + // Before the exit, which cancels the multishot reads later: none may pick a buffer of the freed pool. + io_uring_free_buf_ring(&Ring_, BufRing_, Buffers_.Config().Count, BufferGroup); + Buffers_.SetOwner(nullptr); + } io_uring_queue_exit(&Ring_); close(RingFd_); close(EpollFd_); @@ -212,6 +242,10 @@ bool TUring::Reap(TOperation& op, int* result) { bool found = false; io_uring_for_each_cqe(&Ring_, head, cqe) { completed ++; + if (IsTag(cqe->user_data)) { + OnBuffered(cqe, true); + continue; + } auto* other = static_cast(io_uring_cqe_get_data(cqe)); if (other == &op) { Done(op); @@ -242,11 +276,13 @@ int TUring::Wait(timespec ts) { continue; // a close already in the ring } if (ev.Type & TEvent::CLOSE) { + ResetBuffered(ev.Fd); PrepClose(ev); } else { Cancel(ev.Fd); } } + PrepBuffered(); Reset(); if (Completions_.Carry()) { @@ -278,7 +314,9 @@ int TUring::Wait(timespec ts) { int completed = 0; io_uring_for_each_cqe(&Ring_, head, cqe) { completed ++; - if (auto* op = static_cast(io_uring_cqe_get_data(cqe))) { + if (IsTag(cqe->user_data)) { + OnBuffered(cqe, false); + } else if (auto* op = static_cast(io_uring_cqe_get_data(cqe))) { Done(*op); Completions_.Collect(op, cqe->res); } @@ -295,6 +333,234 @@ void TUring::WakeupReadyHandles() { Completions_.Wakeup(); } +bool TUring::TakeBuffered(int fd, TReadBuffer& lease) { + if (fd < 0) { + throw std::system_error(EBADF, std::generic_category(), "read"); + } + if (fd >= static_cast(Buffered_.size())) { + return false; + } + auto& state = Buffered_[fd]; + if (state.Pending()) { + lease = std::move(state.Ready[state.Head++]); + if (!state.Pending()) { + state.Ready.clear(); + state.Head = 0; + } + return true; + } + if (state.Error) { + throw std::system_error(std::exchange(state.Error, 0), std::generic_category(), "read"); + } + if (state.Eof) { + lease = {}; + return true; + } + return false; +} + +int TUring::ReadBuffered(int fd, void* buf, size_t size) { + if (fd < 0) { + throw std::system_error(EBADF, std::generic_category(), "read"); + } + if (fd >= static_cast(Buffered_.size())) { + return -1; + } + auto& state = Buffered_[fd]; + size_t copied = 0; + while (copied < size && state.Pending()) { + auto& lease = state.Ready[state.Head]; + size_t n = std::min(size - copied, lease.Data().size()); + memcpy(static_cast(buf) + copied, lease.Data().data(), n); + lease.Consume(n); + copied += n; + if (lease.Data().empty()) { + lease = {}; + state.Head++; + } + } + if (!state.Pending()) { + state.Ready.clear(); + state.Head = 0; + } + if (copied || !size) { + return static_cast(copied); + } + if (state.Error) { + throw std::system_error(std::exchange(state.Error, 0), std::generic_category(), "read"); + } + return state.Eof ? 0 : -1; +} + +void TUring::WaitBuffered(int fd, bool file, TOperation& op) { + try { + if (fd >= static_cast(Buffered_.size())) { + Buffered_.resize(fd + 1); + } + if (fd >= static_cast(Operations_.size())) { + Operations_.resize(fd + 1); + } + SetupBuffers(); + auto& state = Buffered_[fd]; + assert(!state.Waiter); // one reader per descriptor + state.File = file; + if (!state.Armed) { + Arm(fd, state); + } + state.Waiter = &op; + } catch (...) { + op.Handle = {}; + throw; + } +} + +void TUring::DropWaiter(int fd, TOperation& op) { + op.Handle = {}; + if (fd >= 0 && fd < static_cast(Buffered_.size()) && Buffered_[fd].Waiter == &op) { + Buffered_[fd].Waiter = nullptr; + } else { + Completions_.Drop(op); // woken already + } +} + +void TUring::SetupBuffers() { + if (BufRing_) { + return; + } + Buffers_.Allocate(); + const auto& config = Buffers_.Config(); + int err = 0; + BufRing_ = io_uring_setup_buf_ring(&Ring_, config.Count, BufferGroup, 0, &err); + if (!BufRing_) { + throw std::system_error(-err, std::generic_category(), "io_uring_setup_buf_ring"); + } + int mask = io_uring_buf_ring_mask(config.Count); + for (uint32_t id = 0; id < config.Count; id++) { + io_uring_buf_ring_add(BufRing_, Buffers_.Buffer(id), config.BufferSize, id, mask, id); + } + io_uring_buf_ring_advance(BufRing_, config.Count); + Buffers_.SetOwner(this); +} + +void TUring::RingAdd(uint16_t id) { + const auto& config = Buffers_.Config(); + io_uring_buf_ring_add(BufRing_, Buffers_.Buffer(id), config.BufferSize, id, io_uring_buf_ring_mask(config.Count), 0); + io_uring_buf_ring_advance(BufRing_, 1); +} + +void TUring::Release(uint16_t id) { + RingAdd(id); + if (!Starved_.empty()) { + Rearm_.push_back(Starved_.back()); + Starved_.pop_back(); + } +} + +void TUring::Arm(int fd, TBufferedFd& state) { + io_uring_sqe* sqe = GetSqe(); + if (state.Single) { + io_uring_prep_read(sqe, fd, nullptr, Buffers_.Config().BufferSize, -1); + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = BufferGroup; + } else if (state.File) { + io_uring_prep_read_multishot(sqe, fd, 0, -1, BufferGroup); + } else { + io_uring_prep_recv_multishot(sqe, fd, nullptr, 0, 0); + sqe->flags |= IOSQE_BUFFER_SELECT; + sqe->buf_group = BufferGroup; + } + io_uring_sqe_set_data64(sqe, Tag(fd, state.Gen)); + Operations_[fd]++; // so that a close cancels it + state.Armed = true; + state.Starved = false; + state.Throttled = false; +} + +void TUring::OnBuffered(const io_uring_cqe* cqe, bool reaped) { + int fd = TagFd(cqe->user_data); + bool more = cqe->flags & IORING_CQE_F_MORE; + if (!more) { + assert(Operations_[fd] > 0); + Operations_[fd]--; + } + auto& state = Buffered_[fd]; + bool own = TagGen(cqe->user_data) == state.Gen; + if (cqe->flags & IORING_CQE_F_BUFFER) { + auto id = static_cast(cqe->flags >> IORING_CQE_BUFFER_SHIFT); + if (own && cqe->res > 0) { + state.Ready.push_back(Buffers_.Commit(id, cqe->res)); + } else { + RingAdd(id); + } + } + if (!own) { + return; + } + if (!more) { + state.Armed = false; + } + if (cqe->res == 0) { + state.Eof = true; + } else if (cqe->res == -ENOBUFS) { + state.Starved = true; + } else if (cqe->res == -EBADFD && state.File && !state.Single) { + state.Single = true; // a regular file: read it one buffer at a time + } else if (cqe->res < 0 && cqe->res != -ECANCELED) { + state.Error = -cqe->res; + } + if (state.Waiter && (state.Pending() || state.Eof || state.Error)) { + Wake(state, 0, reaped); + } else if (state.Waiter && !state.Armed) { + (state.Starved ? Starved_ : Rearm_).emplace_back(fd, state.Gen); + } + if (state.Armed && !state.Throttled && state.Pending() >= Buffers_.Config().PerSocketLimit) { + // Leaves the rest in the socket, so one fast sender does not take the whole ring. + state.Throttled = true; + Throttle_.emplace_back(fd, state.Gen); + } +} + +void TUring::Wake(TBufferedFd& state, int result, bool reaped) { + auto* op = std::exchange(state.Waiter, nullptr); + if (reaped) { + Completions_.Reap(op, result); + } else { + Completions_.Collect(op, result); + } +} + +void TUring::PrepBuffered() { + for (auto [fd, gen] : Throttle_) { + if (Buffered_[fd].Gen == gen && Buffered_[fd].Armed) { + io_uring_sqe* sqe = GetSqe(); + io_uring_prep_cancel64(sqe, Tag(fd, gen), 0); + io_uring_sqe_set_data(sqe, nullptr); + } + } + Throttle_.clear(); + for (auto [fd, gen] : Rearm_) { + auto& state = Buffered_[fd]; + if (state.Gen == gen && !state.Armed && !state.Eof && !state.Error) { + Arm(fd, state); + } + } + Rearm_.clear(); +} + +void TUring::ResetBuffered(int fd) { + if (fd >= static_cast(Buffered_.size())) { + return; + } + auto& state = Buffered_[fd]; + if (state.Waiter) { + Wake(state, -ECANCELED, false); + } + // The read in the kernel is cancelled by the close; its completions carry the old generation. + TBufferedFd next; + next.Gen = state.Gen + 1; + state = std::move(next); +} + void TUring::Submit() { int err; if ((err = io_uring_submit(&Ring_)) < 0) { diff --git a/coroio/backends/uring.hpp b/coroio/backends/uring.hpp index f9292e6..08f1d55 100644 --- a/coroio/backends/uring.hpp +++ b/coroio/backends/uring.hpp @@ -60,7 +60,7 @@ namespace NNet { * @endcode * */ -class TUring: public TPollerBase { +class TUring: public TPollerBase, private NDetail::TBufferPool::IOwner { public: /// Alias for the poller-driven socket type. using TSocket = NNet::TPollerDrivenSocket; @@ -165,6 +165,24 @@ class TUring: public TPollerBase { * or a negative value. */ int Cancel(TOperation& op); + /** + * @brief Takes the next lease read ahead from @p fd, for ReadBuffer(). + * + * The first WaitBuffered() on @p fd starts reading ahead: a multishot recv (a socket) + * or read (a pipe) into the buffer ring, or one read at a time for a regular file. + * The read ahead stops at TReadBufferConfig::PerSocketLimit leases and when the ring + * is empty, and goes on when the reader waits again or a buffer comes back. + * + * @return False if nothing is read yet; true with an empty @p lease at the end of the stream. + * @throws std::system_error with the error of the read, once. + */ + bool TakeBuffered(int fd, TReadBuffer& lease); + /// Copies up to @p size bytes of the leases of @p fd, as TakeBuffered() takes them. Returns -1 if nothing is read yet, 0 at the end of the stream. + int ReadBuffered(int fd, void* buf, size_t size); + /// Resumes op.Handle once TakeBuffered() has a result for @p fd; op.Result is -ECANCELED if @p fd is closed first. One waiter per descriptor. + void WaitBuffered(int fd, bool file, TOperation& op); + /// Drops the wait of @p op, whose coroutine is destroyed. Nothing goes to the kernel: the read ahead goes on. + void DropWaiter(int fd, TOperation& op); /** * @brief Registers a file descriptor with the IO_uring poller. * @@ -240,12 +258,50 @@ class TUring: public TPollerBase { /// Reaps the posted completions, except the one of @p op, whose result goes to @p result. bool Reap(TOperation& op, int* result); + // Read ahead of one descriptor for TakeBuffered(). + struct TBufferedFd { + std::vector Ready; // not taken from Head on + size_t Head = 0; + TOperation* Waiter = nullptr; + int Error = 0; + uint32_t Gen = 0; // in user_data: completions of a closed descriptor's read give their buffers back + bool Armed = false; // a read is in the kernel + bool Eof = false; + bool File = false; + bool Single = false; // a regular file: multishot needs a pollable one + bool Throttled = false; // cancelled at PerSocketLimit + bool Starved = false; // ended by -ENOBUFS + + size_t Pending() const { + return Ready.size() - Head; + } + }; + using TFdGen = std::pair; + + /// Sets up the buffer ring at the first WaitBuffered(). + void SetupBuffers(); + /// Gives the buffer of a lease back to the kernel, as the TBufferPool::IOwner. + void Release(uint16_t id) override; + void RingAdd(uint16_t id); + void Arm(int fd, TBufferedFd& state); + void OnBuffered(const io_uring_cqe* cqe, bool reaped); + void Wake(TBufferedFd& state, int result, bool reaped); + /// Queues the cancels at PerSocketLimit and the reads that go on. + void PrepBuffered(); + /// Drops the read ahead of a descriptor that is closed: its waiter gets -ECANCELED. + void ResetBuffered(int fd); + int RingFd_; ///< File descriptor for the io_uring. int EpollFd_; ///< Epoll file descriptor (for integration with epoll). struct io_uring Ring_; ///< The io_uring structure. NDetail::TCompletions Completions_; ///< Completions not delivered yet. std::vector Operations_; ///< Per descriptor: operations posted whose completion is not taken yet. std::vector Buffer_; ///< Buffer used for internal I/O operations. + std::vector Buffered_; ///< Per descriptor: the read ahead of TakeBuffered(). + io_uring_buf_ring* BufRing_ = nullptr; ///< The buffers of Buffers_, for the kernel to pick. + std::vector Rearm_; ///< Reads to go on at the next Wait(). + std::vector Throttle_; ///< Reads to cancel at the next Wait(). + std::vector Starved_; ///< Reads that go on when a buffer comes back. }; } // namespace NNet diff --git a/coroio/read_buffer.cpp b/coroio/read_buffer.cpp index 51e3474..3569cef 100644 --- a/coroio/read_buffer.cpp +++ b/coroio/read_buffer.cpp @@ -21,7 +21,9 @@ void TBufferPool::Configure(const TReadBufferConfig& config) { void TBufferPool::Allocate() { if (!Arena_) { // One pool, so that a buffer has an id. Its pages are touched only when used. - Arena_.emplace(Config_.BufferSize, 64, Config_.Count, 1); + // One more cache line per buffer: TUring's ring hands them out in turn, and + // 16 KiB apart they would all fall into the same cache sets. + Arena_.emplace(Config_.BufferSize + 64, 64, Config_.Count, 1); Thread_ = std::this_thread::get_id(); } } diff --git a/coroio/socket.hpp b/coroio/socket.hpp index ff06156..0ae668f 100644 --- a/coroio/socket.hpp +++ b/coroio/socket.hpp @@ -52,6 +52,48 @@ TFuture ReadIntoBuffer(THandle& handle, TBufferPool& pool) { }); } +/// Whether poller T reads ahead into its buffers: TakeBuffered(), WaitBuffered() and friends. +template +constexpr bool BufferedReads = requires(T& poller, TReadBuffer& lease) { poller.TakeBuffered(0, lease); }; + +/// ReadBuffer() of a poller that reads ahead. +template +struct TBufferedRead { + ~TBufferedRead() { + if (op.Handle) { + // The coroutine is destroyed while waiting. + poller->DropWaiter(fd, op); + } + } + + bool await_ready() { + return (ready = poller->TakeBuffered(fd, lease)); + } + + void await_suspend(std::coroutine_handle<> h) { + op.Handle = h; + poller->WaitBuffered(fd, file, op); + } + + TReadBuffer await_resume() { + if (!ready) { + if (op.Result < 0) { + throw std::system_error(-op.Result, std::generic_category(), "read"); + } + ready = poller->TakeBuffered(fd, lease); + assert(ready); + } + return std::move(lease); + } + + T* poller; + int fd; + bool file; + bool ready = false; + TReadBuffer lease = {}; + typename T::TOperation op = {}; +}; + } // namespace NDetail /** @@ -851,17 +893,39 @@ class TPollerDrivenSocket: public TSocket struct TAwaitable { ~TAwaitable() { if (op.Handle) { + if constexpr (NDetail::BufferedReads) { + if (buffered) { + poller->DropWaiter(fd, op); + return; + } + } poller->Cancel(op); } } - bool await_ready() const { return false; } + bool await_ready() { + if constexpr (NDetail::BufferedReads) { + return buffered && (copied = poller->ReadBuffered(fd, buf, size)) >= 0; + } + return false; + } void await_suspend(std::coroutine_handle<> h) { op.Handle = h; + if constexpr (NDetail::BufferedReads) { + if (buffered) { + poller->WaitBuffered(fd, false, op); + return; + } + } poller->Recv(fd, buf, size, op); } auto await_resume() { + if constexpr (NDetail::BufferedReads) { + if (buffered) { + return BufferedResume(); + } + } auto ret = op.Result; if (ret < 0) { #ifdef _WIN32 @@ -880,20 +944,44 @@ class TPollerDrivenSocket: public TSocket return ret; } + // After ReadBuffer(): takes from its leases, which the poller reads ahead. + int BufferedResume() { + if (copied < 0) { + if (op.Result < 0) { + throw std::system_error(-op.Result, std::generic_category(), "read"); + } + copied = poller->ReadBuffered(fd, buf, size); + assert(copied >= 0); + } + return copied; + } + T* poller; int fd; void* buf; size_t size; + bool buffered; + int copied = -1; typename T::TOperation op = {}; }; - return TAwaitable{Poller_, Fd_, buf, size}; + return TAwaitable{Poller_, Fd_, buf, size, Buffered_}; } - /// Reads into a buffer of the poller, see TSocketBase::ReadBuffer(). - TFuture ReadBuffer() { - return NDetail::ReadIntoBuffer(*this, Poller_->ReadBuffers()); + /** + * @brief Reads into a buffer of the poller, see TSocketBase::ReadBuffer(). + * + * A poller that reads ahead (TUring) keeps reading into its buffers from the first + * call on: ReadSome() on the socket then takes from the same leases, with a copy. + */ + auto ReadBuffer() { + if constexpr (NDetail::BufferedReads) { + Buffered_ = true; + return NDetail::TBufferedRead{Poller_, Fd_, false}; + } else { + return NDetail::ReadIntoBuffer(*this, Poller_->ReadBuffers()); + } } /** @@ -962,6 +1050,7 @@ class TPollerDrivenSocket: public TSocket private: T* Poller_; + bool Buffered_ = false; ///< ReadBuffer() was called: ReadSome() takes from its leases. }; /** @@ -1018,17 +1107,39 @@ class TPollerDrivenFileHandle: public TFileHandle struct TAwaitable { ~TAwaitable() { if (op.Handle) { + if constexpr (NDetail::BufferedReads) { + if (buffered) { + poller->DropWaiter(fd, op); + return; + } + } poller->Cancel(op); } } - bool await_ready() const { return false; } + bool await_ready() { + if constexpr (NDetail::BufferedReads) { + return buffered && (copied = poller->ReadBuffered(fd, buf, size)) >= 0; + } + return false; + } void await_suspend(std::coroutine_handle<> h) { op.Handle = h; + if constexpr (NDetail::BufferedReads) { + if (buffered) { + poller->WaitBuffered(fd, true, op); + return; + } + } poller->Read(fd, buf, size, op); } auto await_resume() { + if constexpr (NDetail::BufferedReads) { + if (buffered) { + return BufferedResume(); + } + } auto ret = op.Result; if (ret < 0) { #ifdef _WIN32 @@ -1047,20 +1158,39 @@ class TPollerDrivenFileHandle: public TFileHandle return ret; } + // After ReadBuffer(): takes from its leases, which the poller reads ahead. + int BufferedResume() { + if (copied < 0) { + if (op.Result < 0) { + throw std::system_error(-op.Result, std::generic_category(), "read"); + } + copied = poller->ReadBuffered(fd, buf, size); + assert(copied >= 0); + } + return copied; + } + T* poller; int fd; void* buf; size_t size; + bool buffered; + int copied = -1; typename T::TOperation op = {}; }; - return TAwaitable{Poller_, Fd_, buf, size}; + return TAwaitable{Poller_, Fd_, buf, size, Buffered_}; } - /// Reads into a buffer of the poller, see TSocketBase::ReadBuffer(). - TFuture ReadBuffer() { - return NDetail::ReadIntoBuffer(*this, Poller_->ReadBuffers()); + /// Reads into a buffer of the poller, see TPollerDrivenSocket::ReadBuffer(). + auto ReadBuffer() { + if constexpr (NDetail::BufferedReads) { + Buffered_ = true; + return NDetail::TBufferedRead{Poller_, Fd_, true}; + } else { + return NDetail::ReadIntoBuffer(*this, Poller_->ReadBuffers()); + } } /** @@ -1129,6 +1259,7 @@ class TPollerDrivenFileHandle: public TFileHandle private: T* Poller_; + bool Buffered_ = false; ///< ReadBuffer() was called: ReadSome() takes from its leases. }; } // namespace NNet diff --git a/examples/bench.cpp b/examples/bench.cpp index f552e19..63acc30 100644 --- a/examples/bench.cpp +++ b/examples/bench.cpp @@ -19,9 +19,12 @@ using namespace std; namespace { void usage(const char* name) { - printf("%s [-n num_pipes] [-a num_active] [-w num_writes] [-m method]\n", name); + printf("%s [-n num_pipes] [-a num_active] [-w num_writes] [-m method] [-r some|buffer|adapter]\n", name); } +// How a reader reads: ReadSome(); ReadBuffer(); ReadSome() after one ReadBuffer(). +enum class ERead { Some, Buffer, Adapter }; + struct Stat { int writes = 0; int fired = 0; @@ -53,6 +56,36 @@ TFuture pipe_reader(TSocket& r, TSocket& w, Stat& s) { co_return; } +// Reads with ReadBuffer(), or with the adapter only the first time, then as pipe_reader. +template +TFuture pipe_buffer_reader(TSocket& r, TSocket& w, Stat& s, bool adapter) { + try { + for (bool more = true; more; more = !adapter) { + TReadBuffer lease = co_await r.ReadBuffer(); + if (lease.Data().empty()) { + break; + } + s.count += lease.Data().size(); + for (size_t i = 0; i < lease.Data().size() && s.writes; i++) { + s.writes--; + if (co_await w.WriteSome(lease.Data().data() + i, 1) != 1) { + s.failures ++; + } + s.fired++; + } + } + } catch (const std::exception& ex) { + s.failures ++; + } + if (adapter) { + // The lease is gone: a reader that kept one would hold a buffer of the ring forever. + co_await pipe_reader(r, w, s); + co_return; + } + s.out ++; + co_return; +} + template TFuture write_one(TSocket& w, Stat& s) { char buf[1] = {'e'}; @@ -67,7 +100,7 @@ TFuture yield(TPollerBase& poller) { } template -std::chrono::microseconds run_one(int num_pipes, int num_writes, int num_active) { +std::chrono::microseconds run_one(int num_pipes, int num_writes, int num_active, ERead read) { Stat s; using TFileHandle = typename TPoller::TFileHandle; using TSocket = typename TPoller::TSocket; @@ -102,7 +135,13 @@ std::chrono::microseconds run_one(int num_pipes, int num_writes, int num_active) s.writes = num_writes; for (int i = 0; i < num_pipes; i++) { - handles.emplace_back(pipe_reader(pipes[i*2], pipes[((i+1)%num_pipes)*2+1], s)); + auto& r = pipes[i*2]; + auto& w = pipes[((i+1)%num_pipes)*2+1]; + if (read == ERead::Some) { + handles.emplace_back(pipe_reader(r, w, s)); + } else { + handles.emplace_back(pipe_buffer_reader(r, w, s, read == ERead::Adapter)); + } } handles.emplace_back(yield(loop.Poller())); // initialize events (sleep in readers) @@ -135,12 +174,12 @@ std::chrono::microseconds run_one(int num_pipes, int num_writes, int num_active) } template -void run_test(int num_pipes, int num_writes, int num_active) { +void run_test(int num_pipes, int num_writes, int num_active, ERead read) { int runs = 25; vector results; results.reserve(runs); for (int i = 0; i < runs; i++) { - auto d = run_one(num_pipes, num_writes, num_active).count(); + auto d = run_one(num_pipes, num_writes, num_active, read).count(); results.emplace_back(d); } sort(results.begin(), results.end()); @@ -160,6 +199,7 @@ int main(int argc, char** argv) { int num_writes = num_pipes; int num_active = 1; const char* method = "poll"; + ERead read = ERead::Some; for (int i = 1; i < argc; ++i) { if (!strcmp(argv[i], "-n") && i < argc-1) { @@ -170,34 +210,45 @@ int main(int argc, char** argv) { num_writes = atoi(argv[++i]); } else if (!strcmp(argv[i], "-m") && i < argc-1) { method = argv[++i]; + } else if (!strcmp(argv[i], "-r") && i < argc-1) { + const char* name = argv[++i]; + if (!strcmp(name, "some")) { + read = ERead::Some; + } else if (!strcmp(name, "buffer")) { + read = ERead::Buffer; + } else if (!strcmp(name, "adapter")) { + read = ERead::Adapter; + } else { + usage(argv[0]); return 1; + } } else { usage(argv[0]); return 1; } } if (!strcmp(method, "select")) { - run_test(num_pipes, num_writes, num_active); + run_test(num_pipes, num_writes, num_active, read); } else if (!strcmp(method, "poll")) { - run_test(num_pipes, num_writes, num_active); + run_test(num_pipes, num_writes, num_active, read); } #ifdef HAVE_EPOLL else if (!strcmp(method, "epoll")) { - run_test(num_pipes, num_writes, num_active); + run_test(num_pipes, num_writes, num_active, read); } #endif #ifdef HAVE_URING else if (!strcmp(method, "uring")) { - run_test(num_pipes, num_writes, num_active); + run_test(num_pipes, num_writes, num_active, read); } #endif #ifdef HAVE_KQUEUE else if (!strcmp(method, "kqueue")) { - run_test(num_pipes, num_writes, num_active); + run_test(num_pipes, num_writes, num_active, read); } #endif #ifdef HAVE_IOCP else if (!strcmp(method, "iocp")) { - run_test(num_pipes, num_writes, num_active); + run_test(num_pipes, num_writes, num_active, read); } #endif diff --git a/tests/tests.cpp b/tests/tests.cpp index 1f50be7..6cae36c 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -1955,6 +1955,15 @@ constexpr int ConnReset = ECONNRESET; template constexpr bool kDrivenSocket = !std::is_same_v; +// The kernel picks the buffers from a ring: a read waits for one instead of failing. +template +constexpr bool kBufferRing = +#if defined(__linux__) && defined(HAVE_URING) + std::is_same_v; +#else + false; +#endif + template TFuture ReadLease(TFile& file, std::optional* lease) { lease->emplace(co_await file.ReadBuffer()); @@ -2113,8 +2122,8 @@ void test_read_buffer_mixed_order(void**) { assert_string_equal(data.c_str(), "01234567"); } -// A reader that holds every buffer gets no more: the readiness pollers throw. -// Bytes sent faster than they are read stay in order. +// A reader that holds every buffer gets no more: the readiness pollers throw, +// TUring waits until one comes back. Bytes sent faster than they are read stay in order. template void test_read_buffer_back_pressure(void**) { TLoop loop; @@ -2130,8 +2139,18 @@ void test_read_buffer_back_pressure(void**) { std::string data; int result = 0; auto starved = AppendLease(c.Server, &data, &result); - StepUntilDone(loop, starved); - assert_int_equal(result, static_cast(std::errc::no_buffer_space)); + if constexpr (kBufferRing) { + for (int i = 0; i < 3; i++) { + loop.Step(); + } + assert_false(starved.done()); + held[0].reset(); + StepUntilDone(loop, starved); + assert_int_equal(result, 0); + } else { + StepUntilDone(loop, starved); + assert_int_equal(result, static_cast(std::errc::no_buffer_space)); + } held[0].reset(); held[1].reset(); @@ -2141,7 +2160,7 @@ void test_read_buffer_back_pressure(void**) { } TByteWriter writer(c.Client); auto write = writer.Write(sent.data(), sent.size()); - std::string got; + std::string got = data; // "y" if read already while (got.size() < sent.size() + 1) { auto read = AppendLease(c.Server, &got, &result); StepUntilDone(loop, read); @@ -2251,6 +2270,109 @@ void test_read_buffer_files(void**) { } #endif +#if defined(__linux__) && defined(HAVE_URING) +// The bytes read ahead for a socket closed meanwhile go back to the ring, not +// to the socket that reuses its number: their completions carry the old generation. +void test_uring_read_buffer_stale_generation(void**) { + TLoop loop; + loop.Poller().ConfigureReadBuffers({.BufferSize = 64, .Count = 4}); + TAddress address{"127.0.0.1", GetFreePort()}; + TUring::TSocket listener(loop.Poller(), address.Domain()); + listener.Bind(address); + listener.Listen(); + int reused = 0; + int last = -1; + for (int i = 0; i < 8; i++) { + auto [client, server] = ConnectedPair(loop, listener, address); + reused += client.Fd() == last; + last = client.Fd(); + std::string data; + int result = 0; + assert_int_equal(send(server.Fd(), "new", 3, 0), 3); + auto first = AppendLease(client, &data, &result); + StepUntilDone(loop, first); + assert_string_equal(data.c_str(), "new"); + auto reader = AppendLease(client, &data, &result); + loop.Step(); + assert_int_equal(send(server.Fd(), "old", 3, 0), 3); + client.Close(); + StepUntilDone(loop, reader); + assert_int_equal(result, ECANCELED); + loop.Step(); // the close completes, so the next client gets the number + } + assert_true(reused > 0); +} + +// A close cancels the read ahead of its socket, which close() alone leaves holding +// the file, before the number is released: the peer sees the end of the stream, +// and a descriptor opened meanwhile keeps its own read. +void test_uring_read_buffer_close_cancels_read_ahead(void**) { + TLoop loop; + loop.Poller().SetMaxDuration(std::chrono::milliseconds(1)); + int fds[2]; + assert_int_equal(socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + auto handle = std::make_unique(fds[0], loop.Poller()); + std::string data; + int result = 0; + { + auto reader = AppendLease(*handle, &data, &result); // destroyed waiting: the read ahead stays + loop.Step(); + } + handle.reset(); + int next[2]; + assert_int_equal(pipe(next), 0); + TUring::TFileHandle nextHandle(next[0], loop.Poller()); + auto nextRead = AppendLease(nextHandle, &data, &result); + char c; + bool eof = false; + for (int i = 0; i < 100 && !eof; i++) { + loop.Step(); + eof = recv(fds[1], &c, 1, MSG_DONTWAIT) == 0; + } + assert_true(eof); + assert_int_equal(write(next[1], "n", 1), 1); + StepUntilDone(loop, nextRead); + assert_string_equal(data.c_str(), "n"); + close(fds[1]); + close(next[1]); +} + +// A sender faster than its reader stops being read at PerSocketLimit leases, +// so it leaves buffers for the other sockets, and its bytes wait in its socket. +void test_uring_read_buffer_socket_limit(void**) { + TLoop loop; + loop.Poller().SetMaxDuration(std::chrono::milliseconds(1)); + loop.Poller().ConfigureReadBuffers({.BufferSize = 16, .Count = 64, .PerSocketLimit = 2}); + TConnected fast(loop); + TConnected slow(loop); + std::optional held; + assert_int_equal(send(fast.Client.Fd(), "f", 1, 0), 1); + auto first = ReadLease(fast.Server, &held); + StepUntilDone(loop, first); + std::string sent; + for (int i = 0; i < 64; i++) { + std::string chunk(16, static_cast('a' + i % 26)); + assert_int_equal(send(fast.Client.Fd(), chunk.data(), chunk.size(), 0), 16); + sent += chunk; + loop.Step(); + } + assert_int_equal(send(slow.Client.Fd(), "s", 1, 0), 1); + std::string data; + int result = 0; + auto other = AppendLease(slow.Server, &data, &result); + StepUntilDone(loop, other); + assert_string_equal(data.c_str(), "s"); + held.reset(); + data.clear(); + while (data.size() < sent.size()) { + auto read = AppendLease(fast.Server, &data, &result); + StepUntilDone(loop, read); + assert_int_equal(result, 0); + } + assert_true(data == sent); +} +#endif + #ifdef _WIN32 // A failed or aborted read or write gets the error in its packet, not 0 bytes // that look like the end of the stream (#183). @@ -4659,6 +4781,11 @@ int main(int argc, char* argv[]) { ADD_TEST(my_unit_poller, test_read_buffer_destroyed_waiter); #ifndef _WIN32 ADD_TEST(my_unit_poller, test_read_buffer_files); +#endif +#if defined(__linux__) && defined(HAVE_URING) + ADD_TEST(cmocka_unit_test, test_uring_read_buffer_stale_generation); + ADD_TEST(cmocka_unit_test, test_uring_read_buffer_close_cancels_read_ahead); + ADD_TEST(cmocka_unit_test, test_uring_read_buffer_socket_limit); #endif ADD_TEST(cmocka_unit_test, test_completions);