Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -235,18 +235,20 @@ class TcpConnection internal constructor(
releaseParkedFlush()
}

// Recycle: hands back the buffers still queued - shared ones; a connection's own ring is reset whole.
// Hands back the buffers still queued, for a handler that won't read them, the way a handler returns them.
internal fun drainRecv(scratch: RecvItem) {
while (recv.tryDequeue(scratch)) {
if (scratch.hasBuffer && ring == null) {
tcp.returnBufferDirect(scratch.bufferId)
}
returnBuffer(scratch)
}
}

/** Recv completions queued and not taken yet. */
internal val queuedRecvs: Int get() = recv.countUntil(recv.snapshotTail())

/**
* The adapter holding buffers taken out of the queue, if any: recycle asks it to give them back, so a
* buffer stranded by a handler that returned early does not depend on the handler remembering.
* buffer stranded by a handler that returned early does not depend on the handler remembering. The proactor
* asks sooner when the handler has let go and a recv needs the buffers.
*/
internal var bufferHolder: RecvBufferHolder? = null

Expand Down Expand Up @@ -489,7 +491,8 @@ class TcpConnection internal constructor(
/**
* The handler's release, once per life - the proactor makes it when the handler returns, so calling it is
* only for letting go earlier. A peer still connected gets a FIN; the read side stays open until the peer
* closes, bounded by [TcpOptions.readTimeoutMs].
* closes, bounded by [TcpOptions.readTimeoutMs]. Nobody reads that side from here on: what is queued and what a
* reader holds can go back to the kernel at any point, so a handler that goes on must leave them alone.
*/
fun decRef() {
if (!tcp.proactor.onProactorThread) {
Expand Down Expand Up @@ -662,8 +665,9 @@ class TcpConnection internal constructor(
}

/**
* An adapter that takes buffers out of a connection's queue and holds them past one read (a pipe reader):
* recycle asks it to hand them back.
* An adapter that takes buffers out of a connection's queue and holds them past one read (a pipe reader): recycle
* asks it to hand them back, and so can the proactor before that, once the handler has let go. It can be asked more
* than once, and each buffer goes back only once.
*/
internal interface RecvBufferHolder {
fun releaseHeldBuffers()
Expand Down
10 changes: 5 additions & 5 deletions kronos/src/commonMain/kotlin/kronos/connection/tcp/TcpPipe.kt
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class TcpPipeReader(private val connection: TcpConnection) : PipeReader, RecvBuf
private val lostToOverflow: Boolean get() = connectionClosed && connection.recvOverflowed

init {
// So the proactor can reclaim at recycle what this reader dequeued and never gave back.
// So the proactor can reclaim what this reader dequeued and never gave back.
connection.bufferHolder = this
}

Expand Down Expand Up @@ -220,10 +220,10 @@ class TcpPipeReader(private val connection: TcpConnection) : PipeReader, RecvBuf
}

/**
* Hands every held buffer back and forgets the chain. Called by [complete], and by the proactor at recycle
* for a reader its handler never completed. Idempotent: once the chain is empty it does nothing, so
* complete-then-recycle returns each buffer once. Terminal, so a reader kept past its handler fails instead
* of reading the recycled connection's next tenant.
* Hands every held buffer back and forgets the chain. Called by [complete], and by the proactor for a reader
* its handler never completed: at recycle, or sooner once the handler has let go and a recv needs the buffers.
* Idempotent: once the chain is empty it does nothing, so complete-then-recycle returns each buffer once.
* Terminal, so a reader kept past its handler fails instead of reading the recycled connection's next tenant.
*/
override fun releaseHeldBuffers() {
completed = true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,7 @@ internal class TcpTransport(val proactor: Proactor, private val config: ServerCo
// The group ran dry, not a peer error: the multishot ended, so park the connection until buffers
// come back.
if (conn != null) {
conn.readParkedMs = proactor.nowMs // the peer sent; the wait is ours, so the clock restarts
recvStarved.add((gen.toLong() shl 32) or (fd.toLong() and 0xFFFF_FFFFL))
parkStarvedRecv(conn, fd, gen)
}

return
Expand Down Expand Up @@ -442,8 +441,7 @@ internal class TcpTransport(val proactor: Proactor, private val config: ServerCo
// buffers. Only this connection waits - its bytes stay in the kernel and TCP slows its peer - until a
// buffer comes back and the loop re-arms the recv.
if (conn != null) {
conn.readParkedMs = proactor.nowMs // the peer sent; the wait is ours, so the clock restarts
recvStarved.add((gen.toLong() shl 32) or (fd.toLong() and 0xFFFF_FFFFL))
parkStarvedRecv(conn, fd, gen)
}

return
Expand Down Expand Up @@ -473,6 +471,22 @@ internal class TcpTransport(val proactor: Proactor, private val config: ServerCo
}
}

private fun parkStarvedRecv(conn: TcpConnection, fd: Int, gen: Int) {
conn.readParkedMs = proactor.nowMs // the peer sent; the wait is ours, so the clock restarts
if (conn.handlerReleased) {
returnReleasedBuffers(conn)
}
recvStarved.add((gen.toLong() shl 32) or (fd.toLong() and 0xFFFF_FFFFL))
}

// Nobody will read a connection its handler let go, and its recv needs a buffer to reach the peer's close, which
// is what tears it down. So what it holds goes back.
private fun returnReleasedBuffers(conn: TcpConnection) {
// Not claimed the way recycle claims it: a read left parked can still take more, for recycle to ask back.
conn.bufferHolder?.releaseHeldBuffers()
conn.drainRecv(scratchItem)
}

// Once per loop pass, and only when a buffer came back since the last one, so a parked connection
// cannot spin the loop.
fun rearmStarvedRecvs() {
Expand Down Expand Up @@ -603,9 +617,16 @@ internal class TcpTransport(val proactor: Proactor, private val config: ServerCo

val now = Clock.monotonicMs()
val conns = connections
val starved = recvStarved.isNotEmpty()

for (fd in conns.indices) {
val conn = conns[fd] ?: continue

// A recv is waiting for a buffer, maybe its own, and this connection holds some for nobody.
if (starved && conn.handlerReleased) {
returnReleasedBuffers(conn)
}

if (conn.sweepClosed) {
continue
}
Expand All @@ -632,11 +653,25 @@ internal class TcpTransport(val proactor: Proactor, private val config: ServerCo
}
}
}

// A condemned connection whose recv is parked has no completion coming for its shutdown to end.
var i = recvStarved.size - 1
while (i >= 0) {
val parked = recvStarved[i]
val fd = parked.toInt()
val conn = connAt(fd, (parked ushr 32).toInt())
if (conn != null && conn.sweepClosed) {
recvStarved.removeAt(i)
closeFromRecv(conn, fd)
}
i--
}
}

// shutdown() is what the peer sees, and also what releases the connection: it ends the recv a silent
// peer never completes, or the send a stalled one holds, and those completions run the teardown that
// gets the refcount right once the kernel is done with the slab. markClosed wakes a parked read now.
// gets the refcount right once the kernel is done with the slab. markClosed wakes a parked read now. A recv
// parked on -ENOBUFS has no completion to wait for, so [sweep] runs that teardown itself.
private fun sweepClose(conn: TcpConnection) {
conn.sweepClosed = true
conn.suppressFin() // this shutdown is the FIN
Expand Down
6 changes: 3 additions & 3 deletions kronos/src/commonMain/kotlin/kronos/tls/TlsPipe.kt
Original file line number Diff line number Diff line change
Expand Up @@ -334,9 +334,9 @@ class TlsDecryptingPipeReader(
}

/**
* Frees the plaintext buffer. Called by [complete], and by the proactor at recycle for a reader its handler never
* completed; idempotent. Terminal, so a reader kept past its handler fails instead of reading the recycled
* connection's next tenant.
* Frees the plaintext buffer. Called by [complete], and by the proactor for a reader its handler never completed:
* at recycle, or sooner once the handler has let go and a recv needs the buffers. Idempotent. Terminal, so a reader
* kept past its handler fails instead of reading the recycled connection's next tenant.
*/
override fun releaseHeldBuffers() {
completed = true
Expand Down
209 changes: 209 additions & 0 deletions kronos/src/commonTest/kotlin/kronos/LetGoDriver.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
package kronos

import kotlin.concurrent.atomics.AtomicInt
import kotlin.coroutines.Continuation
import kotlin.coroutines.EmptyCoroutineContext
import kotlin.coroutines.startCoroutine
import kotlin.test.assertTrue

/**
* Clients that outlive their handlers, for any handler on the TCP transport. Once a handler has let go nobody reads
* its connection, so what its client goes on sending holds buffers. A server survives that if a new client is still
* served and the connections end, whichever way the client goes and whether or not a read timeout is set.
*/
internal object LetGoDriver {
const val GRACE_MS = 4_000L

private const val CLIENTS = 3

/** Which rings the clients empty, and what that takes: below the 64-entry queue, so nothing overflows. */
class Mode(val name: String, val sendsEach: Int, val bytesEach: Int, val incremental: IncrementalOptions?)

val SHARED = Mode("shared ring", sendsEach = 30, bytesEach = 1, incremental = null)

val INCREMENTAL = Mode(
"incremental",
sendsEach = 8,
bytesEach = 8 * 1024,
incremental = IncrementalOptions(maxConnections = 16, recvSlots = 4, recvBufferSize = 4096),
)

/** What the clients do next, and whether that ends the connections. */
class Ending(val name: String, val leave: Boolean, val readTimeoutMs: Long, val ends: Boolean)

private val THEY_LEAVE = Ending("they leave", leave = true, readTimeoutMs = 300, ends = true)
private val THEY_LEAVE_NO_TIMEOUT =
Ending("they leave, no read timeout", leave = true, readTimeoutMs = 0, ends = true)
private val THEY_STAY = Ending("they stay, until the read timeout", leave = false, readTimeoutMs = 300, ends = true)
private val THEY_STAY_NO_TIMEOUT =
Ending("they stay, no read timeout", leave = false, readTimeoutMs = 0, ends = false)

// IOU_PBUF_RING_INC arrived in Linux 6.12.
val incrementalSupported: Boolean = run {
val parts = kernelRelease().split('.', '-')
val major = parts.getOrNull(0)?.toIntOrNull() ?: 0
val minor = parts.getOrNull(1)?.toIntOrNull() ?: 0
major > 6 || (major == 6 && minor >= 12)
}

val modes: List<Mode> = if (incrementalSupported) listOf(SHARED, INCREMENTAL) else listOf(SHARED)

/**
* Runs [case] for every mode and ending that can show anything, naming the one that failed: a ring of its own takes
* nobody else's buffers, and with no end coming nothing can be seen to close.
*/
fun forEachCase(case: (Mode, Ending) -> Unit) {
for (mode in modes) {
for (ending in listOf(THEY_LEAVE, THEY_LEAVE_NO_TIMEOUT, THEY_STAY, THEY_STAY_NO_TIMEOUT)) {
if (mode !== INCREMENTAL || ending !== THEY_STAY_NO_TIMEOUT) {
naming("${mode.name}, ${ending.name}") { case(mode, ending) }
}
}
}
}

/** Runs [block], putting [name] on what it fails with. */
fun naming(name: String, block: () -> Unit) {
try {
block()
} catch (e: AssertionError) {
throw AssertionError("$name: ${e.message}")
} catch (e: IOException) {
throw AssertionError("$name: ${e.message}")
}
}

fun config(mode: Mode, readTimeoutMs: Long, sendTimeoutMs: Long = 500) = ServerConfig(
proactorCount = 1,
ringEntries = 256,
recvBufferSize = 4096,
recvSlots = 64,
tcp = TcpOptions(port = 0, recvQueueEntries = 64, readTimeoutMs = readTimeoutMs, sendTimeoutMs = sendTimeoutMs),
incremental = mode.incremental,
)

/** A client that outlived its handler, and how it goes: a TLS client has a session of its own to free. */
class Peer(val socket: TestSocket, private val release: () -> Unit = socket::close) {
fun close() = release()

fun trySend(bytes: ByteArray) {
try {
socket.write(bytes)
} catch (e: IOException) {
// The server has already ended this one, which is allowed at any point from here.
}
}
}

/**
* [letGoClient] makes a connected client whose handler has let go, or is about to; [served] says whether a new
* client, using the server as it should, gets its answer.
*/
fun check(proactor: Proactor, mode: Mode, ending: Ending, letGoClient: () -> Peer, served: () -> Boolean) {
val clients = ArrayList<Peer>()
try {
repeat(CLIENTS) {
clients.add(letGoClient())
}

val message = ByteArray(mode.bytesEach)
repeat(mode.sendsEach) {
clients.forEach { it.trySend(message) }
sleepMs(1) // apart, so each is a recv of its own
}

if (ending.leave) {
clients.forEach { it.close() }
clients.clear()
}

sleepMs(if (ending.readTimeoutMs > 0) 1_000 else 300)

assertTrue(served(), "a new client was never answered: the let-go connections kept the buffers")

if (ending.ends) {
assertTrue(
eventually { openConnections(proactor) == 0 },
"${openConnections(proactor)} connections still open, a new client served and their handlers gone",
)
}
} finally {
clients.forEach { it.close() }
}
}

fun connect(port: Int, receiveBuffer: Int = 0): TestSocket {
val socket = TestSocket()
socket.setReceiveTimeout(GRACE_MS.toInt())
socket.setNoDelay()
if (receiveBuffer > 0) {
socket.setReceiveBuffer(receiveBuffer) // before connect, so it is the window advertised
}

socket.connect(port)
return socket
}

/** Connected, and everything the server sent has been read through to its FIN or reset. */
fun refused(port: Int, first: ByteArray): Peer {
val client = connect(port)
client.write(first)
readToEnd(client)
return Peer(client)
}

private fun readToEnd(client: TestSocket) {
val buffer = ByteArray(4096)
try {
while (client.read(buffer, 0, buffer.size) >= 0) {
// what it was refused with
}
} catch (e: IOException) {
// A reset is an end too: the sends that follow are what the test is about. A timeout is not.
if (e.message?.contains("errno 104") != true) {
throw e
}
}
}

/** Whether [request] got its answer: a connection the server ended or never served did not. */
fun answers(request: () -> Boolean): Boolean = try {
request()
} catch (e: IOException) {
false
}

fun openConnections(proactor: Proactor): Int = onProactor(proactor) { proactor.tcp!!.openConnections }

/** Runs [block] on the proactor thread and waits for its answer. */
fun onProactor(proactor: Proactor, block: () -> Int): Int {
val result = AtomicInt(Int.MIN_VALUE)
val done = Latch(1)

proactor.post {
result.store(block())
done.countDown()
}

assertTrue(done.await(10_000), "the proactor never ran the task")
return result.load()
}

fun eventually(condition: () -> Boolean): Boolean {
val deadline = Clock.monotonicMs() + GRACE_MS
while (Clock.monotonicMs() < deadline) {
if (condition()) {
return true
}

sleepMs(10)
}

return condition()
}

/** Starts [block] apart from the handler, which goes on, and returns, while it waits. */
fun detached(block: suspend () -> Unit) {
block.startCoroutine(Continuation(EmptyCoroutineContext) {})
}
}
Loading
Loading