From 46579ba86813216dad6bd8be466ca5464c4dcce9 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Mon, 31 Aug 2026 12:45:20 +0300 Subject: [PATCH 01/11] Add private-link RTT estimation --- code/nettime.cpp | 42 ++++++ code/nettime.h | 38 ++++++ code/nettiming.cpp | 97 ++++++++++++++ code/nettiming.h | 46 +++++++ tests/CMakeLists.txt | 1 + tests/nettiming/CMakeLists.txt | 34 +++++ tests/nettiming/nettiming.cpp | 228 +++++++++++++++++++++++++++++++++ 7 files changed, 486 insertions(+) create mode 100644 code/nettime.cpp create mode 100644 code/nettime.h create mode 100644 code/nettiming.cpp create mode 100644 code/nettiming.h create mode 100644 tests/nettiming/CMakeLists.txt create mode 100644 tests/nettiming/nettiming.cpp diff --git a/code/nettime.cpp b/code/nettime.cpp new file mode 100644 index 000000000..a1b82f3eb --- /dev/null +++ b/code/nettime.cpp @@ -0,0 +1,42 @@ +/******************************************************************************* + * O P E N T S + ******************************************************************************* + * SPDX-License-Identifier: GPL-3.0-or-later + * Copyright 2026 OpenTS contributors + * + * See LICENSE.md for applicable additional terms and warranty disclaimers. + ******************************************************************************/ + + +#include "nettime.h" + +#include +#include + + +namespace NetTiming +{ + namespace + { + class SystemMillisecondClock final : public MillisecondClock + { + public: + Milliseconds Now(void) const override; + }; + } + + + /// Reads the system's wrapping millisecond clock. + Milliseconds SystemMillisecondClock::Now(void) const + { + return(static_cast(::timeGetTime())); + } + + + /// Returns the process-wide network clock. + MillisecondClock const & Default_Clock(void) + { + static SystemMillisecondClock clock; + return(clock); + } +} diff --git a/code/nettime.h b/code/nettime.h new file mode 100644 index 000000000..d07200cd1 --- /dev/null +++ b/code/nettime.h @@ -0,0 +1,38 @@ +/******************************************************************************* + * O P E N T S + ******************************************************************************* + * SPDX-License-Identifier: GPL-3.0-or-later + * Copyright 2026 OpenTS contributors + * + * See LICENSE.md for applicable additional terms and warranty disclaimers. + ******************************************************************************/ + + +#pragma once + +#include + + +namespace NetTiming +{ + using Milliseconds = std::uint32_t; + + class MillisecondClock + { + public: + virtual ~MillisecondClock() = default; + virtual Milliseconds Now(void) const = 0; + }; + + MillisecondClock const & Default_Clock(void); + + constexpr Milliseconds Elapsed_Milliseconds(Milliseconds start, Milliseconds finish) + { + return(finish - start); + } + + constexpr bool Milliseconds_Have_Elapsed(Milliseconds start, Milliseconds now, Milliseconds duration) + { + return(Elapsed_Milliseconds(start, now) >= duration); + } +} diff --git a/code/nettiming.cpp b/code/nettiming.cpp new file mode 100644 index 000000000..74f465a9b --- /dev/null +++ b/code/nettiming.cpp @@ -0,0 +1,97 @@ +/******************************************************************************* + * O P E N T S + ******************************************************************************* + * SPDX-License-Identifier: GPL-3.0-or-later + * Copyright 2026 OpenTS contributors + * + * See LICENSE.md for applicable additional terms and warranty disclaimers. + ******************************************************************************/ + + +#include "nettiming.h" + +#include + + +namespace NetTiming +{ + namespace + { + /// Constrains a retransmission timeout to the supported range. + constexpr Milliseconds Clamp_Rto(std::uint64_t value) + { + return(static_cast(std::clamp(value, MINIMUM_RTO, MAXIMUM_RTO))); + } + } + + + /// Restores the estimator to its unsampled state. + void RttEstimator::Reset(void) + { + Initialized = false; + SmoothedRtt = 0; + RttVariation = 0; + RetransmitTimeout = MINIMUM_RTO; + } + + + /// Updates SRTT, RTTVAR, and RTO from an eligible sample. + bool RttEstimator::Add_Sample(Milliseconds round_trip, bool retransmitted) + { + // Karn's rule excludes ambiguous acknowledgements after retransmission. + if (retransmitted) { + return(false); + } + + if (!Initialized) { + Initialized = true; + SmoothedRtt = round_trip; + RttVariation = (round_trip + 1) / 2; + } else { + Milliseconds const error = SmoothedRtt > round_trip ? SmoothedRtt - round_trip : round_trip - SmoothedRtt; + RttVariation = static_cast((3ull * RttVariation + error + 2) / 4); + SmoothedRtt = static_cast((7ull * SmoothedRtt + round_trip + 4) / 8); + } + + std::uint64_t const variation = std::max(1, 4ull * RttVariation); + RetransmitTimeout = Clamp_Rto(static_cast(SmoothedRtt) + variation); + return(true); + } + + + /// Samples an acknowledgement when its send time is unambiguous. + bool RttEstimator::Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock) + { + if (transmission_count != 1) { + return(false); + } + return(Add_Sample(Elapsed_Milliseconds(sent_at, clock.Now()))); + } + + + /// Derives the connection timeout from smoothed latency. + Milliseconds Connection_Timeout(Milliseconds smoothed_rtt) + { + std::uint64_t const timeout = 8ull * smoothed_rtt + 250; + return(static_cast(std::clamp(timeout, MINIMUM_CONNECTION_TIMEOUT, MAXIMUM_CONNECTION_TIMEOUT))); + } + + + /// Applies bounded exponential backoff to a packet's RTO. + Milliseconds Retransmit_Delay(Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay) + { + maximum_delay = std::max(maximum_delay, MINIMUM_RTO); + std::uint64_t delay = std::clamp(base_rto, MINIMUM_RTO, maximum_delay); + while (prior_retransmissions-- > 0 && delay < maximum_delay) { + delay = std::min(delay * 2, maximum_delay); + } + return(static_cast(delay)); + } + + + /// Checks whether a packet's current backoff interval has elapsed. + bool Retransmit_Is_Due(Milliseconds last_send, Milliseconds now, Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay) + { + return(Milliseconds_Have_Elapsed(last_send, now, Retransmit_Delay(base_rto, prior_retransmissions, maximum_delay))); + } +} diff --git a/code/nettiming.h b/code/nettiming.h new file mode 100644 index 000000000..8dee4a849 --- /dev/null +++ b/code/nettiming.h @@ -0,0 +1,46 @@ +/******************************************************************************* + * O P E N T S + ******************************************************************************* + * SPDX-License-Identifier: GPL-3.0-or-later + * Copyright 2026 OpenTS contributors + * + * See LICENSE.md for applicable additional terms and warranty disclaimers. + ******************************************************************************/ + + +#pragma once + +#include "nettime.h" + + +namespace NetTiming +{ + constexpr Milliseconds MINIMUM_RTO = 100; + constexpr Milliseconds MAXIMUM_RTO = 2000; + constexpr Milliseconds MINIMUM_CONNECTION_TIMEOUT = 2000; + constexpr Milliseconds MAXIMUM_CONNECTION_TIMEOUT = 30000; + + class RttEstimator + { + public: + void Reset(void); + bool Add_Sample(Milliseconds round_trip, bool retransmitted = false); + bool Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock = Default_Clock()); + + bool Has_Sample(void) const {return(Initialized);} + Milliseconds Smoothed_Rtt(void) const {return(SmoothedRtt);} + Milliseconds Rtt_Variation(void) const {return(RttVariation);} + Milliseconds Retransmit_Timeout(void) const {return(RetransmitTimeout);} + + private: + bool Initialized = false; + Milliseconds SmoothedRtt = 0; + Milliseconds RttVariation = 0; + Milliseconds RetransmitTimeout = MINIMUM_RTO; + }; + + Milliseconds Connection_Timeout(Milliseconds smoothed_rtt); + Milliseconds Retransmit_Delay(Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay = MAXIMUM_RTO); + bool Retransmit_Is_Due(Milliseconds last_send, Milliseconds now, Milliseconds base_rto, + unsigned int prior_retransmissions, Milliseconds maximum_delay = MAXIMUM_RTO); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 547b0d61d..c8d1ff844 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,5 +2,6 @@ add_subdirectory(cpudetect) add_subdirectory(gamedirs) add_subdirectory(logstress) add_subdirectory(netpacket) +add_subdirectory(nettiming) add_subdirectory(sosparity) add_subdirectory(spawner) diff --git a/tests/nettiming/CMakeLists.txt b/tests/nettiming/CMakeLists.txt new file mode 100644 index 000000000..7cc49744a --- /dev/null +++ b/tests/nettiming/CMakeLists.txt @@ -0,0 +1,34 @@ +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + cmake_minimum_required(VERSION 3.23) + project(NetTimingContract LANGUAGES CXX) + enable_testing() + set(OPENTS_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../..") +else() + set(OPENTS_ROOT "${CMAKE_SOURCE_DIR}") +endif() + +# Compile the transport timing sources directly so this harness needs no game assets. +add_executable(NetTiming + "${CMAKE_CURRENT_SOURCE_DIR}/nettiming.cpp" + "${OPENTS_ROOT}/code/nettime.cpp" + "${OPENTS_ROOT}/code/nettiming.cpp" +) + +target_compile_features(NetTiming PRIVATE cxx_std_20) + +target_include_directories(NetTiming PRIVATE "${OPENTS_ROOT}/code") + +target_compile_definitions(NetTiming PRIVATE WIN32 _WINDOWS _MBCS) + +target_compile_options(NetTiming PRIVATE + $<$:/MTd /EHsc /Zc:__cplusplus> + $<$:/MT /EHsc /Zc:__cplusplus> +) + +target_link_libraries(NetTiming PRIVATE kernel32 winmm) + +set_target_properties(NetTiming PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" +) + +add_test(NAME nettiming COMMAND NetTiming) diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp new file mode 100644 index 000000000..5e65c1ee1 --- /dev/null +++ b/tests/nettiming/nettiming.cpp @@ -0,0 +1,228 @@ +/******************************************************************************* + * O P E N T S + ******************************************************************************* + * SPDX-License-Identifier: GPL-3.0-or-later + * Copyright 2026 OpenTS contributors + * + * See LICENSE.md for applicable additional terms and warranty disclaimers. + ******************************************************************************/ + + +#include "nettiming.h" + +#include +#include +#include +#include + + +namespace +{ + class FakeClock final : public NetTiming::MillisecondClock + { + public: + NetTiming::Milliseconds Now(void) const override {return(Current);} + void Set(NetTiming::Milliseconds now) {Current = now;} + + private: + NetTiming::Milliseconds Current = 0; + }; + + + class FakeTransport + { + public: + void Send(NetTiming::Milliseconds now) + { + Clock.Set(now); + FirstSend = now; + LastSend = now; + TransmissionCount = 1; + BaseRto = Estimator.Retransmit_Timeout(); + } + + bool Retry(NetTiming::Milliseconds now) + { + Clock.Set(now); + if (!NetTiming::Retransmit_Is_Due(LastSend, now, BaseRto, TransmissionCount - 1, NetTiming::MINIMUM_CONNECTION_TIMEOUT)) { + return(false); + } + LastSend = now; + TransmissionCount++; + return(true); + } + + bool Acknowledge(NetTiming::Milliseconds now) + { + Clock.Set(now); + return(Estimator.Acknowledge(FirstSend, TransmissionCount, Clock)); + } + + NetTiming::RttEstimator const & Rtt(void) const {return(Estimator);} + + private: + FakeClock Clock; + NetTiming::RttEstimator Estimator; + NetTiming::Milliseconds FirstSend = 0; + NetTiming::Milliseconds LastSend = 0; + NetTiming::Milliseconds BaseRto = NetTiming::MINIMUM_RTO; + unsigned int TransmissionCount = 0; + }; + + + int Failures = 0; + + + template + void Expect_Equal(std::string const & name, Actual const & actual, Expected const & expected) + { + if (actual == expected) { + return; + } + + std::cerr << name << ": expected " << expected << ", got " << actual << '\n'; + Failures++; + } + + + void Expect(std::string const & name, bool condition) + { + if (!condition) { + std::cerr << name << " failed\n"; + Failures++; + } + } + + + void Test_Rtt_Estimator(void) + { + using namespace NetTiming; + + RttEstimator estimator; + Expect("estimator starts empty", !estimator.Has_Sample()); + Expect("first sample accepted", estimator.Add_Sample(100)); + Expect_Equal("first smoothed RTT", estimator.Smoothed_Rtt(), 100u); + Expect_Equal("first variation", estimator.Rtt_Variation(), 50u); + Expect_Equal("first RTO", estimator.Retransmit_Timeout(), 300u); + + Expect("second sample accepted", estimator.Add_Sample(140)); + Expect_Equal("alpha one eighth", estimator.Smoothed_Rtt(), 105u); + Expect_Equal("beta one quarter", estimator.Rtt_Variation(), 48u); + Expect_Equal("updated RTO", estimator.Retransmit_Timeout(), 297u); + + Expect("retransmitted sample rejected", !estimator.Add_Sample(900, true)); + Expect_Equal("Karn keeps smoothed RTT", estimator.Smoothed_Rtt(), 105u); + Expect_Equal("Karn keeps RTO", estimator.Retransmit_Timeout(), 297u); + + RttEstimator minimum; + minimum.Add_Sample(0); + Expect_Equal("minimum RTO clamp", minimum.Retransmit_Timeout(), MINIMUM_RTO); + + RttEstimator maximum; + maximum.Add_Sample(2000); + Expect_Equal("maximum RTO clamp", maximum.Retransmit_Timeout(), MAXIMUM_RTO); + + RttEstimator fast_link; + RttEstimator slow_link; + fast_link.Add_Sample(50); + slow_link.Add_Sample(300); + Expect("unequal links keep independent RTOs", fast_link.Retransmit_Timeout() < slow_link.Retransmit_Timeout()); + + estimator.Reset(); + Expect("reset clears estimator", !estimator.Has_Sample()); + Expect_Equal("reset restores RTO", estimator.Retransmit_Timeout(), MINIMUM_RTO); + } + + + void Test_Clock_And_Wrap(void) + { + using namespace NetTiming; + + FakeClock clock; + clock.Set(0x00000020u); + RttEstimator estimator; + Expect("wrap sample accepted", estimator.Acknowledge(0xfffffff0u, 1, clock)); + Expect_Equal("wrap elapsed", estimator.Smoothed_Rtt(), 48u); + Expect("retransmitted acknowledgement ignored", !estimator.Acknowledge(0, 2, clock)); + + Expect("wrapped retry due", Retransmit_Is_Due(0xfffffff0u, 0x00000054u, 100, 0)); + Expect("wrapped retry not early", !Retransmit_Is_Due(0xfffffff0u, 0x00000040u, 100, 0)); + } + + + void Test_Retransmit_Backoff(void) + { + using namespace NetTiming; + + Expect_Equal("base retry", Retransmit_Delay(100, 0), 100u); + Expect_Equal("first backoff", Retransmit_Delay(100, 1), 200u); + Expect_Equal("second backoff", Retransmit_Delay(100, 2), 400u); + Expect_Equal("third backoff", Retransmit_Delay(100, 3), 800u); + Expect_Equal("fourth backoff", Retransmit_Delay(100, 4), 1600u); + Expect_Equal("backoff saturation", Retransmit_Delay(100, 20), MAXIMUM_RTO); + Expect_Equal("base clamp", Retransmit_Delay(1, 0), MINIMUM_RTO); + Expect_Equal("connection timeout minimum", Connection_Timeout(0), 2000u); + Expect_Equal("connection timeout follows RTT", Connection_Timeout(500), 4250u); + Expect_Equal("connection timeout ceiling", Connection_Timeout(10000), 30000u); + Expect_Equal("backoff reaches connection timeout", Retransmit_Delay(500, 8, 4250), 4250u); + } + + + void Test_Loss_Jitter_And_Reordering(void) + { + using namespace NetTiming; + + FakeClock clock; + RttEstimator reordered; + clock.Set(1200); + Expect("newer packet ACK samples first", reordered.Acknowledge(1100, 1, clock)); + clock.Set(1300); + Expect("older packet ACK can sample after reordering", reordered.Acknowledge(1000, 1, clock)); + Expect_Equal("reordered samples keep alpha filter", reordered.Smoothed_Rtt(), 125u); + Expect_Equal("reordered samples keep beta filter", reordered.Rtt_Variation(), 88u); + + clock.Set(2000); + Expect("duplicate ambiguous ACK is excluded by Karn", !reordered.Acknowledge(1500, 2, clock)); + Expect_Equal("ambiguous ACK leaves SRTT unchanged", reordered.Smoothed_Rtt(), 125u); + + RttEstimator jitter; + for (Milliseconds sample : {20u, 400u, 35u, 350u, 40u}) { + jitter.Add_Sample(sample); + } + Expect("jitter raises variation", jitter.Rtt_Variation() > 0); + Expect("jittered RTO remains bounded", jitter.Retransmit_Timeout() >= MINIMUM_RTO && jitter.Retransmit_Timeout() <= MAXIMUM_RTO); + + Expect("loss does not retransmit before the base RTO", !Retransmit_Is_Due(1000, 1099, 100, 0, 2000)); + Expect("first loss retransmits at the base RTO", Retransmit_Is_Due(1000, 1100, 100, 0, 2000)); + Expect("second loss waits for exponential backoff", !Retransmit_Is_Due(1100, 1299, 100, 1, 2000)); + Expect("second loss retransmits at doubled RTO", Retransmit_Is_Due(1100, 1300, 100, 1, 2000)); + + FakeTransport clean_transport; + clean_transport.Send(1000); + Expect("fake transport accepts a clean ACK sample", clean_transport.Acknowledge(1080)); + Expect_Equal("fake transport publishes clean RTT", clean_transport.Rtt().Smoothed_Rtt(), 80u); + + FakeTransport lossy_transport; + lossy_transport.Send(1000); + Expect("fake transport retries a lost packet", lossy_transport.Retry(1100)); + Expect("fake transport applies Karn after loss", !lossy_transport.Acknowledge(1180)); + Expect("lossy fake transport has no ambiguous RTT sample", !lossy_transport.Rtt().Has_Sample()); + } +} + + +int main(void) +{ + Test_Rtt_Estimator(); + Test_Clock_And_Wrap(); + Test_Retransmit_Backoff(); + Test_Loss_Jitter_And_Reordering(); + + if (Failures != 0) { + std::cerr << Failures << " network timing checks failed\n"; + return(1); + } + + std::cout << "All network timing checks passed\n"; + return(0); +} From d8ce1e94f4e9dfab949342314256a9a756a8f3e2 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Mon, 31 Aug 2026 13:41:18 +0300 Subject: [PATCH 02/11] Back off private network retransmissions --- code/combuf.h | 3 + code/connect.cpp | 87 ++++++++++++++----- code/connect.h | 15 ++-- code/ipxgconn.h | 2 + code/ipxmgr.cpp | 3 + code/nettiming.cpp | 18 ++++ code/nettiming.h | 17 ++++ manual/changes/network-transport-timing.md | 12 +++ .../systems/network-transport-timing.md | 15 ++++ tests/nettiming/nettiming.cpp | 29 +++++++ 10 files changed, 176 insertions(+), 25 deletions(-) create mode 100644 manual/changes/network-transport-timing.md create mode 100644 manual/content/systems/network-transport-timing.md diff --git a/code/combuf.h b/code/combuf.h index 54b9b73f7..4f02f0c6e 100644 --- a/code/combuf.h +++ b/code/combuf.h @@ -59,6 +59,9 @@ struct SendQueueType { unsigned int IsUndeliverable : 1; /// 1 = gave up on it (retries or timeout) unsigned int FirstTime; // time this packet was first sent unsigned int LastTime; // time this packet was last sent + unsigned int FirstTimeMilliseconds = 0; // monotonic time of the first transmission + unsigned int LastTimeMilliseconds = 0; // monotonic time of the latest transmission + unsigned int RetransmitTimeoutMilliseconds = 0; // base RTO captured for this packet unsigned int SendCount; // # of times this packet has been sent int BufLen; // size of the packet stored in this entry char *Buffer; // the data packet diff --git a/code/connect.cpp b/code/connect.cpp index c2b08b493..6c173a69a 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -47,8 +47,11 @@ #include "_timer.h" #include "dbgprint.h" +#include +#include #include #include +#include #include @@ -61,6 +64,27 @@ char const * ConnectionClass::Commands[PACKET_COUNT] = { "ACK" }; +namespace { + +/// Converts engine ticks to milliseconds. +NetTiming::Milliseconds Ticks_To_Milliseconds(unsigned int ticks) +{ + std::uint64_t const milliseconds = (static_cast(ticks) * 1000 + TIMER_SECOND - 1) / TIMER_SECOND; + if (milliseconds > std::numeric_limits::max()) { + return(std::numeric_limits::max()); + } + return(static_cast(milliseconds)); +} + + +/// Converts and bounds a legacy connection timeout. +NetTiming::Milliseconds Legacy_Connection_Timeout(unsigned int ticks) +{ + return(std::clamp(Ticks_To_Milliseconds(ticks), NetTiming::MINIMUM_CONNECTION_TIMEOUT, NetTiming::MAXIMUM_CONNECTION_TIMEOUT)); +} + +} + /*************************************************************************** * ConnectionClass::ConnectionClass -- class constructor * @@ -76,6 +100,7 @@ char const * ConnectionClass::Commands[PACKET_COUNT] = { * timeout the max amount of time before we give up on a packet* * (-1 means retry forever, based on this parameter) * * extralen max size of app-specific extra bytes (optional) * + * clock monotonic millisecond clock (default if NULL) * * * * OUTPUT: * * none. * @@ -88,7 +113,7 @@ char const * ConnectionClass::Commands[PACKET_COUNT] = { *=========================================================================*/ ConnectionClass::ConnectionClass (int numsend, int numreceive, int maxlen, unsigned short magicnum, unsigned int retry_delta, - unsigned int max_retries, unsigned int timeout, int extralen) + unsigned int max_retries, unsigned int timeout, int extralen, NetTiming::MillisecondClock const * clock) { /*------------------------------------------------------------------------ Compute our maximum packet length @@ -115,6 +140,7 @@ ConnectionClass::ConnectionClass (int numsend, int numreceive, Set the timeout for this connection. ------------------------------------------------------------------------*/ Timeout = timeout; + MillisecondTime = clock != nullptr ? clock : &NetTiming::Default_Clock(); /*------------------------------------------------------------------------ Allocate the packet staging buffer. This will be used to @@ -191,6 +217,7 @@ void ConnectionClass::Init (void) LastSeqID = 0xffffffff; LastReadID = 0xffffffff; + RoundTripEstimator.Reset(); Queue->Init(); @@ -748,7 +775,7 @@ int ConnectionClass::Service_Send_Queue (void) int i; int num_entries; SendQueueType *send_entry; // ptr to send queue entry - CommHeaderType *packet_hdr; // packet header + CommHeaderType packet_header; // packet header unsigned int curtime; // current time int bad_conn = 0; @@ -769,9 +796,15 @@ int ConnectionClass::Service_Send_Queue (void) /*.................................................................. Update this queue's response time ..................................................................*/ - packet_hdr = (CommHeaderType *)send_entry->Buffer; - if (packet_hdr->Code == PACKET_DATA_ACK) { - Queue->Add_Delay(Time() - send_entry->FirstTime); + if (send_entry->BufLen >= (int)sizeof(CommHeaderType)) { + CommHeaderType header; + memcpy(&header, send_entry->Buffer, sizeof(header)); + if (header.Code == PACKET_DATA_ACK) { + Queue->Add_Delay(Time() - send_entry->FirstTime); + if (Adaptive_Timing_Enabled()) { + RoundTripEstimator.Acknowledge(send_entry->FirstTimeMilliseconds, send_entry->SendCount, *MillisecondTime); + } + } } /*.................................................................. @@ -787,6 +820,16 @@ int ConnectionClass::Service_Send_Queue (void) need it. ------------------------------------------------------------------------*/ num_entries = Queue->Num_Send(); + curtime = Time(); + NetTiming::Milliseconds const current_milliseconds = MillisecondTime->Now(); + bool const adaptive_channel = Adaptive_Timing_Enabled(); + bool const adaptive_timing = adaptive_channel && RoundTripEstimator.Has_Sample(); + bool const timeout_enabled = Timeout != (unsigned int)-1; + NetTiming::Milliseconds const connection_timeout = !timeout_enabled + ? NetTiming::MAXIMUM_CONNECTION_TIMEOUT + : (adaptive_timing ? NetTiming::Connection_Timeout(RoundTripEstimator.Smoothed_Rtt()) + : (adaptive_channel ? Legacy_Connection_Timeout(Timeout) : Ticks_To_Milliseconds(Timeout))); + NetTiming::Milliseconds const base_retry_timeout = adaptive_timing ? RoundTripEstimator.Retransmit_Timeout() : Ticks_To_Milliseconds(RetryDelta); for (i = 0; i < num_entries; i++) { send_entry = Queue->Get_Send(i); @@ -795,13 +838,20 @@ int ConnectionClass::Service_Send_Queue (void) continue; } - /*..................................................................... - Only send the message if time has elapsed. (The message's Time - fields are init'd to 0 when a message is queue'd or unqueue'd, so the - first time through, the delta time will appear large.) - .....................................................................*/ - curtime = Time(); - if (curtime - send_entry->LastTime > RetryDelta) { + NetTiming::RetransmitState const retransmit_state{ + send_entry->FirstTimeMilliseconds, + send_entry->LastTimeMilliseconds, + send_entry->RetransmitTimeoutMilliseconds, + send_entry->SendCount + }; + NetTiming::RetryDecision const retry_decision = NetTiming::Evaluate_Retry( + retransmit_state, current_milliseconds, base_retry_timeout, connection_timeout, timeout_enabled, adaptive_channel); + if (retry_decision == NetTiming::RetryDecision::TIMED_OUT) { + bad_conn = 1; + send_entry->IsUndeliverable = true; + continue; + } + if (retry_decision == NetTiming::RetryDecision::SEND) { /*.................................................................. Send the message @@ -813,16 +863,19 @@ int ConnectionClass::Service_Send_Queue (void) Fill in Time fields ..................................................................*/ send_entry->LastTime = curtime; + send_entry->LastTimeMilliseconds = current_milliseconds; if (send_entry->SendCount==0) { send_entry->FirstTime = curtime; + send_entry->FirstTimeMilliseconds = current_milliseconds; + send_entry->RetransmitTimeoutMilliseconds = base_retry_timeout; /*............................................................... If this is the 1st time we're sending this packet, and it doesn't require an ACK, mark it as ACK'd; then, the next time through, it will just be removed from the queue. ...............................................................*/ - packet_hdr = (CommHeaderType *)send_entry->Buffer; - if (packet_hdr->Code == PACKET_DATA_NOACK) { + memcpy(&packet_header, send_entry->Buffer, sizeof(packet_header)); + if (packet_header.Code == PACKET_DATA_NOACK) { send_entry->IsACK = 1; } } else { @@ -841,12 +894,6 @@ int ConnectionClass::Service_Send_Queue (void) bad_conn = 1; send_entry->IsUndeliverable = true; } - - if (Timeout != -1 && - (send_entry->LastTime - send_entry->FirstTime) > Timeout) { - bad_conn = 1; - send_entry->IsUndeliverable = true; - } } } diff --git a/code/connect.h b/code/connect.h index da735f5e2..99fbedcd3 100644 --- a/code/connect.h +++ b/code/connect.h @@ -98,6 +98,7 @@ */ #include "combuf.h" #include "netadmit.h" +#include "nettiming.h" /* ********************************** Defines ********************************** @@ -143,9 +144,8 @@ class ConnectionClass /*..................................................................... Constructor/destructor. .....................................................................*/ - ConnectionClass (int numsend, int numrecieve, int maxlen, - unsigned short magicnum, unsigned int retry_delta, - unsigned int max_retries, unsigned int timeout, int extralen = 0); + ConnectionClass (int numsend, int numrecieve, int maxlen, unsigned short magicnum, unsigned int retry_delta, + unsigned int max_retries, unsigned int timeout, int extralen = 0, NetTiming::MillisecondClock const *clock = nullptr); virtual ~ConnectionClass (void); /*..................................................................... @@ -185,6 +185,7 @@ class ConnectionClass unsigned int Time_Out (void) { return(Timeout); } void Set_TimeOut (unsigned int t) { Timeout = t;} unsigned int Max_Packet_Len (void) { return(MaxPacketLen); } + void Reset_Round_Trip_Time(void) {RoundTripEstimator.Reset();} static const char * Command_Name(int command); int Num_Resends(void) const { return(NumResends); } @@ -227,8 +228,8 @@ class ConnectionClass is protected; it's only called by the ACK/Retry logic, not the application. .....................................................................*/ - virtual int Send(char *buf, int buflen, void *extrabuf, - int extralen) = 0; + virtual int Send(char *buf, int buflen, void *extrabuf, int extralen) = 0; + virtual bool Adaptive_Timing_Enabled(void) const {return(true);} void Record_Packet_Drop(PacketDropReasonType reason); void Record_Admission_Drop(NetAdmission::Error error, unsigned char code); @@ -293,6 +294,10 @@ class ConnectionClass .....................................................................*/ unsigned int Timeout; + // An injected clock must outlive the connection. + NetTiming::MillisecondClock const *MillisecondTime; + NetTiming::RttEstimator RoundTripEstimator; + /*..................................................................... Running totals of # of packets we send & receive which require an ACK, and those that don't. diff --git a/code/ipxgconn.h b/code/ipxgconn.h index 7d53e4c26..6cd0e6bb8 100644 --- a/code/ipxgconn.h +++ b/code/ipxgconn.h @@ -166,6 +166,8 @@ class IPXGlobalConnClass : public IPXConnClass // stored in the extra buffer within the Queue. //..................................................................... virtual int Send (char *buf, int buflen, void *extrabuf, int extralen) override; + virtual bool Adaptive_Timing_Enabled(void) const override {return(false);} + //..................................................................... // This routine is overloaded from SequencedConnClass, because the // Global Connection needs to ACK its packets differently from the diff --git a/code/ipxmgr.cpp b/code/ipxmgr.cpp index 26a6f106f..78e09c3b9 100644 --- a/code/ipxmgr.cpp +++ b/code/ipxmgr.cpp @@ -1478,6 +1478,9 @@ void IPXManagerClass::Reset_Response_Time(bool zero) for (i = 0; i < NumConnections; i++) { Connection[i]->Queue->Reset_Response_Time(zero); + if (zero) { + Connection[i]->Reset_Round_Trip_Time(); + } } if (GlobalChannel) diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 74f465a9b..0f22cf2f2 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -94,4 +94,22 @@ namespace NetTiming { return(Milliseconds_Have_Elapsed(last_send, now, Retransmit_Delay(base_rto, prior_retransmissions, maximum_delay))); } + + + /// Chooses the next action for one queued packet without changing its timing state. + RetryDecision Evaluate_Retry(RetransmitState const & state, Milliseconds now, Milliseconds current_rto, Milliseconds connection_timeout, + bool timeout_enabled, bool adaptive) + { + if (state.TransmissionCount == 0) { + return(RetryDecision::SEND); + } + if (timeout_enabled && Milliseconds_Have_Elapsed(state.FirstSend, now, connection_timeout)) { + return(RetryDecision::TIMED_OUT); + } + + bool const retry_due = adaptive + ? Retransmit_Is_Due(state.LastSend, now, state.CapturedRto, state.TransmissionCount - 1, connection_timeout) + : Milliseconds_Have_Elapsed(state.LastSend, now, current_rto); + return(retry_due ? RetryDecision::SEND : RetryDecision::WAIT); + } } diff --git a/code/nettiming.h b/code/nettiming.h index 8dee4a849..b7c178647 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -20,6 +20,21 @@ namespace NetTiming constexpr Milliseconds MINIMUM_CONNECTION_TIMEOUT = 2000; constexpr Milliseconds MAXIMUM_CONNECTION_TIMEOUT = 30000; + enum class RetryDecision + { + WAIT, + SEND, + TIMED_OUT + }; + + struct RetransmitState + { + Milliseconds FirstSend = 0; + Milliseconds LastSend = 0; + Milliseconds CapturedRto = MINIMUM_RTO; + unsigned int TransmissionCount = 0; + }; + class RttEstimator { public: @@ -43,4 +58,6 @@ namespace NetTiming Milliseconds Retransmit_Delay(Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay = MAXIMUM_RTO); bool Retransmit_Is_Due(Milliseconds last_send, Milliseconds now, Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay = MAXIMUM_RTO); + RetryDecision Evaluate_Retry(RetransmitState const & state, Milliseconds now, Milliseconds current_rto, Milliseconds connection_timeout, + bool timeout_enabled, bool adaptive); } diff --git a/manual/changes/network-transport-timing.md b/manual/changes/network-transport-timing.md new file mode 100644 index 000000000..928abd2d3 --- /dev/null +++ b/manual/changes/network-transport-timing.md @@ -0,0 +1,12 @@ +--- +title: Adapt private network retries +category: performance +release: 0.2.0 +targets: [] +credit: +- ZivDero +--- + +Each private connection now estimates its own round trip and backs off repeated +transmissions. Lobby traffic keeps its fixed retry cadence. Packet layouts, +event IDs, and configuration remain unchanged. diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md new file mode 100644 index 000000000..f3193f60d --- /dev/null +++ b/manual/content/systems/network-transport-timing.md @@ -0,0 +1,15 @@ +--- +title: Network transport timing +summary: Measures each private link and schedules retries without changing synchronized frame timing. +category: multiplayer-networking +keys: [] +--- + +Each private connection maintains smoothed round trip, variation, and a retry +timeout. Only acknowledgements for first transmissions become samples, avoiding +ambiguous measurements after a retry. + +The retry timeout is limited to 100–2000 ms. Repeated private transmissions +double their wait up to the connection timeout; that timeout follows measured +latency with a 2-second minimum and 30-second ceiling. With no measurement, the +bounded legacy timing is used. Global lobby traffic retains its fixed cadence. diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index 5e65c1ee1..f03019b91 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -168,6 +168,34 @@ namespace } + void Test_Retry_Decisions(void) + { + using namespace NetTiming; + + RetransmitState state; + Expect("new packet sends immediately", Evaluate_Retry(state, 1000, 800, 2000, true, true) == RetryDecision::SEND); + + state = {1000, 1000, 100, 1}; + Expect("adaptive packet keeps captured RTO", Evaluate_Retry(state, 1099, 800, 2000, true, true) == RetryDecision::WAIT); + Expect("adaptive packet sends at captured RTO", Evaluate_Retry(state, 1100, 800, 2000, true, true) == RetryDecision::SEND); + + state = {1000, 1100, 100, 2}; + Expect("adaptive retry waits through backoff", Evaluate_Retry(state, 1299, 800, 2000, true, true) == RetryDecision::WAIT); + Expect("adaptive retry sends after backoff", Evaluate_Retry(state, 1300, 800, 2000, true, true) == RetryDecision::SEND); + + state = {1000, 1000, 100, 4}; + Expect("fixed channel uses current retry delay", Evaluate_Retry(state, 1399, 400, 2000, true, false) == RetryDecision::WAIT); + Expect("fixed channel does not back off", Evaluate_Retry(state, 1400, 400, 2000, true, false) == RetryDecision::SEND); + + state = {1000, 1900, 100, 1}; + Expect("connection timeout wins over retry", Evaluate_Retry(state, 3000, 100, 2000, true, true) == RetryDecision::TIMED_OUT); + Expect("disabled connection timeout still retries", Evaluate_Retry(state, 3000, 100, 2000, false, true) == RetryDecision::SEND); + + state = {0xffffff00u, 0xfffffff0u, 100, 1}; + Expect("retry decision handles clock wrap", Evaluate_Retry(state, 0x00000054u, 800, 2000, true, true) == RetryDecision::SEND); + } + + void Test_Loss_Jitter_And_Reordering(void) { using namespace NetTiming; @@ -216,6 +244,7 @@ int main(void) Test_Rtt_Estimator(); Test_Clock_And_Wrap(); Test_Retransmit_Backoff(); + Test_Retry_Decisions(); Test_Loss_Jitter_And_Reordering(); if (Failures != 0) { From 99855d06ea71aa38f7b94e4439f934c52917b375 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Mon, 31 Aug 2026 19:49:56 +0300 Subject: [PATCH 03/11] Persist retransmission backoff and age RTT samples --- code/connect.cpp | 3 + code/nettiming.cpp | 30 +++++ code/nettiming.h | 8 ++ manual/changes/network-transport-timing.md | 6 +- .../systems/network-transport-timing.md | 9 ++ tests/nettiming/nettiming.cpp | 104 ++++++++++++++++++ 6 files changed, 158 insertions(+), 2 deletions(-) diff --git a/code/connect.cpp b/code/connect.cpp index 6c173a69a..120c67620 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -880,6 +880,9 @@ int ConnectionClass::Service_Send_Queue (void) } } else { NumResends++; + if (adaptive_channel) { + RoundTripEstimator.Note_Retransmit(send_entry->RetransmitTimeoutMilliseconds, current_milliseconds); + } } /*.................................................................. diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 0f22cf2f2..28d54c0a3 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -32,6 +32,8 @@ namespace NetTiming SmoothedRtt = 0; RttVariation = 0; RetransmitTimeout = MINIMUM_RTO; + RetransmitEpochAt = 0; + RetransmittedSinceLastSample = false; } @@ -55,6 +57,7 @@ namespace NetTiming std::uint64_t const variation = std::max(1, 4ull * RttVariation); RetransmitTimeout = Clamp_Rto(static_cast(SmoothedRtt) + variation); + RetransmittedSinceLastSample = false; return(true); } @@ -69,6 +72,33 @@ namespace NetTiming } + /// Backs the timeout off once per retransmission era so a slower link stays measurable. + void RttEstimator::Note_Retransmit(Milliseconds captured_rto, Milliseconds now) + { + if (!Initialized) { + return; + } + + if (!RetransmittedSinceLastSample) { + RetransmittedSinceLastSample = true; + RetransmitEpochAt = now; + } + + // Only a packet sent under the current timeout proves that timeout too short. + if (captured_rto >= RetransmitTimeout) { + RetransmitTimeout = Clamp_Rto(2ull * RetransmitTimeout); + } + } + + + /// Reports whether the estimate still reflects a measured acknowledgement. + bool RttEstimator::Has_Fresh_Sample(Milliseconds now) const + { + return(Initialized && (!RetransmittedSinceLastSample + || Elapsed_Milliseconds(RetransmitEpochAt, now) < RTT_SAMPLE_LIFETIME)); + } + + /// Derives the connection timeout from smoothed latency. Milliseconds Connection_Timeout(Milliseconds smoothed_rtt) { diff --git a/code/nettiming.h b/code/nettiming.h index b7c178647..4974572c6 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -19,6 +19,9 @@ namespace NetTiming constexpr Milliseconds MAXIMUM_RTO = 2000; constexpr Milliseconds MINIMUM_CONNECTION_TIMEOUT = 2000; constexpr Milliseconds MAXIMUM_CONNECTION_TIMEOUT = 30000; + // Long enough for the backoff ladder to climb from the minimum to the maximum RTO and + // still measure a clean acknowledgement, so a recovering link is not reported stale. + constexpr Milliseconds RTT_SAMPLE_LIFETIME = 8000; enum class RetryDecision { @@ -41,8 +44,10 @@ namespace NetTiming void Reset(void); bool Add_Sample(Milliseconds round_trip, bool retransmitted = false); bool Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock = Default_Clock()); + void Note_Retransmit(Milliseconds captured_rto, Milliseconds now); bool Has_Sample(void) const {return(Initialized);} + bool Has_Fresh_Sample(Milliseconds now) const; Milliseconds Smoothed_Rtt(void) const {return(SmoothedRtt);} Milliseconds Rtt_Variation(void) const {return(RttVariation);} Milliseconds Retransmit_Timeout(void) const {return(RetransmitTimeout);} @@ -52,6 +57,9 @@ namespace NetTiming Milliseconds SmoothedRtt = 0; Milliseconds RttVariation = 0; Milliseconds RetransmitTimeout = MINIMUM_RTO; + // Start of the current stretch of retransmissions without an eligible sample. + Milliseconds RetransmitEpochAt = 0; + bool RetransmittedSinceLastSample = false; }; Milliseconds Connection_Timeout(Milliseconds smoothed_rtt); diff --git a/manual/changes/network-transport-timing.md b/manual/changes/network-transport-timing.md index 928abd2d3..1c952e706 100644 --- a/manual/changes/network-transport-timing.md +++ b/manual/changes/network-transport-timing.md @@ -8,5 +8,7 @@ credit: --- Each private connection now estimates its own round trip and backs off repeated -transmissions. Lobby traffic keeps its fixed retry cadence. Packet layouts, -event IDs, and configuration remain unchanged. +transmissions. The retry timeout backs off with them, so a link whose latency +rises above it stays measurable instead of retransmitting every packet. Lobby +traffic keeps its fixed retry cadence. Packet layouts, event IDs, and +configuration remain unchanged. diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index f3193f60d..dfb7b4059 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -13,3 +13,12 @@ The retry timeout is limited to 100–2000 ms. Repeated private transmissions double their wait up to the connection timeout; that timeout follows measured latency with a 2-second minimum and 30-second ceiling. With no measurement, the bounded legacy timing is used. Global lobby traffic retains its fixed cadence. + +A link that is retransmitting also doubles the timeout it measures against, once +per retransmission proven against the current value. This keeps a link whose +latency has risen above its timeout measurable, because every packet would +otherwise be retransmitted before its acknowledgement arrived and no +unambiguous sample could be taken. The next clean acknowledgement recomputes the +timeout from the measured latency. An estimate that has gone 8 seconds of +retransmissions without such an acknowledgement is treated as stale while still +pacing retries. diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index f03019b91..af2282b35 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -49,6 +49,7 @@ namespace } LastSend = now; TransmissionCount++; + Estimator.Note_Retransmit(BaseRto, now); return(true); } @@ -59,6 +60,7 @@ namespace } NetTiming::RttEstimator const & Rtt(void) const {return(Estimator);} + NetTiming::Milliseconds Base_Rto(void) const {return(BaseRto);} private: FakeClock Clock; @@ -236,6 +238,105 @@ namespace Expect("fake transport applies Karn after loss", !lossy_transport.Acknowledge(1180)); Expect("lossy fake transport has no ambiguous RTT sample", !lossy_transport.Rtt().Has_Sample()); } + + + void Test_Backoff_Persistence(void) + { + using namespace NetTiming; + + FakeTransport transport; + transport.Send(0); + Expect("fast link samples cleanly", transport.Acknowledge(10)); + Expect_Equal("fast link floors the RTO", transport.Rtt().Retransmit_Timeout(), MINIMUM_RTO); + + // The link now takes 500 ms, so every packet is retransmitted before its ACK arrives. + transport.Send(1000); + Expect("first era retransmits at the floor RTO", transport.Retry(1100)); + Expect_Equal("first era doubles the RTO", transport.Rtt().Retransmit_Timeout(), 200u); + Expect("same era retransmits again", transport.Retry(1300)); + Expect_Equal("same era does not double twice", transport.Rtt().Retransmit_Timeout(), 200u); + Expect("ambiguous ACK is excluded", !transport.Acknowledge(1500)); + + transport.Send(2000); + Expect_Equal("new packet captures the backed off RTO", transport.Base_Rto(), 200u); + Expect("second era retransmits", transport.Retry(2200)); + Expect_Equal("second era doubles the RTO", transport.Rtt().Retransmit_Timeout(), 400u); + + transport.Send(3000); + Expect_Equal("third packet captures the backed off RTO", transport.Base_Rto(), 400u); + Expect("third era retransmits", transport.Retry(3400)); + Expect_Equal("third era doubles the RTO", transport.Rtt().Retransmit_Timeout(), 800u); + + // The RTO now exceeds the real round trip, so a first transmission is acknowledged. + transport.Send(4000); + Expect("backed off RTO lets a clean sample through", transport.Acknowledge(4500)); + Expect_Equal("recovered smoothed RTT", transport.Rtt().Smoothed_Rtt(), 71u); + Expect_Equal("recovered variation", transport.Rtt().Rtt_Variation(), 126u); + Expect_Equal("recovered RTO covers the slower link", transport.Rtt().Retransmit_Timeout(), 575u); + Expect("recovered estimate is fresh", transport.Rtt().Has_Fresh_Sample(4500)); + } + + + void Test_Sample_Staleness(void) + { + using namespace NetTiming; + + RttEstimator idle; + Expect("unsampled estimator is never fresh", !idle.Has_Fresh_Sample(0)); + idle.Add_Sample(100); + Expect("quiet link stays fresh indefinitely", idle.Has_Fresh_Sample(1000000)); + + RttEstimator starved; + starved.Add_Sample(100); + starved.Note_Retransmit(starved.Retransmit_Timeout(), 1000); + Expect("estimate is fresh before the lifetime", starved.Has_Fresh_Sample(1000 + RTT_SAMPLE_LIFETIME - 1)); + Expect("estimate is stale at the lifetime", !starved.Has_Fresh_Sample(1000 + RTT_SAMPLE_LIFETIME)); + Expect("stale estimate still paces retries", starved.Has_Sample()); + + FakeClock clock; + clock.Set(9500); + Expect("clean ACK restores the estimate", starved.Acknowledge(9400, 1, clock)); + Expect("restored estimate is fresh again", starved.Has_Fresh_Sample(1000000)); + + starved.Reset(); + Expect("reset clears freshness", !starved.Has_Fresh_Sample(0)); + + RttEstimator wrapped; + wrapped.Add_Sample(100); + wrapped.Note_Retransmit(wrapped.Retransmit_Timeout(), 0xfffff000u); + Expect("freshness survives the clock wrap", wrapped.Has_Fresh_Sample(0)); + Expect("staleness is measured across the wrap", !wrapped.Has_Fresh_Sample(0x00001000u)); + } + + + void Test_Note_Retransmit_Guards(void) + { + using namespace NetTiming; + + RttEstimator unsampled; + unsampled.Note_Retransmit(MINIMUM_RTO, 1000); + Expect("retransmission does not invent a sample", !unsampled.Has_Sample()); + Expect_Equal("unsampled RTO is unchanged", unsampled.Retransmit_Timeout(), MINIMUM_RTO); + + RttEstimator ceiling; + ceiling.Add_Sample(300); + Expect_Equal("sampled RTO", ceiling.Retransmit_Timeout(), 900u); + ceiling.Note_Retransmit(900, 1000); + Expect_Equal("backoff doubles below the ceiling", ceiling.Retransmit_Timeout(), 1800u); + ceiling.Note_Retransmit(1800, 2000); + Expect_Equal("backoff clamps at the ceiling", ceiling.Retransmit_Timeout(), MAXIMUM_RTO); + ceiling.Note_Retransmit(MAXIMUM_RTO, 3000); + Expect_Equal("backoff stays at the ceiling", ceiling.Retransmit_Timeout(), MAXIMUM_RTO); + + // A packet captured during backoff can double a freshly lowered RTO once. + RttEstimator recovered; + recovered.Add_Sample(300); + recovered.Note_Retransmit(900, 1000); + recovered.Add_Sample(300); + Expect_Equal("clean sample lowers the RTO", recovered.Retransmit_Timeout(), 752u); + recovered.Note_Retransmit(1800, 2000); + Expect_Equal("stale capture doubles the RTO once", recovered.Retransmit_Timeout(), 1504u); + } } @@ -246,6 +347,9 @@ int main(void) Test_Retransmit_Backoff(); Test_Retry_Decisions(); Test_Loss_Jitter_And_Reordering(); + Test_Backoff_Persistence(); + Test_Sample_Staleness(); + Test_Note_Retransmit_Guards(); if (Failures != 0) { std::cerr << Failures << " network timing checks failed\n"; From 5f57fe280fdddfd8e5469c19317a1dc1be7b3d67 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Wed, 2 Sep 2026 10:50:14 +0300 Subject: [PATCH 04/11] Keep retransmitting private packets past the connection timeout --- code/connect.cpp | 13 +++----- code/connect.h | 2 ++ code/ipxmgr.cpp | 5 ++- code/nettiming.cpp | 13 ++++---- code/nettiming.h | 7 ++-- .../systems/network-transport-timing.md | 7 ++-- tests/nettiming/nettiming.cpp | 33 +++++++++++++------ 7 files changed, 48 insertions(+), 32 deletions(-) diff --git a/code/connect.cpp b/code/connect.cpp index 120c67620..1b889071b 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -218,6 +218,7 @@ void ConnectionClass::Init (void) LastSeqID = 0xffffffff; LastReadID = 0xffffffff; RoundTripEstimator.Reset(); + IsBad = false; Queue->Init(); @@ -746,11 +747,8 @@ int ConnectionClass::Service (void) been ACK'd yet. Entries that the app has read, and have been ACK'd, should be removed. ------------------------------------------------------------------------*/ - if ( Service_Send_Queue() && Service_Receive_Queue() ) { - return(1); - } else { - return(0); - } + IsBad = !(Service_Send_Queue() && Service_Receive_Queue()); + return(IsBad ? 0 : 1); } /* end of Service */ @@ -846,12 +844,11 @@ int ConnectionClass::Service_Send_Queue (void) }; NetTiming::RetryDecision const retry_decision = NetTiming::Evaluate_Retry( retransmit_state, current_milliseconds, base_retry_timeout, connection_timeout, timeout_enabled, adaptive_channel); - if (retry_decision == NetTiming::RetryDecision::TIMED_OUT) { + if (retry_decision.TimedOut) { bad_conn = 1; send_entry->IsUndeliverable = true; - continue; } - if (retry_decision == NetTiming::RetryDecision::SEND) { + if (retry_decision.Send) { /*.................................................................. Send the message diff --git a/code/connect.h b/code/connect.h index 99fbedcd3..d7fd47c44 100644 --- a/code/connect.h +++ b/code/connect.h @@ -193,6 +193,7 @@ class ConnectionClass int Percent_Lost(void) const { return(PercentLost); } int Missed_Overall(void) const { return(MissedOverall); } int Missed_Magic(void) const { return(MissedMagic); } + bool Is_Bad(void) const { return(IsBad); } enum PacketDropReasonType { CONNECTION_DROP_SHORT_HEADER, @@ -297,6 +298,7 @@ class ConnectionClass // An injected clock must outlive the connection. NetTiming::MillisecondClock const *MillisecondTime; NetTiming::RttEstimator RoundTripEstimator; + bool IsBad = false; /*..................................................................... Running totals of # of packets we send & receive which require an ACK, diff --git a/code/ipxmgr.cpp b/code/ipxmgr.cpp index 78e09c3b9..57fb524ac 100644 --- a/code/ipxmgr.cpp +++ b/code/ipxmgr.cpp @@ -1066,10 +1066,13 @@ int IPXManagerClass::Service(void) } } for (i = 0; i < NumConnections; i++) { + bool const was_bad = Connection[i]->Is_Bad(); if (!Connection[i]->Service()) { rc = 0; BadConnection = Connection[i]->ID; - DebugString("Error - Connection %d has gone bad\n", BadConnection); + if (!was_bad) { + DebugString("Error - Connection %d has gone bad\n", BadConnection); + } } } diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 28d54c0a3..7379414d5 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -126,20 +126,19 @@ namespace NetTiming } - /// Chooses the next action for one queued packet without changing its timing state. + /// Chooses the next action for one queued packet; a timed-out packet still retries at its capped backoff. RetryDecision Evaluate_Retry(RetransmitState const & state, Milliseconds now, Milliseconds current_rto, Milliseconds connection_timeout, bool timeout_enabled, bool adaptive) { if (state.TransmissionCount == 0) { - return(RetryDecision::SEND); - } - if (timeout_enabled && Milliseconds_Have_Elapsed(state.FirstSend, now, connection_timeout)) { - return(RetryDecision::TIMED_OUT); + return(RetryDecision{true, false}); } - bool const retry_due = adaptive + RetryDecision decision; + decision.TimedOut = timeout_enabled && Milliseconds_Have_Elapsed(state.FirstSend, now, connection_timeout); + decision.Send = adaptive ? Retransmit_Is_Due(state.LastSend, now, state.CapturedRto, state.TransmissionCount - 1, connection_timeout) : Milliseconds_Have_Elapsed(state.LastSend, now, current_rto); - return(retry_due ? RetryDecision::SEND : RetryDecision::WAIT); + return(decision); } } diff --git a/code/nettiming.h b/code/nettiming.h index 4974572c6..113b09f15 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -23,11 +23,10 @@ namespace NetTiming // still measure a clean acknowledgement, so a recovering link is not reported stale. constexpr Milliseconds RTT_SAMPLE_LIFETIME = 8000; - enum class RetryDecision + struct RetryDecision { - WAIT, - SEND, - TIMED_OUT + bool Send = false; + bool TimedOut = false; }; struct RetransmitState diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index dfb7b4059..7a2d49d58 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -11,8 +11,11 @@ ambiguous measurements after a retry. The retry timeout is limited to 100–2000 ms. Repeated private transmissions double their wait up to the connection timeout; that timeout follows measured -latency with a 2-second minimum and 30-second ceiling. With no measurement, the -bounded legacy timing is used. Global lobby traffic retains its fixed cadence. +latency with a 2-second minimum and 30-second ceiling. A packet older than that +timeout marks the connection bad but is still retransmitted at the capped wait +until it is acknowledged, so a link that recovers drains its backlog. With no +measurement, the bounded legacy timing is used. Global lobby traffic retains its +fixed cadence. A link that is retransmitting also doubles the timeout it measures against, once per retransmission proven against the current value. This keeps a link whose diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index af2282b35..fb411a71f 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -175,26 +175,39 @@ namespace using namespace NetTiming; RetransmitState state; - Expect("new packet sends immediately", Evaluate_Retry(state, 1000, 800, 2000, true, true) == RetryDecision::SEND); + RetryDecision decision = Evaluate_Retry(state, 1000, 800, 2000, true, true); + Expect("new packet sends immediately", decision.Send && !decision.TimedOut); state = {1000, 1000, 100, 1}; - Expect("adaptive packet keeps captured RTO", Evaluate_Retry(state, 1099, 800, 2000, true, true) == RetryDecision::WAIT); - Expect("adaptive packet sends at captured RTO", Evaluate_Retry(state, 1100, 800, 2000, true, true) == RetryDecision::SEND); + Expect("adaptive packet keeps captured RTO", !Evaluate_Retry(state, 1099, 800, 2000, true, true).Send); + Expect("adaptive packet sends at captured RTO", Evaluate_Retry(state, 1100, 800, 2000, true, true).Send); state = {1000, 1100, 100, 2}; - Expect("adaptive retry waits through backoff", Evaluate_Retry(state, 1299, 800, 2000, true, true) == RetryDecision::WAIT); - Expect("adaptive retry sends after backoff", Evaluate_Retry(state, 1300, 800, 2000, true, true) == RetryDecision::SEND); + Expect("adaptive retry waits through backoff", !Evaluate_Retry(state, 1299, 800, 2000, true, true).Send); + Expect("adaptive retry sends after backoff", Evaluate_Retry(state, 1300, 800, 2000, true, true).Send); state = {1000, 1000, 100, 4}; - Expect("fixed channel uses current retry delay", Evaluate_Retry(state, 1399, 400, 2000, true, false) == RetryDecision::WAIT); - Expect("fixed channel does not back off", Evaluate_Retry(state, 1400, 400, 2000, true, false) == RetryDecision::SEND); + Expect("fixed channel uses current retry delay", !Evaluate_Retry(state, 1399, 400, 2000, true, false).Send); + decision = Evaluate_Retry(state, 1400, 400, 2000, true, false); + Expect("fixed channel does not back off", decision.Send && !decision.TimedOut); state = {1000, 1900, 100, 1}; - Expect("connection timeout wins over retry", Evaluate_Retry(state, 3000, 100, 2000, true, true) == RetryDecision::TIMED_OUT); - Expect("disabled connection timeout still retries", Evaluate_Retry(state, 3000, 100, 2000, false, true) == RetryDecision::SEND); + decision = Evaluate_Retry(state, 3000, 100, 2000, true, true); + Expect("connection timeout flags the link", decision.TimedOut); + Expect("timed-out packet still retries when due", decision.Send); + decision = Evaluate_Retry(state, 3000, 100, 2000, false, true); + Expect("disabled connection timeout still retries", !decision.TimedOut && decision.Send); + + state = {1000, 2950, 100, 1}; + decision = Evaluate_Retry(state, 3000, 100, 2000, true, true); + Expect("timed-out packet waits for its backoff", decision.TimedOut && !decision.Send); + + state = {1000, 5000, 100, 6}; + Expect("timed-out packet waits for the connection timeout cap", !Evaluate_Retry(state, 6999, 100, 2000, true, true).Send); + Expect("timed-out packet retries at the connection timeout cap", Evaluate_Retry(state, 7000, 100, 2000, true, true).Send); state = {0xffffff00u, 0xfffffff0u, 100, 1}; - Expect("retry decision handles clock wrap", Evaluate_Retry(state, 0x00000054u, 800, 2000, true, true) == RetryDecision::SEND); + Expect("retry decision handles clock wrap", Evaluate_Retry(state, 0x00000054u, 800, 2000, true, true).Send); } From 3700caeffb2c7cc4feb14cf619e71a75fc8ef578 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Wed, 2 Sep 2026 10:53:08 +0300 Subject: [PATCH 05/11] Drop RTT sample aging from the retransmission estimator --- code/connect.cpp | 2 +- code/nettiming.cpp | 18 +------ code/nettiming.h | 9 +--- .../systems/network-transport-timing.md | 5 +- tests/nettiming/nettiming.cpp | 49 +++---------------- 5 files changed, 13 insertions(+), 70 deletions(-) diff --git a/code/connect.cpp b/code/connect.cpp index 1b889071b..f172cd7f0 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -878,7 +878,7 @@ int ConnectionClass::Service_Send_Queue (void) } else { NumResends++; if (adaptive_channel) { - RoundTripEstimator.Note_Retransmit(send_entry->RetransmitTimeoutMilliseconds, current_milliseconds); + RoundTripEstimator.Note_Retransmit(send_entry->RetransmitTimeoutMilliseconds); } } diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 7379414d5..0115a5efa 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -32,8 +32,6 @@ namespace NetTiming SmoothedRtt = 0; RttVariation = 0; RetransmitTimeout = MINIMUM_RTO; - RetransmitEpochAt = 0; - RetransmittedSinceLastSample = false; } @@ -57,7 +55,6 @@ namespace NetTiming std::uint64_t const variation = std::max(1, 4ull * RttVariation); RetransmitTimeout = Clamp_Rto(static_cast(SmoothedRtt) + variation); - RetransmittedSinceLastSample = false; return(true); } @@ -73,17 +70,12 @@ namespace NetTiming /// Backs the timeout off once per retransmission era so a slower link stays measurable. - void RttEstimator::Note_Retransmit(Milliseconds captured_rto, Milliseconds now) + void RttEstimator::Note_Retransmit(Milliseconds captured_rto) { if (!Initialized) { return; } - if (!RetransmittedSinceLastSample) { - RetransmittedSinceLastSample = true; - RetransmitEpochAt = now; - } - // Only a packet sent under the current timeout proves that timeout too short. if (captured_rto >= RetransmitTimeout) { RetransmitTimeout = Clamp_Rto(2ull * RetransmitTimeout); @@ -91,14 +83,6 @@ namespace NetTiming } - /// Reports whether the estimate still reflects a measured acknowledgement. - bool RttEstimator::Has_Fresh_Sample(Milliseconds now) const - { - return(Initialized && (!RetransmittedSinceLastSample - || Elapsed_Milliseconds(RetransmitEpochAt, now) < RTT_SAMPLE_LIFETIME)); - } - - /// Derives the connection timeout from smoothed latency. Milliseconds Connection_Timeout(Milliseconds smoothed_rtt) { diff --git a/code/nettiming.h b/code/nettiming.h index 113b09f15..1621eadf9 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -19,9 +19,6 @@ namespace NetTiming constexpr Milliseconds MAXIMUM_RTO = 2000; constexpr Milliseconds MINIMUM_CONNECTION_TIMEOUT = 2000; constexpr Milliseconds MAXIMUM_CONNECTION_TIMEOUT = 30000; - // Long enough for the backoff ladder to climb from the minimum to the maximum RTO and - // still measure a clean acknowledgement, so a recovering link is not reported stale. - constexpr Milliseconds RTT_SAMPLE_LIFETIME = 8000; struct RetryDecision { @@ -43,10 +40,9 @@ namespace NetTiming void Reset(void); bool Add_Sample(Milliseconds round_trip, bool retransmitted = false); bool Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock = Default_Clock()); - void Note_Retransmit(Milliseconds captured_rto, Milliseconds now); + void Note_Retransmit(Milliseconds captured_rto); bool Has_Sample(void) const {return(Initialized);} - bool Has_Fresh_Sample(Milliseconds now) const; Milliseconds Smoothed_Rtt(void) const {return(SmoothedRtt);} Milliseconds Rtt_Variation(void) const {return(RttVariation);} Milliseconds Retransmit_Timeout(void) const {return(RetransmitTimeout);} @@ -56,9 +52,6 @@ namespace NetTiming Milliseconds SmoothedRtt = 0; Milliseconds RttVariation = 0; Milliseconds RetransmitTimeout = MINIMUM_RTO; - // Start of the current stretch of retransmissions without an eligible sample. - Milliseconds RetransmitEpochAt = 0; - bool RetransmittedSinceLastSample = false; }; Milliseconds Connection_Timeout(Milliseconds smoothed_rtt); diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index 7a2d49d58..010df04d4 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -22,6 +22,5 @@ per retransmission proven against the current value. This keeps a link whose latency has risen above its timeout measurable, because every packet would otherwise be retransmitted before its acknowledgement arrived and no unambiguous sample could be taken. The next clean acknowledgement recomputes the -timeout from the measured latency. An estimate that has gone 8 seconds of -retransmissions without such an acknowledgement is treated as stale while still -pacing retries. +timeout from the measured latency; until then the estimate keeps its last +measured value while pacing retries. diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index fb411a71f..5dfa613ad 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -49,7 +49,7 @@ namespace } LastSend = now; TransmissionCount++; - Estimator.Note_Retransmit(BaseRto, now); + Estimator.Note_Retransmit(BaseRto); return(true); } @@ -286,39 +286,7 @@ namespace Expect_Equal("recovered smoothed RTT", transport.Rtt().Smoothed_Rtt(), 71u); Expect_Equal("recovered variation", transport.Rtt().Rtt_Variation(), 126u); Expect_Equal("recovered RTO covers the slower link", transport.Rtt().Retransmit_Timeout(), 575u); - Expect("recovered estimate is fresh", transport.Rtt().Has_Fresh_Sample(4500)); - } - - - void Test_Sample_Staleness(void) - { - using namespace NetTiming; - - RttEstimator idle; - Expect("unsampled estimator is never fresh", !idle.Has_Fresh_Sample(0)); - idle.Add_Sample(100); - Expect("quiet link stays fresh indefinitely", idle.Has_Fresh_Sample(1000000)); - - RttEstimator starved; - starved.Add_Sample(100); - starved.Note_Retransmit(starved.Retransmit_Timeout(), 1000); - Expect("estimate is fresh before the lifetime", starved.Has_Fresh_Sample(1000 + RTT_SAMPLE_LIFETIME - 1)); - Expect("estimate is stale at the lifetime", !starved.Has_Fresh_Sample(1000 + RTT_SAMPLE_LIFETIME)); - Expect("stale estimate still paces retries", starved.Has_Sample()); - - FakeClock clock; - clock.Set(9500); - Expect("clean ACK restores the estimate", starved.Acknowledge(9400, 1, clock)); - Expect("restored estimate is fresh again", starved.Has_Fresh_Sample(1000000)); - - starved.Reset(); - Expect("reset clears freshness", !starved.Has_Fresh_Sample(0)); - - RttEstimator wrapped; - wrapped.Add_Sample(100); - wrapped.Note_Retransmit(wrapped.Retransmit_Timeout(), 0xfffff000u); - Expect("freshness survives the clock wrap", wrapped.Has_Fresh_Sample(0)); - Expect("staleness is measured across the wrap", !wrapped.Has_Fresh_Sample(0x00001000u)); + Expect("recovered estimate is measured", transport.Rtt().Has_Sample()); } @@ -327,27 +295,27 @@ namespace using namespace NetTiming; RttEstimator unsampled; - unsampled.Note_Retransmit(MINIMUM_RTO, 1000); + unsampled.Note_Retransmit(MINIMUM_RTO); Expect("retransmission does not invent a sample", !unsampled.Has_Sample()); Expect_Equal("unsampled RTO is unchanged", unsampled.Retransmit_Timeout(), MINIMUM_RTO); RttEstimator ceiling; ceiling.Add_Sample(300); Expect_Equal("sampled RTO", ceiling.Retransmit_Timeout(), 900u); - ceiling.Note_Retransmit(900, 1000); + ceiling.Note_Retransmit(900); Expect_Equal("backoff doubles below the ceiling", ceiling.Retransmit_Timeout(), 1800u); - ceiling.Note_Retransmit(1800, 2000); + ceiling.Note_Retransmit(1800); Expect_Equal("backoff clamps at the ceiling", ceiling.Retransmit_Timeout(), MAXIMUM_RTO); - ceiling.Note_Retransmit(MAXIMUM_RTO, 3000); + ceiling.Note_Retransmit(MAXIMUM_RTO); Expect_Equal("backoff stays at the ceiling", ceiling.Retransmit_Timeout(), MAXIMUM_RTO); // A packet captured during backoff can double a freshly lowered RTO once. RttEstimator recovered; recovered.Add_Sample(300); - recovered.Note_Retransmit(900, 1000); + recovered.Note_Retransmit(900); recovered.Add_Sample(300); Expect_Equal("clean sample lowers the RTO", recovered.Retransmit_Timeout(), 752u); - recovered.Note_Retransmit(1800, 2000); + recovered.Note_Retransmit(1800); Expect_Equal("stale capture doubles the RTO once", recovered.Retransmit_Timeout(), 1504u); } } @@ -361,7 +329,6 @@ int main(void) Test_Retry_Decisions(); Test_Loss_Jitter_And_Reordering(); Test_Backoff_Persistence(); - Test_Sample_Staleness(); Test_Note_Retransmit_Guards(); if (Failures != 0) { From 159ef228be4e4f2f7ece6839e31841479edb2308 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Wed, 2 Sep 2026 10:57:09 +0300 Subject: [PATCH 06/11] Seed the RTT estimate from an ambiguous first acknowledgement --- code/nettiming.cpp | 22 +++++++- code/nettiming.h | 7 ++- manual/changes/network-transport-timing.md | 8 ++- .../systems/network-transport-timing.md | 8 ++- tests/nettiming/nettiming.cpp | 55 ++++++++++++++++++- 5 files changed, 87 insertions(+), 13 deletions(-) diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 0115a5efa..8f1cb9f43 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -32,6 +32,7 @@ namespace NetTiming SmoothedRtt = 0; RttVariation = 0; RetransmitTimeout = MINIMUM_RTO; + Provisional = false; } @@ -59,13 +60,28 @@ namespace NetTiming } - /// Samples an acknowledgement when its send time is unambiguous. + /// Samples an acknowledgement; an unmeasured link takes an ambiguous one as a provisional upper bound. bool RttEstimator::Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock) { - if (transmission_count != 1) { + if (transmission_count == 0) { return(false); } - return(Add_Sample(Elapsed_Milliseconds(sent_at, clock.Now()))); + + Milliseconds const elapsed = Elapsed_Milliseconds(sent_at, clock.Now()); + if (transmission_count != 1) { + if (Initialized) { + return(false); + } + Provisional = Add_Sample(elapsed); + return(Provisional); + } + + // The first clean sample replaces a provisional seed instead of blending with it. + if (Provisional) { + Initialized = false; + Provisional = false; + } + return(Add_Sample(elapsed)); } diff --git a/code/nettiming.h b/code/nettiming.h index 1621eadf9..cb473aad6 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -16,7 +16,9 @@ namespace NetTiming { constexpr Milliseconds MINIMUM_RTO = 100; - constexpr Milliseconds MAXIMUM_RTO = 2000; + // Above any round trip a private link is expected to carry, so a slow link's first retry + // does not precede its acknowledgement. + constexpr Milliseconds MAXIMUM_RTO = 4000; constexpr Milliseconds MINIMUM_CONNECTION_TIMEOUT = 2000; constexpr Milliseconds MAXIMUM_CONNECTION_TIMEOUT = 30000; @@ -43,6 +45,7 @@ namespace NetTiming void Note_Retransmit(Milliseconds captured_rto); bool Has_Sample(void) const {return(Initialized);} + bool Is_Provisional(void) const {return(Provisional);} Milliseconds Smoothed_Rtt(void) const {return(SmoothedRtt);} Milliseconds Rtt_Variation(void) const {return(RttVariation);} Milliseconds Retransmit_Timeout(void) const {return(RetransmitTimeout);} @@ -52,6 +55,8 @@ namespace NetTiming Milliseconds SmoothedRtt = 0; Milliseconds RttVariation = 0; Milliseconds RetransmitTimeout = MINIMUM_RTO; + // Set while the estimate comes from an ambiguous first acknowledgement. + bool Provisional = false; }; Milliseconds Connection_Timeout(Milliseconds smoothed_rtt); diff --git a/manual/changes/network-transport-timing.md b/manual/changes/network-transport-timing.md index 1c952e706..1959de0eb 100644 --- a/manual/changes/network-transport-timing.md +++ b/manual/changes/network-transport-timing.md @@ -9,6 +9,8 @@ credit: Each private connection now estimates its own round trip and backs off repeated transmissions. The retry timeout backs off with them, so a link whose latency -rises above it stays measurable instead of retransmitting every packet. Lobby -traffic keeps its fixed retry cadence. Packet layouts, event IDs, and -configuration remain unchanged. +rises above it stays measurable instead of retransmitting every packet. A link +slower than the initial retry delay is still measured, and a packet that +outlives the connection timeout keeps retransmitting instead of blocking the +link. Lobby traffic keeps its fixed retry cadence. Packet layouts, event IDs, +and configuration remain unchanged. diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index 010df04d4..9341ac115 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -6,10 +6,12 @@ keys: [] --- Each private connection maintains smoothed round trip, variation, and a retry -timeout. Only acknowledgements for first transmissions become samples, avoiding -ambiguous measurements after a retry. +timeout. Acknowledgements of first transmissions are the measurements. Until a +link has one, its first acknowledgement seeds a provisional estimate even after +a retry, so a link slower than the initial retry delay becomes measurable; the +first clean acknowledgement replaces the seed. -The retry timeout is limited to 100–2000 ms. Repeated private transmissions +The retry timeout is limited to 100–4000 ms. Repeated private transmissions double their wait up to the connection timeout; that timeout follows measured latency with a 2-second minimum and 30-second ceiling. A packet older than that timeout marks the connection bad but is still retransmitted at the capped wait diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index 5dfa613ad..b16782e09 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -248,8 +248,15 @@ namespace FakeTransport lossy_transport; lossy_transport.Send(1000); Expect("fake transport retries a lost packet", lossy_transport.Retry(1100)); - Expect("fake transport applies Karn after loss", !lossy_transport.Acknowledge(1180)); - Expect("lossy fake transport has no ambiguous RTT sample", !lossy_transport.Rtt().Has_Sample()); + Expect("ambiguous first ACK seeds a provisional sample", lossy_transport.Acknowledge(1180)); + Expect("provisional seed is the elapsed upper bound", lossy_transport.Rtt().Smoothed_Rtt() == 180u && lossy_transport.Rtt().Is_Provisional()); + lossy_transport.Send(2000); + Expect_Equal("provisional seed paces the next packet", lossy_transport.Base_Rto(), 540u); + Expect("clean sample replaces the provisional seed", lossy_transport.Acknowledge(2080)); + Expect_Equal("replaced smoothed RTT", lossy_transport.Rtt().Smoothed_Rtt(), 80u); + Expect_Equal("replaced variation", lossy_transport.Rtt().Rtt_Variation(), 40u); + Expect_Equal("replaced RTO", lossy_transport.Rtt().Retransmit_Timeout(), 240u); + Expect("replaced estimate is measured", !lossy_transport.Rtt().Is_Provisional()); } @@ -286,7 +293,46 @@ namespace Expect_Equal("recovered smoothed RTT", transport.Rtt().Smoothed_Rtt(), 71u); Expect_Equal("recovered variation", transport.Rtt().Rtt_Variation(), 126u); Expect_Equal("recovered RTO covers the slower link", transport.Rtt().Retransmit_Timeout(), 575u); - Expect("recovered estimate is measured", transport.Rtt().Has_Sample()); + Expect("recovered estimate is measured", transport.Rtt().Has_Sample() && !transport.Rtt().Is_Provisional()); + } + + + void Test_Provisional_Seed(void) + { + using namespace NetTiming; + + FakeClock clock; + RttEstimator estimator; + Expect("unsent packet never samples", !estimator.Acknowledge(0, 0, clock)); + Expect("unsent acknowledgement leaves the estimator empty", !estimator.Has_Sample()); + + clock.Set(2000); + Expect("two-second link seeds through a retransmitted ACK", estimator.Acknowledge(0, 2, clock)); + Expect("seed is provisional", estimator.Has_Sample() && estimator.Is_Provisional()); + Expect_Equal("seed smoothed RTT", estimator.Smoothed_Rtt(), 2000u); + Expect_Equal("seed RTO reaches the ceiling", estimator.Retransmit_Timeout(), MAXIMUM_RTO); + + clock.Set(4500); + Expect("second ambiguous ACK does not move a provisional seed", !estimator.Acknowledge(1000, 3, clock)); + Expect_Equal("seed unchanged by a second ambiguous ACK", estimator.Smoothed_Rtt(), 2000u); + + clock.Set(6900); + Expect("clean sample replaces the seed", estimator.Acknowledge(5000, 1, clock)); + Expect_Equal("clean sample replaces rather than blends", estimator.Smoothed_Rtt(), 1900u); + Expect("replaced seed is measured", !estimator.Is_Provisional()); + + clock.Set(9000); + Expect("Karn applies once the estimate is measured", !estimator.Acknowledge(7000, 2, clock)); + + RttEstimator backed_off; + clock.Set(300); + backed_off.Acknowledge(0, 2, clock); + Expect_Equal("provisional RTO", backed_off.Retransmit_Timeout(), 900u); + backed_off.Note_Retransmit(900); + Expect_Equal("provisional estimate backs off like a measured one", backed_off.Retransmit_Timeout(), 1800u); + + backed_off.Reset(); + Expect("reset clears the provisional flag", !backed_off.Is_Provisional() && !backed_off.Has_Sample()); } @@ -305,6 +351,8 @@ namespace ceiling.Note_Retransmit(900); Expect_Equal("backoff doubles below the ceiling", ceiling.Retransmit_Timeout(), 1800u); ceiling.Note_Retransmit(1800); + Expect_Equal("backoff doubles again below the ceiling", ceiling.Retransmit_Timeout(), 3600u); + ceiling.Note_Retransmit(3600); Expect_Equal("backoff clamps at the ceiling", ceiling.Retransmit_Timeout(), MAXIMUM_RTO); ceiling.Note_Retransmit(MAXIMUM_RTO); Expect_Equal("backoff stays at the ceiling", ceiling.Retransmit_Timeout(), MAXIMUM_RTO); @@ -329,6 +377,7 @@ int main(void) Test_Retry_Decisions(); Test_Loss_Jitter_And_Reordering(); Test_Backoff_Persistence(); + Test_Provisional_Seed(); Test_Note_Retransmit_Guards(); if (Failures != 0) { From cac13ffaa783e05a86c668c008c93f8f7967db70 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Wed, 2 Sep 2026 12:27:51 +0300 Subject: [PATCH 07/11] Bound the first retry so the timeout allows three transmissions --- code/connect.cpp | 4 +++- code/nettiming.cpp | 7 +++++++ code/nettiming.h | 1 + manual/content/systems/network-transport-timing.md | 11 ++++++----- tests/nettiming/nettiming.cpp | 6 ++++++ 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/code/connect.cpp b/code/connect.cpp index f172cd7f0..ccde7c016 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -827,7 +827,9 @@ int ConnectionClass::Service_Send_Queue (void) ? NetTiming::MAXIMUM_CONNECTION_TIMEOUT : (adaptive_timing ? NetTiming::Connection_Timeout(RoundTripEstimator.Smoothed_Rtt()) : (adaptive_channel ? Legacy_Connection_Timeout(Timeout) : Ticks_To_Milliseconds(Timeout))); - NetTiming::Milliseconds const base_retry_timeout = adaptive_timing ? RoundTripEstimator.Retransmit_Timeout() : Ticks_To_Milliseconds(RetryDelta); + NetTiming::Milliseconds const base_retry_timeout = adaptive_timing + ? NetTiming::Initial_Retry_Timeout(RoundTripEstimator.Retransmit_Timeout(), connection_timeout) + : Ticks_To_Milliseconds(RetryDelta); for (i = 0; i < num_entries; i++) { send_entry = Queue->Get_Send(i); diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 8f1cb9f43..3dee007f4 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -107,6 +107,13 @@ namespace NetTiming } + /// Bounds a packet's first retry so the connection timeout allows at least three transmissions. + Milliseconds Initial_Retry_Timeout(Milliseconds retransmit_timeout, Milliseconds connection_timeout) + { + return(std::max(MINIMUM_RTO, std::min(retransmit_timeout, connection_timeout / 4))); + } + + /// Applies bounded exponential backoff to a packet's RTO. Milliseconds Retransmit_Delay(Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay) { diff --git a/code/nettiming.h b/code/nettiming.h index cb473aad6..dcc141c9b 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -60,6 +60,7 @@ namespace NetTiming }; Milliseconds Connection_Timeout(Milliseconds smoothed_rtt); + Milliseconds Initial_Retry_Timeout(Milliseconds retransmit_timeout, Milliseconds connection_timeout); Milliseconds Retransmit_Delay(Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay = MAXIMUM_RTO); bool Retransmit_Is_Due(Milliseconds last_send, Milliseconds now, Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay = MAXIMUM_RTO); diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index 9341ac115..4dbfc84a3 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -13,11 +13,12 @@ first clean acknowledgement replaces the seed. The retry timeout is limited to 100–4000 ms. Repeated private transmissions double their wait up to the connection timeout; that timeout follows measured -latency with a 2-second minimum and 30-second ceiling. A packet older than that -timeout marks the connection bad but is still retransmitted at the capped wait -until it is acknowledged, so a link that recovers drains its backlog. With no -measurement, the bounded legacy timing is used. Global lobby traffic retains its -fixed cadence. +latency with a 2-second minimum and 30-second ceiling. A packet's first retry +waits at most a quarter of that timeout, so every packet is sent at least three +times before it. A packet older than the timeout marks the connection bad but is +still retransmitted at the capped wait until it is acknowledged, so a link that +recovers drains its backlog. With no measurement, the bounded legacy timing is +used. Global lobby traffic retains its fixed cadence. A link that is retransmitting also doubles the timeout it measures against, once per retransmission proven against the current value. This keeps a link whose diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index b16782e09..a791beb2a 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -167,6 +167,12 @@ namespace Expect_Equal("connection timeout follows RTT", Connection_Timeout(500), 4250u); Expect_Equal("connection timeout ceiling", Connection_Timeout(10000), 30000u); Expect_Equal("backoff reaches connection timeout", Retransmit_Delay(500, 8, 4250), 4250u); + + Expect_Equal("first retry keeps a small RTO", Initial_Retry_Timeout(300, 2000), 300u); + Expect_Equal("first retry is bounded by a quarter of the timeout", Initial_Retry_Timeout(1600, 2000), 500u); + Expect_Equal("first retry bound keeps the floor", Initial_Retry_Timeout(1600, 300), MINIMUM_RTO); + Expect_Equal("slow link keeps its RTO under a long timeout", Initial_Retry_Timeout(2055, Connection_Timeout(2015)), 2055u); + Expect_Equal("bounded first retry allows three sends before the timeout", Retransmit_Delay(500, 0) + Retransmit_Delay(500, 1), 1500u); } From 6de5100f468845a66f91c32c02ff4002004b08ab Mon Sep 17 00:00:00 2001 From: ZivDero Date: Wed, 2 Sep 2026 20:50:09 +0300 Subject: [PATCH 08/11] Seed the provisional RTT estimate from the last transmission --- code/connect.cpp | 3 +- code/nettiming.cpp | 19 ++++++-- code/nettiming.h | 3 +- .../systems/network-transport-timing.md | 6 ++- tests/nettiming/nettiming.cpp | 46 +++++++++++-------- 5 files changed, 48 insertions(+), 29 deletions(-) diff --git a/code/connect.cpp b/code/connect.cpp index ccde7c016..a4de8d88d 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -800,7 +800,8 @@ int ConnectionClass::Service_Send_Queue (void) if (header.Code == PACKET_DATA_ACK) { Queue->Add_Delay(Time() - send_entry->FirstTime); if (Adaptive_Timing_Enabled()) { - RoundTripEstimator.Acknowledge(send_entry->FirstTimeMilliseconds, send_entry->SendCount, *MillisecondTime); + RoundTripEstimator.Acknowledge(send_entry->FirstTimeMilliseconds, send_entry->LastTimeMilliseconds, send_entry->SendCount, + send_entry->RetransmitTimeoutMilliseconds, *MillisecondTime); } } } diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 3dee007f4..470a5ffbd 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -60,19 +60,28 @@ namespace NetTiming } - /// Samples an acknowledgement; an unmeasured link takes an ambiguous one as a provisional upper bound. - bool RttEstimator::Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock) + /// + /// Samples an acknowledgement. An unmeasured link takes an ambiguous one as a provisional seed: + /// the time since the packet's last transmission, with the retry delay it outlived as the floor + /// of the timeout. + /// + bool RttEstimator::Acknowledge(Milliseconds first_sent_at, Milliseconds last_sent_at, unsigned int transmission_count, Milliseconds retry_timeout, + MillisecondClock const & clock) { if (transmission_count == 0) { return(false); } - Milliseconds const elapsed = Elapsed_Milliseconds(sent_at, clock.Now()); + Milliseconds const now = clock.Now(); if (transmission_count != 1) { if (Initialized) { return(false); } - Provisional = Add_Sample(elapsed); + // The first transmission may predate the peer entirely; the last one bounds the round trip from below. + Provisional = Add_Sample(Elapsed_Milliseconds(last_sent_at, now)); + if (Provisional && RetransmitTimeout < retry_timeout) { + RetransmitTimeout = Clamp_Rto(retry_timeout); + } return(Provisional); } @@ -81,7 +90,7 @@ namespace NetTiming Initialized = false; Provisional = false; } - return(Add_Sample(elapsed)); + return(Add_Sample(Elapsed_Milliseconds(first_sent_at, now))); } diff --git a/code/nettiming.h b/code/nettiming.h index dcc141c9b..2e6e54a22 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -41,7 +41,8 @@ namespace NetTiming public: void Reset(void); bool Add_Sample(Milliseconds round_trip, bool retransmitted = false); - bool Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock = Default_Clock()); + bool Acknowledge(Milliseconds first_sent_at, Milliseconds last_sent_at, unsigned int transmission_count, Milliseconds retry_timeout, + MillisecondClock const & clock = Default_Clock()); void Note_Retransmit(Milliseconds captured_rto); bool Has_Sample(void) const {return(Initialized);} diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index 4dbfc84a3..b80d20a3b 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -8,8 +8,10 @@ keys: [] Each private connection maintains smoothed round trip, variation, and a retry timeout. Acknowledgements of first transmissions are the measurements. Until a link has one, its first acknowledgement seeds a provisional estimate even after -a retry, so a link slower than the initial retry delay becomes measurable; the -first clean acknowledgement replaces the seed. +a retry: the time since the packet's last transmission, paced no faster than the +retry delay that went unanswered. A link slower than the initial retry delay +therefore becomes measurable, and a peer that was still loading does not +inflate the seed. The first clean acknowledgement replaces it. The retry timeout is limited to 100–4000 ms. Repeated private transmissions double their wait up to the connection timeout; that timeout follows measured diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index a791beb2a..d7b4fe1f1 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -56,7 +56,7 @@ namespace bool Acknowledge(NetTiming::Milliseconds now) { Clock.Set(now); - return(Estimator.Acknowledge(FirstSend, TransmissionCount, Clock)); + return(Estimator.Acknowledge(FirstSend, LastSend, TransmissionCount, BaseRto, Clock)); } NetTiming::RttEstimator const & Rtt(void) const {return(Estimator);} @@ -143,9 +143,9 @@ namespace FakeClock clock; clock.Set(0x00000020u); RttEstimator estimator; - Expect("wrap sample accepted", estimator.Acknowledge(0xfffffff0u, 1, clock)); + Expect("wrap sample accepted", estimator.Acknowledge(0xfffffff0u, 0xfffffff0u, 1, MINIMUM_RTO, clock)); Expect_Equal("wrap elapsed", estimator.Smoothed_Rtt(), 48u); - Expect("retransmitted acknowledgement ignored", !estimator.Acknowledge(0, 2, clock)); + Expect("retransmitted acknowledgement ignored", !estimator.Acknowledge(0, 0, 2, MINIMUM_RTO, clock)); Expect("wrapped retry due", Retransmit_Is_Due(0xfffffff0u, 0x00000054u, 100, 0)); Expect("wrapped retry not early", !Retransmit_Is_Due(0xfffffff0u, 0x00000040u, 100, 0)); @@ -224,14 +224,14 @@ namespace FakeClock clock; RttEstimator reordered; clock.Set(1200); - Expect("newer packet ACK samples first", reordered.Acknowledge(1100, 1, clock)); + Expect("newer packet ACK samples first", reordered.Acknowledge(1100, 1100, 1, MINIMUM_RTO, clock)); clock.Set(1300); - Expect("older packet ACK can sample after reordering", reordered.Acknowledge(1000, 1, clock)); + Expect("older packet ACK can sample after reordering", reordered.Acknowledge(1000, 1000, 1, MINIMUM_RTO, clock)); Expect_Equal("reordered samples keep alpha filter", reordered.Smoothed_Rtt(), 125u); Expect_Equal("reordered samples keep beta filter", reordered.Rtt_Variation(), 88u); clock.Set(2000); - Expect("duplicate ambiguous ACK is excluded by Karn", !reordered.Acknowledge(1500, 2, clock)); + Expect("duplicate ambiguous ACK is excluded by Karn", !reordered.Acknowledge(1500, 1500, 2, MINIMUM_RTO, clock)); Expect_Equal("ambiguous ACK leaves SRTT unchanged", reordered.Smoothed_Rtt(), 125u); RttEstimator jitter; @@ -255,9 +255,9 @@ namespace lossy_transport.Send(1000); Expect("fake transport retries a lost packet", lossy_transport.Retry(1100)); Expect("ambiguous first ACK seeds a provisional sample", lossy_transport.Acknowledge(1180)); - Expect("provisional seed is the elapsed upper bound", lossy_transport.Rtt().Smoothed_Rtt() == 180u && lossy_transport.Rtt().Is_Provisional()); + Expect("provisional seed is the time since the last transmission", lossy_transport.Rtt().Smoothed_Rtt() == 80u && lossy_transport.Rtt().Is_Provisional()); lossy_transport.Send(2000); - Expect_Equal("provisional seed paces the next packet", lossy_transport.Base_Rto(), 540u); + Expect_Equal("provisional seed paces the next packet", lossy_transport.Base_Rto(), 240u); Expect("clean sample replaces the provisional seed", lossy_transport.Acknowledge(2080)); Expect_Equal("replaced smoothed RTT", lossy_transport.Rtt().Smoothed_Rtt(), 80u); Expect_Equal("replaced variation", lossy_transport.Rtt().Rtt_Variation(), 40u); @@ -309,33 +309,39 @@ namespace FakeClock clock; RttEstimator estimator; - Expect("unsent packet never samples", !estimator.Acknowledge(0, 0, clock)); + Expect("unsent packet never samples", !estimator.Acknowledge(0, 0, 0, MINIMUM_RTO, clock)); Expect("unsent acknowledgement leaves the estimator empty", !estimator.Has_Sample()); clock.Set(2000); - Expect("two-second link seeds through a retransmitted ACK", estimator.Acknowledge(0, 2, clock)); + Expect("two-second link seeds through a retransmitted ACK", estimator.Acknowledge(0, 1000, 2, 1000, clock)); Expect("seed is provisional", estimator.Has_Sample() && estimator.Is_Provisional()); - Expect_Equal("seed smoothed RTT", estimator.Smoothed_Rtt(), 2000u); - Expect_Equal("seed RTO reaches the ceiling", estimator.Retransmit_Timeout(), MAXIMUM_RTO); + Expect_Equal("seed is the time since the last transmission", estimator.Smoothed_Rtt(), 1000u); + Expect_Equal("seed RTO covers the two-second link", estimator.Retransmit_Timeout(), 3000u); clock.Set(4500); - Expect("second ambiguous ACK does not move a provisional seed", !estimator.Acknowledge(1000, 3, clock)); - Expect_Equal("seed unchanged by a second ambiguous ACK", estimator.Smoothed_Rtt(), 2000u); + Expect("second ambiguous ACK does not move a provisional seed", !estimator.Acknowledge(1000, 3000, 3, 1000, clock)); + Expect_Equal("seed unchanged by a second ambiguous ACK", estimator.Smoothed_Rtt(), 1000u); clock.Set(6900); - Expect("clean sample replaces the seed", estimator.Acknowledge(5000, 1, clock)); + Expect("clean sample replaces the seed", estimator.Acknowledge(5000, 5000, 1, 3000, clock)); Expect_Equal("clean sample replaces rather than blends", estimator.Smoothed_Rtt(), 1900u); Expect("replaced seed is measured", !estimator.Is_Provisional()); clock.Set(9000); - Expect("Karn applies once the estimate is measured", !estimator.Acknowledge(7000, 2, clock)); + Expect("Karn applies once the estimate is measured", !estimator.Acknowledge(7000, 8000, 2, 1900, clock)); + + RttEstimator loading_peer; + clock.Set(3100); + Expect("a peer that was still loading seeds from the last transmission", loading_peer.Acknowledge(0, 3000, 4, 1000, clock)); + Expect_Equal("loading time does not inflate the seed", loading_peer.Smoothed_Rtt(), 100u); + Expect_Equal("the seed keeps the retry delay it outlived", loading_peer.Retransmit_Timeout(), 1000u); RttEstimator backed_off; clock.Set(300); - backed_off.Acknowledge(0, 2, clock); - Expect_Equal("provisional RTO", backed_off.Retransmit_Timeout(), 900u); - backed_off.Note_Retransmit(900); - Expect_Equal("provisional estimate backs off like a measured one", backed_off.Retransmit_Timeout(), 1800u); + backed_off.Acknowledge(0, 100, 2, 100, clock); + Expect_Equal("provisional RTO", backed_off.Retransmit_Timeout(), 600u); + backed_off.Note_Retransmit(600); + Expect_Equal("provisional estimate backs off like a measured one", backed_off.Retransmit_Timeout(), 1200u); backed_off.Reset(); Expect("reset clears the provisional flag", !backed_off.Is_Provisional() && !backed_off.Has_Sample()); From 27e6e6f069af227805372065a2b2782e8f7f80b2 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Thu, 3 Sep 2026 10:38:46 +0300 Subject: [PATCH 09/11] Seed the provisional RTT estimate from the first transmission again --- code/connect.cpp | 3 +- code/nettiming.cpp | 19 ++------ code/nettiming.h | 3 +- .../systems/network-transport-timing.md | 6 +-- tests/nettiming/nettiming.cpp | 46 ++++++++----------- 5 files changed, 29 insertions(+), 48 deletions(-) diff --git a/code/connect.cpp b/code/connect.cpp index a4de8d88d..ccde7c016 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -800,8 +800,7 @@ int ConnectionClass::Service_Send_Queue (void) if (header.Code == PACKET_DATA_ACK) { Queue->Add_Delay(Time() - send_entry->FirstTime); if (Adaptive_Timing_Enabled()) { - RoundTripEstimator.Acknowledge(send_entry->FirstTimeMilliseconds, send_entry->LastTimeMilliseconds, send_entry->SendCount, - send_entry->RetransmitTimeoutMilliseconds, *MillisecondTime); + RoundTripEstimator.Acknowledge(send_entry->FirstTimeMilliseconds, send_entry->SendCount, *MillisecondTime); } } } diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 470a5ffbd..3dee007f4 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -60,28 +60,19 @@ namespace NetTiming } - /// - /// Samples an acknowledgement. An unmeasured link takes an ambiguous one as a provisional seed: - /// the time since the packet's last transmission, with the retry delay it outlived as the floor - /// of the timeout. - /// - bool RttEstimator::Acknowledge(Milliseconds first_sent_at, Milliseconds last_sent_at, unsigned int transmission_count, Milliseconds retry_timeout, - MillisecondClock const & clock) + /// Samples an acknowledgement; an unmeasured link takes an ambiguous one as a provisional upper bound. + bool RttEstimator::Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock) { if (transmission_count == 0) { return(false); } - Milliseconds const now = clock.Now(); + Milliseconds const elapsed = Elapsed_Milliseconds(sent_at, clock.Now()); if (transmission_count != 1) { if (Initialized) { return(false); } - // The first transmission may predate the peer entirely; the last one bounds the round trip from below. - Provisional = Add_Sample(Elapsed_Milliseconds(last_sent_at, now)); - if (Provisional && RetransmitTimeout < retry_timeout) { - RetransmitTimeout = Clamp_Rto(retry_timeout); - } + Provisional = Add_Sample(elapsed); return(Provisional); } @@ -90,7 +81,7 @@ namespace NetTiming Initialized = false; Provisional = false; } - return(Add_Sample(Elapsed_Milliseconds(first_sent_at, now))); + return(Add_Sample(elapsed)); } diff --git a/code/nettiming.h b/code/nettiming.h index 2e6e54a22..dcc141c9b 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -41,8 +41,7 @@ namespace NetTiming public: void Reset(void); bool Add_Sample(Milliseconds round_trip, bool retransmitted = false); - bool Acknowledge(Milliseconds first_sent_at, Milliseconds last_sent_at, unsigned int transmission_count, Milliseconds retry_timeout, - MillisecondClock const & clock = Default_Clock()); + bool Acknowledge(Milliseconds sent_at, unsigned int transmission_count, MillisecondClock const & clock = Default_Clock()); void Note_Retransmit(Milliseconds captured_rto); bool Has_Sample(void) const {return(Initialized);} diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index b80d20a3b..4dbfc84a3 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -8,10 +8,8 @@ keys: [] Each private connection maintains smoothed round trip, variation, and a retry timeout. Acknowledgements of first transmissions are the measurements. Until a link has one, its first acknowledgement seeds a provisional estimate even after -a retry: the time since the packet's last transmission, paced no faster than the -retry delay that went unanswered. A link slower than the initial retry delay -therefore becomes measurable, and a peer that was still loading does not -inflate the seed. The first clean acknowledgement replaces it. +a retry, so a link slower than the initial retry delay becomes measurable; the +first clean acknowledgement replaces the seed. The retry timeout is limited to 100–4000 ms. Repeated private transmissions double their wait up to the connection timeout; that timeout follows measured diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index d7b4fe1f1..a791beb2a 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -56,7 +56,7 @@ namespace bool Acknowledge(NetTiming::Milliseconds now) { Clock.Set(now); - return(Estimator.Acknowledge(FirstSend, LastSend, TransmissionCount, BaseRto, Clock)); + return(Estimator.Acknowledge(FirstSend, TransmissionCount, Clock)); } NetTiming::RttEstimator const & Rtt(void) const {return(Estimator);} @@ -143,9 +143,9 @@ namespace FakeClock clock; clock.Set(0x00000020u); RttEstimator estimator; - Expect("wrap sample accepted", estimator.Acknowledge(0xfffffff0u, 0xfffffff0u, 1, MINIMUM_RTO, clock)); + Expect("wrap sample accepted", estimator.Acknowledge(0xfffffff0u, 1, clock)); Expect_Equal("wrap elapsed", estimator.Smoothed_Rtt(), 48u); - Expect("retransmitted acknowledgement ignored", !estimator.Acknowledge(0, 0, 2, MINIMUM_RTO, clock)); + Expect("retransmitted acknowledgement ignored", !estimator.Acknowledge(0, 2, clock)); Expect("wrapped retry due", Retransmit_Is_Due(0xfffffff0u, 0x00000054u, 100, 0)); Expect("wrapped retry not early", !Retransmit_Is_Due(0xfffffff0u, 0x00000040u, 100, 0)); @@ -224,14 +224,14 @@ namespace FakeClock clock; RttEstimator reordered; clock.Set(1200); - Expect("newer packet ACK samples first", reordered.Acknowledge(1100, 1100, 1, MINIMUM_RTO, clock)); + Expect("newer packet ACK samples first", reordered.Acknowledge(1100, 1, clock)); clock.Set(1300); - Expect("older packet ACK can sample after reordering", reordered.Acknowledge(1000, 1000, 1, MINIMUM_RTO, clock)); + Expect("older packet ACK can sample after reordering", reordered.Acknowledge(1000, 1, clock)); Expect_Equal("reordered samples keep alpha filter", reordered.Smoothed_Rtt(), 125u); Expect_Equal("reordered samples keep beta filter", reordered.Rtt_Variation(), 88u); clock.Set(2000); - Expect("duplicate ambiguous ACK is excluded by Karn", !reordered.Acknowledge(1500, 1500, 2, MINIMUM_RTO, clock)); + Expect("duplicate ambiguous ACK is excluded by Karn", !reordered.Acknowledge(1500, 2, clock)); Expect_Equal("ambiguous ACK leaves SRTT unchanged", reordered.Smoothed_Rtt(), 125u); RttEstimator jitter; @@ -255,9 +255,9 @@ namespace lossy_transport.Send(1000); Expect("fake transport retries a lost packet", lossy_transport.Retry(1100)); Expect("ambiguous first ACK seeds a provisional sample", lossy_transport.Acknowledge(1180)); - Expect("provisional seed is the time since the last transmission", lossy_transport.Rtt().Smoothed_Rtt() == 80u && lossy_transport.Rtt().Is_Provisional()); + Expect("provisional seed is the elapsed upper bound", lossy_transport.Rtt().Smoothed_Rtt() == 180u && lossy_transport.Rtt().Is_Provisional()); lossy_transport.Send(2000); - Expect_Equal("provisional seed paces the next packet", lossy_transport.Base_Rto(), 240u); + Expect_Equal("provisional seed paces the next packet", lossy_transport.Base_Rto(), 540u); Expect("clean sample replaces the provisional seed", lossy_transport.Acknowledge(2080)); Expect_Equal("replaced smoothed RTT", lossy_transport.Rtt().Smoothed_Rtt(), 80u); Expect_Equal("replaced variation", lossy_transport.Rtt().Rtt_Variation(), 40u); @@ -309,39 +309,33 @@ namespace FakeClock clock; RttEstimator estimator; - Expect("unsent packet never samples", !estimator.Acknowledge(0, 0, 0, MINIMUM_RTO, clock)); + Expect("unsent packet never samples", !estimator.Acknowledge(0, 0, clock)); Expect("unsent acknowledgement leaves the estimator empty", !estimator.Has_Sample()); clock.Set(2000); - Expect("two-second link seeds through a retransmitted ACK", estimator.Acknowledge(0, 1000, 2, 1000, clock)); + Expect("two-second link seeds through a retransmitted ACK", estimator.Acknowledge(0, 2, clock)); Expect("seed is provisional", estimator.Has_Sample() && estimator.Is_Provisional()); - Expect_Equal("seed is the time since the last transmission", estimator.Smoothed_Rtt(), 1000u); - Expect_Equal("seed RTO covers the two-second link", estimator.Retransmit_Timeout(), 3000u); + Expect_Equal("seed smoothed RTT", estimator.Smoothed_Rtt(), 2000u); + Expect_Equal("seed RTO reaches the ceiling", estimator.Retransmit_Timeout(), MAXIMUM_RTO); clock.Set(4500); - Expect("second ambiguous ACK does not move a provisional seed", !estimator.Acknowledge(1000, 3000, 3, 1000, clock)); - Expect_Equal("seed unchanged by a second ambiguous ACK", estimator.Smoothed_Rtt(), 1000u); + Expect("second ambiguous ACK does not move a provisional seed", !estimator.Acknowledge(1000, 3, clock)); + Expect_Equal("seed unchanged by a second ambiguous ACK", estimator.Smoothed_Rtt(), 2000u); clock.Set(6900); - Expect("clean sample replaces the seed", estimator.Acknowledge(5000, 5000, 1, 3000, clock)); + Expect("clean sample replaces the seed", estimator.Acknowledge(5000, 1, clock)); Expect_Equal("clean sample replaces rather than blends", estimator.Smoothed_Rtt(), 1900u); Expect("replaced seed is measured", !estimator.Is_Provisional()); clock.Set(9000); - Expect("Karn applies once the estimate is measured", !estimator.Acknowledge(7000, 8000, 2, 1900, clock)); - - RttEstimator loading_peer; - clock.Set(3100); - Expect("a peer that was still loading seeds from the last transmission", loading_peer.Acknowledge(0, 3000, 4, 1000, clock)); - Expect_Equal("loading time does not inflate the seed", loading_peer.Smoothed_Rtt(), 100u); - Expect_Equal("the seed keeps the retry delay it outlived", loading_peer.Retransmit_Timeout(), 1000u); + Expect("Karn applies once the estimate is measured", !estimator.Acknowledge(7000, 2, clock)); RttEstimator backed_off; clock.Set(300); - backed_off.Acknowledge(0, 100, 2, 100, clock); - Expect_Equal("provisional RTO", backed_off.Retransmit_Timeout(), 600u); - backed_off.Note_Retransmit(600); - Expect_Equal("provisional estimate backs off like a measured one", backed_off.Retransmit_Timeout(), 1200u); + backed_off.Acknowledge(0, 2, clock); + Expect_Equal("provisional RTO", backed_off.Retransmit_Timeout(), 900u); + backed_off.Note_Retransmit(900); + Expect_Equal("provisional estimate backs off like a measured one", backed_off.Retransmit_Timeout(), 1800u); backed_off.Reset(); Expect("reset clears the provisional flag", !backed_off.Is_Provisional() && !backed_off.Has_Sample()); From 289295640e86ceb1769dd1e88a4f4dd109f85e73 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Sat, 5 Sep 2026 03:54:42 +0300 Subject: [PATCH 10/11] Fix private-link recovery after timeouts and latency increases --- code/connect.cpp | 6 +- code/nettiming.cpp | 6 +- code/nettiming.h | 2 +- manual/changes/network-transport-timing.md | 8 +- .../systems/network-transport-timing.md | 18 +- tests/nettiming/CMakeLists.txt | 39 ++- tests/nettiming/connectionstubs.cpp | 41 ++++ tests/nettiming/connectiontiming.cpp | 232 ++++++++++++++++++ tests/nettiming/nettiming.cpp | 71 +++++- 9 files changed, 387 insertions(+), 36 deletions(-) create mode 100644 tests/nettiming/connectionstubs.cpp create mode 100644 tests/nettiming/connectiontiming.cpp diff --git a/code/connect.cpp b/code/connect.cpp index ccde7c016..015ec5aba 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -747,7 +747,9 @@ int ConnectionClass::Service (void) been ACK'd yet. Entries that the app has read, and have been ACK'd, should be removed. ------------------------------------------------------------------------*/ - IsBad = !(Service_Send_Queue() && Service_Receive_Queue()); + int const send_status = Service_Send_Queue(); + int const receive_status = Service_Receive_Queue(); + IsBad = !(send_status && receive_status); return(IsBad ? 0 : 1); } /* end of Service */ @@ -825,7 +827,7 @@ int ConnectionClass::Service_Send_Queue (void) bool const timeout_enabled = Timeout != (unsigned int)-1; NetTiming::Milliseconds const connection_timeout = !timeout_enabled ? NetTiming::MAXIMUM_CONNECTION_TIMEOUT - : (adaptive_timing ? NetTiming::Connection_Timeout(RoundTripEstimator.Smoothed_Rtt()) + : (adaptive_timing ? NetTiming::Connection_Timeout(RoundTripEstimator.Smoothed_Rtt(), RoundTripEstimator.Retransmit_Timeout()) : (adaptive_channel ? Legacy_Connection_Timeout(Timeout) : Ticks_To_Milliseconds(Timeout))); NetTiming::Milliseconds const base_retry_timeout = adaptive_timing ? NetTiming::Initial_Retry_Timeout(RoundTripEstimator.Retransmit_Timeout(), connection_timeout) diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 3dee007f4..2f62b5f51 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -99,10 +99,10 @@ namespace NetTiming } - /// Derives the connection timeout from smoothed latency. - Milliseconds Connection_Timeout(Milliseconds smoothed_rtt) + /// Derives a timeout that covers measured latency and three transmissions at the current RTO. + Milliseconds Connection_Timeout(Milliseconds smoothed_rtt, Milliseconds retransmit_timeout) { - std::uint64_t const timeout = 8ull * smoothed_rtt + 250; + std::uint64_t const timeout = std::max(8ull * smoothed_rtt + 250, 4ull * retransmit_timeout); return(static_cast(std::clamp(timeout, MINIMUM_CONNECTION_TIMEOUT, MAXIMUM_CONNECTION_TIMEOUT))); } diff --git a/code/nettiming.h b/code/nettiming.h index dcc141c9b..ec34f56bc 100644 --- a/code/nettiming.h +++ b/code/nettiming.h @@ -59,7 +59,7 @@ namespace NetTiming bool Provisional = false; }; - Milliseconds Connection_Timeout(Milliseconds smoothed_rtt); + Milliseconds Connection_Timeout(Milliseconds smoothed_rtt, Milliseconds retransmit_timeout); Milliseconds Initial_Retry_Timeout(Milliseconds retransmit_timeout, Milliseconds connection_timeout); Milliseconds Retransmit_Delay(Milliseconds base_rto, unsigned int prior_retransmissions, Milliseconds maximum_delay = MAXIMUM_RTO); bool Retransmit_Is_Due(Milliseconds last_send, Milliseconds now, Milliseconds base_rto, diff --git a/manual/changes/network-transport-timing.md b/manual/changes/network-transport-timing.md index 1959de0eb..eef69e120 100644 --- a/manual/changes/network-transport-timing.md +++ b/manual/changes/network-transport-timing.md @@ -10,7 +10,7 @@ credit: Each private connection now estimates its own round trip and backs off repeated transmissions. The retry timeout backs off with them, so a link whose latency rises above it stays measurable instead of retransmitting every packet. A link -slower than the initial retry delay is still measured, and a packet that -outlives the connection timeout keeps retransmitting instead of blocking the -link. Lobby traffic keeps its fixed retry cadence. Packet layouts, event IDs, -and configuration remain unchanged. +slower than the initial retry delay is still measured. Timed-out packets keep +retrying, and receive queues keep freeing space so a recovered link can drain +its backlog. Lobby traffic keeps its fixed retry cadence. Packet layouts, +event IDs, and configuration remain unchanged. diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index 4dbfc84a3..94cf2fabc 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -12,13 +12,17 @@ a retry, so a link slower than the initial retry delay becomes measurable; the first clean acknowledgement replaces the seed. The retry timeout is limited to 100–4000 ms. Repeated private transmissions -double their wait up to the connection timeout; that timeout follows measured -latency with a 2-second minimum and 30-second ceiling. A packet's first retry -waits at most a quarter of that timeout, so every packet is sent at least three -times before it. A packet older than the timeout marks the connection bad but is -still retransmitted at the capped wait until it is acknowledged, so a link that -recovers drains its backlog. With no measurement, the bounded legacy timing is -used. Global lobby traffic retains its fixed cadence. +double their wait up to the connection timeout. For a measured link, the +connection timeout is the larger of eight times the smoothed round trip plus +250 ms and four times the current retry timeout, bounded to 2–30 seconds. +This lets the first retry wait for the backed-off timeout while allowing at +least three transmissions before the connection timeout. + +A packet older than the connection timeout marks the connection bad but keeps +retrying until acknowledged. Receive-queue cleanup continues during these +retries, freeing space for the backlog when the link recovers. With no +measurement, the bounded legacy timing is used. Global lobby traffic retains +its fixed cadence. A link that is retransmitting also doubles the timeout it measures against, once per retransmission proven against the current value. This keeps a link whose diff --git a/tests/nettiming/CMakeLists.txt b/tests/nettiming/CMakeLists.txt index 7cc49744a..96454fd21 100644 --- a/tests/nettiming/CMakeLists.txt +++ b/tests/nettiming/CMakeLists.txt @@ -14,21 +14,34 @@ add_executable(NetTiming "${OPENTS_ROOT}/code/nettiming.cpp" ) -target_compile_features(NetTiming PRIVATE cxx_std_20) - -target_include_directories(NetTiming PRIVATE "${OPENTS_ROOT}/code") - -target_compile_definitions(NetTiming PRIVATE WIN32 _WINDOWS _MBCS) - -target_compile_options(NetTiming PRIVATE - $<$:/MTd /EHsc /Zc:__cplusplus> - $<$:/MT /EHsc /Zc:__cplusplus> +add_executable(NetConnection + "${CMAKE_CURRENT_SOURCE_DIR}/connectiontiming.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/connectionstubs.cpp" + "${OPENTS_ROOT}/code/combuf.cpp" + "${OPENTS_ROOT}/code/connect.cpp" + "${OPENTS_ROOT}/code/netadmit.cpp" + "${OPENTS_ROOT}/code/nettime.cpp" + "${OPENTS_ROOT}/code/nettiming.cpp" ) -target_link_libraries(NetTiming PRIVATE kernel32 winmm) - -set_target_properties(NetTiming PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" +target_compile_definitions(NetConnection PRIVATE NOMINMAX) +set_source_files_properties("${OPENTS_ROOT}/code/combuf.cpp" PROPERTIES + COMPILE_OPTIONS "/source-charset:437;/execution-charset:437" ) +foreach(target NetTiming NetConnection) + target_compile_features(${target} PRIVATE cxx_std_20) + target_include_directories(${target} PRIVATE "${OPENTS_ROOT}/code") + target_compile_definitions(${target} PRIVATE WIN32 _WINDOWS _MBCS) + target_compile_options(${target} PRIVATE + $<$:/MTd /EHsc /Zc:__cplusplus> + $<$:/MT /EHsc /Zc:__cplusplus> + ) + target_link_libraries(${target} PRIVATE kernel32 winmm) + set_target_properties(${target} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" + ) +endforeach() + add_test(NAME nettiming COMMAND NetTiming) +add_test(NAME netconnection COMMAND NetConnection) diff --git a/tests/nettiming/connectionstubs.cpp b/tests/nettiming/connectionstubs.cpp new file mode 100644 index 000000000..5eba0f596 --- /dev/null +++ b/tests/nettiming/connectionstubs.cpp @@ -0,0 +1,41 @@ +/******************************************************************************* + * O P E N T S + ******************************************************************************* + * SPDX-License-Identifier: GPL-3.0-or-later + * Copyright 2026 OpenTS contributors + * + * See LICENSE.md for applicable additional terms and warranty disclaimers. + ******************************************************************************/ + + +#include "_timer.h" +#include "mono.h" + + +// Connection timing uses its injected clock; the engine timer and debug display stay idle. +TTimerClass TickCount; +MonoClass Mono; + + +int SystemTimerClass::operator () (void) const {return(0);} + + +SystemTimerClass::operator int(void) const {return(0);} + + +void __cdecl DebugString(char const *, ...) {} + + +MonoClass::MonoClass(void) {} + + +MonoClass::~MonoClass(void) {} + + +void MonoClass::Clear(void) {} + + +void MonoClass::Set_Cursor(int, int) {} + + +void __cdecl MonoClass::Printf(char const *, ...) {} diff --git a/tests/nettiming/connectiontiming.cpp b/tests/nettiming/connectiontiming.cpp new file mode 100644 index 000000000..39618d47d --- /dev/null +++ b/tests/nettiming/connectiontiming.cpp @@ -0,0 +1,232 @@ +/******************************************************************************* + * O P E N T S + ******************************************************************************* + * SPDX-License-Identifier: GPL-3.0-or-later + * Copyright 2026 OpenTS contributors + * + * See LICENSE.md for applicable additional terms and warranty disclaimers. + ******************************************************************************/ + + +#include "connect.h" + +#include +#include +#include +#include +#include + + +namespace +{ + class TestClock final : public NetTiming::MillisecondClock + { + public: + NetTiming::Milliseconds Now(void) const override {return(Current);} + NetTiming::Milliseconds Current = 0; + }; + + + class TestTransport + { + public: + explicit TestTransport(TestClock const & clock) : Clock(clock) {} + + void Send(ConnectionClass * destination, char const * buffer, int length) + { + if (Connected) { + Packets.push_back({destination, {buffer, buffer + length}, Clock.Now() + OneWayDelay}); + } + } + + void Deliver(void) + { + while (!Packets.empty() && Packets.front().Arrival <= Clock.Now()) { + Packet packet = std::move(Packets.front()); + Packets.pop_front(); + packet.Destination->Receive_Packet(packet.Bytes.data(), static_cast(packet.Bytes.size())); + } + } + + bool Connected = true; + NetTiming::Milliseconds OneWayDelay = 5; + + private: + struct Packet + { + ConnectionClass * Destination; + std::vector Bytes; + NetTiming::Milliseconds Arrival; + }; + + TestClock const & Clock; + std::deque Packets; + }; + + + class TestConnection final : public ConnectionClass + { + public: + TestConnection(TestClock const & clock, TestTransport & transport, int capacity) + : ConnectionClass(capacity, capacity, sizeof(int), 1234, 6, -1, 120, 0, &clock), Transport(transport) + { + Init(); + } + + void Read(void) + { + int payload; + int length; + while (Get_Packet(&payload, sizeof(payload), &length)) { + Received.push_back(payload); + } + } + + NetTiming::RttEstimator const & Rtt(void) const {return(RoundTripEstimator);} + + ConnectionClass * Remote = nullptr; + std::vector Received; + + private: + int Send(char * buffer, int length, void *, int) override + { + Transport.Send(Remote, buffer, length); + return(1); + } + + TestTransport & Transport; + }; + + + class ConnectionFixture + { + public: + explicit ConnectionFixture(int capacity) : Transport(Clock), First(Clock, Transport, capacity), Second(Clock, Transport, capacity) + { + First.Remote = &Second; + Second.Remote = &First; + } + + void Service(void) + { + Transport.Deliver(); + First.Service(); + Second.Service(); + First.Read(); + Second.Read(); + } + + void Advance_To(NetTiming::Milliseconds time) + { + while (Clock.Current < time) { + Clock.Current++; + Service(); + } + } + + TestClock Clock; + TestTransport Transport; + TestConnection First; + TestConnection Second; + }; + + + int Failures = 0; + + + void Expect(std::string const & name, bool condition) + { + if (!condition) { + std::cerr << name << " failed\n"; + Failures++; + } + } + + + void Establish_Rtt(ConnectionFixture & fixture) + { + int payload = 0; + Expect("first peer queues its initial packet", fixture.First.Send_Packet(&payload, sizeof(payload), true) != 0); + Expect("second peer queues its initial packet", fixture.Second.Send_Packet(&payload, sizeof(payload), true) != 0); + fixture.Service(); + fixture.Advance_To(10); + Expect("both connections measure the initial round trip", fixture.First.Rtt().Smoothed_Rtt() == 10 && fixture.Second.Rtt().Smoothed_Rtt() == 10); + } + + + void Test_Unsampled_Retry(void) + { + ConnectionFixture fixture(2); + fixture.Transport.Connected = false; + int payload = 0; + fixture.First.Send_Packet(&payload, sizeof(payload), true); + fixture.Service(); + Expect("initial send needs no elapsed clock time", fixture.First.Queue->Get_Send(0)->SendCount == 1); + fixture.Advance_To(99); + Expect("unsampled connection retains its legacy retry delay", fixture.First.Queue->Get_Send(0)->SendCount == 1); + fixture.Advance_To(100); + Expect("unsampled connection retries after six engine ticks", fixture.First.Queue->Get_Send(0)->SendCount == 2); + Expect("retry alone does not establish RTT", !fixture.First.Rtt().Has_Sample()); + } + + + void Test_Saturated_Outage_Recovery(int capacity) + { + ConnectionFixture fixture(capacity); + Establish_Rtt(fixture); + Expect("read packets retain the latest sequence entry", fixture.First.Queue->Num_Receive() == 1 && fixture.Second.Queue->Num_Receive() == 1); + fixture.Advance_To(100); + fixture.Transport.Connected = false; + for (int payload = 1; payload <= capacity; payload++) { + Expect("first peer fills its send queue", fixture.First.Send_Packet(&payload, sizeof(payload), true) != 0); + Expect("second peer fills its send queue", fixture.Second.Send_Packet(&payload, sizeof(payload), true) != 0); + } + fixture.Service(); + fixture.Advance_To(3000); + Expect("the outage times out both peers", fixture.First.Is_Bad() && fixture.Second.Is_Bad()); + fixture.Transport.Connected = true; + fixture.Advance_To(8000); + + std::vector expected; + for (int payload = 0; payload <= capacity; payload++) { + expected.push_back(payload); + } + Expect("first peer receives the complete ordered backlog", fixture.First.Received == expected); + Expect("second peer receives the complete ordered backlog", fixture.Second.Received == expected); + Expect("restored connectivity drains both send queues", fixture.First.Queue->Num_Send() == 0 && fixture.Second.Queue->Num_Send() == 0); + Expect("both connections recover from timeout", !fixture.First.Is_Bad() && !fixture.Second.Is_Bad()); + } + + + void Test_Latency_Increase(void) + { + ConnectionFixture fixture(2); + Establish_Rtt(fixture); + fixture.Transport.OneWayDelay = 300; + for (int payload = 1; payload <= 12; payload++) { + Expect("slower link accepts another packet", fixture.First.Send_Packet(&payload, sizeof(payload), true) != 0); + fixture.Service(); + fixture.Advance_To(fixture.Clock.Now() + 1000); + } + Expect("production retry capture lets the slower link become measurable", fixture.First.Rtt().Smoothed_Rtt() > 100); + Expect("slower link obtains an unambiguous estimate", fixture.First.Rtt().Has_Sample() && !fixture.First.Rtt().Is_Provisional()); + Expect("slower link delivers every packet", fixture.Second.Received.size() == 13); + Expect("slower link completes its acknowledgements", fixture.First.Queue->Num_Send() == 0); + } +} + + +int main(void) +{ + Test_Unsampled_Retry(); + Test_Saturated_Outage_Recovery(2); + Test_Saturated_Outage_Recovery(32); + Test_Latency_Increase(); + + if (Failures != 0) { + std::cerr << Failures << " connection timing checks failed\n"; + return(1); + } + std::cout << "All connection timing checks passed\n"; + return(0); +} diff --git a/tests/nettiming/nettiming.cpp b/tests/nettiming/nettiming.cpp index a791beb2a..9e6c0cc6e 100644 --- a/tests/nettiming/nettiming.cpp +++ b/tests/nettiming/nettiming.cpp @@ -38,13 +38,14 @@ namespace FirstSend = now; LastSend = now; TransmissionCount = 1; - BaseRto = Estimator.Retransmit_Timeout(); + BaseRto = NetTiming::Initial_Retry_Timeout(Estimator.Retransmit_Timeout(), Timeout()); } bool Retry(NetTiming::Milliseconds now) { Clock.Set(now); - if (!NetTiming::Retransmit_Is_Due(LastSend, now, BaseRto, TransmissionCount - 1, NetTiming::MINIMUM_CONNECTION_TIMEOUT)) { + NetTiming::RetransmitState const state{FirstSend, LastSend, BaseRto, TransmissionCount}; + if (!NetTiming::Evaluate_Retry(state, now, BaseRto, Timeout(), true, true).Send) { return(false); } LastSend = now; @@ -61,6 +62,10 @@ namespace NetTiming::RttEstimator const & Rtt(void) const {return(Estimator);} NetTiming::Milliseconds Base_Rto(void) const {return(BaseRto);} + NetTiming::Milliseconds Timeout(void) const + { + return(NetTiming::Connection_Timeout(Estimator.Smoothed_Rtt(), Estimator.Retransmit_Timeout())); + } private: FakeClock Clock; @@ -163,19 +168,71 @@ namespace Expect_Equal("fourth backoff", Retransmit_Delay(100, 4), 1600u); Expect_Equal("backoff saturation", Retransmit_Delay(100, 20), MAXIMUM_RTO); Expect_Equal("base clamp", Retransmit_Delay(1, 0), MINIMUM_RTO); - Expect_Equal("connection timeout minimum", Connection_Timeout(0), 2000u); - Expect_Equal("connection timeout follows RTT", Connection_Timeout(500), 4250u); - Expect_Equal("connection timeout ceiling", Connection_Timeout(10000), 30000u); + Expect_Equal("connection timeout minimum", Connection_Timeout(0, MINIMUM_RTO), 2000u); + Expect_Equal("connection timeout follows RTT", Connection_Timeout(500, MINIMUM_RTO), 4250u); + Expect_Equal("connection timeout ceiling", Connection_Timeout(10000, MAXIMUM_RTO), 30000u); Expect_Equal("backoff reaches connection timeout", Retransmit_Delay(500, 8, 4250), 4250u); Expect_Equal("first retry keeps a small RTO", Initial_Retry_Timeout(300, 2000), 300u); Expect_Equal("first retry is bounded by a quarter of the timeout", Initial_Retry_Timeout(1600, 2000), 500u); Expect_Equal("first retry bound keeps the floor", Initial_Retry_Timeout(1600, 300), MINIMUM_RTO); - Expect_Equal("slow link keeps its RTO under a long timeout", Initial_Retry_Timeout(2055, Connection_Timeout(2015)), 2055u); + Expect_Equal("slow link keeps its RTO under a long timeout", Initial_Retry_Timeout(2055, Connection_Timeout(2015, 2055)), 2055u); Expect_Equal("bounded first retry allows three sends before the timeout", Retransmit_Delay(500, 0) + Retransmit_Delay(500, 1), 1500u); } + void Test_Backoff_Timeout(void) + { + using namespace NetTiming; + + Expect_Equal("backed off RTO extends the timeout", Connection_Timeout(10, 800), 3200u); + Expect_Equal("maximum RTO fits below the timeout ceiling", Connection_Timeout(10, MAXIMUM_RTO), 16000u); + Expect_Equal("large RTO calculation keeps the hard ceiling", Connection_Timeout(10, 0xffffffffu), MAXIMUM_CONNECTION_TIMEOUT); + + for (Milliseconds rto : {100u, 500u, 800u, 1600u, MAXIMUM_RTO}) { + Milliseconds const timeout = Connection_Timeout(10, rto); + Milliseconds const initial_retry = Initial_Retry_Timeout(rto, timeout); + Expect_Equal("packet captures the full backed off RTO", initial_retry, rto); + + RetransmitState state{1000, 1000, initial_retry, 1}; + RetryDecision decision = Evaluate_Retry(state, 1000 + rto, rto, timeout, true, true); + Expect("second transmission precedes the extended timeout", decision.Send && !decision.TimedOut); + state.LastSend += rto; + state.TransmissionCount++; + decision = Evaluate_Retry(state, 1000 + 3 * rto, rto, timeout, true, true); + Expect("third transmission precedes the extended timeout", decision.Send && !decision.TimedOut); + Expect("extended timeout still expires", Evaluate_Retry(state, 1000 + timeout, rto, timeout, true, true).TimedOut); + } + } + + + void Test_Latency_Increase(void) + { + using namespace NetTiming; + + for (Milliseconds round_trip : {600u, 1500u}) { + FakeTransport transport; + transport.Send(0); + Expect("latency increase starts from a measured fast link", transport.Acknowledge(10)); + + bool measured = false; + for (unsigned int packet = 0; packet < 10 && !measured; packet++) { + Milliseconds const sent_at = 1000 + packet * (round_trip + 1000); + transport.Send(sent_at); + for (Milliseconds elapsed = 1; elapsed < round_trip; elapsed++) { + transport.Retry(sent_at + elapsed); + } + measured = transport.Acknowledge(sent_at + round_trip); + } + + Expect("slower link eventually produces a clean sample", measured); + Expect("clean sample updates the stale fast-link RTT", transport.Rtt().Smoothed_Rtt() > 10); + Expect("clean sample arrives before the captured retry", transport.Base_Rto() > round_trip); + Expect("recovery keeps the connection timeout bounded", transport.Timeout() <= MAXIMUM_CONNECTION_TIMEOUT); + } + } + + void Test_Retry_Decisions(void) { using namespace NetTiming; @@ -380,6 +437,8 @@ int main(void) Test_Rtt_Estimator(); Test_Clock_And_Wrap(); Test_Retransmit_Backoff(); + Test_Backoff_Timeout(); + Test_Latency_Increase(); Test_Retry_Decisions(); Test_Loss_Jitter_And_Reordering(); Test_Backoff_Persistence(); From 8c5370d7a065abf328b09e54a6b5b55d20904af2 Mon Sep 17 00:00:00 2001 From: ZivDero Date: Sat, 5 Sep 2026 23:09:19 +0300 Subject: [PATCH 11/11] Tighten transport timing documentation and comments --- code/combuf.h | 4 ++-- code/connect.cpp | 3 +-- code/nettiming.cpp | 2 -- manual/changes/network-transport-timing.md | 16 +++++++++------- .../systems/network-transport-timing.md | 19 +++++++++---------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/code/combuf.h b/code/combuf.h index 4f02f0c6e..2c6f0dd0f 100644 --- a/code/combuf.h +++ b/code/combuf.h @@ -59,8 +59,8 @@ struct SendQueueType { unsigned int IsUndeliverable : 1; /// 1 = gave up on it (retries or timeout) unsigned int FirstTime; // time this packet was first sent unsigned int LastTime; // time this packet was last sent - unsigned int FirstTimeMilliseconds = 0; // monotonic time of the first transmission - unsigned int LastTimeMilliseconds = 0; // monotonic time of the latest transmission + unsigned int FirstTimeMilliseconds = 0; // millisecond clock at the first transmission + unsigned int LastTimeMilliseconds = 0; // millisecond clock at the latest transmission unsigned int RetransmitTimeoutMilliseconds = 0; // base RTO captured for this packet unsigned int SendCount; // # of times this packet has been sent int BufLen; // size of the packet stored in this entry diff --git a/code/connect.cpp b/code/connect.cpp index 015ec5aba..b273378b7 100644 --- a/code/connect.cpp +++ b/code/connect.cpp @@ -66,7 +66,6 @@ char const * ConnectionClass::Commands[PACKET_COUNT] = { namespace { -/// Converts engine ticks to milliseconds. NetTiming::Milliseconds Ticks_To_Milliseconds(unsigned int ticks) { std::uint64_t const milliseconds = (static_cast(ticks) * 1000 + TIMER_SECOND - 1) / TIMER_SECOND; @@ -77,7 +76,7 @@ NetTiming::Milliseconds Ticks_To_Milliseconds(unsigned int ticks) } -/// Converts and bounds a legacy connection timeout. +/// Converts a legacy tick timeout and clamps it to the supported range. NetTiming::Milliseconds Legacy_Connection_Timeout(unsigned int ticks) { return(std::clamp(Ticks_To_Milliseconds(ticks), NetTiming::MINIMUM_CONNECTION_TIMEOUT, NetTiming::MAXIMUM_CONNECTION_TIMEOUT)); diff --git a/code/nettiming.cpp b/code/nettiming.cpp index 2f62b5f51..e08a3fcf0 100644 --- a/code/nettiming.cpp +++ b/code/nettiming.cpp @@ -17,7 +17,6 @@ namespace NetTiming { namespace { - /// Constrains a retransmission timeout to the supported range. constexpr Milliseconds Clamp_Rto(std::uint64_t value) { return(static_cast(std::clamp(value, MINIMUM_RTO, MAXIMUM_RTO))); @@ -25,7 +24,6 @@ namespace NetTiming } - /// Restores the estimator to its unsampled state. void RttEstimator::Reset(void) { Initialized = false; diff --git a/manual/changes/network-transport-timing.md b/manual/changes/network-transport-timing.md index eef69e120..297192675 100644 --- a/manual/changes/network-transport-timing.md +++ b/manual/changes/network-transport-timing.md @@ -2,15 +2,17 @@ title: Adapt private network retries category: performance release: 0.2.0 -targets: [] +targets: +- type: system + id: network-transport-timing + effect: added credit: - ZivDero --- -Each private connection now estimates its own round trip and backs off repeated +Each private connection estimates its own round trip and backs off repeated transmissions. The retry timeout backs off with them, so a link whose latency -rises above it stays measurable instead of retransmitting every packet. A link -slower than the initial retry delay is still measured. Timed-out packets keep -retrying, and receive queues keep freeing space so a recovered link can drain -its backlog. Lobby traffic keeps its fixed retry cadence. Packet layouts, -event IDs, and configuration remain unchanged. +rises above it stays measurable instead of retransmitting every packet. +Timed-out packets keep retrying, and receive queues keep freeing space so a +recovered link can drain its backlog. Lobby traffic keeps its fixed retry +cadence. Packet layouts, event IDs, and configuration are unchanged. diff --git a/manual/content/systems/network-transport-timing.md b/manual/content/systems/network-transport-timing.md index 94cf2fabc..f933dc91e 100644 --- a/manual/content/systems/network-transport-timing.md +++ b/manual/content/systems/network-transport-timing.md @@ -20,14 +20,13 @@ least three transmissions before the connection timeout. A packet older than the connection timeout marks the connection bad but keeps retrying until acknowledged. Receive-queue cleanup continues during these -retries, freeing space for the backlog when the link recovers. With no -measurement, the bounded legacy timing is used. Global lobby traffic retains -its fixed cadence. +retries, freeing space for the backlog when the link recovers. An unmeasured +link uses the bounded legacy timing, and global lobby traffic keeps its fixed +cadence. -A link that is retransmitting also doubles the timeout it measures against, once -per retransmission proven against the current value. This keeps a link whose -latency has risen above its timeout measurable, because every packet would -otherwise be retransmitted before its acknowledgement arrived and no -unambiguous sample could be taken. The next clean acknowledgement recomputes the -timeout from the measured latency; until then the estimate keeps its last -measured value while pacing retries. +A retransmitting link also doubles the timeout it measures against, once per +retransmission proven against the current value. Without that, a link whose +latency has risen above its timeout retransmits every packet before its +acknowledgement arrives and never yields an unambiguous sample. The next clean +acknowledgement recomputes the timeout from the measured latency; until then the +estimate keeps its last measured value while pacing retries.