diff --git a/apps/counterstrikesharp/src/FiveStack.Commands/Match.cs b/apps/counterstrikesharp/src/FiveStack.Commands/Match.cs index 3e653012..e2bde509 100644 --- a/apps/counterstrikesharp/src/FiveStack.Commands/Match.cs +++ b/apps/counterstrikesharp/src/FiveStack.Commands/Match.cs @@ -35,6 +35,8 @@ public void SetMatchState(CCSPlayerController? player, CommandInfo command) ); return; } + + _gameBackupRounds.AbandonRestore(); } match?.UpdateMapStatus(status); diff --git a/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs b/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs index b231f780..93e45d9a 100644 --- a/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs +++ b/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs @@ -35,7 +35,7 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event, GameE match.UpdateMapStatus(eMapStatus.Overtime); } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.HoldsRoundResults()) { _logger.LogInformation("OnRoundOfficiallyEnded skipping capture: restoring round"); return HookResult.Continue; @@ -55,7 +55,7 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event, GameE [GameEventHandler] public HookResult OnRoundEnd(EventRoundEnd @event, GameEventInfo info) { - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.HoldsRoundResults()) { _logger.LogInformation( $"OnRoundEnd ignored (restoring round): message={@event.Message}" diff --git a/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs b/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs index f00504bf..643a4659 100644 --- a/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs +++ b/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs @@ -35,7 +35,7 @@ public HookResult OnRoundStart(EventRoundStart @event, GameEventInfo info) return HookResult.Continue; } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.HoldsRoundResults()) { // A restart can unpause the round the hold paused. if (_gameBackupRounds.IsHoldingForRestore()) diff --git a/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs b/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs index dd113fb2..8099eeee 100644 --- a/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs +++ b/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs @@ -13,12 +13,8 @@ namespace FiveStack; public class GameBackUpRounds { - private int? _resetRound; + private readonly RoundRestoreState _restore; - // A server that came back behind the backend, holding the round it can - // restore until every player is back to vote on it. Meanwhile nothing is - // played, published or written over the backups being restored. - private int? _pendingRestore; private readonly MatchEvents _matchEvents; private readonly GameServer _gameServer; private readonly MatchService _matchService; @@ -38,9 +34,12 @@ public GameBackUpRounds( MatchService matchService, IServiceProvider serviceProvider, EnvironmentService environmentService, - IStringLocalizer localizer + IStringLocalizer localizer, + RoundRestoreState restore ) { + _restore = restore; + _restore.LoadStarted += matchEvents.ClearPendingRoundResult; _logger = logger; _matchEvents = matchEvents; _gameServer = gameServer; @@ -77,12 +76,21 @@ public void Setup() public bool IsResettingRound() { - return _resetRound != null || _pendingRestore != null; + return _restore.Resetting; + } + + public bool HoldsRoundResults() + { + return RoundRestoreGateUtility.HoldsRoundResults( + IsResettingRound(), + restoreRoundVote?.IsVoteActive() == true, + _restore.HoldingRounds + ); } public bool IsHoldingForRestore() { - return _pendingRestore != null; + return _restore.HoldingRounds; } public void SetBackupWriting(bool enabled) @@ -94,12 +102,12 @@ public void SetBackupWriting(bool enabled) // player is back. Called when the match loads and as players connect. public void OfferPendingRestore() { - if (_pendingRestore == null || restoreRoundVote != null) + if (!_restore.CanOfferVote || restoreRoundVote != null) { return; } - int round = _pendingRestore.Value; + int round = _restore.HeldRound!.Value; int connectedPlayers = MatchUtility.Players().Count; int expectedPlayers = _matchService.GetCurrentMatch()?.GetExpectedPlayerCount() ?? 10; @@ -112,13 +120,13 @@ public void OfferPendingRestore() return; } - _pendingRestore = null; + _restore.ReleaseHold(); RequestRestoreBackupRound(round, null, true, recovering: true); // Refused for another reason: keep holding for the next player. - if (_resetRound == null) + if (!_restore.InProgress && HoldStaysOn(round)) { - _pendingRestore = round; + _restore.Hold(round, ActiveMapId()); } } @@ -162,14 +170,158 @@ public bool CheckForBackupRestore(eMapStatus status) _logger.LogInformation( $"Backup round {highestNumber} is ahead of current round {totalRoundsPlayed}, holding the match for a restore vote" ); - _pendingRestore = highestNumber; + HoldForRestore(highestNumber); + + return false; + } + + private void HoldForRestore(int round) + { + _restore.Hold(round, ActiveMapId()); SetBackupWriting(false); _gameServer.SendCommands(["mp_pause_match"]); OfferPendingRestore(); + } + + // A load that is owed and not coming goes back to the hold, and never to a + // replay: only players deciding that gives up rounds the backend recorded. + private void ReturnToHold(int round) + { + if (!HoldStaysOn(round)) + { + return; + } + + _logger.LogWarning($"Round {round} was not restored, holding the match for another vote"); + HoldForRestore(round); + } + + // False once the hold has moved to another round or ended. + private bool HoldStaysOn(int round) + { + MatchManager? matchManager = _matchService.GetCurrentMatch(); + eMapStatus status = matchManager?.GetCurrentMapStatus() ?? eMapStatus.Unknown; + int? next = RoundRestoreGateUtility.RoundToHold( + status, + matchManager?.GetCurrentMap()?.rounds ?? Array.Empty(), + round, + _gameServer.GetTotalRoundsPlayed() + ); + + if (next == round) + { + return true; + } + + if (RoundRestoreGateUtility.IsInPlay(status)) + { + _gameServer.Message( + HudDestination.Alert, + RoundRestoreGateUtility.WhyTheHoldMoved(round, next) + ); + } + + if (next is int moved) + { + HoldForRestore(moved); + return false; + } + + SetBackupWriting(!IsHoldingForRestore()); return false; } + private void ReturnOwedToHold() + { + if (_restore.FailedLoad() is int round) + { + ReturnToHold(round); + } + } + + // A hold turns backups off itself and offers its vote straight away, so + // nothing here may read the hold after that. + private void FailLoad() + { + if (_restore.FailedLoad() is int round) + { + ReturnToHold(round); + return; + } + + SetBackupWriting(!IsHoldingForRestore()); + } + + private Guid? ActiveMapId() + { + MatchManager? matchManager = _matchService.GetCurrentMatch(); + + return matchManager?.GetActiveMapId() + ?? matchManager?.GetMatchData()?.current_match_map_id; + } + + private void OweLoad(int round) + { + int owing = _restore.StartOwing(round, ActiveMapId()); + + TimerUtility.AddTimer(RoundRestoreState.OwedSeconds, () => OwedLoadTimedOut(owing)); + } + + private void OwedLoadTimedOut(int owing) + { + OwedLoadTimeout timeout = _restore.TimedOut(owing, ActiveMapId()); + + if (!timeout.Expired) + { + return; + } + + _matchEvents.WithdrawRestoreRequests(); + + if (timeout.HoldRound is int round) + { + ReturnToHold(round); + return; + } + + SetBackupWriting(!IsHoldingForRestore()); + } + + // A dedicated server keeps this state from one match to the next. + public void ForgetOtherMaps(Guid? map) + { + if (_restore.LeaveMap(map)) + { + restoreRoundVote?.Abandon(); + restoreRoundVote = null; + SetBackupWriting(!IsHoldingForRestore()); + } + } + + public void TellWhyResumeWaits() + { + string? refusal = RoundRestoreGateUtility.ResumeRefusal( + _restore.OwedRound, + _restore.NeedsRestartMap + ); + + if (refusal != null) + { + _gameServer.Message(HudDestination.Alert, refusal); + } + } + + // Restart Map starts the map over, so whatever a restore was protecting is + // gone and the restore with it. + public void AbandonRestore() + { + restoreRoundVote?.Abandon(); + restoreRoundVote = null; + _restore.Abandon(); + SetBackupWriting(true); + } + // A match whose rounds can't be restored replays the map from warmup, but // only while the map is still being played: a restart after it ended must // keep the rounds it recorded. @@ -236,7 +388,7 @@ public void RequestRestoreBackupRound( if (player != null || vote == true) { - _resetRound = round; + _restore.StartVote(recovering, ActiveMapId()); restoreRoundVote = _serviceProvider.GetRequiredService(typeof(VoteSystem)) as VoteSystem; @@ -252,7 +404,8 @@ public void RequestRestoreBackupRound( () => { _logger.LogInformation("restore round vote passed"); - _resetRound = null; + _restore.End(); + restoreRoundVote = null; string? refusal = RestoreRefusal(round); if (refusal != null) @@ -264,14 +417,31 @@ public void RequestRestoreBackupRound( HudDestination.Alert, $" Restore round blocked: {refusal}." ); - restoreRoundVote = null; - if (recovering) + if (recovering && HoldStaysOn(round)) { - _pendingRestore = round; + _restore.Hold(round, ActiveMapId()); } return; } + // A round that ended during the vote is the one being undone, + // and the next round start can come before the load does. + _matchEvents.ClearPendingRoundResult(); + + if ( + RoundRestoreGateUtility.OwesLoad( + recovering, + _environmentService.IsOfflineMode() + ) + ) + { + OweLoad(round); + } + else if (recovering) + { + SetBackupWriting(true); + } + SendRestoreRoundToBackend(round); }, () => @@ -279,7 +449,7 @@ public void RequestRestoreBackupRound( _logger.LogInformation("restore round vote failed"); restoreRoundVote = null; - _resetRound = null; + _restore.End(); // Declined after a restart: the map starts again from 0-0 and // the rounds the backend recorded are set aside. @@ -378,7 +548,7 @@ public void SendRestoreRoundToBackend(int round) public void RestoreRound(int round) { - if (_resetRound != null) + if (_restore.VotingOrLoading) { _logger.LogWarning($"Restore already in progress, ignoring RestoreRound({round})"); return; @@ -386,6 +556,7 @@ public void RestoreRound(int round) if (!CanRestoreRound(round)) { + ReturnOwedToHold(); return; } @@ -394,6 +565,7 @@ public void RestoreRound(int round) if (match == null || matchMap == null) { + ReturnOwedToHold(); return; } @@ -402,6 +574,7 @@ public void RestoreRound(int round) if (backupRound == null) { _logger.LogWarning($"missing backup round: {round}"); + ReturnOwedToHold(); return; } @@ -412,37 +585,52 @@ public void RestoreRound(int round) backupRoundFileName ); - File.WriteAllText(backupRoundFilePath, backupRound.backup_file); + int? held = _restore.ReleaseHold(); - _pendingRestore = null; - _resetRound = round; + _restore.StartLoad(held); + + try + { + File.WriteAllText(backupRoundFilePath, backupRound.backup_file); + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed writing restore backup round file {File}", + backupRoundFileName + ); + FailLoad(); + return; + } _logger.LogInformation($"Loading backup round file {backupRoundFileName}"); - Server.NextFrame(async () => + Server.NextFrame(() => { - _gameServer.SendCommands([$"mp_backup_restore_load_file {backupRoundFileName}"]); - _matchService.GetCurrentMatch()?.PauseMatch(); + _ = Task.Delay(5 * 1000) + .ContinueWith(_ => + Server.NextFrame(() => + { + _restore.End(); + SetBackupWriting(true); - await Task.Delay(5 * 1000); + _logger.LogInformation($"Sending Message for Round {round}"); - Server.NextFrame(() => - { - _resetRound = null; - SetBackupWriting(true); - - _logger.LogInformation($"Sending Message for Round {round}"); - - _gameServer.Message( - HudDestination.Alert, - _localizer[ - "backup.round_restored", - ChatColors.Red, - round, - CommandUtility.PublicChatTrigger - ] + _gameServer.Message( + HudDestination.Alert, + _localizer[ + "backup.round_restored", + ChatColors.Red, + round, + CommandUtility.PublicChatTrigger + ] + ); + }) ); - }); + + _gameServer.SendCommands([$"mp_backup_restore_load_file {backupRoundFileName}"]); + _matchService.GetCurrentMatch()?.PauseMatch(); }); } @@ -456,7 +644,7 @@ public void RestoreRound(int round) return RoundRestoreGateUtility.Refusal( match != null && !match.options.CanRestoreRounds(), - _resetRound != null, + _restore.VotingOrLoading, _matchService.GetCurrentMatch()?.GetCurrentMapStatus() ?? eMapStatus.Unknown, round, matchMap != null diff --git a/apps/counterstrikesharp/src/FiveStack.Services/MatchEvents.cs b/apps/counterstrikesharp/src/FiveStack.Services/MatchEvents.cs index f45f2fe6..be6c02af 100644 --- a/apps/counterstrikesharp/src/FiveStack.Services/MatchEvents.cs +++ b/apps/counterstrikesharp/src/FiveStack.Services/MatchEvents.cs @@ -46,14 +46,17 @@ EventData> Payload private readonly MatchService _matchService; private readonly EnvironmentService _environmentService; private readonly GameServer _gameServer; + private readonly RoundRestoreState _restore; public MatchEvents( ILogger logger, EnvironmentService environmentService, MatchService matchService, - GameServer gameServer + GameServer gameServer, + RoundRestoreState restore ) { + _restore = restore; _logger = logger; _matchService = matchService; _environmentService = environmentService; @@ -148,8 +151,31 @@ public void PublishMapStatus(eMapStatus status, Guid? winningLineupId) PublishGameEvent("mapStatus", data); } + // The api answers whatever reaches it, however late, and an unacknowledged + // event is resent. A request given up on has to stop going out, or it can land + // on top of what the players decided next. + public void WithdrawRestoreRequests() + { + foreach (var message in _pendingMessages) + { + if ( + message.Value.Event.data?.TryGetValue("event", out object? name) == true + && name as string == "restoreRound" + ) + { + _pendingMessages.TryRemove(message.Key, out _); + } + } + } + public void PublishGameEvent(string Event, Dictionary Data) { + if (RoundRestoreGateUtility.DroppedDuringLoad(Event, _restore.LoadingBackup)) + { + _logger.LogInformation($"Game event '{Event}' dropped: a backup round is loading"); + return; + } + if (_environmentService.IsOfflineMode()) { return; diff --git a/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs b/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs index 0b970f07..7437e017 100644 --- a/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs +++ b/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs @@ -305,6 +305,7 @@ public void ResumeMatch(string? message = null, bool skipUpdate = false) if (_backUpManagement.IsResettingRound()) { _logger.LogInformation("Resetting round, cannot resume match"); + _backUpManagement.TellWhyResumeWaits(); return; } @@ -318,12 +319,6 @@ public void ResumeMatch(string? message = null, bool skipUpdate = false) _timeoutSystem.resumeVote = null; } - if (_backUpManagement.restoreRoundVote != null) - { - _backUpManagement.restoreRoundVote.CancelVote(); - _backUpManagement.restoreRoundVote = null; - } - _timeoutSystem.ClearPendingTeamResumes(); if (!IsPaused()) @@ -565,6 +560,7 @@ public void SetupMatch(MatchData match) bool wasAlreadySetup = _configuredMapId == _currentMap.id; _activeMapId = _currentMap.id; + _backUpManagement.ForgetOtherMaps(_activeMapId); if (_currentMapStatus == eMapStatus.WaitingForTV) { diff --git a/apps/counterstrikesharp/src/FiveStack.Services/VoteSystem.cs b/apps/counterstrikesharp/src/FiveStack.Services/VoteSystem.cs index 75200b4d..65bbf57e 100644 --- a/apps/counterstrikesharp/src/FiveStack.Services/VoteSystem.cs +++ b/apps/counterstrikesharp/src/FiveStack.Services/VoteSystem.cs @@ -90,8 +90,23 @@ public void StartVote( SendVoteMessage(); } + // Over with no outcome: neither callback runs. + public void Abandon() + { + _voteFailedCallback = null; + _voteSuccessCallback = null; + CancelVote(); + } + public void CancelVote() { + // The failure callback can replay a map from 0-0, so a vote that has + // already passed must not reach it. + if (!IsVoteActive()) + { + return; + } + _voteStartTime = null; _voteTimeout = null; KillTimers(); diff --git a/apps/counterstrikesharp/src/FiveStackServiceCollection.cs b/apps/counterstrikesharp/src/FiveStackServiceCollection.cs index 77957669..acd4d4bc 100644 --- a/apps/counterstrikesharp/src/FiveStackServiceCollection.cs +++ b/apps/counterstrikesharp/src/FiveStackServiceCollection.cs @@ -1,4 +1,5 @@ using CounterStrikeSharp.API.Core; +using FiveStack.Utilities; using Microsoft.Extensions.DependencyInjection; namespace FiveStack; @@ -9,6 +10,7 @@ public void ConfigureServices(IServiceCollection serviceCollection) { serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); diff --git a/apps/counterstrikesharp/test/BackupWritingTests.cs b/apps/counterstrikesharp/test/BackupWritingTests.cs new file mode 100644 index 00000000..6a8f3790 --- /dev/null +++ b/apps/counterstrikesharp/test/BackupWritingTests.cs @@ -0,0 +1,122 @@ +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using FiveStack.Utilities; +using Xunit; + +// A restore that could not write its file never reaches the timer that turns +// backups back on. What the state decides is tested. RestoreRound needs the engine +// from its first line, so its half of this is read. +public class BackupWritingTests +{ + private const int HeldRound = 5; + private const int Round = 8; + + private static readonly Guid Map = Guid.NewGuid(); + + private readonly RoundRestoreState _state = new(); + + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Method(string signature) + { + string source = Regex.Replace( + File.ReadAllText(Path.Combine(SourceDir(), "FiveStack.Services", "GameBackUpRounds.cs")), + @"//[^\n]*", + "" + ); + + int from = source.IndexOf(signature, StringComparison.Ordinal); + + Assert.True(from >= 0, $"{signature} is gone"); + + int to = source.IndexOf("\n }\n", from, StringComparison.Ordinal); + + Assert.True(to > from, $"the end of {signature} was not found"); + + return source[from..to]; + } + + private static string FailedWrite() + { + string restore = Method("public void RestoreRound(int round)"); + int write = restore.IndexOf("File.WriteAllText(", StringComparison.Ordinal); + int caught = restore.IndexOf("catch (Exception", write, StringComparison.Ordinal); + int load = restore.IndexOf("mp_backup_restore_load_file", StringComparison.Ordinal); + + Assert.True(write >= 0, "RestoreRound no longer writes the backup file"); + Assert.True(caught > write && caught < load, "a failed write is not caught before the load"); + + return restore[caught..load]; + } + + [Fact] + public void AnOwedLoadWhoseWriteFailedHoldsItsRoundAgain() + { + _state.StartOwing(Round, Map); + _state.StartLoad(); + + Assert.Equal(Round, _state.FailedLoad()); + Assert.False(_state.LoadingBackup); + } + + [Fact] + public void ALoadThatTookOverAHoldPutsItBack() + { + _state.StartLoad(HeldRound); + + Assert.Equal(HeldRound, _state.FailedLoad()); + } + + [Fact] + public void AnOwedLoadKeepsItsOwnRoundOverAHoldItTookOver() + { + _state.StartOwing(Round, Map); + _state.StartLoad(HeldRound); + + Assert.Equal(Round, _state.FailedLoad()); + } + + [Fact] + public void ALoadWithNothingHeldHoldsNothing() + { + _state.StartLoad(); + + Assert.Null(_state.FailedLoad()); + } + + [Fact] + public void TheLoadIsToldWhatItTookOver() + { + string restore = Method("public void RestoreRound(int round)"); + int taken = restore.IndexOf("int? held = _restore.ReleaseHold();", StringComparison.Ordinal); + int load = restore.IndexOf("_restore.StartLoad(held);", StringComparison.Ordinal); + int write = restore.IndexOf("File.WriteAllText(", StringComparison.Ordinal); + + Assert.True(taken >= 0 && load > taken, "the hold is let go with nothing keeping it"); + Assert.True(load < write, "a failed write has nothing to put back"); + } + + [Fact] + public void AFailedWriteIsSettledOneWay() + { + Assert.Contains("FailLoad", FailedWrite()); + } + + // A hold offers its vote straight away, which empties it, so anything that + // reads the hold afterwards would turn backups on during that vote. + [Fact] + public void BackupsOnlyComeBackWhenTheFailedLoadHoldsNothing() + { + string failed = Method("private void FailLoad()"); + int asked = failed.IndexOf("_restore.FailedLoad()", StringComparison.Ordinal); + int hold = failed.IndexOf("ReturnToHold(", StringComparison.Ordinal); + int returned = failed.IndexOf("return;", hold, StringComparison.Ordinal); + int writing = failed.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.True(asked >= 0 && hold > asked, "the state is not asked before acting"); + Assert.True(returned > hold && writing > returned, "backups can come back during the vote"); + } +} diff --git a/apps/counterstrikesharp/test/DeclinedVoteRoundTests.cs b/apps/counterstrikesharp/test/DeclinedVoteRoundTests.cs new file mode 100644 index 00000000..17594e23 --- /dev/null +++ b/apps/counterstrikesharp/test/DeclinedVoteRoundTests.cs @@ -0,0 +1,183 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using FiveStack; +using FiveStack.Utilities; +using Xunit; + +// A .reset vote leaves the round running until the next freezetime, so a round +// that ends during one is real unless the vote passes. A recovery vote is the +// other way round: the server is behind the backend, and has nothing to report. +public class DeclinedVoteRoundTests +{ + private const int Round = 4; + + private readonly RoundRestoreState _restore = new(); + + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Code(params string[] path) + { + return Regex.Replace( + File.ReadAllText(Path.Combine(new[] { SourceDir() }.Concat(path).ToArray())), + @"//[^\n]*", + "" + ); + } + + private static void Set(object target, string field, object? value) + { + target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(target, value); + } + + private static VoteSystem FinishedVote() + { + return (VoteSystem)RuntimeHelpers.GetUninitializedObject(typeof(VoteSystem)); + } + + private static VoteSystem OpenVote() + { + VoteSystem vote = FinishedVote(); + FieldInfo timer = typeof(VoteSystem).GetField( + "_playerMessageTimer", + BindingFlags.NonPublic | BindingFlags.Instance + )!; + + timer.SetValue(vote, RuntimeHelpers.GetUninitializedObject(timer.FieldType)); + + return vote; + } + + private GameBackUpRounds Backups(VoteSystem? vote = null) + { + GameBackUpRounds backups = (GameBackUpRounds) + RuntimeHelpers.GetUninitializedObject(typeof(GameBackUpRounds)); + + Set(backups, "_restore", _restore); + backups.restoreRoundVote = vote; + + return backups; + } + + [Fact] + public void ARoundEndingDuringAResetVoteIsKept() + { + _restore.StartVote(); + + Assert.False(Backups(OpenVote()).HoldsRoundResults()); + } + + [Fact] + public void NothingIsHeldWithNoRestoreGoingOn() + { + Assert.False(Backups().HoldsRoundResults()); + } + + [Fact] + public void ARecoveryVoteHoldsResultsWhileItIsOpen() + { + _restore.StartVote(recovering: true); + + Assert.True(Backups(OpenVote()).HoldsRoundResults()); + } + + [Fact] + public void ARecoveryVoteKeepsTheBackupsHeld() + { + _restore.StartVote(recovering: true); + + Assert.True(Backups(OpenVote()).IsHoldingForRestore()); + } + + [Fact] + public void AResetVoteHoldsNoBackups() + { + _restore.StartVote(); + + Assert.False(Backups(OpenVote()).IsHoldingForRestore()); + } + + [Fact] + public void ARestoreHeldForPlayersHoldsResults() + { + _restore.Hold(Round); + + Assert.True(Backups().HoldsRoundResults()); + } + + [Fact] + public void ALoadingBackupHoldsResults() + { + _restore.StartLoad(); + + Assert.True(Backups().HoldsRoundResults()); + } + + [Fact] + public void ALoadAfterAVoteFinishedHoldsResults() + { + _restore.StartLoad(); + + Assert.True(Backups(FinishedVote()).HoldsRoundResults()); + } + + [Fact] + public void NoScoreSlipsOutWhileALoadIsOwed() + { + _restore.StartOwing(Round, Guid.NewGuid()); + + Assert.True(Backups(FinishedVote()).HoldsRoundResults()); + } + + [Fact] + public void ARestoreSaysWhenItsLoadStarts() + { + int started = 0; + _restore.LoadStarted += () => started++; + + _restore.StartVote(); + _restore.End(); + _restore.StartOwing(Round, Guid.NewGuid()); + + Assert.Equal(0, started); + + _restore.StartLoad(); + + Assert.Equal(1, started); + } + + [Fact] + public void ALoadForgetsTheResultOfTheRoundItUndoes() + { + Assert.Contains( + "_restore.LoadStarted += matchEvents.ClearPendingRoundResult;", + Code("FiveStack.Services", "GameBackUpRounds.cs") + ); + } + + [Theory] + [InlineData("RoundEnd.cs", 2)] + [InlineData("RoundStart.cs", 1)] + public void TheRoundHooksAskWhetherResultsAreHeld(string file, int sites) + { + string source = Code("FiveStack.Events", file); + + Assert.DoesNotContain("IsResettingRound", source); + Assert.Equal(sites, Regex.Matches(source, @"_gameBackupRounds\.HoldsRoundResults\(\)").Count); + } + + [Fact] + public void AVoteSaysWhetherItIsARecoveryVote() + { + Assert.Contains( + "_restore.StartVote(recovering, ", + Code("FiveStack.Services", "GameBackUpRounds.cs") + ); + } +} diff --git a/apps/counterstrikesharp/test/PendingRoundResultTests.cs b/apps/counterstrikesharp/test/PendingRoundResultTests.cs new file mode 100644 index 00000000..305a88b1 --- /dev/null +++ b/apps/counterstrikesharp/test/PendingRoundResultTests.cs @@ -0,0 +1,64 @@ +using System.Runtime.CompilerServices; +using FiveStack; +using FiveStack.Utilities; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +// The real state and the real clear, joined the way GameBackUpRounds' constructor +// joins them: that constructor can reach native code, so the line itself is read +// in DeclinedVoteRoundTests. Only here: the snapshot holds a team enum, and the +// other plugin's enum cannot load without its native package. +public class PendingRoundResultTests : IAsyncLifetime +{ + private readonly RoundRestoreState _restore = new(); + private readonly MatchEvents _events; + + public PendingRoundResultTests() + { + _events = new MatchEvents( + NullLogger.Instance, + (EnvironmentService)RuntimeHelpers.GetUninitializedObject(typeof(EnvironmentService)), + (MatchService)RuntimeHelpers.GetUninitializedObject(typeof(MatchService)), + null!, + _restore + ); + + _restore.LoadStarted += _events.ClearPendingRoundResult; + + _events.PendingRoundResult = new MatchEvents.RoundResultSnapshot { Round = 9 }; + } + + public Task InitializeAsync() + { + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + return _events.Disconnect(); + } + + [Fact] + public void ALoadForgetsTheRoundItUndoes() + { + _restore.StartLoad(); + + Assert.Null(_events.PendingRoundResult); + } + + [Fact] + public void AVoteLeavesItAlone() + { + _restore.StartVote(); + + Assert.NotNull(_events.PendingRoundResult); + } + + [Fact] + public void AnOwedLoadLeavesItAloneUntilItStarts() + { + _restore.StartOwing(9, Guid.NewGuid()); + + Assert.NotNull(_events.PendingRoundResult); + } +} diff --git a/apps/counterstrikesharp/test/RestoreVoteTests.cs b/apps/counterstrikesharp/test/RestoreVoteTests.cs new file mode 100644 index 00000000..46a5083e --- /dev/null +++ b/apps/counterstrikesharp/test/RestoreVoteTests.cs @@ -0,0 +1,61 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using FiveStack; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +// A recovery vote's failure callback replays the map from 0-0, which sets every +// round the backend recorded aside. It must only ever run for a vote that lost. +public class RestoreVoteTests +{ + private int _failures; + private int _passes; + + private static void Set(object target, string field, object? value) + { + target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(target, value); + } + + private static object? Get(object target, string field) + { + return target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(target); + } + + private VoteSystem FinishedVote() + { + VoteSystem vote = (VoteSystem)RuntimeHelpers.GetUninitializedObject(typeof(VoteSystem)); + + Set(vote, "_logger", NullLogger.Instance); + Set(vote, "_voteFailedCallback", (Action)(() => _failures++)); + Set(vote, "_voteSuccessCallback", (Action)(() => _passes++)); + + return vote; + } + + [Fact] + public void AFinishedVoteCannotBeCancelledIntoItsFailure() + { + FinishedVote().CancelVote(); + + Assert.Equal(0, _failures); + } + + [Fact] + public void AbandoningAFinishedVoteRunsNoCallback() + { + VoteSystem vote = FinishedVote(); + + vote.Abandon(); + + Assert.Equal(0, _failures); + Assert.Equal(0, _passes); + Assert.Null(Get(vote, "_voteFailedCallback")); + Assert.Null(Get(vote, "_voteSuccessCallback")); + } +} diff --git a/apps/counterstrikesharp/test/RestoreWiringTests.cs b/apps/counterstrikesharp/test/RestoreWiringTests.cs new file mode 100644 index 00000000..bd6d21b9 --- /dev/null +++ b/apps/counterstrikesharp/test/RestoreWiringTests.cs @@ -0,0 +1,335 @@ +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Xunit; + +// What the state decides is tested in RoundRestoreStateTests. Calling it needs +// the engine, so these read the plugin. +public class RestoreWiringTests +{ + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Code(params string[] path) + { + return Regex.Replace( + File.ReadAllText(Path.Combine(new[] { SourceDir() }.Concat(path).ToArray())), + @"//[^\n]*", + "" + ); + } + + private static string Backups() + { + return Code("FiveStack.Services", "GameBackUpRounds.cs"); + } + + private static string Method(string source, string signature) + { + int from = source.IndexOf(signature, StringComparison.Ordinal); + + Assert.True(from >= 0, $"{signature} is gone"); + + int to = source.IndexOf("\n }\n", from, StringComparison.Ordinal); + + Assert.True(to > from, $"the end of {signature} was not found"); + + return source[from..to]; + } + + private static string Between(string source, string start, string end) + { + int from = source.IndexOf(start, StringComparison.Ordinal); + int to = source.IndexOf(end, from + 1, StringComparison.Ordinal); + + Assert.True(from >= 0 && to > from, $"{start} has moved"); + + return source[from..to]; + } + + private static string PassedVote() + { + return Between( + Method(Backups(), "public void RequestRestoreBackupRound("), + "\"restore round vote passed\"", + "\"restore round vote failed\"" + ); + } + + [Fact] + public void AnOrganizerRestoreIsOnlyRefusedForAVoteOrALoadAsOnMain() + { + string refusal = Method(Backups(), "public string? RestoreRefusal(int round)"); + + Assert.Contains("_restore.VotingOrLoading,", refusal); + Assert.DoesNotContain("_restore.InProgress", refusal); + } + + [Fact] + public void AVoteIsLetGoAsSoonAsItPasses() + { + string passed = PassedVote(); + + Assert.Contains("restoreRoundVote = null;", Between(passed, "passed", "RestoreRefusal(round)")); + } + + [Fact] + public void APassedVoteAsksWhetherItOwesALoadBeforeSendingIt() + { + string passed = PassedVote(); + Match owes = Regex.Match( + passed, + @"RoundRestoreGateUtility\.OwesLoad\(\s*recovering,\s*_environmentService\.IsOfflineMode\(\)\s*\)" + ); + int sent = passed.IndexOf("SendRestoreRoundToBackend(round);", StringComparison.Ordinal); + + Assert.True(owes.Success, "a passed vote does not ask"); + Assert.True(sent > owes.Index, "the answer can beat the debt"); + } + + // A result captured during the vote is from a round the restore is about to + // undo, and it would publish at the next round start, before the load begins. + // A refused restore undoes nothing, so its result is real and stays. + [Fact] + public void APassedVoteForgetsTheResultOfARoundItIsAboutToUndo() + { + string passed = PassedVote(); + int refused = passed.IndexOf("return;", StringComparison.Ordinal); + int forgotten = passed.IndexOf("_matchEvents.ClearPendingRoundResult();", StringComparison.Ordinal); + int sent = passed.IndexOf("SendRestoreRoundToBackend(round);", StringComparison.Ordinal); + + Assert.True(forgotten >= 0, "a passed vote leaves the result to publish"); + Assert.True(forgotten > refused, "a refused restore throws away a real result"); + Assert.True(forgotten < sent, "the result can publish after the restore is sent"); + } + + [Fact] + public void AnOfflineRecoveryTurnsBackupsBackOnAndDeletesNothing() + { + string passed = PassedVote(); + + Assert.Matches( + @"\{\s*OweLoad\(round\);\s*\}\s*else if \(recovering\)\s*\{\s*SetBackupWriting\(true\);\s*\}", + passed + ); + Assert.DoesNotContain("ReplayMap", passed); + Assert.DoesNotContain("File.Delete", passed); + Assert.DoesNotContain("Backups(", passed); + } + + [Fact] + public void OnlyADeclinedVoteReplaysTheMap() + { + string vote = Method(Backups(), "public void RequestRestoreBackupRound("); + string declined = vote[vote.IndexOf("\"restore round vote failed\"", StringComparison.Ordinal)..]; + + Assert.Contains("ReplayMap(", declined); + Assert.DoesNotContain("ReplayMap", Method(Backups(), "private void OwedLoadTimedOut(int owing)")); + Assert.DoesNotContain("ReplayMap", Method(Backups(), "private void ReturnToHold(int round)")); + Assert.DoesNotContain("ReplayMap", Method(Backups(), "public void RestoreRound(int round)")); + } + + [Fact] + public void AnOwedLoadIsTimedFromTheMomentItIsOwed() + { + string owe = Method(Backups(), "private void OweLoad(int round)"); + + Assert.Contains("_restore.StartOwing(round, ActiveMapId())", owe); + Assert.Contains("RoundRestoreState.OwedSeconds", owe); + } + + [Fact] + public void TheAnswerIsNotGatedOnWhatWasAskedFor() + { + string restore = Method(Backups(), "public void RestoreRound(int round)"); + string opening = restore[restore.IndexOf('{')..restore.IndexOf("if (!CanRestoreRound(round))", StringComparison.Ordinal)]; + + Assert.Contains("if (_restore.VotingOrLoading)", opening); + Assert.Single(Regex.Matches(opening, @"\bif\s*\(")); + Assert.DoesNotContain("OwedRound", restore); + Assert.DoesNotMatch(@"\bround\s*(==|!=|<|>)", restore); + } + + [Fact] + public void EveryWayOutOfRestoreRoundSettlesAnOwedLoad() + { + string restore = Method(Backups(), "public void RestoreRound(int round)"); + string guarded = Between(restore, "CanRestoreRound(round)", "_restore.StartLoad("); + + MatchCollection exits = Regex.Matches(guarded, @"[^\n]*\n\s*return;"); + + Assert.Equal(3, exits.Count); + Assert.All(exits.Select(exit => exit.Value), exit => Assert.Contains("ReturnOwedToHold();", exit)); + } + + [Fact] + public void AHoldAsksWhereToGoAndOnlySpeaksWhenItMoves() + { + string check = Method(Backups(), "private bool HoldStaysOn(int round)"); + int asked = check.IndexOf("RoundRestoreGateUtility.RoundToHold(", StringComparison.Ordinal); + int stays = check.IndexOf("if (next == round)", StringComparison.Ordinal); + int told = check.IndexOf("RoundRestoreGateUtility.WhyTheHoldMoved(round, next)", StringComparison.Ordinal); + int moved = check.IndexOf("HoldForRestore(moved);", StringComparison.Ordinal); + int writing = check.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.Contains("_gameServer.GetTotalRoundsPlayed()", check); + Assert.True(asked >= 0 && stays > asked, "a hold stays on a round that can't load"); + Assert.True(told > stays, "players are told again by a hold that didn't move"); + Assert.True(moved > told, "a new hold is offered before players hear why"); + Assert.True(writing > moved, "backups come on with a hold still in place"); + Assert.DoesNotContain("ReplayMap", check); + } + + [Fact] + public void EveryHoldThatComesBackAsksWhereItBelongs() + { + string hold = Method(Backups(), "private void ReturnToHold(int round)"); + string offer = Method(Backups(), "public void OfferPendingRestore()"); + string vote = Method(Backups(), "public void RequestRestoreBackupRound("); + + Assert.True( + hold.IndexOf("HoldStaysOn(round)", StringComparison.Ordinal) >= 0 + && hold.IndexOf("HoldStaysOn(round)", StringComparison.Ordinal) + < hold.IndexOf("HoldForRestore(round);", StringComparison.Ordinal), + "a hold comes back for a round that can't load" + ); + Assert.Matches(@"HoldStaysOn\(round\)\s*\)\s*\{\s*_restore\.Hold\(", offer); + Assert.Matches(@"HoldStaysOn\(round\)\s*\)\s*\{\s*_restore\.Hold\(", vote); + } + + [Fact] + public void AResumeOnNothingToLoadSaysWhy() + { + Assert.Matches( + @"ResumeRefusal\(\s*_restore\.OwedRound,\s*_restore\.NeedsRestartMap\s*\)", + Method(Backups(), "public void TellWhyResumeWaits()") + ); + } + + [Fact] + public void AnOpenVoteIsAbandonedWithItsMap() + { + string forget = Method(Backups(), "public void ForgetOtherMaps(Guid? map)"); + int left = forget.IndexOf("_restore.LeaveMap(map)", StringComparison.Ordinal); + int abandoned = forget.IndexOf("restoreRoundVote?.Abandon();", StringComparison.Ordinal); + int cleared = forget.IndexOf("restoreRoundVote = null;", StringComparison.Ordinal); + int writing = forget.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.True(abandoned > left, "a vote outlives its map"); + Assert.True(cleared > abandoned && writing > cleared, "backups are put right too early"); + Assert.DoesNotContain("Withdraw", forget); + } + + [Fact] + public void AVoteRemembersItsMap() + { + Assert.Contains("_restore.StartVote(recovering, ActiveMapId());", Backups()); + } + + [Fact] + public void AnOwedLoadOnAnotherMapTurnsBackupsBackOnIfNothingHolds() + { + Assert.Contains( + "SetBackupWriting(!IsHoldingForRestore());", + Method(Backups(), "private void OwedLoadTimedOut(int owing)") + ); + } + + [Fact] + public void AnOwedLoadThatTimedOutStopsBeingResentBeforeTheHoldReturns() + { + string timedOut = Method(Backups(), "private void OwedLoadTimedOut(int owing)"); + int withdrawn = timedOut.IndexOf("_matchEvents.WithdrawRestoreRequests();", StringComparison.Ordinal); + int held = timedOut.IndexOf("ReturnToHold(", StringComparison.Ordinal); + + Assert.True(withdrawn >= 0, "the request can land after a declined vote"); + Assert.True(withdrawn < held, "the hold can be offered with the request still going out"); + } + + [Fact] + public void TheNextMatchLeavesWhatTheLastOneHeldAndPutsBackupsRight() + { + string forget = Method(Backups(), "public void ForgetOtherMaps(Guid? map)"); + int left = forget.IndexOf("_restore.LeaveMap(map)", StringComparison.Ordinal); + int writing = forget.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.True(left >= 0 && writing > left, "backups stay off with nothing holding them"); + } + + [Fact] + public void AMatchIsSetUpBeforeAnythingElseLooksAtItsHold() + { + string setup = Method(Code("FiveStack.Services", "MatchManager.cs"), "public void SetupMatch("); + int active = setup.IndexOf("_activeMapId = _currentMap.id;", StringComparison.Ordinal); + int forgotten = setup.IndexOf("_backUpManagement.ForgetOtherMaps(_activeMapId);", StringComparison.Ordinal); + int waiting = setup.IndexOf("eMapStatus.WaitingForTV", active, StringComparison.Ordinal); + + Assert.True(active >= 0 && forgotten > active, "a hold outlives its match"); + Assert.True(forgotten < waiting, "an early return skips it"); + } + + [Fact] + public void AHoldRemembersItsMapEverywhereItIsSet() + { + MatchCollection holds = Regex.Matches(Backups(), @"_restore\.Hold\(([^;]*)\);"); + + Assert.Equal(3, holds.Count); + Assert.All(holds.Select(hold => hold.Groups[1].Value), hold => Assert.EndsWith("ActiveMapId()", hold)); + } + + [Fact] + public void AHoldIsOnlyOfferedThroughTheState() + { + string offer = Method(Backups(), "public void OfferPendingRestore()"); + int asked = offer.IndexOf("_restore.CanOfferVote", StringComparison.Ordinal); + int taken = offer.IndexOf("_restore.ReleaseHold()", StringComparison.Ordinal); + + Assert.True(asked >= 0 && taken > asked, "a hold can be taken during a load"); + } + + [Fact] + public void RestartMapAbandonsTheRestore() + { + string abandon = Method(Backups(), "public void AbandonRestore()"); + + Assert.Contains("_restore.Abandon();", abandon); + Assert.Contains("SetBackupWriting(true);", abandon); + Assert.Contains(".Abandon();", abandon.Replace("_restore.Abandon();", "")); + Assert.DoesNotContain("CancelVote", abandon); + } + + [Fact] + public void RestartMapOnlyAbandonsOnceItWasAccepted() + { + string state = Between(Code("FiveStack.Commands", "Match.cs"), "SetMatchState(", "match?.UpdateMapStatus(status);"); + int refused = state.IndexOf("refusal != null", StringComparison.Ordinal); + int abandoned = state.IndexOf("_gameBackupRounds.AbandonRestore();", StringComparison.Ordinal); + + Assert.True(abandoned > refused, "a refused restart can drop a hold"); + } + + [Fact] + public void ResumingNoLongerReachesARestoreVote() + { + Assert.DoesNotContain("restoreRoundVote", Code("FiveStack.Services", "MatchManager.cs")); + } + + [Fact] + public void ResumeSaysWhyItIsWaiting() + { + string resume = Method(Code("FiveStack.Services", "MatchManager.cs"), "public void ResumeMatch("); + + Assert.Contains("TellWhyResumeWaits();", Between(resume, "IsResettingRound()", "return;")); + } + + [Fact] + public void AnOpenVoteStillRunsItsFailure() + { + string cancel = Method(Code("FiveStack.Services", "VoteSystem.cs"), "public void CancelVote()"); + int guarded = cancel.IndexOf("if (!IsVoteActive())", StringComparison.Ordinal); + int failed = cancel.IndexOf("_voteFailedCallback?.Invoke();", StringComparison.Ordinal); + + Assert.True(guarded >= 0 && failed > guarded, "CancelVote no longer fails an open vote"); + } +} diff --git a/apps/counterstrikesharp/test/RoundRestorePublishTests.cs b/apps/counterstrikesharp/test/RoundRestorePublishTests.cs new file mode 100644 index 00000000..ef55b7dd --- /dev/null +++ b/apps/counterstrikesharp/test/RoundRestorePublishTests.cs @@ -0,0 +1,154 @@ +using System.Collections.Concurrent; +using System.Reflection; +using System.Runtime.CompilerServices; +using FiveStack; +using FiveStack.Entities; +using FiveStack.Utilities; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +// Runs the plugin's own MatchEvents. With no socket a published event stops in +// _pendingMessages, and the publish queue keeps its order, so once the marker +// has arrived everything sent before it has been dealt with. +public class RoundRestorePublishTests : IAsyncLifetime +{ + private const string Marker = "restoreRound"; + + private readonly RoundRestoreState _restore = new(); + private readonly MatchEvents _events; + + public RoundRestorePublishTests() + { + MatchManager match = Bare(); + Set(match, "_matchData", new MatchData { id = Guid.NewGuid() }); + Set(match, "_activeMapId", (Guid?)Guid.NewGuid()); + + MatchService matchService = Bare(); + Set(matchService, "_currentMatch", match); + + _events = new MatchEvents( + NullLogger.Instance, + Bare(), + matchService, + null!, + _restore + ); + } + + public Task InitializeAsync() + { + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + return _events.Disconnect(); + } + + private static T Bare() + { + return (T)RuntimeHelpers.GetUninitializedObject(typeof(T)); + } + + private static void Set(object target, string field, object? value) + { + target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(target, value); + } + + private List Published() + { + var pending = (ConcurrentDictionary< + Guid, + (MatchEvents.EventData> Event, DateTime Timestamp) + >) + typeof(MatchEvents) + .GetField("_pendingMessages", BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(_events)!; + + return pending.Values.Select(entry => (string)entry.Event.data!["event"]).ToList(); + } + + private async Task> PublishThenSettle(string gameEvent) + { + _events.PublishGameEvent(gameEvent, new Dictionary()); + _events.PublishGameEvent(Marker, new Dictionary()); + + DateTime giveUp = DateTime.UtcNow.AddSeconds(5); + + while (!Published().Contains(Marker)) + { + Assert.True(DateTime.UtcNow < giveUp, "the marker never reached the publish step"); + await Task.Delay(5); + } + + return Published(); + } + + [Theory] + [InlineData("kill")] + [InlineData("assist")] + [InlineData("damage")] + [InlineData("utility")] + [InlineData("flash")] + [InlineData("objective")] + public async Task AStatEventIsNotPublishedWhileABackupLoads(string gameEvent) + { + _restore.StartLoad(); + + Assert.DoesNotContain(gameEvent, await PublishThenSettle(gameEvent)); + } + + [Theory] + [InlineData("kill")] + [InlineData("damage")] + [InlineData("objective")] + public async Task ARestoreVoteKeepsPublishingTheRoundBeingPlayed(string gameEvent) + { + _restore.StartVote(); + + Assert.Contains(gameEvent, await PublishThenSettle(gameEvent)); + } + + [Fact] + public async Task ARequestThatTimedOutStopsBeingResent() + { + Assert.Contains("kill", await PublishThenSettle("kill")); + + _events.WithdrawRestoreRequests(); + + Assert.DoesNotContain(Marker, Published()); + Assert.Contains("kill", Published()); + } + + [Fact] + public async Task AKillIsPublishedOnceTheLoadHasEnded() + { + _restore.StartLoad(); + _restore.End(); + + Assert.Contains("kill", await PublishThenSettle("kill")); + } + + [Fact] + public async Task ALoadAfterAPassedVoteStopsPublishing() + { + _restore.StartVote(); + _restore.End(); + _restore.StartLoad(); + + Assert.DoesNotContain("kill", await PublishThenSettle("kill")); + } + + [Theory] + [InlineData("score")] + [InlineData("mapStatus")] + public async Task ALoadLeavesEventsThatAreNotRoundStatsAlone(string gameEvent) + { + _restore.StartLoad(); + + Assert.Contains(gameEvent, await PublishThenSettle(gameEvent)); + } +} diff --git a/apps/counterstrikesharp/test/RoundRestoreStateTests.cs b/apps/counterstrikesharp/test/RoundRestoreStateTests.cs new file mode 100644 index 00000000..3ce2feac --- /dev/null +++ b/apps/counterstrikesharp/test/RoundRestoreStateTests.cs @@ -0,0 +1,583 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using FiveStack; +using FiveStack.Entities; +using FiveStack.Enums; +using FiveStack.Utilities; +using Xunit; + +// One debt: a recovery vote passed and the server is owed its load. Nothing else +// about a restore changes, so .reset, organizer restores and declined votes are +// pinned here as well as the new paths. +public class RoundRestoreStateTests +{ + private const int Round = 8; + + private static readonly Guid Map = Guid.NewGuid(); + private static readonly Guid OtherMap = Guid.NewGuid(); + + private readonly RoundRestoreState _state = new(); + + private GameBackUpRounds Backups() + { + GameBackUpRounds backups = (GameBackUpRounds) + RuntimeHelpers.GetUninitializedObject(typeof(GameBackUpRounds)); + + typeof(GameBackUpRounds) + .GetField("_restore", BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(backups, _state); + + return backups; + } + + [Fact] + public void AnOwedLoadCannotBeResumedOver() + { + _state.StartOwing(Round, Map); + + Assert.Equal(Round, _state.OwedRound); + Assert.True(Backups().IsResettingRound()); + Assert.True(Backups().IsHoldingForRestore()); + } + + [Fact] + public void AnOwedLoadDoesNotStandInTheWayOfItsAnswer() + { + _state.StartOwing(Round, Map); + + Assert.False(_state.VotingOrLoading); + } + + [Fact] + public void TheAnswerSettlesTheDebt() + { + _state.StartOwing(Round, Map); + _state.StartLoad(); + + Assert.Null(_state.OwedRound); + Assert.True(_state.LoadingBackup); + Assert.True(Backups().IsResettingRound()); + + _state.End(); + + Assert.False(Backups().IsResettingRound()); + } + + [Fact] + public void AnOwedLoadThatCouldNotStartGoesBackToTheHold() + { + _state.StartOwing(Round, Map); + + Assert.Equal(Round, _state.FailedLoad()); + Assert.False(_state.InProgress); + } + + [Fact] + public void AFailedLoadNobodyWasOwedHoldsNothing() + { + Assert.Null(_state.FailedLoad()); + } + + [Fact] + public void AnOwedLoadNobodyAnsweredGoesBackToTheHold() + { + int owing = _state.StartOwing(Round, Map); + + Assert.Equal(new OwedLoadTimeout(true, Round), _state.TimedOut(owing, Map)); + Assert.False(_state.InProgress); + } + + [Fact] + public void ADebtThatNeverWentOutStillComesBackToTheHold() + { + int owing = _state.StartOwing(Round, null); + + Assert.Equal(new OwedLoadTimeout(true, Round), _state.TimedOut(owing, null)); + } + + [Fact] + public void ATimerOnAnotherMapEndsQuietly() + { + int owing = _state.StartOwing(Round, Map); + + Assert.Equal(new OwedLoadTimeout(true), _state.TimedOut(owing, OtherMap)); + Assert.False(_state.InProgress); + } + + [Fact] + public void ATimerFromAnEarlierDebtDoesNothing() + { + int earlier = _state.StartOwing(Round, Map); + _state.End(); + _state.StartOwing(Round, Map); + + Assert.Equal(default, _state.TimedOut(earlier, Map)); + Assert.Equal(Round, _state.OwedRound); + } + + [Fact] + public void ATimerAfterTheLoadStartedDoesNothing() + { + int owing = _state.StartOwing(Round, Map); + _state.StartLoad(); + + Assert.Equal(default, _state.TimedOut(owing, Map)); + Assert.True(_state.LoadingBackup); + } + + [Fact] + public void ATimerAfterRestartMapDoesNothing() + { + int owing = _state.StartOwing(Round, Map); + _state.Abandon(); + + Assert.Equal(default, _state.TimedOut(owing, Map)); + } + + [Fact] + public void RestartMapClearsAHold() + { + _state.Hold(Round); + _state.Abandon(); + + Assert.Null(_state.HeldRound); + Assert.False(Backups().IsResettingRound()); + Assert.False(Backups().IsHoldingForRestore()); + } + + [Fact] + public void RestartMapClearsAnOwedLoad() + { + _state.StartOwing(Round, Map); + _state.Abandon(); + + Assert.False(Backups().IsResettingRound()); + Assert.False(Backups().IsHoldingForRestore()); + } + + [Fact] + public void RestartMapClearsAVoteAndAHoldTogether() + { + _state.Hold(Round); + _state.StartVote(); + _state.Abandon(); + + Assert.False(Backups().IsResettingRound()); + } + + [Fact] + public void AHoldIsOfferedItsVoteWhenNothingElseIsRunning() + { + _state.Hold(Round); + + Assert.True(_state.CanOfferVote); + } + + [Fact] + public void NoHoldMeansNoVote() + { + Assert.False(_state.CanOfferVote); + } + + [Fact] + public void ALoadInProgressKeepsAHoldFromBeingOffered() + { + _state.Hold(Round); + _state.StartLoad(); + + Assert.False(_state.CanOfferVote); + Assert.Equal(Round, _state.HeldRound); + } + + [Fact] + public void AnOwedLoadKeepsAHoldFromBeingOffered() + { + _state.Hold(Round); + _state.StartOwing(Round, Map); + + Assert.False(_state.CanOfferVote); + } + + [Fact] + public void AVoteInProgressKeepsAHoldFromBeingOffered() + { + _state.Hold(Round); + _state.StartVote(); + + Assert.False(_state.CanOfferVote); + } + + [Fact] + public void TakingTheHoldHandsBackItsRound() + { + _state.Hold(Round); + + Assert.Equal(Round, _state.ReleaseHold()); + Assert.Null(_state.HeldRound); + } + + [Fact] + public void AnAnswerDuringAVoteOrALoadIsIgnoredAsOnMain() + { + _state.StartVote(); + Assert.True(_state.VotingOrLoading); + + _state.End(); + _state.StartLoad(); + Assert.True(_state.VotingOrLoading); + } + + [Fact] + public void AnAnswerWithNothingGoingOnIsLoadedAsOnMain() + { + Assert.False(_state.VotingOrLoading); + } + + [Fact] + public void ResetToRoundZeroFindsItsBackupAsOnMain() + { + BackupRound first = new() { round = 0, backup_file = "round00" }; + BackupRound newest = new() { round = 0, backup_file = "round00 again" }; + BackupRound later = new() { round = Round, backup_file = "round08" }; + + Assert.Same(newest, RoundRestoreGateUtility.LatestBackup(new[] { first, later, newest }, 0)); + } + + [Fact] + public void ResetToRoundZeroIsRefusedOnlyForAMissingBackup() + { + Assert.Equal( + "the server has no backup of round 0", + RoundRestoreGateUtility.Refusal(false, false, eMapStatus.Live, 0, false, 10, 10, "Competitive") + ); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AnOwedLoadDoesNotStopRoundZeroLoading(bool owing) + { + if (owing) + { + _state.StartOwing(Round, Map); + } + + Assert.False(_state.VotingOrLoading); + Assert.Null( + RoundRestoreGateUtility.Refusal( + false, + _state.VotingOrLoading, + eMapStatus.Live, + 0, + true, + 10, + 10, + "Competitive" + ) + ); + } + + [Fact] + public void TheNextMatchEndsALoadOwedOnTheLastOne() + { + _state.StartOwing(Round, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.Null(_state.OwedRound); + Assert.False(_state.Resetting); + } + + [Fact] + public void TheNextMatchEndsAHoldOnTheLastOne() + { + _state.Hold(Round, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.Null(_state.HeldRound); + Assert.False(_state.HoldingRounds); + } + + [Fact] + public void SetupOnTheSameMapKeepsAHoldAndAnOwedLoad() + { + _state.Hold(Round, Map); + _state.StartOwing(Round, Map); + + Assert.False(_state.LeaveMap(Map)); + Assert.Equal(Round, _state.HeldRound); + Assert.Equal(Round, _state.OwedRound); + } + + [Fact] + public void ANewMatchWithNothingToLeaveChangesNothing() + { + Assert.False(_state.LeaveMap(OtherMap)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void AnOpenVoteEndsWithItsMap(bool recovering) + { + _state.StartVote(recovering, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.False(_state.VotingOrLoading); + Assert.False(_state.HoldingRounds); + Assert.False(_state.Resetting); + } + + [Fact] + public void AnOpenVoteStaysOnItsOwnMap() + { + _state.StartVote(map: Map); + + Assert.False(_state.LeaveMap(Map)); + Assert.True(_state.VotingOrLoading); + } + + [Theory] + [InlineData(5, true)] + [InlineData(4, true)] + [InlineData(6, false)] + [InlineData(Round, false)] + [InlineData(3, false)] + public void OnlyADiskRoundTheBackendHasARowForIsUsable(int round, bool usable) + { + BackupRound[] rounds = + { + new() { round = 4, backup_file = "kept" }, + new() { round = 5 }, + }; + + Assert.Equal(usable, RoundRestoreGateUtility.IsUsableOnDisk(rounds, round, false)); + } + + [Fact] + public void WithNoRowThereIsNothingOnDiskToUse() + { + Assert.False( + RoundRestoreGateUtility.IsUsableOnDisk(Array.Empty(), Round, false) + ); + } + + [Fact] + public void ARowARestoreUndidStaysUnusable() + { + BackupRound[] rounds = { new() { round = Round, deleted_at = "2026-11-02" } }; + + Assert.False(RoundRestoreGateUtility.IsUsableOnDisk(rounds, Round, false)); + } + + [Fact] + public void OfflineKeepsItsDiskRoundsAsOnMain() + { + BackupRound[] undone = { new() { round = 5, deleted_at = "2026-11-02" } }; + + Assert.True( + RoundRestoreGateUtility.IsUsableOnDisk(Array.Empty(), Round, true) + ); + Assert.False(RoundRestoreGateUtility.IsUsableOnDisk(undone, 5, true)); + } + + private static BackupRound Row(int round, string file = "", string deleted = "") + { + return new BackupRound + { + round = round, + backup_file = file, + deleted_at = deleted, + }; + } + + private static int? RoundToHold(BackupRound[] rounds, int round = Round, int played = 0) + { + return RoundRestoreGateUtility.RoundToHold(eMapStatus.Paused, rounds, round, played); + } + + [Fact] + public void AHoldStaysOnARoundThatCanStillLoad() + { + Assert.Equal(Round, RoundToHold(new[] { Row(5, "kept"), Row(Round, "kept") })); + } + + [Fact] + public void AGoneBackupFallsBackToTheNewestTheBackendKept() + { + Assert.Equal(5, RoundToHold(new[] { Row(4, "kept"), Row(5, "kept"), Row(Round) })); + } + + [Fact] + public void AGoneBackupDoesNotFallBackToARoundARestoreUndid() + { + BackupRound[] rounds = { Row(4, "kept"), Row(5, "undone", "2026-11-02"), Row(Round) }; + + Assert.Equal(4, RoundToHold(rounds)); + } + + [Fact] + public void NoBackupWithRoundsRecordedAheadNeedsRestartMap() + { + Assert.Equal(RoundRestoreState.NothingToLoad, RoundToHold(new[] { Row(5), Row(Round) })); + } + + [Fact] + public void NoRowLeftForTheHeldRoundStillProtectsTheOnesAhead() + { + Assert.Equal(RoundRestoreState.NothingToLoad, RoundToHold(new[] { Row(5) }, played: 2)); + } + + [Theory] + [InlineData(5)] + [InlineData(6)] + public void NothingRecordedAheadEndsTheHold(int played) + { + Assert.Null(RoundToHold(new[] { Row(5) }, played: played)); + } + + [Fact] + public void NoRowsAtAllEndsTheHold() + { + Assert.Null(RoundToHold(Array.Empty())); + } + + [Fact] + public void AnOldBackupIsNoReasonToHoldWithNothingAhead() + { + Assert.Null(RoundToHold(new[] { Row(5, "kept") }, played: 5)); + } + + [Fact] + public void AnOlderBackupStillRecoversRoundsRecordedAhead() + { + Assert.Equal(3, RoundToHold(new[] { Row(3, "kept"), Row(5) }, played: 4)); + } + + [Theory] + [InlineData(eMapStatus.Warmup)] + [InlineData(eMapStatus.Finished)] + public void AMapNotInPlayEndsTheHold(eMapStatus status) + { + BackupRound[] rounds = { Row(5, "kept"), Row(Round, "kept") }; + + Assert.Null(RoundRestoreGateUtility.RoundToHold(status, rounds, Round, 0)); + } + + [Fact] + public void AHoldOnNothingToLoadStaysPutSoPlayersAreOnlyToldOnce() + { + Assert.Equal( + RoundRestoreState.NothingToLoad, + RoundToHold(new[] { Row(Round) }, RoundRestoreState.NothingToLoad) + ); + } + + [Fact] + public void AHoldOnNothingToLoadKeepsTheMatchHeldWithNoVote() + { + _state.Hold(RoundRestoreState.NothingToLoad, Map); + + Assert.True(_state.NeedsRestartMap); + Assert.False(_state.CanOfferVote); + Assert.True(_state.HoldingRounds); + Assert.True(_state.Resetting); + } + + [Fact] + public void AnOrdinaryHoldDoesNotNeedRestartMap() + { + _state.Hold(Round, Map); + + Assert.False(_state.NeedsRestartMap); + Assert.True(_state.CanOfferVote); + } + + [Fact] + public void RestartMapEndsAHoldOnNothingToLoad() + { + _state.Hold(RoundRestoreState.NothingToLoad, Map); + _state.Abandon(); + + Assert.False(_state.NeedsRestartMap); + Assert.False(_state.HoldingRounds); + } + + [Fact] + public void AHoldOnNothingToLoadEndsWithItsMap() + { + _state.Hold(RoundRestoreState.NothingToLoad, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.False(_state.HoldingRounds); + } + + [Fact] + public void PlayersAreToldAnAdminNeedsRestartMap() + { + Assert.Contains( + "Restart Map", + RoundRestoreGateUtility.WhyTheHoldMoved(Round, RoundRestoreState.NothingToLoad) + ); + } + + [Theory] + [InlineData(5)] + [InlineData(null)] + public void PlayersAreToldWhichRoundCanNoLongerBeRestored(int? next) + { + string told = RoundRestoreGateUtility.WhyTheHoldMoved(Round, next); + + Assert.Contains(Round.ToString(), told); + Assert.DoesNotContain("Restart Map", told); + } + + [Fact] + public void AResumeSaysWhyItWaitsForRestartMap() + { + Assert.Equal( + RoundRestoreGateUtility.RestartMapNeeded, + RoundRestoreGateUtility.ResumeRefusal(null, needsRestartMap: true) + ); + } + + [Fact] + public void AnOwedLoadDoesNotRefuseAnOrganizerRestore() + { + _state.StartOwing(Round, Map); + + Assert.Null( + RoundRestoreGateUtility.Refusal( + false, + _state.VotingOrLoading, + eMapStatus.Paused, + Round, + true, + 10, + 10, + "Competitive" + ) + ); + } + + [Theory] + [InlineData(true, false, true)] + [InlineData(false, false, false)] + [InlineData(true, true, false)] + [InlineData(false, true, false)] + public void OnlyARecoveryOnAServerWithAPanelIsOwedALoad(bool recovering, bool offline, bool owed) + { + Assert.Equal(owed, RoundRestoreGateUtility.OwesLoad(recovering, offline)); + } + + [Fact] + public void PlayersAreToldWhyResumeWaits() + { + Assert.Contains(Round.ToString(), RoundRestoreGateUtility.ResumeRefusal(Round)); + } + + [Fact] + public void NothingIsSaidWhenNoLoadIsOwed() + { + Assert.Null(RoundRestoreGateUtility.ResumeRefusal(null)); + } +} diff --git a/apps/counterstrikesharp/test/RoundRestoreWiringTests.cs b/apps/counterstrikesharp/test/RoundRestoreWiringTests.cs new file mode 100644 index 00000000..dfa9c595 --- /dev/null +++ b/apps/counterstrikesharp/test/RoundRestoreWiringTests.cs @@ -0,0 +1,97 @@ +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Xunit; + +// RoundRestorePublishTests hands MatchEvents a state of its own, so it cannot +// see whether the plugin wires the same one in or calls it from the right place. +// That code needs the engine, so these read it. +public class RoundRestoreWiringTests +{ + private static readonly string[] BuildOutput = ["bin", "obj", "build"]; + + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Strip(string source) + { + return Regex.Replace(source, @"//[^\n]*", ""); + } + + private static string Service(string file) + { + return Strip(File.ReadAllText(Path.Combine(SourceDir(), "FiveStack.Services", file))); + } + + private static string Method(string source, string signature) + { + int from = source.IndexOf(signature, StringComparison.Ordinal); + + Assert.True(from >= 0, $"{signature} is gone"); + + int to = source.IndexOf("\n }\n", from, StringComparison.Ordinal); + + Assert.True(to > from, $"the end of {signature} was not found"); + + return source[from..to]; + } + + [Fact] + public void ThePluginHasOneRestoreState() + { + IEnumerable sources = Directory + .EnumerateFiles(SourceDir(), "*.cs", SearchOption.AllDirectories) + .Where(file => + !Path.GetRelativePath(SourceDir(), file) + .Split(Path.DirectorySeparatorChar) + .Any(BuildOutput.Contains) + ) + .Select(file => Strip(File.ReadAllText(file))); + + Assert.Single(sources, source => source.Contains("AddSingleton()")); + } + + [Theory] + [InlineData("MatchEvents.cs", "public MatchEvents(")] + [InlineData("GameBackUpRounds.cs", "public GameBackUpRounds(")] + public void BothSidesAreHandedThatState(string file, string constructor) + { + Assert.Contains("RoundRestoreState restore", Method(Service(file), constructor)); + } + + [Fact] + public void ALoadIsMarkedAsALoad() + { + string restore = Method(Service("GameBackUpRounds.cs"), "public void RestoreRound(int round)"); + + Assert.Contains("_restore.StartLoad(", restore); + Assert.DoesNotContain("_restore.StartVote(", restore); + } + + [Fact] + public void AVoteIsNotMarkedAsALoad() + { + string request = Method( + Service("GameBackUpRounds.cs"), + "public void RequestRestoreBackupRound(" + ); + + Assert.Contains("_restore.StartVote(", request); + Assert.DoesNotContain("_restore.StartLoad(", request); + } + + [Fact] + public void TheFilterComesBeforeAnythingElseInThePublish() + { + string publish = Method(Service("MatchEvents.cs"), "public void PublishGameEvent("); + int filter = publish.IndexOf( + "RoundRestoreGateUtility.DroppedDuringLoad(Event, _restore.LoadingBackup)", + StringComparison.Ordinal + ); + int offline = publish.IndexOf("IsOfflineMode", StringComparison.Ordinal); + + Assert.True(filter >= 0, "PublishGameEvent does not filter"); + Assert.True(filter < offline, "the filter runs after the first way out"); + } +} diff --git a/apps/swiftly/src/FiveStack.Commands/Match.cs b/apps/swiftly/src/FiveStack.Commands/Match.cs index d105ce1a..e1010f37 100644 --- a/apps/swiftly/src/FiveStack.Commands/Match.cs +++ b/apps/swiftly/src/FiveStack.Commands/Match.cs @@ -39,6 +39,8 @@ public void SetMatchState(ICommandContext context) context.Reply($"{RoundRestoreGateUtility.MatchStateRefusedPrefix} {refusal}"); return; } + + _gameBackupRounds.AbandonRestore(); } match?.UpdateMapStatus(status); diff --git a/apps/swiftly/src/FiveStack.Events/RoundEnd.cs b/apps/swiftly/src/FiveStack.Events/RoundEnd.cs index 3c562cf9..c3296340 100644 --- a/apps/swiftly/src/FiveStack.Events/RoundEnd.cs +++ b/apps/swiftly/src/FiveStack.Events/RoundEnd.cs @@ -36,7 +36,7 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event) match.UpdateMapStatus(eMapStatus.Overtime); } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.HoldsRoundResults()) { _logger.LogInformation("OnRoundOfficiallyEnded skipping capture: restoring round"); return HookResult.Continue; @@ -56,7 +56,7 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event) [GameEventHandler(HookMode.Post)] public HookResult OnRoundEnd(EventRoundEnd @event) { - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.HoldsRoundResults()) { _logger.LogInformation( $"OnRoundEnd ignored (restoring round): message={@event.Message}" diff --git a/apps/swiftly/src/FiveStack.Events/RoundStart.cs b/apps/swiftly/src/FiveStack.Events/RoundStart.cs index c76f7140..90bf3446 100644 --- a/apps/swiftly/src/FiveStack.Events/RoundStart.cs +++ b/apps/swiftly/src/FiveStack.Events/RoundStart.cs @@ -36,7 +36,7 @@ public HookResult OnRoundStart(EventRoundStart @event) return HookResult.Continue; } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.HoldsRoundResults()) { // A restart can unpause the round the hold paused. if (_gameBackupRounds.IsHoldingForRestore()) diff --git a/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs b/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs index acd3d752..cb7c3a42 100644 --- a/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs +++ b/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs @@ -13,12 +13,8 @@ namespace FiveStack; public class GameBackUpRounds { - private int? _resetRound; + private readonly RoundRestoreState _restore; - // A server that came back behind the backend, holding the round it can - // restore until every player is back to vote on it. Meanwhile nothing is - // played, published or written over the backups being restored. - private int? _pendingRestore; private readonly ISwiftlyCore _core; private readonly MatchEvents _matchEvents; private readonly GameServer _gameServer; @@ -40,9 +36,12 @@ public GameBackUpRounds( MatchService matchService, IServiceProvider serviceProvider, EnvironmentService environmentService, - ILocalizer localizer + ILocalizer localizer, + RoundRestoreState restore ) { + _restore = restore; + _restore.LoadStarted += matchEvents.ClearPendingRoundResult; _core = core; _logger = logger; _matchEvents = matchEvents; @@ -112,7 +111,7 @@ public void Setup() fileConVar.ValueAsString = prefix; } - SetBackupWriting(_pendingRestore == null); + SetBackupWriting(!IsHoldingForRestore()); // Read the values straight back from the engine to confirm the sets applied. string autoReadback = @@ -123,7 +122,7 @@ public void Setup() _logger.LogInformation( "Backup Setup: set mp_backup_round_file={Prefix}, mp_backup_round_auto={Writing} | readback auto={Auto} file={File} (files: {Dir}/{Prefix}_round.txt)", prefix, - _pendingRestore == null ? 1 : 0, + IsHoldingForRestore() ? 0 : 1, autoReadback, fileReadback, BackupDirectory, @@ -133,12 +132,21 @@ public void Setup() public bool IsResettingRound() { - return _resetRound != null || _pendingRestore != null; + return _restore.Resetting; + } + + public bool HoldsRoundResults() + { + return RoundRestoreGateUtility.HoldsRoundResults( + IsResettingRound(), + restoreRoundVote?.IsVoteActive() == true, + _restore.HoldingRounds + ); } public bool IsHoldingForRestore() { - return _pendingRestore != null; + return _restore.HoldingRounds; } private void SetBackupWriting(bool enabled) @@ -153,12 +161,12 @@ private void SetBackupWriting(bool enabled) // player is back. Called when the match loads and as players connect. public void OfferPendingRestore() { - if (_pendingRestore == null || restoreRoundVote != null) + if (!_restore.CanOfferVote || restoreRoundVote != null) { return; } - int round = _pendingRestore.Value; + int round = _restore.HeldRound!.Value; int connectedPlayers = MatchUtility.PlayerCount(); int expectedPlayers = _matchService.GetCurrentMatch()?.GetExpectedPlayerCount() ?? 10; @@ -171,13 +179,13 @@ public void OfferPendingRestore() return; } - _pendingRestore = null; + _restore.ReleaseHold(); RequestRestoreBackupRound(round, null, true, recovering: true); // Refused for another reason: keep holding for the next player. - if (_resetRound == null) + if (!_restore.InProgress && HoldStaysOn(round)) { - _pendingRestore = round; + _restore.Hold(round, ActiveMapId()); } } @@ -192,17 +200,22 @@ public bool CheckForBackupRestore(eMapStatus status) return false; } - // The backup files on disk (written by mp_backup_round_auto) hold rounds - // the backend may not have recorded, as with offline match data. A server - // that came back on another node or a fresh pod has none, only the - // backups the backend kept. + // The backup files on disk (written by mp_backup_round_auto) can fill in a + // round the backend recorded without one, and are all offline match data + // has. A server that came back on another node or a fresh pod has none, + // only the backups the backend kept. string csgoDir = BackupDirectory; string prefix = MatchUtility.GetSafeMatchPrefix(match); int highestOnDisk = GetHighestBackupRoundOnDisk( csgoDir, prefix, - round => RoundRestoreGateUtility.IsUndone(matchMap.rounds, round) + round => + !RoundRestoreGateUtility.IsUsableOnDisk( + matchMap.rounds, + round, + _environmentService.IsOfflineMode() + ) ); int highestNumber = RoundRestoreGateUtility.HighestRestorableRound( matchMap.rounds, @@ -241,9 +254,7 @@ public bool CheckForBackupRestore(eMapStatus status) { string content = File.ReadAllText(backupFilePath); matchMap.rounds = matchMap - .rounds.Append( - new BackupRound { round = highestNumber, backup_file = content } - ) + .rounds.Append(new BackupRound { round = highestNumber, backup_file = content }) .ToArray(); } } @@ -266,14 +277,158 @@ public bool CheckForBackupRestore(eMapStatus status) _logger.LogInformation( $"Backup round {highestNumber} is ahead of current round {totalRoundsPlayed}, holding the match for a restore vote" ); - _pendingRestore = highestNumber; + HoldForRestore(highestNumber); + + return false; + } + + private void HoldForRestore(int round) + { + _restore.Hold(round, ActiveMapId()); SetBackupWriting(false); _gameServer.SendCommands(["mp_pause_match"]); OfferPendingRestore(); + } + + // A load that is owed and not coming goes back to the hold, and never to a + // replay: only players deciding that gives up rounds the backend recorded. + private void ReturnToHold(int round) + { + if (!HoldStaysOn(round)) + { + return; + } + + _logger.LogWarning($"Round {round} was not restored, holding the match for another vote"); + HoldForRestore(round); + } + + // False once the hold has moved to another round or ended. + private bool HoldStaysOn(int round) + { + MatchManager? matchManager = _matchService.GetCurrentMatch(); + eMapStatus status = matchManager?.GetCurrentMapStatus() ?? eMapStatus.Unknown; + int? next = RoundRestoreGateUtility.RoundToHold( + status, + matchManager?.GetCurrentMap()?.rounds ?? Array.Empty(), + round, + _gameServer.GetTotalRoundsPlayed() + ); + + if (next == round) + { + return true; + } + + if (RoundRestoreGateUtility.IsInPlay(status)) + { + _gameServer.Message( + MessageType.Alert, + RoundRestoreGateUtility.WhyTheHoldMoved(round, next) + ); + } + + if (next is int moved) + { + HoldForRestore(moved); + return false; + } + + SetBackupWriting(!IsHoldingForRestore()); return false; } + private void ReturnOwedToHold() + { + if (_restore.FailedLoad() is int round) + { + ReturnToHold(round); + } + } + + // A hold turns backups off itself and offers its vote straight away, so + // nothing here may read the hold after that. + private void FailLoad() + { + if (_restore.FailedLoad() is int round) + { + ReturnToHold(round); + return; + } + + SetBackupWriting(!IsHoldingForRestore()); + } + + private Guid? ActiveMapId() + { + MatchManager? matchManager = _matchService.GetCurrentMatch(); + + return matchManager?.GetActiveMapId() + ?? matchManager?.GetMatchData()?.current_match_map_id; + } + + private void OweLoad(int round) + { + int owing = _restore.StartOwing(round, ActiveMapId()); + + TimerUtility.AddTimer(RoundRestoreState.OwedSeconds, () => OwedLoadTimedOut(owing)); + } + + private void OwedLoadTimedOut(int owing) + { + OwedLoadTimeout timeout = _restore.TimedOut(owing, ActiveMapId()); + + if (!timeout.Expired) + { + return; + } + + _matchEvents.WithdrawRestoreRequests(); + + if (timeout.HoldRound is int round) + { + ReturnToHold(round); + return; + } + + SetBackupWriting(!IsHoldingForRestore()); + } + + // A dedicated server keeps this state from one match to the next. + public void ForgetOtherMaps(Guid? map) + { + if (_restore.LeaveMap(map)) + { + restoreRoundVote?.Abandon(); + restoreRoundVote = null; + SetBackupWriting(!IsHoldingForRestore()); + } + } + + public void TellWhyResumeWaits() + { + string? refusal = RoundRestoreGateUtility.ResumeRefusal( + _restore.OwedRound, + _restore.NeedsRestartMap + ); + + if (refusal != null) + { + _gameServer.Message(MessageType.Alert, refusal); + } + } + + // Restart Map starts the map over, so whatever a restore was protecting is + // gone and the restore with it. + public void AbandonRestore() + { + restoreRoundVote?.Abandon(); + restoreRoundVote = null; + _restore.Abandon(); + SetBackupWriting(true); + } + // A match whose rounds can't be restored replays the map from warmup, but // only while the map is still being played: a restart after it ended must // keep the rounds it recorded. The old backup files go too, or the next @@ -433,7 +588,7 @@ public void RequestRestoreBackupRound( if (player != null || vote == true) { - _resetRound = round; + _restore.StartVote(recovering, ActiveMapId()); restoreRoundVote = _serviceProvider.GetRequiredService(typeof(VoteSystem)) as VoteSystem; @@ -449,7 +604,8 @@ public void RequestRestoreBackupRound( () => { _logger.LogInformation("restore round vote passed"); - _resetRound = null; + _restore.End(); + restoreRoundVote = null; string? refusal = RestoreRefusal(round); if (refusal != null) @@ -461,14 +617,31 @@ public void RequestRestoreBackupRound( MessageType.Alert, $" Restore round blocked: {refusal}." ); - restoreRoundVote = null; - if (recovering) + if (recovering && HoldStaysOn(round)) { - _pendingRestore = round; + _restore.Hold(round, ActiveMapId()); } return; } + // A round that ended during the vote is the one being undone, + // and the next round start can come before the load does. + _matchEvents.ClearPendingRoundResult(); + + if ( + RoundRestoreGateUtility.OwesLoad( + recovering, + _environmentService.IsOfflineMode() + ) + ) + { + OweLoad(round); + } + else if (recovering) + { + SetBackupWriting(true); + } + SendRestoreRoundToBackend(round); }, () => @@ -476,7 +649,7 @@ public void RequestRestoreBackupRound( _logger.LogInformation("restore round vote failed"); restoreRoundVote = null; - _resetRound = null; + _restore.End(); // Declined after a restart: the map starts again from 0-0 and // the rounds the backend recorded are set aside. @@ -611,7 +784,7 @@ public void SendRestoreRoundToBackend(int round) public void RestoreRound(int round) { - if (_resetRound != null) + if (_restore.VotingOrLoading) { _logger.LogWarning($"Restore already in progress, ignoring RestoreRound({round})"); return; @@ -619,6 +792,7 @@ public void RestoreRound(int round) if (!CanRestoreRound(round)) { + ReturnOwedToHold(); return; } @@ -633,6 +807,7 @@ public void RestoreRound(int round) match != null, matchMap != null ); + ReturnOwedToHold(); return; } @@ -645,10 +820,11 @@ public void RestoreRound(int round) round, string.Join(", ", matchMap.rounds.Select(r => r.round)) ); + ReturnOwedToHold(); return; } - _pendingRestore = null; + int? held = _restore.ReleaseHold(); string backupRoundFileName = $"restore-{MatchUtility.GetSafeMatchPrefix(match)}round{round.ToString().PadLeft(2, '0')}.txt"; @@ -657,7 +833,7 @@ public void RestoreRound(int round) backupRoundFileName ); - _resetRound = round; + _restore.StartLoad(held); _logger.LogInformation($"Loading backup round file {backupRoundFileName}"); @@ -676,22 +852,17 @@ public void RestoreRound(int round) "Failed writing restore backup round file {File}", backupRoundFileName ); - _core.Scheduler.NextTick(() => _resetRound = null); + _core.Scheduler.NextTick(FailLoad); return; } _core.Scheduler.NextTick(() => { - _gameServer.SendCommands( - [$"mp_backup_restore_load_file {backupRoundFileName}"] - ); - _matchService.GetCurrentMatch()?.PauseMatch(); - TimerUtility.AddTimer( 5, () => { - _resetRound = null; + _restore.End(); SetBackupWriting(true); _logger.LogInformation($"Sending Message for Round {round}"); @@ -707,6 +878,11 @@ public void RestoreRound(int round) ); } ); + + _gameServer.SendCommands( + [$"mp_backup_restore_load_file {backupRoundFileName}"] + ); + _matchService.GetCurrentMatch()?.PauseMatch(); }); }); } @@ -721,7 +897,7 @@ public void RestoreRound(int round) return RoundRestoreGateUtility.Refusal( match != null && !match.options.CanRestoreRounds(), - _resetRound != null, + _restore.VotingOrLoading, _matchService.GetCurrentMatch()?.GetCurrentMapStatus() ?? eMapStatus.Unknown, round, matchMap != null diff --git a/apps/swiftly/src/FiveStack.Services/MatchEvents.cs b/apps/swiftly/src/FiveStack.Services/MatchEvents.cs index 6fc615ad..4d08de75 100644 --- a/apps/swiftly/src/FiveStack.Services/MatchEvents.cs +++ b/apps/swiftly/src/FiveStack.Services/MatchEvents.cs @@ -46,14 +46,17 @@ EventData> Payload private readonly MatchService _matchService; private readonly EnvironmentService _environmentService; private readonly GameServer _gameServer; + private readonly RoundRestoreState _restore; public MatchEvents( ILogger logger, EnvironmentService environmentService, MatchService matchService, - GameServer gameServer + GameServer gameServer, + RoundRestoreState restore ) { + _restore = restore; _logger = logger; _matchService = matchService; _environmentService = environmentService; @@ -154,8 +157,31 @@ public void PublishMapStatus(eMapStatus status, Guid? winningLineupId) PublishGameEvent("mapStatus", data); } + // The api answers whatever reaches it, however late, and an unacknowledged + // event is resent. A request given up on has to stop going out, or it can land + // on top of what the players decided next. + public void WithdrawRestoreRequests() + { + foreach (var message in _pendingMessages) + { + if ( + message.Value.Event.data?.TryGetValue("event", out object? name) == true + && name as string == "restoreRound" + ) + { + _pendingMessages.TryRemove(message.Key, out _); + } + } + } + public void PublishGameEvent(string Event, Dictionary Data) { + if (RoundRestoreGateUtility.DroppedDuringLoad(Event, _restore.LoadingBackup)) + { + _logger.LogInformation($"Game event '{Event}' dropped: a backup round is loading"); + return; + } + if (_environmentService.IsOfflineMode()) { _logger.LogWarning($"Game event '{Event}' dropped: offline mode"); diff --git a/apps/swiftly/src/FiveStack.Services/MatchManager.cs b/apps/swiftly/src/FiveStack.Services/MatchManager.cs index 2ddac2bc..1fc7cf6d 100644 --- a/apps/swiftly/src/FiveStack.Services/MatchManager.cs +++ b/apps/swiftly/src/FiveStack.Services/MatchManager.cs @@ -304,6 +304,7 @@ public void ResumeMatch(string? message = null, bool skipUpdate = false) if (_backUpManagement.IsResettingRound()) { _logger.LogInformation("Resetting round, cannot resume match"); + _backUpManagement.TellWhyResumeWaits(); return; } @@ -317,12 +318,6 @@ public void ResumeMatch(string? message = null, bool skipUpdate = false) _timeoutSystem.resumeVote = null; } - if (_backUpManagement.restoreRoundVote != null) - { - _backUpManagement.restoreRoundVote.CancelVote(); - _backUpManagement.restoreRoundVote = null; - } - _timeoutSystem.ClearPendingTeamResumes(); if (!IsPaused()) @@ -552,6 +547,7 @@ public void SetupMatch(MatchData match) bool wasAlreadySetup = _configuredMapId == _currentMap.id; _activeMapId = _currentMap.id; + _backUpManagement.ForgetOtherMaps(_activeMapId); if (_currentMapStatus == eMapStatus.WaitingForTV) { diff --git a/apps/swiftly/src/FiveStack.Services/VoteSystem.cs b/apps/swiftly/src/FiveStack.Services/VoteSystem.cs index 2aee5bc1..b7c1601e 100644 --- a/apps/swiftly/src/FiveStack.Services/VoteSystem.cs +++ b/apps/swiftly/src/FiveStack.Services/VoteSystem.cs @@ -88,8 +88,23 @@ public void StartVote( SendVoteMessage(); } + // Over with no outcome: neither callback runs. + public void Abandon() + { + _voteFailedCallback = null; + _voteSuccessCallback = null; + CancelVote(); + } + public void CancelVote() { + // The failure callback can replay a map from 0-0, so a vote that has + // already passed must not reach it. + if (!IsVoteActive()) + { + return; + } + _voteStartTime = null; _voteTimeout = null; KillTimers(); diff --git a/apps/swiftly/src/FiveStack.Utilities/TimerUtility.cs b/apps/swiftly/src/FiveStack.Utilities/TimerUtility.cs index 00287d2d..e94fc7da 100644 --- a/apps/swiftly/src/FiveStack.Utilities/TimerUtility.cs +++ b/apps/swiftly/src/FiveStack.Utilities/TimerUtility.cs @@ -6,7 +6,7 @@ namespace FiveStack.Utilities { public static class TimerUtility { - private static ISwiftlyCore _core = null!; + private static ISwiftlyCore? _core; public static readonly List Timers = new List(); @@ -18,7 +18,7 @@ public static void Initialize(ISwiftlyCore core) public static CancellationTokenSource AddTimer(float interval, Action callback) { - CancellationTokenSource timer = _core.Scheduler.DelayBySeconds(interval, callback); + CancellationTokenSource timer = _core!.Scheduler.DelayBySeconds(interval, callback); Timers.Add(timer); return timer; } @@ -29,7 +29,7 @@ public static CancellationTokenSource Repeat(float interval, Action callback) // SwiftlyS2 never reschedules a repeating timer whose callback throws, // and still runs a timer's first pass if it was cancelled before then - timer = _core.Scheduler.RepeatBySeconds( + timer = _core!.Scheduler.RepeatBySeconds( interval, () => { @@ -44,7 +44,7 @@ public static CancellationTokenSource Repeat(float interval, Action callback) } catch (Exception ex) { - _core.Logger.LogError(ex, "repeating timer callback failed"); + _core!.Logger.LogError(ex, "repeating timer callback failed"); } } ); diff --git a/apps/swiftly/src/FiveStack.cs b/apps/swiftly/src/FiveStack.cs index 5a9b8975..6eeb1ef1 100644 --- a/apps/swiftly/src/FiveStack.cs +++ b/apps/swiftly/src/FiveStack.cs @@ -79,6 +79,7 @@ public override void Load(bool hotReload) .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/apps/swiftly/test/BackupWritingTests.cs b/apps/swiftly/test/BackupWritingTests.cs new file mode 100644 index 00000000..6a8f3790 --- /dev/null +++ b/apps/swiftly/test/BackupWritingTests.cs @@ -0,0 +1,122 @@ +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using FiveStack.Utilities; +using Xunit; + +// A restore that could not write its file never reaches the timer that turns +// backups back on. What the state decides is tested. RestoreRound needs the engine +// from its first line, so its half of this is read. +public class BackupWritingTests +{ + private const int HeldRound = 5; + private const int Round = 8; + + private static readonly Guid Map = Guid.NewGuid(); + + private readonly RoundRestoreState _state = new(); + + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Method(string signature) + { + string source = Regex.Replace( + File.ReadAllText(Path.Combine(SourceDir(), "FiveStack.Services", "GameBackUpRounds.cs")), + @"//[^\n]*", + "" + ); + + int from = source.IndexOf(signature, StringComparison.Ordinal); + + Assert.True(from >= 0, $"{signature} is gone"); + + int to = source.IndexOf("\n }\n", from, StringComparison.Ordinal); + + Assert.True(to > from, $"the end of {signature} was not found"); + + return source[from..to]; + } + + private static string FailedWrite() + { + string restore = Method("public void RestoreRound(int round)"); + int write = restore.IndexOf("File.WriteAllText(", StringComparison.Ordinal); + int caught = restore.IndexOf("catch (Exception", write, StringComparison.Ordinal); + int load = restore.IndexOf("mp_backup_restore_load_file", StringComparison.Ordinal); + + Assert.True(write >= 0, "RestoreRound no longer writes the backup file"); + Assert.True(caught > write && caught < load, "a failed write is not caught before the load"); + + return restore[caught..load]; + } + + [Fact] + public void AnOwedLoadWhoseWriteFailedHoldsItsRoundAgain() + { + _state.StartOwing(Round, Map); + _state.StartLoad(); + + Assert.Equal(Round, _state.FailedLoad()); + Assert.False(_state.LoadingBackup); + } + + [Fact] + public void ALoadThatTookOverAHoldPutsItBack() + { + _state.StartLoad(HeldRound); + + Assert.Equal(HeldRound, _state.FailedLoad()); + } + + [Fact] + public void AnOwedLoadKeepsItsOwnRoundOverAHoldItTookOver() + { + _state.StartOwing(Round, Map); + _state.StartLoad(HeldRound); + + Assert.Equal(Round, _state.FailedLoad()); + } + + [Fact] + public void ALoadWithNothingHeldHoldsNothing() + { + _state.StartLoad(); + + Assert.Null(_state.FailedLoad()); + } + + [Fact] + public void TheLoadIsToldWhatItTookOver() + { + string restore = Method("public void RestoreRound(int round)"); + int taken = restore.IndexOf("int? held = _restore.ReleaseHold();", StringComparison.Ordinal); + int load = restore.IndexOf("_restore.StartLoad(held);", StringComparison.Ordinal); + int write = restore.IndexOf("File.WriteAllText(", StringComparison.Ordinal); + + Assert.True(taken >= 0 && load > taken, "the hold is let go with nothing keeping it"); + Assert.True(load < write, "a failed write has nothing to put back"); + } + + [Fact] + public void AFailedWriteIsSettledOneWay() + { + Assert.Contains("FailLoad", FailedWrite()); + } + + // A hold offers its vote straight away, which empties it, so anything that + // reads the hold afterwards would turn backups on during that vote. + [Fact] + public void BackupsOnlyComeBackWhenTheFailedLoadHoldsNothing() + { + string failed = Method("private void FailLoad()"); + int asked = failed.IndexOf("_restore.FailedLoad()", StringComparison.Ordinal); + int hold = failed.IndexOf("ReturnToHold(", StringComparison.Ordinal); + int returned = failed.IndexOf("return;", hold, StringComparison.Ordinal); + int writing = failed.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.True(asked >= 0 && hold > asked, "the state is not asked before acting"); + Assert.True(returned > hold && writing > returned, "backups can come back during the vote"); + } +} diff --git a/apps/swiftly/test/DeclinedVoteRoundTests.cs b/apps/swiftly/test/DeclinedVoteRoundTests.cs new file mode 100644 index 00000000..17594e23 --- /dev/null +++ b/apps/swiftly/test/DeclinedVoteRoundTests.cs @@ -0,0 +1,183 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using FiveStack; +using FiveStack.Utilities; +using Xunit; + +// A .reset vote leaves the round running until the next freezetime, so a round +// that ends during one is real unless the vote passes. A recovery vote is the +// other way round: the server is behind the backend, and has nothing to report. +public class DeclinedVoteRoundTests +{ + private const int Round = 4; + + private readonly RoundRestoreState _restore = new(); + + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Code(params string[] path) + { + return Regex.Replace( + File.ReadAllText(Path.Combine(new[] { SourceDir() }.Concat(path).ToArray())), + @"//[^\n]*", + "" + ); + } + + private static void Set(object target, string field, object? value) + { + target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(target, value); + } + + private static VoteSystem FinishedVote() + { + return (VoteSystem)RuntimeHelpers.GetUninitializedObject(typeof(VoteSystem)); + } + + private static VoteSystem OpenVote() + { + VoteSystem vote = FinishedVote(); + FieldInfo timer = typeof(VoteSystem).GetField( + "_playerMessageTimer", + BindingFlags.NonPublic | BindingFlags.Instance + )!; + + timer.SetValue(vote, RuntimeHelpers.GetUninitializedObject(timer.FieldType)); + + return vote; + } + + private GameBackUpRounds Backups(VoteSystem? vote = null) + { + GameBackUpRounds backups = (GameBackUpRounds) + RuntimeHelpers.GetUninitializedObject(typeof(GameBackUpRounds)); + + Set(backups, "_restore", _restore); + backups.restoreRoundVote = vote; + + return backups; + } + + [Fact] + public void ARoundEndingDuringAResetVoteIsKept() + { + _restore.StartVote(); + + Assert.False(Backups(OpenVote()).HoldsRoundResults()); + } + + [Fact] + public void NothingIsHeldWithNoRestoreGoingOn() + { + Assert.False(Backups().HoldsRoundResults()); + } + + [Fact] + public void ARecoveryVoteHoldsResultsWhileItIsOpen() + { + _restore.StartVote(recovering: true); + + Assert.True(Backups(OpenVote()).HoldsRoundResults()); + } + + [Fact] + public void ARecoveryVoteKeepsTheBackupsHeld() + { + _restore.StartVote(recovering: true); + + Assert.True(Backups(OpenVote()).IsHoldingForRestore()); + } + + [Fact] + public void AResetVoteHoldsNoBackups() + { + _restore.StartVote(); + + Assert.False(Backups(OpenVote()).IsHoldingForRestore()); + } + + [Fact] + public void ARestoreHeldForPlayersHoldsResults() + { + _restore.Hold(Round); + + Assert.True(Backups().HoldsRoundResults()); + } + + [Fact] + public void ALoadingBackupHoldsResults() + { + _restore.StartLoad(); + + Assert.True(Backups().HoldsRoundResults()); + } + + [Fact] + public void ALoadAfterAVoteFinishedHoldsResults() + { + _restore.StartLoad(); + + Assert.True(Backups(FinishedVote()).HoldsRoundResults()); + } + + [Fact] + public void NoScoreSlipsOutWhileALoadIsOwed() + { + _restore.StartOwing(Round, Guid.NewGuid()); + + Assert.True(Backups(FinishedVote()).HoldsRoundResults()); + } + + [Fact] + public void ARestoreSaysWhenItsLoadStarts() + { + int started = 0; + _restore.LoadStarted += () => started++; + + _restore.StartVote(); + _restore.End(); + _restore.StartOwing(Round, Guid.NewGuid()); + + Assert.Equal(0, started); + + _restore.StartLoad(); + + Assert.Equal(1, started); + } + + [Fact] + public void ALoadForgetsTheResultOfTheRoundItUndoes() + { + Assert.Contains( + "_restore.LoadStarted += matchEvents.ClearPendingRoundResult;", + Code("FiveStack.Services", "GameBackUpRounds.cs") + ); + } + + [Theory] + [InlineData("RoundEnd.cs", 2)] + [InlineData("RoundStart.cs", 1)] + public void TheRoundHooksAskWhetherResultsAreHeld(string file, int sites) + { + string source = Code("FiveStack.Events", file); + + Assert.DoesNotContain("IsResettingRound", source); + Assert.Equal(sites, Regex.Matches(source, @"_gameBackupRounds\.HoldsRoundResults\(\)").Count); + } + + [Fact] + public void AVoteSaysWhetherItIsARecoveryVote() + { + Assert.Contains( + "_restore.StartVote(recovering, ", + Code("FiveStack.Services", "GameBackUpRounds.cs") + ); + } +} diff --git a/apps/swiftly/test/OpenVoteTests.cs b/apps/swiftly/test/OpenVoteTests.cs new file mode 100644 index 00000000..7510b13f --- /dev/null +++ b/apps/swiftly/test/OpenVoteTests.cs @@ -0,0 +1,55 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using FiveStack; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +// Only here: the other plugin's vote timers are native handles. +public class OpenVoteTests +{ + private int _failures; + private int _passes; + + private static void Set(object target, string field, object? value) + { + target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(target, value); + } + + private VoteSystem OpenVote() + { + VoteSystem vote = (VoteSystem)RuntimeHelpers.GetUninitializedObject(typeof(VoteSystem)); + + Set(vote, "_logger", NullLogger.Instance); + Set(vote, "_voteFailedCallback", (Action)(() => _failures++)); + Set(vote, "_voteSuccessCallback", (Action)(() => _passes++)); + Set(vote, "_playerMessageTimer", new CancellationTokenSource()); + + return vote; + } + + [Fact] + public void CancellingAnOpenVoteRunsItsFailure() + { + VoteSystem vote = OpenVote(); + + vote.CancelVote(); + + Assert.Equal(1, _failures); + Assert.False(vote.IsVoteActive()); + } + + [Fact] + public void AbandoningAnOpenVoteRunsNeitherCallback() + { + VoteSystem vote = OpenVote(); + + vote.Abandon(); + + Assert.Equal(0, _failures); + Assert.Equal(0, _passes); + Assert.False(vote.IsVoteActive()); + } +} diff --git a/apps/swiftly/test/RestoreParityTests.cs b/apps/swiftly/test/RestoreParityTests.cs new file mode 100644 index 00000000..a3f18fd3 --- /dev/null +++ b/apps/swiftly/test/RestoreParityTests.cs @@ -0,0 +1,61 @@ +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Xunit; + +// The two plugins share RoundRestoreState, but each has its own copy of the code +// that acts on it, and a restore has to behave the same on both. +public class RestoreParityTests +{ + private static string Apps([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "..")); + } + + private static string Method(string app, string signature) + { + string source = File.ReadAllText( + Path.Combine(Apps(), app, "src", "FiveStack.Services", "GameBackUpRounds.cs") + ); + + int from = source.IndexOf(signature, StringComparison.Ordinal); + + Assert.True(from >= 0, $"{app} has no {signature}"); + + int to = source.IndexOf("\n }\n", from, StringComparison.Ordinal); + + return Regex.Replace(source[from..to], @"\s+", " ").Replace("HudDestination.", "MessageType."); + } + + // Only this plugin reads backups off disk. The api raises P0002 for a round it + // has no row for and answers nothing, so offering one could never load. + [Fact] + public void OnlyDiskRoundsTheBackendHasARowForAreOffered() + { + string source = File.ReadAllText( + Path.Combine(Apps(), "swiftly", "src", "FiveStack.Services", "GameBackUpRounds.cs") + ); + + Assert.Contains("RoundRestoreGateUtility.IsUsableOnDisk(", source); + Assert.DoesNotContain("from_disk", source); + } + + [Theory] + [InlineData("private void HoldForRestore(int round)")] + [InlineData("private bool HoldStaysOn(int round)")] + [InlineData("private void ReturnToHold(int round)")] + [InlineData("private void ReturnOwedToHold()")] + [InlineData("private void FailLoad()")] + [InlineData("private void OweLoad(int round)")] + [InlineData("private void OwedLoadTimedOut(int owing)")] + [InlineData("public void TellWhyResumeWaits()")] + [InlineData("public void AbandonRestore()")] + [InlineData("private Guid? ActiveMapId()")] + [InlineData("public void ForgetOtherMaps(Guid? map)")] + [InlineData("public bool IsResettingRound()")] + [InlineData("public bool IsHoldingForRestore()")] + [InlineData("public bool HoldsRoundResults()")] + public void BothPluginsActTheSame(string signature) + { + Assert.Equal(Method("swiftly", signature), Method("counterstrikesharp", signature)); + } +} diff --git a/apps/swiftly/test/RestoreVoteTests.cs b/apps/swiftly/test/RestoreVoteTests.cs new file mode 100644 index 00000000..46a5083e --- /dev/null +++ b/apps/swiftly/test/RestoreVoteTests.cs @@ -0,0 +1,61 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using FiveStack; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +// A recovery vote's failure callback replays the map from 0-0, which sets every +// round the backend recorded aside. It must only ever run for a vote that lost. +public class RestoreVoteTests +{ + private int _failures; + private int _passes; + + private static void Set(object target, string field, object? value) + { + target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(target, value); + } + + private static object? Get(object target, string field) + { + return target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(target); + } + + private VoteSystem FinishedVote() + { + VoteSystem vote = (VoteSystem)RuntimeHelpers.GetUninitializedObject(typeof(VoteSystem)); + + Set(vote, "_logger", NullLogger.Instance); + Set(vote, "_voteFailedCallback", (Action)(() => _failures++)); + Set(vote, "_voteSuccessCallback", (Action)(() => _passes++)); + + return vote; + } + + [Fact] + public void AFinishedVoteCannotBeCancelledIntoItsFailure() + { + FinishedVote().CancelVote(); + + Assert.Equal(0, _failures); + } + + [Fact] + public void AbandoningAFinishedVoteRunsNoCallback() + { + VoteSystem vote = FinishedVote(); + + vote.Abandon(); + + Assert.Equal(0, _failures); + Assert.Equal(0, _passes); + Assert.Null(Get(vote, "_voteFailedCallback")); + Assert.Null(Get(vote, "_voteSuccessCallback")); + } +} diff --git a/apps/swiftly/test/RestoreWiringTests.cs b/apps/swiftly/test/RestoreWiringTests.cs new file mode 100644 index 00000000..bd6d21b9 --- /dev/null +++ b/apps/swiftly/test/RestoreWiringTests.cs @@ -0,0 +1,335 @@ +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Xunit; + +// What the state decides is tested in RoundRestoreStateTests. Calling it needs +// the engine, so these read the plugin. +public class RestoreWiringTests +{ + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Code(params string[] path) + { + return Regex.Replace( + File.ReadAllText(Path.Combine(new[] { SourceDir() }.Concat(path).ToArray())), + @"//[^\n]*", + "" + ); + } + + private static string Backups() + { + return Code("FiveStack.Services", "GameBackUpRounds.cs"); + } + + private static string Method(string source, string signature) + { + int from = source.IndexOf(signature, StringComparison.Ordinal); + + Assert.True(from >= 0, $"{signature} is gone"); + + int to = source.IndexOf("\n }\n", from, StringComparison.Ordinal); + + Assert.True(to > from, $"the end of {signature} was not found"); + + return source[from..to]; + } + + private static string Between(string source, string start, string end) + { + int from = source.IndexOf(start, StringComparison.Ordinal); + int to = source.IndexOf(end, from + 1, StringComparison.Ordinal); + + Assert.True(from >= 0 && to > from, $"{start} has moved"); + + return source[from..to]; + } + + private static string PassedVote() + { + return Between( + Method(Backups(), "public void RequestRestoreBackupRound("), + "\"restore round vote passed\"", + "\"restore round vote failed\"" + ); + } + + [Fact] + public void AnOrganizerRestoreIsOnlyRefusedForAVoteOrALoadAsOnMain() + { + string refusal = Method(Backups(), "public string? RestoreRefusal(int round)"); + + Assert.Contains("_restore.VotingOrLoading,", refusal); + Assert.DoesNotContain("_restore.InProgress", refusal); + } + + [Fact] + public void AVoteIsLetGoAsSoonAsItPasses() + { + string passed = PassedVote(); + + Assert.Contains("restoreRoundVote = null;", Between(passed, "passed", "RestoreRefusal(round)")); + } + + [Fact] + public void APassedVoteAsksWhetherItOwesALoadBeforeSendingIt() + { + string passed = PassedVote(); + Match owes = Regex.Match( + passed, + @"RoundRestoreGateUtility\.OwesLoad\(\s*recovering,\s*_environmentService\.IsOfflineMode\(\)\s*\)" + ); + int sent = passed.IndexOf("SendRestoreRoundToBackend(round);", StringComparison.Ordinal); + + Assert.True(owes.Success, "a passed vote does not ask"); + Assert.True(sent > owes.Index, "the answer can beat the debt"); + } + + // A result captured during the vote is from a round the restore is about to + // undo, and it would publish at the next round start, before the load begins. + // A refused restore undoes nothing, so its result is real and stays. + [Fact] + public void APassedVoteForgetsTheResultOfARoundItIsAboutToUndo() + { + string passed = PassedVote(); + int refused = passed.IndexOf("return;", StringComparison.Ordinal); + int forgotten = passed.IndexOf("_matchEvents.ClearPendingRoundResult();", StringComparison.Ordinal); + int sent = passed.IndexOf("SendRestoreRoundToBackend(round);", StringComparison.Ordinal); + + Assert.True(forgotten >= 0, "a passed vote leaves the result to publish"); + Assert.True(forgotten > refused, "a refused restore throws away a real result"); + Assert.True(forgotten < sent, "the result can publish after the restore is sent"); + } + + [Fact] + public void AnOfflineRecoveryTurnsBackupsBackOnAndDeletesNothing() + { + string passed = PassedVote(); + + Assert.Matches( + @"\{\s*OweLoad\(round\);\s*\}\s*else if \(recovering\)\s*\{\s*SetBackupWriting\(true\);\s*\}", + passed + ); + Assert.DoesNotContain("ReplayMap", passed); + Assert.DoesNotContain("File.Delete", passed); + Assert.DoesNotContain("Backups(", passed); + } + + [Fact] + public void OnlyADeclinedVoteReplaysTheMap() + { + string vote = Method(Backups(), "public void RequestRestoreBackupRound("); + string declined = vote[vote.IndexOf("\"restore round vote failed\"", StringComparison.Ordinal)..]; + + Assert.Contains("ReplayMap(", declined); + Assert.DoesNotContain("ReplayMap", Method(Backups(), "private void OwedLoadTimedOut(int owing)")); + Assert.DoesNotContain("ReplayMap", Method(Backups(), "private void ReturnToHold(int round)")); + Assert.DoesNotContain("ReplayMap", Method(Backups(), "public void RestoreRound(int round)")); + } + + [Fact] + public void AnOwedLoadIsTimedFromTheMomentItIsOwed() + { + string owe = Method(Backups(), "private void OweLoad(int round)"); + + Assert.Contains("_restore.StartOwing(round, ActiveMapId())", owe); + Assert.Contains("RoundRestoreState.OwedSeconds", owe); + } + + [Fact] + public void TheAnswerIsNotGatedOnWhatWasAskedFor() + { + string restore = Method(Backups(), "public void RestoreRound(int round)"); + string opening = restore[restore.IndexOf('{')..restore.IndexOf("if (!CanRestoreRound(round))", StringComparison.Ordinal)]; + + Assert.Contains("if (_restore.VotingOrLoading)", opening); + Assert.Single(Regex.Matches(opening, @"\bif\s*\(")); + Assert.DoesNotContain("OwedRound", restore); + Assert.DoesNotMatch(@"\bround\s*(==|!=|<|>)", restore); + } + + [Fact] + public void EveryWayOutOfRestoreRoundSettlesAnOwedLoad() + { + string restore = Method(Backups(), "public void RestoreRound(int round)"); + string guarded = Between(restore, "CanRestoreRound(round)", "_restore.StartLoad("); + + MatchCollection exits = Regex.Matches(guarded, @"[^\n]*\n\s*return;"); + + Assert.Equal(3, exits.Count); + Assert.All(exits.Select(exit => exit.Value), exit => Assert.Contains("ReturnOwedToHold();", exit)); + } + + [Fact] + public void AHoldAsksWhereToGoAndOnlySpeaksWhenItMoves() + { + string check = Method(Backups(), "private bool HoldStaysOn(int round)"); + int asked = check.IndexOf("RoundRestoreGateUtility.RoundToHold(", StringComparison.Ordinal); + int stays = check.IndexOf("if (next == round)", StringComparison.Ordinal); + int told = check.IndexOf("RoundRestoreGateUtility.WhyTheHoldMoved(round, next)", StringComparison.Ordinal); + int moved = check.IndexOf("HoldForRestore(moved);", StringComparison.Ordinal); + int writing = check.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.Contains("_gameServer.GetTotalRoundsPlayed()", check); + Assert.True(asked >= 0 && stays > asked, "a hold stays on a round that can't load"); + Assert.True(told > stays, "players are told again by a hold that didn't move"); + Assert.True(moved > told, "a new hold is offered before players hear why"); + Assert.True(writing > moved, "backups come on with a hold still in place"); + Assert.DoesNotContain("ReplayMap", check); + } + + [Fact] + public void EveryHoldThatComesBackAsksWhereItBelongs() + { + string hold = Method(Backups(), "private void ReturnToHold(int round)"); + string offer = Method(Backups(), "public void OfferPendingRestore()"); + string vote = Method(Backups(), "public void RequestRestoreBackupRound("); + + Assert.True( + hold.IndexOf("HoldStaysOn(round)", StringComparison.Ordinal) >= 0 + && hold.IndexOf("HoldStaysOn(round)", StringComparison.Ordinal) + < hold.IndexOf("HoldForRestore(round);", StringComparison.Ordinal), + "a hold comes back for a round that can't load" + ); + Assert.Matches(@"HoldStaysOn\(round\)\s*\)\s*\{\s*_restore\.Hold\(", offer); + Assert.Matches(@"HoldStaysOn\(round\)\s*\)\s*\{\s*_restore\.Hold\(", vote); + } + + [Fact] + public void AResumeOnNothingToLoadSaysWhy() + { + Assert.Matches( + @"ResumeRefusal\(\s*_restore\.OwedRound,\s*_restore\.NeedsRestartMap\s*\)", + Method(Backups(), "public void TellWhyResumeWaits()") + ); + } + + [Fact] + public void AnOpenVoteIsAbandonedWithItsMap() + { + string forget = Method(Backups(), "public void ForgetOtherMaps(Guid? map)"); + int left = forget.IndexOf("_restore.LeaveMap(map)", StringComparison.Ordinal); + int abandoned = forget.IndexOf("restoreRoundVote?.Abandon();", StringComparison.Ordinal); + int cleared = forget.IndexOf("restoreRoundVote = null;", StringComparison.Ordinal); + int writing = forget.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.True(abandoned > left, "a vote outlives its map"); + Assert.True(cleared > abandoned && writing > cleared, "backups are put right too early"); + Assert.DoesNotContain("Withdraw", forget); + } + + [Fact] + public void AVoteRemembersItsMap() + { + Assert.Contains("_restore.StartVote(recovering, ActiveMapId());", Backups()); + } + + [Fact] + public void AnOwedLoadOnAnotherMapTurnsBackupsBackOnIfNothingHolds() + { + Assert.Contains( + "SetBackupWriting(!IsHoldingForRestore());", + Method(Backups(), "private void OwedLoadTimedOut(int owing)") + ); + } + + [Fact] + public void AnOwedLoadThatTimedOutStopsBeingResentBeforeTheHoldReturns() + { + string timedOut = Method(Backups(), "private void OwedLoadTimedOut(int owing)"); + int withdrawn = timedOut.IndexOf("_matchEvents.WithdrawRestoreRequests();", StringComparison.Ordinal); + int held = timedOut.IndexOf("ReturnToHold(", StringComparison.Ordinal); + + Assert.True(withdrawn >= 0, "the request can land after a declined vote"); + Assert.True(withdrawn < held, "the hold can be offered with the request still going out"); + } + + [Fact] + public void TheNextMatchLeavesWhatTheLastOneHeldAndPutsBackupsRight() + { + string forget = Method(Backups(), "public void ForgetOtherMaps(Guid? map)"); + int left = forget.IndexOf("_restore.LeaveMap(map)", StringComparison.Ordinal); + int writing = forget.IndexOf("SetBackupWriting(!IsHoldingForRestore());", StringComparison.Ordinal); + + Assert.True(left >= 0 && writing > left, "backups stay off with nothing holding them"); + } + + [Fact] + public void AMatchIsSetUpBeforeAnythingElseLooksAtItsHold() + { + string setup = Method(Code("FiveStack.Services", "MatchManager.cs"), "public void SetupMatch("); + int active = setup.IndexOf("_activeMapId = _currentMap.id;", StringComparison.Ordinal); + int forgotten = setup.IndexOf("_backUpManagement.ForgetOtherMaps(_activeMapId);", StringComparison.Ordinal); + int waiting = setup.IndexOf("eMapStatus.WaitingForTV", active, StringComparison.Ordinal); + + Assert.True(active >= 0 && forgotten > active, "a hold outlives its match"); + Assert.True(forgotten < waiting, "an early return skips it"); + } + + [Fact] + public void AHoldRemembersItsMapEverywhereItIsSet() + { + MatchCollection holds = Regex.Matches(Backups(), @"_restore\.Hold\(([^;]*)\);"); + + Assert.Equal(3, holds.Count); + Assert.All(holds.Select(hold => hold.Groups[1].Value), hold => Assert.EndsWith("ActiveMapId()", hold)); + } + + [Fact] + public void AHoldIsOnlyOfferedThroughTheState() + { + string offer = Method(Backups(), "public void OfferPendingRestore()"); + int asked = offer.IndexOf("_restore.CanOfferVote", StringComparison.Ordinal); + int taken = offer.IndexOf("_restore.ReleaseHold()", StringComparison.Ordinal); + + Assert.True(asked >= 0 && taken > asked, "a hold can be taken during a load"); + } + + [Fact] + public void RestartMapAbandonsTheRestore() + { + string abandon = Method(Backups(), "public void AbandonRestore()"); + + Assert.Contains("_restore.Abandon();", abandon); + Assert.Contains("SetBackupWriting(true);", abandon); + Assert.Contains(".Abandon();", abandon.Replace("_restore.Abandon();", "")); + Assert.DoesNotContain("CancelVote", abandon); + } + + [Fact] + public void RestartMapOnlyAbandonsOnceItWasAccepted() + { + string state = Between(Code("FiveStack.Commands", "Match.cs"), "SetMatchState(", "match?.UpdateMapStatus(status);"); + int refused = state.IndexOf("refusal != null", StringComparison.Ordinal); + int abandoned = state.IndexOf("_gameBackupRounds.AbandonRestore();", StringComparison.Ordinal); + + Assert.True(abandoned > refused, "a refused restart can drop a hold"); + } + + [Fact] + public void ResumingNoLongerReachesARestoreVote() + { + Assert.DoesNotContain("restoreRoundVote", Code("FiveStack.Services", "MatchManager.cs")); + } + + [Fact] + public void ResumeSaysWhyItIsWaiting() + { + string resume = Method(Code("FiveStack.Services", "MatchManager.cs"), "public void ResumeMatch("); + + Assert.Contains("TellWhyResumeWaits();", Between(resume, "IsResettingRound()", "return;")); + } + + [Fact] + public void AnOpenVoteStillRunsItsFailure() + { + string cancel = Method(Code("FiveStack.Services", "VoteSystem.cs"), "public void CancelVote()"); + int guarded = cancel.IndexOf("if (!IsVoteActive())", StringComparison.Ordinal); + int failed = cancel.IndexOf("_voteFailedCallback?.Invoke();", StringComparison.Ordinal); + + Assert.True(guarded >= 0 && failed > guarded, "CancelVote no longer fails an open vote"); + } +} diff --git a/apps/swiftly/test/RoundRestorePublishTests.cs b/apps/swiftly/test/RoundRestorePublishTests.cs new file mode 100644 index 00000000..ef55b7dd --- /dev/null +++ b/apps/swiftly/test/RoundRestorePublishTests.cs @@ -0,0 +1,154 @@ +using System.Collections.Concurrent; +using System.Reflection; +using System.Runtime.CompilerServices; +using FiveStack; +using FiveStack.Entities; +using FiveStack.Utilities; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +// Runs the plugin's own MatchEvents. With no socket a published event stops in +// _pendingMessages, and the publish queue keeps its order, so once the marker +// has arrived everything sent before it has been dealt with. +public class RoundRestorePublishTests : IAsyncLifetime +{ + private const string Marker = "restoreRound"; + + private readonly RoundRestoreState _restore = new(); + private readonly MatchEvents _events; + + public RoundRestorePublishTests() + { + MatchManager match = Bare(); + Set(match, "_matchData", new MatchData { id = Guid.NewGuid() }); + Set(match, "_activeMapId", (Guid?)Guid.NewGuid()); + + MatchService matchService = Bare(); + Set(matchService, "_currentMatch", match); + + _events = new MatchEvents( + NullLogger.Instance, + Bare(), + matchService, + null!, + _restore + ); + } + + public Task InitializeAsync() + { + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + return _events.Disconnect(); + } + + private static T Bare() + { + return (T)RuntimeHelpers.GetUninitializedObject(typeof(T)); + } + + private static void Set(object target, string field, object? value) + { + target + .GetType() + .GetField(field, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(target, value); + } + + private List Published() + { + var pending = (ConcurrentDictionary< + Guid, + (MatchEvents.EventData> Event, DateTime Timestamp) + >) + typeof(MatchEvents) + .GetField("_pendingMessages", BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(_events)!; + + return pending.Values.Select(entry => (string)entry.Event.data!["event"]).ToList(); + } + + private async Task> PublishThenSettle(string gameEvent) + { + _events.PublishGameEvent(gameEvent, new Dictionary()); + _events.PublishGameEvent(Marker, new Dictionary()); + + DateTime giveUp = DateTime.UtcNow.AddSeconds(5); + + while (!Published().Contains(Marker)) + { + Assert.True(DateTime.UtcNow < giveUp, "the marker never reached the publish step"); + await Task.Delay(5); + } + + return Published(); + } + + [Theory] + [InlineData("kill")] + [InlineData("assist")] + [InlineData("damage")] + [InlineData("utility")] + [InlineData("flash")] + [InlineData("objective")] + public async Task AStatEventIsNotPublishedWhileABackupLoads(string gameEvent) + { + _restore.StartLoad(); + + Assert.DoesNotContain(gameEvent, await PublishThenSettle(gameEvent)); + } + + [Theory] + [InlineData("kill")] + [InlineData("damage")] + [InlineData("objective")] + public async Task ARestoreVoteKeepsPublishingTheRoundBeingPlayed(string gameEvent) + { + _restore.StartVote(); + + Assert.Contains(gameEvent, await PublishThenSettle(gameEvent)); + } + + [Fact] + public async Task ARequestThatTimedOutStopsBeingResent() + { + Assert.Contains("kill", await PublishThenSettle("kill")); + + _events.WithdrawRestoreRequests(); + + Assert.DoesNotContain(Marker, Published()); + Assert.Contains("kill", Published()); + } + + [Fact] + public async Task AKillIsPublishedOnceTheLoadHasEnded() + { + _restore.StartLoad(); + _restore.End(); + + Assert.Contains("kill", await PublishThenSettle("kill")); + } + + [Fact] + public async Task ALoadAfterAPassedVoteStopsPublishing() + { + _restore.StartVote(); + _restore.End(); + _restore.StartLoad(); + + Assert.DoesNotContain("kill", await PublishThenSettle("kill")); + } + + [Theory] + [InlineData("score")] + [InlineData("mapStatus")] + public async Task ALoadLeavesEventsThatAreNotRoundStatsAlone(string gameEvent) + { + _restore.StartLoad(); + + Assert.Contains(gameEvent, await PublishThenSettle(gameEvent)); + } +} diff --git a/apps/swiftly/test/RoundRestoreStateTests.cs b/apps/swiftly/test/RoundRestoreStateTests.cs new file mode 100644 index 00000000..3ce2feac --- /dev/null +++ b/apps/swiftly/test/RoundRestoreStateTests.cs @@ -0,0 +1,583 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using FiveStack; +using FiveStack.Entities; +using FiveStack.Enums; +using FiveStack.Utilities; +using Xunit; + +// One debt: a recovery vote passed and the server is owed its load. Nothing else +// about a restore changes, so .reset, organizer restores and declined votes are +// pinned here as well as the new paths. +public class RoundRestoreStateTests +{ + private const int Round = 8; + + private static readonly Guid Map = Guid.NewGuid(); + private static readonly Guid OtherMap = Guid.NewGuid(); + + private readonly RoundRestoreState _state = new(); + + private GameBackUpRounds Backups() + { + GameBackUpRounds backups = (GameBackUpRounds) + RuntimeHelpers.GetUninitializedObject(typeof(GameBackUpRounds)); + + typeof(GameBackUpRounds) + .GetField("_restore", BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(backups, _state); + + return backups; + } + + [Fact] + public void AnOwedLoadCannotBeResumedOver() + { + _state.StartOwing(Round, Map); + + Assert.Equal(Round, _state.OwedRound); + Assert.True(Backups().IsResettingRound()); + Assert.True(Backups().IsHoldingForRestore()); + } + + [Fact] + public void AnOwedLoadDoesNotStandInTheWayOfItsAnswer() + { + _state.StartOwing(Round, Map); + + Assert.False(_state.VotingOrLoading); + } + + [Fact] + public void TheAnswerSettlesTheDebt() + { + _state.StartOwing(Round, Map); + _state.StartLoad(); + + Assert.Null(_state.OwedRound); + Assert.True(_state.LoadingBackup); + Assert.True(Backups().IsResettingRound()); + + _state.End(); + + Assert.False(Backups().IsResettingRound()); + } + + [Fact] + public void AnOwedLoadThatCouldNotStartGoesBackToTheHold() + { + _state.StartOwing(Round, Map); + + Assert.Equal(Round, _state.FailedLoad()); + Assert.False(_state.InProgress); + } + + [Fact] + public void AFailedLoadNobodyWasOwedHoldsNothing() + { + Assert.Null(_state.FailedLoad()); + } + + [Fact] + public void AnOwedLoadNobodyAnsweredGoesBackToTheHold() + { + int owing = _state.StartOwing(Round, Map); + + Assert.Equal(new OwedLoadTimeout(true, Round), _state.TimedOut(owing, Map)); + Assert.False(_state.InProgress); + } + + [Fact] + public void ADebtThatNeverWentOutStillComesBackToTheHold() + { + int owing = _state.StartOwing(Round, null); + + Assert.Equal(new OwedLoadTimeout(true, Round), _state.TimedOut(owing, null)); + } + + [Fact] + public void ATimerOnAnotherMapEndsQuietly() + { + int owing = _state.StartOwing(Round, Map); + + Assert.Equal(new OwedLoadTimeout(true), _state.TimedOut(owing, OtherMap)); + Assert.False(_state.InProgress); + } + + [Fact] + public void ATimerFromAnEarlierDebtDoesNothing() + { + int earlier = _state.StartOwing(Round, Map); + _state.End(); + _state.StartOwing(Round, Map); + + Assert.Equal(default, _state.TimedOut(earlier, Map)); + Assert.Equal(Round, _state.OwedRound); + } + + [Fact] + public void ATimerAfterTheLoadStartedDoesNothing() + { + int owing = _state.StartOwing(Round, Map); + _state.StartLoad(); + + Assert.Equal(default, _state.TimedOut(owing, Map)); + Assert.True(_state.LoadingBackup); + } + + [Fact] + public void ATimerAfterRestartMapDoesNothing() + { + int owing = _state.StartOwing(Round, Map); + _state.Abandon(); + + Assert.Equal(default, _state.TimedOut(owing, Map)); + } + + [Fact] + public void RestartMapClearsAHold() + { + _state.Hold(Round); + _state.Abandon(); + + Assert.Null(_state.HeldRound); + Assert.False(Backups().IsResettingRound()); + Assert.False(Backups().IsHoldingForRestore()); + } + + [Fact] + public void RestartMapClearsAnOwedLoad() + { + _state.StartOwing(Round, Map); + _state.Abandon(); + + Assert.False(Backups().IsResettingRound()); + Assert.False(Backups().IsHoldingForRestore()); + } + + [Fact] + public void RestartMapClearsAVoteAndAHoldTogether() + { + _state.Hold(Round); + _state.StartVote(); + _state.Abandon(); + + Assert.False(Backups().IsResettingRound()); + } + + [Fact] + public void AHoldIsOfferedItsVoteWhenNothingElseIsRunning() + { + _state.Hold(Round); + + Assert.True(_state.CanOfferVote); + } + + [Fact] + public void NoHoldMeansNoVote() + { + Assert.False(_state.CanOfferVote); + } + + [Fact] + public void ALoadInProgressKeepsAHoldFromBeingOffered() + { + _state.Hold(Round); + _state.StartLoad(); + + Assert.False(_state.CanOfferVote); + Assert.Equal(Round, _state.HeldRound); + } + + [Fact] + public void AnOwedLoadKeepsAHoldFromBeingOffered() + { + _state.Hold(Round); + _state.StartOwing(Round, Map); + + Assert.False(_state.CanOfferVote); + } + + [Fact] + public void AVoteInProgressKeepsAHoldFromBeingOffered() + { + _state.Hold(Round); + _state.StartVote(); + + Assert.False(_state.CanOfferVote); + } + + [Fact] + public void TakingTheHoldHandsBackItsRound() + { + _state.Hold(Round); + + Assert.Equal(Round, _state.ReleaseHold()); + Assert.Null(_state.HeldRound); + } + + [Fact] + public void AnAnswerDuringAVoteOrALoadIsIgnoredAsOnMain() + { + _state.StartVote(); + Assert.True(_state.VotingOrLoading); + + _state.End(); + _state.StartLoad(); + Assert.True(_state.VotingOrLoading); + } + + [Fact] + public void AnAnswerWithNothingGoingOnIsLoadedAsOnMain() + { + Assert.False(_state.VotingOrLoading); + } + + [Fact] + public void ResetToRoundZeroFindsItsBackupAsOnMain() + { + BackupRound first = new() { round = 0, backup_file = "round00" }; + BackupRound newest = new() { round = 0, backup_file = "round00 again" }; + BackupRound later = new() { round = Round, backup_file = "round08" }; + + Assert.Same(newest, RoundRestoreGateUtility.LatestBackup(new[] { first, later, newest }, 0)); + } + + [Fact] + public void ResetToRoundZeroIsRefusedOnlyForAMissingBackup() + { + Assert.Equal( + "the server has no backup of round 0", + RoundRestoreGateUtility.Refusal(false, false, eMapStatus.Live, 0, false, 10, 10, "Competitive") + ); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AnOwedLoadDoesNotStopRoundZeroLoading(bool owing) + { + if (owing) + { + _state.StartOwing(Round, Map); + } + + Assert.False(_state.VotingOrLoading); + Assert.Null( + RoundRestoreGateUtility.Refusal( + false, + _state.VotingOrLoading, + eMapStatus.Live, + 0, + true, + 10, + 10, + "Competitive" + ) + ); + } + + [Fact] + public void TheNextMatchEndsALoadOwedOnTheLastOne() + { + _state.StartOwing(Round, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.Null(_state.OwedRound); + Assert.False(_state.Resetting); + } + + [Fact] + public void TheNextMatchEndsAHoldOnTheLastOne() + { + _state.Hold(Round, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.Null(_state.HeldRound); + Assert.False(_state.HoldingRounds); + } + + [Fact] + public void SetupOnTheSameMapKeepsAHoldAndAnOwedLoad() + { + _state.Hold(Round, Map); + _state.StartOwing(Round, Map); + + Assert.False(_state.LeaveMap(Map)); + Assert.Equal(Round, _state.HeldRound); + Assert.Equal(Round, _state.OwedRound); + } + + [Fact] + public void ANewMatchWithNothingToLeaveChangesNothing() + { + Assert.False(_state.LeaveMap(OtherMap)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void AnOpenVoteEndsWithItsMap(bool recovering) + { + _state.StartVote(recovering, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.False(_state.VotingOrLoading); + Assert.False(_state.HoldingRounds); + Assert.False(_state.Resetting); + } + + [Fact] + public void AnOpenVoteStaysOnItsOwnMap() + { + _state.StartVote(map: Map); + + Assert.False(_state.LeaveMap(Map)); + Assert.True(_state.VotingOrLoading); + } + + [Theory] + [InlineData(5, true)] + [InlineData(4, true)] + [InlineData(6, false)] + [InlineData(Round, false)] + [InlineData(3, false)] + public void OnlyADiskRoundTheBackendHasARowForIsUsable(int round, bool usable) + { + BackupRound[] rounds = + { + new() { round = 4, backup_file = "kept" }, + new() { round = 5 }, + }; + + Assert.Equal(usable, RoundRestoreGateUtility.IsUsableOnDisk(rounds, round, false)); + } + + [Fact] + public void WithNoRowThereIsNothingOnDiskToUse() + { + Assert.False( + RoundRestoreGateUtility.IsUsableOnDisk(Array.Empty(), Round, false) + ); + } + + [Fact] + public void ARowARestoreUndidStaysUnusable() + { + BackupRound[] rounds = { new() { round = Round, deleted_at = "2026-11-02" } }; + + Assert.False(RoundRestoreGateUtility.IsUsableOnDisk(rounds, Round, false)); + } + + [Fact] + public void OfflineKeepsItsDiskRoundsAsOnMain() + { + BackupRound[] undone = { new() { round = 5, deleted_at = "2026-11-02" } }; + + Assert.True( + RoundRestoreGateUtility.IsUsableOnDisk(Array.Empty(), Round, true) + ); + Assert.False(RoundRestoreGateUtility.IsUsableOnDisk(undone, 5, true)); + } + + private static BackupRound Row(int round, string file = "", string deleted = "") + { + return new BackupRound + { + round = round, + backup_file = file, + deleted_at = deleted, + }; + } + + private static int? RoundToHold(BackupRound[] rounds, int round = Round, int played = 0) + { + return RoundRestoreGateUtility.RoundToHold(eMapStatus.Paused, rounds, round, played); + } + + [Fact] + public void AHoldStaysOnARoundThatCanStillLoad() + { + Assert.Equal(Round, RoundToHold(new[] { Row(5, "kept"), Row(Round, "kept") })); + } + + [Fact] + public void AGoneBackupFallsBackToTheNewestTheBackendKept() + { + Assert.Equal(5, RoundToHold(new[] { Row(4, "kept"), Row(5, "kept"), Row(Round) })); + } + + [Fact] + public void AGoneBackupDoesNotFallBackToARoundARestoreUndid() + { + BackupRound[] rounds = { Row(4, "kept"), Row(5, "undone", "2026-11-02"), Row(Round) }; + + Assert.Equal(4, RoundToHold(rounds)); + } + + [Fact] + public void NoBackupWithRoundsRecordedAheadNeedsRestartMap() + { + Assert.Equal(RoundRestoreState.NothingToLoad, RoundToHold(new[] { Row(5), Row(Round) })); + } + + [Fact] + public void NoRowLeftForTheHeldRoundStillProtectsTheOnesAhead() + { + Assert.Equal(RoundRestoreState.NothingToLoad, RoundToHold(new[] { Row(5) }, played: 2)); + } + + [Theory] + [InlineData(5)] + [InlineData(6)] + public void NothingRecordedAheadEndsTheHold(int played) + { + Assert.Null(RoundToHold(new[] { Row(5) }, played: played)); + } + + [Fact] + public void NoRowsAtAllEndsTheHold() + { + Assert.Null(RoundToHold(Array.Empty())); + } + + [Fact] + public void AnOldBackupIsNoReasonToHoldWithNothingAhead() + { + Assert.Null(RoundToHold(new[] { Row(5, "kept") }, played: 5)); + } + + [Fact] + public void AnOlderBackupStillRecoversRoundsRecordedAhead() + { + Assert.Equal(3, RoundToHold(new[] { Row(3, "kept"), Row(5) }, played: 4)); + } + + [Theory] + [InlineData(eMapStatus.Warmup)] + [InlineData(eMapStatus.Finished)] + public void AMapNotInPlayEndsTheHold(eMapStatus status) + { + BackupRound[] rounds = { Row(5, "kept"), Row(Round, "kept") }; + + Assert.Null(RoundRestoreGateUtility.RoundToHold(status, rounds, Round, 0)); + } + + [Fact] + public void AHoldOnNothingToLoadStaysPutSoPlayersAreOnlyToldOnce() + { + Assert.Equal( + RoundRestoreState.NothingToLoad, + RoundToHold(new[] { Row(Round) }, RoundRestoreState.NothingToLoad) + ); + } + + [Fact] + public void AHoldOnNothingToLoadKeepsTheMatchHeldWithNoVote() + { + _state.Hold(RoundRestoreState.NothingToLoad, Map); + + Assert.True(_state.NeedsRestartMap); + Assert.False(_state.CanOfferVote); + Assert.True(_state.HoldingRounds); + Assert.True(_state.Resetting); + } + + [Fact] + public void AnOrdinaryHoldDoesNotNeedRestartMap() + { + _state.Hold(Round, Map); + + Assert.False(_state.NeedsRestartMap); + Assert.True(_state.CanOfferVote); + } + + [Fact] + public void RestartMapEndsAHoldOnNothingToLoad() + { + _state.Hold(RoundRestoreState.NothingToLoad, Map); + _state.Abandon(); + + Assert.False(_state.NeedsRestartMap); + Assert.False(_state.HoldingRounds); + } + + [Fact] + public void AHoldOnNothingToLoadEndsWithItsMap() + { + _state.Hold(RoundRestoreState.NothingToLoad, Map); + + Assert.True(_state.LeaveMap(OtherMap)); + Assert.False(_state.HoldingRounds); + } + + [Fact] + public void PlayersAreToldAnAdminNeedsRestartMap() + { + Assert.Contains( + "Restart Map", + RoundRestoreGateUtility.WhyTheHoldMoved(Round, RoundRestoreState.NothingToLoad) + ); + } + + [Theory] + [InlineData(5)] + [InlineData(null)] + public void PlayersAreToldWhichRoundCanNoLongerBeRestored(int? next) + { + string told = RoundRestoreGateUtility.WhyTheHoldMoved(Round, next); + + Assert.Contains(Round.ToString(), told); + Assert.DoesNotContain("Restart Map", told); + } + + [Fact] + public void AResumeSaysWhyItWaitsForRestartMap() + { + Assert.Equal( + RoundRestoreGateUtility.RestartMapNeeded, + RoundRestoreGateUtility.ResumeRefusal(null, needsRestartMap: true) + ); + } + + [Fact] + public void AnOwedLoadDoesNotRefuseAnOrganizerRestore() + { + _state.StartOwing(Round, Map); + + Assert.Null( + RoundRestoreGateUtility.Refusal( + false, + _state.VotingOrLoading, + eMapStatus.Paused, + Round, + true, + 10, + 10, + "Competitive" + ) + ); + } + + [Theory] + [InlineData(true, false, true)] + [InlineData(false, false, false)] + [InlineData(true, true, false)] + [InlineData(false, true, false)] + public void OnlyARecoveryOnAServerWithAPanelIsOwedALoad(bool recovering, bool offline, bool owed) + { + Assert.Equal(owed, RoundRestoreGateUtility.OwesLoad(recovering, offline)); + } + + [Fact] + public void PlayersAreToldWhyResumeWaits() + { + Assert.Contains(Round.ToString(), RoundRestoreGateUtility.ResumeRefusal(Round)); + } + + [Fact] + public void NothingIsSaidWhenNoLoadIsOwed() + { + Assert.Null(RoundRestoreGateUtility.ResumeRefusal(null)); + } +} diff --git a/apps/swiftly/test/RoundRestoreWiringTests.cs b/apps/swiftly/test/RoundRestoreWiringTests.cs new file mode 100644 index 00000000..dfa9c595 --- /dev/null +++ b/apps/swiftly/test/RoundRestoreWiringTests.cs @@ -0,0 +1,97 @@ +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Xunit; + +// RoundRestorePublishTests hands MatchEvents a state of its own, so it cannot +// see whether the plugin wires the same one in or calls it from the right place. +// That code needs the engine, so these read it. +public class RoundRestoreWiringTests +{ + private static readonly string[] BuildOutput = ["bin", "obj", "build"]; + + private static string SourceDir([CallerFilePath] string testFile = "") + { + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(testFile)!, "..", "src")); + } + + private static string Strip(string source) + { + return Regex.Replace(source, @"//[^\n]*", ""); + } + + private static string Service(string file) + { + return Strip(File.ReadAllText(Path.Combine(SourceDir(), "FiveStack.Services", file))); + } + + private static string Method(string source, string signature) + { + int from = source.IndexOf(signature, StringComparison.Ordinal); + + Assert.True(from >= 0, $"{signature} is gone"); + + int to = source.IndexOf("\n }\n", from, StringComparison.Ordinal); + + Assert.True(to > from, $"the end of {signature} was not found"); + + return source[from..to]; + } + + [Fact] + public void ThePluginHasOneRestoreState() + { + IEnumerable sources = Directory + .EnumerateFiles(SourceDir(), "*.cs", SearchOption.AllDirectories) + .Where(file => + !Path.GetRelativePath(SourceDir(), file) + .Split(Path.DirectorySeparatorChar) + .Any(BuildOutput.Contains) + ) + .Select(file => Strip(File.ReadAllText(file))); + + Assert.Single(sources, source => source.Contains("AddSingleton()")); + } + + [Theory] + [InlineData("MatchEvents.cs", "public MatchEvents(")] + [InlineData("GameBackUpRounds.cs", "public GameBackUpRounds(")] + public void BothSidesAreHandedThatState(string file, string constructor) + { + Assert.Contains("RoundRestoreState restore", Method(Service(file), constructor)); + } + + [Fact] + public void ALoadIsMarkedAsALoad() + { + string restore = Method(Service("GameBackUpRounds.cs"), "public void RestoreRound(int round)"); + + Assert.Contains("_restore.StartLoad(", restore); + Assert.DoesNotContain("_restore.StartVote(", restore); + } + + [Fact] + public void AVoteIsNotMarkedAsALoad() + { + string request = Method( + Service("GameBackUpRounds.cs"), + "public void RequestRestoreBackupRound(" + ); + + Assert.Contains("_restore.StartVote(", request); + Assert.DoesNotContain("_restore.StartLoad(", request); + } + + [Fact] + public void TheFilterComesBeforeAnythingElseInThePublish() + { + string publish = Method(Service("MatchEvents.cs"), "public void PublishGameEvent("); + int filter = publish.IndexOf( + "RoundRestoreGateUtility.DroppedDuringLoad(Event, _restore.LoadingBackup)", + StringComparison.Ordinal + ); + int offline = publish.IndexOf("IsOfflineMode", StringComparison.Ordinal); + + Assert.True(filter >= 0, "PublishGameEvent does not filter"); + Assert.True(filter < offline, "the filter runs after the first way out"); + } +} diff --git a/shared/dotnet/FiveStack.Utilities/RoundRestoreGateUtility.cs b/shared/dotnet/FiveStack.Utilities/RoundRestoreGateUtility.cs index 9e185c2b..734e3b9a 100644 --- a/shared/dotnet/FiveStack.Utilities/RoundRestoreGateUtility.cs +++ b/shared/dotnet/FiveStack.Utilities/RoundRestoreGateUtility.cs @@ -12,6 +12,81 @@ public static class RoundRestoreGateUtility // Same for match_state, which the api sends to restart a map from warmup. public const string MatchStateRefusedPrefix = "match_state refused:"; + // A restore vote leaves the round running, because mp_pause_match waits for + // the next freezetime, so a round ending during one is real and is lost if + // the vote fails. A recovery vote is not that: the server is behind the + // backend, and anything it reported would land over rounds already kept. + public static bool HoldsRoundResults(bool resetting, bool voteOpen, bool recovering) + { + return resetting && (!voteOpen || recovering); + } + + // Only a server behind the backend has anything to protect, and offline + // nothing would ever answer. + public static bool OwesLoad(bool recovering, bool offline) + { + return recovering && !offline; + } + + // The api refuses a round it has no row for and tells nobody, so a backup + // file of one can never load. Offline there is no api to refuse it. + public static bool IsUsableOnDisk(IEnumerable rounds, int round, bool offline) + { + return !IsUndone(rounds, round) + && (offline || rounds.Any(backup => backup.round == round)); + } + + public const string RestartMapNeeded = + " No round can be restored, an admin needs to use Restart Map."; + + // Where a hold goes when it comes back, as a hold on a round that can't + // load would come back for ever. The newest round the backend can load, or + // NothingToLoad if it has none. Null when no recorded round is ahead of the + // server: there is nothing left to play over. + public static int? RoundToHold( + eMapStatus status, + IEnumerable rounds, + int round, + int totalRoundsPlayed + ) + { + if (!IsInPlay(status)) + { + return null; + } + + if (LatestBackup(rounds, round) != null) + { + return round; + } + + if (HighestRecordedRound(rounds, 0) <= totalRoundsPlayed) + { + return null; + } + + return HighestRestorableRound(rounds, 0); + } + + public static string WhyTheHoldMoved(int round, int? next) + { + return next == RoundRestoreState.NothingToLoad + ? RestartMapNeeded + : $" Round {round} can't be restored, its backup is gone."; + } + + public static string? ResumeRefusal(int? owedRound, bool needsRestartMap = false) + { + if (needsRestartMap) + { + return RestartMapNeeded; + } + + return owedRound == null + ? null + : $" Round {owedRound} is still being restored, the match can't resume yet."; + } + public static string? Refusal( bool restoresDisabled, bool alreadyRestoring, @@ -58,6 +133,24 @@ string matchType return null; } + // Every stat row the api keeps for a round comes from one of these. A + // backup load outlasts what the api clears when it restores a round, so + // whatever the engine reports while it runs would stay in the stats. + private static readonly HashSet RoundStatEvents = new() + { + "kill", + "assist", + "damage", + "utility", + "flash", + "objective", + }; + + public static bool DroppedDuringLoad(string gameEvent, bool loadingBackup) + { + return loadingBackup && RoundStatEvents.Contains(gameEvent); + } + // A map's backups are appended as rounds are played, so a round played // again after a restore is listed twice and the newer backup is the one // that matches what was replayed. A round the api marked deleted was diff --git a/shared/dotnet/FiveStack.Utilities/RoundRestoreState.cs b/shared/dotnet/FiveStack.Utilities/RoundRestoreState.cs new file mode 100644 index 00000000..7f8ce9b9 --- /dev/null +++ b/shared/dotnet/FiveStack.Utilities/RoundRestoreState.cs @@ -0,0 +1,162 @@ +namespace FiveStack.Utilities +{ + public readonly record struct OwedLoadTimeout(bool Expired, int? HoldRound = null); + + // A restore vote leaves the round running, because mp_pause_match waits for the + // next freezetime, so its kills are real. A backup load outlasts what the api + // clears when it restores a round, so anything reported then would stay in the + // stats. + // + // A server that came back behind the backend is held on the round it can + // restore until every player is back to vote on it. Meanwhile nothing is + // played, published or written over the backups being restored. A passed vote + // ends the hold before the load has started, and the api replies nothing to + // the command, so the load is owed until it starts: resuming before then would + // write round 1 over the round 1 the backend already has. + public class RoundRestoreState + { + // Long, because a slow answer is not a lost one: the event is resent until + // the api has handled it, so a panel deploy only delays it. Going back to + // the hold early costs a second vote. Waiting costs a few paused minutes. + public const int OwedSeconds = 300; + + // A hold on this has rounds recorded ahead of it and no backup of any of + // them. There is nothing to vote on, and playing would go over them, so it + // stays held until Restart Map. + public const int NothingToLoad = 0; + + private bool _voting; + private bool _recoveryVote; + private int _owing; + private int? _owedRound; + private Guid? _map; + private Guid? _heldMap; + private int? _loadOwed; + + public int? HeldRound { get; private set; } + + public int? OwedRound => _owedRound; + + public bool LoadingBackup { get; private set; } + + public bool VotingOrLoading => _voting || LoadingBackup; + + public bool InProgress => VotingOrLoading || _owedRound != null; + + public bool Resetting => InProgress || HeldRound != null; + + public bool HoldingRounds => HeldRound != null || _recoveryVote || _owedRound != null; + + // Taking a hold to vote on it while something else is running loses it: + // the vote is refused, and nothing puts the hold back. + public bool CanOfferVote => HeldRound > NothingToLoad && !InProgress; + + public bool NeedsRestartMap => HeldRound == NothingToLoad; + + public void Hold(int round, Guid? map = null) + { + HeldRound = round; + _heldMap = map; + } + + // Says whether anything went, so the caller can put backups right. + public bool LeaveMap(Guid? map) + { + bool left = false; + + if ((_voting || _owedRound != null) && _map != map) + { + End(); + left = true; + } + + if (HeldRound != null && _heldMap != map) + { + HeldRound = null; + left = true; + } + + return left; + } + + public int? ReleaseHold() + { + int? held = HeldRound; + HeldRound = null; + + return held; + } + + // A result waiting for the next round start is from the round this undoes. + public event Action? LoadStarted; + + public void StartVote(bool recovering = false, Guid? map = null) + { + End(); + _voting = true; + _recoveryVote = recovering; + _map = map; + } + + public int StartOwing(int round, Guid? map) + { + End(); + _owedRound = round; + _map = map; + + return ++_owing; + } + + // heldRound is a hold this load took over, which a failed load puts back. + public void StartLoad(int? heldRound = null) + { + _loadOwed = _owedRound ?? heldRound; + _owedRound = null; + _voting = false; + _recoveryVote = false; + LoadingBackup = true; + LoadStarted?.Invoke(); + } + + // The round to hold again, if a load was owed. + public int? FailedLoad() + { + int? owed = _owedRound ?? _loadOwed; + + End(); + + return owed; + } + + public OwedLoadTimeout TimedOut(int owing, Guid? map) + { + if (owing != _owing || _owedRound == null) + { + return default; + } + + int round = _owedRound.Value; + bool sameMap = _map == map; + + End(); + + return new OwedLoadTimeout(true, sameMap ? round : null); + } + + public void End() + { + _voting = false; + _recoveryVote = false; + _owedRound = null; + _map = null; + _loadOwed = null; + LoadingBackup = false; + } + + public void Abandon() + { + End(); + HeldRound = null; + } + } +}