From 16b43fecbb077047ac59cb5e0e6542f4606de826 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 30 Jun 2026 23:26:30 -0400 Subject: [PATCH 1/4] chore: attempt to upload orphaned demo --- src/FiveStack.Services/GameDemos.cs | 120 ++++++++++++++++++++++------ src/FiveStackPlugin.cs | 16 ++++ 2 files changed, 111 insertions(+), 25 deletions(-) diff --git a/src/FiveStack.Services/GameDemos.cs b/src/FiveStack.Services/GameDemos.cs index bfcf367..e9c7f6f 100644 --- a/src/FiveStack.Services/GameDemos.cs +++ b/src/FiveStack.Services/GameDemos.cs @@ -141,6 +141,69 @@ public async Task UploadDemos() _logger.LogInformation("Uploaded all demos"); } + // Scans the demos directory for leftover .dem files and attempts to upload + // them. Demos are normally uploaded by the GameEnd timer chain, but that + // lives entirely in memory — a server crash/restart during the post-match + // window drops the pending upload and orphans the file on disk. Running this + // on startup self-heals those cases. Safe to retry: the API rejects demos + // for maps that aren't finished (409) and cleans up ones already uploaded + // (406) or whose map is gone (410). + public async Task UploadOrphanedDemos() + { + if (_environmentService.IsOfflineMode()) + { + return; + } + + string demosRoot = $"{_rootDir}/demos"; + if (!Directory.Exists(demosRoot)) + { + return; + } + + string[] files; + try + { + files = Directory.GetFiles(demosRoot, "*.dem", SearchOption.AllDirectories); + } + catch (Exception ex) + { + _logger.LogError($"Failed to scan for orphaned demos: {ex.Message}"); + return; + } + + if (files.Length == 0) + { + return; + } + + _logger.LogInformation( + $"Found {files.Length} demo(s) on disk, attempting to recover uploads" + ); + + foreach (string file in files) + { + // Skip a demo that belongs to a match that is actively recording right + // now, so we never upload a partially-written file. + string? matchId = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(file))); + if ( + Guid.TryParse(matchId, out Guid parsedMatchId) + && File.Exists(GetLockFilePath(parsedMatchId)) + ) + { + _logger.LogInformation( + $"Skipping demo for actively recording match {matchId}: {file}" + ); + continue; + } + + _logger.LogInformation($"Recovering orphaned demo {file}"); + await UploadDemo(file); + } + + _logger.LogInformation("Finished recovering orphaned demos"); + } + public async Task UploadDemo(string filePath) { try @@ -150,27 +213,40 @@ public async Task UploadDemo(string filePath) return; } - MatchData? match = _matchService.GetCurrentMatch()?.GetMatchData(); - string? serverId = _environmentService.GetServerId(); string? apiPassword = _environmentService.GetServerApiPassword(); - if (serverId == null || apiPassword == null || match == null) + if (serverId == null || apiPassword == null) { return; } + // Demos live at {_rootDir}/demos/{matchId}/{mapId}/{demo}.dem — derive + // the ids from the path rather than GetCurrentMatch() so uploads work + // even when the demo belongs to a match that is no longer current + // (e.g. recovered on startup after a crash/restart). string demoName = Path.GetFileName(filePath); + string? mapId = Path.GetFileName(Path.GetDirectoryName(filePath)); + string? matchId = Path.GetFileName( + Path.GetDirectoryName(Path.GetDirectoryName(filePath)) + ); - string? presignedUrl = await GetPresignedUrl(filePath); - if (string.IsNullOrEmpty(presignedUrl)) + if (!Guid.TryParse(matchId, out _) || !Guid.TryParse(mapId, out _)) { - _logger.LogCritical( - $"Failed to get presigned URL (match {match.id} map {match.current_match_map_id} demo {demoName})" + _logger.LogWarning( + $"Skipping demo with unexpected path (cannot derive match/map ids): {filePath}" ); return; } + string? presignedUrl = await GetPresignedUrl(matchId, mapId, filePath); + if (string.IsNullOrEmpty(presignedUrl)) + { + // GetPresignedUrl already logs the reason (and cleans up the file + // when the map is already uploaded or gone). + return; + } + using var httpClient = new HttpClient(); httpClient.Timeout = System.Threading.Timeout.InfiniteTimeSpan; @@ -185,25 +261,25 @@ public async Task UploadDemo(string filePath) request.Content.Headers.ContentLength = fileInfo.Length; _logger.LogInformation( - $"PUT demo {demoName} ({fileInfo.Length} bytes) for match {match.id}" + $"PUT demo {demoName} ({fileInfo.Length} bytes) for match {matchId}" ); var response = await httpClient.SendAsync(request); _logger.LogInformation( - $"demo PUT response {(int)response.StatusCode} {response.StatusCode} (match {match.id} demo {demoName})" + $"demo PUT response {(int)response.StatusCode} {response.StatusCode} (match {matchId} demo {demoName})" ); if (response.IsSuccessStatusCode) { - _logger.LogInformation($"demo uploaded (match {match.id} demo {demoName})"); + _logger.LogInformation($"demo uploaded (match {matchId} demo {demoName})"); var notifyEndpoint = - $"{_environmentService.GetDemosUrl()}/demos/{match.id}/uploaded"; + $"{_environmentService.GetDemosUrl()}/demos/{matchId}/uploaded"; var notifyRequest = new { demo = demoName, - mapId = match.current_match_map_id, + mapId = mapId, size = fileInfo.Length, }; @@ -216,7 +292,7 @@ public async Task UploadDemo(string filePath) ); _logger.LogInformation( - $"demo uploaded notify response {(int)notifyResponse.StatusCode} {notifyResponse.StatusCode} (match {match.id} demo {demoName})" + $"demo uploaded notify response {(int)notifyResponse.StatusCode} {notifyResponse.StatusCode} (match {matchId} demo {demoName})" ); if (notifyResponse.IsSuccessStatusCode) @@ -242,22 +318,16 @@ public async Task UploadDemo(string filePath) } } - private async Task GetPresignedUrl(string filePath) + private async Task GetPresignedUrl(string matchId, string mapId, string filePath) { - MatchData? match = _matchService.GetCurrentMatch()?.GetMatchData(); - - if (match == null) - { - return null; - } - string? apiPassword = _environmentService.GetServerApiPassword(); if (apiPassword == null) { return null; } - string endpoint = $"{_environmentService.GetDemosUrl()}/demos/{match.id}/pre-signed"; + string demoName = Path.GetFileName(filePath); + string endpoint = $"{_environmentService.GetDemosUrl()}/demos/{matchId}/pre-signed"; using var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( @@ -267,14 +337,14 @@ public async Task UploadDemo(string filePath) var requestBody = new { - demo = Path.GetFileName(filePath), - mapId = _matchService.GetCurrentMatch()?.GetMatchData()?.current_match_map_id, + demo = demoName, + mapId = mapId, }; var response = await httpClient.PostAsJsonAsync(endpoint, requestBody); _logger.LogInformation( - $"presigned url response {(int)response.StatusCode} {response.StatusCode} (match {match.id} map {match.current_match_map_id} demo {Path.GetFileName(filePath)})" + $"presigned url response {(int)response.StatusCode} {response.StatusCode} (match {matchId} map {mapId} demo {demoName})" ); switch (response.StatusCode) diff --git a/src/FiveStackPlugin.cs b/src/FiveStackPlugin.cs index 619b603..ca583b4 100644 --- a/src/FiveStackPlugin.cs +++ b/src/FiveStackPlugin.cs @@ -103,6 +103,22 @@ public override void Load(bool hotReload) { _matchService.GetMatchFromOffline(); } + + // Recover any demos left on disk by a previous crash/restart. Deferred so + // env + API connectivity have settled, and off the main thread since it is + // pure file IO + HTTP and never touches game state. + _ = Task.Run(async () => + { + await Task.Delay(TimeSpan.FromSeconds(15)); + try + { + await _gameDemos.UploadOrphanedDemos(); + } + catch (Exception ex) + { + _logger.LogError($"Failed to recover orphaned demos: {ex.Message}"); + } + }); } public override void Unload(bool hotReload) From 94e69c22af7ac95de7eda0a33a6ec31e1c0141b3 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Wed, 1 Jul 2026 12:25:48 -0400 Subject: [PATCH 2/4] wip --- src/FiveStack.Commands/Demo.cs | 5 +- src/FiveStack.Events/GameEnd.cs | 70 +++- src/FiveStack.Services/EnvironmentService.cs | 12 + src/FiveStack.Services/GameDemos.cs | 340 ++++++++++++------- src/FiveStackPlugin.cs | 21 +- 5 files changed, 307 insertions(+), 141 deletions(-) diff --git a/src/FiveStack.Commands/Demo.cs b/src/FiveStack.Commands/Demo.cs index 8cbe1cf..a9502a0 100644 --- a/src/FiveStack.Commands/Demo.cs +++ b/src/FiveStack.Commands/Demo.cs @@ -19,7 +19,10 @@ public async void OnUploadDemo(CCSPlayerController? player, CommandInfo command) match.UpdateMapStatus(eMapStatus.UploadingDemo); - await _gameDemos.UploadDemos(); + using var cts = new System.Threading.CancellationTokenSource( + TimeSpan.FromSeconds(_environmentService.GetDemoUploadTimeLimitSeconds()) + ); + await _gameDemos.UploadDemos(cts.Token); } [ConsoleCommand("test_start_demo", "start demo recording")] diff --git a/src/FiveStack.Events/GameEnd.cs b/src/FiveStack.Events/GameEnd.cs index d2412d5..9e3c77d 100644 --- a/src/FiveStack.Events/GameEnd.cs +++ b/src/FiveStack.Events/GameEnd.cs @@ -197,19 +197,64 @@ private void HandleEndOfMap(Guid? winningLineupId) currentMap.id ); - try + // Only finish the map once the demo is confirmed uploaded, so + // the server stays reserved until it's safe. If the budget + // runs out, give up loudly and finish anyway (retried on next + // startup) so the match never gets stuck. + int uploadTimeLimit = + _environmentService.GetDemoUploadTimeLimitSeconds(); + using var uploadCts = new System.Threading.CancellationTokenSource( + TimeSpan.FromSeconds(uploadTimeLimit) + ); + + bool uploaded = false; + int attempt = 0; + while (!uploadCts.IsCancellationRequested) { - await _gameDemos.UploadDemos(); - _logger.LogInformation( - "Demo upload finished (match={MatchId})", - expectedMatchId - ); + attempt++; + try + { + uploaded = await _gameDemos.UploadDemos(uploadCts.Token); + if (uploaded) + { + _logger.LogInformation( + "Demo upload finished (match={MatchId} attempts={Attempts})", + expectedMatchId, + attempt + ); + break; + } + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "UploadDemos failed (match={MatchId} attempt={Attempt})", + expectedMatchId, + attempt + ); + } + + try + { + await Task.Delay(TimeSpan.FromSeconds(10), uploadCts.Token); + } + catch (OperationCanceledException) + { + break; + } } - catch (Exception ex) + + if (!uploaded) { - _logger.LogError( - ex, - "UploadDemos failed after map end for match {MatchId}", + _logger.LogCritical( + "Demo upload did NOT complete within {Limit}s ({Attempts} attempt(s)) — this server may be too slow to upload demos. Finishing the map without a confirmed upload (match={MatchId}); the demo remains on disk and will be retried on next startup.", + uploadTimeLimit, + attempt, expectedMatchId ); } @@ -232,8 +277,9 @@ private void HandleEndOfMap(Guid? winningLineupId) bool isSurrenderedNow = wasSurrendered || next.isSurrendered(); _logger.LogInformation( - "Demo upload done — finishing map and switching (match={MatchId})", - expectedMatchId + "Finishing map and switching (match={MatchId} demoUploaded={Uploaded})", + expectedMatchId, + uploaded ); if (isSurrenderedNow) diff --git a/src/FiveStack.Services/EnvironmentService.cs b/src/FiveStack.Services/EnvironmentService.cs index 5ea45de..32a9deb 100644 --- a/src/FiveStack.Services/EnvironmentService.cs +++ b/src/FiveStack.Services/EnvironmentService.cs @@ -49,6 +49,18 @@ public bool AllowBots() return Environment.GetEnvironmentVariable("ALLOW_BOTS") == "true"; } + // Total budget for uploading a map's demos after it ends, across retries. + public int GetDemoUploadTimeLimitSeconds() + { + var raw = Environment.GetEnvironmentVariable("DEMO_UPLOAD_TIME_LIMIT_SECONDS"); + if (int.TryParse(raw, out int seconds) && seconds > 0) + { + return seconds; + } + + return 300; + } + public bool isOnGameServerNode() { return Environment.GetEnvironmentVariable("GAME_NODE_SERVER") == "true"; diff --git a/src/FiveStack.Services/GameDemos.cs b/src/FiveStack.Services/GameDemos.cs index e9c7f6f..1005ea7 100644 --- a/src/FiveStack.Services/GameDemos.cs +++ b/src/FiveStack.Services/GameDemos.cs @@ -27,6 +27,22 @@ public class GameDemos private readonly IStringLocalizer _localizer; private string _rootDir = "/opt"; + // A finalized demo hasn't been written since tv_stoprecord; a file touched + // more recently than this may still be recording, so recovery skips it. + private const int RecordingFinalizeWindowSeconds = 60; + + // Shared to avoid socket churn; per-request auth is set on each message. + // Timeout is infinite by design — the caller's CancellationToken bounds it. + private static readonly HttpClient _httpClient = new HttpClient + { + Timeout = System.Threading.Timeout.InfiniteTimeSpan, + }; + + // Guards against two paths (GameEnd, orphan recovery, the console command) + // uploading/deleting the same demo at once. + private readonly HashSet _uploadsInProgress = new(); + private readonly object _uploadsLock = new(); + public GameDemos( ILogger logger, GameServer gameServer, @@ -89,7 +105,9 @@ public void Stop() } string demoPath = GetMatchDemoPath(); - int demoCount = Directory.Exists(demoPath) ? Directory.GetFiles(demoPath, "*").Length : 0; + int demoCount = Directory.Exists(demoPath) + ? Directory.GetFiles(demoPath, "*.dem").Length + : 0; File.Delete(GetLockFilePath(match.id)); Server.NextFrame(() => @@ -111,7 +129,10 @@ public void StopTV() }); } - public async Task UploadDemos() + // Returns false if any demo still needs another attempt. + public async Task UploadDemos( + System.Threading.CancellationToken cancellationToken = default + ) { string demoPath = GetMatchDemoPath(); _logger.LogInformation($"Uploading demos from {demoPath}"); @@ -119,36 +140,61 @@ public async Task UploadDemos() if (!Directory.Exists(demoPath)) { _logger.LogCritical($"Demo directory does not exist: {demoPath}"); - return; + return true; } - string[] files = Directory.GetFiles(demoPath, "*"); + string[] files = Directory.GetFiles(demoPath, "*.dem"); if (files.Length == 0) { _logger.LogWarning($"No demo files found in {demoPath}"); - return; + return true; } _logger.LogInformation($"Found {files.Length} demo file(s) in {demoPath}"); + bool allUploaded = true; foreach (string file in files) { - long size = new FileInfo(file).Length; - _logger.LogInformation($"Uploading demo {file} ({size} bytes)"); - await UploadDemo(file); + _logger.LogInformation($"Uploading demo {file}"); + if (!await UploadDemo(file, cancellationToken)) + { + allUploaded = false; + } + } + + if (allUploaded) + { + _logger.LogInformation("Uploaded all demos"); } - _logger.LogInformation("Uploaded all demos"); + + return allUploaded; } - // Scans the demos directory for leftover .dem files and attempts to upload - // them. Demos are normally uploaded by the GameEnd timer chain, but that - // lives entirely in memory — a server crash/restart during the post-match - // window drops the pending upload and orphans the file on disk. Running this - // on startup self-heals those cases. Safe to retry: the API rejects demos - // for maps that aren't finished (409) and cleans up ones already uploaded - // (406) or whose map is gone (410). - public async Task UploadOrphanedDemos() + // Recording locks never survive a process restart, so any left on disk at + // startup are stale from a crash. Clear them so Start() doesn't refuse to + // record a resumed match. + public void ClearStaleRecordingLocks() + { + try + { + foreach (string lockFile in Directory.GetFiles(_rootDir, ".recording-demo-*")) + { + File.Delete(lockFile); + _logger.LogInformation($"Cleared stale recording lock {lockFile}"); + } + } + catch (Exception ex) + { + _logger.LogError($"Failed to clear stale recording locks: {ex.Message}"); + } + } + + // Uploads .dem files left on disk by a crash/restart during the post-match + // upload window. Runs on startup to self-heal. + public async Task UploadOrphanedDemos( + System.Threading.CancellationToken cancellationToken = default + ) { if (_environmentService.IsOfflineMode()) { @@ -181,167 +227,205 @@ public async Task UploadOrphanedDemos() $"Found {files.Length} demo(s) on disk, attempting to recover uploads" ); + int timeLimit = _environmentService.GetDemoUploadTimeLimitSeconds(); + foreach (string file in files) { - // Skip a demo that belongs to a match that is actively recording right - // now, so we never upload a partially-written file. - string? matchId = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(file))); + cancellationToken.ThrowIfCancellationRequested(); + + // Skip a file that's still being written (an active recording); a + // finalized demo hasn't been touched since tv_stoprecord. if ( - Guid.TryParse(matchId, out Guid parsedMatchId) - && File.Exists(GetLockFilePath(parsedMatchId)) + File.GetLastWriteTimeUtc(file) + > DateTime.UtcNow.AddSeconds(-RecordingFinalizeWindowSeconds) ) { _logger.LogInformation( - $"Skipping demo for actively recording match {matchId}: {file}" + $"Skipping recently-written demo (may still be recording): {file}" ); continue; } _logger.LogInformation($"Recovering orphaned demo {file}"); - await UploadDemo(file); + using var cts = System.Threading.CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken + ); + cts.CancelAfter(TimeSpan.FromSeconds(timeLimit)); + await UploadDemo(file, cts.Token); } _logger.LogInformation("Finished recovering orphaned demos"); } - public async Task UploadDemo(string filePath) + // Returns true when resolved (uploaded, or already-uploaded/map-gone); + // false on a retryable failure. + public async Task UploadDemo( + string filePath, + System.Threading.CancellationToken cancellationToken = default + ) { - try + if (_environmentService.IsOfflineMode()) { - if (_environmentService.IsOfflineMode()) - { - return; - } + return true; + } + + string? serverId = _environmentService.GetServerId(); + string? apiPassword = _environmentService.GetServerApiPassword(); - string? serverId = _environmentService.GetServerId(); - string? apiPassword = _environmentService.GetServerApiPassword(); + if (serverId == null || apiPassword == null) + { + _logger.LogWarning("Cannot upload demo: server id / api password not configured"); + return true; + } - if (serverId == null || apiPassword == null) + // Derive ids from the path ({root}/demos/{matchId}/{mapId}/{demo}.dem) + // so this works for demos that aren't the current match. + string demoName = Path.GetFileName(filePath); + string? mapId = Path.GetFileName(Path.GetDirectoryName(filePath)); + string? matchId = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(filePath))); + + if (!Guid.TryParse(matchId, out _) || !Guid.TryParse(mapId, out _)) + { + _logger.LogWarning( + $"Skipping demo with unexpected path (cannot derive match/map ids): {filePath}" + ); + return true; + } + + lock (_uploadsLock) + { + if (!_uploadsInProgress.Add(filePath)) { - return; + _logger.LogInformation($"Demo already being uploaded, skipping: {demoName}"); + return true; } + } - // Demos live at {_rootDir}/demos/{matchId}/{mapId}/{demo}.dem — derive - // the ids from the path rather than GetCurrentMatch() so uploads work - // even when the demo belongs to a match that is no longer current - // (e.g. recovered on startup after a crash/restart). - string demoName = Path.GetFileName(filePath); - string? mapId = Path.GetFileName(Path.GetDirectoryName(filePath)); - string? matchId = Path.GetFileName( - Path.GetDirectoryName(Path.GetDirectoryName(filePath)) + try + { + var (presignedUrl, resolved) = await GetPresignedUrl( + matchId, + mapId, + filePath, + apiPassword, + cancellationToken ); - if (!Guid.TryParse(matchId, out _) || !Guid.TryParse(mapId, out _)) + if (resolved) { - _logger.LogWarning( - $"Skipping demo with unexpected path (cannot derive match/map ids): {filePath}" - ); - return; + return true; } - string? presignedUrl = await GetPresignedUrl(matchId, mapId, filePath); if (string.IsNullOrEmpty(presignedUrl)) { - // GetPresignedUrl already logs the reason (and cleans up the file - // when the map is already uploaded or gone). - return; + _logger.LogCritical( + $"Failed to get presigned URL (match {matchId} map {mapId} demo {demoName})" + ); + return false; } - using var httpClient = new HttpClient(); - httpClient.Timeout = System.Threading.Timeout.InfiniteTimeSpan; - var fileInfo = new FileInfo(filePath); - using (var fileStream = File.OpenRead(filePath)) - { - var request = new HttpRequestMessage(HttpMethod.Put, presignedUrl); - request.Content = new StreamContent(fileStream); - request.Content.Headers.ContentType = new MediaTypeHeaderValue( - "application/octet-stream" - ); - request.Content.Headers.ContentLength = fileInfo.Length; + using var fileStream = File.OpenRead(filePath); - _logger.LogInformation( - $"PUT demo {demoName} ({fileInfo.Length} bytes) for match {matchId}" - ); + var request = new HttpRequestMessage(HttpMethod.Put, presignedUrl); + request.Content = new StreamContent(fileStream); + request.Content.Headers.ContentType = new MediaTypeHeaderValue( + "application/octet-stream" + ); + request.Content.Headers.ContentLength = fileInfo.Length; - var response = await httpClient.SendAsync(request); + _logger.LogInformation( + $"PUT demo {demoName} ({fileInfo.Length} bytes) for match {matchId}" + ); - _logger.LogInformation( - $"demo PUT response {(int)response.StatusCode} {response.StatusCode} (match {matchId} demo {demoName})" - ); + var response = await _httpClient.SendAsync(request, cancellationToken); - if (response.IsSuccessStatusCode) - { - _logger.LogInformation($"demo uploaded (match {matchId} demo {demoName})"); + _logger.LogInformation( + $"demo PUT response {(int)response.StatusCode} {response.StatusCode} (match {matchId} demo {demoName})" + ); - var notifyEndpoint = - $"{_environmentService.GetDemosUrl()}/demos/{matchId}/uploaded"; - var notifyRequest = new + if (!response.IsSuccessStatusCode) + { + _logger.LogCritical($"unable to upload demo {response.StatusCode}"); + return false; + } + + _logger.LogInformation($"demo uploaded (match {matchId} demo {demoName})"); + + var notifyRequest = new HttpRequestMessage( + HttpMethod.Post, + $"{_environmentService.GetDemosUrl()}/demos/{matchId}/uploaded" + ) + { + Content = JsonContent.Create( + new { demo = demoName, mapId = mapId, size = fileInfo.Length, - }; + } + ), + }; + notifyRequest.Headers.Authorization = new AuthenticationHeaderValue( + "Bearer", + apiPassword + ); - using var notifyClient = new HttpClient(); - notifyClient.DefaultRequestHeaders.Authorization = - new AuthenticationHeaderValue("Bearer", apiPassword); - var notifyResponse = await notifyClient.PostAsJsonAsync( - notifyEndpoint, - notifyRequest - ); + var notifyResponse = await _httpClient.SendAsync(notifyRequest, cancellationToken); - _logger.LogInformation( - $"demo uploaded notify response {(int)notifyResponse.StatusCode} {notifyResponse.StatusCode} (match {matchId} demo {demoName})" - ); + _logger.LogInformation( + $"demo uploaded notify response {(int)notifyResponse.StatusCode} {notifyResponse.StatusCode} (match {matchId} demo {demoName})" + ); - if (notifyResponse.IsSuccessStatusCode) - { - File.Delete(filePath); - } - else - { - _logger.LogCritical( - $"Failed to notify about demo upload: {notifyResponse.StatusCode}" - ); - } - } - else - { - _logger.LogCritical($"unable to upload demo {response.StatusCode}"); - } + if (!notifyResponse.IsSuccessStatusCode) + { + _logger.LogCritical( + $"Failed to notify about demo upload: {notifyResponse.StatusCode}" + ); + return false; } + + File.Delete(filePath); + return true; + } + catch (OperationCanceledException) + { + _logger.LogWarning($"Demo upload canceled (time budget reached): {filePath}"); + return false; } catch (Exception ex) { _logger.LogCritical($"An error occurred during file upload: {ex.Message}"); + return false; } - } - - private async Task GetPresignedUrl(string matchId, string mapId, string filePath) - { - string? apiPassword = _environmentService.GetServerApiPassword(); - if (apiPassword == null) + finally { - return null; + lock (_uploadsLock) + { + _uploadsInProgress.Remove(filePath); + } } + } + // "resolved" = nothing left to upload (already uploaded / map gone); file cleaned up. + private async Task<(string? PresignedUrl, bool Resolved)> GetPresignedUrl( + string matchId, + string mapId, + string filePath, + string apiPassword, + System.Threading.CancellationToken cancellationToken = default + ) + { string demoName = Path.GetFileName(filePath); string endpoint = $"{_environmentService.GetDemosUrl()}/demos/{matchId}/pre-signed"; - using var httpClient = new HttpClient(); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( - "Bearer", - apiPassword - ); - - var requestBody = new + var request = new HttpRequestMessage(HttpMethod.Post, endpoint) { - demo = demoName, - mapId = mapId, + Content = JsonContent.Create(new { demo = demoName, mapId = mapId }), }; + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiPassword); - var response = await httpClient.PostAsJsonAsync(endpoint, requestBody); + var response = await _httpClient.SendAsync(request, cancellationToken); _logger.LogInformation( $"presigned url response {(int)response.StatusCode} {response.StatusCode} (match {matchId} map {mapId} demo {demoName})" @@ -355,22 +439,32 @@ public async Task UploadDemo(string filePath) case HttpStatusCode.NotAcceptable: _logger.LogInformation("demo is already uploaded"); File.Delete(filePath); - break; + return (null, true); case HttpStatusCode.Gone: _logger.LogInformation("match map not found"); File.Delete(filePath); - break; + return (null, true); } if (!response.IsSuccessStatusCode) { _logger.LogCritical($"unable to get presigned url: {response.StatusCode}"); - return null; + return (null, false); } - var responseContent = await response.Content.ReadFromJsonAsync(); + var responseContent = await response.Content.ReadFromJsonAsync( + cancellationToken + ); + + if (string.IsNullOrEmpty(responseContent?.PresignedUrl)) + { + _logger.LogCritical( + $"presigned url response had no url (match {matchId} map {mapId} demo {demoName})" + ); + return (null, false); + } - return responseContent?.PresignedUrl; + return (responseContent.PresignedUrl, false); } private string GetMatchDemoPath() diff --git a/src/FiveStackPlugin.cs b/src/FiveStackPlugin.cs index ca583b4..66f4d3b 100644 --- a/src/FiveStackPlugin.cs +++ b/src/FiveStackPlugin.cs @@ -66,6 +66,7 @@ IStringLocalizer localizer } private Timer _pingTimer; + private readonly System.Threading.CancellationTokenSource _shutdownCts = new(); public override void Load(bool hotReload) { @@ -104,15 +105,24 @@ public override void Load(bool hotReload) _matchService.GetMatchFromOffline(); } - // Recover any demos left on disk by a previous crash/restart. Deferred so - // env + API connectivity have settled, and off the main thread since it is - // pure file IO + HTTP and never touches game state. + // On a fresh start (not a hot reload, where a recording may still be + // live) any recording lock left on disk is stale from a crash. + if (!hotReload) + { + _gameDemos.ClearStaleRecordingLocks(); + } + + // Recover demos left on disk by a previous crash/restart, once env/API have settled. _ = Task.Run(async () => { - await Task.Delay(TimeSpan.FromSeconds(15)); try { - await _gameDemos.UploadOrphanedDemos(); + await Task.Delay(TimeSpan.FromSeconds(15), _shutdownCts.Token); + await _gameDemos.UploadOrphanedDemos(_shutdownCts.Token); + } + catch (OperationCanceledException) + { + // plugin unloaded before/while recovering } catch (Exception ex) { @@ -123,6 +133,7 @@ public override void Load(bool hotReload) public override void Unload(bool hotReload) { + _shutdownCts.Cancel(); _pingTimer?.Dispose(); ConnectClientFunc.Unhook(ConnectClientHook, HookMode.Pre); From 340176b1042fc700b519da890f8db3e15d500d4b Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Wed, 1 Jul 2026 13:05:36 -0400 Subject: [PATCH 3/4] wip --- src/FiveStack.Events/GameEnd.cs | 15 ++++-- src/FiveStack.Services/GameDemos.cs | 75 +++++++++++++++++++---------- 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/src/FiveStack.Events/GameEnd.cs b/src/FiveStack.Events/GameEnd.cs index 9e3c77d..7133de8 100644 --- a/src/FiveStack.Events/GameEnd.cs +++ b/src/FiveStack.Events/GameEnd.cs @@ -207,6 +207,11 @@ private void HandleEndOfMap(Guid? winningLineupId) TimeSpan.FromSeconds(uploadTimeLimit) ); + // Upload the specific map that just ended, not whatever the + // current match/map resolves to when the timer fires. + string uploadMatchId = expectedMatchId.ToString(); + string uploadMapId = currentMap.id.ToString(); + bool uploaded = false; int attempt = 0; while (!uploadCts.IsCancellationRequested) @@ -214,7 +219,11 @@ private void HandleEndOfMap(Guid? winningLineupId) attempt++; try { - uploaded = await _gameDemos.UploadDemos(uploadCts.Token); + uploaded = await _gameDemos.UploadDemos( + uploadMatchId, + uploadMapId, + uploadCts.Token + ); if (uploaded) { _logger.LogInformation( @@ -225,10 +234,6 @@ private void HandleEndOfMap(Guid? winningLineupId) break; } } - catch (OperationCanceledException) - { - break; - } catch (Exception ex) { _logger.LogError( diff --git a/src/FiveStack.Services/GameDemos.cs b/src/FiveStack.Services/GameDemos.cs index 1005ea7..66a4802 100644 --- a/src/FiveStack.Services/GameDemos.cs +++ b/src/FiveStack.Services/GameDemos.cs @@ -129,12 +129,29 @@ public void StopTV() }); } - // Returns false if any demo still needs another attempt. - public async Task UploadDemos( + // Uploads the current match/map's demos. + public Task UploadDemos(System.Threading.CancellationToken cancellationToken = default) + { + return UploadDemosFromPath(GetMatchDemoPath(), cancellationToken); + } + + // Uploads a specific match/map's demos, independent of the current match + // state (which can change during the post-match upload window). + public Task UploadDemos( + string matchId, + string mapId, System.Threading.CancellationToken cancellationToken = default ) { - string demoPath = GetMatchDemoPath(); + return UploadDemosFromPath($"{_rootDir}/demos/{matchId}/{mapId}", cancellationToken); + } + + // Returns false if any demo still needs another attempt. + private async Task UploadDemosFromPath( + string demoPath, + System.Threading.CancellationToken cancellationToken + ) + { _logger.LogInformation($"Uploading demos from {demoPath}"); if (!Directory.Exists(demoPath)) @@ -227,30 +244,36 @@ public async Task UploadOrphanedDemos( $"Found {files.Length} demo(s) on disk, attempting to recover uploads" ); - int timeLimit = _environmentService.GetDemoUploadTimeLimitSeconds(); + // One overall budget for the whole pass so a backlog can't run for + // N × timeLimit; whatever isn't reached is retried on the next start. + using var cts = System.Threading.CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken + ); + cts.CancelAfter(TimeSpan.FromSeconds(_environmentService.GetDemoUploadTimeLimitSeconds())); foreach (string file in files) { - cancellationToken.ThrowIfCancellationRequested(); - - // Skip a file that's still being written (an active recording); a - // finalized demo hasn't been touched since tv_stoprecord. - if ( + cts.Token.ThrowIfCancellationRequested(); + + // Skip anything still being recorded: an active recording holds a lock + // for its match, and a finalized demo hasn't been written since + // tv_stoprecord. The lock check covers the case where an active + // recording simply hasn't flushed within the freshness window. + string? matchId = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(file))); + bool activelyRecording = + Guid.TryParse(matchId, out Guid parsedMatchId) + && File.Exists(GetLockFilePath(parsedMatchId)); + bool recentlyWritten = File.GetLastWriteTimeUtc(file) - > DateTime.UtcNow.AddSeconds(-RecordingFinalizeWindowSeconds) - ) + > DateTime.UtcNow.AddSeconds(-RecordingFinalizeWindowSeconds); + + if (activelyRecording || recentlyWritten) { - _logger.LogInformation( - $"Skipping recently-written demo (may still be recording): {file}" - ); + _logger.LogInformation($"Skipping demo that may still be recording: {file}"); continue; } _logger.LogInformation($"Recovering orphaned demo {file}"); - using var cts = System.Threading.CancellationTokenSource.CreateLinkedTokenSource( - cancellationToken - ); - cts.CancelAfter(TimeSpan.FromSeconds(timeLimit)); await UploadDemo(file, cts.Token); } @@ -269,13 +292,12 @@ public async Task UploadDemo( return true; } - string? serverId = _environmentService.GetServerId(); string? apiPassword = _environmentService.GetServerApiPassword(); - if (serverId == null || apiPassword == null) + if (apiPassword == null) { - _logger.LogWarning("Cannot upload demo: server id / api password not configured"); - return true; + _logger.LogWarning("Cannot upload demo: server api password not configured"); + return false; } // Derive ids from the path ({root}/demos/{matchId}/{mapId}/{demo}.dem) @@ -296,8 +318,10 @@ public async Task UploadDemo( { if (!_uploadsInProgress.Add(filePath)) { + // Another path owns this upload; report not-done so the caller + // retries rather than treating it as a confirmed success. _logger.LogInformation($"Demo already being uploaded, skipping: {demoName}"); - return true; + return false; } } @@ -318,9 +342,8 @@ public async Task UploadDemo( if (string.IsNullOrEmpty(presignedUrl)) { - _logger.LogCritical( - $"Failed to get presigned URL (match {matchId} map {mapId} demo {demoName})" - ); + // GetPresignedUrl already logged the specific reason (e.g. 409 + // "not finished yet", which is a normal retry, not an error). return false; } From 356750ce06140af2dd2a3a8563df9dde8b0416be Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Wed, 1 Jul 2026 13:14:17 -0400 Subject: [PATCH 4/4] wip --- src/FiveStack.Services/GameDemos.cs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/FiveStack.Services/GameDemos.cs b/src/FiveStack.Services/GameDemos.cs index 66a4802..d4d1d40 100644 --- a/src/FiveStack.Services/GameDemos.cs +++ b/src/FiveStack.Services/GameDemos.cs @@ -70,12 +70,12 @@ public void Start() { MatchData? match = _matchService.GetCurrentMatch()?.GetMatchData(); - if (match == null) + if (match == null || match.current_match_map_id == null) { return; } - string lockFilePath = GetLockFilePath(match.id); + string lockFilePath = GetLockFilePath(match.current_match_map_id.Value); if (File.Exists(lockFilePath)) { _logger.LogInformation("Demo is already recording"); @@ -99,7 +99,7 @@ public void Stop() { MatchData? match = _matchService.GetCurrentMatch()?.GetMatchData(); - if (match == null) + if (match == null || match.current_match_map_id == null) { return; } @@ -109,7 +109,7 @@ public void Stop() ? Directory.GetFiles(demoPath, "*.dem").Length : 0; - File.Delete(GetLockFilePath(match.id)); + File.Delete(GetLockFilePath(match.current_match_map_id.Value)); Server.NextFrame(() => { _logger.LogInformation( @@ -256,13 +256,14 @@ public async Task UploadOrphanedDemos( cts.Token.ThrowIfCancellationRequested(); // Skip anything still being recorded: an active recording holds a lock - // for its match, and a finalized demo hasn't been written since + // for its map, and a finalized demo hasn't been written since // tv_stoprecord. The lock check covers the case where an active - // recording simply hasn't flushed within the freshness window. - string? matchId = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(file))); + // recording simply hasn't flushed within the freshness window. Keyed + // per-map so a finished map isn't skipped while a later map records. + string? mapId = Path.GetFileName(Path.GetDirectoryName(file)); bool activelyRecording = - Guid.TryParse(matchId, out Guid parsedMatchId) - && File.Exists(GetLockFilePath(parsedMatchId)); + Guid.TryParse(mapId, out Guid parsedMapId) + && File.Exists(GetLockFilePath(parsedMapId)); bool recentlyWritten = File.GetLastWriteTimeUtc(file) > DateTime.UtcNow.AddSeconds(-RecordingFinalizeWindowSeconds); @@ -501,8 +502,8 @@ private string GetMatchDemoPath() return $"{_rootDir}/demos/{match.id}/{match.current_match_map_id}"; } - private string GetLockFilePath(Guid matchId) + private string GetLockFilePath(Guid mapId) { - return $"{_rootDir}/.recording-demo-{matchId}"; + return $"{_rootDir}/.recording-demo-{mapId}"; } }