Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/FiveStack.Commands/Demo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@ public async void OnUploadDemo(CCSPlayerController? player, CommandInfo command)

match.UpdateMapStatus(eMapStatus.UploadingDemo);

await _gameDemos.UploadDemos();
using var cts = new System.Threading.CancellationTokenSource(
TimeSpan.FromSeconds(_environmentService.GetDemoUploadTimeLimitSeconds())
);
await _gameDemos.UploadDemos(cts.Token);
}

[ConsoleCommand("test_start_demo", "start demo recording")]
Expand Down
75 changes: 63 additions & 12 deletions src/FiveStack.Events/GameEnd.cs
Original file line number Diff line number Diff line change
Expand Up @@ -197,19 +197,69 @@ private void HandleEndOfMap(Guid? winningLineupId)
currentMap.id
);

try
// Only finish the map once the demo is confirmed uploaded, so
// the server stays reserved until it's safe. If the budget
// runs out, give up loudly and finish anyway (retried on next
// startup) so the match never gets stuck.
int uploadTimeLimit =
_environmentService.GetDemoUploadTimeLimitSeconds();
using var uploadCts = new System.Threading.CancellationTokenSource(
TimeSpan.FromSeconds(uploadTimeLimit)
);

// Upload the specific map that just ended, not whatever the
// current match/map resolves to when the timer fires.
string uploadMatchId = expectedMatchId.ToString();
string uploadMapId = currentMap.id.ToString();

bool uploaded = false;
int attempt = 0;
while (!uploadCts.IsCancellationRequested)
{
await _gameDemos.UploadDemos();
_logger.LogInformation(
"Demo upload finished (match={MatchId})",
expectedMatchId
);
attempt++;
try
{
uploaded = await _gameDemos.UploadDemos(
uploadMatchId,
uploadMapId,
uploadCts.Token
);
if (uploaded)
{
_logger.LogInformation(
"Demo upload finished (match={MatchId} attempts={Attempts})",
expectedMatchId,
attempt
);
break;
}
}
catch (Exception ex)
{
_logger.LogError(
ex,
"UploadDemos failed (match={MatchId} attempt={Attempt})",
expectedMatchId,
attempt
);
}
Comment on lines +237 to +245

try
{
await Task.Delay(TimeSpan.FromSeconds(10), uploadCts.Token);
}
catch (OperationCanceledException)
{
break;
}
}
catch (Exception ex)

if (!uploaded)
{
_logger.LogError(
ex,
"UploadDemos failed after map end for match {MatchId}",
_logger.LogCritical(
"Demo upload did NOT complete within {Limit}s ({Attempts} attempt(s)) — this server may be too slow to upload demos. Finishing the map without a confirmed upload (match={MatchId}); the demo remains on disk and will be retried on next startup.",
uploadTimeLimit,
attempt,
expectedMatchId
);
}
Expand All @@ -232,8 +282,9 @@ private void HandleEndOfMap(Guid? winningLineupId)
bool isSurrenderedNow = wasSurrendered || next.isSurrendered();

_logger.LogInformation(
"Demo upload done — finishing map and switching (match={MatchId})",
expectedMatchId
"Finishing map and switching (match={MatchId} demoUploaded={Uploaded})",
expectedMatchId,
uploaded
);

if (isSurrenderedNow)
Expand Down
12 changes: 12 additions & 0 deletions src/FiveStack.Services/EnvironmentService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ public bool AllowBots()
return Environment.GetEnvironmentVariable("ALLOW_BOTS") == "true";
}

// Total budget for uploading a map's demos after it ends, across retries.
public int GetDemoUploadTimeLimitSeconds()
{
var raw = Environment.GetEnvironmentVariable("DEMO_UPLOAD_TIME_LIMIT_SECONDS");
if (int.TryParse(raw, out int seconds) && seconds > 0)
{
return seconds;
}

return 300;
}

public bool isOnGameServerNode()
{
return Environment.GetEnvironmentVariable("GAME_NODE_SERVER") == "true";
Expand Down
Loading