diff --git a/Core/Resgrid.Config/SystemBehaviorConfig.cs b/Core/Resgrid.Config/SystemBehaviorConfig.cs index cca308e30..a886eb1aa 100644 --- a/Core/Resgrid.Config/SystemBehaviorConfig.cs +++ b/Core/Resgrid.Config/SystemBehaviorConfig.cs @@ -112,6 +112,14 @@ public static class SystemBehaviorConfig /// public static string ApiTokenEncryptionPassphrase = ""; + /// + /// Shared key trusted monitoring must present (X-Resgrid-Health-Key header) to call + /// the deep /health/full endpoint, which runs real SQL/Redis/TTS probes and returns + /// dependency detail. Empty (the default) disables /health/full entirely; the + /// shallow /health liveness endpoint is unaffected. + /// + public static string FullHealthCheckKey = ""; + /// /// The length the API token will be valid for once a user logs into the app /// diff --git a/Core/Resgrid.Config/TtsConfig.cs b/Core/Resgrid.Config/TtsConfig.cs index 61b436326..535a0871e 100644 --- a/Core/Resgrid.Config/TtsConfig.cs +++ b/Core/Resgrid.Config/TtsConfig.cs @@ -24,6 +24,14 @@ public static class TtsConfig public static int S3PresignedUrlExpiryMinutes = 60; public static string S3PublicBaseUrl = ""; + /// + /// When true, voice webhooks degrade to Twilio's native <Say> verb (billed per + /// use by Twilio) if TTS audio can't be produced in time or generation fails. + /// Default off: an unavailable prompt is skipped so a TTS outage surfaces as + /// missing audio in the call — and in the logs — rather than as extra spend. + /// + public static bool TwilioSayFallbackEnabled = false; + public static string DefaultVoice = "en-us+klatt4"; public static int DefaultSpeed = 150; public static int MaxConcurrentGenerations = 4; @@ -70,7 +78,8 @@ public static class TtsConfig "No status selection made. Returning to the main menu.", "Invalid staffing selection. Returning to the main menu.", "No staffing selection made. Returning to the main menu.", - "Thank you. Your response has been recorded." + "Thank you. Your response has been recorded.", + "Please wait while we prepare your dispatch information." }); public static int RateLimitPermitLimit = 600; diff --git a/Core/Resgrid.Services/DepartmentsService.cs b/Core/Resgrid.Services/DepartmentsService.cs index c87cdcdff..a3cd49bf6 100644 --- a/Core/Resgrid.Services/DepartmentsService.cs +++ b/Core/Resgrid.Services/DepartmentsService.cs @@ -211,6 +211,9 @@ public void InvalidateDepartmentUserInCache(string userId, IdentityUser user = n await _departmentSettingsService.SaveOrUpdateSettingAsync(d.DepartmentId, dispatchCode, DepartmentSettingTypes.InternalDispatchEmail, cancellationToken); } + // New departments get modern notification sounds enabled by default + await _departmentSettingsService.SaveOrUpdateSettingAsync(d.DepartmentId, true.ToString(), DepartmentSettingTypes.EnableModernNotifications, cancellationToken); + return d; } diff --git a/Providers/Resgrid.Providers.Number/OutboundVoiceProvider.cs b/Providers/Resgrid.Providers.Number/OutboundVoiceProvider.cs index bad7b6197..b82e2f43c 100644 --- a/Providers/Resgrid.Providers.Number/OutboundVoiceProvider.cs +++ b/Providers/Resgrid.Providers.Number/OutboundVoiceProvider.cs @@ -37,8 +37,12 @@ public async Task CommunicateCallAsync(string phoneNumber, UserProfile pro var options = new CreateCallOptions(new PhoneNumber(profile.GetPhoneNumber()), new PhoneNumber(number)); options.Url = new Uri(string.Format(Config.NumberProviderConfig.TwilioVoiceCallApiUrl, profile.UserId, call.CallId)); options.Method = "GET"; - options.MachineDetection = "Enable"; - //options.IfMachine = "Continue"; + // No machine detection: nothing consumes AnsweredBy (no AMD status + // callback endpoint exists, IfMachine handling was never enabled), so + // AMD would only add Twilio's per-call detection fee — and, in its + // synchronous form, a multi-second answer delay. The webhook fires the + // moment the call is answered and dispatch playback starts immediately; + // a voicemail simply records the dispatch prompt. var phoneCall = await CallResource.CreateAsync(options); return true; @@ -52,8 +56,7 @@ public async Task CommunicateCallAsync(string phoneNumber, UserProfile pro var options = new CreateCallOptions(new PhoneNumber(profile.GetHomePhoneNumber()), new PhoneNumber(number)); options.Url = new Uri(string.Format(Config.NumberProviderConfig.TwilioVoiceCallApiUrl, profile.UserId, call.CallId)); options.Method = "GET"; - options.MachineDetection = "Enable"; - //options.IfMachine = "Continue"; + // No machine detection — see the mobile branch above. var phoneCall = await CallResource.CreateAsync(options); return true; diff --git a/Tests/Resgrid.Tests/Web/Services/HealthChecksTests.cs b/Tests/Resgrid.Tests/Web/Services/HealthChecksTests.cs new file mode 100644 index 000000000..322dbff58 --- /dev/null +++ b/Tests/Resgrid.Tests/Web/Services/HealthChecksTests.cs @@ -0,0 +1,356 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; +using Resgrid.Model.Providers; +using Resgrid.Web.Services.Health; +using Resgrid.Web.Tts.Configuration; +using Resgrid.Web.Tts.Health; +using Resgrid.Web.Tts.Services; + +namespace Resgrid.Tests.Web.Services +{ + [TestFixture] + public class TtsFullHealthCheckTests + { + private static TtsFullHealthCheck CreateCheck( + Mock distributedCache, + Mock storageService, + Mock audioProcessingService) + { + return new TtsFullHealthCheck( + distributedCache.Object, + storageService.Object, + audioProcessingService.Object, + Options.Create(new TtsOptions())); + } + + private static (Mock Cache, Mock Storage, Mock Audio) CreateHealthyMocks() + { + var distributedCache = new Mock(); + distributedCache + .Setup(x => x.GetAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new byte[] { 1 }); + + var storageService = new Mock(); + storageService + .Setup(x => x.ExistsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(false); + + var audioProcessingService = new Mock(); + audioProcessingService + .Setup(x => x.GenerateNormalizedWavAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new byte[] { 82, 73, 70, 70 }); + + return (distributedCache, storageService, audioProcessingService); + } + + [Test] + public async Task check_health_should_report_healthy_when_all_probes_pass() + { + var (cache, storage, audio) = CreateHealthyMocks(); + var check = CreateCheck(cache, storage, audio); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Healthy); + result.Data.Keys.Should().Contain(new[] { "redis", "s3", "synthesis" }); + } + + [Test] + public async Task check_health_should_report_unhealthy_when_synthesis_fails() + { + var (cache, storage, audio) = CreateHealthyMocks(); + audio + .Setup(x => x.GenerateNormalizedWavAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Piper exited with code 1.")); + + var check = CreateCheck(cache, storage, audio); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Description.Should().Contain("synthesis"); + } + + [Test] + public async Task check_health_should_report_unhealthy_when_redis_round_trip_returns_nothing() + { + var (cache, storage, audio) = CreateHealthyMocks(); + cache + .Setup(x => x.GetAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[])null); + + var check = CreateCheck(cache, storage, audio); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Description.Should().Contain("redis"); + } + + [Test] + public async Task check_health_should_memoize_the_result_between_close_polls() + { + var (cache, storage, audio) = CreateHealthyMocks(); + var check = CreateCheck(cache, storage, audio); + + await check.CheckHealthAsync(new HealthCheckContext()); + await check.CheckHealthAsync(new HealthCheckContext()); + + audio.Verify( + x => x.GenerateNormalizedWavAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } + } + + [TestFixture] + public class RedisHealthCheckTests + { + [Test] + public async Task check_health_should_report_healthy_when_round_trip_succeeds() + { + var cacheProvider = new Mock(); + cacheProvider.Setup(x => x.SetStringAsync(It.IsAny(), "ok", It.IsAny())).ReturnsAsync(true); + cacheProvider.Setup(x => x.GetStringAsync(It.IsAny())).ReturnsAsync("ok"); + + var check = new RedisHealthCheck(cacheProvider.Object); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Healthy); + } + + [Test] + public async Task check_health_should_report_unhealthy_when_write_fails() + { + var cacheProvider = new Mock(); + cacheProvider.Setup(x => x.SetStringAsync(It.IsAny(), "ok", It.IsAny())).ReturnsAsync(false); + + var check = new RedisHealthCheck(cacheProvider.Object); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Unhealthy); + } + } + + [TestFixture] + public class FullHealthCheckAccessTests + { + private string _originalKey; + + [SetUp] + public void SetUp() + { + _originalKey = Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey; + } + + [TearDown] + public void TearDown() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = _originalKey; + } + + private static HttpRequest CreateRequest(string headerValue = null) + { + var context = new DefaultHttpContext(); + + if (headerValue != null) + context.Request.Headers[FullHealthCheckAccess.HeaderName] = headerValue; + + return context.Request; + } + + [Test] + public void is_authorized_should_fail_closed_when_no_key_is_configured() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = ""; + + FullHealthCheckAccess.IsAuthorized(CreateRequest("anything")).Should().BeFalse(); + } + + [Test] + public void is_authorized_should_reject_missing_header() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = "monitor-key"; + + FullHealthCheckAccess.IsAuthorized(CreateRequest()).Should().BeFalse(); + } + + [Test] + public void is_authorized_should_reject_wrong_key() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = "monitor-key"; + + FullHealthCheckAccess.IsAuthorized(CreateRequest("wrong-key")).Should().BeFalse(); + } + + [Test] + public void is_authorized_should_accept_matching_key() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = "monitor-key"; + + FullHealthCheckAccess.IsAuthorized(CreateRequest("monitor-key")).Should().BeTrue(); + } + } + + [TestFixture] + public class TtsHealthCheckAccessTests + { + private string _originalKey; + + [SetUp] + public void SetUp() + { + _originalKey = Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey; + } + + [TearDown] + public void TearDown() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = _originalKey; + } + + private static HttpRequest CreateRequest(string headerValue = null) + { + var context = new DefaultHttpContext(); + + if (headerValue != null) + context.Request.Headers[TtsHealthCheckAccess.HeaderName] = headerValue; + + return context.Request; + } + + [Test] + public void is_authorized_should_fail_closed_when_no_key_is_configured() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = ""; + + TtsHealthCheckAccess.IsAuthorized(CreateRequest("anything")).Should().BeFalse(); + } + + [Test] + public void is_authorized_should_reject_missing_or_wrong_key() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = "monitor-key"; + + TtsHealthCheckAccess.IsAuthorized(CreateRequest()).Should().BeFalse(); + TtsHealthCheckAccess.IsAuthorized(CreateRequest("wrong-key")).Should().BeFalse(); + } + + [Test] + public void is_authorized_should_accept_matching_key() + { + Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey = "monitor-key"; + + TtsHealthCheckAccess.IsAuthorized(CreateRequest("monitor-key")).Should().BeTrue(); + } + + [Test] + public void header_name_should_match_the_api_gate_so_one_monitoring_key_works_for_both() + { + TtsHealthCheckAccess.HeaderName.Should().Be(FullHealthCheckAccess.HeaderName); + } + } + + [TestFixture] + public class TtsServiceHealthCheckTests + { + private string _originalServiceBaseUrl; + + [SetUp] + public void SetUp() + { + _originalServiceBaseUrl = Resgrid.Config.TtsConfig.ServiceBaseUrl; + } + + [TearDown] + public void TearDown() + { + Resgrid.Config.TtsConfig.ServiceBaseUrl = _originalServiceBaseUrl; + } + + [Test] + public async Task check_health_should_report_degraded_when_tts_is_not_configured() + { + Resgrid.Config.TtsConfig.ServiceBaseUrl = ""; + + var check = new TtsServiceHealthCheck(new StubHttpClientFactory(_ => throw new InvalidOperationException("Should not be called."))); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Degraded); + } + + [Test] + public async Task check_health_should_report_healthy_when_tts_health_endpoint_returns_success() + { + Resgrid.Config.TtsConfig.ServiceBaseUrl = "https://tts.example.com"; + + string requestedUrl = null; + var check = new TtsServiceHealthCheck(new StubHttpClientFactory(request => + { + requestedUrl = request.RequestUri.ToString(); + return new HttpResponseMessage(HttpStatusCode.OK); + })); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Healthy); + requestedUrl.Should().Be("https://tts.example.com/health"); + } + + [Test] + public async Task check_health_should_report_unhealthy_when_tts_health_endpoint_errors() + { + Resgrid.Config.TtsConfig.ServiceBaseUrl = "https://tts.example.com"; + + var check = new TtsServiceHealthCheck(new StubHttpClientFactory(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError))); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Unhealthy); + result.Description.Should().Contain("500"); + } + + private sealed class StubHttpClientFactory : IHttpClientFactory + { + private readonly Func _responder; + + public StubHttpClientFactory(Func responder) + { + _responder = responder; + } + + public HttpClient CreateClient(string name) + { + return new HttpClient(new StubHandler(_responder)); + } + + private sealed class StubHandler : HttpMessageHandler + { + private readonly Func _responder; + + public StubHandler(Func responder) + { + _responder = responder; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(_responder(request)); + } + } + } + } +} diff --git a/Tests/Resgrid.Tests/Web/Services/TwilioControllerVoiceVerificationTests.cs b/Tests/Resgrid.Tests/Web/Services/TwilioControllerVoiceVerificationTests.cs index 9be4c29bd..951af8ad8 100644 --- a/Tests/Resgrid.Tests/Web/Services/TwilioControllerVoiceVerificationTests.cs +++ b/Tests/Resgrid.Tests/Web/Services/TwilioControllerVoiceVerificationTests.cs @@ -288,7 +288,7 @@ public async System.Threading.Tasks.Task should_play_dispatch_before_outbound_re var result = await BuildController().VoiceCall("user1", 42); var content = ((ContentResult)result).Content; - var dispatchPrompt = Uri.EscapeDataString("Call 42, Priority High Address 123 Main St Nature Structure fire."); + var dispatchPrompt = Uri.EscapeDataString("Call 42, Priority High. Address 123 Main St. Nature Structure fire."); var menuPrompt = Uri.EscapeDataString(TwilioVoicePromptCatalog.OutboundDispatchMenu); content.Should().Contain(dispatchPrompt); @@ -308,9 +308,9 @@ public void dispatch_prompt_helpers_should_end_with_sentence_punctuation() }; InvokeBuildDispatchPrompt(typeof(TwilioController), call, "123 Main St") - .Should().Be("Call 42, Priority High Address 123 Main St Nature Structure fire."); + .Should().Be("Call 42, Priority High. Address 123 Main St. Nature Structure fire."); InvokeBuildDispatchPrompt(typeof(TwilioController), call, null) - .Should().Be("Call 42, Priority High Nature Structure fire."); + .Should().Be("Call 42, Priority High. Nature Structure fire."); } [TestCase("1", "https://resgridapi.local/api/Twilio/VoiceCall?userId=user1&callId=42")] diff --git a/Tests/Resgrid.Tests/Web/Services/TwilioVoiceResponseServiceTests.cs b/Tests/Resgrid.Tests/Web/Services/TwilioVoiceResponseServiceTests.cs index 19ffc662c..1196ed90d 100644 --- a/Tests/Resgrid.Tests/Web/Services/TwilioVoiceResponseServiceTests.cs +++ b/Tests/Resgrid.Tests/Web/Services/TwilioVoiceResponseServiceTests.cs @@ -161,6 +161,134 @@ public async Task pre_warm_prompt_async_should_not_throw_for_multi_chunk_text() } } + [Test] + public async Task append_prompt_async_should_fall_back_to_say_when_tts_generation_fails_and_fallback_enabled() + { + // A TTS microservice outage previously bubbled InvalidOperationException out of the + // voice webhooks, returning a 500 that Twilio reads to callers as "an application + // error has occurred". With the paid fallback enabled the service must degrade to + // a native verb instead. + var originalFallbackEnabled = Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled; + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled = true; + try + { + var ttsAudioService = new Mock(); + ttsAudioService + .Setup(x => x.GenerateSpeechUrlAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("The TTS service failed to generate speech audio.")); + + var service = new TwilioVoiceResponseService(ttsAudioService.Object); + var response = new VoiceResponse(); + + await service.AppendPromptAsync(response, "Hello from Resgrid", CancellationToken.None); + + var xml = response.ToString(); + xml.Should().Contain("Hello from Resgrid"); + xml.Should().NotContain(""); + } + finally + { + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled = originalFallbackEnabled; + } + } + + [Test] + public async Task append_prompt_async_should_fall_back_to_say_within_a_gather_when_tts_generation_fails_and_fallback_enabled() + { + var originalFallbackEnabled = Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled; + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled = true; + try + { + var ttsAudioService = new Mock(); + ttsAudioService + .Setup(x => x.GenerateSpeechUrlAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("The TTS service failed to generate speech audio.")); + + var service = new TwilioVoiceResponseService(ttsAudioService.Object); + var gather = new global::Twilio.TwiML.Voice.Gather(numDigits: 1); + + await service.AppendPromptAsync(gather, "Press 1 to respond", CancellationToken.None); + + gather.ToString().Should().Contain("Press 1 to respond"); + } + finally + { + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled = originalFallbackEnabled; + } + } + + [Test] + public async Task append_prompt_async_should_skip_prompt_when_tts_generation_fails_and_fallback_disabled() + { + // Default posture: the paid fallback is off, so a failed prompt is skipped + // (no , no ) and no exception escapes the webhook. + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled.Should().BeFalse( + "the Say fallback must default to disabled so TTS failures don't incur Twilio charges"); + + var ttsAudioService = new Mock(); + ttsAudioService + .Setup(x => x.GenerateSpeechUrlAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("The TTS service failed to generate speech audio.")); + + var service = new TwilioVoiceResponseService(ttsAudioService.Object); + var response = new VoiceResponse(); + + await service.AppendPromptAsync(response, "Hello from Resgrid", CancellationToken.None); + + var xml = response.ToString(); + xml.Should().NotContain(""); + xml.Should().NotContain(""); + } + + [Test] + public void append_say_fallback_should_be_a_no_op_when_disabled_and_emit_say_when_enabled() + { + var originalFallbackEnabled = Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled; + try + { + var service = new TwilioVoiceResponseService(Mock.Of()); + + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled = false; + var disabledResponse = new VoiceResponse(); + service.AppendSayFallback(disabledResponse, "Hello from Resgrid"); + disabledResponse.ToString().Should().NotContain(""); + + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled = true; + var enabledResponse = new VoiceResponse(); + service.AppendSayFallback(enabledResponse, "Hello from Resgrid"); + enabledResponse.ToString().Should().Contain("Hello from Resgrid"); + } + finally + { + Resgrid.Config.TtsConfig.TwilioSayFallbackEnabled = originalFallbackEnabled; + } + } + + [Test] + public async Task append_prompt_async_should_still_throw_when_request_is_cancelled_even_if_generation_faults() + { + // Caller-driven cancellation is control flow (the dispatch playback timeout uses it + // to fall back to the pre-warm/redirect loop) and must not be swallowed by the + // degradation path. + var ttsAudioService = new Mock(); + ttsAudioService + .Setup(x => x.GenerateSpeechUrlAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (_, _, _, _) => + { + await Task.Delay(Timeout.Infinite); + return new Uri("https://tts.example.com/tts/audio/never.wav"); + }); + + var service = new TwilioVoiceResponseService(ttsAudioService.Object); + var response = new VoiceResponse(); + using var requestCancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + + await FluentActions + .Awaiting(() => service.AppendPromptAsync(response, "Hello from Resgrid", requestCancellation.Token)) + .Should() + .ThrowAsync(); + } + [Test] public async Task append_prompt_async_should_emit_a_play_per_chunk_for_multi_chunk_text() { diff --git a/Tests/Resgrid.Tests/Web/Tts/TextPreprocessorTests.cs b/Tests/Resgrid.Tests/Web/Tts/TextPreprocessorTests.cs new file mode 100644 index 000000000..a62b7856c --- /dev/null +++ b/Tests/Resgrid.Tests/Web/Tts/TextPreprocessorTests.cs @@ -0,0 +1,126 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using Resgrid.Web.Tts.Services; + +namespace Resgrid.Tests.Web.Tts +{ + [TestFixture] + public class TextPreprocessorTests + { + private TextPreprocessor _preprocessor; + + private const string EnglishVoice = "en-us+klatt4"; + + [SetUp] + public void SetUp() + { + _preprocessor = new TextPreprocessor(NullLogger.Instance); + } + + // ----------------------------------------------------------- + // Address abbreviation expansion must preserve the house + // number and street name — only the suffix is rewritten. + // ----------------------------------------------------------- + + [TestCase("123 Main St", "123 Main Street.")] + [TestCase("456 Oak Ave", "456 Oak Avenue.")] + [TestCase("789 Sunset Blvd", "789 Sunset Boulevard.")] + public void Preprocess_ExpandsAddressSuffix_WithoutDroppingHouseNumberOrStreetName(string input, string expected) + { + _preprocessor.Preprocess(input, EnglishVoice).Should().Be(expected); + } + + [Test] + public void Preprocess_ExpandsMultipleAddressSuffixesInOneAddress() + { + _preprocessor.Preprocess("100 Main St Apt 4", EnglishVoice) + .Should().Be("100 Main Street Apartment 4."); + } + + [Test] + public void Preprocess_DoesNotExpandAddressSuffixWithoutLeadingNumber() + { + // "St" with no house number ahead of it is left alone. + _preprocessor.Preprocess("Meet at Main St.", EnglishVoice) + .Should().Be("Meet at Main St."); + } + + // ----------------------------------------------------------- + // Abbreviation expansion is case-sensitive so ordinary words + // are never rewritten into dispatch jargon. + // ----------------------------------------------------------- + + [TestCase("Please do so now", "Please do so now.")] + [TestCase("please pass the tools", "please pass the tools.")] + [TestCase("an apt description", "an apt description.")] + [TestCase("the co responder", "the co responder.")] + public void Preprocess_LeavesOrdinaryEnglishWordsAlone(string input, string expected) + { + _preprocessor.Preprocess(input, EnglishVoice).Should().Be(expected); + } + + [TestCase("MVC with ENTRP", "Motor Vehicle Collision with Entrapment.")] + [TestCase("SO on scene", "Sheriff's Office on scene.")] + [TestCase("RP ADV 2 VEH MVC", "Reporting Party Advised two Vehicle Motor Vehicle Collision.")] + [TestCase("HAZMAT spill", "Hazardous Materials spill.")] + [TestCase("HazMat spill", "Hazardous Materials spill.")] + [TestCase("debris etc on roadway", "debris et cetera on roadway.")] + [TestCase("debris ETC on roadway", "debris et cetera on roadway.")] + [TestCase("debris Etc on roadway", "debris et cetera on roadway.")] + public void Preprocess_ExpandsUppercaseDispatchCodes(string input, string expected) + { + _preprocessor.Preprocess(input, EnglishVoice).Should().Be(expected); + } + + // ----------------------------------------------------------- + // Number handling for spoken clarity. + // ----------------------------------------------------------- + + [Test] + public void Preprocess_ReadsLongNumbersDigitByDigit() + { + _preprocessor.Preprocess("12345 Elm Rd", EnglishVoice) + .Should().Be("1 2 3 4 5 Elm Road."); + } + + [Test] + public void Preprocess_SpeaksSmallCountsAsWords() + { + _preprocessor.Preprocess("2 patients trapped", EnglishVoice) + .Should().Be("two patients trapped."); + } + + [Test] + public void Preprocess_SplitsUnitIdentifiers() + { + _preprocessor.Preprocess("E1 and L14 responding", EnglishVoice) + .Should().Be("E one and L fourteen responding."); + } + + // ----------------------------------------------------------- + // Slash notation and sentence termination. + // ----------------------------------------------------------- + + [Test] + public void Preprocess_ExpandsSlashNotation() + { + _preprocessor.Preprocess("75 Y/O male", EnglishVoice) + .Should().Be("75 Year Old male."); + } + + [Test] + public void Preprocess_AppendsTerminalPunctuation() + { + _preprocessor.Preprocess("Structure fire reported", EnglishVoice) + .Should().EndWith("."); + } + + [Test] + public void Preprocess_NonEnglishVoicePassesThroughUntouched() + { + _preprocessor.Preprocess("123 Main St", "es") + .Should().Be("123 Main St."); + } + } +} diff --git a/Web/Resgrid.Web.Services/Controllers/TwilioController.cs b/Web/Resgrid.Web.Services/Controllers/TwilioController.cs index d5e1d05c4..0107be771 100644 --- a/Web/Resgrid.Web.Services/Controllers/TwilioController.cs +++ b/Web/Resgrid.Web.Services/Controllers/TwilioController.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Globalization; using System.Linq; using System.Net.Http; using System.Security.Cryptography; @@ -93,6 +94,34 @@ public TwilioController(IDepartmentSettingsService departmentSettingsService, IN private const int MAX_DISPATCH_RETRY = 3; + // Shared time budget for ALL TTS prompt playback within a single webhook request. + // A TTS service that is down fails fast and degrades to inside + // TwilioVoiceResponseService, but a HUNG service eats the full RestClient + // timeout per attempt (with retries, ~16s for one prompt) — enough to blow + // Twilio's 15-second webhook limit before any fallback fires. The budget + // starts on the first prompt append; once it expires every remaining prompt + // in the request falls back to Twilio's native voice immediately. + private static readonly TimeSpan TtsPromptBudget = TimeSpan.FromSeconds(6); + private CancellationTokenSource _ttsBudgetCts; + + private CancellationToken GetTtsPromptBudgetToken() + { + if (_ttsBudgetCts == null) + { + _ttsBudgetCts = CancellationTokenSource.CreateLinkedTokenSource(HttpContext?.RequestAborted ?? CancellationToken.None); + _ttsBudgetCts.CancelAfter(TtsPromptBudget); + HttpContext?.Response.RegisterForDispose(_ttsBudgetCts); + } + + return _ttsBudgetCts.Token; + } + + // True when the TTS budget expired on its own (the caller hasn't hung up), + // which is the only cancellation we convert into a fallback. + private bool TtsBudgetExpired => + _ttsBudgetCts != null && _ttsBudgetCts.IsCancellationRequested + && !(HttpContext?.RequestAborted.IsCancellationRequested ?? false); + [HttpGet("IncomingMessage")] [Produces("application/xml")] public async Task IncomingMessage([FromQuery] TwilioMessage request) @@ -562,9 +591,18 @@ private async System.Threading.Tasks.Task ProcessTextCommandsAsync(TextMessage t { string[] points = call.GeoLocationData.Split(char.Parse(",")); - if (points != null && points.Length == 2) + // The task must be awaited — appending it directly stringifies as + // "System.Threading.Tasks.Task`1[System.String]" in the SMS reply. + // Bounded like the voice path so a slow geocoder can't stall the webhook. + // TryParse with InvariantCulture: malformed coordinates skip the lookup + // instead of throwing, and a comma-decimal server culture can't silently + // misread "47.606" as 47606. + if (points != null && points.Length == 2 + && double.TryParse(points[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var latitude) + && double.TryParse(points[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var longitude)) { - callText.Append(_geoLocationProvider.GetAproxAddressFromLatLong(double.Parse(points[0]), double.Parse(points[1])) + Environment.NewLine); + callText.Append(await _geoLocationProvider.GetAproxAddressFromLatLong(latitude, longitude) + .WaitAsync(TimeSpan.FromSeconds(2), HttpContext?.RequestAborted ?? CancellationToken.None) + Environment.NewLine); } } catch @@ -700,7 +738,7 @@ public async Task VoiceCall(string userId, int callId, [FromQuery] // not yet cached, we play a brief "please wait" prompt and redirect // back to this endpoint, giving the TTS service time to complete // generation in the background. - var dispatchReady = await TryAppendDispatchPlaybackAsync(response, call); + var (dispatchReady, dispatchText) = await TryAppendDispatchPlaybackAsync(response, call); if (!dispatchReady) { // Parse and increment the retry counter from the incoming request. @@ -709,37 +747,43 @@ public async Task VoiceCall(string userId, int callId, [FromQuery] if (retryCount >= MAX_DISPATCH_RETRY) { - // Exceeded retry cap — fall back to a static error prompt. - await AppendVoicePromptAsync(response, TwilioVoicePromptCatalog.CallClosed); - response.Hangup(); + // The dispatch audio never became ready within the retry budget. If the paid + // fallback is enabled, speak the dispatch with Twilio's native voice; + // otherwise this is a no-op and the caller goes straight to the menu, where + // "press 1" re-enters VoiceCall with a fresh retry budget. Either way do NOT + // play the "call closed" prompt (a new call is not closed) and hang up. + _twilioVoiceResponseService.AppendSayFallback(response, dispatchText); + } + else + { + // Dispatch audio isn't ready yet. Pre-warm it in the background + // and redirect back — by the time Twilio re-fetches this endpoint + // the audio should be cached. + var ttsLanguage = await GetDepartmentTtsLanguageAsync(call.DepartmentId); + + // Fire off TTS generation in the background. The TTS microservice + // caches the result, so the redirect will find it once ready. + _twilioVoiceResponseService.PreWarmPromptAsync(dispatchText, ttsLanguage) + .ContinueWith(t => + { + if (t.IsFaulted && t.Exception != null) + Logging.LogException(t.Exception); + }, TaskContinuationOptions.OnlyOnFaulted); + + // We're only in this loop because TTS is already slow — the shared + // per-request budget bounds the "please wait" prompt (falling back to + // ) so a hung TTS service can't stall this webhook past Twilio's + // 15-second limit. + await AppendVoicePromptAsync(response, TwilioVoicePromptCatalog.PleaseWaitForDispatch, call.DepartmentId); + var nextRetry = retryCount + 1; + response.Redirect( + new Uri($"{Config.SystemBehaviorConfig.ResgridApiBaseUrl}/api/Twilio/VoiceCall?userId={userId}&callId={callId}&retry={nextRetry}"), + "GET"); return CreateVoiceContentResult(response); } - - // Dispatch audio isn't ready yet. Pre-warm it in the background - // and redirect back — by the time Twilio re-fetches this endpoint - // the audio should be cached. - var address = await ResolveCallAddressAsync(call); - var dispatchText = BuildDispatchPrompt(call, address); - var ttsLanguage = await GetDepartmentTtsLanguageAsync(call.DepartmentId); - - // Fire off TTS generation in the background. The TTS microservice - // caches the result, so the redirect will find it once ready. - _twilioVoiceResponseService.PreWarmPromptAsync(dispatchText, ttsLanguage) - .ContinueWith(t => - { - if (t.IsFaulted && t.Exception != null) - Logging.LogException(t.Exception); - }, TaskContinuationOptions.OnlyOnFaulted); - - await AppendVoicePromptAsync(response, TwilioVoicePromptCatalog.PleaseWaitForDispatch); - var nextRetry = retryCount + 1; - response.Redirect( - new Uri($"{Config.SystemBehaviorConfig.ResgridApiBaseUrl}/api/Twilio/VoiceCall?userId={userId}&callId={callId}&retry={nextRetry}"), - "GET"); - return CreateVoiceContentResult(response); } - // Dispatch is ready (fast path or retry with cached audio). + // Dispatch is ready (fast path, retry with cached audio, or native-voice fallback). var gather = new Gather(numDigits: 1, action: new Uri($"{Config.SystemBehaviorConfig.ResgridApiBaseUrl}/api/Twilio/VoiceCallAction?userId={userId}&callId={callId}"), method: "GET") { BargeIn = true @@ -1062,12 +1106,21 @@ public async Task InboundVoiceAction(string userId, [FromQuery] Vo if (calls != null && calls.Any()) { - prompts.Add($"There are {calls.Count()} active calls for department {department.Name}."); + // Join the listing into ONE prompt: the TTS layer chunks long text on + // sentence boundaries and generates the chunks in parallel, whereas one + // prompt per call would issue that many sequential TTS round-trips and + // risk running past Twilio's 15-second webhook limit on big lists. + var lines = new List + { + $"There are {calls.Count()} active calls for department {department.Name}." + }; foreach (var call in calls) { - prompts.Add($"{call.Name}, Priority {call.GetPriorityText()} Address {call.Address} Nature {StringHelpers.StripHtmlTagsCharArray(call.NatureOfCall)}."); + lines.Add($"{call.Name}, Priority {call.GetPriorityText()}. Address {call.Address}. Nature {StringHelpers.StripHtmlTagsCharArray(call.NatureOfCall)}."); } + + prompts.Add(string.Join(" ", lines)); } else { @@ -1082,6 +1135,11 @@ public async Task InboundVoiceAction(string userId, [FromQuery] Vo if (allUsers != null && allUsers.Any()) { + // One joined prompt (see the active-calls branch above): chunked and + // generated in parallel by the TTS layer instead of one sequential + // round-trip per person. + var lines = new List(); + foreach (var user in allUsers) { var lastActionLog = lastUserActionlogs.FirstOrDefault(x => x.UserId == user.UserId); @@ -1089,8 +1147,12 @@ public async Task InboundVoiceAction(string userId, [FromQuery] Vo var staffingLevel = await _customStateService.GetCustomPersonnelStaffingAsync(department.DepartmentId, userState); var status = await _customStateService.GetCustomPersonnelStatusAsync(department.DepartmentId, lastActionLog); - prompts.Add($"{user.LastName}, {user.FirstName}, Status {status.ButtonText} Staffing Level {staffingLevel.ButtonText}."); + // A user with no recorded action log or state resolves to a null detail; + // dereferencing it here 500s the webhook and Twilio speaks "application error". + lines.Add($"{user.LastName}, {user.FirstName}, Status {status?.ButtonText ?? "Unknown"} Staffing Level {staffingLevel?.ButtonText ?? "Unknown"}."); } + + prompts.Add(string.Join(" ", lines)); } } else if (twilioRequest.Digits == "3") @@ -1101,13 +1163,21 @@ public async Task InboundVoiceAction(string userId, [FromQuery] Vo if (units != null && units.Any()) { + // One joined prompt (see the active-calls branch above). + var lines = new List(); + foreach (var unit in units) { var unitState = states.FirstOrDefault(x => x.UnitId == unit.UnitId); var unitStatus = await _customStateService.GetCustomUnitStateAsync(unitState); - prompts.Add($"{unit.Name}, Status {unitStatus.ButtonText}."); + // Units beyond the plan limit (or with a deleted custom state) have no + // resolvable state; a null dereference here 500s the webhook and Twilio + // speaks "application error" mid-menu. + lines.Add($"{unit.Name}, Status {unitStatus?.ButtonText ?? "Unknown"}."); } + + prompts.Add(string.Join(" ", lines)); } else { @@ -1120,10 +1190,16 @@ public async Task InboundVoiceAction(string userId, [FromQuery] Vo if (upcomingItems != null && upcomingItems.Any()) { + // One joined prompt (see the active-calls branch above). Each line gets a + // terminal period so the joined text chunks on sentence boundaries. + var lines = new List(); + foreach (var item in upcomingItems) { - prompts.Add($"{item.Title}, {item.Start.TimeConverter(department).ToShortDateString()}, {item.Start.TimeConverter(department).ToShortTimeString()}, {item.Location}"); + lines.Add($"{item.Title}, {item.Start.TimeConverter(department).ToShortDateString()}, {item.Start.TimeConverter(department).ToShortTimeString()}, {item.Location}."); } + + prompts.Add(string.Join(" ", lines)); } else { @@ -1275,25 +1351,48 @@ public async Task InboundVoiceActionStaffing(string userId, [FromQ private async System.Threading.Tasks.Task AppendVoicePromptAsync(VoiceResponse response, string text, int? departmentId = null) { var ttsLanguage = await GetDepartmentTtsLanguageAsync(departmentId); - await _twilioVoiceResponseService.AppendPromptAsync(response, text, HttpContext?.RequestAborted ?? CancellationToken.None, ttsLanguage); + + try + { + await _twilioVoiceResponseService.AppendPromptAsync(response, text, GetTtsPromptBudgetToken(), ttsLanguage); + } + catch (OperationCanceledException) when (TtsBudgetExpired) + { + _twilioVoiceResponseService.AppendSayFallback(response, text); + } } private async System.Threading.Tasks.Task AppendVoicePromptAsync(Gather gather, string text, int? departmentId = null) { var ttsLanguage = await GetDepartmentTtsLanguageAsync(departmentId); - await _twilioVoiceResponseService.AppendPromptAsync(gather, text, HttpContext?.RequestAborted ?? CancellationToken.None, ttsLanguage); + + try + { + await _twilioVoiceResponseService.AppendPromptAsync(gather, text, GetTtsPromptBudgetToken(), ttsLanguage); + } + catch (OperationCanceledException) when (TtsBudgetExpired) + { + _twilioVoiceResponseService.AppendSayFallback(gather, text); + } } + // The plural variants append prompt-by-prompt (rather than delegating to the + // service's AppendPromptsAsync) so a budget expiry mid-list only downgrades + // the remaining prompts to — prompts already appended are kept. private async System.Threading.Tasks.Task AppendVoicePromptsAsync(VoiceResponse response, IEnumerable prompts, int? departmentId = null) { - var ttsLanguage = await GetDepartmentTtsLanguageAsync(departmentId); - await _twilioVoiceResponseService.AppendPromptsAsync(response, prompts, HttpContext?.RequestAborted ?? CancellationToken.None, ttsLanguage); + foreach (var prompt in prompts) + { + await AppendVoicePromptAsync(response, prompt, departmentId); + } } private async System.Threading.Tasks.Task AppendVoicePromptsAsync(Gather gather, IEnumerable prompts, int? departmentId = null) { - var ttsLanguage = await GetDepartmentTtsLanguageAsync(departmentId); - await _twilioVoiceResponseService.AppendPromptsAsync(gather, prompts, HttpContext?.RequestAborted ?? CancellationToken.None, ttsLanguage); + foreach (var prompt in prompts) + { + await AppendVoicePromptAsync(gather, prompt, departmentId); + } } private async Task GetDepartmentTtsLanguageAsync(int? departmentId) @@ -1314,7 +1413,10 @@ private async Task GetDepartmentTtsLanguageAsync(int? departmentId) return ttsLanguage; } - private async System.Threading.Tasks.Task TryAppendDispatchPlaybackAsync(VoiceResponse response, Call call) + // Returns the dispatch text alongside readiness so the caller's retry branch + // reuses it instead of re-resolving the address (a second geocoder round-trip). + // DispatchText is null only when a recorded dispatch-audio attachment was played. + private async System.Threading.Tasks.Task<(bool Ready, string DispatchText)> TryAppendDispatchPlaybackAsync(VoiceResponse response, Call call) { if (call.Attachments != null) { @@ -1329,7 +1431,7 @@ private async System.Threading.Tasks.Task TryAppendDispatchPlaybackAsync(V { Url = audioUri }); - return true; + return (true, null); } } } @@ -1352,13 +1454,13 @@ private async System.Threading.Tasks.Task TryAppendDispatchPlaybackAsync(V // multi-chunk-aware AppendPromptAsync (one per chunk) instead of GetPromptUrlAsync, // which only supports single-chunk text and throws ArgumentException otherwise. await _twilioVoiceResponseService.AppendPromptAsync(response, dispatchText, linkedCts.Token, ttsLanguage); - return true; + return (true, dispatchText); } catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested) { // TTS generation is taking too long — return false so the caller // can pre-warm in the background and redirect. - return false; + return (false, dispatchText); } } @@ -1372,8 +1474,17 @@ private async Task ResolveCallAddressAsync(Call call) { string[] points = call.GeoLocationData.Split(char.Parse(",")); - if (points != null && points.Length == 2) - address = await _geoLocationProvider.GetAproxAddressFromLatLong(double.Parse(points[0]), double.Parse(points[1])); + // Bound the reverse-geocode: it's an external HTTP call with no timeout of + // its own, and it runs inside a Twilio webhook whose total budget is 15s. + // On timeout the catch swallows and the dispatch is spoken without an address. + // TryParse with InvariantCulture: malformed coordinates skip the lookup + // instead of throwing, and a comma-decimal server culture can't silently + // misread "47.606" as 47606. + if (points != null && points.Length == 2 + && double.TryParse(points[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var latitude) + && double.TryParse(points[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var longitude)) + address = await _geoLocationProvider.GetAproxAddressFromLatLong(latitude, longitude) + .WaitAsync(TimeSpan.FromSeconds(2), HttpContext?.RequestAborted ?? CancellationToken.None); } catch { @@ -1385,10 +1496,13 @@ private async Task ResolveCallAddressAsync(Call call) private static string BuildDispatchPrompt(Call call, string address) { + // Periods between the segments give the TTS engine sentence boundaries + // (Piper inserts 0.35s of silence per sentence), which keeps the priority, + // address and nature audibly separated instead of running together. var nature = StringHelpers.StripHtmlTagsCharArray(call.NatureOfCall); var prompt = !String.IsNullOrWhiteSpace(address) - ? string.Format("{0}, Priority {1} Address {2} Nature {3}", call.Name, call.GetPriorityText(), address, nature) - : string.Format("{0}, Priority {1} Nature {2}", call.Name, call.GetPriorityText(), nature); + ? string.Format("{0}, Priority {1}. Address {2}. Nature {3}", call.Name, call.GetPriorityText(), address, nature) + : string.Format("{0}, Priority {1}. Nature {2}", call.Name, call.GetPriorityText(), nature); return prompt.EndsWith(".", StringComparison.Ordinal) || prompt.EndsWith("!", StringComparison.Ordinal) || prompt.EndsWith("?", StringComparison.Ordinal) ? prompt diff --git a/Web/Resgrid.Web.Services/Health/FullHealthCheckAccess.cs b/Web/Resgrid.Web.Services/Health/FullHealthCheckAccess.cs new file mode 100644 index 000000000..dd2fc672a --- /dev/null +++ b/Web/Resgrid.Web.Services/Health/FullHealthCheckAccess.cs @@ -0,0 +1,38 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.AspNetCore.Http; + +namespace Resgrid.Web.Services.Health +{ + /// + /// Gates the deep /health/full endpoint. The probes it triggers (SQL, Redis, TTS + /// microservice) and the dependency detail it returns are for trusted monitoring + /// only, so callers must present the configured shared key. An unconfigured key + /// fails closed and disables the endpoint. The shallow /health liveness endpoint + /// stays public. + /// + public static class FullHealthCheckAccess + { + public const string HeaderName = "X-Resgrid-Health-Key"; + + public static bool IsAuthorized(HttpRequest request) + { + var configuredKey = Config.SystemBehaviorConfig.FullHealthCheckKey; + + if (string.IsNullOrWhiteSpace(configuredKey)) + return false; + + if (request == null || !request.Headers.TryGetValue(HeaderName, out var providedValues)) + return false; + + var providedKey = providedValues.ToString(); + + if (string.IsNullOrEmpty(providedKey)) + return false; + + return CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(providedKey), + Encoding.UTF8.GetBytes(configuredKey)); + } + } +} diff --git a/Web/Resgrid.Web.Services/Health/HealthResponseWriter.cs b/Web/Resgrid.Web.Services/Health/HealthResponseWriter.cs new file mode 100644 index 000000000..988d55730 --- /dev/null +++ b/Web/Resgrid.Web.Services/Health/HealthResponseWriter.cs @@ -0,0 +1,37 @@ +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Resgrid.Web.Services.Health +{ + /// + /// Shared JSON writer for the /health and /health/full endpoints. Matches the payload + /// shape the TTS microservice emits so monitoring can consume both uniformly. + /// + public static class HealthResponseWriter + { + public static async Task WriteAsync(HttpContext context, HealthReport report) + { + context.Response.ContentType = "application/json"; + + var payload = new + { + status = report.Status.ToString(), + checks = report.Entries.ToDictionary( + entry => entry.Key, + entry => new + { + status = entry.Value.Status.ToString(), + description = entry.Value.Description, + data = entry.Value.Data.Count > 0 + ? entry.Value.Data.ToDictionary(item => item.Key, item => item.Value?.ToString()) + : null + }) + }; + + await context.Response.WriteAsync(JsonSerializer.Serialize(payload), context.RequestAborted); + } + } +} diff --git a/Web/Resgrid.Web.Services/Health/RedisHealthCheck.cs b/Web/Resgrid.Web.Services/Health/RedisHealthCheck.cs new file mode 100644 index 000000000..0d60dc1f5 --- /dev/null +++ b/Web/Resgrid.Web.Services/Health/RedisHealthCheck.cs @@ -0,0 +1,49 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Resgrid.Model.Providers; + +namespace Resgrid.Web.Services.Health +{ + /// + /// Full-health probe: proves Redis is reachable through the same cache provider the + /// application uses, via a write/read round-trip. Not part of the shallow /health + /// liveness endpoint. + /// + public sealed class RedisHealthCheck : IHealthCheck + { + private const string ProbeKey = "health-probe"; + + private readonly ICacheProvider _cacheProvider; + + public RedisHealthCheck(ICacheProvider cacheProvider) + { + _cacheProvider = cacheProvider; + } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + // The provider swallows connection failures and returns false/null, so a + // failed round-trip surfaces here as a value mismatch rather than a throw. + var stored = await _cacheProvider.SetStringAsync(ProbeKey, "ok", TimeSpan.FromMinutes(1)); + + if (!stored) + return HealthCheckResult.Unhealthy("Redis write failed; the cache provider reports it is unavailable."); + + var value = await _cacheProvider.GetStringAsync(ProbeKey); + + if (value != "ok") + return HealthCheckResult.Unhealthy("Redis round-trip returned an unexpected value."); + + return HealthCheckResult.Healthy("Redis round-trip succeeded."); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy("Redis probe failed.", ex); + } + } + } +} diff --git a/Web/Resgrid.Web.Services/Health/SqlDatabaseHealthCheck.cs b/Web/Resgrid.Web.Services/Health/SqlDatabaseHealthCheck.cs new file mode 100644 index 000000000..1dcec1d9b --- /dev/null +++ b/Web/Resgrid.Web.Services/Health/SqlDatabaseHealthCheck.cs @@ -0,0 +1,44 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Resgrid.Config; + +namespace Resgrid.Web.Services.Health +{ + /// + /// Full-health probe: proves the primary SQL database accepts connections and + /// executes a query. Not part of the shallow /health liveness endpoint. + /// + public sealed class SqlDatabaseHealthCheck : IHealthCheck + { + private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5); + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + using var timeout = new CancellationTokenSource(ProbeTimeout); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken); + + try + { + await using var connection = new SqlConnection(DataConfig.ConnectionString); + await connection.OpenAsync(linked.Token); + + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT 1"; + await command.ExecuteScalarAsync(linked.Token); + + return HealthCheckResult.Healthy("Database connection and query succeeded."); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + return HealthCheckResult.Unhealthy($"Database probe timed out after {ProbeTimeout.TotalSeconds:0} seconds."); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy("Database probe failed.", ex); + } + } + } +} diff --git a/Web/Resgrid.Web.Services/Health/TtsServiceHealthCheck.cs b/Web/Resgrid.Web.Services/Health/TtsServiceHealthCheck.cs new file mode 100644 index 000000000..293fa6c22 --- /dev/null +++ b/Web/Resgrid.Web.Services/Health/TtsServiceHealthCheck.cs @@ -0,0 +1,57 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Resgrid.Config; + +namespace Resgrid.Web.Services.Health +{ + /// + /// Full-health probe: proves the TTS microservice is reachable by calling its shallow + /// /health endpoint. Deliberately does NOT call the TTS /health/full endpoint — that + /// spawns real synthesis work and belongs to the TTS service's own monitoring. Not + /// part of the shallow /health liveness endpoint. + /// + public sealed class TtsServiceHealthCheck : IHealthCheck + { + private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5); + + private readonly IHttpClientFactory _httpClientFactory; + + public TtsServiceHealthCheck(IHttpClientFactory httpClientFactory) + { + _httpClientFactory = httpClientFactory; + } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(TtsConfig.ServiceBaseUrl)) + return HealthCheckResult.Degraded("TTS service base URL is not configured; dynamic voice prompts fall back to native Twilio speech."); + + using var timeout = new CancellationTokenSource(ProbeTimeout); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken); + + try + { + var client = _httpClientFactory.CreateClient("ByPassSSLHttpClient"); + var healthUrl = $"{TtsConfig.ServiceBaseUrl.TrimEnd('/')}/health"; + + using var response = await client.GetAsync(healthUrl, linked.Token); + + if (response.IsSuccessStatusCode) + return HealthCheckResult.Healthy("TTS service is reachable and reports healthy."); + + return HealthCheckResult.Unhealthy($"TTS service /health returned HTTP {(int)response.StatusCode}."); + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + return HealthCheckResult.Unhealthy($"TTS service probe timed out after {ProbeTimeout.TotalSeconds:0} seconds."); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy("TTS service probe failed.", ex); + } + } + } +} diff --git a/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml b/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml index ce23e9947..d5d7f8907 100644 --- a/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml +++ b/Web/Resgrid.Web.Services/Resgrid.Web.Services.xml @@ -4736,6 +4736,42 @@ to its Call and department-ownership checked, then (3) a CallId on a bound request body object. + + + Gates the deep /health/full endpoint. The probes it triggers (SQL, Redis, TTS + microservice) and the dependency detail it returns are for trusted monitoring + only, so callers must present the configured shared key. An unconfigured key + fails closed and disables the endpoint. The shallow /health liveness endpoint + stays public. + + + + + Shared JSON writer for the /health and /health/full endpoints. Matches the payload + shape the TTS microservice emits so monitoring can consume both uniformly. + + + + + Full-health probe: proves Redis is reachable through the same cache provider the + application uses, via a write/read round-trip. Not part of the shallow /health + liveness endpoint. + + + + + Full-health probe: proves the primary SQL database accepts connections and + executes a query. Not part of the shallow /health liveness endpoint. + + + + + Full-health probe: proves the TTS microservice is reachable by calling its shallow + /health endpoint. Deliberately does NOT call the TTS /health/full endpoint — that + spawns real synthesis work and belongs to the TTS service's own monitoring. Not + part of the shallow /health liveness endpoint. + + Gets or sets the on authentication failed. @@ -11007,6 +11043,15 @@ resource lookups using this strongly typed resource class. + + + Appends the text as Twilio-native <Say> verbs to the given TwiML + parent (VoiceResponse, Gather, ...), split into the same chunks the TTS + pipeline would use so no single verb exceeds Twilio's per-<Say> + text limit. No-op unless TtsConfig.TwilioSayFallbackEnabled. Used when + the TTS time budget is exhausted or generation has already failed. + + Kicks off TTS generation for the given text in the background so that diff --git a/Web/Resgrid.Web.Services/Startup.cs b/Web/Resgrid.Web.Services/Startup.cs index 480c29add..235a11293 100644 --- a/Web/Resgrid.Web.Services/Startup.cs +++ b/Web/Resgrid.Web.Services/Startup.cs @@ -183,6 +183,16 @@ public void ConfigureServices(IServiceCollection services) }); services.AddMemoryCache(); + + // Shallow "self" check backs /health (liveness probes). The full checks run real + // dependency probes (SQL, Redis, TTS microservice) and only execute when + // /health/full is called explicitly. + services.AddHealthChecks() + .AddCheck("self", () => Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy("API process is responsive."), tags: new[] { "live" }) + .AddCheck("database", tags: new[] { "full" }) + .AddCheck("redis", tags: new[] { "full" }) + .AddCheck("tts_service", tags: new[] { "full" }); + services.Configure(Configuration.GetSection("IpRateLimiting")); services.AddSingleton(); services.AddInMemoryRateLimiting(); @@ -724,11 +734,45 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerF app.UseIpRateLimiting(); + // Deep-probe gate: /health/full runs real SQL/Redis/TTS probes and returns + // dependency detail, so it is restricted to trusted monitoring via the + // configured shared key (SystemBehaviorConfig.FullHealthCheckKey). An + // unconfigured key fails closed. The shallow /health endpoint stays public. + app.UseWhen( + context => context.Request.Path.StartsWithSegments("/health/full", StringComparison.OrdinalIgnoreCase), + healthApp => healthApp.Use(async (context, next) => + { + if (!Resgrid.Web.Services.Health.FullHealthCheckAccess.IsAuthorized(context.Request)) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + await next.Invoke(); + })); + app.UseEndpoints(endpoints => { endpoints.MapControllers(); endpoints.MapHub("/eventingHub"); + + // Shallow liveness: process is up and serving requests, no external calls. + // Point k8s liveness probes here. + endpoints.MapHealthChecks("/health", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions + { + Predicate = registration => !registration.Tags.Contains("full"), + ResponseWriter = Resgrid.Web.Services.Health.HealthResponseWriter.WriteAsync + }); + + // Deep dependency probe: SQL, Redis, and TTS microservice reachability. + // For monitoring and diagnostics — do not wire probes to this endpoint. + // Requires the X-Resgrid-Health-Key header (gate registered above); + // without a configured FullHealthCheckKey the endpoint returns 401. + endpoints.MapHealthChecks("/health/full", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions + { + ResponseWriter = Resgrid.Web.Services.Health.HealthResponseWriter.WriteAsync + }); }); } } diff --git a/Web/Resgrid.Web.Services/Twilio/ITwilioVoiceResponseService.cs b/Web/Resgrid.Web.Services/Twilio/ITwilioVoiceResponseService.cs index 23b2e67d9..1440933fc 100644 --- a/Web/Resgrid.Web.Services/Twilio/ITwilioVoiceResponseService.cs +++ b/Web/Resgrid.Web.Services/Twilio/ITwilioVoiceResponseService.cs @@ -17,6 +17,15 @@ public interface ITwilioVoiceResponseService System.Threading.Tasks.Task AppendPromptsAsync(Gather gather, IEnumerable prompts, CancellationToken cancellationToken = default, string voice = null); + /// + /// Appends the text as Twilio-native <Say> verbs to the given TwiML + /// parent (VoiceResponse, Gather, ...), split into the same chunks the TTS + /// pipeline would use so no single verb exceeds Twilio's per-<Say> + /// text limit. No-op unless TtsConfig.TwilioSayFallbackEnabled. Used when + /// the TTS time budget is exhausted or generation has already failed. + /// + void AppendSayFallback(TwiML parent, string text); + /// /// Kicks off TTS generation for the given text in the background so that /// a subsequent call to GetPromptUrlAsync (or AppendPromptAsync) will find diff --git a/Web/Resgrid.Web.Services/Twilio/TwilioVoiceResponseService.cs b/Web/Resgrid.Web.Services/Twilio/TwilioVoiceResponseService.cs index 249df747a..09f5d6c96 100644 --- a/Web/Resgrid.Web.Services/Twilio/TwilioVoiceResponseService.cs +++ b/Web/Resgrid.Web.Services/Twilio/TwilioVoiceResponseService.cs @@ -26,17 +26,17 @@ public TwilioVoiceResponseService(ITtsAudioService ttsAudioService) public async System.Threading.Tasks.Task AppendPromptAsync(VoiceResponse response, string text, CancellationToken cancellationToken = default, string voice = null) { - foreach (var play in await CreatePlayVerbsAsync(text, voice, cancellationToken)) + foreach (var verb in await CreatePlayVerbsAsync(text, voice, cancellationToken)) { - response.Append(play); + response.Append(verb); } } public async System.Threading.Tasks.Task AppendPromptAsync(Gather gather, string text, CancellationToken cancellationToken = default, string voice = null) { - foreach (var play in await CreatePlayVerbsAsync(text, voice, cancellationToken)) + foreach (var verb in await CreatePlayVerbsAsync(text, voice, cancellationToken)) { - gather.Append(play); + gather.Append(verb); } } @@ -56,17 +56,64 @@ public async System.Threading.Tasks.Task AppendPromptsAsync(Gather gather, IEnum } } - private async Task> CreatePlayVerbsAsync(string text, string voice, CancellationToken cancellationToken) + public void AppendSayFallback(TwiML parent, string text) + { + if (!TtsConfig.TwilioSayFallbackEnabled) + { + LogSayFallbackSkipped(text); + return; + } + + foreach (var chunk in ChunkText(text)) + { + parent.Append(new Say(chunk)); + } + } + + private static void LogSayFallbackSkipped(string text) + { + // Deliberate silence: TTS couldn't produce this prompt and the paid + // fallback is disabled (TtsConfig.TwilioSayFallbackEnabled). Log so a TTS + // outage is visible as skipped prompts rather than passing unnoticed. + Logging.LogInfo($"[Twilio Voice] TTS unavailable and Say fallback is disabled; skipping prompt ({text?.Length ?? 0} chars)."); + } + + private async Task> CreatePlayVerbsAsync(string text, string voice, CancellationToken cancellationToken) { var chunks = ChunkText(text).ToList(); if (!chunks.Any()) { - return new List(); + return new List(); } - var urls = await System.Threading.Tasks.Task.WhenAll(chunks.Select(chunk => GetOrCreatePromptUrlAsync(chunk, voice, cancellationToken))); - return urls.Select(CreatePlay).ToList(); + try + { + var urls = await System.Threading.Tasks.Task.WhenAll(chunks.Select(chunk => GetOrCreatePromptUrlAsync(chunk, voice, cancellationToken))); + return urls.Select(CreatePlay).Cast().ToList(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Caller-driven cancellation (e.g. the dispatch playback timeout) is control + // flow, not a failure — the caller decides what to do next. + throw; + } + catch (Exception ex) + { + // The TTS microservice is unavailable or errored. Never bubble a 500 out of + // a voice webhook (Twilio reads it to the caller as "an application error + // has occurred"): degrade to Twilio's native voice when the paid + // fallback is enabled, otherwise skip the prompt and keep the call flowing. + Logging.LogException(ex); + + if (!TtsConfig.TwilioSayFallbackEnabled) + { + LogSayFallbackSkipped(text); + return new List(); + } + + return chunks.Select(chunk => (TwiML)new Say(chunk)).ToList(); + } } private IEnumerable ChunkText(string text) diff --git a/Web/Resgrid.Web.Tts/Health/TtsFullHealthCheck.cs b/Web/Resgrid.Web.Tts/Health/TtsFullHealthCheck.cs new file mode 100644 index 000000000..99d336259 --- /dev/null +++ b/Web/Resgrid.Web.Tts/Health/TtsFullHealthCheck.cs @@ -0,0 +1,155 @@ +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using Resgrid.Web.Tts.Configuration; +using Resgrid.Web.Tts.Services; +using System.Diagnostics; + +namespace Resgrid.Web.Tts.Health +{ + /// + /// Deep dependency probe backing /health/full: proves Redis, S3 and the Piper/ffmpeg + /// synthesis pipeline actually work, not merely that they are configured (the shallow + /// wired to the k8s probes covers configuration). + /// The result is memoized briefly so aggressive monitoring cannot spawn a Piper process + /// per poll. + /// + public sealed class TtsFullHealthCheck : IHealthCheck + { + private static readonly TimeSpan ResultMemoDuration = TimeSpan.FromSeconds(30); + private static readonly TimeSpan ComponentTimeout = TimeSpan.FromSeconds(10); + private const string SynthesisProbeText = "Resgrid health check."; + + private readonly IDistributedCache _distributedCache; + private readonly IStorageService _storageService; + private readonly IAudioProcessingService _audioProcessingService; + private readonly TtsOptions _options; + private readonly SemaphoreSlim _probeLock = new(1, 1); + + private HealthCheckResult? _lastResult; + private DateTimeOffset _lastResultAt; + + public TtsFullHealthCheck( + IDistributedCache distributedCache, + IStorageService storageService, + IAudioProcessingService audioProcessingService, + IOptions options) + { + _distributedCache = distributedCache; + _storageService = storageService; + _audioProcessingService = audioProcessingService; + _options = options.Value; + } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + await _probeLock.WaitAsync(cancellationToken); + + try + { + if (_lastResult.HasValue && DateTimeOffset.UtcNow - _lastResultAt < ResultMemoDuration) + { + return _lastResult.Value; + } + + var result = await RunProbesAsync(cancellationToken); + + _lastResult = result; + _lastResultAt = DateTimeOffset.UtcNow; + + return result; + } + finally + { + _probeLock.Release(); + } + } + + private async Task RunProbesAsync(CancellationToken cancellationToken) + { + var data = new Dictionary(); + var errors = new List(); + + await ProbeAsync("redis", data, errors, async token => + { + const string probeKey = "tts-health::probe"; + var expiry = new DistributedCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(1) + }; + + await _distributedCache.SetAsync(probeKey, new byte[] { 1 }, expiry, token); + + var payload = await _distributedCache.GetAsync(probeKey, token); + + if (payload is null || payload.Length == 0) + { + throw new InvalidOperationException("Redis round-trip returned no payload."); + } + }, cancellationToken); + + await ProbeAsync("s3", data, errors, async token => + { + // The existence result is irrelevant — a completed call proves the bucket is + // reachable with the configured credentials. + await _storageService.ExistsAsync(GetProbeObjectKey(), token); + }, cancellationToken); + + await ProbeAsync("synthesis", data, errors, async token => + { + var audio = await _audioProcessingService.GenerateNormalizedWavAsync( + SynthesisProbeText, _options.DefaultVoice, _options.DefaultSpeed, token); + + if (audio is null || audio.Length == 0) + { + throw new InvalidOperationException("Synthesis returned no audio bytes."); + } + }, cancellationToken); + + if (errors.Count == 0) + { + return HealthCheckResult.Healthy("All TTS dependencies passed functional probes.", data); + } + + return HealthCheckResult.Unhealthy(string.Join(" ", errors), data: data); + } + + private string GetProbeObjectKey() + { + var prefix = string.IsNullOrWhiteSpace(_options.CachePrefix) + ? string.Empty + : $"{_options.CachePrefix.Trim().Trim('/')}/"; + + return $"{prefix}health-probe.wav"; + } + + private static async Task ProbeAsync( + string name, + Dictionary data, + List errors, + Func probe, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + + using var timeout = new CancellationTokenSource(ComponentTimeout); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken); + + try + { + await probe(linked.Token); + data[name] = $"healthy ({stopwatch.ElapsedMilliseconds}ms)"; + } + catch (OperationCanceledException) when (timeout.IsCancellationRequested) + { + data[name] = $"timeout ({stopwatch.ElapsedMilliseconds}ms)"; + errors.Add($"The {name} probe timed out after {ComponentTimeout.TotalSeconds:0} seconds."); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + data[name] = $"failed ({stopwatch.ElapsedMilliseconds}ms)"; + errors.Add($"The {name} probe failed: {ex.Message}"); + } + } + } +} diff --git a/Web/Resgrid.Web.Tts/Health/TtsHealthCheckAccess.cs b/Web/Resgrid.Web.Tts/Health/TtsHealthCheckAccess.cs new file mode 100644 index 000000000..9e04a241a --- /dev/null +++ b/Web/Resgrid.Web.Tts/Health/TtsHealthCheckAccess.cs @@ -0,0 +1,39 @@ +using System.Security.Cryptography; +using System.Text; + +namespace Resgrid.Web.Tts.Health +{ + /// + /// Gates the deep /health/full endpoint. Its probes run a real Piper/ffmpeg + /// synthesis plus Redis and S3 round-trips, and this service is internet-reachable + /// (Twilio fetches playback audio from it), so callers must present the same + /// shared monitoring key the API's /health/full uses + /// (SystemBehaviorConfig.FullHealthCheckKey, X-Resgrid-Health-Key header). + /// An unconfigured key fails closed and disables the endpoint. The shallow + /// /health probe endpoint stays public. + /// + public static class TtsHealthCheckAccess + { + public const string HeaderName = "X-Resgrid-Health-Key"; + + public static bool IsAuthorized(HttpRequest request) + { + var configuredKey = Resgrid.Config.SystemBehaviorConfig.FullHealthCheckKey; + + if (string.IsNullOrWhiteSpace(configuredKey)) + return false; + + if (request == null || !request.Headers.TryGetValue(HeaderName, out var providedValues)) + return false; + + var providedKey = providedValues.ToString(); + + if (string.IsNullOrEmpty(providedKey)) + return false; + + return CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(providedKey), + Encoding.UTF8.GetBytes(configuredKey)); + } + } +} diff --git a/Web/Resgrid.Web.Tts/Program.cs b/Web/Resgrid.Web.Tts/Program.cs index 9a609dcbb..f805197f4 100644 --- a/Web/Resgrid.Web.Tts/Program.cs +++ b/Web/Resgrid.Web.Tts/Program.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.HttpOverrides; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using Resgrid.Config; @@ -40,7 +41,7 @@ if (samplingContext.CustomSamplingContext.TryGetValue("__HttpPath", out var httpPath)) { var pathValue = httpPath?.ToString(); - if (string.Equals(pathValue, "/health", StringComparison.OrdinalIgnoreCase) || + if ((pathValue is not null && pathValue.StartsWith("/health", StringComparison.OrdinalIgnoreCase)) || string.Equals(pathValue, "/livez", StringComparison.OrdinalIgnoreCase) || string.Equals(pathValue, "/readyz", StringComparison.OrdinalIgnoreCase)) { @@ -69,8 +70,13 @@ builder.Services.AddHttpClient(); builder.Services.AddTtsConfiguration(); builder.Services.Configure(TtsRequestIdentity.ConfigureForwardedHeaders); +// Shallow (config-presence) check backs /health for the k8s probes; the full check runs +// functional probes against Redis, S3 and the Piper/ffmpeg pipeline and only executes +// when /health/full is called explicitly. +builder.Services.AddSingleton(); builder.Services.AddHealthChecks() - .AddCheck("tts_dependencies"); + .AddCheck("tts_dependencies", tags: new[] { "live" }) + .AddCheck("tts_full", tags: new[] { "full" }); builder.Services.AddRateLimiter(options => { options.RejectionStatusCode = StatusCodes.Status429TooManyRequests; @@ -124,29 +130,63 @@ await context.HttpContext.Response.WriteAsJsonAsync( app.UseForwardedHeaders(); app.UseRateLimiter(); -app.MapHealthChecks("/health", new HealthCheckOptions -{ - ResponseWriter = static async (context, report) => +// Deep-probe gate: /health/full runs a real Piper/ffmpeg synthesis plus Redis and +// S3 round-trips, and this service is internet-reachable (Twilio fetches playback +// audio from it). Restricted to trusted monitoring via the shared key +// (SystemBehaviorConfig.FullHealthCheckKey); an unconfigured key fails closed. +// The shallow /health endpoint stays public for the k8s probes. +app.UseWhen( + context => context.Request.Path.StartsWithSegments("/health/full", StringComparison.OrdinalIgnoreCase), + healthApp => healthApp.Use(async (context, next) => { - context.Response.ContentType = "application/json"; - - var payload = new + if (!TtsHealthCheckAccess.IsAuthorized(context.Request)) { - status = report.Status.ToString(), - checks = report.Entries.ToDictionary( - entry => entry.Key, - entry => new - { - status = entry.Value.Status.ToString(), - description = entry.Value.Description - }) - }; + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } - await context.Response.WriteAsync(JsonSerializer.Serialize(payload), context.RequestAborted); - } + await next(); + })); + +// Shallow liveness endpoint: configuration presence only, no external calls. This is +// what the k8s liveness/readiness/startup probes point at. +app.MapHealthChecks("/health", new HealthCheckOptions +{ + Predicate = registration => !registration.Tags.Contains("full"), + ResponseWriter = WriteHealthResponseAsync +}); + +// Deep functional probe: Redis round-trip, S3 reachability, and a real Piper/ffmpeg +// synthesis. For monitoring and diagnostics — do not wire probes to this endpoint. +// Requires the X-Resgrid-Health-Key header (gate registered above). +app.MapHealthChecks("/health/full", new HealthCheckOptions +{ + ResponseWriter = WriteHealthResponseAsync }); app.MapControllers(); +static async Task WriteHealthResponseAsync(HttpContext context, HealthReport report) +{ + context.Response.ContentType = "application/json"; + + var payload = new + { + status = report.Status.ToString(), + checks = report.Entries.ToDictionary( + entry => entry.Key, + entry => new + { + status = entry.Value.Status.ToString(), + description = entry.Value.Description, + data = entry.Value.Data.Count > 0 + ? entry.Value.Data.ToDictionary(item => item.Key, item => item.Value?.ToString()) + : null + }) + }; + + await context.Response.WriteAsync(JsonSerializer.Serialize(payload), context.RequestAborted); +} + app.Run(); public partial class Program; diff --git a/Web/Resgrid.Web.Tts/Services/TextPreprocessor.cs b/Web/Resgrid.Web.Tts/Services/TextPreprocessor.cs index 344c3f8b7..a1b560e12 100644 --- a/Web/Resgrid.Web.Tts/Services/TextPreprocessor.cs +++ b/Web/Resgrid.Web.Tts/Services/TextPreprocessor.cs @@ -8,9 +8,10 @@ namespace Resgrid.Web.Tts.Services /// Transforms dispatch jargon, abbreviations, and codes into expanded, /// pronounceable English that the TTS engine renders clearly. /// - /// The preprocessor runs before the cache key is computed so that - /// two requests that differ only by abbreviation style share the same - /// synthesised audio. + /// The preprocessor runs inside audio generation, after the cache + /// key has been computed from the raw request text (see TtsService) — two + /// requests that only differ by abbreviation style therefore cache as + /// separate entries even though they synthesise identical speech. /// /// Expansion rules are deliberately conservative — we only touch terms /// that the engine routinely gets wrong. Everything else is passed through @@ -124,7 +125,12 @@ public sealed partial class TextPreprocessor : ITextPreprocessor // Communications { "PX", "Phone Extension" }, + // All casings of "etc" are safe to expand (no English-word collision), + // so list each explicitly for the case-sensitive matcher — same pattern + // as HAZMAT/HazMat in AbbreviationMap. { "etc", "et cetera" }, + { "ETC", "et cetera" }, + { "Etc", "et cetera" }, // Geographical { "NH", "Northbound" }, @@ -245,10 +251,14 @@ public string Preprocess(string text, string voice) private static string ExpandAbbreviations(string text) { // Sort keys longest-first so "ALSEMS" is matched before "ALS". + // Matching is case-sensitive: short tokens like "SO", "PASS", "CO" and "PI" + // collide with ordinary English words when lowercased, and CAD feeds emit + // these codes in upper case (the map carries explicit casing variants such + // as "HAZMAT"/"HazMat" where more than one form is expected). foreach (var kvp in AbbreviationMap.OrderByDescending(k => k.Key.Length)) { var pattern = $@"\b{Regex.Escape(kvp.Key)}\b"; - text = Regex.Replace(text, pattern, kvp.Value, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + text = Regex.Replace(text, pattern, kvp.Value, RegexOptions.CultureInvariant); } return text; @@ -262,10 +272,13 @@ private static string ExpandAbbreviations(string text) /// private static string ExpandDispatchShorthand(string text) { + // Case-sensitive for the same reason as ExpandAbbreviations: "APT", "PT", + // "RM" and "ADV" lowercased are (parts of) ordinary words, and CAD systems + // emit shorthand in upper case. foreach (var kvp in DispatchShorthandMap.OrderByDescending(k => k.Key.Length)) { var pattern = $@"\b{Regex.Escape(kvp.Key)}\b"; - text = Regex.Replace(text, pattern, kvp.Value, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + text = Regex.Replace(text, pattern, kvp.Value, RegexOptions.CultureInvariant); } return text; @@ -335,11 +348,13 @@ private static string ExpandAddressAbbreviations(string text) // after a house/building number (e.g. "123 Main St" → "123 Main Street"). // The pattern anchors to a leading digit (\b\d+\b), then lazily skips // over the street name (one or more words) before matching the suffix. + // The house number and street name are captured and re-emitted — only + // the suffix itself is rewritten. foreach (var kvp in AddressAbbreviationMap.OrderByDescending(k => k.Key.Length)) { - var pattern = $@"\b\d+\b[\s\w,]*?\b{Regex.Escape(kvp.Key)}\b"; - text = Regex.Replace(text, pattern, kvp.Value, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + var pattern = $@"(\b\d+\b[\s\w,]*?)\b{Regex.Escape(kvp.Key)}\b"; + text = Regex.Replace(text, pattern, "${1}" + kvp.Value, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); } return text; @@ -411,8 +426,13 @@ private static string ExpandLongNumbers(string text) [GeneratedRegex(@"\b(?[A-Z])(?\d+)\b", RegexOptions.CultureInvariant)] private static partial Regex UnitIdentifierExpandoRegex(); - /// Matches standalone digits 1-20 followed by a space and a letter. - [GeneratedRegex(@"\b(?(?:[1-9]|1[0-9]|20))\s(?[A-Za-z])", RegexOptions.CultureInvariant)] + /// + /// Matches standalone digits 1-20 followed by a space and a letter. + /// The negative lookbehind skips digits that are part of a spaced + /// digit-by-digit run produced by ExpandLongNumbers (e.g. "1 2 3 4 5 Elm"), + /// which must stay uniformly digit-form rather than ending "...4 five Elm". + /// + [GeneratedRegex(@"(?(?:[1-9]|1[0-9]|20))\s(?[A-Za-z])", RegexOptions.CultureInvariant)] private static partial Regex NumberToWordRegex(); /// Matches a run of 4+ consecutive digits not surrounded by other digits.