From 0cffffd624ce14d5451c2bf043f20306dd7381f3 Mon Sep 17 00:00:00 2001 From: Alexander Zekelin Date: Tue, 21 Jul 2026 18:02:27 +0200 Subject: [PATCH] fix(data): fresh headers per paged request, cap SQL results at TOP --- .../Sdk/SolutionComponentQueryReader.cs | 6 +- .../DataverseQueryService.cs | 55 ++++++++++++++----- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/SolutionComponentQueryReader.cs b/src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/SolutionComponentQueryReader.cs index bafbb4cd..5f6d92fe 100644 --- a/src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/SolutionComponentQueryReader.cs +++ b/src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/SolutionComponentQueryReader.cs @@ -48,12 +48,12 @@ public static async Task> ListAsync( "&api-version=9.1" + (top.HasValue ? $"&$top={top.Value}" : ""); - var headers = new Dictionary> + static Dictionary> BuildHeaders() => new() { ["Prefer"] = new() { "odata.maxpagesize=5000" }, }; - var response = client.ExecuteWebRequest(HttpMethod.Get, path, string.Empty, headers); + var response = client.ExecuteWebRequest(HttpMethod.Get, path, string.Empty, BuildHeaders()); response.EnsureSuccessStatusCode(); var rows = new List(); @@ -80,7 +80,7 @@ public static async Task> ListAsync( relativePath = relativePath[(relativePath.IndexOf('/') + 1)..]; response.Dispose(); - response = client.ExecuteWebRequest(HttpMethod.Get, relativePath, string.Empty, headers); + response = client.ExecuteWebRequest(HttpMethod.Get, relativePath, string.Empty, BuildHeaders()); response.EnsureSuccessStatusCode(); await ParsePageAsync(response, rows, ct).ConfigureAwait(false); } diff --git a/src/TALXIS.CLI.Platform.Dataverse.Data/DataverseQueryService.cs b/src/TALXIS.CLI.Platform.Dataverse.Data/DataverseQueryService.cs index 9c6b158a..8f9eecbb 100644 --- a/src/TALXIS.CLI.Platform.Dataverse.Data/DataverseQueryService.cs +++ b/src/TALXIS.CLI.Platform.Dataverse.Data/DataverseQueryService.cs @@ -50,17 +50,19 @@ public async Task QuerySqlAsync( // Web API request targets the correct OData entity set. var entitySetName = ResolveEntitySetName(conn, sql); - var headers = BuildHeaders(includeAnnotations); + var effectiveTop = MinTop(top, TryGetSqlTop(sql)); + var queryPath = $"{entitySetName}?sql={Uri.EscapeDataString(sql)}"; var records = new List(); try { - using var response = conn.Client.ExecuteWebRequest(HttpMethod.Get, queryPath, string.Empty, headers); + using var response = conn.Client.ExecuteWebRequest( + HttpMethod.Get, queryPath, string.Empty, BuildHeaders(includeAnnotations)); await ParseValueArrayAsync(response, records, ct).ConfigureAwait(false); // Follow @odata.nextLink pages until exhausted or top limit reached. - await FollowNextLinksAsync(conn, response, records, top, headers, ct).ConfigureAwait(false); + await FollowNextLinksAsync(conn, response, records, effectiveTop, includeAnnotations, ct).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -71,8 +73,8 @@ public async Task QuerySqlAsync( if (!includeAnnotations) StripAnnotations(records); - if (top.HasValue && records.Count > top.Value) - records.RemoveRange(top.Value, records.Count - top.Value); + if (effectiveTop.HasValue && records.Count > effectiveTop.Value) + records.RemoveRange(effectiveTop.Value, records.Count - effectiveTop.Value); return new DataverseQueryResult(records, records.Count); } @@ -140,19 +142,19 @@ public async Task QueryODataAsync( using var conn = await DataverseCommandBridge.ConnectAsync(profileName, ct).ConfigureAwait(false); var queryPath = BuildODataQueryPath(entitySetOrPath, select, filter, orderBy, top); - var headers = BuildHeaders(includeAnnotations); var records = new List(); try { - using var response = conn.Client.ExecuteWebRequest(HttpMethod.Get, queryPath, string.Empty, headers); + using var response = conn.Client.ExecuteWebRequest( + HttpMethod.Get, queryPath, string.Empty, BuildHeaders(includeAnnotations)); await ParseValueArrayAsync(response, records, ct).ConfigureAwait(false); // Dataverse can still return @odata.nextLink when $top is specified // (for example when the requested top exceeds the server page size). // Always follow pagination links and let the helper stop once the // requested number of records has been accumulated. - await FollowNextLinksAsync(conn, response, records, top, headers, ct).ConfigureAwait(false); + await FollowNextLinksAsync(conn, response, records, top, includeAnnotations, ct).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -182,14 +184,34 @@ private static void ValidateSqlReadOnly(string sql) "Write operations (INSERT, UPDATE, DELETE, DROP, ALTER, TRUNCATE, MERGE) are not allowed. Only SELECT queries are supported."); } + /// + /// Extracts the row limit from a SQL TOP n / TOP (n) clause, + /// or null when the query has no TOP clause. + /// + private static int? TryGetSqlTop(string sql) + { + var match = Regex.Match(sql, @"\bTOP\s*\(?\s*(\d+)\s*\)?", RegexOptions.IgnoreCase); + + return match.Success && int.TryParse(match.Groups[1].Value, out var n) ? n : null; + } + + /// Returns the smaller of two optional row limits. + private static int? MinTop(int? a, int? b) + { + if (a.HasValue && b.HasValue) return Math.Min(a.Value, b.Value); + + return a ?? b; + } + /// /// Extracts the table logical name from the SQL FROM clause and /// retrieves its EntitySetName via entity metadata. + /// Supports an optional schema prefix and bracketed identifiers + /// (FROM account, FROM dbo.account, FROM [dbo].[account]). /// private static string ResolveEntitySetName(DataverseConnection conn, string sql) { - // Simple regex to grab the first table name after FROM. - var match = Regex.Match(sql, @"\bFROM\s+(\w+)", RegexOptions.IgnoreCase); + var match = Regex.Match(sql, @"\bFROM\s+(?:\[?dbo\]?\s*\.\s*)?\[?(\w+)\]?", RegexOptions.IgnoreCase); if (!match.Success) throw new InvalidOperationException($"Could not parse a table name from the SQL FROM clause in: '{sql}'."); @@ -309,13 +331,15 @@ private static async Task ParseValueArrayAsync( /// Follows @odata.nextLink URLs for automatic pagination. /// NextLink URLs are absolute; we extract the relative path to pass /// to . + /// A fresh header dictionary is built per request because the SDK mutates + /// the passed dictionary (adds a Cookie entry) and throws on reuse. /// private static async Task FollowNextLinksAsync( DataverseConnection conn, HttpResponseMessage lastResponse, List records, int? top, - Dictionary> headers, + bool includeAnnotations, CancellationToken ct) { var currentResponse = lastResponse; @@ -324,6 +348,9 @@ private static async Task FollowNextLinksAsync( { ct.ThrowIfCancellationRequested(); + if (top.HasValue && records.Count >= top.Value) + break; + var content = await currentResponse.Content.ReadAsStringAsync(ct).ConfigureAwait(false); using var doc = JsonDocument.Parse(content); @@ -338,7 +365,8 @@ private static async Task FollowNextLinksAsync( var relativePath = ExtractRelativePath(nextLinkUrl, conn); // Dispose paginated responses to avoid socket/handler leaks. - var nextResponse = conn.Client.ExecuteWebRequest(HttpMethod.Get, relativePath, string.Empty, headers); + var nextResponse = conn.Client.ExecuteWebRequest( + HttpMethod.Get, relativePath, string.Empty, BuildHeaders(includeAnnotations)); try { await ParseValueArrayAsync(nextResponse, records, ct).ConfigureAwait(false); @@ -350,9 +378,6 @@ private static async Task FollowNextLinksAsync( currentResponse.Dispose(); currentResponse = nextResponse; } - - if (top.HasValue && records.Count >= top.Value) - break; } // Dispose the last paginated response (but not the caller's initial response).