Skip to content

fix: fresh headers per paged request, cap SQL results at TOP (#172) #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zekelinAlex wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: fresh headers per paged request, cap SQL results at TOP (#172) #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/TALXIS.CLI.Platform.Dataverse.Application/Sdk/SolutionComponentQueryReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ public static async Task<IReadOnlyList<ComponentSummaryRow>> ListAsync(
"&api-version=9.1" +
(top.HasValue ? $"&$top={top.Value}" : "");

var headers = new Dictionary<string, List<string>>
static Dictionary<string, List<string>> BuildHeaders() => new()
{
["Prefer"] = new() { "odata.maxpagesize=5000" },
};

var response = client.ExecuteWebRequest(HttpMethod.Get, path, string.Empty, headers);
var response = client.ExecuteWebRequest(HttpMethod.Get, path, string.Empty, BuildHeaders());
response.EnsureSuccessStatusCode();

var rows = new List<ComponentSummaryRow>();
Expand All @@ -80,7 +80,7 @@ public static async Task<IReadOnlyList<ComponentSummaryRow>> ListAsync(
relativePath = relativePath[(relativePath.IndexOf('/') + 1)..];

response.Dispose();
response = client.ExecuteWebRequest(HttpMethod.Get, relativePath, string.Empty, headers);
response = client.ExecuteWebRequest(HttpMethod.Get, relativePath, string.Empty, BuildHeaders());
response.EnsureSuccessStatusCode();
await ParsePageAsync(response, rows, ct).ConfigureAwait(false);
}
Expand Down
55 changes: 40 additions & 15 deletions src/TALXIS.CLI.Platform.Dataverse.Data/DataverseQueryService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,17 +50,19 @@ public async Task<DataverseQueryResult> QuerySqlAsync(
// Web API request targets the correct OData entity set.
var entitySetName = ResolveEntitySetName(conn, sql);

var headers = BuildHeaders(includeAnnotations);
var effectiveTop = MinTop(top, TryGetSqlTop(sql));

var queryPath = $"{entitySetName}?sql={Uri.EscapeDataString(sql)}";

var records = new List<JsonElement>();
try
{
using var response = conn.Client.ExecuteWebRequest(HttpMethod.Get, queryPath, string.Empty, headers);
using var response = conn.Client.ExecuteWebRequest(
HttpMethod.Get, queryPath, string.Empty, BuildHeaders(includeAnnotations));
await ParseValueArrayAsync(response, records, ct).ConfigureAwait(false);

// Follow @odata.nextLink pages until exhausted or top limit reached.
await FollowNextLinksAsync(conn, response, records, top, headers, ct).ConfigureAwait(false);
await FollowNextLinksAsync(conn, response, records, effectiveTop, includeAnnotations, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
Expand All @@ -71,8 +73,8 @@ public async Task<DataverseQueryResult> QuerySqlAsync(
if (!includeAnnotations)
StripAnnotations(records);

if (top.HasValue && records.Count > top.Value)
records.RemoveRange(top.Value, records.Count - top.Value);
if (effectiveTop.HasValue && records.Count > effectiveTop.Value)
records.RemoveRange(effectiveTop.Value, records.Count - effectiveTop.Value);

return new DataverseQueryResult(records, records.Count);
}
Expand Down Expand Up @@ -140,19 +142,19 @@ public async Task<DataverseQueryResult> QueryODataAsync(
using var conn = await DataverseCommandBridge.ConnectAsync(profileName, ct).ConfigureAwait(false);

var queryPath = BuildODataQueryPath(entitySetOrPath, select, filter, orderBy, top);
var headers = BuildHeaders(includeAnnotations);

var records = new List<JsonElement>();
try
{
using var response = conn.Client.ExecuteWebRequest(HttpMethod.Get, queryPath, string.Empty, headers);
using var response = conn.Client.ExecuteWebRequest(
HttpMethod.Get, queryPath, string.Empty, BuildHeaders(includeAnnotations));
await ParseValueArrayAsync(response, records, ct).ConfigureAwait(false);

// Dataverse can still return @odata.nextLink when $top is specified
// (for example when the requested top exceeds the server page size).
// Always follow pagination links and let the helper stop once the
// requested number of records has been accumulated.
await FollowNextLinksAsync(conn, response, records, top, headers, ct).ConfigureAwait(false);
await FollowNextLinksAsync(conn, response, records, top, includeAnnotations, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
Expand Down Expand Up @@ -182,14 +184,34 @@ private static void ValidateSqlReadOnly(string sql)
"Write operations (INSERT, UPDATE, DELETE, DROP, ALTER, TRUNCATE, MERGE) are not allowed. Only SELECT queries are supported.");
}

/// <summary>
/// Extracts the row limit from a SQL <c>TOP n</c> / <c>TOP (n)</c> clause,
/// or <c>null</c> when the query has no TOP clause.
/// </summary>
private static int? TryGetSqlTop(string sql)
{
var match = Regex.Match(sql, @"\bTOP\s*\(?\s*(\d+)\s*\)?", RegexOptions.IgnoreCase);

return match.Success && int.TryParse(match.Groups[1].Value, out var n) ? n : null;
}

/// <summary>Returns the smaller of two optional row limits.</summary>
private static int? MinTop(int? a, int? b)
{
if (a.HasValue && b.HasValue) return Math.Min(a.Value, b.Value);

return a ?? b;
}

/// <summary>
/// Extracts the table logical name from the SQL FROM clause and
/// retrieves its <c>EntitySetName</c> via entity metadata.
/// Supports an optional schema prefix and bracketed identifiers
/// (<c>FROM account</c>, <c>FROM dbo.account</c>, <c>FROM [dbo].[account]</c>).
/// </summary>
private static string ResolveEntitySetName(DataverseConnection conn, string sql)
{
// Simple regex to grab the first table name after FROM.
var match = Regex.Match(sql, @"\bFROM\s+(\w+)", RegexOptions.IgnoreCase);
var match = Regex.Match(sql, @"\bFROM\s+(?:\[?dbo\]?\s*\.\s*)?\[?(\w+)\]?", RegexOptions.IgnoreCase);
if (!match.Success)
throw new InvalidOperationException($"Could not parse a table name from the SQL FROM clause in: '{sql}'.");

Expand Down Expand Up @@ -309,13 +331,15 @@ private static async Task ParseValueArrayAsync(
/// Follows <c>@odata.nextLink</c> URLs for automatic pagination.
/// NextLink URLs are absolute; we extract the relative path to pass
/// to <see cref="Microsoft.PowerPlatform.Dataverse.Client.ServiceClient.ExecuteWebRequest"/>.
/// A fresh header dictionary is built per request because the SDK mutates
/// the passed dictionary (adds a <c>Cookie</c> entry) and throws on reuse.
/// </summary>
private static async Task FollowNextLinksAsync(
DataverseConnection conn,
HttpResponseMessage lastResponse,
List<JsonElement> records,
int? top,
Dictionary<string, List<string>> headers,
bool includeAnnotations,
CancellationToken ct)
{
var currentResponse = lastResponse;
Expand All @@ -324,6 +348,9 @@ private static async Task FollowNextLinksAsync(
{
ct.ThrowIfCancellationRequested();

if (top.HasValue && records.Count >= top.Value)
break;

var content = await currentResponse.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
using var doc = JsonDocument.Parse(content);

Expand All @@ -338,7 +365,8 @@ private static async Task FollowNextLinksAsync(
var relativePath = ExtractRelativePath(nextLinkUrl, conn);

// Dispose paginated responses to avoid socket/handler leaks.
var nextResponse = conn.Client.ExecuteWebRequest(HttpMethod.Get, relativePath, string.Empty, headers);
var nextResponse = conn.Client.ExecuteWebRequest(
HttpMethod.Get, relativePath, string.Empty, BuildHeaders(includeAnnotations));
try
{
await ParseValueArrayAsync(nextResponse, records, ct).ConfigureAwait(false);
Expand All @@ -350,9 +378,6 @@ private static async Task FollowNextLinksAsync(
currentResponse.Dispose();
currentResponse = nextResponse;
}

if (top.HasValue && records.Count >= top.Value)
break;
}

// Dispose the last paginated response (but not the caller's initial response).
Expand Down