diff --git a/docs/4-data-queries/query-limits-and-performance.md b/docs/4-data-queries/query-limits-and-performance.md index 9f4e154c3..eafa054f6 100644 --- a/docs/4-data-queries/query-limits-and-performance.md +++ b/docs/4-data-queries/query-limits-and-performance.md @@ -38,6 +38,241 @@ When the limit is reached, additional queries are rejected with an `HTTP 429` (t !!! tip If you regularly run automation or dashboards that need more headroom, contact support to request a higher concurrent-query limit for your organization. +### Seeing What Is Using Your Limit + +When queries start being rejected, the question is which ones are responsible. `GET /v1/search/{oid}/queries` lists the queries your organization currently has open, and reports how many of them are consuming the limit. + +Those are two different numbers, and the response reports them separately: + +- **`slotsHeld`** is how many queries are consuming concurrency right now. This is what the limit applies to. +- **`count`** is how many queries are open in total, including ones that are paused between pages. + +A paginated query is open from the moment it is submitted until it finishes, is cancelled, or its state expires - but it only consumes a slot while a page is actually running. A client that fetched a page and has not asked for the next one holds no slot, so an organization can legitimately have many queries open while few of them count against the limit. + +Every entry says which case it falls into: + +| `state` | Consumes a slot | Meaning | +|---------|-----------------|---------| +| `executing` | Yes | A page of this query is running. | +| `queued` | Yes | A slot is held but the work has not started yet. Many of these at once indicates a backlog. | +| `idle` | No | The query is open and can be resumed, but nothing is running. | +| `unknown` | Not reported | Per-query slot tracking is not available in this deployment. | + +Each entry also carries the query text and time range as submitted, who submitted it and from which client (`submittedBy`, `userAgent`), how long the current page has been running (`runningForMs`), how far along it is (`batchesCompleted` / `batchesInScope` and `progressPercent`), how much it has scanned and been billed for (`eventsScanned`, `billedEvents`), and two separate expiries: `slotExpiresAt` for the concurrency slot, and `resumableUntil` for how long the query can still be resumed. + +The `?state=` parameter accepts `all` (the default), `executing` for only what is consuming the limit, and `idle` for what is open but consuming nothing. Page with `?limit=` (default 50, maximum 200) and `?offset=`. + +!!! info "Prerequisites" + This endpoint requires an API key with the `insight.evt.get` permission. See [API Keys](../7-administration/access/api-keys.md) for setup. The REST examples use `$LC_JWT` for your JWT and `$SEARCH_HOST` for your organization's search endpoint; see [Search API Endpoint](index.md#search-api-endpoint) for how to discover the host once and reuse it. + +=== "CLI" + + ```bash + limacharlie search queries # Everything this org has open + limacharlie search queries --state executing # Only what is using a slot + ``` + +=== "Python SDK" + + ```python + --8<-- "snippets/python/open_queries.py" + ``` + +=== "Go SDK" + + ```go + --8<-- "snippets/golang/open_queries/main.go" + ``` + +=== "REST" + + ```bash + # Who is using the limit right now, worst offender first. + curl -s "https://$SEARCH_HOST/v1/search/YOUR_OID/queries?state=executing" \ + -H "Authorization: Bearer $LC_JWT" \ + | jq '{slots: .slotsHeld, limit: .limit, open: .count, + queries: [.queries[] | {queryId, submittedBy, runningForMs, + progressPercent, eventsScanned, query}] + | sort_by(-.eventsScanned)}' + ``` + +The CLI prints the two numbers first and then one row per open query, which is enough to tell at a glance whether the limit is genuinely saturated or merely has queries parked against it: + +```text +1 of 10 concurrency slots in use; 2 search(es) open. +queryId state slot progress pages scanned running submittedBy +------------------------------------ --------- ------ ---------- ------- ----------- --------- ------------------- +9f1c0a7e-3d51-4c8a-9f2b-7d6e5a4b3c21 executing yes 37% 4 184,320,000 26s analyst@example.com +2b4d6f80-11ac-4e39-8a55-c0de1f2a3b44 idle no 2 41,200,000 dashboard-key +(1 more field hidden, use -W to show all or --output json for full data) +``` + +A representative response, with one query running and one parked between pages: + +```json +{ + "oid": "YOUR_OID", + "limit": 10, + "slotsHeld": 1, + "count": 2, + "truncated": false, + "queries": [ + { + "queryId": "9f1c0a7e-3d51-4c8a-9f2b-7d6e5a4b3c21", + "state": "executing", + "holdsSlot": true, + "page": "b7c19f42", + "query": "-24h | * | NEW_PROCESS | event/FILE_PATH contains 'powershell'", + "stream": "event", + "startTime": "1753500000", + "endTime": "1753586400", + "submittedBy": "analyst@example.com", + "userAgent": "limacharlie-python/5.5.1", + "submittedAt": "2026-07-26T18:31:04Z", + "startedAt": "2026-07-26T18:33:12Z", + "runningForMs": 26400, + "lastActivityAt": "2026-07-26T18:33:10Z", + "pagesCompleted": 4, + "hasMorePages": true, + "batchesCompleted": 740, + "batchesInScope": 2000, + "progressPercent": 37, + "eventsScanned": 184320000, + "billedEvents": 184320000, + "slotExpiresAt": "2026-07-26T18:38:12Z", + "resumableUntil": "2026-07-27T18:31:04Z" + }, + { + "queryId": "2b4d6f80-11ac-4e39-8a55-c0de1f2a3b44", + "state": "idle", + "holdsSlot": false, + "query": "-7d | plat == windows | WEL | event/EVENT/System/EventID == '4625'", + "stream": "event", + "startTime": "1752981600", + "endTime": "1753586400", + "submittedBy": "dashboard-key", + "userAgent": "curl/8.5.0", + "submittedAt": "2026-07-26T17:02:41Z", + "lastActivityAt": "2026-07-26T17:04:55Z", + "pagesCompleted": 2, + "batchesCompleted": 120, + "batchesInScope": 0, + "eventsScanned": 41200000, + "billedEvents": 39900000, + "resumableUntil": "2026-07-27T17:02:41Z" + } + ] +} +``` + +Two queries are open, but only the executing one holds a slot - which is why `slotsHeld` is `1` while `count` is `2`. The envelope also carries `limit` (your organization's concurrent-query limit) and `truncated`. `truncated` is computed against the unfiltered listing, so a request filtered to `idle` can legitimately report `true` alongside an empty `queries` array. + +What each query entry means: + +| Field | Meaning | +| --- | --- | +| `queryId` | Identifier of the query. This is what you pass to the cancel endpoint below. | +| `state` | `executing`, `queued`, `idle` or `unknown`, as described in the table above. | +| `holdsSlot` | Whether this query is counted against the concurrency limit. It is `null` **only** when `state` is `unknown`, which means slot tracking is unavailable rather than that the query holds no slot. An `idle` query is always `false`. | +| `page` | Which page holds the slot, for a paginated query past its first. A digest of the continuation token, never the token itself. | +| `query`, `stream`, `startTime`, `endTime` | Echoed back as submitted, and absent once the query's record has expired. A long query is shortened by the server and marked with a trailing `...`. | +| `submittedBy`, `userAgent` | The authenticated identity that submitted the query, and the client it arrived from. | +| `submittedAt`, `startedAt`, `lastActivityAt` | When the query was submitted, when the current page started (absent when nothing is running), and when the server last finished producing a page. `lastActivityAt` is absent until one has completed, and polling for results does not move it. | +| `runningForMs` | How long the current page has been running. Absent when nothing is running, so an idle query has no value here. | +| `pagesCompleted` | Pages successfully produced, not pages a client collected. A failed page is not counted, and a page still executing counts only once it completes. | +| `hasMorePages` | Whether more pages remain. **Absent until a page has completed**, because that is when the answer is known - absent means "not yet determined", not "no more pages". | +| `batchesCompleted`, `batchesInScope` | The progress pair, both advancing at page boundaries. **`batchesInScope` of `0` means the scope estimate was unavailable**, so progress cannot be computed - it does not mean no work has been done. | +| `progressPercent` | The pair above as a percentage, clamped to 0-100. Absent when there is no denominator to divide by. | +| `eventsScanned`, `billedEvents` | The total scanned so far and the charged portion of it, both cumulative across pages. In practice the most actionable pair: the query worth cancelling is usually the one that has scanned the most. | +| `slotExpiresAt` | When the concurrency slot is reclaimed if the query neither finishes nor is cancelled. Absent when no slot is held. | +| `resumableUntil` | When the query's state expires and it can no longer be resumed. Unrelated to `slotExpiresAt`, which is why they are separate fields. | + +To free a slot, cancel the query with `DELETE /v1/search/{queryId}`, using the `queryId` from the listing. Cancelling is available through the REST API only; there is no CLI command for it. + +```bash +curl -s -X DELETE "https://$SEARCH_HOST/v1/search/$QUERY_ID" \ + -H "Authorization: Bearer $LC_JWT" +``` + +!!! note + `progressPercent` advances at page boundaries, so a query that returns everything in one response - any aggregation, which does not paginate - reports `0` for its whole life and then leaves the listing. It is absent entirely when the scope estimate was unavailable, which means progress cannot be computed rather than that no work has been done. + +## Reading Your Limits Directly + +Rather than discovering a limit by hitting it, ask for it. `GET /v1/search/{oid}/limits` reports the limits actually in effect for your organization: how many queries may run at once, the shape of a page, how long results stay resumable, and any enforced execution deadlines. + +!!! info "Prerequisites" + This endpoint requires an API key with the `insight.evt.get` permission. See [API Keys](../7-administration/access/api-keys.md) for setup. The REST example uses `$LC_JWT` for your JWT and `$SEARCH_HOST` for your organization's search endpoint; see [Search API Endpoint](index.md#search-api-endpoint) for how to discover the host once and reuse it. + +=== "CLI" + + ```bash + limacharlie search limits + ``` + +=== "Python SDK" + + ```python + --8<-- "snippets/python/search_limits.py" + ``` + +=== "Go SDK" + + ```go + --8<-- "snippets/golang/search_limits/main.go" + ``` + +=== "REST" + + ```bash + curl -s "https://$SEARCH_HOST/v1/search/YOUR_OID/limits" \ + -H "Authorization: Bearer $LC_JWT" + ``` + +A representative response: + +```json +{ + "oid": "YOUR_OID", + "concurrency": { "maxConcurrentQueries": 10 }, + "pagination": { + "resultsPerPage": 200, + "maxPageDurationSeconds": 30, + "maxCursorBytes": 4096 + }, + "retention": { + "resumableForSeconds": 86400, + "pageResultsForSeconds": 900 + }, + "execution": { + "maxQueryDurationSeconds": 480, + "maxAggregationDurationSeconds": 540, + "maxResponseBytes": null + }, + "request": { "maxRequestBodyBytes": 102400 }, + "capabilities": { "openQueryListing": true } +} +``` + +What the groups mean: + +| Field | Meaning | +| --- | --- | +| `concurrency.maxConcurrentQueries` | How many queries may be **executing** at once. A paginated query parked between pages consumes nothing, so this is not a cap on how many you may have open. | +| `pagination.resultsPerPage` | Events returned per page before you get a continuation token. | +| `pagination.maxPageDurationSeconds` | How long the server spends on one page before returning what it has plus a token. Reaching it is normal for a broad query and does not mean the query failed. | +| `pagination.maxCursorBytes` | The largest continuation token the server accepts back when you ask for the next page. | +| `retention.resumableForSeconds` | How long a query can be resumed for, measured from when it was submitted and never extended by activity. Leave a paginated query paused longer than this and it cannot be continued. | +| `retention.pageResultsForSeconds` | How long a page's results are kept. Can be shorter than the above, in which case re-reading an older page recomputes it rather than failing - a latency characteristic, not a deadline. | +| `execution.maxQueryDurationSeconds` | The deadline for a non-aggregation query. | +| `execution.maxAggregationDurationSeconds` | The deadline for an aggregation, which gets its own budget because it cannot return partial pages. | +| `execution.maxResponseBytes` | The ceiling on a single response's accumulated size. The `null` above is the common case and means no ceiling is enforced, as described in the warning below. | +| `request.maxRequestBodyBytes` | The largest request body accepted, which in practice bounds query length and how many sensors you may name. | +| `capabilities.openQueryListing` | Whether the open-query listing above can report queries that are open but idle on your deployment. | + +!!! warning "`null` means unlimited, not zero" + A limit that is not enforced is reported as `null`, never `0`. In a document of limits a zero would read as "nothing allowed", which is the opposite. Treat `null` as "no limit applies", and treat a field you do not recognise as not applicable rather than as zero - the response is additive and gains fields over time. + ## Query Timeouts A single query has a maximum execution time of roughly **8 to 9 minutes**. If a query exceeds this deadline it returns an error rather than partial results. @@ -236,9 +471,23 @@ If an aggregation over a wide time range is slow or times out, break it into sma ### The query is rejected as too busy or times out -- **`HTTP 429` (too many concurrent queries).** You have reached the [concurrent-query limit](#concurrent-queries). Wait for an in-flight query to finish, then retry. +- **`HTTP 429` (too many concurrent queries).** You have reached the [concurrent-query limit](#concurrent-queries). Confirm what that limit actually is with `limacharlie search limits` (see [Reading Your Limits Directly](#reading-your-limits-directly)), then list what is holding the slots with `limacharlie search queries --state executing` (see [Seeing What Is Using Your Limit](#seeing-what-is-using-your-limit)). Cancelling is available through the REST API only - there is no CLI command for it - so free a slot with a `DELETE` against a `queryId` from that listing: + + ```bash + curl -s -X DELETE "https://$SEARCH_HOST/v1/search/$QUERY_ID" \ + -H "Authorization: Bearer $LC_JWT" + ``` + + If nothing in the listing looks unexpected, nothing needs cancelling: wait for an in-flight query to finish and retry. + - **Timeout.** Long-running aggregations over large ranges can hit the [query timeout](#query-timeouts). Narrow the range or split the query into smaller windows. +### A paginated query cannot be resumed + +The continuation token stopped working, or the query disappeared from the open-query listing. A query stays resumable for a fixed window measured from when it was **submitted**, not from its last page, so one left paused long enough expires even if it was producing pages recently. Read the window with `limacharlie search limits` (`retention.resumableForSeconds`) and page through faster than it, or re-run the query. + +This is distinct from `retention.pageResultsForSeconds`, which is shorter on some deployments: re-reading a page whose results have aged out recomputes it rather than failing, so that shows up as a slow page, not a broken one. + ## See Also - [LCQL Examples](lcql-examples.md) diff --git a/snippets/golang/go.mod b/snippets/golang/go.mod index 82c481842..7b6d63165 100644 --- a/snippets/golang/go.mod +++ b/snippets/golang/go.mod @@ -2,7 +2,7 @@ module github.com/refractionPOINT/documentation/snippets/golang go 1.25.9 -require github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20260706232626-4212dfffc585 +require github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20260726172822-e1078d02bc52 require ( cel.dev/expr v0.25.1 // indirect diff --git a/snippets/golang/go.sum b/snippets/golang/go.sum index f70bab1ba..3dd44570a 100644 --- a/snippets/golang/go.sum +++ b/snippets/golang/go.sum @@ -85,8 +85,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20260706232626-4212dfffc585 h1:Cmo1IowDLTBhZP/LDphh9/t5NreFXNKUHmwcAukRzfc= -github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20260706232626-4212dfffc585/go.mod h1:clcfnNYeCNJGU3VyYd8vKbGue52Vv4tsoDRpEkfRpws= +github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20260726172822-e1078d02bc52 h1:YqtdL4UVs0obgcWoYyD0/xFbFCo0DinGwjPiV4KV/j4= +github.com/refractionPOINT/go-limacharlie/limacharlie v0.0.0-20260726172822-e1078d02bc52/go.mod h1:6ukUHtDzOQm+3Cu5ysth6+m4PoW3/ImR3QkYXpYO/Gc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= diff --git a/snippets/golang/open_queries/main.go b/snippets/golang/open_queries/main.go new file mode 100644 index 000000000..05826ae00 --- /dev/null +++ b/snippets/golang/open_queries/main.go @@ -0,0 +1,62 @@ +// List the searches an organization currently has open. +// +// SlotsHeld is what the concurrent-query limit applies to. Count is everything +// the organization has open, including paginated searches parked between pages, +// which are resumable but consume no slot. +package main + +import ( + "context" + "fmt" + "sort" + + limacharlie "github.com/refractionPOINT/go-limacharlie/limacharlie" +) + +func main() { + org, err := limacharlie.NewOrganizationFromClientOptions(limacharlie.ClientOptions{ + OID: "YOUR_OID", + APIKey: "YOUR_API_KEY", + }, nil) + if err != nil { + panic(err) + } + + // State accepts "all" (the default), "executing" and "idle". + open, err := org.ListOpenQueries(limacharlie.OpenSearchQueriesFilters{State: "executing"}) + if err != nil { + panic(err) + } + fmt.Printf("%d of %d slots in use, %d search(es) open\n", + open.SlotsHeld, open.Limit, open.Count) + + // Biggest scanner first - usually the one worth cancelling. + sort.Slice(open.Queries, func(i, j int) bool { + return open.Queries[i].EventsScanned > open.Queries[j].EventsScanned + }) + for _, q := range open.Queries { + fmt.Printf("%s by=%s scanned=%d billed=%d\n", + q.QueryID, q.SubmittedBy, q.EventsScanned, q.BilledEvents) + } + + // ListAllOpenQueries walks the whole listing rather than one server page. + all, err := org.ListAllOpenQueries(context.Background(), "all") + if err != nil { + panic(err) + } + for _, q := range all { + // HoldsSlot is nil only when State is "unknown", which means slot + // tracking is unavailable - not that the search holds no slot. + slot := "unknown" + if q.HoldsSlot != nil { + slot = fmt.Sprintf("%t", *q.HoldsSlot) + } + // ProgressPercent is nil when there is no denominator to divide by. + progress := "unknown" + if q.ProgressPercent != nil { + progress = fmt.Sprintf("%.0f%%", *q.ProgressPercent) + } + fmt.Printf("%s state=%s holdsSlot=%s pages=%d progress=%s\n", + q.QueryID, q.State, slot, q.PagesCompleted, progress) + } +} diff --git a/snippets/golang/search_limits/main.go b/snippets/golang/search_limits/main.go new file mode 100644 index 000000000..a33ebd029 --- /dev/null +++ b/snippets/golang/search_limits/main.go @@ -0,0 +1,48 @@ +// Read an organization's resolved search limits. +// +// The execution limits are pointers because a limit that is not enforced is +// nil, never 0: in a set of limits a zero would read as "nothing allowed". +package main + +import ( + "fmt" + + limacharlie "github.com/refractionPOINT/go-limacharlie/limacharlie" +) + +func main() { + org, err := limacharlie.NewOrganizationFromClientOptions(limacharlie.ClientOptions{ + OID: "YOUR_OID", + APIKey: "YOUR_API_KEY", + }, nil) + if err != nil { + panic(err) + } + + limits, err := org.GetSearchLimits() + if err != nil { + panic(err) + } + + fmt.Printf("concurrent queries: %d\n", limits.Concurrency.MaxConcurrentQueries) + fmt.Printf("results per page: %d\n", limits.Pagination.ResultsPerPage) + fmt.Printf("max page duration: %ds\n", limits.Pagination.MaxPageDurationSeconds) + fmt.Printf("max cursor bytes: %d\n", limits.Pagination.MaxCursorBytes) + fmt.Printf("resumable for: %ds\n", limits.Retention.ResumableForSeconds) + fmt.Printf("page results kept: %ds\n", limits.Retention.PageResultsForSeconds) + fmt.Printf("max request body: %d bytes\n", limits.Request.MaxRequestBodyBytes) + fmt.Printf("open-query listing: %t\n", limits.Capabilities.OpenQueryListing) + + // Nil is "no limit applies". Dereferencing without the check panics, and + // reading a zero value as the limit gets the contract exactly backwards. + if d := limits.Execution.MaxQueryDurationSeconds; d != nil { + fmt.Printf("query duration: cut off after %ds\n", *d) + } else { + fmt.Println("query duration: not enforced") + } + if n := limits.Execution.MaxResponseBytes; n != nil { + fmt.Printf("max response: %d bytes\n", *n) + } else { + fmt.Println("max response: not enforced") + } +} diff --git a/snippets/python/open_queries.py b/snippets/python/open_queries.py new file mode 100644 index 000000000..0856f0777 --- /dev/null +++ b/snippets/python/open_queries.py @@ -0,0 +1,34 @@ +"""List the searches an organization currently has open. + +``slotsHeld`` is what the concurrent-query limit applies to. ``count`` is +everything the organization has open, including paginated searches parked +between pages, which are resumable but consume no slot. +""" + +from limacharlie.client import Client +from limacharlie.sdk.organization import Organization +from limacharlie.sdk.search import Search + +client = Client(oid="YOUR_OID", api_key="YOUR_API_KEY") +org = Organization(client) +search = Search(org) + +# state defaults to "all"; "executing" and "idle" are the other choices. +listing = search.list_open_queries(state="all") +print( + f"{listing['slotsHeld']} of {listing['limit']} slots in use, " + f"{listing['count']} search(es) open" +) + +# Only what is consuming the limit, biggest scanner first - usually the one +# worth cancelling. +executing = search.list_open_queries(state="executing") +for entry in sorted(executing["queries"], key=lambda q: q["eventsScanned"], reverse=True): + print(entry["queryId"], entry["submittedBy"], entry["eventsScanned"], entry["query"]) + +# iter_open_queries walks every page, so it is not capped at one server page. +for entry in search.iter_open_queries(state="all"): + # progressPercent is absent when the scope estimate was unavailable, which + # means progress cannot be computed rather than that nothing has been done. + progress = entry.get("progressPercent") + print(entry["queryId"], entry["state"], "unknown" if progress is None else f"{progress:.0f}%") diff --git a/snippets/python/search_limits.py b/snippets/python/search_limits.py new file mode 100644 index 000000000..04557f049 --- /dev/null +++ b/snippets/python/search_limits.py @@ -0,0 +1,30 @@ +"""Read an organization's resolved search limits. + +Every value here is otherwise discoverable only by hitting it, so read it once +and size the client to it. A limit that is not enforced comes back as ``None``, +never ``0`` - in a document of limits a zero would read as "nothing allowed". +""" + +from limacharlie.client import Client +from limacharlie.sdk.organization import Organization +from limacharlie.sdk.search import Search + +client = Client(oid="YOUR_OID", api_key="YOUR_API_KEY") +org = Organization(client) + +limits = Search(org).get_limits() + +print(f"concurrent queries: {limits['concurrency']['maxConcurrentQueries']}") +print(f"results per page: {limits['pagination']['resultsPerPage']}") +print(f"max page duration: {limits['pagination']['maxPageDurationSeconds']}s") +print(f"resumable for: {limits['retention']['resumableForSeconds']}s") +print(f"page results kept: {limits['retention']['pageResultsForSeconds']}s") +print(f"open-query listing: {limits['capabilities']['openQueryListing']}") + +# None means the limit is not enforced, so check for it rather than treating a +# falsy value as "no time allowed". +max_query_seconds = limits["execution"]["maxQueryDurationSeconds"] +if max_query_seconds is None: + print("query duration: not enforced") +else: + print(f"query duration: cut off after {max_query_seconds}s")