From cdbefb26ba276ab0016c19a4b9121dedcc8c1fcd Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Thu, 2 Jul 2026 21:50:46 -0700 Subject: [PATCH] Warn when AT ALL drops ungrouped filters --- include/yardstick_ffi.h | 1 + src/yardstick_extension.cpp | 420 +- test/sql/measures.test | 435 ++ yardstick-rs/src/ffi.rs | 10 + yardstick-rs/src/sql/measures.rs | 10335 +++++++++++++++++------------ 5 files changed, 7069 insertions(+), 4132 deletions(-) diff --git a/include/yardstick_ffi.h b/include/yardstick_ffi.h index d9b14df..f524cfa 100644 --- a/include/yardstick_ffi.h +++ b/include/yardstick_ffi.h @@ -313,6 +313,7 @@ struct YardstickAggregateResult { bool had_aggregate; char* expanded_sql; char* error; + char* warnings; }; struct YardstickMeasureAggResult { diff --git a/src/yardstick_extension.cpp b/src/yardstick_extension.cpp index f830749..0c779fd 100644 --- a/src/yardstick_extension.cpp +++ b/src/yardstick_extension.cpp @@ -228,6 +228,9 @@ static std::string RewritePercentileWithinGroup(const std::string &sql) { // TABLE FUNCTION: yardstick(sql) - Execute SQL with AGGREGATE() expansion //============================================================================= +static std::string AggregateWarnings(YardstickAggregateResult &result); +static void HandleAggregateWarnings(ClientContext &context, const string &warnings); + struct YardstickQueryData : public TableFunctionData { string original_sql; string rewritten_sql; @@ -241,6 +244,9 @@ static unique_ptr YardstickQueryBind(ClientContext &context, vector &names) { auto data = make_uniq(); data->original_sql = input.inputs[0].GetValue(); + if (input.inputs.size() > 1) { + HandleAggregateWarnings(context, input.inputs[1].GetValue()); + } // Rewrite the SQL using our Rust code if (yardstick_has_aggregate(data->original_sql.c_str())) { @@ -252,6 +258,7 @@ static unique_ptr YardstickQueryBind(ClientContext &context, } if (result.had_aggregate) { data->rewritten_sql = string(result.expanded_sql); + HandleAggregateWarnings(context, AggregateWarnings(result)); } else { data->rewritten_sql = data->original_sql; } @@ -304,6 +311,18 @@ static void YardstickQueryFunction(ClientContext &context, TableFunctionInput &d } } +static void YardstickWarningFunction(DataChunk &input, ExpressionState &state, Vector &result) { + if (input.ColumnCount() > 0 && input.size() > 0) { + auto warnings = input.data[0].GetValue(0); + if (!warnings.IsNull()) { + HandleAggregateWarnings(state.GetContext(), warnings.GetValue()); + } + } + result.SetVectorType(VectorType::CONSTANT_VECTOR); + ConstantVector::SetNull(result, false); + ConstantVector::GetData(result)[0] = true; +} + //============================================================================= // PARSER EXTENSION //============================================================================= @@ -1118,8 +1137,43 @@ static string EscapeSqlStringLiteral(const string &sql) { return escaped_sql; } +static std::string AggregateWarnings(YardstickAggregateResult &result) { + return result.warnings ? string(result.warnings) : string(); +} + +static bool WarningsAsErrors(ClientContext &context) { + Value setting; + if (!context.TryGetCurrentSetting("warnings_as_errors", setting) || setting.IsNull()) { + return false; + } + return setting.GetValue(); +} + +static void HandleAggregateWarnings(ClientContext &context, const string &warnings) { + if (warnings.empty()) { + return; + } + if (WarningsAsErrors(context)) { + throw InvalidInputException("%s", warnings); + } + DUCKDB_LOG_WARNING(context, "%s", warnings.c_str()); +} + +static string YardstickWrapperSql(const string &expanded_sql, const string &warnings) { + string wrapper_sql = "SELECT * FROM yardstick('" + EscapeSqlStringLiteral(expanded_sql) + "'"; + if (!warnings.empty()) { + wrapper_sql += ", '" + EscapeSqlStringLiteral(warnings) + "'"; + } + wrapper_sql += ")"; + return wrapper_sql; +} + static string WrapYardstickSelect(const string &sql) { - return "SELECT * FROM yardstick('" + EscapeSqlStringLiteral(sql) + "')"; + return YardstickWrapperSql(sql, ""); +} + +static string WarningStatementSql(const string &warnings) { + return "SELECT yardstick_warning('" + EscapeSqlStringLiteral(warnings) + "')"; } static bool ParsesAsSingleSelect(const string &sql) { @@ -1133,6 +1187,314 @@ static bool ParsesAsSingleSelect(const string &sql) { parser.statements[0]->type == StatementType::SELECT_STATEMENT; } +static bool IsKeywordBoundary(char c) { + return !std::isalnum(static_cast(c)) && c != '_'; +} + +static idx_t FindTopLevelKeyword(const string &sql, const string &keyword, idx_t start = 0) { + idx_t depth = 0; + for (idx_t i = start; i < sql.size(); i++) { + char c = sql[i]; + if (c == '\'') { + i++; + while (i < sql.size()) { + if (sql[i] == '\'' && i + 1 < sql.size() && sql[i + 1] == '\'') { + i += 2; + continue; + } + if (sql[i] == '\'') { + break; + } + i++; + } + continue; + } + if (c == '"') { + i++; + while (i < sql.size()) { + if (sql[i] == '"' && i + 1 < sql.size() && sql[i + 1] == '"') { + i += 2; + continue; + } + if (sql[i] == '"') { + break; + } + i++; + } + continue; + } + if (c == '-' && i + 1 < sql.size() && sql[i + 1] == '-') { + i += 2; + while (i < sql.size() && sql[i] != '\n' && sql[i] != '\r') { + i++; + } + continue; + } + if (c == '/' && i + 1 < sql.size() && sql[i + 1] == '*') { + i += 2; + while (i + 1 < sql.size() && !(sql[i] == '*' && sql[i + 1] == '/')) { + i++; + } + if (i + 1 < sql.size()) { + i++; + } + continue; + } + if (c == '(') { + depth++; + continue; + } + if (c == ')' && depth > 0) { + depth--; + continue; + } + if (depth != 0 || i + keyword.size() > sql.size()) { + continue; + } + bool matched = true; + for (idx_t j = 0; j < keyword.size(); j++) { + if (std::toupper(static_cast(sql[i + j])) != keyword[j]) { + matched = false; + break; + } + } + if (!matched) { + continue; + } + bool left_ok = i == 0 || IsKeywordBoundary(sql[i - 1]); + bool right_ok = i + keyword.size() >= sql.size() || IsKeywordBoundary(sql[i + keyword.size()]); + if (left_ok && right_ok) { + return i; + } + } + return std::string::npos; +} + +static idx_t SkipSqlWhitespace(const string &sql, idx_t i) { + while (i < sql.size() && std::isspace(static_cast(sql[i]))) { + i++; + } + return i; +} + +static idx_t SkipSqlWhitespaceAndComments(const string &sql, idx_t i) { + while (i < sql.size()) { + i = SkipSqlWhitespace(sql, i); + if (i + 1 < sql.size() && sql[i] == '-' && sql[i + 1] == '-') { + i += 2; + while (i < sql.size() && sql[i] != '\n' && sql[i] != '\r') { + i++; + } + continue; + } + if (i + 1 < sql.size() && sql[i] == '/' && sql[i + 1] == '*') { + i += 2; + while (i + 1 < sql.size() && !(sql[i] == '*' && sql[i + 1] == '/')) { + i++; + } + if (i + 1 < sql.size()) { + i += 2; + } + continue; + } + return i; + } + return i; +} + +static bool KeywordAt(const string &sql, idx_t pos, const string &keyword) { + if (pos + keyword.size() > sql.size()) { + return false; + } + for (idx_t i = 0; i < keyword.size(); i++) { + if (std::toupper(static_cast(sql[pos + i])) != keyword[i]) { + return false; + } + } + bool left_ok = pos == 0 || IsKeywordBoundary(sql[pos - 1]); + bool right_ok = pos + keyword.size() >= sql.size() || IsKeywordBoundary(sql[pos + keyword.size()]); + return left_ok && right_ok; +} + +static string StripTrailingSemicolon(string sql) { + StringUtil::RTrim(sql); + if (!sql.empty() && sql.back() == ';') { + sql.pop_back(); + StringUtil::RTrim(sql); + } + return sql; +} + +static idx_t FindMatchingParen(const string &sql, idx_t open_pos) { + if (open_pos >= sql.size() || sql[open_pos] != '(') { + return std::string::npos; + } + idx_t depth = 0; + for (idx_t i = open_pos; i < sql.size(); i++) { + char c = sql[i]; + if (c == '\'') { + i++; + while (i < sql.size()) { + if (sql[i] == '\'' && i + 1 < sql.size() && sql[i + 1] == '\'') { + i += 2; + continue; + } + if (sql[i] == '\'') { + break; + } + i++; + } + continue; + } + if (c == '"') { + i++; + while (i < sql.size()) { + if (sql[i] == '"' && i + 1 < sql.size() && sql[i + 1] == '"') { + i += 2; + continue; + } + if (sql[i] == '"') { + break; + } + i++; + } + continue; + } + if (c == '-' && i + 1 < sql.size() && sql[i + 1] == '-') { + i += 2; + while (i < sql.size() && sql[i] != '\n' && sql[i] != '\r') { + i++; + } + continue; + } + if (c == '/' && i + 1 < sql.size() && sql[i + 1] == '*') { + i += 2; + while (i + 1 < sql.size() && !(sql[i] == '*' && sql[i + 1] == '/')) { + i++; + } + if (i + 1 < sql.size()) { + i++; + } + continue; + } + if (c == '(') { + depth++; + } else if (c == ')') { + depth--; + if (depth == 0) { + return i; + } + } + } + return std::string::npos; +} + +static bool IsWrappedInParens(const string &sql) { + idx_t start = SkipSqlWhitespace(sql, 0); + if (start >= sql.size() || sql[start] != '(') { + return false; + } + idx_t close = FindMatchingParen(sql, start); + if (close == std::string::npos) { + return false; + } + idx_t tail = SkipSqlWhitespace(sql, close + 1); + return tail == sql.size() || sql[tail] == ';'; +} + +static string UnwrapQueryBody(string sql) { + sql = StripTrailingSemicolon(sql); + while (IsWrappedInParens(sql)) { + idx_t start = SkipSqlWhitespace(sql, 0); + idx_t close = FindMatchingParen(sql, start); + sql = sql.substr(start + 1, close - start - 1); + sql = StripTrailingSemicolon(sql); + } + return sql; +} + +static string WarningWrappedSelectSql(const string &query_sql, const string &warnings) { + string query = UnwrapQueryBody(query_sql); + return "WITH __yardstick_warning AS MATERIALIZED (" + WarningStatementSql(warnings) + + "), __yardstick_query AS MATERIALIZED (\n" + query + + "\n) SELECT __yardstick_query.* FROM __yardstick_warning CROSS JOIN __yardstick_query"; +} + +static idx_t FindParenthesizedQueryBodyAfter(const string &sql, idx_t start) { + idx_t i = start; + while (i < sql.size()) { + i = SkipSqlWhitespace(sql, i); + if (i >= sql.size()) { + return std::string::npos; + } + if (sql[i] == '(') { + auto close = FindMatchingParen(sql, i); + if (close == std::string::npos) { + return std::string::npos; + } + auto body_start = SkipSqlWhitespaceAndComments(sql, i + 1); + if (KeywordAt(sql, body_start, "SELECT") || KeywordAt(sql, body_start, "WITH")) { + return i; + } + i = close + 1; + continue; + } + i++; + } + return std::string::npos; +} + +static bool TryWarningWrappedNonSelectSql(const string &expanded_sql, + const string &warnings, + string &rewritten_sql) { + string trimmed = expanded_sql; + StringUtil::LTrim(trimmed); + string upper = StringUtil::Upper(trimmed); + + auto insert_pos = FindTopLevelKeyword(expanded_sql, "INSERT"); + if (insert_pos != std::string::npos) { + auto select_pos = FindTopLevelKeyword(expanded_sql, "SELECT", insert_pos + strlen("INSERT")); + auto with_pos = FindTopLevelKeyword(expanded_sql, "WITH", insert_pos + strlen("INSERT")); + idx_t body_pos = std::string::npos; + if (select_pos != std::string::npos && with_pos != std::string::npos) { + body_pos = select_pos < with_pos ? select_pos : with_pos; + } else if (select_pos != std::string::npos) { + body_pos = select_pos; + } else { + body_pos = with_pos; + } + if (body_pos != std::string::npos) { + rewritten_sql = expanded_sql.substr(0, body_pos) + + WarningWrappedSelectSql(expanded_sql.substr(body_pos), warnings); + return true; + } + body_pos = FindParenthesizedQueryBodyAfter(expanded_sql, insert_pos + strlen("INSERT")); + if (body_pos != std::string::npos) { + rewritten_sql = expanded_sql.substr(0, body_pos) + + WarningWrappedSelectSql(expanded_sql.substr(body_pos), warnings); + return true; + } + } + + auto create_pos = FindTopLevelKeyword(expanded_sql, "CREATE"); + auto table_pos = create_pos == std::string::npos + ? std::string::npos + : FindTopLevelKeyword(expanded_sql, "TABLE", create_pos + strlen("CREATE")); + auto as_pos = table_pos == std::string::npos + ? std::string::npos + : FindTopLevelKeyword(expanded_sql, "AS", table_pos + strlen("TABLE")); + if (as_pos != std::string::npos && !StringUtil::StartsWith(upper, "CREATE VIEW")) { + auto body_pos = SkipSqlWhitespace(expanded_sql, as_pos + strlen("AS")); + if (body_pos < expanded_sql.size()) { + rewritten_sql = expanded_sql.substr(0, as_pos + strlen("AS")) + " " + + WarningWrappedSelectSql(expanded_sql.substr(body_pos), warnings); + return true; + } + } + + return false; +} + static MeasureRewriteResult RewriteMeasureViewsStatementByStatement( const std::string &query, std::vector &permanent_snapshots) { @@ -1339,6 +1701,7 @@ static MeasureRewriteResult RewriteMeasureViewsStatementByStatement( } } string expanded_sql(result.expanded_sql); + string warnings = AggregateWarnings(result); if (ParsesAsSingleSelect(expanded_sql)) { if (reads_pending_temporary_view) { rewrite_result.error = "TEMPORARY AS MEASURE views cannot be returned directly from a statement batch"; @@ -1347,9 +1710,18 @@ static MeasureRewriteResult RewriteMeasureViewsStatementByStatement( cleanup_temporary_measure_views(); return rewrite_result; } - rewritten_statements.push_back(WrapYardstickSelect(expanded_sql)); + rewritten_statements.push_back(YardstickWrapperSql(expanded_sql, warnings)); } else { - rewritten_statements.push_back(expanded_sql); + if (!warnings.empty()) { + string warning_sql; + if (TryWarningWrappedNonSelectSql(expanded_sql, warnings, warning_sql)) { + rewritten_statements.push_back(warning_sql); + } else { + rewritten_statements.push_back(WarningStatementSql(warnings) + ";\n" + expanded_sql); + } + } else { + rewritten_statements.push_back(expanded_sql); + } } } else { rewritten_statements.push_back(statement); @@ -1433,10 +1805,11 @@ ParserExtensionParseResult yardstick_parse(ParserExtensionInfo *, if (result.had_aggregate) { string expanded_sql(result.expanded_sql); + string warnings = AggregateWarnings(result); yardstick_free_aggregate_result(result); // Wrap in table function call - string wrapper_sql = WrapYardstickSelect(expanded_sql); + string wrapper_sql = YardstickWrapperSql(expanded_sql, warnings); Parser parser; parser.ParseQuery(wrapper_sql); @@ -1538,6 +1911,7 @@ ParserOverrideResult yardstick_parser_override(ParserExtensionInfo *, if (result.had_aggregate) { string expanded_sql(result.expanded_sql); + string warnings = AggregateWarnings(result); yardstick_free_aggregate_result(result); // Validate the expanded SQL parses. If expansion produced garbage @@ -1559,13 +1933,27 @@ ParserOverrideResult yardstick_parser_override(ParserExtensionInfo *, validation_parser.statements[0]->type == StatementType::SELECT_STATEMENT; if (is_select) { - string wrapper_sql = WrapYardstickSelect(expanded_sql); + string wrapper_sql = YardstickWrapperSql(expanded_sql, warnings); Parser parser; parser.ParseQuery(wrapper_sql); RestoreMeasureViewSnapshots(permanent_snapshots); return ParserOverrideResult(std::move(parser.statements)); } + if (!warnings.empty()) { + string warning_sql; + if (TryWarningWrappedNonSelectSql(expanded_sql, warnings, warning_sql)) { + Parser parser; + parser.ParseQuery(warning_sql); + RestoreMeasureViewSnapshots(permanent_snapshots); + return ParserOverrideResult(std::move(parser.statements)); + } + Parser parser; + parser.ParseQuery(WarningStatementSql(warnings) + "; " + expanded_sql); + RestoreMeasureViewSnapshots(permanent_snapshots); + return ParserOverrideResult(std::move(parser.statements)); + } + RestoreMeasureViewSnapshots(permanent_snapshots); return ParserOverrideResult(std::move(validation_parser.statements)); } @@ -1640,20 +2028,11 @@ BoundStatement yardstick_bind(ClientContext &context, Binder &binder, if (result.had_aggregate) { string expanded_sql(result.expanded_sql); + string warnings = AggregateWarnings(result); yardstick_free_aggregate_result(result); - // Escape single quotes for embedding in string literal - string escaped_sql; - for (char c : expanded_sql) { - if (c == '\'') { - escaped_sql += "''"; - } else { - escaped_sql += c; - } - } - // Rebind through table function so rewritten SQL executes with normal planning - string wrapper_sql = "SELECT * FROM yardstick('" + escaped_sql + "')"; + string wrapper_sql = YardstickWrapperSql(expanded_sql, warnings); Parser parser; parser.ParseQuery(wrapper_sql); auto statements = std::move(parser.statements); @@ -1724,6 +2103,15 @@ static void LoadInternal(ExtensionLoader &loader) { TableFunction query_func("yardstick", {LogicalType::VARCHAR}, YardstickQueryFunction, YardstickQueryBind); loader.RegisterFunction(query_func); + TableFunction query_func_with_warnings("yardstick", {LogicalType::VARCHAR, LogicalType::VARCHAR}, + YardstickQueryFunction, YardstickQueryBind); + loader.RegisterFunction(query_func_with_warnings); + + ScalarFunction warning_func("yardstick_warning", {LogicalType::VARCHAR}, LogicalType::BOOLEAN, + YardstickWarningFunction); + warning_func.SetVolatile(); + warning_func.SetFallible(); + loader.RegisterFunction(warning_func); } void YardstickExtension::Load(ExtensionLoader &loader) { diff --git a/test/sql/measures.test b/test/sql/measures.test index 4119a0e..50e22d0 100644 --- a/test/sql/measures.test +++ b/test/sql/measures.test @@ -728,6 +728,414 @@ FROM sales_v; 2023 EU 225.0 2023 US 225.0 +# AT (ALL dim) drops outer WHERE filters on dimensions that are not grouped. +# Surface a warning through DuckDB; warnings_as_errors makes this test assert it is emitted. +statement ok +SET enable_logging = true; + +statement ok +SET warnings_as_errors = true; + +statement error +SEMANTIC SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023; +---- + +statement error +SELECT * FROM yardstick('SELECT region, AGGREGATE(revenue) AT (ALL region) AS pct_of_total FROM sales_v WHERE year = 2023 GROUP BY region'); +---- + +statement error +SELECT region, AGGREGATE(revenue) AT (ALL) AS grand_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +SELECT region, AGGREGATE(revenue) AT (ALL) AT (VISIBLE) AS grand_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +SELECT region, AGGREGATE(revenue) AT (WHERE year = 2023) AT (ALL) AS grand_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +SELECT region, AGGREGATE(revenue) AT (ALL) AT (WHERE year = 2023) AS grand_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +SELECT region, AGGREGATE(revenue) AT (ALL) AT (SET year = 2023) AS grand_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +WITH cte_warning AS ( + SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total + FROM sales_v + WHERE year = 2023 + GROUP BY region +) +SELECT * FROM cte_warning; +---- + +statement error +CREATE VIEW warning_error_view AS +SELECT region, AGGREGATE(revenue) AT (ALL region) AS year_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +query TR rowsort +SELECT region, AGGREGATE(revenue) AT (ALL region) AS all_region_revenue +FROM sales_v +WHERE EXISTS (/* calendar */ SELECT 1 FROM sales_v c WHERE c.year = 2023) +GROUP BY region; +---- +EU 375.0 +US 375.0 + +statement ok +CREATE TABLE warning_calendar (year INT); + +statement ok +INSERT INTO warning_calendar VALUES (2023); + +query TR rowsort +SELECT s.region, AGGREGATE(revenue) AT (ALL region) AS all_region_revenue +FROM sales_v s JOIN warning_calendar c ON TRUE +WHERE c.year = 2023 +GROUP BY s.region; +---- +EU 375.0 +US 375.0 + +query TR +SELECT region, ROUND(100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region WHERE year = 2023), 1) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region +ORDER BY region; +---- +EU 33.3 +US 66.7 + +query TR +SELECT region, ROUND(100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region SET year = 2023), 1) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region +ORDER BY region; +---- +EU 33.3 +US 66.7 + +query TR +SELECT region, AGGREGATE(revenue) AT (ALL year) AT (SET year = 2023) AS region_total +FROM sales_v +WHERE year = 2023 +GROUP BY region +ORDER BY region; +---- +EU 75.0 +US 150.0 + +query TR +SELECT region, ROUND(100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region VISIBLE), 1) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region +ORDER BY region; +---- +EU 33.3 +US 66.7 + +statement error +SELECT region, AGGREGATE(revenue) AT (ALL region) AT (WHERE region = 'US') AT (VISIBLE) AS region_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +SELECT region, AGGREGATE(revenue) AT (ALL region VISIBLE SET region = CURRENT(region)) AS region_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement ok +CREATE TABLE alias_warning_sales (fiscal_year INT, region TEXT, amount DOUBLE); + +statement ok +INSERT INTO alias_warning_sales VALUES + (2023, 'US', 150), (2023, 'EU', 75), + (2024, 'US', 300), (2024, 'EU', 100); + +statement ok +CREATE VIEW alias_warning_sales_v AS +SELECT fiscal_year AS year, region, SUM(amount) AS MEASURE alias_warning_revenue +FROM alias_warning_sales; + +statement error +SELECT region, AGGREGATE(alias_warning_revenue) AT (ALL region) AS pct_of_total +FROM alias_warning_sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement ok +CREATE TABLE date_warning_sales (date DATE, region TEXT, amount DOUBLE); + +statement ok +INSERT INTO date_warning_sales VALUES + (DATE '2023-01-01', 'US', 150), (DATE '2023-01-01', 'EU', 75), + (DATE '2024-01-01', 'US', 300), (DATE '2024-01-01', 'EU', 100); + +statement ok +CREATE VIEW date_warning_sales_v AS +SELECT date, region, SUM(amount) AS MEASURE date_warning_revenue +FROM date_warning_sales; + +statement error +SELECT region, AGGREGATE(date_warning_revenue) AT (ALL region) AS pct_of_total +FROM date_warning_sales_v +WHERE date = DATE '2023-01-01' +GROUP BY region; +---- + +statement ok +CREATE TABLE warning_cast_calendar (dt TIMESTAMP); + +statement ok +INSERT INTO warning_cast_calendar VALUES (TIMESTAMP '2023-01-01 12:00:00'); + +query TR rowsort +SELECT s.region, AGGREGATE(date_warning_revenue) AT (ALL region) AS all_region_revenue +FROM date_warning_sales_v s JOIN warning_cast_calendar c ON TRUE +WHERE CAST(c.dt AS DATE) = DATE '2023-01-01' +GROUP BY s.region; +---- +EU 625.0 +US 625.0 + +statement error +SEMANTIC SELECT year, region, AGGREGATE(revenue) AT (ALL year) AS region_total +FROM sales_v +WHERE year = 2023 +GROUP BY year, region; +---- + +statement error +SELECT year, region, AGGREGATE(revenue) AT (SET year = 2023) AT (ALL year) AS region_total +FROM sales_v +WHERE year = 2023 +GROUP BY year, region; +---- + +statement error +CREATE TABLE ctas_warning_result AS +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +CREATE TABLE ctas_warning_parenthesized_result AS ( +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region +); +---- + +statement error +CREATE TABLE ctas_warning_parenthesized_cte_result AS ( +WITH cte_warning_parenthesized AS ( + SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total + FROM sales_v + WHERE year = 2023 + GROUP BY region +) +SELECT * FROM cte_warning_parenthesized +); +---- + +statement error +CREATE TABLE ctas_warning_empty_result AS +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 9999 +GROUP BY region; +---- + +statement ok +CREATE TABLE warning_insert_target (region TEXT, pct_of_total DOUBLE); + +statement error +INSERT INTO warning_insert_target +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; +---- + +statement error +INSERT INTO warning_insert_target BY NAME ( +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region +); +---- + +statement ok +SET warnings_as_errors = false; + +statement ok +PREPARE warning_insert_stmt AS +INSERT INTO warning_insert_target +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; + +statement ok +EXECUTE warning_insert_stmt; + +statement ok +DEALLOCATE warning_insert_stmt; + +query I +SELECT COUNT(*) FROM warning_insert_target; +---- +2 + +statement ok +CREATE TABLE warning_insert_by_name_target (region TEXT, pct_of_total DOUBLE); + +statement ok +PREPARE warning_insert_parenthesized_stmt AS +INSERT INTO warning_insert_by_name_target BY NAME ( +/* leading query comment */ +SELECT region, 100.0 * AGGREGATE(date_warning_revenue) / AGGREGATE(date_warning_revenue) AT (ALL region) AS pct_of_total +FROM date_warning_sales_v +WHERE date = DATE '2023-01-01' +GROUP BY region +); + +statement ok +EXECUTE warning_insert_parenthesized_stmt; + +statement ok +DEALLOCATE warning_insert_parenthesized_stmt; + +query I +SELECT COUNT(*) FROM warning_insert_by_name_target; +---- +2 + +statement ok +CREATE TABLE ctas_warning_cte_result AS +WITH cte_warning AS ( + SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total + FROM sales_v + WHERE year = 2023 + GROUP BY region +) +SELECT * FROM cte_warning; + +query I +SELECT COUNT(*) FROM ctas_warning_cte_result; +---- +2 + +statement ok +CREATE TEMP TABLE warning_temp_sales (year INT, region TEXT, amount DOUBLE); + +statement ok +INSERT INTO warning_temp_sales VALUES + (2023, 'US', 150), (2023, 'EU', 75), + (2024, 'US', 300), (2024, 'EU', 100); + +statement ok +CREATE TEMP VIEW warning_temp_sales_v AS +SELECT year, region, SUM(amount) AS MEASURE warning_temp_revenue +FROM warning_temp_sales; +CREATE TABLE ctas_warning_temp_result AS +SELECT region, 100.0 * AGGREGATE(warning_temp_revenue) / AGGREGATE(warning_temp_revenue) AT (ALL region) AS pct_of_total +FROM warning_temp_sales_v +WHERE year = 2023 +GROUP BY region; + +query I +SELECT COUNT(*) FROM ctas_warning_temp_result; +---- +2 + +statement ok +CREATE TABLE ctas_warning_comment_result AS -- SELECT ignored by warning wrapper +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; + +query I +SELECT COUNT(*) FROM ctas_warning_comment_result; +---- +2 + +statement ok +CREATE TABLE ctas_warning_trailing_comment_result AS +SELECT region, 100.0 * AGGREGATE(revenue) / AGGREGATE(revenue) AT (ALL region) AS pct_of_total +FROM sales_v +WHERE year = 2023 +GROUP BY region -- warning wrapper should not append after this comment +; + +query I +SELECT COUNT(*) FROM ctas_warning_trailing_comment_result; +---- +2 + +statement ok +CREATE VIEW warning_persist_view AS +SELECT region, AGGREGATE(revenue) AT (ALL region) AS year_total +FROM sales_v +WHERE year = 2023 +GROUP BY region; + +statement ok +SET warnings_as_errors = true; + +query I +SELECT COUNT(*) FROM warning_persist_view; +---- +2 + +statement ok +SET warnings_as_errors = false; + +statement ok +DROP TABLE ctas_warning_cte_result; + +statement ok +DROP TABLE warning_insert_target; + # ORDER BY expression referencing a named aggregate that expands to a subquery (#28) query IIRR SEMANTIC SELECT @@ -1612,6 +2020,20 @@ CREATE VIEW daily_orders_v AS SELECT order_date, SUM(amount) AS MEASURE revenue FROM daily_orders; +statement ok +SET warnings_as_errors = true; + +query IR +SEMANTIC SELECT MONTH(order_date), AGGREGATE(revenue) AT (ALL MONTH(order_date) SET MONTH(order_date) = 2) AS feb_revenue +FROM daily_orders_v +WHERE MONTH(order_date) = 2 +GROUP BY MONTH(order_date); +---- +2 320.0 + +statement ok +SET warnings_as_errors = false; + # SET with expression dimension: fix to a specific month # This generates WHERE MONTH(_inner.order_date) = 2 query IRR rowsort @@ -2246,6 +2668,19 @@ CREATE VIEW dated_sales_v AS SELECT sale_date, SUM(amount) AS MEASURE revenue FROM dated_sales; +statement ok +SET warnings_as_errors = true; + +statement error +SELECT DATE_TRUNC('month', sale_date) AS month, AGGREGATE(revenue) AT (ALL) AS total +FROM dated_sales_v +WHERE sale_date >= DATE '2023-01-15' +GROUP BY DATE_TRUNC('month', sale_date); +---- + +statement ok +SET warnings_as_errors = false; + # YEAR() and MONTH() in GROUP BY with AGGREGATE query IIR rowsort SEMANTIC SELECT YEAR(sale_date), MONTH(sale_date), AGGREGATE(revenue) diff --git a/yardstick-rs/src/ffi.rs b/yardstick-rs/src/ffi.rs index b9f0663..d410a34 100644 --- a/yardstick-rs/src/ffi.rs +++ b/yardstick-rs/src/ffi.rs @@ -40,6 +40,8 @@ pub struct YardstickAggregateResult { pub expanded_sql: *mut c_char, /// Error message (null if success) pub error: *mut c_char, + /// Warning message(s), separated by newlines (null if none) + pub warnings: *mut c_char, } /// Check if SQL contains "AS MEASURE" pattern @@ -280,6 +282,7 @@ pub extern "C" fn yardstick_expand_aggregate(sql: *const c_char) -> YardstickAgg had_aggregate: false, expanded_sql: ptr::null_mut(), error: to_c_string("Error: null sql pointer"), + warnings: ptr::null_mut(), }; } @@ -291,6 +294,7 @@ pub extern "C" fn yardstick_expand_aggregate(sql: *const c_char) -> YardstickAgg had_aggregate: false, expanded_sql: ptr::null_mut(), error: to_c_string(&format!("Error: invalid UTF-8: {e}")), + warnings: ptr::null_mut(), }; } } @@ -305,6 +309,11 @@ pub extern "C" fn yardstick_expand_aggregate(sql: *const c_char) -> YardstickAgg .error .map(|s| to_c_string(&s)) .unwrap_or(ptr::null_mut()), + warnings: if result.warnings.is_empty() { + ptr::null_mut() + } else { + to_c_string(&result.warnings.join("\n")) + }, } } @@ -400,6 +409,7 @@ pub extern "C" fn yardstick_free_create_view_result(result: YardstickCreateViewR pub extern "C" fn yardstick_free_aggregate_result(result: YardstickAggregateResult) { yardstick_free(result.expanded_sql); yardstick_free(result.error); + yardstick_free(result.warnings); } // Helper: convert Rust string to C string diff --git a/yardstick-rs/src/sql/measures.rs b/yardstick-rs/src/sql/measures.rs index a854fd6..127f0cb 100644 --- a/yardstick-rs/src/sql/measures.rs +++ b/yardstick-rs/src/sql/measures.rs @@ -74,6 +74,7 @@ pub struct AggregateExpandResult { pub had_aggregate: bool, pub expanded_sql: String, pub error: Option, + pub warnings: Vec, } /// Context modifier for AGGREGATE() AT (...) syntax @@ -1825,6 +1826,7 @@ fn find_top_level_keyword(sql: &str, keyword: &str, start: usize) -> Option = None; let mut in_line_comment = false; let mut in_block_comment = false; @@ -1832,6 +1834,16 @@ fn find_top_level_keyword(sql: &str, keyword: &str, start: usize) -> Option Option Option { struct CteExpansion { sql: String, had_aggregate: bool, + warnings: Vec, } fn expand_cte_queries(sql: &str) -> CteExpansion { @@ -2096,6 +2117,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate: false, + warnings: Vec::new(), }; } }; @@ -2104,6 +2126,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { let mut idx = with_pos + "WITH".len(); let mut replacements: Vec<(usize, usize, String)> = Vec::new(); let mut had_aggregate = false; + let mut warnings = Vec::new(); idx = skip_whitespace(sql, idx); if matches_keyword_at(&upper, idx, "RECURSIVE") { @@ -2116,6 +2139,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate: false, + warnings: Vec::new(), }; } @@ -2126,6 +2150,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate: false, + warnings: Vec::new(), }; } }; @@ -2140,6 +2165,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate: false, + warnings: Vec::new(), }; } }; @@ -2152,6 +2178,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate: false, + warnings: Vec::new(), }; } idx += "AS".len(); @@ -2161,6 +2188,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate: false, + warnings: Vec::new(), }; } @@ -2171,6 +2199,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate: false, + warnings: Vec::new(), }; } }; @@ -2182,6 +2211,11 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { if expanded.had_aggregate { had_aggregate = true; } + for warning in expanded.warnings { + if !warnings.contains(&warning) { + warnings.push(warning); + } + } if expanded.expanded_sql != query_sql { replacements.push((query_start, query_end, expanded.expanded_sql)); } @@ -2199,6 +2233,7 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { return CteExpansion { sql: sql.to_string(), had_aggregate, + warnings, }; } @@ -2211,1134 +2246,978 @@ fn expand_cte_queries(sql: &str) -> CteExpansion { CteExpansion { sql: result, had_aggregate, + warnings, } } -fn extract_base_relation_sql(view_query: &str) -> Option { - let query = view_query.trim().trim_end_matches(';').trim(); - if query.is_empty() { - return None; - } - - let has_set_op = ["UNION", "INTERSECT", "EXCEPT"] - .iter() - .any(|kw| find_top_level_keyword(query, kw, 0).is_some()); - if has_set_op { - return Some(format!("SELECT * FROM ({query})")); +fn top_level_parenthesized_query_body_range(sql: &str) -> Option<(usize, usize)> { + let upper = sql.to_uppercase(); + let mut idx = 0; + while let Some(as_pos) = find_top_level_keyword(sql, "AS", idx) { + let open_pos = skip_ws_and_comments(sql, as_pos + "AS".len()); + if open_pos < sql.len() && sql.as_bytes()[open_pos] == b'(' { + let body_start = skip_ws_and_comments(sql, open_pos + 1); + if matches_keyword_at(&upper, body_start, "SELECT") + || matches_keyword_at(&upper, body_start, "WITH") + { + if let Ok((_, body)) = balanced_parens(&sql[open_pos + 1..]) { + let start = open_pos + 1; + return Some((start, start + body.len())); + } + } + } + idx = as_pos + "AS".len(); } + None +} - let select_pos = find_top_level_keyword(query, "SELECT", 0)?; - let from_pos = find_top_level_keyword(query, "FROM", select_pos)?; - let from_start = from_pos + 4; - let from_end = find_first_top_level_keyword( - query, - from_start, - &[ - "WHERE", - "GROUP BY", - "HAVING", - "QUALIFY", - "ORDER BY", - "LIMIT", - "WINDOW", - "UNION", - "INTERSECT", - "EXCEPT", - ], - ) - .unwrap_or(query.len()); +fn top_level_parenthesized_query_body(sql: &str) -> Option<&str> { + top_level_parenthesized_query_body_range(sql).map(|(start, end)| &sql[start..end]) +} - let from_clause = query[from_start..from_end].trim(); - if from_clause.is_empty() { - return None; - } +fn find_top_level_open_paren(sql: &str, start: usize) -> Option { + let bytes = sql.as_bytes(); + let mut depth: i32 = 0; + let mut in_single = false; + let mut in_double = false; + let mut in_backtick = false; + let mut in_bracket = false; + let mut in_line_comment = false; + let mut in_block_comment = false; - let where_clause = find_top_level_keyword(query, "WHERE", from_pos).map(|pos| { - let where_start = pos + 5; - let where_end = find_first_top_level_keyword( - query, - where_start, - &[ - "GROUP BY", - "HAVING", - "QUALIFY", - "ORDER BY", - "LIMIT", - "WINDOW", - "UNION", - "INTERSECT", - "EXCEPT", - ], - ) - .unwrap_or(query.len()); - query[where_start..where_end].trim().to_string() - }); + let mut i = start; + while i < bytes.len() { + let c = bytes[i] as char; - let cte_prefix = query[..select_pos].trim(); - let mut base_sql = String::new(); - if !cte_prefix.is_empty() { - base_sql.push_str(cte_prefix); - base_sql.push(' '); - } - base_sql.push_str("SELECT * FROM "); - base_sql.push_str(from_clause); - if let Some(w) = where_clause { - if !w.is_empty() { - base_sql.push_str(" WHERE "); - base_sql.push_str(&w); + if in_line_comment { + if c == '\n' || c == '\r' { + in_line_comment = false; + } + i += 1; + continue; } - } - Some(base_sql) -} + if in_block_comment { + if c == '*' && i + 1 < bytes.len() && bytes[i + 1] as char == '/' { + in_block_comment = false; + i += 2; + continue; + } + i += 1; + continue; + } -fn normalize_group_by_col(col: &str) -> String { - let trimmed = col.trim(); - let unquoted = trimmed - .strip_prefix('"') - .and_then(|s| s.strip_suffix('"')) - .unwrap_or(trimmed); - if unquoted.contains('(') { - let mut result = String::new(); - let mut last_space = false; - for ch in unquoted.chars() { - if ch.is_whitespace() { - if !last_space { - result.push(' '); - last_space = true; + if in_single { + if c == '\'' { + if i + 1 < bytes.len() && bytes[i + 1] as char == '\'' { + i += 1; + } else { + in_single = false; } - } else { - result.push(ch.to_ascii_lowercase()); - last_space = false; } + i += 1; + continue; } - result.trim().to_string() - } else { - let base = unquoted.split('.').next_back().unwrap_or(unquoted); - base.to_lowercase() - } -} -fn extract_view_group_by_cols(view_query: &str) -> Vec { - let query = view_query.trim().trim_end_matches(';').trim(); - let group_pos = match find_top_level_keyword(query, "GROUP BY", 0) { - Some(pos) => pos, - None => { - if let Ok(info) = parser_ffi::parse_select(query) { - return info - .items - .iter() - .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) - .map(|item| { - item.alias - .clone() - .unwrap_or_else(|| item.expression_sql.clone()) - }) - .collect(); + if in_double { + if c == '"' { + in_double = false; } - return Vec::new(); + i += 1; + continue; } - }; - let start = advance_after_group_by(query, group_pos) - .unwrap_or_else(|| group_pos + "GROUP BY".len()); - let end = find_first_top_level_keyword( - query, - start, - &[ - "HAVING", - "QUALIFY", - "ORDER BY", - "LIMIT", - "WINDOW", - "UNION", - "INTERSECT", - "EXCEPT", - ], - ) - .unwrap_or(query.len()); + if in_backtick { + if c == '`' { + in_backtick = false; + } + i += 1; + continue; + } - let group_content = query[start..end].trim(); - if group_content.is_empty() { - return Vec::new(); - } + if in_bracket { + if c == ']' { + in_bracket = false; + } + i += 1; + continue; + } - let group_upper = group_content.to_uppercase(); - if group_upper == "ALL" || group_upper.starts_with("ALL ") { - return extract_dimension_columns_from_select(query); - } + if c == '-' && i + 1 < bytes.len() && bytes[i + 1] as char == '-' { + in_line_comment = true; + i += 2; + continue; + } + + if c == '/' && i + 1 < bytes.len() && bytes[i + 1] as char == '*' { + in_block_comment = true; + i += 2; + continue; + } - let mut columns = Vec::new(); - let mut depth = 0; - let mut current = String::new(); - for c in group_content.chars() { match c { + '\'' => in_single = true, + '"' => in_double = true, + '`' => in_backtick = true, + '[' => in_bracket = true, '(' => { + if depth == 0 { + return Some(i); + } depth += 1; - current.push(c); } ')' => { depth -= 1; - current.push(c); } - ',' if depth == 0 => { - let col = current.trim(); - if !col.is_empty() && !col.chars().all(|ch| ch.is_ascii_digit()) { - columns.push(col.to_string()); - } - current.clear(); - } - _ => current.push(c), + _ => {} } - } - let col = current.trim(); - if !col.is_empty() && !col.chars().all(|ch| ch.is_ascii_digit()) { - columns.push(col.to_string()); + i += 1; } - columns + None } -fn find_from_clause_end(sql: &str) -> Option { - // Preserve leading whitespace so returned indices match the original SQL. - let query = sql.trim_end(); - let query = query.strip_suffix(';').unwrap_or(query); - let from_pos = find_top_level_keyword(query, "FROM", 0)?; - let start = from_pos + 4; - Some( - find_first_top_level_keyword( - query, - start, - &[ - "WHERE", - "GROUP BY", - "HAVING", - "QUALIFY", - "ORDER BY", - "LIMIT", - "WINDOW", - "UNION", - "INTERSECT", - "EXCEPT", - ], - ) - .unwrap_or(query.len()), - ) -} - -fn group_by_matches_view(outer_cols: &[String], view_cols: &[String]) -> bool { - if view_cols.is_empty() || outer_cols.is_empty() { - return false; +fn find_matching_paren_sql(sql: &str, open_pos: usize) -> Option { + let bytes = sql.as_bytes(); + if open_pos >= bytes.len() || bytes[open_pos] != b'(' { + return None; } - let outer_set: std::collections::HashSet = - outer_cols.iter().map(|c| normalize_group_by_col(c)).collect(); - let view_set: std::collections::HashSet = - view_cols.iter().map(|c| normalize_group_by_col(c)).collect(); + let mut depth: i32 = 0; + let mut in_single = false; + let mut in_double = false; + let mut in_backtick = false; + let mut in_bracket = false; + let mut in_dollar_quote: Option = None; + let mut in_line_comment = false; + let mut in_block_comment = false; - !outer_set.is_empty() && outer_set == view_set -} + let mut i = open_pos; + while i < bytes.len() { + let c = bytes[i] as char; -fn filter_group_by_cols_for_measure( - outer_cols: &[String], - view_cols: &[String], - dimension_exprs: &HashMap, -) -> Vec { - if view_cols.is_empty() { - return outer_cols.to_vec(); - } + if let Some(ref delimiter) = in_dollar_quote { + if sql[i..].starts_with(delimiter) { + i += delimiter.len(); + in_dollar_quote = None; + } else { + i += sql[i..].chars().next().map(|ch| ch.len_utf8()).unwrap_or(1); + } + continue; + } - let view_set: HashSet = view_cols - .iter() - .map(|col| normalize_group_by_col(col)) - .collect(); + if in_line_comment { + if c == '\n' || c == '\r' { + in_line_comment = false; + } + i += 1; + continue; + } - let mut alias_expr_norms: Vec<(String, String)> = Vec::new(); - alias_expr_norms.reserve(dimension_exprs.len()); - for (alias, expr) in dimension_exprs.iter() { - alias_expr_norms.push(( - normalize_group_by_col(alias), - normalize_group_by_col(expr), - )); - } + if in_block_comment { + if c == '*' && i + 1 < bytes.len() && bytes[i + 1] as char == '/' { + in_block_comment = false; + i += 2; + continue; + } + i += 1; + continue; + } - outer_cols - .iter() - .filter(|col| { - let normalized_outer = normalize_group_by_col(col); - if view_set.contains(&normalized_outer) { - return true; + if in_single { + if c == '\'' { + if i + 1 < bytes.len() && bytes[i + 1] as char == '\'' { + i += 2; + continue; + } + in_single = false; } + i += 1; + continue; + } - if let Some(expr) = dimension_exprs.get(&normalized_outer) { - let normalized_expr = normalize_group_by_col(expr); - if view_set.contains(&normalized_expr) { - return true; + if in_double { + if c == '"' { + if i + 1 < bytes.len() && bytes[i + 1] as char == '"' { + i += 2; + continue; } + in_double = false; } + i += 1; + continue; + } - alias_expr_norms.iter().any(|(alias_norm, expr_norm)| { - expr_norm == &normalized_outer && view_set.contains(alias_norm) - }) - }) - .cloned() - .collect() -} + if in_backtick { + if c == '`' { + in_backtick = false; + } + i += 1; + continue; + } -fn source_dimension_names(source_view: &str) -> HashSet { - let views = MEASURE_VIEWS.lock().unwrap(); - let Some((_, view)) = views - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(source_view)) - else { - return HashSet::new(); - }; + if in_bracket { + if c == ']' { + in_bracket = false; + } + i += 1; + continue; + } - let view_query = extract_view_query(&view.base_query).unwrap_or_else(|| view.base_query.clone()); - extract_dimension_columns_from_select(&view_query) - .into_iter() - .map(|col| { - let dim_name = col.split('.').next_back().unwrap_or(col.as_str()).trim(); - normalize_dimension_key(dim_name) - }) - .collect() -} + if let Some(delimiter) = dollar_quote_delimiter_at(sql, i) { + if sql[i + delimiter.len()..].contains(delimiter) { + in_dollar_quote = Some(delimiter.to_string()); + i += delimiter.len(); + continue; + } + } -fn can_use_view_measure_directly(resolved: &ResolvedMeasure, outer_group_by: &[String]) -> bool { - group_by_matches_view(outer_group_by, &resolved.view_group_by_cols) -} + if c == '-' && i + 1 < bytes.len() && bytes[i + 1] as char == '-' { + in_line_comment = true; + i += 2; + continue; + } -/// Extract WHERE clause from SQL query -pub fn extract_where_clause(sql: &str) -> Option { - let query = sql.trim().trim_end_matches(';').trim(); - let where_pos = find_top_level_keyword(query, "WHERE", 0)?; - let start = where_pos + 5; - let end = find_first_top_level_keyword( - query, - start, - &[ - "GROUP BY", - "HAVING", - "QUALIFY", - "ORDER BY", - "LIMIT", - "WINDOW", - "UNION", - "INTERSECT", - "EXCEPT", - ], - ) - .unwrap_or(query.len()); + if c == '/' && i + 1 < bytes.len() && bytes[i + 1] as char == '*' { + in_block_comment = true; + i += 2; + continue; + } - let where_content = query[start..end].trim().to_string(); - if where_content.is_empty() { - None - } else { - Some(where_content) + match c { + '\'' => in_single = true, + '"' => in_double = true, + '`' => in_backtick = true, + '[' => in_bracket = true, + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + return Some(i); + } + } + _ => {} + } + i += 1; } + + None } -// ============================================================================= -// Nom Parsers - Expression Helpers -// ============================================================================= +fn top_level_parenthesized_insert_query_body_range(sql: &str) -> Option<(usize, usize)> { + let upper = sql.to_uppercase(); + let insert_pos = find_top_level_keyword(sql, "INSERT", 0)?; + if insert_pos != skip_ws_and_comments(sql, 0) { + return None; + } -/// Parse aggregation function name - accepts any identifier followed by ( -/// This allows all DuckDB aggregate functions to work as measures -fn agg_function_name(input: &str) -> IResult<&str, &str> { - // Match any identifier (alphanumeric + underscore) that's followed by ( - let (remaining, name) = nom::bytes::complete::take_while1(|c: char| c.is_alphanumeric() || c == '_')(input)?; - // Verify it's followed by a paren (but don't consume it) - let _ = nom::character::complete::char('(')(remaining)?; - Ok((remaining, name)) -} + let mut idx = insert_pos + "INSERT".len(); + while let Some(open_pos) = find_top_level_open_paren(sql, idx) { + let close_pos = find_matching_paren_sql(sql, open_pos)?; + let body_start = skip_ws_and_comments(sql, open_pos + 1); + if matches_keyword_at(&upper, body_start, "SELECT") + || matches_keyword_at(&upper, body_start, "WITH") + { + return Some((open_pos + 1, close_pos)); + } + idx = close_pos + 1; + } -/// Extract aggregation function from expression like "SUM(amount)" -pub fn extract_agg_function(expr: &str) -> String { - agg_function_name(expr.trim()) - .map(|(_, name)| name.to_uppercase()) - .unwrap_or_else(|_| "SUM".to_string()) + None } -/// Extract aggregation function name from full expression -/// "SUM(amount)" -> "sum" -/// "COUNT(DISTINCT x)" -> "count" -pub fn extract_aggregation_function(expr: &str) -> Option { - let (_, (name, _)) = function_call(expr.trim()).ok()?; - Some(name.to_lowercase()) -} +fn top_level_create_as_query_body_range(sql: &str) -> Option<(usize, usize)> { + let upper = sql.to_uppercase(); + let create_pos = find_top_level_keyword(sql, "CREATE", 0)?; + if create_pos != skip_ws_and_comments(sql, 0) { + return None; + } -/// Check if expression contains DISTINCT modifier -/// "COUNT(DISTINCT region)" -> true -fn has_distinct_modifier(expr: &str) -> bool { - let expr_upper = expr.to_uppercase(); - // Look for DISTINCT after opening paren - if let Some(paren_pos) = expr_upper.find('(') { - let after_paren = &expr_upper[paren_pos + 1..]; - after_paren.trim_start().starts_with("DISTINCT") - } else { - false + let as_pos = find_top_level_keyword(sql, "AS", create_pos + "CREATE".len())?; + let body_start = skip_ws_and_comments(sql, as_pos + "AS".len()); + if body_start >= sql.len() { + return None; } -} -/// Returns true if expression appears to use a SQL window clause. -fn is_window_expression(expr: &str) -> bool { - let bytes = expr.as_bytes(); - let mut i = 0; + if sql.as_bytes()[body_start] == b'(' { + let close_pos = find_matching_paren_sql(sql, body_start)?; + let inner_start = skip_ws_and_comments(sql, body_start + 1); + if matches_keyword_at(&upper, inner_start, "SELECT") + || matches_keyword_at(&upper, inner_start, "WITH") + { + return Some((body_start + 1, close_pos)); + } + return None; + } - let is_word_boundary = |b: Option| b.map_or(true, |c| !c.is_ascii_alphanumeric() && c != b'_'); + if matches_keyword_at(&upper, body_start, "SELECT") + || matches_keyword_at(&upper, body_start, "WITH") + { + return Some((body_start, sql.len())); + } - while i < bytes.len() { - if bytes[i] == b'\'' { - i += 1; - while i < bytes.len() { - if bytes[i] == b'\'' { - if i + 1 < bytes.len() && bytes[i + 1] == b'\'' { - i += 2; - continue; - } - i += 1; - break; - } - i += 1; - } - continue; - } + None +} - if bytes[i] == b'"' { - i += 1; - while i < bytes.len() { - if bytes[i] == b'"' { - if i + 1 < bytes.len() && bytes[i + 1] == b'"' { - i += 2; - continue; - } - i += 1; - break; - } - i += 1; - } - continue; - } +fn top_level_with_main_query_start(sql: &str) -> Option { + let with_pos = find_top_level_keyword(sql, "WITH", 0)?; + if with_pos != skip_ws_and_comments(sql, 0) { + return None; + } - if bytes[i] == b'`' { - i += 1; - while i < bytes.len() && bytes[i] != b'`' { - i += 1; - } - if i < bytes.len() { - i += 1; - } - continue; + let upper = sql.to_uppercase(); + let mut idx = with_pos + "WITH".len(); + idx = skip_ws_and_comments(sql, idx); + if matches_keyword_at(&upper, idx, "RECURSIVE") { + idx += "RECURSIVE".len(); + } + + loop { + idx = skip_ws_and_comments(sql, idx); + if idx >= sql.len() { + return None; } - if bytes[i] == b'[' { - i += 1; - while i < bytes.len() && bytes[i] != b']' { - i += 1; - } - if i < bytes.len() { - i += 1; - } - continue; + let rest = &sql[idx..]; + let (after_ident, _) = identifier(rest).ok()?; + idx += rest.len() - after_ident.len(); + + idx = skip_ws_and_comments(sql, idx); + if idx < sql.len() && sql.as_bytes()[idx] == b'(' { + let sub = &sql[idx + 1..]; + let (rest_after_cols, _) = balanced_parens(sub).ok()?; + let cols_len = sub.len() - rest_after_cols.len(); + idx = idx + 1 + cols_len + 1; + idx = skip_ws_and_comments(sql, idx); } - if bytes[i] == b'-' && i + 1 < bytes.len() && bytes[i + 1] == b'-' { - i += 2; - while i < bytes.len() { - let ch = bytes[i]; - i += 1; - if ch == b'\n' || ch == b'\r' { - break; - } - } - continue; + if !matches_keyword_at(&upper, idx, "AS") { + return None; } + idx += "AS".len(); + idx = skip_ws_and_comments(sql, idx); - if bytes[i] == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' { - i += 2; - while i + 1 < bytes.len() { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - if i + 1 >= bytes.len() { - i = bytes.len(); - } - continue; + if idx >= sql.len() || sql.as_bytes()[idx] != b'(' { + return None; } - // Window clauses follow a function invocation: ") OVER (...)|OVER window_name" - if bytes[i] == b')' { - let mut j = skip_ws_and_comments(expr, i + 1); - let has_over = j + 4 <= bytes.len() - && bytes[j..j + 4] - .iter() - .zip(b"OVER") - .all(|(b, kw)| b.to_ascii_uppercase() == *kw) - && is_word_boundary(if j == 0 { None } else { Some(bytes[j - 1]) }) - && is_word_boundary(if j + 4 >= bytes.len() { - None - } else { - Some(bytes[j + 4]) - }); + let sub = &sql[idx + 1..]; + let (rest_after_query, _) = balanced_parens(sub).ok()?; + let query_len = sub.len() - rest_after_query.len(); + idx = idx + 1 + query_len + 1; + idx = skip_ws_and_comments(sql, idx); - if has_over { - j += 4; - j = skip_ws_and_comments(expr, j); - if j < bytes.len() && (bytes[j] == b'(' || parse_identifier_token(expr, j).is_some()) { - return true; - } - } + if idx < sql.len() && sql.as_bytes()[idx] == b',' { + idx += 1; + continue; } - i += 1; + return Some(idx); } - - false } -/// Non-decomposable aggregate functions that require recompute from base rows -const NON_DECOMPOSABLE_AGGREGATES: &[&str] = &[ - "MEDIAN", - "PERCENTILE_CONT", - "PERCENTILE_DISC", - "MODE", - "QUANTILE", - "QUANTILE_CONT", - "QUANTILE_DISC", -]; - -/// Returns true if expression uses a non-decomposable aggregate -/// (including COUNT DISTINCT and ordered-set aggregates) -fn is_non_decomposable(expr: &str) -> bool { - if has_distinct_modifier(expr) { - return true; +fn aggregate_context_sql(sql: &str) -> &str { + let body_sql = if top_level_with_main_query_start(sql).is_some() { + sql + } else { + top_level_create_as_query_body_range(sql) + .map(|(start, end)| &sql[start..end]) + .or_else(|| { + top_level_parenthesized_insert_query_body_range(sql) + .map(|(start, end)| &sql[start..end]) + }) + .or_else(|| top_level_parenthesized_query_body(sql)) + .unwrap_or(sql) + }; + if let Some(main_query_start) = top_level_with_main_query_start(body_sql) { + &body_sql[main_query_start..] + } else { + body_sql } - - let expr_upper = expr.to_uppercase(); - NON_DECOMPOSABLE_AGGREGATES - .iter() - .any(|agg| expr_upper.contains(&format!("{agg}("))) } -/// Find any aggregation function inside an expression -/// "CASE WHEN SUM(x) > 100 THEN 1 ELSE 0 END" -> Some("sum") -fn find_aggregation_in_expression(expr: &str) -> Option { - // Look for any function call pattern: identifier followed by ( - // This allows all DuckDB aggregate functions to work - let expr_upper = expr.to_uppercase(); - - // Common aggregates to check first (for performance) - let common_aggs = [ - "SUM", "COUNT", "AVG", "MIN", "MAX", "MEDIAN", "STDDEV", "STDDEV_POP", - "STDDEV_SAMP", "VARIANCE", "VAR_POP", "VAR_SAMP", "STRING_AGG", - "ARRAY_AGG", "LIST", "FIRST", "LAST", "MODE", "QUANTILE", - ]; - - for agg in common_aggs { - if expr_upper.contains(&format!("{agg}(")) { - return Some(agg.to_lowercase()); - } +fn extract_base_relation_sql(view_query: &str) -> Option { + let query = view_query.trim().trim_end_matches(';').trim(); + if query.is_empty() { + return None; } - // Fallback: look for any identifier followed by ( that could be an aggregate - // This catches custom aggregates - let chars: Vec = expr.chars().collect(); - let mut i = 0; - while i < chars.len() { - // Look for start of identifier - if chars[i].is_alphabetic() || chars[i] == '_' { - let start = i; - while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') { - i += 1; - } - // Check if followed by ( - if i < chars.len() && chars[i] == '(' { - let name: String = chars[start..i].iter().collect(); - // Skip known non-aggregates - let name_upper = name.to_uppercase(); - if !["CASE", "WHEN", "THEN", "ELSE", "END", "AND", "OR", "NOT", "IN", "IS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", "CAST", "COALESCE", "NULLIF", "IF", "IIF"].contains(&name_upper.as_str()) { - return Some(name.to_lowercase()); - } - } - } - i += 1; + let has_set_op = ["UNION", "INTERSECT", "EXCEPT"] + .iter() + .any(|kw| find_top_level_keyword(query, kw, 0).is_some()); + if has_set_op { + return Some(format!("SELECT * FROM ({query})")); } - None -} + let select_pos = find_top_level_keyword(query, "SELECT", 0)?; + let from_pos = find_top_level_keyword(query, "FROM", select_pos)?; + let from_start = from_pos + 4; + let from_end = find_first_top_level_keyword( + query, + from_start, + &[ + "WHERE", + "GROUP BY", + "HAVING", + "QUALIFY", + "ORDER BY", + "LIMIT", + "WINDOW", + "UNION", + "INTERSECT", + "EXCEPT", + ], + ) + .unwrap_or(query.len()); -/// Check if an expression contains COUNT(DISTINCT ...) which is non-decomposable -/// "COUNT(DISTINCT user_id)" -> true -/// "COUNT(user_id)" -> false -/// "SUM(amount)" -> false -pub fn is_count_distinct(expr: &str) -> bool { - let expr_upper = expr.to_uppercase(); - // Match COUNT followed by optional whitespace, (, optional whitespace, DISTINCT - // This handles: COUNT(DISTINCT x), COUNT( DISTINCT x), COUNT (DISTINCT x) - let patterns = ["COUNT(DISTINCT", "COUNT( DISTINCT", "COUNT (DISTINCT"]; - patterns.iter().any(|p| expr_upper.contains(p)) -} - -/// Expand derived measure expression by replacing measure references with their aggregations -/// "revenue - cost" with measures [revenue=SUM(amount), cost=SUM(expense)] -/// -> "SUM(revenue) - SUM(cost)" -fn expand_derived_measure_expr(expr: &str, measure_view: &MeasureView) -> String { - let mut result = String::new(); - let mut chars = expr.chars().peekable(); + let from_clause = query[from_start..from_end].trim(); + if from_clause.is_empty() { + return None; + } - while let Some(c) = chars.next() { - if c == '\'' { - result.push(c); - while let Some(next) = chars.next() { - result.push(next); - if next == '\'' { - if chars.peek() == Some(&'\'') { - result.push(chars.next().unwrap()); - } else { - break; - } - } - } - continue; - } - if c == '"' { - result.push(c); - while let Some(next) = chars.next() { - result.push(next); - if next == '"' { - if chars.peek() == Some(&'"') { - result.push(chars.next().unwrap()); - } else { - break; - } - } - } - continue; - } - if c == '`' { - result.push(c); - while let Some(next) = chars.next() { - result.push(next); - if next == '`' { - break; - } - } - continue; - } - if c == '[' { - result.push(c); - while let Some(next) = chars.next() { - result.push(next); - if next == ']' { - break; - } - } - continue; - } - if c.is_alphabetic() || c == '_' { - // Collect identifier - let mut ident = String::from(c); - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - ident.push(chars.next().unwrap()); - } else { - break; - } - } + let where_clause = find_top_level_keyword(query, "WHERE", from_pos).map(|pos| { + let where_start = pos + 5; + let where_end = find_first_top_level_keyword( + query, + where_start, + &[ + "GROUP BY", + "HAVING", + "QUALIFY", + "ORDER BY", + "LIMIT", + "WINDOW", + "UNION", + "INTERSECT", + "EXCEPT", + ], + ) + .unwrap_or(query.len()); + query[where_start..where_end].trim().to_string() + }); - // Check if this identifier is a measure name - if let Some(m) = measure_view - .measures - .iter() - .find(|m| m.column_name.eq_ignore_ascii_case(&ident)) - { - result.push('('); - result.push_str(&m.expression); - result.push(')'); - } else { - // Not a measure, keep as-is - result.push_str(&ident); - } - } else { - result.push(c); + let cte_prefix = query[..select_pos].trim(); + let mut base_sql = String::new(); + if !cte_prefix.is_empty() { + base_sql.push_str(cte_prefix); + base_sql.push(' '); + } + base_sql.push_str("SELECT * FROM "); + base_sql.push_str(from_clause); + if let Some(w) = where_clause { + if !w.is_empty() { + base_sql.push_str(" WHERE "); + base_sql.push_str(&w); } } - result + Some(base_sql) } -/// Qualify dimension reference in expression for correlated subquery -/// "year - 1" with table "sales" and dim "year" -> "sales.year - 1" -pub fn qualify_outer_reference(expr: &str, table_name: &str, dim: &str) -> String { - // Parse expression into tokens and replace matching identifiers - let mut result = String::new(); - let mut chars = expr.chars().peekable(); - - while let Some(c) = chars.next() { - if c.is_alphabetic() || c == '_' { - // Collect identifier - let mut ident = String::from(c); - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - ident.push(chars.next().unwrap()); - } else { - break; +fn normalize_group_by_col(col: &str) -> String { + let trimmed = col.trim(); + let unquoted = trimmed + .strip_prefix('"') + .and_then(|s| s.strip_suffix('"')) + .unwrap_or(trimmed); + if unquoted.contains('(') { + let mut result = String::new(); + let mut last_space = false; + for ch in unquoted.chars() { + if ch.is_whitespace() { + if !last_space { + result.push(' '); + last_space = true; } + } else { + result.push(ch.to_ascii_lowercase()); + last_space = false; } - - // Check if this identifier matches the dimension - if ident == dim { - result.push_str(table_name); - result.push('.'); - } - result.push_str(&ident); - } else { - result.push(c); } + result.trim().to_string() + } else { + let base = unquoted.split('.').next_back().unwrap_or(unquoted); + base.to_lowercase() } - - result } -fn expr_mentions_identifier(expr: &str, ident: &str) -> bool { - let mut chars = expr.chars().peekable(); - - while let Some(c) = chars.next() { - if c == '\'' { - // Skip SQL single-quoted string literal content, including escaped ''. - while let Some(next) = chars.next() { - if next == '\'' { - if chars.peek() == Some(&'\'') { - chars.next(); - } else { - break; - } - } +fn extract_view_group_by_cols(view_query: &str) -> Vec { + let query = view_query.trim().trim_end_matches(';').trim(); + let group_pos = match find_top_level_keyword(query, "GROUP BY", 0) { + Some(pos) => pos, + None => { + if let Ok(info) = parser_ffi::parse_select(query) { + return info + .items + .iter() + .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) + .map(|item| { + item.alias + .clone() + .unwrap_or_else(|| item.expression_sql.clone()) + }) + .collect(); } - continue; + return Vec::new(); } + }; - if c.is_alphabetic() || c == '_' { - let mut token = String::from(c); - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - token.push(chars.next().unwrap()); - } else { - break; - } - } + let start = advance_after_group_by(query, group_pos) + .unwrap_or_else(|| group_pos + "GROUP BY".len()); + let end = find_first_top_level_keyword( + query, + start, + &[ + "HAVING", + "QUALIFY", + "ORDER BY", + "LIMIT", + "WINDOW", + "UNION", + "INTERSECT", + "EXCEPT", + ], + ) + .unwrap_or(query.len()); - if token.eq_ignore_ascii_case(ident) { - return true; - } - } + let group_content = query[start..end].trim(); + if group_content.is_empty() { + return Vec::new(); } - false -} - -fn dimension_in_group_by( - dim: &str, - group_by_cols: &[String], - default_qualifier: Option<&str>, -) -> bool { - let dim_trim = dim.trim(); - let dim_name = dim_trim.split('.').next_back().unwrap_or(dim_trim).trim(); - let explicit_dim_qualifier = dim_trim - .rsplit_once('.') - .map(|(qualifier, _)| qualifier.trim()); - let expected_qualifier = explicit_dim_qualifier.or(default_qualifier); - let dim_lower = dim_trim.to_lowercase(); + let group_upper = group_content.to_uppercase(); + if group_upper == "ALL" || group_upper.starts_with("ALL ") { + return extract_dimension_columns_from_select(query); + } - if dim_trim.contains('(') { - return group_by_cols - .iter() - .any(|col| col.to_lowercase() == dim_lower); - } - - group_by_cols.iter().any(|col| { - let col_trim = col.trim(); - let col_name = col_trim.split('.').next_back().unwrap_or(col_trim).trim(); - - if !col_name.eq_ignore_ascii_case(dim_name) { - return false; - } - - // When we know the expected outer qualifier, GROUP BY qualifiers must match it. - // Unqualified GROUP BY columns still count. - match (expected_qualifier, col_trim.rsplit_once('.')) { - (Some(expected), Some((col_qualifier, _))) => { - col_qualifier.trim().eq_ignore_ascii_case(expected) + let mut columns = Vec::new(); + let mut depth = 0; + let mut current = String::new(); + for c in group_content.chars() { + match c { + '(' => { + depth += 1; + current.push(c); } - _ => true, - } - }) -} - -fn expr_mentions_identifier_outside_current(expr: &str, ident: &str) -> bool { - let mut chars = expr.chars().peekable(); - let mut pending_current = false; - - while let Some(c) = chars.next() { - if c == '\'' { - while let Some(next) = chars.next() { - if next == '\'' { - if chars.peek() == Some(&'\'') { - chars.next(); - } else { - break; - } + ')' => { + depth -= 1; + current.push(c); + } + ',' if depth == 0 => { + let col = current.trim(); + if !col.is_empty() && !col.chars().all(|ch| ch.is_ascii_digit()) { + columns.push(col.to_string()); } + current.clear(); } - continue; + _ => current.push(c), } + } + let col = current.trim(); + if !col.is_empty() && !col.chars().all(|ch| ch.is_ascii_digit()) { + columns.push(col.to_string()); + } - if c.is_alphabetic() || c == '_' { - let mut token = String::from(c); - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - token.push(chars.next().unwrap()); - } else { - break; - } - } + columns +} - if token.eq_ignore_ascii_case("CURRENT") { - pending_current = true; - continue; - } +fn find_from_clause_end(sql: &str) -> Option { + // Preserve leading whitespace so returned indices match the original SQL. + let query = sql.trim_end(); + let query = query.strip_suffix(';').unwrap_or(query); + let from_pos = find_top_level_keyword(query, "FROM", 0)?; + let start = from_pos + 4; + Some( + find_first_top_level_keyword( + query, + start, + &[ + "WHERE", + "GROUP BY", + "HAVING", + "QUALIFY", + "ORDER BY", + "LIMIT", + "WINDOW", + "UNION", + "INTERSECT", + "EXCEPT", + ], + ) + .unwrap_or(query.len()), + ) +} - if token.eq_ignore_ascii_case(ident) { - if pending_current { - pending_current = false; - } else { - return true; - } - } else { - pending_current = false; - } - } +fn group_by_matches_view(outer_cols: &[String], view_cols: &[String]) -> bool { + if view_cols.is_empty() || outer_cols.is_empty() { + return false; } - false + let outer_set: std::collections::HashSet = + outer_cols.iter().map(|c| normalize_group_by_col(c)).collect(); + let view_set: std::collections::HashSet = + view_cols.iter().map(|c| normalize_group_by_col(c)).collect(); + + !outer_set.is_empty() && outer_set == view_set } -fn where_has_simple_equality_constraint( - where_clause: &str, - dim_name: &str, - default_qualifier: Option<&str>, -) -> bool { - // Conservative: if OR is present we don't claim single-valued context. - if where_clause.to_uppercase().contains(" OR ") { - return false; +fn filter_group_by_cols_for_measure( + outer_cols: &[String], + view_cols: &[String], + dimension_exprs: &HashMap, +) -> Vec { + if view_cols.is_empty() { + return outer_cols.to_vec(); } - let expected_qualifier = default_qualifier.map(normalize_identifier_name); - let bytes = where_clause.as_bytes(); - let mut i = 0usize; + let view_set: HashSet = view_cols + .iter() + .map(|col| normalize_group_by_col(col)) + .collect(); - while i < bytes.len() { - if bytes[i] == b'=' { - if (i > 0 && matches!(bytes[i - 1], b'<' | b'>' | b'!' | b'=')) - || (i + 1 < bytes.len() && bytes[i + 1] == b'=') - { - i += 1; - continue; - } + let mut alias_expr_norms: Vec<(String, String)> = Vec::new(); + alias_expr_norms.reserve(dimension_exprs.len()); + for (alias, expr) in dimension_exprs.iter() { + alias_expr_norms.push(( + normalize_group_by_col(alias), + normalize_group_by_col(expr), + )); + } - let mut end = i; - while end > 0 && bytes[end - 1].is_ascii_whitespace() { - end -= 1; - } - let mut start = end; - while start > 0 && is_measure_ref_char(bytes[start - 1]) { - start -= 1; + outer_cols + .iter() + .filter(|col| { + let normalized_outer = normalize_group_by_col(col); + if view_set.contains(&normalized_outer) { + return true; } - if start < end { - let left = where_clause[start..end].trim(); - if let Some((qualifier, name)) = parse_simple_measure_ref(left) { - if name.eq_ignore_ascii_case(dim_name) { - let qualifier_ok = match (&expected_qualifier, qualifier) { - (Some(expected), Some(found)) => found.eq_ignore_ascii_case(expected), - _ => true, - }; - if qualifier_ok { - return true; - } - } + if let Some(expr) = dimension_exprs.get(&normalized_outer) { + let normalized_expr = normalize_group_by_col(expr); + if view_set.contains(&normalized_expr) { + return true; } } - } - i += 1; - } - false + alias_expr_norms.iter().any(|(alias_norm, expr_norm)| { + expr_norm == &normalized_outer && view_set.contains(alias_norm) + }) + }) + .cloned() + .collect() } -fn current_dimension_is_single_valued( - dim: &str, - group_by_cols: &[String], - outer_where: Option<&str>, - default_qualifier: Option<&str>, -) -> bool { - if dimension_in_group_by(dim, group_by_cols, default_qualifier) { - return true; - } +fn source_dimension_names(source_view: &str) -> HashSet { + let views = MEASURE_VIEWS.lock().unwrap(); + let Some((_, view)) = views + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(source_view)) + else { + return HashSet::new(); + }; - let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - outer_where - .map(|w| where_has_simple_equality_constraint(w, dim_name, default_qualifier)) - .unwrap_or(false) + let view_query = extract_view_query(&view.base_query).unwrap_or_else(|| view.base_query.clone()); + let mut dims: HashSet = extract_dimension_columns_from_select(&view_query) + .into_iter() + .map(|col| { + let dim_name = col.split('.').next_back().unwrap_or(col.as_str()).trim(); + normalize_dimension_key(dim_name) + }) + .collect(); + dims.extend(view.dimension_exprs.keys().map(|alias| { + normalize_dimension_key(alias.split('.').next_back().unwrap_or(alias).trim()) + })); + dims } -fn resolve_current_in_expr( - expr: &str, - group_by_cols: &[String], - outer_where: Option<&str>, - default_qualifier: Option<&str>, -) -> String { - let bytes = expr.as_bytes(); - let mut out = String::new(); - let mut i = 0usize; +fn can_use_view_measure_directly(resolved: &ResolvedMeasure, outer_group_by: &[String]) -> bool { + group_by_matches_view(outer_group_by, &resolved.view_group_by_cols) +} - while i < bytes.len() { - match bytes[i] { - b'\'' => { - out.push('\''); - i += 1; - while i < bytes.len() { - out.push(bytes[i] as char); - if bytes[i] == b'\'' { - if i + 1 < bytes.len() && bytes[i + 1] == b'\'' { - i += 1; - out.push(bytes[i] as char); - } else { - i += 1; - break; - } - } - i += 1; - } - } - c if (c as char).is_alphabetic() || c == b'_' => { - let token_start = i; - i += 1; - while i < bytes.len() && ((bytes[i] as char).is_alphanumeric() || bytes[i] == b'_') { - i += 1; - } - let token = &expr[token_start..i]; +/// Extract WHERE clause from SQL query +pub fn extract_where_clause(sql: &str) -> Option { + let query = sql.trim().trim_end_matches(';').trim(); + let where_pos = find_top_level_keyword(query, "WHERE", 0)?; + let start = where_pos + 5; + let end = find_first_top_level_keyword( + query, + start, + &[ + "GROUP BY", + "HAVING", + "QUALIFY", + "ORDER BY", + "LIMIT", + "WINDOW", + "UNION", + "INTERSECT", + "EXCEPT", + ], + ) + .unwrap_or(query.len()); - if token.eq_ignore_ascii_case("CURRENT") { - let mut j = i; - while j < bytes.len() && bytes[j].is_ascii_whitespace() { - j += 1; - } + let where_content = query[start..end].trim().to_string(); + if where_content.is_empty() { + None + } else { + Some(where_content) + } +} - if j < bytes.len() - && (((bytes[j] as char).is_alphabetic()) || bytes[j] == b'_') - { - let dim_start = j; - j += 1; - while j < bytes.len() - && ((bytes[j] as char).is_alphanumeric() || bytes[j] == b'_') - { - j += 1; - } - while j < bytes.len() && bytes[j] == b'.' { - let mut k = j + 1; - if k >= bytes.len() - || !(((bytes[k] as char).is_alphabetic()) || bytes[k] == b'_') - { - break; - } - k += 1; - while k < bytes.len() - && ((bytes[k] as char).is_alphanumeric() || bytes[k] == b'_') - { - k += 1; - } - j = k; - } +// ============================================================================= +// Nom Parsers - Expression Helpers +// ============================================================================= - let dim = expr[dim_start..j].trim(); - if current_dimension_is_single_valued( - dim, - group_by_cols, - outer_where, - default_qualifier, - ) { - out.push_str(dim); - } else { - out.push_str("NULL"); - } - i = j; - continue; - } - } +/// Parse aggregation function name - accepts any identifier followed by ( +/// This allows all DuckDB aggregate functions to work as measures +fn agg_function_name(input: &str) -> IResult<&str, &str> { + // Match any identifier (alphanumeric + underscore) that's followed by ( + let (remaining, name) = nom::bytes::complete::take_while1(|c: char| c.is_alphanumeric() || c == '_')(input)?; + // Verify it's followed by a paren (but don't consume it) + let _ = nom::character::complete::char('(')(remaining)?; + Ok((remaining, name)) +} - out.push_str(token); - } - _ => { - out.push(bytes[i] as char); - i += 1; - } - } - } +/// Extract aggregation function from expression like "SUM(amount)" +pub fn extract_agg_function(expr: &str) -> String { + agg_function_name(expr.trim()) + .map(|(_, name)| name.to_uppercase()) + .unwrap_or_else(|_| "SUM".to_string()) +} - out +/// Extract aggregation function name from full expression +/// "SUM(amount)" -> "sum" +/// "COUNT(DISTINCT x)" -> "count" +pub fn extract_aggregation_function(expr: &str) -> Option { + let (_, (name, _)) = function_call(expr.trim()).ok()?; + Some(name.to_lowercase()) } -/// Qualify column references in a WHERE clause for use inside _inner subquery -/// "region = 'US'" -> "_inner.region = 'US'" -/// "year > 2020 AND region = 'US'" -> "_inner.year > 2020 AND _inner.region = 'US'" -fn qualify_where_for_inner(where_clause: &str) -> String { - if let Ok(result) = parser_ffi::qualify_expression(where_clause, "_inner") { - return result; +/// Check if expression contains DISTINCT modifier +/// "COUNT(DISTINCT region)" -> true +fn has_distinct_modifier(expr: &str) -> bool { + let expr_upper = expr.to_uppercase(); + // Look for DISTINCT after opening paren + if let Some(paren_pos) = expr_upper.find('(') { + let after_paren = &expr_upper[paren_pos + 1..]; + after_paren.trim_start().starts_with("DISTINCT") + } else { + false } - qualify_where_for_inner_fallback(where_clause) } -fn qualify_where_for_inner_fallback(where_clause: &str) -> String { - let mut result = String::new(); - let mut chars = where_clause.chars().peekable(); - let mut previous_was_dot = false; - let mut previous_was_as = false; - // Track interval state: 0=normal, 1=saw INTERVAL keyword, 2=saw INTERVAL + string literal - let mut interval_state: u8 = 0; +/// Returns true if expression appears to use a SQL window clause. +fn is_window_expression(expr: &str) -> bool { + let bytes = expr.as_bytes(); + let mut i = 0; - while let Some(c) = chars.next() { - if c.is_alphabetic() || c == '_' { - // Collect identifier - let mut ident = String::from(c); - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - ident.push(chars.next().unwrap()); - } else { + let is_word_boundary = |b: Option| b.map_or(true, |c| !c.is_ascii_alphanumeric() && c != b'_'); + + while i < bytes.len() { + if bytes[i] == b'\'' { + i += 1; + while i < bytes.len() { + if bytes[i] == b'\'' { + if i + 1 < bytes.len() && bytes[i + 1] == b'\'' { + i += 2; + continue; + } + i += 1; break; } + i += 1; } + continue; + } - // Check if followed by a dot (already qualified) or paren (function call) - let already_qualified = previous_was_dot || chars.peek() == Some(&'.'); - let is_function = chars.peek() == Some(&'('); - - // SQL keywords that should not be prefixed - let keywords = [ - "AND", "OR", "NOT", "IN", "IS", "AS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", - "EXISTS", "CASE", "WHEN", "THEN", "ELSE", "END", - ]; - let is_keyword = keywords.iter().any(|kw| kw.eq_ignore_ascii_case(&ident)); - let is_type_context = previous_was_as; - let is_typed_literal = is_typed_literal_keyword(&ident, &chars); - let is_date_part = interval_state == 2 && is_interval_date_part_keyword(&ident); - - if !already_qualified && !is_keyword && !is_function && !is_typed_literal && !is_type_context && !is_date_part { - result.push_str("_inner."); - } - result.push_str(&ident); - previous_was_dot = false; - previous_was_as = ident.eq_ignore_ascii_case("AS"); - // Update interval state - if ident.eq_ignore_ascii_case("INTERVAL") && is_typed_literal { - interval_state = 1; - } else { - interval_state = 0; - } - } else if c == '\'' { - // String literal - copy as-is until closing quote - result.push(c); - previous_was_dot = false; - while let Some(next) = chars.next() { - result.push(next); - if next == '\'' { - // Check for escaped quote '' - if chars.peek() == Some(&'\'') { - result.push(chars.next().unwrap()); - } else { - break; + if bytes[i] == b'"' { + i += 1; + while i < bytes.len() { + if bytes[i] == b'"' { + if i + 1 < bytes.len() && bytes[i + 1] == b'"' { + i += 2; + continue; } + i += 1; + break; } + i += 1; } - if interval_state == 1 { - interval_state = 2; - } - } else if c == '(' { - // Check if this opens a subquery; if so, emit it verbatim - if let Some(subquery) = try_consume_subquery_parens(&mut chars) { - result.push_str(&subquery); - } else { - result.push(c); + continue; + } + + if bytes[i] == b'`' { + i += 1; + while i < bytes.len() && bytes[i] != b'`' { + i += 1; } - previous_was_dot = false; - } else { - result.push(c); - previous_was_dot = c == '.'; - // Non-whitespace after interval string means no date part follows - if interval_state == 2 && !c.is_whitespace() { - interval_state = 0; + if i < bytes.len() { + i += 1; } + continue; } - } - result -} + if bytes[i] == b'[' { + i += 1; + while i < bytes.len() && bytes[i] != b']' { + i += 1; + } + if i < bytes.len() { + i += 1; + } + continue; + } -fn qualify_where_for_outer(where_clause: &str, outer_alias: &str) -> String { - if let Ok(result) = parser_ffi::qualify_expression(where_clause, outer_alias) { - return result; - } - qualify_where_for_outer_fallback(where_clause, outer_alias) -} + if bytes[i] == b'-' && i + 1 < bytes.len() && bytes[i + 1] == b'-' { + i += 2; + while i < bytes.len() { + let ch = bytes[i]; + i += 1; + if ch == b'\n' || ch == b'\r' { + break; + } + } + continue; + } -fn qualify_where_for_outer_fallback(where_clause: &str, outer_alias: &str) -> String { - let mut result = String::new(); - let mut chars = where_clause.chars().peekable(); - let mut previous_was_dot = false; - let mut previous_was_as = false; - let mut interval_state: u8 = 0; + if bytes[i] == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' { + i += 2; + while i + 1 < bytes.len() { + if bytes[i] == b'*' && bytes[i + 1] == b'/' { + i += 2; + break; + } + i += 1; + } + if i + 1 >= bytes.len() { + i = bytes.len(); + } + continue; + } - while let Some(c) = chars.next() { - if c.is_alphabetic() || c == '_' { - let mut ident = String::from(c); - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - ident.push(chars.next().unwrap()); + // Window clauses follow a function invocation: ") OVER (...)|OVER window_name" + if bytes[i] == b')' { + let mut j = skip_ws_and_comments(expr, i + 1); + let has_over = j + 4 <= bytes.len() + && bytes[j..j + 4] + .iter() + .zip(b"OVER") + .all(|(b, kw)| b.to_ascii_uppercase() == *kw) + && is_word_boundary(if j == 0 { None } else { Some(bytes[j - 1]) }) + && is_word_boundary(if j + 4 >= bytes.len() { + None } else { - break; + Some(bytes[j + 4]) + }); + + if has_over { + j += 4; + j = skip_ws_and_comments(expr, j); + if j < bytes.len() && (bytes[j] == b'(' || parse_identifier_token(expr, j).is_some()) { + return true; } } + } - let already_qualified = previous_was_dot || chars.peek() == Some(&'.'); - let is_function = chars.peek() == Some(&'('); + i += 1; + } - let keywords = [ - "AND", "OR", "NOT", "IN", "IS", "AS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", - "EXISTS", "CASE", "WHEN", "THEN", "ELSE", "END", - ]; - let is_keyword = keywords.iter().any(|kw| kw.eq_ignore_ascii_case(&ident)); - let is_type_context = previous_was_as; - let is_typed_literal = is_typed_literal_keyword(&ident, &chars); - let is_date_part = interval_state == 2 && is_interval_date_part_keyword(&ident); + false +} - if !already_qualified && !is_keyword && !is_function && !is_typed_literal && !is_type_context && !is_date_part { - result.push_str(outer_alias); - result.push('.'); +/// Non-decomposable aggregate functions that require recompute from base rows +const NON_DECOMPOSABLE_AGGREGATES: &[&str] = &[ + "MEDIAN", + "PERCENTILE_CONT", + "PERCENTILE_DISC", + "MODE", + "QUANTILE", + "QUANTILE_CONT", + "QUANTILE_DISC", +]; + +/// Returns true if expression uses a non-decomposable aggregate +/// (including COUNT DISTINCT and ordered-set aggregates) +fn is_non_decomposable(expr: &str) -> bool { + if has_distinct_modifier(expr) { + return true; + } + + let expr_upper = expr.to_uppercase(); + NON_DECOMPOSABLE_AGGREGATES + .iter() + .any(|agg| expr_upper.contains(&format!("{agg}("))) +} + +/// Find any aggregation function inside an expression +/// "CASE WHEN SUM(x) > 100 THEN 1 ELSE 0 END" -> Some("sum") +fn find_aggregation_in_expression(expr: &str) -> Option { + // Look for any function call pattern: identifier followed by ( + // This allows all DuckDB aggregate functions to work + let expr_upper = expr.to_uppercase(); + + // Common aggregates to check first (for performance) + let common_aggs = [ + "SUM", "COUNT", "AVG", "MIN", "MAX", "MEDIAN", "STDDEV", "STDDEV_POP", + "STDDEV_SAMP", "VARIANCE", "VAR_POP", "VAR_SAMP", "STRING_AGG", + "ARRAY_AGG", "LIST", "FIRST", "LAST", "MODE", "QUANTILE", + ]; + + for agg in common_aggs { + if expr_upper.contains(&format!("{agg}(")) { + return Some(agg.to_lowercase()); + } + } + + // Fallback: look for any identifier followed by ( that could be an aggregate + // This catches custom aggregates + let chars: Vec = expr.chars().collect(); + let mut i = 0; + while i < chars.len() { + // Look for start of identifier + if chars[i].is_alphabetic() || chars[i] == '_' { + let start = i; + while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') { + i += 1; } - result.push_str(&ident); - previous_was_dot = false; - previous_was_as = ident.eq_ignore_ascii_case("AS"); - if ident.eq_ignore_ascii_case("INTERVAL") && is_typed_literal { - interval_state = 1; - } else { - interval_state = 0; + // Check if followed by ( + if i < chars.len() && chars[i] == '(' { + let name: String = chars[start..i].iter().collect(); + // Skip known non-aggregates + let name_upper = name.to_uppercase(); + if !["CASE", "WHEN", "THEN", "ELSE", "END", "AND", "OR", "NOT", "IN", "IS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", "CAST", "COALESCE", "NULLIF", "IF", "IIF"].contains(&name_upper.as_str()) { + return Some(name.to_lowercase()); + } } - } else if c == '\'' { + } + i += 1; + } + + None +} + +/// Check if an expression contains COUNT(DISTINCT ...) which is non-decomposable +/// "COUNT(DISTINCT user_id)" -> true +/// "COUNT(user_id)" -> false +/// "SUM(amount)" -> false +pub fn is_count_distinct(expr: &str) -> bool { + let expr_upper = expr.to_uppercase(); + // Match COUNT followed by optional whitespace, (, optional whitespace, DISTINCT + // This handles: COUNT(DISTINCT x), COUNT( DISTINCT x), COUNT (DISTINCT x) + let patterns = ["COUNT(DISTINCT", "COUNT( DISTINCT", "COUNT (DISTINCT"]; + patterns.iter().any(|p| expr_upper.contains(p)) +} + +/// Expand derived measure expression by replacing measure references with their aggregations +/// "revenue - cost" with measures [revenue=SUM(amount), cost=SUM(expense)] +/// -> "SUM(revenue) - SUM(cost)" +fn expand_derived_measure_expr(expr: &str, measure_view: &MeasureView) -> String { + let mut result = String::new(); + let mut chars = expr.chars().peekable(); + + while let Some(c) = chars.next() { + if c == '\'' { result.push(c); - previous_was_dot = false; while let Some(next) = chars.next() { result.push(next); if next == '\'' { @@ -3349,34 +3228,84 @@ fn qualify_where_for_outer_fallback(where_clause: &str, outer_alias: &str) -> St } } } - if interval_state == 1 { - interval_state = 2; + continue; + } + if c == '"' { + result.push(c); + while let Some(next) = chars.next() { + result.push(next); + if next == '"' { + if chars.peek() == Some(&'"') { + result.push(chars.next().unwrap()); + } else { + break; + } + } } - } else if c == '(' { - if let Some(subquery) = try_consume_subquery_parens(&mut chars) { - result.push_str(&subquery); + continue; + } + if c == '`' { + result.push(c); + while let Some(next) = chars.next() { + result.push(next); + if next == '`' { + break; + } + } + continue; + } + if c == '[' { + result.push(c); + while let Some(next) = chars.next() { + result.push(next); + if next == ']' { + break; + } + } + continue; + } + if c.is_alphabetic() || c == '_' { + // Collect identifier + let mut ident = String::from(c); + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + ident.push(chars.next().unwrap()); + } else { + break; + } + } + + // Check if this identifier is a measure name + if let Some(m) = measure_view + .measures + .iter() + .find(|m| m.column_name.eq_ignore_ascii_case(&ident)) + { + result.push('('); + result.push_str(&m.expression); + result.push(')'); } else { - result.push(c); + // Not a measure, keep as-is + result.push_str(&ident); } - previous_was_dot = false; } else { result.push(c); - previous_was_dot = c == '.'; - if interval_state == 2 && !c.is_whitespace() { - interval_state = 0; - } } } result } -fn strip_at_where_qualifiers(condition: &str) -> String { +/// Qualify dimension reference in expression for correlated subquery +/// "year - 1" with table "sales" and dim "year" -> "sales.year - 1" +pub fn qualify_outer_reference(expr: &str, table_name: &str, dim: &str) -> String { + // Parse expression into tokens and replace matching identifiers let mut result = String::new(); - let mut chars = condition.chars().peekable(); + let mut chars = expr.chars().peekable(); while let Some(c) = chars.next() { if c.is_alphabetic() || c == '_' { + // Collect identifier let mut ident = String::from(c); while let Some(&next) = chars.peek() { if next.is_alphanumeric() || next == '_' { @@ -3386,228 +3315,373 @@ fn strip_at_where_qualifiers(condition: &str) -> String { } } - let mut last_ident = ident; - while chars.peek() == Some(&'.') { - chars.next(); // consume '.' - if let Some(&next) = chars.peek() { - if next.is_alphabetic() || next == '_' { - let mut next_ident = String::new(); - while let Some(&next_ch) = chars.peek() { - if next_ch.is_alphanumeric() || next_ch == '_' { - next_ident.push(chars.next().unwrap()); - } else { - break; - } - } - last_ident = next_ident; - continue; - } else { - result.push_str(&last_ident); - result.push('.'); - break; - } - } else { - result.push_str(&last_ident); - result.push('.'); - break; - } + // Check if this identifier matches the dimension + if ident == dim { + result.push_str(table_name); + result.push('.'); } - - result.push_str(&last_ident); - } else if c == '\'' { + result.push_str(&ident); + } else { result.push(c); + } + } + + result +} + +fn expr_mentions_identifier(expr: &str, ident: &str) -> bool { + let mut chars = expr.chars().peekable(); + + while let Some(c) = chars.next() { + if c == '\'' { + // Skip SQL single-quoted string literal content, including escaped ''. while let Some(next) = chars.next() { - result.push(next); if next == '\'' { if chars.peek() == Some(&'\'') { - result.push(chars.next().unwrap()); + chars.next(); } else { break; } } } - } else { - result.push(c); + continue; + } + + if c.is_alphabetic() || c == '_' { + let mut token = String::from(c); + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + token.push(chars.next().unwrap()); + } else { + break; + } + } + + if token.eq_ignore_ascii_case(ident) { + return true; + } } } - result + false } -fn rewrite_percentile_within_group(sql: &str) -> String { - let upper = sql.to_uppercase(); - let upper_bytes = upper.as_bytes(); - let bytes = sql.as_bytes(); - let mut out = String::new(); - let mut i = 0; +fn dimension_in_group_by( + dim: &str, + group_by_cols: &[String], + default_qualifier: Option<&str>, +) -> bool { + let dim_trim = dim.trim(); + let dim_name = dim_trim.split('.').next_back().unwrap_or(dim_trim).trim(); + let explicit_dim_qualifier = dim_trim + .rsplit_once('.') + .map(|(qualifier, _)| qualifier.trim()); + let expected_qualifier = explicit_dim_qualifier.or(default_qualifier); + let dim_lower = dim_trim.to_lowercase(); - while i < bytes.len() { - let (func_name, quantile_name) = if upper_bytes[i..].starts_with(b"PERCENTILE_CONT") { - ("PERCENTILE_CONT", "QUANTILE_CONT") - } else if upper_bytes[i..].starts_with(b"PERCENTILE_DISC") { - ("PERCENTILE_DISC", "QUANTILE_DISC") - } else { - out.push(bytes[i] as char); - i += 1; - continue; - }; + if dim_trim.contains('(') { + return group_by_cols + .iter() + .any(|col| col.to_lowercase() == dim_lower); + } - let name_len = func_name.len(); - let mut j = i + name_len; - while j < bytes.len() && bytes[j].is_ascii_whitespace() { - j += 1; - } - if j >= bytes.len() || bytes[j] != b'(' { - out.push(bytes[i] as char); - i += 1; - continue; + group_by_cols.iter().any(|col| { + let col_trim = col.trim(); + let col_name = col_trim.split('.').next_back().unwrap_or(col_trim).trim(); + + if !col_name.eq_ignore_ascii_case(dim_name) { + return false; } - let args_start = j + 1; - let (after_args, args) = match balanced_parens(&sql[args_start..]) { - Ok(res) => res, - Err(_) => { - out.push(bytes[i] as char); - i += 1; - continue; + // When we know the expected outer qualifier, GROUP BY qualifiers must match it. + // Unqualified GROUP BY columns still count. + match (expected_qualifier, col_trim.rsplit_once('.')) { + (Some(expected), Some((col_qualifier, _))) => { + col_qualifier.trim().eq_ignore_ascii_case(expected) } - }; - let args_len = args.len(); - let args_end = args_start + args_len; - if args_end >= sql.len() || !after_args.starts_with(')') { - out.push(bytes[i] as char); - i += 1; - continue; + _ => true, } + }) +} - let mut k = args_end + 1; - while k < bytes.len() && bytes[k].is_ascii_whitespace() { - k += 1; - } - if k >= bytes.len() || !upper_bytes[k..].starts_with(b"WITHIN") { - out.push(bytes[i] as char); - i += 1; - continue; - } - k += "WITHIN".len(); - while k < bytes.len() && bytes[k].is_ascii_whitespace() { - k += 1; - } - if k >= bytes.len() || !upper_bytes[k..].starts_with(b"GROUP") { - out.push(bytes[i] as char); - i += 1; - continue; - } - k += "GROUP".len(); - while k < bytes.len() && bytes[k].is_ascii_whitespace() { - k += 1; - } - if k >= bytes.len() || bytes[k] != b'(' { - out.push(bytes[i] as char); - i += 1; - continue; - } +fn expr_mentions_identifier_outside_current(expr: &str, ident: &str) -> bool { + let mut chars = expr.chars().peekable(); + let mut pending_current = false; - let inner_start = k + 1; - let (after_inner, inner) = match balanced_parens(&sql[inner_start..]) { - Ok(res) => res, - Err(_) => { - out.push(bytes[i] as char); - i += 1; - continue; + while let Some(c) = chars.next() { + if c == '\'' { + while let Some(next) = chars.next() { + if next == '\'' { + if chars.peek() == Some(&'\'') { + chars.next(); + } else { + break; + } + } } - }; - let inner_len = inner.len(); - let inner_end = inner_start + inner_len; - if inner_end >= sql.len() || !after_inner.starts_with(')') { - out.push(bytes[i] as char); - i += 1; - continue; - } - - let inner_trim = inner.trim(); - let inner_upper = inner_trim.to_uppercase(); - if !inner_upper.starts_with("ORDER BY") { - out.push(bytes[i] as char); - i += 1; - continue; - } - let order_expr = inner_trim["ORDER BY".len()..].trim(); - if order_expr.is_empty() { - out.push(bytes[i] as char); - i += 1; continue; } - out.push_str(&format!( - "{quantile_name}({order_expr}, {})", - args.trim() - )); - i = inner_end + 1; - } - - out -} - -fn qualify_where_for_inner_with_dimensions( - where_clause: &str, - dimension_exprs: &HashMap, -) -> String { - let mut result = String::new(); - let mut chars = where_clause.chars().peekable(); - let mut previous_was_dot = false; - let mut previous_was_as = false; - let mut interval_state: u8 = 0; - - while let Some(c) = chars.next() { if c.is_alphabetic() || c == '_' { - let mut ident = String::from(c); + let mut token = String::from(c); while let Some(&next) = chars.peek() { if next.is_alphanumeric() || next == '_' { - ident.push(chars.next().unwrap()); + token.push(chars.next().unwrap()); } else { break; } } - let already_qualified = previous_was_dot || chars.peek() == Some(&'.'); - let is_function = chars.peek() == Some(&'('); - - let keywords = [ - "AND", "OR", "NOT", "IN", "IS", "AS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", - "EXISTS", "CASE", "WHEN", "THEN", "ELSE", "END", - ]; - let is_keyword = keywords.iter().any(|kw| kw.eq_ignore_ascii_case(&ident)); - let is_type_context = previous_was_as; - let is_typed_literal = is_typed_literal_keyword(&ident, &chars); - let is_date_part = interval_state == 2 && is_interval_date_part_keyword(&ident); + if token.eq_ignore_ascii_case("CURRENT") { + pending_current = true; + continue; + } - if !already_qualified && !is_keyword && !is_function && !is_typed_literal && !is_type_context && !is_date_part { - let key = normalize_dimension_key(&ident); - if let Some(expr) = dimension_exprs.get(&key) { - let inner_expr = qualify_where_for_inner(expr); - result.push('('); - result.push_str(&inner_expr); - result.push(')'); - interval_state = 0; - continue; + if token.eq_ignore_ascii_case(ident) { + if pending_current { + pending_current = false; + } else { + return true; } - result.push_str("_inner."); + } else { + pending_current = false; + } + } + } + + false +} + +fn where_has_simple_equality_constraint( + where_clause: &str, + dim_name: &str, + default_qualifier: Option<&str>, +) -> bool { + // Conservative: if OR is present we don't claim single-valued context. + if where_clause.to_uppercase().contains(" OR ") { + return false; + } + + let expected_qualifier = default_qualifier.map(normalize_identifier_name); + let bytes = where_clause.as_bytes(); + let mut i = 0usize; + + while i < bytes.len() { + if bytes[i] == b'=' { + if (i > 0 && matches!(bytes[i - 1], b'<' | b'>' | b'!' | b'=')) + || (i + 1 < bytes.len() && bytes[i + 1] == b'=') + { + i += 1; + continue; + } + + let mut end = i; + while end > 0 && bytes[end - 1].is_ascii_whitespace() { + end -= 1; + } + let mut start = end; + while start > 0 && is_measure_ref_char(bytes[start - 1]) { + start -= 1; + } + + if start < end { + let left = where_clause[start..end].trim(); + if let Some((qualifier, name)) = parse_simple_measure_ref(left) { + if name.eq_ignore_ascii_case(dim_name) { + let qualifier_ok = match (&expected_qualifier, qualifier) { + (Some(expected), Some(found)) => found.eq_ignore_ascii_case(expected), + _ => true, + }; + if qualifier_ok { + return true; + } + } + } + } + } + i += 1; + } + + false +} + +fn current_dimension_is_single_valued( + dim: &str, + group_by_cols: &[String], + outer_where: Option<&str>, + default_qualifier: Option<&str>, +) -> bool { + if dimension_in_group_by(dim, group_by_cols, default_qualifier) { + return true; + } + + let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); + outer_where + .map(|w| where_has_simple_equality_constraint(w, dim_name, default_qualifier)) + .unwrap_or(false) +} + +fn resolve_current_in_expr( + expr: &str, + group_by_cols: &[String], + outer_where: Option<&str>, + default_qualifier: Option<&str>, +) -> String { + let bytes = expr.as_bytes(); + let mut out = String::new(); + let mut i = 0usize; + + while i < bytes.len() { + match bytes[i] { + b'\'' => { + out.push('\''); + i += 1; + while i < bytes.len() { + out.push(bytes[i] as char); + if bytes[i] == b'\'' { + if i + 1 < bytes.len() && bytes[i + 1] == b'\'' { + i += 1; + out.push(bytes[i] as char); + } else { + i += 1; + break; + } + } + i += 1; + } + } + c if (c as char).is_alphabetic() || c == b'_' => { + let token_start = i; + i += 1; + while i < bytes.len() && ((bytes[i] as char).is_alphanumeric() || bytes[i] == b'_') { + i += 1; + } + let token = &expr[token_start..i]; + + if token.eq_ignore_ascii_case("CURRENT") { + let mut j = i; + while j < bytes.len() && bytes[j].is_ascii_whitespace() { + j += 1; + } + + if j < bytes.len() + && (((bytes[j] as char).is_alphabetic()) || bytes[j] == b'_') + { + let dim_start = j; + j += 1; + while j < bytes.len() + && ((bytes[j] as char).is_alphanumeric() || bytes[j] == b'_') + { + j += 1; + } + while j < bytes.len() && bytes[j] == b'.' { + let mut k = j + 1; + if k >= bytes.len() + || !(((bytes[k] as char).is_alphabetic()) || bytes[k] == b'_') + { + break; + } + k += 1; + while k < bytes.len() + && ((bytes[k] as char).is_alphanumeric() || bytes[k] == b'_') + { + k += 1; + } + j = k; + } + + let dim = expr[dim_start..j].trim(); + if current_dimension_is_single_valued( + dim, + group_by_cols, + outer_where, + default_qualifier, + ) { + out.push_str(dim); + } else { + out.push_str("NULL"); + } + i = j; + continue; + } + } + + out.push_str(token); + } + _ => { + out.push(bytes[i] as char); + i += 1; + } + } + } + + out +} + +/// Qualify column references in a WHERE clause for use inside _inner subquery +/// "region = 'US'" -> "_inner.region = 'US'" +/// "year > 2020 AND region = 'US'" -> "_inner.year > 2020 AND _inner.region = 'US'" +fn qualify_where_for_inner(where_clause: &str) -> String { + if let Ok(result) = parser_ffi::qualify_expression(where_clause, "_inner") { + return result; + } + qualify_where_for_inner_fallback(where_clause) +} + +fn qualify_where_for_inner_fallback(where_clause: &str) -> String { + let mut result = String::new(); + let mut chars = where_clause.chars().peekable(); + let mut previous_was_dot = false; + let mut previous_was_as = false; + // Track interval state: 0=normal, 1=saw INTERVAL keyword, 2=saw INTERVAL + string literal + let mut interval_state: u8 = 0; + + while let Some(c) = chars.next() { + if c.is_alphabetic() || c == '_' { + // Collect identifier + let mut ident = String::from(c); + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + ident.push(chars.next().unwrap()); + } else { + break; + } + } + + // Check if followed by a dot (already qualified) or paren (function call) + let already_qualified = previous_was_dot || chars.peek() == Some(&'.'); + let is_function = chars.peek() == Some(&'('); + + // SQL keywords that should not be prefixed + let keywords = [ + "AND", "OR", "NOT", "IN", "IS", "AS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", + "EXISTS", "CASE", "WHEN", "THEN", "ELSE", "END", + ]; + let is_keyword = keywords.iter().any(|kw| kw.eq_ignore_ascii_case(&ident)); + let is_type_context = previous_was_as; + let is_typed_literal = is_typed_literal_keyword(&ident, &chars); + let is_date_part = interval_state == 2 && is_interval_date_part_keyword(&ident); + + if !already_qualified && !is_keyword && !is_function && !is_typed_literal && !is_type_context && !is_date_part { + result.push_str("_inner."); } result.push_str(&ident); previous_was_dot = false; previous_was_as = ident.eq_ignore_ascii_case("AS"); + // Update interval state if ident.eq_ignore_ascii_case("INTERVAL") && is_typed_literal { interval_state = 1; } else { interval_state = 0; } } else if c == '\'' { + // String literal - copy as-is until closing quote result.push(c); previous_was_dot = false; while let Some(next) = chars.next() { result.push(next); if next == '\'' { + // Check for escaped quote '' if chars.peek() == Some(&'\'') { result.push(chars.next().unwrap()); } else { @@ -3619,6 +3693,7 @@ fn qualify_where_for_inner_with_dimensions( interval_state = 2; } } else if c == '(' { + // Check if this opens a subquery; if so, emit it verbatim if let Some(subquery) = try_consume_subquery_parens(&mut chars) { result.push_str(&subquery); } else { @@ -3628,6 +3703,7 @@ fn qualify_where_for_inner_with_dimensions( } else { result.push(c); previous_was_dot = c == '.'; + // Non-whitespace after interval string means no date part follows if interval_state == 2 && !c.is_whitespace() { interval_state = 0; } @@ -3637,2080 +3713,1761 @@ fn qualify_where_for_inner_with_dimensions( result } -/// After encountering '(', check if it starts a subquery (SELECT/WITH). -/// If so, consume everything up to and including the matching ')' from the -/// original iterator and return the full subquery text including outer parens. -/// If not a subquery, return None and leave the iterator unchanged. -fn try_consume_subquery_parens( - chars: &mut std::iter::Peekable>, -) -> Option { - // Clone the iterator to peek ahead without consuming - let mut lookahead = chars.clone(); - - // Skip whitespace - while let Some(&c) = lookahead.peek() { - if c.is_whitespace() { - lookahead.next(); - } else { - break; - } +fn qualify_where_for_outer(where_clause: &str, outer_alias: &str) -> String { + if let Ok(result) = parser_ffi::qualify_expression(where_clause, outer_alias) { + return result; } + qualify_where_for_outer_fallback(where_clause, outer_alias) +} - // Collect first token (include digits so e.g. "select1" is not confused with "SELECT") - let mut first_token = String::new(); - while let Some(&c) = lookahead.peek() { - if c.is_alphanumeric() || c == '_' { - first_token.push(c); - lookahead.next(); - } else { - break; - } - } - - // Only treat as subquery if it starts with SELECT or WITH - if !first_token.eq_ignore_ascii_case("SELECT") && !first_token.eq_ignore_ascii_case("WITH") { - return None; - } - - // It's a subquery: consume everything from the original iterator until the - // matching closing paren. We already consumed the opening '(' before this - // function was called, so we start at depth 1. - let mut content = String::from("("); - let mut depth = 1; +fn qualify_where_for_outer_fallback(where_clause: &str, outer_alias: &str) -> String { + let mut result = String::new(); + let mut chars = where_clause.chars().peekable(); + let mut previous_was_dot = false; + let mut previous_was_as = false; + let mut interval_state: u8 = 0; while let Some(c) = chars.next() { - content.push(c); - match c { - '(' => depth += 1, - ')' => { - depth -= 1; - if depth == 0 { - return Some(content); + if c.is_alphabetic() || c == '_' { + let mut ident = String::from(c); + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + ident.push(chars.next().unwrap()); + } else { + break; } } - '\'' => { - // Handle single-quoted string literals - while let Some(next) = chars.next() { - content.push(next); - if next == '\'' { - if chars.peek() == Some(&'\'') { - content.push(chars.next().unwrap()); - } else { - break; - } + + let already_qualified = previous_was_dot || chars.peek() == Some(&'.'); + let is_function = chars.peek() == Some(&'('); + + let keywords = [ + "AND", "OR", "NOT", "IN", "IS", "AS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", + "EXISTS", "CASE", "WHEN", "THEN", "ELSE", "END", + ]; + let is_keyword = keywords.iter().any(|kw| kw.eq_ignore_ascii_case(&ident)); + let is_type_context = previous_was_as; + let is_typed_literal = is_typed_literal_keyword(&ident, &chars); + let is_date_part = interval_state == 2 && is_interval_date_part_keyword(&ident); + + if !already_qualified && !is_keyword && !is_function && !is_typed_literal && !is_type_context && !is_date_part { + result.push_str(outer_alias); + result.push('.'); + } + result.push_str(&ident); + previous_was_dot = false; + previous_was_as = ident.eq_ignore_ascii_case("AS"); + if ident.eq_ignore_ascii_case("INTERVAL") && is_typed_literal { + interval_state = 1; + } else { + interval_state = 0; + } + } else if c == '\'' { + result.push(c); + previous_was_dot = false; + while let Some(next) = chars.next() { + result.push(next); + if next == '\'' { + if chars.peek() == Some(&'\'') { + result.push(chars.next().unwrap()); + } else { + break; } } } - '"' => { - // Handle double-quoted identifiers (may contain parens) - while let Some(next) = chars.next() { - content.push(next); - if next == '"' { - if chars.peek() == Some(&'"') { - content.push(chars.next().unwrap()); - } else { - break; - } - } + if interval_state == 1 { + interval_state = 2; + } + } else if c == '(' { + if let Some(subquery) = try_consume_subquery_parens(&mut chars) { + result.push_str(&subquery); + } else { + result.push(c); + } + previous_was_dot = false; + } else { + result.push(c); + previous_was_dot = c == '.'; + if interval_state == 2 && !c.is_whitespace() { + interval_state = 0; + } + } + } + + result +} + +fn strip_at_where_qualifiers(condition: &str) -> String { + let mut result = String::new(); + let mut chars = condition.chars().peekable(); + + while let Some(c) = chars.next() { + if c.is_alphabetic() || c == '_' { + let mut ident = String::from(c); + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + ident.push(chars.next().unwrap()); + } else { + break; } } - '-' if chars.peek() == Some(&'-') => { - // Line comment: skip until end of line - content.push(chars.next().unwrap()); // consume second '-' - while let Some(next) = chars.next() { - content.push(next); - if next == '\n' { + + let mut last_ident = ident; + while chars.peek() == Some(&'.') { + chars.next(); // consume '.' + if let Some(&next) = chars.peek() { + if next.is_alphabetic() || next == '_' { + let mut next_ident = String::new(); + while let Some(&next_ch) = chars.peek() { + if next_ch.is_alphanumeric() || next_ch == '_' { + next_ident.push(chars.next().unwrap()); + } else { + break; + } + } + last_ident = next_ident; + continue; + } else { + result.push_str(&last_ident); + result.push('.'); break; } + } else { + result.push_str(&last_ident); + result.push('.'); + break; } } - '/' if chars.peek() == Some(&'*') => { - // Block comment: skip until closing */ - content.push(chars.next().unwrap()); // consume '*' - let mut prev = ' '; - while let Some(next) = chars.next() { - content.push(next); - if prev == '*' && next == '/' { + + result.push_str(&last_ident); + } else if c == '\'' { + result.push(c); + while let Some(next) = chars.next() { + result.push(next); + if next == '\'' { + if chars.peek() == Some(&'\'') { + result.push(chars.next().unwrap()); + } else { break; } - prev = next; } } - _ => {} + } else { + result.push(c); } } - Some(content) + result } -fn is_typed_literal_keyword( - ident: &str, - chars: &std::iter::Peekable>, -) -> bool { - let is_keyword = ident.eq_ignore_ascii_case("DATE") - || ident.eq_ignore_ascii_case("TIME") - || ident.eq_ignore_ascii_case("TIMESTAMP") - || ident.eq_ignore_ascii_case("TIMESTAMPTZ") - || ident.eq_ignore_ascii_case("INTERVAL"); - if !is_keyword { - return false; - } +fn rewrite_percentile_within_group(sql: &str) -> String { + let upper = sql.to_uppercase(); + let upper_bytes = upper.as_bytes(); + let bytes = sql.as_bytes(); + let mut out = String::new(); + let mut i = 0; - let mut lookahead = chars.clone(); - while let Some(&next) = lookahead.peek() { - if next.is_whitespace() { - lookahead.next(); + while i < bytes.len() { + let (func_name, quantile_name) = if upper_bytes[i..].starts_with(b"PERCENTILE_CONT") { + ("PERCENTILE_CONT", "QUANTILE_CONT") + } else if upper_bytes[i..].starts_with(b"PERCENTILE_DISC") { + ("PERCENTILE_DISC", "QUANTILE_DISC") + } else { + out.push(bytes[i] as char); + i += 1; continue; - } - break; - } - - matches!(lookahead.peek(), Some(&'\'')) -} - -/// Check if an identifier is an interval date-part keyword (e.g. DAYS, MONTHS, YEARS). -/// These follow the SQL-standard INTERVAL '' syntax and should not -/// be qualified as column references when they appear after an interval literal. -fn is_interval_date_part_keyword(ident: &str) -> bool { - let parts = [ - "DAY", "DAYS", "HOUR", "HOURS", "MINUTE", "MINUTES", - "SECOND", "SECONDS", "MONTH", "MONTHS", "YEAR", "YEARS", - "WEEK", "WEEKS", "MICROSECOND", "MICROSECONDS", - "MILLISECOND", "MILLISECONDS", - ]; - parts.iter().any(|p| p.eq_ignore_ascii_case(ident)) -} + }; -fn push_comment_char_sanitized(out: &mut String, ch: char) { - if ch.is_ascii() { - out.push(ch); - } else { - for _ in 0..ch.len_utf8() { - out.push(' '); + let name_len = func_name.len(); + let mut j = i + name_len; + while j < bytes.len() && bytes[j].is_ascii_whitespace() { + j += 1; + } + if j >= bytes.len() || bytes[j] != b'(' { + out.push(bytes[i] as char); + i += 1; + continue; } - } -} - -fn dollar_quote_delimiter_at(sql: &str, start: usize) -> Option<&str> { - let bytes = sql.as_bytes(); - if start >= bytes.len() || bytes[start] != b'$' { - return None; - } - - let mut end = start + 1; - while end < bytes.len() && (bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_') { - end += 1; - } - if end < bytes.len() && bytes[end] == b'$' { - Some(&sql[start..=end]) - } else { - None - } -} - -fn is_escape_string_quote_at(sql: &str, quote_pos: usize) -> bool { - let bytes = sql.as_bytes(); - if quote_pos == 0 || quote_pos >= bytes.len() || bytes[quote_pos] != b'\'' { - return false; - } - let prefix_pos = quote_pos - 1; - if !matches!(bytes[prefix_pos], b'e' | b'E') { - return false; - } - prefix_pos == 0 - || !(bytes[prefix_pos - 1].is_ascii_alphanumeric() || bytes[prefix_pos - 1] == b'_') -} - -fn sanitize_non_ascii_in_sql_comments(sql: &str) -> String { - let bytes = sql.as_bytes(); - let mut out = String::with_capacity(sql.len()); - let mut i = 0; - let mut in_single = false; - let mut single_allows_backslash = false; - let mut in_double = false; - let mut in_backtick = false; - let mut in_dollar_quote: Option = None; - let mut in_line_comment = false; - let mut block_comment_depth = 0usize; - - while i < bytes.len() { - if let Some(ref delimiter) = in_dollar_quote { - if sql[i..].starts_with(delimiter) { - out.push_str(delimiter); - i += delimiter.len(); - in_dollar_quote = None; + let args_start = j + 1; + let (after_args, args) = match balanced_parens(&sql[args_start..]) { + Ok(res) => res, + Err(_) => { + out.push(bytes[i] as char); + i += 1; continue; } - - let ch = sql[i..].chars().next().unwrap(); - out.push(ch); - i += ch.len_utf8(); + }; + let args_len = args.len(); + let args_end = args_start + args_len; + if args_end >= sql.len() || !after_args.starts_with(')') { + out.push(bytes[i] as char); + i += 1; continue; } - if in_line_comment { - let ch = sql[i..].chars().next().unwrap(); - push_comment_char_sanitized(&mut out, ch); - i += ch.len_utf8(); - if ch == '\n' || ch == '\r' { - in_line_comment = false; - } - continue; + let mut k = args_end + 1; + while k < bytes.len() && bytes[k].is_ascii_whitespace() { + k += 1; } - - if block_comment_depth > 0 { - if i + 1 < bytes.len() && bytes[i] == b'/' && bytes[i + 1] == b'*' { - out.push_str("/*"); - i += 2; - block_comment_depth += 1; - continue; - } - - if i + 1 < bytes.len() && bytes[i] == b'*' && bytes[i + 1] == b'/' { - out.push_str("*/"); - i += 2; - block_comment_depth -= 1; - continue; - } - - let ch = sql[i..].chars().next().unwrap(); - push_comment_char_sanitized(&mut out, ch); - i += ch.len_utf8(); + if k >= bytes.len() || !upper_bytes[k..].starts_with(b"WITHIN") { + out.push(bytes[i] as char); + i += 1; continue; } - - let ch = sql[i..].chars().next().unwrap(); - - if in_single { - out.push(ch); - i += ch.len_utf8(); - if single_allows_backslash && ch == '\\' { - if let Some(next) = sql[i..].chars().next() { - out.push(next); - i += next.len_utf8(); - } - continue; - } - if ch == '\'' { - if i < bytes.len() && bytes[i] == b'\'' { - out.push('\''); - i += 1; - } else { - in_single = false; - single_allows_backslash = false; - } - } - continue; + k += "WITHIN".len(); + while k < bytes.len() && bytes[k].is_ascii_whitespace() { + k += 1; } - - if in_double { - out.push(ch); - i += ch.len_utf8(); - if ch == '"' { - if i < bytes.len() && bytes[i] == b'"' { - out.push('"'); - i += 1; - } else { - in_double = false; - } - } + if k >= bytes.len() || !upper_bytes[k..].starts_with(b"GROUP") { + out.push(bytes[i] as char); + i += 1; continue; } - - if in_backtick { - out.push(ch); - i += ch.len_utf8(); - if ch == '`' { - in_backtick = false; - } + k += "GROUP".len(); + while k < bytes.len() && bytes[k].is_ascii_whitespace() { + k += 1; + } + if k >= bytes.len() || bytes[k] != b'(' { + out.push(bytes[i] as char); + i += 1; continue; } - if let Some(delimiter) = dollar_quote_delimiter_at(sql, i) { - if sql[i + delimiter.len()..].contains(delimiter) { - out.push_str(delimiter); - i += delimiter.len(); - in_dollar_quote = Some(delimiter.to_string()); + let inner_start = k + 1; + let (after_inner, inner) = match balanced_parens(&sql[inner_start..]) { + Ok(res) => res, + Err(_) => { + out.push(bytes[i] as char); + i += 1; continue; } + }; + let inner_len = inner.len(); + let inner_end = inner_start + inner_len; + if inner_end >= sql.len() || !after_inner.starts_with(')') { + out.push(bytes[i] as char); + i += 1; + continue; } - if i + 1 < bytes.len() && bytes[i] == b'-' && bytes[i + 1] == b'-' { - out.push_str("--"); - i += 2; - in_line_comment = true; + let inner_trim = inner.trim(); + let inner_upper = inner_trim.to_uppercase(); + if !inner_upper.starts_with("ORDER BY") { + out.push(bytes[i] as char); + i += 1; continue; } - - if i + 1 < bytes.len() && bytes[i] == b'/' && bytes[i + 1] == b'*' { - out.push_str("/*"); - i += 2; - block_comment_depth = 1; + let order_expr = inner_trim["ORDER BY".len()..].trim(); + if order_expr.is_empty() { + out.push(bytes[i] as char); + i += 1; continue; } - out.push(ch); - i += ch.len_utf8(); - - match ch { - '\'' => { - in_single = true; - single_allows_backslash = is_escape_string_quote_at(sql, i - ch.len_utf8()); - } - '"' => in_double = true, - '`' => in_backtick = true, - _ => {} - } + out.push_str(&format!( + "{quantile_name}({order_expr}, {})", + args.trim() + )); + i = inner_end + 1; } out } -// ============================================================================= -// Core Functions - Process CREATE VIEW -// ============================================================================= - -/// Process CREATE VIEW statement, extracting AS MEASURE definitions -pub fn process_create_view(sql: &str) -> CreateViewResult { - if !has_as_measure(sql) { - return CreateViewResult { - is_measure_view: false, - view_name: None, - clean_sql: sql.to_string(), - measures: vec![], - error: None, - }; - } - - let result = extract_measures_from_sql(sql); - - match result { - Ok((clean_sql, measures, view_name, base_table)) => { - if !measures.is_empty() { - if let Some(ref vn) = view_name { - let view_query = - extract_view_query(&clean_sql).unwrap_or_else(|| clean_sql.clone()); - let base_relation_sql = extract_base_relation_sql(&view_query); - let dimension_exprs = extract_dimension_exprs_from_query(&view_query); - let mut group_by_cols = extract_view_group_by_cols(&view_query); - let measure_col_names: HashSet = measures - .iter() - .map(|m| normalize_group_by_col(&m.column_name)) - .collect(); - group_by_cols.retain(|col| !measure_col_names.contains(&normalize_group_by_col(col))); - let measure_view = MeasureView { - view_name: vn.clone(), - measures: measures.clone(), - base_query: clean_sql.clone(), - base_table, - base_relation_sql, - dimension_exprs, - group_by_cols, - }; +fn qualify_where_for_inner_with_dimensions( + where_clause: &str, + dimension_exprs: &HashMap, +) -> String { + let mut result = String::new(); + let mut chars = where_clause.chars().peekable(); + let mut previous_was_dot = false; + let mut previous_was_as = false; + let mut interval_state: u8 = 0; - let mut views = MEASURE_VIEWS.lock().unwrap(); - views.insert(vn.clone(), measure_view); + while let Some(c) = chars.next() { + if c.is_alphabetic() || c == '_' { + let mut ident = String::from(c); + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + ident.push(chars.next().unwrap()); + } else { + break; } } - CreateViewResult { - is_measure_view: !measures.is_empty(), - view_name, - clean_sql, - measures, - error: None, - } - } - Err(e) => CreateViewResult { - is_measure_view: false, - view_name: None, - clean_sql: sql.to_string(), - measures: vec![], - error: Some(e.to_string()), - }, - } -} + let already_qualified = previous_was_dot || chars.peek() == Some(&'.'); + let is_function = chars.peek() == Some(&'('); -/// Extract measures from SQL using nom-based parsing -/// Returns (clean_sql, measures, view_name, base_table) -fn extract_measures_from_sql( - sql: &str, -) -> Result<(String, Vec, Option, Option)> { - let sanitized_sql = sanitize_non_ascii_in_sql_comments(sql); - let sql = sanitized_sql.as_str(); - let view_name = extract_view_name(sql); - let base_table = extract_table_name_from_sql(sql); - let sql_upper = sql.to_ascii_uppercase(); + let keywords = [ + "AND", "OR", "NOT", "IN", "IS", "AS", "NULL", "TRUE", "FALSE", "LIKE", "BETWEEN", + "EXISTS", "CASE", "WHEN", "THEN", "ELSE", "END", + ]; + let is_keyword = keywords.iter().any(|kw| kw.eq_ignore_ascii_case(&ident)); + let is_type_context = previous_was_as; + let is_typed_literal = is_typed_literal_keyword(&ident, &chars); + let is_date_part = interval_state == 2 && is_interval_date_part_keyword(&ident); - // First pass: collect all measures with positions - struct MeasureInfo { - name: String, - expression: String, - expr_start: usize, - name_end: usize, + if !already_qualified && !is_keyword && !is_function && !is_typed_literal && !is_type_context && !is_date_part { + let key = normalize_dimension_key(&ident); + if let Some(expr) = dimension_exprs.get(&key) { + let inner_expr = qualify_where_for_inner(expr); + result.push('('); + result.push_str(&inner_expr); + result.push(')'); + interval_state = 0; + continue; + } + result.push_str("_inner."); + } + result.push_str(&ident); + previous_was_dot = false; + previous_was_as = ident.eq_ignore_ascii_case("AS"); + if ident.eq_ignore_ascii_case("INTERVAL") && is_typed_literal { + interval_state = 1; + } else { + interval_state = 0; + } + } else if c == '\'' { + result.push(c); + previous_was_dot = false; + while let Some(next) = chars.next() { + result.push(next); + if next == '\'' { + if chars.peek() == Some(&'\'') { + result.push(chars.next().unwrap()); + } else { + break; + } + } + } + if interval_state == 1 { + interval_state = 2; + } + } else if c == '(' { + if let Some(subquery) = try_consume_subquery_parens(&mut chars) { + result.push_str(&subquery); + } else { + result.push(c); + } + previous_was_dot = false; + } else { + result.push(c); + previous_was_dot = c == '.'; + if interval_state == 2 && !c.is_whitespace() { + interval_state = 0; + } + } } - let mut measure_infos: Vec = Vec::new(); - - let mut search_pos = 0; - while let Some(offset) = sql_upper[search_pos..].find(" AS MEASURE ") { - let pattern_start = search_pos + offset; - let after_measure = pattern_start + " AS MEASURE ".len(); - if let Ok((rest, name)) = identifier(&sql[after_measure..]) { - let name_end = after_measure + (sql[after_measure..].len() - rest.len()); - let expr_start = find_expression_start(sql, pattern_start); - let expression = sql[expr_start..pattern_start].trim().to_string(); + result +} - measure_infos.push(MeasureInfo { - name: name.to_string(), - expression, - expr_start, - name_end, - }); +/// After encountering '(', check if it starts a subquery (SELECT/WITH). +/// If so, consume everything up to and including the matching ')' from the +/// original iterator and return the full subquery text including outer parens. +/// If not a subquery, return None and leave the iterator unchanged. +fn try_consume_subquery_parens( + chars: &mut std::iter::Peekable>, +) -> Option { + // Clone the iterator to peek ahead without consuming + let mut lookahead = chars.clone(); - search_pos = name_end; + // Skip whitespace + while let Some(&c) = lookahead.peek() { + if c.is_whitespace() { + lookahead.next(); } else { - search_pos = pattern_start + 1; + break; } } - // Replace measured expressions: - // - decomposable measures become NULL placeholders (virtual columns) - // - non-decomposable measures keep their aggregate expression for direct querying - // - window measures keep their expression materialized in the view - let mut replacements: Vec<(usize, usize, String)> = Vec::new(); - let mut has_materialized_non_decomposable = false; - - for info in &measure_infos { - let is_non_decomp = is_non_decomposable(&info.expression); - let is_window = is_window_expression(&info.expression); - if is_non_decomp { - has_materialized_non_decomposable = true; + // Collect first token (include digits so e.g. "select1" is not confused with "SELECT") + let mut first_token = String::new(); + while let Some(&c) = lookahead.peek() { + if c.is_alphanumeric() || c == '_' { + first_token.push(c); + lookahead.next(); + } else { + break; } - replacements.push(( - info.expr_start, - info.name_end, - if is_non_decomp || is_window { - format!("{} AS {}", info.expression.trim(), info.name) - } else { - format!("NULL AS {}", info.name) - }, - )); - } - - // Build measures list - let measures: Vec = measure_infos - .into_iter() - .map(|m| ViewMeasure { - column_name: m.name, - is_decomposable: !is_non_decomposable(&m.expression) && !is_window_expression(&m.expression), - expression: m.expression, - }) - .collect(); - - // Apply replacements in reverse order - let mut clean_sql = sql.to_string(); - replacements.sort_by(|a, b| b.0.cmp(&a.0)); - for (start, end, replacement) in replacements { - clean_sql = format!( - "{}{}{}", - &clean_sql[..start], - replacement, - &clean_sql[end..] - ); - } - - clean_sql = rewrite_percentile_within_group(&clean_sql); - - // Non-decomposable measures (e.g., COUNT DISTINCT, MEDIAN) kept as aggregates - // require grouping to form a valid view if dimensions are projected. - if has_materialized_non_decomposable && !has_group_by_anywhere(&clean_sql) { - let upper = clean_sql.to_uppercase(); - let insert_pos = ["ORDER BY", "LIMIT", "HAVING", ";"] - .iter() - .filter_map(|kw| upper.find(kw)) - .min() - .unwrap_or(clean_sql.len()); - clean_sql = format!( - "{} GROUP BY ALL{}", - clean_sql[..insert_pos].trim_end(), - if insert_pos < clean_sql.len() { - format!(" {}", clean_sql[insert_pos..].trim_start()) - } else { - String::new() - } - ); } - Ok((clean_sql, measures, view_name, base_table)) -} - -/// Find the start of an expression before AS MEASURE -fn find_expression_start(sql: &str, end: usize) -> usize { - let chars: Vec = sql[..end].chars().collect(); - let mut pos = chars.len(); - let mut paren_depth = 0; - - // Skip trailing whitespace - while pos > 0 && chars[pos - 1].is_whitespace() { - pos -= 1; + // Only treat as subquery if it starts with SELECT or WITH + if !first_token.eq_ignore_ascii_case("SELECT") && !first_token.eq_ignore_ascii_case("WITH") { + return None; } - // Walk backward through the expression - while pos > 0 { - let c = chars[pos - 1]; + // It's a subquery: consume everything from the original iterator until the + // matching closing paren. We already consumed the opening '(' before this + // function was called, so we start at depth 1. + let mut content = String::from("("); + let mut depth = 1; + while let Some(c) = chars.next() { + content.push(c); match c { + '(' => depth += 1, ')' => { - paren_depth += 1; - pos -= 1; - } - '(' => { - if paren_depth > 0 { - paren_depth -= 1; - pos -= 1; - } else { - break; + depth -= 1; + if depth == 0 { + return Some(content); } } - ',' if paren_depth == 0 => break, - _ if c.is_whitespace() && paren_depth == 0 => { - // Check if previous word is a SQL keyword - let remaining = &sql[..pos - 1]; - let trimmed = remaining.trim_end(); - if trimmed.is_empty() { - break; + '\'' => { + // Handle single-quoted string literals + while let Some(next) = chars.next() { + content.push(next); + if next == '\'' { + if chars.peek() == Some(&'\'') { + content.push(chars.next().unwrap()); + } else { + break; + } + } } - let last_word = trimmed.split_whitespace().last().unwrap_or(""); - let last_upper = last_word.to_uppercase(); - if matches!( - last_upper.as_str(), - "SELECT" | "FROM" | "WHERE" | "GROUP" | "ORDER" | "HAVING" - ) { - break; + } + '"' => { + // Handle double-quoted identifiers (may contain parens) + while let Some(next) = chars.next() { + content.push(next); + if next == '"' { + if chars.peek() == Some(&'"') { + content.push(chars.next().unwrap()); + } else { + break; + } + } } - pos -= 1; } - _ => pos -= 1, + '-' if chars.peek() == Some(&'-') => { + // Line comment: skip until end of line + content.push(chars.next().unwrap()); // consume second '-' + while let Some(next) = chars.next() { + content.push(next); + if next == '\n' { + break; + } + } + } + '/' if chars.peek() == Some(&'*') => { + // Block comment: skip until closing */ + content.push(chars.next().unwrap()); // consume '*' + let mut prev = ' '; + while let Some(next) = chars.next() { + content.push(next); + if prev == '*' && next == '/' { + break; + } + prev = next; + } + } + _ => {} } } - // Find actual start by looking for comma - let byte_pos: usize = chars[..pos].iter().collect::().len(); - sql[..byte_pos].rfind(',').map(|p| p + 1).unwrap_or(0) + Some(content) } -// ============================================================================= -// Core Functions - AGGREGATE Expansion -// ============================================================================= - -/// Expand AGGREGATE() function calls in a SELECT statement -/// Uses position-based replacement with the C++ FFI parser -pub fn expand_aggregate(sql: &str) -> AggregateExpandResult { - let cte_expansion = expand_cte_queries(sql); - let sql = cte_expansion.sql; - let mut had_aggregate = cte_expansion.had_aggregate; - - if !has_aggregate_function(&sql) { - return AggregateExpandResult { - had_aggregate, - expanded_sql: sql, - error: None, - }; +fn is_typed_literal_keyword( + ident: &str, + chars: &std::iter::Peekable>, +) -> bool { + let is_keyword = ident.eq_ignore_ascii_case("DATE") + || ident.eq_ignore_ascii_case("TIME") + || ident.eq_ignore_ascii_case("TIMESTAMP") + || ident.eq_ignore_ascii_case("TIMESTAMPTZ") + || ident.eq_ignore_ascii_case("INTERVAL"); + if !is_keyword { + return false; } - had_aggregate = true; - // Parse the SQL to get table info and GROUP BY columns - let select_info = match parser_ffi::parse_select(&sql) { - Ok(info) => info, - Err(e) => { - return AggregateExpandResult { - had_aggregate, - expanded_sql: sql, - error: Some(format!("SQL parse error: {e}")), - }; + let mut lookahead = chars.clone(); + while let Some(&next) = lookahead.peek() { + if next.is_whitespace() { + lookahead.next(); + continue; } - }; - - // Get the primary table name - let table_name = select_info.primary_table.clone().unwrap_or_default(); - - // Get measure view if this table has one - let views = MEASURE_VIEWS.lock().unwrap(); - let measure_view = views.get(&table_name); - - // Extract all AGGREGATE() calls (without AT modifiers) - let mut aggregate_calls = extract_all_aggregate_calls(&sql); - - if aggregate_calls.is_empty() { - return AggregateExpandResult { - had_aggregate: false, - expanded_sql: sql.to_string(), - error: None, - }; + break; } - // Sort by position descending for safe replacement - aggregate_calls.sort_by(|a, b| b.1.cmp(&a.1)); + matches!(lookahead.peek(), Some(&'\'')) +} - // Non-decomposable measures require the recompute path - if let Some(mv) = measure_view { - let uses_non_decomposable = aggregate_calls.iter().any(|(measure_name, _, _)| { - mv.measures - .iter() - .find(|m| m.column_name.eq_ignore_ascii_case(measure_name)) - .map(|m| is_non_decomposable(&m.expression)) - .unwrap_or(false) - }); - if uses_non_decomposable { - return expand_aggregate_with_at(&sql); +/// Check if an identifier is an interval date-part keyword (e.g. DAYS, MONTHS, YEARS). +/// These follow the SQL-standard INTERVAL '' syntax and should not +/// be qualified as column references when they appear after an interval literal. +fn is_interval_date_part_keyword(ident: &str) -> bool { + let parts = [ + "DAY", "DAYS", "HOUR", "HOURS", "MINUTE", "MINUTES", + "SECOND", "SECONDS", "MONTH", "MONTHS", "YEAR", "YEARS", + "WEEK", "WEEKS", "MICROSECOND", "MICROSECONDS", + "MILLISECOND", "MILLISECONDS", + ]; + parts.iter().any(|p| p.eq_ignore_ascii_case(ident)) +} + +fn push_comment_char_sanitized(out: &mut String, ch: char) { + if ch.is_ascii() { + out.push(ch); + } else { + for _ in 0..ch.len_utf8() { + out.push(' '); } } +} - // Build replacements - let mut result_sql = sql; - - for (measure_name, start, end) in aggregate_calls { - // Look up measure definition - let expanded = if let Some(mv) = measure_view { - if let Some(m) = mv - .measures - .iter() - .find(|m| m.column_name.eq_ignore_ascii_case(&measure_name)) - { - // Get aggregation function from measure expression - if let Some(agg_fn) = extract_aggregation_function(&m.expression) { - format!("{agg_fn}({measure_name})") - } else if let Some(agg_fn) = find_aggregation_in_expression(&m.expression) { - format!("{agg_fn}({measure_name})") - } else { - // Check if derived measure - let expanded_expr = expand_derived_measure_expr(&m.expression, mv); - if expanded_expr != m.expression { - expanded_expr - } else { - format!("SUM({measure_name})") - } - } - } else { - format!("SUM({measure_name})") - } - } else { - format!("SUM({measure_name})") - }; - - result_sql = format!("{}{}{}", &result_sql[..start], expanded, &result_sql[end..]); +fn dollar_quote_delimiter_at(sql: &str, start: usize) -> Option<&str> { + let bytes = sql.as_bytes(); + if start >= bytes.len() || bytes[start] != b'$' { + return None; } - // Check if we need to add GROUP BY - if !has_group_by_anywhere(&result_sql) { - // Extract dimension columns (non-aggregate items) - let dim_cols = extract_dimension_columns_from_select_info(&select_info); - if !dim_cols.is_empty() { - // Find insertion point - let insert_pos = find_group_by_insert_pos(&result_sql); - - result_sql = format!( - "{} GROUP BY {}{}", - result_sql[..insert_pos].trim_end(), - dim_cols.join(", "), - if insert_pos < result_sql.len() { - format!(" {}", result_sql[insert_pos..].trim_start()) - } else { - String::new() - } - ); - } + let mut end = start + 1; + while end < bytes.len() && (bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_') { + end += 1; } - AggregateExpandResult { - had_aggregate, - expanded_sql: result_sql, - error: None, + if end < bytes.len() && bytes[end] == b'$' { + Some(&sql[start..=end]) + } else { + None } } -/// Extract dimension (non-aggregate) columns from SelectInfo -fn extract_dimension_columns_from_select_info(info: &SelectInfo) -> Vec { - info.items - .iter() - .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) - .filter(|item| !is_literal_constant(&item.expression_sql)) - .map(|item| { - // Use alias if present, otherwise expression - item.alias - .clone() - .unwrap_or_else(|| item.expression_sql.clone()) - }) - .collect() +fn is_escape_string_quote_at(sql: &str, quote_pos: usize) -> bool { + let bytes = sql.as_bytes(); + if quote_pos == 0 || quote_pos >= bytes.len() || bytes[quote_pos] != b'\'' { + return false; + } + let prefix_pos = quote_pos - 1; + if !matches!(bytes[prefix_pos], b'e' | b'E') { + return false; + } + prefix_pos == 0 + || !(bytes[prefix_pos - 1].is_ascii_alphanumeric() || bytes[prefix_pos - 1] == b'_') } -fn normalize_dimension_key(ident: &str) -> String { - let trimmed = ident.trim(); - let unquoted = trimmed - .strip_prefix('"') - .and_then(|s| s.strip_suffix('"')) - .unwrap_or(trimmed); - unquoted.to_lowercase() -} +fn sanitize_non_ascii_in_sql_comments(sql: &str) -> String { + let bytes = sql.as_bytes(); + let mut out = String::with_capacity(sql.len()); + let mut i = 0; + let mut in_single = false; + let mut single_allows_backslash = false; + let mut in_double = false; + let mut in_backtick = false; + let mut in_dollar_quote: Option = None; + let mut in_line_comment = false; + let mut block_comment_depth = 0usize; -fn extract_dimension_exprs_from_query(sql: &str) -> HashMap { - let mut exprs = HashMap::new(); - let info = std::panic::catch_unwind(|| parser_ffi::parse_select(sql)) - .ok() - .and_then(|result| result.ok()); - if let Some(info) = info { - for item in info - .items - .iter() - .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) - { - if let Some(alias) = &item.alias { - exprs.insert(normalize_dimension_key(alias), item.expression_sql.clone()); + while i < bytes.len() { + if let Some(ref delimiter) = in_dollar_quote { + if sql[i..].starts_with(delimiter) { + out.push_str(delimiter); + i += delimiter.len(); + in_dollar_quote = None; + continue; } - } - } - exprs -} - -// ============================================================================= -// FROM clause extraction (for JOIN support) -// ============================================================================= -/// Extract all tables from a SELECT's FROM clause using the FFI parser -/// Returns a FromClauseInfo with tables from the parsed SQL -#[allow(dead_code)] // Used in tests (marked as ignore, require C++ library) -fn extract_from_clause_info_ffi(sql: &str) -> FromClauseInfo { - let mut info = FromClauseInfo::default(); + let ch = sql[i..].chars().next().unwrap(); + out.push(ch); + i += ch.len_utf8(); + continue; + } - if let Ok(select_info) = parser_ffi::parse_select(sql) { - for (i, table_ref) in select_info.tables.iter().enumerate() { - let effective_name = table_ref - .alias - .clone() - .unwrap_or_else(|| table_ref.table_name.clone()); - let has_alias = table_ref.alias.is_some(); + if in_line_comment { + let ch = sql[i..].chars().next().unwrap(); + push_comment_char_sanitized(&mut out, ch); + i += ch.len_utf8(); + if ch == '\n' || ch == '\r' { + in_line_comment = false; + } + continue; + } - let table_info = TableInfo { - name: table_ref.table_name.clone(), - effective_name: effective_name.clone(), - has_alias, - }; + if block_comment_depth > 0 { + if i + 1 < bytes.len() && bytes[i] == b'/' && bytes[i + 1] == b'*' { + out.push_str("/*"); + i += 2; + block_comment_depth += 1; + continue; + } - if i == 0 && info.primary_table.is_none() { - info.primary_table = Some(table_info.clone()); + if i + 1 < bytes.len() && bytes[i] == b'*' && bytes[i + 1] == b'/' { + out.push_str("*/"); + i += 2; + block_comment_depth -= 1; + continue; } - info.tables.insert(effective_name, table_info); + + let ch = sql[i..].chars().next().unwrap(); + push_comment_char_sanitized(&mut out, ch); + i += ch.len_utf8(); + continue; } - } - info -} + let ch = sql[i..].chars().next().unwrap(); -/// Information about a resolved measure -#[derive(Debug, Clone)] -pub struct ResolvedMeasure { - /// The aggregation function (SUM, COUNT, etc.) - pub agg_fn: String, - /// The source view name - pub source_view: String, - /// For derived measures: the expanded expression - pub derived_expr: Option, - /// Whether this measure can be re-aggregated - pub is_decomposable: bool, - /// Base table for non-decomposable measures (for correlated subquery) - pub base_table: Option, - /// Base relation SQL for non-decomposable recomputation - pub base_relation_sql: Option, - /// Dimension alias -> expression mapping from the source view - pub dimension_exprs: HashMap, - /// GROUP BY columns from the source view definition - pub view_group_by_cols: Vec, - /// The original measure expression (e.g., "COUNT(DISTINCT user_id)") - pub expression: String, -} + if in_single { + out.push(ch); + i += ch.len_utf8(); + if single_allows_backslash && ch == '\\' { + if let Some(next) = sql[i..].chars().next() { + out.push(next); + i += next.len_utf8(); + } + continue; + } + if ch == '\'' { + if i < bytes.len() && bytes[i] == b'\'' { + out.push('\''); + i += 1; + } else { + in_single = false; + single_allows_backslash = false; + } + } + continue; + } -/// Look up which view contains a measure and return resolved measure info -/// Prioritizes default_table (the query's FROM table), then searches other views for JOINs -fn resolve_measure_source(measure_name: &str, default_table: &str) -> ResolvedMeasure { - let views = MEASURE_VIEWS.lock().unwrap(); + if in_double { + out.push(ch); + i += ch.len_utf8(); + if ch == '"' { + if i < bytes.len() && bytes[i] == b'"' { + out.push('"'); + i += 1; + } else { + in_double = false; + } + } + continue; + } - // Helper to build ResolvedMeasure from a found measure - let build_resolved = |m: &ViewMeasure, v: &MeasureView, source_view: &str| -> ResolvedMeasure { - let agg_fn = extract_agg_function(&m.expression); - let derived_expr = if extract_aggregation_function(&m.expression).is_none() { - let expanded = expand_derived_measure_expr(&m.expression, v); - if expanded != m.expression { - Some(expanded) - } else { - None + if in_backtick { + out.push(ch); + i += ch.len_utf8(); + if ch == '`' { + in_backtick = false; } - } else { - None - }; + continue; + } - ResolvedMeasure { - agg_fn, - source_view: source_view.to_string(), - derived_expr, - is_decomposable: m.is_decomposable, - base_table: v.base_table.clone(), - base_relation_sql: v.base_relation_sql.clone(), - dimension_exprs: v.dimension_exprs.clone(), - view_group_by_cols: v.group_by_cols.clone(), - expression: m.expression.clone(), + if let Some(delimiter) = dollar_quote_delimiter_at(sql, i) { + if sql[i + delimiter.len()..].contains(delimiter) { + out.push_str(delimiter); + i += delimiter.len(); + in_dollar_quote = Some(delimiter.to_string()); + continue; + } } - }; - // First, check if the measure exists in the default table (query's primary table) - if let Some(v) = views.get(default_table) { - if let Some(m) = v - .measures - .iter() - .find(|m| m.column_name.eq_ignore_ascii_case(measure_name)) - { - return build_resolved(m, v, default_table); + if i + 1 < bytes.len() && bytes[i] == b'-' && bytes[i + 1] == b'-' { + out.push_str("--"); + i += 2; + in_line_comment = true; + continue; } - } - // If not found in default table, search other views (for JOIN support) - for (view_name, v) in views.iter() { - if let Some(m) = v - .measures - .iter() - .find(|m| m.column_name.eq_ignore_ascii_case(measure_name)) - { - return build_resolved(m, v, view_name); + if i + 1 < bytes.len() && bytes[i] == b'/' && bytes[i + 1] == b'*' { + out.push_str("/*"); + i += 2; + block_comment_depth = 1; + continue; } - } - // Fallback: measure not found, return defaults - ResolvedMeasure { - agg_fn: "SUM".to_string(), - source_view: default_table.to_string(), - derived_expr: None, - is_decomposable: true, - base_table: None, - base_relation_sql: None, - dimension_exprs: HashMap::new(), - view_group_by_cols: Vec::new(), - expression: String::new(), + out.push(ch); + i += ch.len_utf8(); + + match ch { + '\'' => { + in_single = true; + single_allows_backslash = is_escape_string_quote_at(sql, i - ch.len_utf8()); + } + '"' => in_double = true, + '`' => in_backtick = true, + _ => {} + } } -} -/// Find the alias used for a view in the FROM clause -/// Returns the effective_name (alias) if the view is in the FROM clause -fn find_alias_for_view<'a>(from_info: &'a FromClauseInfo, view_name: &str) -> Option<&'a str> { - from_info - .tables - .values() - .find(|t| t.name.eq_ignore_ascii_case(view_name)) - .map(|t| t.effective_name.as_str()) + out } // ============================================================================= -// Core Functions - Non-Decomposable Measure Expansion +// Core Functions - Process CREATE VIEW // ============================================================================= -fn base_relation_for_subquery(base_relation_sql: &str) -> String { - let trimmed = base_relation_sql.trim().trim_end_matches(';').trim(); - format!("({trimmed})") -} +/// Process CREATE VIEW statement, extracting AS MEASURE definitions +pub fn process_create_view(sql: &str) -> CreateViewResult { + if !has_as_measure(sql) { + return CreateViewResult { + is_measure_view: false, + view_name: None, + clean_sql: sql.to_string(), + measures: vec![], + error: None, + }; + } -/// Check if a dimension string is an expression (not a simple column reference). -/// Expressions contain function calls, operators, or CASE expressions. -fn is_expression_dim(dim: &str) -> bool { - let trimmed = dim.trim(); - let upper = trimmed.to_uppercase(); - trimmed.contains('(') - || trimmed.contains("||") - || trimmed.contains(" + ") - || trimmed.contains(" - ") - || trimmed.contains(" * ") - || trimmed.contains(" / ") - || upper.starts_with("CASE ") - || upper.contains(" CASE ") -} + let result = extract_measures_from_sql(sql); -/// Strip a specific table qualifier from column references in an expression so -/// it can be re-qualified with `_inner` or an outer alias. -/// Only removes `table_name.column` prefixes where the target is a plain column; -/// schema-qualified function calls (`s.bucket(ts)`) and struct field -/// dereferences are preserved even when the qualifier matches `table_name`. -fn strip_table_qualifier(expr: &str, table_name: &str) -> String { - let mut result = String::new(); - let mut chars = expr.chars().peekable(); + match result { + Ok((clean_sql, measures, view_name, base_table)) => { + if !measures.is_empty() { + if let Some(ref vn) = view_name { + let view_query = + extract_view_query(&clean_sql).unwrap_or_else(|| clean_sql.clone()); + let base_relation_sql = extract_base_relation_sql(&view_query); + let dimension_exprs = extract_dimension_exprs_from_query(&view_query); + let mut group_by_cols = extract_view_group_by_cols(&view_query); + let measure_col_names: HashSet = measures + .iter() + .map(|m| normalize_group_by_col(&m.column_name)) + .collect(); + group_by_cols.retain(|col| !measure_col_names.contains(&normalize_group_by_col(col))); + let measure_view = MeasureView { + view_name: vn.clone(), + measures: measures.clone(), + base_query: clean_sql.clone(), + base_table, + base_relation_sql, + dimension_exprs, + group_by_cols, + }; - while let Some(c) = chars.next() { - if c == '\'' { - // Single-quoted string literal: copy verbatim - result.push(c); - while let Some(next) = chars.next() { - result.push(next); - if next == '\'' { - if chars.peek() == Some(&'\'') { - result.push(chars.next().unwrap()); - } else { - break; - } - } - } - } else if c == '"' || c.is_alphabetic() || c == '_' { - // Collect identifier (possibly double-quoted) - let is_quoted = c == '"'; - let mut ident_raw = String::from(c); // full text including quotes - let ident_name; // unquoted name for comparison - if is_quoted { - while let Some(next) = chars.next() { - ident_raw.push(next); - if next == '"' { - break; - } - } - ident_name = ident_raw[1..ident_raw.len().saturating_sub(1).max(1)].to_string(); - } else { - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - ident_raw.push(chars.next().unwrap()); - } else { - break; - } + let mut views = MEASURE_VIEWS.lock().unwrap(); + views.insert(vn.clone(), measure_view); } - ident_name = ident_raw.clone(); } - if chars.peek() == Some(&'.') && ident_name.eq_ignore_ascii_case(table_name) { - // Peek past the dot to see what follows. If the next token - // is a function call (identifier immediately followed by '('), - // this is a schema-qualified function, not a table.column - // reference, so preserve the qualifier. - chars.next(); // consume the dot - // Collect the next identifier (if any) to check for '(' - let mut next_ident = String::new(); - if chars.peek() == Some(&'"') { - next_ident.push(chars.next().unwrap()); - while let Some(next) = chars.next() { - next_ident.push(next); - if next == '"' { break; } - } - } else { - while let Some(&next) = chars.peek() { - if next.is_alphanumeric() || next == '_' { - next_ident.push(chars.next().unwrap()); - } else { - break; - } - } - } - // Skip whitespace before checking for '(' so that - // `s.bucket (ts)` is recognized as a function call. - let mut ws_after = String::new(); - while let Some(&next) = chars.peek() { - if next == ' ' || next == '\t' || next == '\n' || next == '\r' { - ws_after.push(chars.next().unwrap()); - } else { - break; - } - } - let is_function_call = chars.peek() == Some(&'('); - if is_function_call { - // Schema-qualified function: restore qualifier.dot.name(ws) - result.push_str(&ident_raw); - result.push('.'); - result.push_str(&next_ident); - result.push_str(&ws_after); - } else { - // Table-qualified column: drop qualifier, keep column - result.push_str(&next_ident); - result.push_str(&ws_after); - } - } else { - result.push_str(&ident_raw); + + CreateViewResult { + is_measure_view: !measures.is_empty(), + view_name, + clean_sql, + measures, + error: None, } - } else { - result.push(c); } + Err(e) => CreateViewResult { + is_measure_view: false, + view_name: None, + clean_sql: sql.to_string(), + measures: vec![], + error: Some(e.to_string()), + }, } - - result } -fn correlation_exprs_for_dim( - dim: &str, - dimension_exprs: &HashMap, - outer_alias: Option<&str>, -) -> (String, String) { - let dim_trim = dim.trim(); - let dim_name = dim_trim.split('.').next_back().unwrap_or(dim_trim).trim(); - let dim_is_qualified = dim_trim.contains('.'); - let dim_key = normalize_dimension_key(dim_name); - if let Some(expr) = dimension_exprs.get(&dim_key) { - let inner_expr = qualify_where_for_inner(expr); - let outer_expr = if dim_is_qualified { - dim_trim.to_string() - } else { - outer_alias - .map(|alias| format!("{alias}.{dim_name}")) - .unwrap_or_else(|| dim_name.to_string()) - }; - return (inner_expr, outer_expr); - } +/// Extract measures from SQL using nom-based parsing +/// Returns (clean_sql, measures, view_name, base_table) +fn extract_measures_from_sql( + sql: &str, +) -> Result<(String, Vec, Option, Option)> { + let sanitized_sql = sanitize_non_ascii_in_sql_comments(sql); + let sql = sanitized_sql.as_str(); + let view_name = extract_view_name(sql); + let base_table = extract_table_name_from_sql(sql); + let sql_upper = sql.to_ascii_uppercase(); - // Expression dimensions (function calls, operators, CASE): strip the source - // table qualifier so inner/outer qualification works, then wrap the outer side - // in ANY_VALUE so DuckDB accepts it in grouped context. Only the known table - // name is stripped; schema qualifiers and struct field access are preserved. - if is_expression_dim(dim_trim) { - let table_to_strip = outer_alias.unwrap_or(""); - let unqualified = strip_table_qualifier(dim_trim, table_to_strip); - let inner_expr = qualify_where_for_inner(&unqualified); - let outer_expr = outer_alias - .map(|alias| format!("ANY_VALUE({})", qualify_where_for_outer(&unqualified, alias))) - .unwrap_or_else(|| dim_trim.to_string()); - return (inner_expr, outer_expr); + // First pass: collect all measures with positions + struct MeasureInfo { + name: String, + expression: String, + expr_start: usize, + name_end: usize, } + let mut measure_infos: Vec = Vec::new(); - let inner_expr = format!("_inner.{dim_name}"); - let outer_expr = if dim_is_qualified { - dim_trim.to_string() - } else { - outer_alias - .map(|alias| format!("{alias}.{dim_name}")) - .unwrap_or_else(|| dim_name.to_string()) - }; - (inner_expr, outer_expr) -} - -fn correlation_condition_for_dim( - dim: &str, - dimension_exprs: &HashMap, - outer_alias: Option<&str>, -) -> String { - let (inner_expr, outer_expr) = correlation_exprs_for_dim(dim, dimension_exprs, outer_alias); - format!("{inner_expr} IS NOT DISTINCT FROM {outer_expr}") -} + let mut search_pos = 0; + while let Some(offset) = sql_upper[search_pos..].find(" AS MEASURE ") { + let pattern_start = search_pos + offset; + let after_measure = pattern_start + " AS MEASURE ".len(); -struct NonDecompJoinPlan { - join_sql: String, - replacement: String, -} + if let Ok((rest, name)) = identifier(&sql[after_measure..]) { + let name_end = after_measure + (sql[after_measure..].len() - rest.len()); + let expr_start = find_expression_start(sql, pattern_start); + let expression = sql[expr_start..pattern_start].trim().to_string(); -fn expand_non_decomposable_default_context( - expression: &str, - base_relation_sql: &str, - outer_alias: Option<&str>, - group_by_cols: &[String], - dimension_exprs: &HashMap, -) -> String { - let base_relation = base_relation_for_subquery(base_relation_sql); + measure_infos.push(MeasureInfo { + name: name.to_string(), + expression, + expr_start, + name_end, + }); - if group_by_cols.is_empty() { - return format!("(SELECT {expression} FROM {base_relation})"); + search_pos = name_end; + } else { + search_pos = pattern_start + 1; + } } - let where_clauses: Vec<_> = group_by_cols - .iter() - .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) - .collect(); + // Replace measured expressions: + // - decomposable measures become NULL placeholders (virtual columns) + // - non-decomposable measures keep their aggregate expression for direct querying + // - window measures keep their expression materialized in the view + let mut replacements: Vec<(usize, usize, String)> = Vec::new(); + let mut has_materialized_non_decomposable = false; - format!( - "(SELECT {} FROM {} _inner WHERE {})", - expression, - base_relation, - where_clauses.join(" AND ") - ) -} + for info in &measure_infos { + let is_non_decomp = is_non_decomposable(&info.expression); + let is_window = is_window_expression(&info.expression); + if is_non_decomp { + has_materialized_non_decomposable = true; + } + replacements.push(( + info.expr_start, + info.name_end, + if is_non_decomp || is_window { + format!("{} AS {}", info.expression.trim(), info.name) + } else { + format!("NULL AS {}", info.name) + }, + )); + } -fn scalar_subquery_to_rows_sql(scalar_sql: &str, value_alias: &str) -> Option { - let trimmed = scalar_sql.trim(); - let inner = trimmed.strip_prefix('(')?.strip_suffix(')')?.trim(); - if !inner.to_uppercase().starts_with("SELECT") { - return None; + // Build measures list + let measures: Vec = measure_infos + .into_iter() + .map(|m| ViewMeasure { + column_name: m.name, + is_decomposable: !is_non_decomposable(&m.expression) && !is_window_expression(&m.expression), + expression: m.expression, + }) + .collect(); + + // Apply replacements in reverse order + let mut clean_sql = sql.to_string(); + replacements.sort_by(|a, b| b.0.cmp(&a.0)); + for (start, end, replacement) in replacements { + clean_sql = format!( + "{}{}{}", + &clean_sql[..start], + replacement, + &clean_sql[end..] + ); } - let select_start = "SELECT".len(); - let from_pos = find_top_level_keyword(inner, "FROM", select_start)?; - let select_expr = inner[select_start..from_pos].trim(); - let from_clause = inner[from_pos..].trim(); - Some(format!("SELECT {select_expr} AS {value_alias} {from_clause}")) -} + clean_sql = rewrite_percentile_within_group(&clean_sql); -fn wrap_window_rows_as_single_value(row_sql: &str, measure_name: &str) -> String { - let escaped_measure = measure_name.replace('\'', "''"); - format!( - "(SELECT CASE \ - WHEN EXISTS (\ - SELECT 1 \ - FROM ({row_sql}) __window_vals \ - CROSS JOIN (SELECT __window_value AS __first FROM ({row_sql}) LIMIT 1) __window_first \ - WHERE __window_vals.__window_value IS DISTINCT FROM __window_first.__first\ - ) \ - THEN error('Window measure {escaped_measure} returned multiple values for the evaluation context') \ - ELSE (SELECT __window_value FROM ({row_sql}) LIMIT 1) \ - END)" - ) + // Non-decomposable measures (e.g., COUNT DISTINCT, MEDIAN) kept as aggregates + // require grouping to form a valid view if dimensions are projected. + if has_materialized_non_decomposable && !has_group_by_anywhere(&clean_sql) { + let upper = clean_sql.to_uppercase(); + let insert_pos = ["ORDER BY", "LIMIT", "HAVING", ";"] + .iter() + .filter_map(|kw| upper.find(kw)) + .min() + .unwrap_or(clean_sql.len()); + clean_sql = format!( + "{} GROUP BY ALL{}", + clean_sql[..insert_pos].trim_end(), + if insert_pos < clean_sql.len() { + format!(" {}", clean_sql[insert_pos..].trim_start()) + } else { + String::new() + } + ); + } + + Ok((clean_sql, measures, view_name, base_table)) } -fn build_non_decomposable_join_plan( - expression: &str, - base_relation_sql: &str, - outer_alias: Option<&str>, - outer_where: Option<&str>, - group_by_cols: &[String], - modifiers: &[ContextModifier], - dimension_exprs: &HashMap, - join_alias: &str, -) -> Option { - let base_relation = base_relation_for_subquery(base_relation_sql); - let mut removed_dims: Vec = Vec::new(); - let mut effective_where: Option = None; - let mut set_overrides: HashMap = HashMap::new(); - let mut has_all_global = false; +/// Find the start of an expression before AS MEASURE +fn find_expression_start(sql: &str, end: usize) -> usize { + let chars: Vec = sql[..end].chars().collect(); + let mut pos = chars.len(); + let mut paren_depth = 0; - let has_set = modifiers - .iter() - .any(|m| matches!(m, ContextModifier::Set(_, _))); + // Skip trailing whitespace + while pos > 0 && chars[pos - 1].is_whitespace() { + pos -= 1; + } - if modifiers.is_empty() { - if let Some(w) = outer_where { - effective_where = Some(qualify_where_for_inner_with_dimensions(w, dimension_exprs)); - } - } else { - let all_are_all = modifiers - .iter() - .all(|m| matches!(m, ContextModifier::All(_) | ContextModifier::AllGlobal)); - if all_are_all { - if modifiers - .iter() - .any(|m| matches!(m, ContextModifier::AllGlobal)) - { - return None; + // Walk backward through the expression + while pos > 0 { + let c = chars[pos - 1]; + + match c { + ')' => { + paren_depth += 1; + pos -= 1; } - for modifier in modifiers { - if let ContextModifier::All(dim) = modifier { - removed_dims.push(dim.to_lowercase()); + '(' => { + if paren_depth > 0 { + paren_depth -= 1; + pos -= 1; + } else { + break; } } - } else { - for modifier in modifiers.iter().rev() { - match modifier { - ContextModifier::AllGlobal => { - has_all_global = true; - effective_where = None; - set_overrides.clear(); - removed_dims.clear(); - } - ContextModifier::All(dim) => { - removed_dims.push(dim.to_lowercase()); - } - ContextModifier::Visible => { - if !has_set && !has_all_global { - if let Some(w) = outer_where { - let stripped = strip_at_where_qualifiers(w); - effective_where = Some(qualify_where_for_inner_with_dimensions( - &stripped, - dimension_exprs, - )); - } - } - } - ContextModifier::Where(cond) => { - if !has_all_global { - let stripped = strip_at_where_qualifiers(cond); - effective_where = - Some(qualify_where_for_inner_with_dimensions( - &stripped, - dimension_exprs, - )); - } - } - ContextModifier::Set(dim, expr) => { - let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - let dim_key = normalize_dimension_key(dim_name); - if !has_all_global && !removed_dims.contains(&dim_key) { - let resolved_expr = - resolve_current_in_expr(expr, group_by_cols, outer_where, outer_alias); - let outer_expr = if let Some(alias) = outer_alias { - if dim.contains('(') { - qualify_where_for_outer(&resolved_expr, alias) - } else { - qualify_outer_reference(&resolved_expr, alias, dim_name) - } - } else { - resolved_expr - }; - set_overrides.insert(dim_key, outer_expr); - } - } + ',' if paren_depth == 0 => break, + _ if c.is_whitespace() && paren_depth == 0 => { + // Check if previous word is a SQL keyword + let remaining = &sql[..pos - 1]; + let trimmed = remaining.trim_end(); + if trimmed.is_empty() { + break; + } + let last_word = trimmed.split_whitespace().last().unwrap_or(""); + let last_upper = last_word.to_uppercase(); + if matches!( + last_upper.as_str(), + "SELECT" | "FROM" | "WHERE" | "GROUP" | "ORDER" | "HAVING" + ) { + break; } + pos -= 1; } + _ => pos -= 1, } } - if has_all_global && set_overrides.is_empty() { - return None; - } + // Find actual start by looking for comma + let byte_pos: usize = chars[..pos].iter().collect::().len(); + sql[..byte_pos].rfind(',').map(|p| p + 1).unwrap_or(0) +} - let remaining_cols: Vec<&String> = group_by_cols - .iter() - .filter(|col| { - let col_lower = col.to_lowercase(); - let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); - !removed_dims - .iter() - .any(|d| *d == col_lower || *d == col_name) - }) - .collect(); +// ============================================================================= +// Core Functions - AGGREGATE Expansion +// ============================================================================= - if remaining_cols.is_empty() { - return None; - } +/// Expand AGGREGATE() function calls in a SELECT statement +/// Uses position-based replacement with the C++ FFI parser +pub fn expand_aggregate(sql: &str) -> AggregateExpandResult { + let cte_expansion = expand_cte_queries(sql); + let sql = cte_expansion.sql; + let mut had_aggregate = cte_expansion.had_aggregate; - let mut select_parts = Vec::new(); - let mut group_parts = Vec::new(); - let mut join_conditions = Vec::new(); + if !has_aggregate_function(&sql) { + return AggregateExpandResult { + had_aggregate, + expanded_sql: sql, + error: None, + warnings: Vec::new(), + }; + } + had_aggregate = true; - for (idx, col) in remaining_cols.iter().enumerate() { - let alias = format!("dim_{idx}"); - let (inner_expr, default_outer_expr) = - correlation_exprs_for_dim(col, dimension_exprs, outer_alias); - let dim_name = col.split('.').next_back().unwrap_or(col).trim(); - let dim_key = normalize_dimension_key(dim_name); - let outer_expr = set_overrides - .get(&dim_key) - .cloned() - .unwrap_or(default_outer_expr); - - select_parts.push(format!("{inner_expr} AS {alias}")); - group_parts.push(alias.clone()); - join_conditions.push(format!("{join_alias}.{alias} IS NOT DISTINCT FROM {outer_expr}")); - } + // Parse the SQL to get table info and GROUP BY columns + let select_info = match parser_ffi::parse_select(&sql) { + Ok(info) => info, + Err(e) => { + return AggregateExpandResult { + had_aggregate, + expanded_sql: sql, + error: Some(format!("SQL parse error: {e}")), + warnings: Vec::new(), + }; + } + }; - let mut agg_query = format!( - "SELECT {}, {} AS value FROM {} _inner", - select_parts.join(", "), - expression, - base_relation - ); - if let Some(w) = effective_where { - agg_query.push_str(" WHERE "); - agg_query.push_str(&w); - } - if !group_parts.is_empty() { - agg_query.push_str(" GROUP BY "); - agg_query.push_str(&group_parts.join(", ")); - } + // Get the primary table name + let table_name = select_info.primary_table.clone().unwrap_or_default(); - Some(NonDecompJoinPlan { - join_sql: format!(" LEFT JOIN ({agg_query}) {join_alias} ON {}", join_conditions.join(" AND ")), - replacement: format!("{join_alias}.value"), - }) -} + // Get measure view if this table has one + let views = MEASURE_VIEWS.lock().unwrap(); + let measure_view = views.get(&table_name); -/// Expand a non-decomposable measure (like COUNT DISTINCT) to a correlated subquery -/// against the base table. This is used when the measure cannot be re-aggregated. -/// -/// Example: -/// - expression: "COUNT(DISTINCT user_id)" -/// - base_relation_sql: "SELECT * FROM orders" -/// - group_by_cols: ["year", "region"] -/// - Result: (SELECT COUNT(DISTINCT user_id) FROM (SELECT * FROM orders) _inner WHERE _inner.year IS NOT DISTINCT FROM _outer.year AND _inner.region IS NOT DISTINCT FROM _outer.region) -fn expand_non_decomposable_to_sql( - expression: &str, - base_relation_sql: &str, - outer_alias: Option<&str>, - outer_where: Option<&str>, - group_by_cols: &[String], - modifiers: &[ContextModifier], - dimension_exprs: &HashMap, -) -> String { - let base_relation = base_relation_for_subquery(base_relation_sql); + // Extract all AGGREGATE() calls (without AT modifiers) + let mut aggregate_calls = extract_all_aggregate_calls(&sql); - if modifiers.is_empty() { - // No modifiers = default VISIBLE behavior (respect outer WHERE) - return expand_non_decomposable_at_to_sql( - expression, - &ContextModifier::Visible, - base_relation_sql, - outer_alias, - outer_where, - group_by_cols, - dimension_exprs, - ); + if aggregate_calls.is_empty() { + return AggregateExpandResult { + had_aggregate: false, + expanded_sql: sql.to_string(), + error: None, + warnings: Vec::new(), + }; } - if modifiers.len() == 1 { - return expand_non_decomposable_at_to_sql( - expression, - &modifiers[0], - base_relation_sql, - outer_alias, - outer_where, - group_by_cols, - dimension_exprs, - ); - } + // Sort by position descending for safe replacement + aggregate_calls.sort_by(|a, b| b.1.cmp(&a.1)); - // Check if all modifiers are ALL (dimension or global) - let all_are_all = modifiers - .iter() - .all(|m| matches!(m, ContextModifier::All(_) | ContextModifier::AllGlobal)); - if all_are_all { - // Check for explicit AllGlobal - that means grand total - if modifiers - .iter() - .any(|m| matches!(m, ContextModifier::AllGlobal)) - { - return expand_non_decomposable_at_to_sql( - expression, - &ContextModifier::AllGlobal, - base_relation_sql, - outer_alias, - outer_where, - group_by_cols, - dimension_exprs, - ); + // Non-decomposable measures require the recompute path + if let Some(mv) = measure_view { + let uses_non_decomposable = aggregate_calls.iter().any(|(measure_name, _, _)| { + mv.measures + .iter() + .find(|m| m.column_name.eq_ignore_ascii_case(measure_name)) + .map(|m| is_non_decomposable(&m.expression)) + .unwrap_or(false) + }); + if uses_non_decomposable { + return expand_aggregate_with_at(&sql); } + } - // Accumulate all dimensions to remove - let removed_dims: Vec<&str> = modifiers - .iter() - .filter_map(|m| match m { - ContextModifier::All(dim) => Some(dim.as_str()), - _ => None, - }) - .collect(); + // Build replacements + let mut result_sql = sql; - // Filter group_by_cols to get remaining dimensions - let remaining_cols: Vec<&String> = group_by_cols - .iter() - .filter(|col| { - let col_lower = col.to_lowercase(); - let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); - !removed_dims - .iter() - .any(|d| d.to_lowercase() == col_lower || d.to_lowercase() == col_name) - }) - .collect(); + for (measure_name, start, end) in aggregate_calls { + // Look up measure definition + let expanded = if let Some(mv) = measure_view { + if let Some(m) = mv + .measures + .iter() + .find(|m| m.column_name.eq_ignore_ascii_case(&measure_name)) + { + // Get aggregation function from measure expression + if let Some(agg_fn) = extract_aggregation_function(&m.expression) { + format!("{agg_fn}({measure_name})") + } else if let Some(agg_fn) = find_aggregation_in_expression(&m.expression) { + format!("{agg_fn}({measure_name})") + } else { + // Check if derived measure + let expanded_expr = expand_derived_measure_expr(&m.expression, mv); + if expanded_expr != m.expression { + expanded_expr + } else { + format!("SUM({measure_name})") + } + } + } else { + format!("SUM({measure_name})") + } + } else { + format!("SUM({measure_name})") + }; - if remaining_cols.is_empty() { - // All dimensions removed = grand total - return expand_non_decomposable_at_to_sql( - expression, - &ContextModifier::AllGlobal, - base_relation_sql, - outer_alias, - outer_where, - group_by_cols, - dimension_exprs, + result_sql = format!("{}{}{}", &result_sql[..start], expanded, &result_sql[end..]); + } + + // Check if we need to add GROUP BY + if !has_group_by_anywhere(&result_sql) { + // Extract dimension columns (non-aggregate items) + let dim_cols = extract_dimension_columns_from_select_info(&select_info); + if !dim_cols.is_empty() { + // Find insertion point + let insert_pos = find_group_by_insert_pos(&result_sql); + + result_sql = format!( + "{} GROUP BY {}{}", + result_sql[..insert_pos].trim_end(), + dim_cols.join(", "), + if insert_pos < result_sql.len() { + format!(" {}", result_sql[insert_pos..].trim_start()) + } else { + String::new() + } ); } - - // Generate correlation on remaining dimensions only - let where_clauses: Vec<_> = remaining_cols - .iter() - .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) - .collect(); - return format!( - "(SELECT {} FROM {} _inner WHERE {})", - expression, - base_relation, - where_clauses.join(" AND ") - ); } - // Apply modifiers right-to-left - // - VISIBLE adds outer WHERE (but SET bypasses it per paper) - // - ALL removes dimensions from correlation - // - SET changes a dimension and bypasses outer WHERE - // - WHERE adds a filter + AggregateExpandResult { + had_aggregate, + expanded_sql: result_sql, + error: None, + warnings: Vec::new(), + } +} - let has_set = modifiers +/// Extract dimension (non-aggregate) columns from SelectInfo +fn extract_dimension_columns_from_select_info(info: &SelectInfo) -> Vec { + info.items .iter() - .any(|m| matches!(m, ContextModifier::Set(_, _))); + .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) + .filter(|item| !is_literal_constant(&item.expression_sql)) + .map(|item| { + // Use alias if present, otherwise expression + item.alias + .clone() + .unwrap_or_else(|| item.expression_sql.clone()) + }) + .collect() +} - let mut effective_where: Option = None; - let mut has_all_global = false; - let mut set_conditions: Vec = Vec::new(); - let mut removed_dims: Vec = Vec::new(); +fn is_star_select_expression(expr: &str) -> bool { + let trimmed = expr.trim(); + trimmed == "*" || trimmed.ends_with(".*") || trimmed.starts_with("* ") +} - for modifier in modifiers.iter().rev() { - match modifier { - ContextModifier::AllGlobal => { - has_all_global = true; - effective_where = None; - set_conditions.clear(); - } - ContextModifier::All(dim) => { - removed_dims.push(dim.to_lowercase()); - } - ContextModifier::Visible => { - if !has_set && !has_all_global { - if let Some(w) = outer_where { - let stripped = strip_at_where_qualifiers(w); - effective_where = Some(qualify_where_for_inner_with_dimensions( - &stripped, - dimension_exprs, - )); - } - } - } - ContextModifier::Where(cond) => { - if !has_all_global { - let stripped = strip_at_where_qualifiers(cond); - effective_where = - Some(qualify_where_for_inner_with_dimensions( - &stripped, - dimension_exprs, - )); - } - } - ContextModifier::Set(dim, expr) => { - let dim_lower = dim.to_lowercase(); - if !has_all_global && !removed_dims.contains(&dim_lower) { - let outer_ref = outer_alias.unwrap_or("_outer"); - let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - let dim_key = normalize_dimension_key(dim_name); - let resolved_expr = resolve_current_in_expr( - expr, - group_by_cols, - outer_where, - Some(outer_ref), - ); - let qualified_expr = if dim.contains('(') { - qualify_where_for_outer(&resolved_expr, outer_ref) - } else { - qualify_outer_reference(&resolved_expr, outer_ref, dim_name) - }; - let inner_dim = if let Some(expr) = dimension_exprs.get(&dim_key) { - qualify_where_for_inner(expr) - } else if dim.contains('(') { - qualify_where_for_inner(dim) - } else { - format!("_inner.{dim_name}") - }; - set_conditions.push(format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}")); - } +fn normalize_dimension_key(ident: &str) -> String { + let trimmed = ident.trim(); + let unquoted = trimmed + .strip_prefix('"') + .and_then(|s| s.strip_suffix('"')) + .unwrap_or(trimmed); + unquoted.to_lowercase() +} + +fn extract_dimension_exprs_from_query(sql: &str) -> HashMap { + let mut exprs = HashMap::new(); + let info = std::panic::catch_unwind(|| parser_ffi::parse_select(sql)) + .ok() + .and_then(|result| result.ok()); + if let Some(info) = info { + for item in info + .items + .iter() + .filter(|item| !item.is_aggregate && !item.is_star && !item.is_measure_ref) + { + if let Some(alias) = &item.alias { + exprs.insert(normalize_dimension_key(alias), item.expression_sql.clone()); } } } + exprs +} - if has_all_global && set_conditions.is_empty() { - return format!("(SELECT {expression} FROM {base_relation})"); - } +// ============================================================================= +// FROM clause extraction (for JOIN support) +// ============================================================================= - // Filter group_by_cols to exclude removed dimensions - let remaining_cols: Vec<&String> = group_by_cols - .iter() - .filter(|col| { - let col_lower = col.to_lowercase(); - let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); - !removed_dims - .iter() - .any(|d| *d == col_lower || *d == col_name) - }) - .collect(); +/// Extract all tables from a SELECT's FROM clause using the FFI parser +/// Returns a FromClauseInfo with tables from the parsed SQL +#[allow(dead_code)] // Used in tests (marked as ignore, require C++ library) +fn extract_from_clause_info_ffi(sql: &str) -> FromClauseInfo { + let mut info = FromClauseInfo::default(); - // Build correlation conditions for remaining dimensions - let correlation_conditions: Vec = remaining_cols - .iter() - .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) - .collect(); + if let Ok(Ok(select_info)) = std::panic::catch_unwind(|| parser_ffi::parse_select(sql)) { + for (i, table_ref) in select_info.tables.iter().enumerate() { + let effective_name = table_ref + .alias + .clone() + .unwrap_or_else(|| table_ref.table_name.clone()); + let has_alias = table_ref.alias.is_some(); - // Combine all conditions: correlation + SET conditions + WHERE - let mut all_conditions: Vec = correlation_conditions; - all_conditions.extend(set_conditions); - if let Some(w) = effective_where { - all_conditions.push(w); - } + let table_info = TableInfo { + name: table_ref.table_name.clone(), + effective_name: effective_name.clone(), + has_alias, + }; - if all_conditions.is_empty() { - format!("(SELECT {expression} FROM {base_relation})") - } else { - format!( - "(SELECT {} FROM {} _inner WHERE {})", - expression, - base_relation, - all_conditions.join(" AND ") - ) + if i == 0 && info.primary_table.is_none() { + info.primary_table = Some(table_info.clone()); + } + info.tables.insert(effective_name, table_info); + } } + + info } -/// Expand a single AT modifier for non-decomposable measures -fn expand_non_decomposable_at_to_sql( - expression: &str, - modifier: &ContextModifier, - base_relation_sql: &str, - outer_alias: Option<&str>, - outer_where: Option<&str>, - group_by_cols: &[String], - dimension_exprs: &HashMap, -) -> String { - let base_relation = base_relation_for_subquery(base_relation_sql); +fn extract_from_clause_info(sql: &str) -> FromClauseInfo { + #[cfg(test)] + let mut info = FromClauseInfo::default(); + #[cfg(not(test))] + let mut info = extract_from_clause_info_ffi(sql); - match modifier { - ContextModifier::AllGlobal => { - format!("(SELECT {expression} FROM {base_relation})") - } - ContextModifier::All(dim) => { - let dim_lower = dim.to_lowercase(); - let is_expression = dim.contains('('); - let correlating_dims: Vec<_> = group_by_cols - .iter() - .filter(|col| { - if is_expression { - col.to_lowercase() != dim_lower - } else { - let col_name = col.split('.').next_back().unwrap_or(col); - col_name.to_lowercase() != dim_lower - } - }) - .collect(); + supplement_from_clause_info_from_sql(&mut info, sql); + info +} - if correlating_dims.is_empty() { - format!("(SELECT {expression} FROM {base_relation})") - } else { - let where_clauses: Vec<_> = correlating_dims - .iter() - .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) - .collect(); - format!( - "(SELECT {} FROM {} _inner WHERE {})", - expression, - base_relation, - where_clauses.join(" AND ") - ) - } +fn supplement_from_clause_info_from_sql(info: &mut FromClauseInfo, sql: &str) { + let mut search_pos = 0; + while let Some((keyword_pos, keyword)) = next_from_or_join_keyword(sql, search_pos) { + let table_start = skip_ws_and_comments(sql, keyword_pos + keyword.len()); + if table_start >= sql.len() || sql.as_bytes()[table_start] == b'(' { + search_pos = table_start.saturating_add(1); + continue; } - ContextModifier::Set(dim, expr) => { - let outer_ref = outer_alias.unwrap_or("_outer"); - let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - let dim_key = normalize_dimension_key(dim_name); - let resolved_expr = - resolve_current_in_expr(expr, group_by_cols, outer_where, Some(outer_ref)); - let qualified_expr = if dim.contains('(') { - qualify_where_for_outer(&resolved_expr, outer_ref) - } else { - qualify_outer_reference(&resolved_expr, outer_ref, dim_name) - }; - let inner_dim = if let Some(expr) = dimension_exprs.get(&dim_key) { - qualify_where_for_inner(expr) - } else if dim.contains('(') { - qualify_where_for_inner(dim) + + let Some(table_end) = parse_qualified_name_span(sql, table_start) else { + search_pos = table_start.saturating_add(1); + continue; + }; + let raw_table = sql[table_start..table_end].trim(); + let table_name = + extract_last_qualified_identifier(raw_table).unwrap_or_else(|| raw_table.to_string()); + + let alias_start = skip_ws_and_comments(sql, table_end); + let (effective_name, has_alias, next_pos) = + if let Some((alias_end, alias)) = parse_alias_span(sql, alias_start) { + (alias, true, alias_end) } else { - format!("_inner.{dim_name}") + (table_name.clone(), false, table_end) }; - let set_condition = format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}"); - let dim_lower = dim.to_lowercase(); - let is_expression = dim.contains('('); - let correlation_conditions: Vec = group_by_cols - .iter() - .filter(|col| { - if is_expression { - col.to_lowercase() != dim_lower - } else { - let col_name = col.split('.').next_back().unwrap_or(col); - col_name.to_lowercase() != dim_lower - } - }) - .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) - .collect(); + let table_info = TableInfo { + name: table_name, + effective_name: effective_name.clone(), + has_alias, + }; + if keyword.eq_ignore_ascii_case("FROM") && info.primary_table.is_none() { + info.primary_table = Some(table_info.clone()); + } + info.tables.entry(effective_name).or_insert(table_info); + search_pos = next_pos; + } +} - let mut all_conditions = vec![set_condition]; - all_conditions.extend(correlation_conditions); +fn next_from_or_join_keyword(sql: &str, search_pos: usize) -> Option<(usize, &'static str)> { + let from_pos = find_top_level_keyword(sql, "FROM", search_pos); + let join_pos = find_top_level_keyword(sql, "JOIN", search_pos); + match (from_pos, join_pos) { + (Some(from), Some(join)) if from <= join => Some((from, "FROM")), + (Some(_), Some(join)) => Some((join, "JOIN")), + (Some(from), None) => Some((from, "FROM")), + (None, Some(join)) => Some((join, "JOIN")), + (None, None) => None, + } +} - format!( - "(SELECT {} FROM {} _inner WHERE {})", - expression, - base_relation, - all_conditions.join(" AND ") - ) - } - ContextModifier::Where(condition) => { - let stripped = strip_at_where_qualifiers(condition); - let qualified = - qualify_where_for_inner_with_dimensions(&stripped, dimension_exprs); - format!( - "(SELECT {expression} FROM {base_relation} _inner WHERE {qualified})" - ) - } - ContextModifier::Visible => { - if group_by_cols.is_empty() { - match outer_where { - Some(w) => { - let stripped = strip_at_where_qualifiers(w); - let qualified = - qualify_where_for_inner_with_dimensions(&stripped, dimension_exprs); - format!( - "(SELECT {expression} FROM {base_relation} _inner WHERE {qualified})" - ) - } - None => format!("(SELECT {expression} FROM {base_relation})"), - } - } else { - let where_clauses: Vec<_> = group_by_cols - .iter() - .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) - .collect(); - let full_where = match outer_where { - Some(w) => { - let stripped = strip_at_where_qualifiers(w); - let qualified = qualify_where_for_inner_with_dimensions( - &stripped, - dimension_exprs, - ); - format!("{} AND {}", where_clauses.join(" AND "), qualified) - } - None => where_clauses.join(" AND "), - }; - format!( - "(SELECT {} FROM {} _inner WHERE {full_where})", - expression, base_relation - ) +/// Information about a resolved measure +#[derive(Debug, Clone)] +pub struct ResolvedMeasure { + /// The aggregation function (SUM, COUNT, etc.) + pub agg_fn: String, + /// The source view name + pub source_view: String, + /// For derived measures: the expanded expression + pub derived_expr: Option, + /// Whether this measure can be re-aggregated + pub is_decomposable: bool, + /// Base table for non-decomposable measures (for correlated subquery) + pub base_table: Option, + /// Base relation SQL for non-decomposable recomputation + pub base_relation_sql: Option, + /// Dimension alias -> expression mapping from the source view + pub dimension_exprs: HashMap, + /// GROUP BY columns from the source view definition + pub view_group_by_cols: Vec, + /// The original measure expression (e.g., "COUNT(DISTINCT user_id)") + pub expression: String, +} + +/// Look up which view contains a measure and return resolved measure info +/// Prioritizes default_table (the query's FROM table), then searches other views for JOINs +fn resolve_measure_source(measure_name: &str, default_table: &str) -> ResolvedMeasure { + let views = MEASURE_VIEWS.lock().unwrap(); + + // Helper to build ResolvedMeasure from a found measure + let build_resolved = |m: &ViewMeasure, v: &MeasureView, source_view: &str| -> ResolvedMeasure { + let agg_fn = extract_agg_function(&m.expression); + let derived_expr = if extract_aggregation_function(&m.expression).is_none() { + let expanded = expand_derived_measure_expr(&m.expression, v); + if expanded != m.expression { + Some(expanded) + } else { + None } + } else { + None + }; + + ResolvedMeasure { + agg_fn, + source_view: source_view.to_string(), + derived_expr, + is_decomposable: m.is_decomposable, + base_table: v.base_table.clone(), + base_relation_sql: v.base_relation_sql.clone(), + dimension_exprs: v.dimension_exprs.clone(), + view_group_by_cols: v.group_by_cols.clone(), + expression: m.expression.clone(), + } + }; + + // First, check if the measure exists in the default table (query's primary table) + if let Some(v) = views.get(default_table) { + if let Some(m) = v + .measures + .iter() + .find(|m| m.column_name.eq_ignore_ascii_case(measure_name)) + { + return build_resolved(m, v, default_table); + } + } + + // If not found in default table, search other views (for JOIN support) + for (view_name, v) in views.iter() { + if let Some(m) = v + .measures + .iter() + .find(|m| m.column_name.eq_ignore_ascii_case(measure_name)) + { + return build_resolved(m, v, view_name); } } + + // Fallback: measure not found, return defaults + ResolvedMeasure { + agg_fn: "SUM".to_string(), + source_view: default_table.to_string(), + derived_expr: None, + is_decomposable: true, + base_table: None, + base_relation_sql: None, + dimension_exprs: HashMap::new(), + view_group_by_cols: Vec::new(), + expression: String::new(), + } +} + +/// Find the alias used for a view in the FROM clause +/// Returns the effective_name (alias) if the view is in the FROM clause +fn find_alias_for_view<'a>(from_info: &'a FromClauseInfo, view_name: &str) -> Option<&'a str> { + from_info + .tables + .values() + .find(|t| t.name.eq_ignore_ascii_case(view_name)) + .map(|t| t.effective_name.as_str()) } // ============================================================================= -// Core Functions - AT Expansion +// Core Functions - Non-Decomposable Measure Expansion // ============================================================================= -/// Expand AT modifier to SQL subquery -/// When outer_alias is provided, it's used for correlated references in SET/ALL modifiers -/// When outer_where is provided, it's used for VISIBLE expansion -/// When group_by_cols is provided, it's used for ALL(dim) to correlate on other dimensions -pub fn expand_at_to_sql( - measure_col: &str, - agg_fn: &str, - modifier: &ContextModifier, - table_name: &str, - outer_alias: Option<&str>, - outer_where: Option<&str>, - group_by_cols: &[String], -) -> String { - let measure_expr = format!("{agg_fn}({measure_col})"); +fn base_relation_for_subquery(base_relation_sql: &str) -> String { + let trimmed = base_relation_sql.trim().trim_end_matches(';').trim(); + format!("({trimmed})") +} - match modifier { - ContextModifier::AllGlobal => { - // Grand total - no WHERE clause, aggregate over entire table - format!("(SELECT {measure_expr} FROM {table_name})") - } - ContextModifier::All(dim) => { - // Remove dimension from context - correlate on other GROUP BY dimensions - let outer_ref = outer_alias.unwrap_or(table_name); - // Filter group_by_cols to exclude the removed dimension (case-insensitive) - let dim_lower = dim.to_lowercase(); - let is_expression = dim.contains('('); - let correlating_dims: Vec<_> = group_by_cols - .iter() - .filter(|col| { - if is_expression { - // For expressions like MONTH(date), compare full expression - col.to_lowercase() != dim_lower +/// Check if a dimension string is an expression (not a simple column reference). +/// Expressions contain function calls, operators, or CASE expressions. +fn is_expression_dim(dim: &str) -> bool { + let trimmed = dim.trim(); + let upper = trimmed.to_uppercase(); + trimmed.contains('(') + || trimmed.contains("||") + || trimmed.contains(" + ") + || trimmed.contains(" - ") + || trimmed.contains(" * ") + || trimmed.contains(" / ") + || upper.starts_with("CASE ") + || upper.contains(" CASE ") +} + +/// Strip a specific table qualifier from column references in an expression so +/// it can be re-qualified with `_inner` or an outer alias. +/// Only removes `table_name.column` prefixes where the target is a plain column; +/// schema-qualified function calls (`s.bucket(ts)`) and struct field +/// dereferences are preserved even when the qualifier matches `table_name`. +fn strip_table_qualifier(expr: &str, table_name: &str) -> String { + let mut result = String::new(); + let mut chars = expr.chars().peekable(); + + while let Some(c) = chars.next() { + if c == '\'' { + // Single-quoted string literal: copy verbatim + result.push(c); + while let Some(next) = chars.next() { + result.push(next); + if next == '\'' { + if chars.peek() == Some(&'\'') { + result.push(chars.next().unwrap()); } else { - // For simple columns, extract just the column name (handle qualified refs like "s.year") - let col_name = col.split('.').next_back().unwrap_or(col); - col_name.to_lowercase() != dim_lower + break; } - }) - .collect(); - - if correlating_dims.is_empty() { - // No other dimensions - same as AllGlobal - format!("(SELECT {measure_expr} FROM {table_name})") + } + } + } else if c == '"' || c.is_alphabetic() || c == '_' { + // Collect identifier (possibly double-quoted) + let is_quoted = c == '"'; + let mut ident_raw = String::from(c); // full text including quotes + let ident_name; // unquoted name for comparison + if is_quoted { + while let Some(next) = chars.next() { + ident_raw.push(next); + if next == '"' { + break; + } + } + ident_name = ident_raw[1..ident_raw.len().saturating_sub(1).max(1)].to_string(); } else { - // Correlate on remaining dimensions - let where_clauses: Vec<_> = correlating_dims - .iter() - .map(|col| { - let col_is_expr = col.contains('('); - if col_is_expr { - // For expression dimensions, qualify column refs inside - let inner_expr = qualify_where_for_inner(col); - format!("{inner_expr} IS NOT DISTINCT FROM {col}") + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + ident_raw.push(chars.next().unwrap()); + } else { + break; + } + } + ident_name = ident_raw.clone(); + } + if chars.peek() == Some(&'.') && ident_name.eq_ignore_ascii_case(table_name) { + // Peek past the dot to see what follows. If the next token + // is a function call (identifier immediately followed by '('), + // this is a schema-qualified function, not a table.column + // reference, so preserve the qualifier. + chars.next(); // consume the dot + // Collect the next identifier (if any) to check for '(' + let mut next_ident = String::new(); + if chars.peek() == Some(&'"') { + next_ident.push(chars.next().unwrap()); + while let Some(next) = chars.next() { + next_ident.push(next); + if next == '"' { break; } + } + } else { + while let Some(&next) = chars.peek() { + if next.is_alphanumeric() || next == '_' { + next_ident.push(chars.next().unwrap()); } else { - // Extract just the column name for _inner reference - let col_name = col.split('.').next_back().unwrap_or(col); - format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + break; } - }) - .collect(); - format!( - "(SELECT {} FROM {} _inner WHERE {})", - measure_expr, - table_name, - where_clauses.join(" AND ") - ) - } - } - ContextModifier::Set(dim, expr) => { - // Use outer_alias for the correlated reference, falling back to table_name - let outer_ref = outer_alias.unwrap_or(table_name); - let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - let resolved_expr = - resolve_current_in_expr(expr, group_by_cols, outer_where, Some(outer_ref)); - let qualified_expr = if dim.contains('(') { - qualify_where_for_outer(&resolved_expr, outer_ref) - } else { - qualify_outer_reference(&resolved_expr, outer_ref, dim_name) - }; - // For ad hoc dimensions (expressions like MONTH(date)), qualify column refs with _inner - // For simple columns, just prefix with _inner. - let inner_dim = if dim.contains('(') { - // Expression: qualify column refs inside it - qualify_where_for_inner(dim) - } else { - format!("_inner.{dim_name}") - }; - - // SET condition for the specified dimension - let set_condition = format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}"); - - // Build correlation conditions for OTHER GROUP BY columns (not the SET dim) - // Per paper: SET only removes terms for the specified dimension, correlates on others - let dim_lower = dim.to_lowercase(); - let is_expression = dim.contains('('); - let correlation_conditions: Vec = group_by_cols - .iter() - .filter(|col| { - if is_expression { - col.to_lowercase() != dim_lower - } else { - let col_name = col.split('.').next_back().unwrap_or(col); - col_name.to_lowercase() != dim_lower } - }) - .map(|col| { - let col_is_expr = col.contains('('); - if col_is_expr { - let inner_expr = qualify_where_for_inner(col); - format!("{inner_expr} IS NOT DISTINCT FROM {col}") + } + // Skip whitespace before checking for '(' so that + // `s.bucket (ts)` is recognized as a function call. + let mut ws_after = String::new(); + while let Some(&next) = chars.peek() { + if next == ' ' || next == '\t' || next == '\n' || next == '\r' { + ws_after.push(chars.next().unwrap()); } else { - let col_name = col.split('.').next_back().unwrap_or(col); - format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + break; } - }) - .collect(); - - // Combine: SET condition + correlation on other dims - // NOTE: Do NOT include outer_where - SET bypasses outer WHERE per paper - let mut all_conditions = vec![set_condition]; - all_conditions.extend(correlation_conditions); - - format!( - "(SELECT {} FROM {} _inner WHERE {})", - measure_expr, - table_name, - all_conditions.join(" AND ") - ) - } - ContextModifier::Where(condition) => { - let stripped = strip_at_where_qualifiers(condition); - format!( - "(SELECT {measure_expr} FROM {table_name} WHERE {stripped})" - ) - } - ContextModifier::Visible => { - // VISIBLE means include outer query's WHERE clause AND respect GROUP BY context - let outer_ref = outer_alias.unwrap_or(table_name); - if group_by_cols.is_empty() { - // No GROUP BY - just apply WHERE - match outer_where { - Some(w) => format!("(SELECT {measure_expr} FROM {table_name} WHERE {w})"), - None => measure_expr, + } + let is_function_call = chars.peek() == Some(&'('); + if is_function_call { + // Schema-qualified function: restore qualifier.dot.name(ws) + result.push_str(&ident_raw); + result.push('.'); + result.push_str(&next_ident); + result.push_str(&ws_after); + } else { + // Table-qualified column: drop qualifier, keep column + result.push_str(&next_ident); + result.push_str(&ws_after); } } else { - // Correlate on GROUP BY columns and apply WHERE - let where_clauses: Vec<_> = group_by_cols - .iter() - .map(|col| { - // Extract just the column name for _inner reference - let col_name = col.split('.').next_back().unwrap_or(col); - format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") - }) - .collect(); - let full_where = match outer_where { - Some(w) => format!("{} AND {}", where_clauses.join(" AND "), w), - None => where_clauses.join(" AND "), - }; - format!( - "(SELECT {measure_expr} FROM {table_name} _inner WHERE {full_where})" - ) + result.push_str(&ident_raw); } + } else { + result.push(c); } } + + result } -/// Expand multiple AT modifiers sequentially (right-to-left per paper spec) -pub fn expand_modifiers_to_sql( - measure_col: &str, - agg_fn: &str, - modifiers: &[ContextModifier], - table_name: &str, +fn correlation_exprs_for_dim( + dim: &str, + dimension_exprs: &HashMap, outer_alias: Option<&str>, - outer_where: Option<&str>, - group_by_cols: &[String], -) -> String { - if modifiers.is_empty() { - // No modifiers = default VISIBLE behavior (respect outer WHERE) - return expand_at_to_sql( - measure_col, - agg_fn, - &ContextModifier::Visible, - table_name, - outer_alias, - outer_where, - group_by_cols, - ); +) -> (String, String) { + let dim_trim = dim.trim(); + let dim_name = dim_trim.split('.').next_back().unwrap_or(dim_trim).trim(); + let dim_is_qualified = dim_trim.contains('.'); + let dim_key = normalize_dimension_key(dim_name); + if let Some(expr) = dimension_exprs.get(&dim_key) { + let inner_expr = qualify_where_for_inner(expr); + let outer_expr = if dim_is_qualified { + dim_trim.to_string() + } else { + outer_alias + .map(|alias| format!("{alias}.{dim_name}")) + .unwrap_or_else(|| dim_name.to_string()) + }; + return (inner_expr, outer_expr); } - if modifiers.len() == 1 { - return expand_at_to_sql( - measure_col, - agg_fn, - &modifiers[0], - table_name, - outer_alias, - outer_where, - group_by_cols, - ); + // Expression dimensions (function calls, operators, CASE): strip the source + // table qualifier so inner/outer qualification works, then wrap the outer side + // in ANY_VALUE so DuckDB accepts it in grouped context. Only the known table + // name is stripped; schema qualifiers and struct field access are preserved. + if is_expression_dim(dim_trim) { + let table_to_strip = outer_alias.unwrap_or(""); + let unqualified = strip_table_qualifier(dim_trim, table_to_strip); + let inner_expr = qualify_where_for_inner(&unqualified); + let outer_expr = outer_alias + .map(|alias| format!("ANY_VALUE({})", qualify_where_for_outer(&unqualified, alias))) + .unwrap_or_else(|| dim_trim.to_string()); + return (inner_expr, outer_expr); } - // Check if all modifiers are ALL (dimension or global) - let all_are_all = modifiers - .iter() - .all(|m| matches!(m, ContextModifier::All(_) | ContextModifier::AllGlobal)); - if all_are_all { - // Check for explicit AllGlobal - that means grand total - if modifiers - .iter() - .any(|m| matches!(m, ContextModifier::AllGlobal)) - { - return expand_at_to_sql( - measure_col, - agg_fn, - &ContextModifier::AllGlobal, - table_name, - outer_alias, - outer_where, - group_by_cols, - ); - } + let inner_expr = format!("_inner.{dim_name}"); + let outer_expr = if dim_is_qualified { + dim_trim.to_string() + } else { + outer_alias + .map(|alias| format!("{alias}.{dim_name}")) + .unwrap_or_else(|| dim_name.to_string()) + }; + (inner_expr, outer_expr) +} - // Accumulate all dimensions to remove - let removed_dims: Vec<&str> = modifiers - .iter() - .filter_map(|m| match m { - ContextModifier::All(dim) => Some(dim.as_str()), - _ => None, - }) - .collect(); +fn correlation_condition_for_dim( + dim: &str, + dimension_exprs: &HashMap, + outer_alias: Option<&str>, +) -> String { + let (inner_expr, outer_expr) = correlation_exprs_for_dim(dim, dimension_exprs, outer_alias); + format!("{inner_expr} IS NOT DISTINCT FROM {outer_expr}") +} - // Filter group_by_cols to get remaining dimensions - let remaining_cols: Vec<&String> = group_by_cols - .iter() - .filter(|col| { - let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); - !removed_dims.iter().any(|d| d.to_lowercase() == col_name) - }) - .collect(); +struct NonDecompJoinPlan { + join_sql: String, + replacement: String, +} - if remaining_cols.is_empty() { - // All dimensions removed = grand total - return expand_at_to_sql( - measure_col, - agg_fn, - &ContextModifier::AllGlobal, - table_name, - outer_alias, - outer_where, - group_by_cols, - ); - } +fn expand_non_decomposable_default_context( + expression: &str, + base_relation_sql: &str, + outer_alias: Option<&str>, + group_by_cols: &[String], + dimension_exprs: &HashMap, +) -> String { + let base_relation = base_relation_for_subquery(base_relation_sql); - // Generate correlation on remaining dimensions only - let outer_ref = outer_alias.unwrap_or(table_name); - let measure_expr = format!("{agg_fn}({measure_col})"); - let where_clauses: Vec<_> = remaining_cols - .iter() - .map(|col| { - let col_name = col.split('.').next_back().unwrap_or(col); - format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") - }) - .collect(); - return format!( - "(SELECT {} FROM {} _inner WHERE {})", - measure_expr, - table_name, - where_clauses.join(" AND ") - ); + if group_by_cols.is_empty() { + return format!("(SELECT {expression} FROM {base_relation})"); } - // Apply modifiers right-to-left - // For now, collect the effects: - // - VISIBLE adds outer WHERE (but SET bypasses it per paper) - // - ALL removes all filters - // - SET changes a dimension and bypasses outer WHERE - // - WHERE adds a filter - - // Check if SET is present - per paper, SET bypasses outer WHERE - let has_set = modifiers + let where_clauses: Vec<_> = group_by_cols .iter() - .any(|m| matches!(m, ContextModifier::Set(_, _))); + .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) + .collect(); + + format!( + "(SELECT {} FROM {} _inner WHERE {})", + expression, + base_relation, + where_clauses.join(" AND ") + ) +} + +fn scalar_subquery_to_rows_sql(scalar_sql: &str, value_alias: &str) -> Option { + let trimmed = scalar_sql.trim(); + let inner = trimmed.strip_prefix('(')?.strip_suffix(')')?.trim(); + if !inner.to_uppercase().starts_with("SELECT") { + return None; + } + + let select_start = "SELECT".len(); + let from_pos = find_top_level_keyword(inner, "FROM", select_start)?; + let select_expr = inner[select_start..from_pos].trim(); + let from_clause = inner[from_pos..].trim(); + Some(format!("SELECT {select_expr} AS {value_alias} {from_clause}")) +} + +fn wrap_window_rows_as_single_value(row_sql: &str, measure_name: &str) -> String { + let escaped_measure = measure_name.replace('\'', "''"); + format!( + "(SELECT CASE \ + WHEN EXISTS (\ + SELECT 1 \ + FROM ({row_sql}) __window_vals \ + CROSS JOIN (SELECT __window_value AS __first FROM ({row_sql}) LIMIT 1) __window_first \ + WHERE __window_vals.__window_value IS DISTINCT FROM __window_first.__first\ + ) \ + THEN error('Window measure {escaped_measure} returned multiple values for the evaluation context') \ + ELSE (SELECT __window_value FROM ({row_sql}) LIMIT 1) \ + END)" + ) +} +fn build_non_decomposable_join_plan( + expression: &str, + base_relation_sql: &str, + outer_alias: Option<&str>, + outer_where: Option<&str>, + group_by_cols: &[String], + modifiers: &[ContextModifier], + dimension_exprs: &HashMap, + join_alias: &str, +) -> Option { + let base_relation = base_relation_for_subquery(base_relation_sql); + let mut removed_dims: Vec = Vec::new(); let mut effective_where: Option = None; + let mut set_overrides: HashMap = HashMap::new(); let mut has_all_global = false; - let mut set_conditions: Vec = Vec::new(); - let mut removed_dims: Vec = Vec::new(); - // Process modifiers (in order, which is right-to-left per paper) - for modifier in modifiers.iter().rev() { - match modifier { - ContextModifier::AllGlobal => { - has_all_global = true; - effective_where = None; - set_conditions.clear(); - } - ContextModifier::All(dim) => { - // ALL dim removes that dimension from context - // Track which dimensions are removed for later filtering - removed_dims.push(dim.to_lowercase()); - } - ContextModifier::Visible => { - // Per paper: SET bypasses outer WHERE, so VISIBLE has no effect when SET is present - if !has_set && !has_all_global { - if let Some(w) = outer_where { - let stripped = strip_at_where_qualifiers(w); - effective_where = Some(qualify_where_for_inner(&stripped)); - } - } + let has_set = modifiers + .iter() + .any(|m| matches!(m, ContextModifier::Set(_, _))); + + if modifiers.is_empty() { + if let Some(w) = outer_where { + effective_where = Some(qualify_where_for_inner_with_dimensions(w, dimension_exprs)); + } + } else { + let all_are_all = modifiers + .iter() + .all(|m| matches!(m, ContextModifier::All(_) | ContextModifier::AllGlobal)); + if all_are_all { + if modifiers + .iter() + .any(|m| matches!(m, ContextModifier::AllGlobal)) + { + return None; } - ContextModifier::Where(cond) => { - if !has_all_global { - let stripped = strip_at_where_qualifiers(cond); - // Qualify column references with _inner - effective_where = Some(qualify_where_for_inner(&stripped)); + for modifier in modifiers { + if let ContextModifier::All(dim) = modifier { + removed_dims.push(dim.to_lowercase()); } } - ContextModifier::Set(dim, expr) => { - // Skip SET if ALL(dim) was already processed (dimension removed from context) - let dim_lower = dim.to_lowercase(); - if !has_all_global && !removed_dims.contains(&dim_lower) { - let outer_ref = outer_alias.unwrap_or(table_name); - let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - let resolved_expr = resolve_current_in_expr( - expr, - group_by_cols, - outer_where, - Some(outer_ref), - ); - let qualified_expr = if dim.contains('(') { - qualify_where_for_outer(&resolved_expr, outer_ref) - } else { - qualify_outer_reference(&resolved_expr, outer_ref, dim_name) - }; - // For ad hoc dimensions (expressions), qualify column refs inside - let inner_dim = if dim.contains('(') { - qualify_where_for_inner(dim) - } else { - format!("_inner.{dim_name}") - }; - set_conditions.push(format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}")); + } else { + for modifier in modifiers.iter().rev() { + match modifier { + ContextModifier::AllGlobal => { + has_all_global = true; + effective_where = None; + set_overrides.clear(); + removed_dims.clear(); + } + ContextModifier::All(dim) => { + removed_dims.push(dim.to_lowercase()); + } + ContextModifier::Visible => { + if !has_set && !has_all_global { + if let Some(w) = outer_where { + let stripped = strip_at_where_qualifiers(w); + effective_where = Some(qualify_where_for_inner_with_dimensions( + &stripped, + dimension_exprs, + )); + } + } + } + ContextModifier::Where(cond) => { + if !has_all_global { + let stripped = strip_at_where_qualifiers(cond); + effective_where = + Some(qualify_where_for_inner_with_dimensions( + &stripped, + dimension_exprs, + )); + } + } + ContextModifier::Set(dim, expr) => { + let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); + let dim_key = normalize_dimension_key(dim_name); + if !has_all_global && !removed_dims.contains(&dim_key) { + let resolved_expr = + resolve_current_in_expr(expr, group_by_cols, outer_where, outer_alias); + let outer_expr = if let Some(alias) = outer_alias { + if dim.contains('(') { + qualify_where_for_outer(&resolved_expr, alias) + } else { + qualify_outer_reference(&resolved_expr, alias, dim_name) + } + } else { + resolved_expr + }; + set_overrides.insert(dim_key, outer_expr); + } + } } } } } - // Build final SQL - let measure_expr = format!("{agg_fn}({measure_col})"); - - if has_all_global && set_conditions.is_empty() { - // Pure grand total - return format!("(SELECT {measure_expr} FROM {table_name})"); + if has_all_global && set_overrides.is_empty() { + return None; } - // Filter group_by_cols to exclude removed dimensions let remaining_cols: Vec<&String> = group_by_cols .iter() .filter(|col| { let col_lower = col.to_lowercase(); let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); - // Check both full expression match and simple column match !removed_dims .iter() .any(|d| *d == col_lower || *d == col_name) }) .collect(); - // Build correlation conditions for remaining dimensions - let outer_ref = outer_alias.unwrap_or(table_name); - let correlation_conditions: Vec = remaining_cols - .iter() - .map(|col| { - let col_is_expr = col.contains('('); - if col_is_expr { - // For expression dimensions, qualify column refs inside - let inner_expr = qualify_where_for_inner(col); - format!("{inner_expr} IS NOT DISTINCT FROM {col}") - } else { - let col_name = col.split('.').next_back().unwrap_or(col); - format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") - } - }) - .collect(); - - // Combine all conditions: correlation + SET conditions + WHERE - let mut all_conditions: Vec = correlation_conditions; - all_conditions.extend(set_conditions); - if let Some(w) = effective_where { - all_conditions.push(w); + if remaining_cols.is_empty() { + return None; } - if all_conditions.is_empty() { - format!("(SELECT {measure_expr} FROM {table_name})") - } else { - format!( - "(SELECT {} FROM {} _inner WHERE {})", - measure_expr, - table_name, - all_conditions.join(" AND ") - ) - } -} + let mut select_parts = Vec::new(); + let mut group_parts = Vec::new(); + let mut join_conditions = Vec::new(); -/// Expand AT modifiers for derived measures (pre-expanded expression like "SUM(revenue) - SUM(cost)") -fn expand_modifiers_to_sql_derived( - derived_expr: &str, - modifiers: &[ContextModifier], - table_name: &str, + for (idx, col) in remaining_cols.iter().enumerate() { + let alias = format!("dim_{idx}"); + let (inner_expr, default_outer_expr) = + correlation_exprs_for_dim(col, dimension_exprs, outer_alias); + let dim_name = col.split('.').next_back().unwrap_or(col).trim(); + let dim_key = normalize_dimension_key(dim_name); + let outer_expr = set_overrides + .get(&dim_key) + .cloned() + .unwrap_or(default_outer_expr); + + select_parts.push(format!("{inner_expr} AS {alias}")); + group_parts.push(alias.clone()); + join_conditions.push(format!("{join_alias}.{alias} IS NOT DISTINCT FROM {outer_expr}")); + } + + let mut agg_query = format!( + "SELECT {}, {} AS value FROM {} _inner", + select_parts.join(", "), + expression, + base_relation + ); + if let Some(w) = effective_where { + agg_query.push_str(" WHERE "); + agg_query.push_str(&w); + } + if !group_parts.is_empty() { + agg_query.push_str(" GROUP BY "); + agg_query.push_str(&group_parts.join(", ")); + } + + Some(NonDecompJoinPlan { + join_sql: format!(" LEFT JOIN ({agg_query}) {join_alias} ON {}", join_conditions.join(" AND ")), + replacement: format!("{join_alias}.value"), + }) +} + +/// Expand a non-decomposable measure (like COUNT DISTINCT) to a correlated subquery +/// against the base table. This is used when the measure cannot be re-aggregated. +/// +/// Example: +/// - expression: "COUNT(DISTINCT user_id)" +/// - base_relation_sql: "SELECT * FROM orders" +/// - group_by_cols: ["year", "region"] +/// - Result: (SELECT COUNT(DISTINCT user_id) FROM (SELECT * FROM orders) _inner WHERE _inner.year IS NOT DISTINCT FROM _outer.year AND _inner.region IS NOT DISTINCT FROM _outer.region) +fn expand_non_decomposable_to_sql( + expression: &str, + base_relation_sql: &str, outer_alias: Option<&str>, outer_where: Option<&str>, group_by_cols: &[String], + modifiers: &[ContextModifier], + dimension_exprs: &HashMap, ) -> String { - // For derived measures, we use the pre-expanded expression directly - // The logic is similar to expand_modifiers_to_sql but uses derived_expr instead of AGG(measure) + let base_relation = base_relation_for_subquery(base_relation_sql); if modifiers.is_empty() { - // No modifiers = just use the expression - return format!("(SELECT {derived_expr} FROM {table_name})"); + // No modifiers = default VISIBLE behavior (respect outer WHERE) + return expand_non_decomposable_at_to_sql( + expression, + &ContextModifier::Visible, + base_relation_sql, + outer_alias, + outer_where, + group_by_cols, + dimension_exprs, + ); } - // Check for AllGlobal (grand total) - if modifiers - .iter() - .any(|m| matches!(m, ContextModifier::AllGlobal)) - { - return format!("(SELECT {derived_expr} FROM {table_name})"); + if modifiers.len() == 1 { + return expand_non_decomposable_at_to_sql( + expression, + &modifiers[0], + base_relation_sql, + outer_alias, + outer_where, + group_by_cols, + dimension_exprs, + ); } - // Check if all modifiers are ALL (dimension) + // Check if all modifiers are ALL (dimension or global) let all_are_all = modifiers .iter() - .all(|m| matches!(m, ContextModifier::All(_))); + .all(|m| matches!(m, ContextModifier::All(_) | ContextModifier::AllGlobal)); if all_are_all { - // Accumulate dimensions to remove + // Check for explicit AllGlobal - that means grand total + if modifiers + .iter() + .any(|m| matches!(m, ContextModifier::AllGlobal)) + { + return expand_non_decomposable_at_to_sql( + expression, + &ContextModifier::AllGlobal, + base_relation_sql, + outer_alias, + outer_where, + group_by_cols, + dimension_exprs, + ); + } + + // Accumulate all dimensions to remove let removed_dims: Vec<&str> = modifiers .iter() .filter_map(|m| match m { @@ -5725,7 +5482,6 @@ fn expand_modifiers_to_sql_derived( .filter(|col| { let col_lower = col.to_lowercase(); let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); - // Check both full expression match and simple column match !removed_dims .iter() .any(|d| d.to_lowercase() == col_lower || d.to_lowercase() == col_name) @@ -5734,41 +5490,43 @@ fn expand_modifiers_to_sql_derived( if remaining_cols.is_empty() { // All dimensions removed = grand total - return format!("(SELECT {derived_expr} FROM {table_name})"); + return expand_non_decomposable_at_to_sql( + expression, + &ContextModifier::AllGlobal, + base_relation_sql, + outer_alias, + outer_where, + group_by_cols, + dimension_exprs, + ); } - // Generate correlation on remaining dimensions - let outer_ref = outer_alias.unwrap_or(table_name); + // Generate correlation on remaining dimensions only let where_clauses: Vec<_> = remaining_cols .iter() - .map(|col| { - let col_is_expr = col.contains('('); - if col_is_expr { - // For expression dimensions, qualify column refs inside - let inner_expr = qualify_where_for_inner(col); - format!("{inner_expr} IS NOT DISTINCT FROM {col}") - } else { - let col_name = col.split('.').next_back().unwrap_or(col); - format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") - } - }) + .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) .collect(); return format!( "(SELECT {} FROM {} _inner WHERE {})", - derived_expr, - table_name, + expression, + base_relation, where_clauses.join(" AND ") ); } - // For other modifiers (SET, WHERE, VISIBLE), build conditions - // Check if SET is present - per paper, SET bypasses outer WHERE + // Apply modifiers right-to-left + // - VISIBLE adds outer WHERE (but SET bypasses it per paper) + // - ALL removes dimensions from correlation + // - SET changes a dimension and bypasses outer WHERE + // - WHERE adds a filter + let has_set = modifiers .iter() .any(|m| matches!(m, ContextModifier::Set(_, _))); let mut effective_where: Option = None; let mut has_all_global = false; + let mut set_conditions: Vec = Vec::new(); let mut removed_dims: Vec = Vec::new(); for modifier in modifiers.iter().rev() { @@ -5776,1485 +5534,3788 @@ fn expand_modifiers_to_sql_derived( ContextModifier::AllGlobal => { has_all_global = true; effective_where = None; + set_conditions.clear(); } ContextModifier::All(dim) => { removed_dims.push(dim.to_lowercase()); } ContextModifier::Visible => { - // Per paper: SET bypasses outer WHERE, so VISIBLE has no effect when SET is present if !has_set && !has_all_global { if let Some(w) = outer_where { let stripped = strip_at_where_qualifiers(w); - effective_where = Some(qualify_where_for_inner(&stripped)); + effective_where = Some(qualify_where_for_inner_with_dimensions( + &stripped, + dimension_exprs, + )); } } } ContextModifier::Where(cond) => { if !has_all_global { let stripped = strip_at_where_qualifiers(cond); - effective_where = Some(qualify_where_for_inner(&stripped)); + effective_where = + Some(qualify_where_for_inner_with_dimensions( + &stripped, + dimension_exprs, + )); } } - ContextModifier::Set(_, _) => { - // SET doesn't apply directly to derived measures in the same way - // The derived expression already includes the aggregations + ContextModifier::Set(dim, expr) => { + let dim_lower = dim.to_lowercase(); + if !has_all_global && !removed_dims.contains(&dim_lower) { + let outer_ref = outer_alias.unwrap_or("_outer"); + let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); + let dim_key = normalize_dimension_key(dim_name); + let resolved_expr = resolve_current_in_expr( + expr, + group_by_cols, + outer_where, + Some(outer_ref), + ); + let qualified_expr = if dim.contains('(') { + qualify_where_for_outer(&resolved_expr, outer_ref) + } else { + qualify_outer_reference(&resolved_expr, outer_ref, dim_name) + }; + let inner_dim = if let Some(expr) = dimension_exprs.get(&dim_key) { + qualify_where_for_inner(expr) + } else if dim.contains('(') { + qualify_where_for_inner(dim) + } else { + format!("_inner.{dim_name}") + }; + set_conditions.push(format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}")); + } } } } - if has_all_global { - return format!("(SELECT {derived_expr} FROM {table_name})"); + if has_all_global && set_conditions.is_empty() { + return format!("(SELECT {expression} FROM {base_relation})"); } - // Filter remaining dimensions + // Filter group_by_cols to exclude removed dimensions let remaining_cols: Vec<&String> = group_by_cols .iter() .filter(|col| { let col_lower = col.to_lowercase(); let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); - // Check both full expression match and simple column match !removed_dims .iter() .any(|d| *d == col_lower || *d == col_name) }) .collect(); - // Build conditions - let outer_ref = outer_alias.unwrap_or(table_name); - let mut all_conditions: Vec = remaining_cols + // Build correlation conditions for remaining dimensions + let correlation_conditions: Vec = remaining_cols .iter() - .map(|col| { - let col_is_expr = col.contains('('); - if col_is_expr { - // For expression dimensions, qualify column refs inside - let inner_expr = qualify_where_for_inner(col); - format!("{inner_expr} IS NOT DISTINCT FROM {col}") - } else { - let col_name = col.split('.').next_back().unwrap_or(col); - format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") - } - }) + .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) .collect(); + // Combine all conditions: correlation + SET conditions + WHERE + let mut all_conditions: Vec = correlation_conditions; + all_conditions.extend(set_conditions); if let Some(w) = effective_where { all_conditions.push(w); } if all_conditions.is_empty() { - format!("(SELECT {derived_expr} FROM {table_name})") + format!("(SELECT {expression} FROM {base_relation})") } else { format!( "(SELECT {} FROM {} _inner WHERE {})", - derived_expr, - table_name, + expression, + base_relation, all_conditions.join(" AND ") ) } } -fn validate_set_expression_requirements( - at_patterns: &[(String, Vec, usize, usize)], +/// Expand a single AT modifier for non-decomposable measures +fn expand_non_decomposable_at_to_sql( + expression: &str, + modifier: &ContextModifier, + base_relation_sql: &str, + outer_alias: Option<&str>, + outer_where: Option<&str>, group_by_cols: &[String], - default_qualifier: Option<&str>, -) -> Option { - for (_, modifiers, _, _) in at_patterns { - for modifier in modifiers { - if let ContextModifier::Set(dim, expr) = modifier { - if dim.contains('(') { - continue; - } - let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); - if expr_mentions_identifier_outside_current(expr, dim_name) - && !dimension_in_group_by(dim, group_by_cols, default_qualifier) - { - return Some(format!( - "AT (SET {dim} = {expr}) references {dim_name}, but the query does not group by {dim_name}. Add {dim_name} to SELECT/GROUP BY or use a constant SET value." - )); - } - } - } - } - - None -} + dimension_exprs: &HashMap, +) -> String { + let base_relation = base_relation_for_subquery(base_relation_sql); -/// Expand AGGREGATE() with AT modifiers in SQL -pub fn expand_aggregate_with_at(sql: &str) -> AggregateExpandResult { - let cte_expansion = expand_cte_queries(sql); + match modifier { + ContextModifier::AllGlobal => { + format!("(SELECT {expression} FROM {base_relation})") + } + ContextModifier::All(dim) => { + let dim_lower = dim.to_lowercase(); + let is_expression = dim.contains('('); + let correlating_dims: Vec<_> = group_by_cols + .iter() + .filter(|col| { + if is_expression { + col.to_lowercase() != dim_lower + } else { + let col_name = col.split('.').next_back().unwrap_or(col); + col_name.to_lowercase() != dim_lower + } + }) + .collect(); + + if correlating_dims.is_empty() { + format!("(SELECT {expression} FROM {base_relation})") + } else { + let where_clauses: Vec<_> = correlating_dims + .iter() + .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) + .collect(); + format!( + "(SELECT {} FROM {} _inner WHERE {})", + expression, + base_relation, + where_clauses.join(" AND ") + ) + } + } + ContextModifier::Set(dim, expr) => { + let outer_ref = outer_alias.unwrap_or("_outer"); + let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); + let dim_key = normalize_dimension_key(dim_name); + let resolved_expr = + resolve_current_in_expr(expr, group_by_cols, outer_where, Some(outer_ref)); + let qualified_expr = if dim.contains('(') { + qualify_where_for_outer(&resolved_expr, outer_ref) + } else { + qualify_outer_reference(&resolved_expr, outer_ref, dim_name) + }; + let inner_dim = if let Some(expr) = dimension_exprs.get(&dim_key) { + qualify_where_for_inner(expr) + } else if dim.contains('(') { + qualify_where_for_inner(dim) + } else { + format!("_inner.{dim_name}") + }; + let set_condition = format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}"); + + let dim_lower = dim.to_lowercase(); + let is_expression = dim.contains('('); + let correlation_conditions: Vec = group_by_cols + .iter() + .filter(|col| { + if is_expression { + col.to_lowercase() != dim_lower + } else { + let col_name = col.split('.').next_back().unwrap_or(col); + col_name.to_lowercase() != dim_lower + } + }) + .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) + .collect(); + + let mut all_conditions = vec![set_condition]; + all_conditions.extend(correlation_conditions); + + format!( + "(SELECT {} FROM {} _inner WHERE {})", + expression, + base_relation, + all_conditions.join(" AND ") + ) + } + ContextModifier::Where(condition) => { + let stripped = strip_at_where_qualifiers(condition); + let qualified = + qualify_where_for_inner_with_dimensions(&stripped, dimension_exprs); + format!( + "(SELECT {expression} FROM {base_relation} _inner WHERE {qualified})" + ) + } + ContextModifier::Visible => { + if group_by_cols.is_empty() { + match outer_where { + Some(w) => { + let stripped = strip_at_where_qualifiers(w); + let qualified = + qualify_where_for_inner_with_dimensions(&stripped, dimension_exprs); + format!( + "(SELECT {expression} FROM {base_relation} _inner WHERE {qualified})" + ) + } + None => format!("(SELECT {expression} FROM {base_relation})"), + } + } else { + let where_clauses: Vec<_> = group_by_cols + .iter() + .map(|col| correlation_condition_for_dim(col, dimension_exprs, outer_alias)) + .collect(); + let full_where = match outer_where { + Some(w) => { + let stripped = strip_at_where_qualifiers(w); + let qualified = qualify_where_for_inner_with_dimensions( + &stripped, + dimension_exprs, + ); + format!("{} AND {}", where_clauses.join(" AND "), qualified) + } + None => where_clauses.join(" AND "), + }; + format!( + "(SELECT {} FROM {} _inner WHERE {full_where})", + expression, base_relation + ) + } + } + } +} + +// ============================================================================= +// Core Functions - AT Expansion +// ============================================================================= + +/// Expand AT modifier to SQL subquery +/// When outer_alias is provided, it's used for correlated references in SET/ALL modifiers +/// When outer_where is provided, it's used for VISIBLE expansion +/// When group_by_cols is provided, it's used for ALL(dim) to correlate on other dimensions +pub fn expand_at_to_sql( + measure_col: &str, + agg_fn: &str, + modifier: &ContextModifier, + table_name: &str, + outer_alias: Option<&str>, + outer_where: Option<&str>, + group_by_cols: &[String], +) -> String { + let measure_expr = format!("{agg_fn}({measure_col})"); + + match modifier { + ContextModifier::AllGlobal => { + // Grand total - no WHERE clause, aggregate over entire table + format!("(SELECT {measure_expr} FROM {table_name})") + } + ContextModifier::All(dim) => { + // Remove dimension from context - correlate on other GROUP BY dimensions + let outer_ref = outer_alias.unwrap_or(table_name); + // Filter group_by_cols to exclude the removed dimension (case-insensitive) + let dim_lower = dim.to_lowercase(); + let is_expression = dim.contains('('); + let correlating_dims: Vec<_> = group_by_cols + .iter() + .filter(|col| { + if is_expression { + // For expressions like MONTH(date), compare full expression + col.to_lowercase() != dim_lower + } else { + // For simple columns, extract just the column name (handle qualified refs like "s.year") + let col_name = col.split('.').next_back().unwrap_or(col); + col_name.to_lowercase() != dim_lower + } + }) + .collect(); + + if correlating_dims.is_empty() { + // No other dimensions - same as AllGlobal + format!("(SELECT {measure_expr} FROM {table_name})") + } else { + // Correlate on remaining dimensions + let where_clauses: Vec<_> = correlating_dims + .iter() + .map(|col| { + let col_is_expr = col.contains('('); + if col_is_expr { + // For expression dimensions, qualify column refs inside + let inner_expr = qualify_where_for_inner(col); + format!("{inner_expr} IS NOT DISTINCT FROM {col}") + } else { + // Extract just the column name for _inner reference + let col_name = col.split('.').next_back().unwrap_or(col); + format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + } + }) + .collect(); + format!( + "(SELECT {} FROM {} _inner WHERE {})", + measure_expr, + table_name, + where_clauses.join(" AND ") + ) + } + } + ContextModifier::Set(dim, expr) => { + // Use outer_alias for the correlated reference, falling back to table_name + let outer_ref = outer_alias.unwrap_or(table_name); + let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); + let resolved_expr = + resolve_current_in_expr(expr, group_by_cols, outer_where, Some(outer_ref)); + let qualified_expr = if dim.contains('(') { + qualify_where_for_outer(&resolved_expr, outer_ref) + } else { + qualify_outer_reference(&resolved_expr, outer_ref, dim_name) + }; + // For ad hoc dimensions (expressions like MONTH(date)), qualify column refs with _inner + // For simple columns, just prefix with _inner. + let inner_dim = if dim.contains('(') { + // Expression: qualify column refs inside it + qualify_where_for_inner(dim) + } else { + format!("_inner.{dim_name}") + }; + + // SET condition for the specified dimension + let set_condition = format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}"); + + // Build correlation conditions for OTHER GROUP BY columns (not the SET dim) + // Per paper: SET only removes terms for the specified dimension, correlates on others + let dim_lower = dim.to_lowercase(); + let is_expression = dim.contains('('); + let correlation_conditions: Vec = group_by_cols + .iter() + .filter(|col| { + if is_expression { + col.to_lowercase() != dim_lower + } else { + let col_name = col.split('.').next_back().unwrap_or(col); + col_name.to_lowercase() != dim_lower + } + }) + .map(|col| { + let col_is_expr = col.contains('('); + if col_is_expr { + let inner_expr = qualify_where_for_inner(col); + format!("{inner_expr} IS NOT DISTINCT FROM {col}") + } else { + let col_name = col.split('.').next_back().unwrap_or(col); + format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + } + }) + .collect(); + + // Combine: SET condition + correlation on other dims + // NOTE: Do NOT include outer_where - SET bypasses outer WHERE per paper + let mut all_conditions = vec![set_condition]; + all_conditions.extend(correlation_conditions); + + format!( + "(SELECT {} FROM {} _inner WHERE {})", + measure_expr, + table_name, + all_conditions.join(" AND ") + ) + } + ContextModifier::Where(condition) => { + let stripped = strip_at_where_qualifiers(condition); + format!( + "(SELECT {measure_expr} FROM {table_name} WHERE {stripped})" + ) + } + ContextModifier::Visible => { + // VISIBLE means include outer query's WHERE clause AND respect GROUP BY context + let outer_ref = outer_alias.unwrap_or(table_name); + if group_by_cols.is_empty() { + // No GROUP BY - just apply WHERE + match outer_where { + Some(w) => format!("(SELECT {measure_expr} FROM {table_name} WHERE {w})"), + None => measure_expr, + } + } else { + // Correlate on GROUP BY columns and apply WHERE + let where_clauses: Vec<_> = group_by_cols + .iter() + .map(|col| { + // Extract just the column name for _inner reference + let col_name = col.split('.').next_back().unwrap_or(col); + format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + }) + .collect(); + let full_where = match outer_where { + Some(w) => format!("{} AND {}", where_clauses.join(" AND "), w), + None => where_clauses.join(" AND "), + }; + format!( + "(SELECT {measure_expr} FROM {table_name} _inner WHERE {full_where})" + ) + } + } + } +} + +/// Expand multiple AT modifiers sequentially (right-to-left per paper spec) +pub fn expand_modifiers_to_sql( + measure_col: &str, + agg_fn: &str, + modifiers: &[ContextModifier], + table_name: &str, + outer_alias: Option<&str>, + outer_where: Option<&str>, + group_by_cols: &[String], +) -> String { + if modifiers.is_empty() { + // No modifiers = default VISIBLE behavior (respect outer WHERE) + return expand_at_to_sql( + measure_col, + agg_fn, + &ContextModifier::Visible, + table_name, + outer_alias, + outer_where, + group_by_cols, + ); + } + + if modifiers.len() == 1 { + return expand_at_to_sql( + measure_col, + agg_fn, + &modifiers[0], + table_name, + outer_alias, + outer_where, + group_by_cols, + ); + } + + // Check if all modifiers are ALL (dimension or global) + let all_are_all = modifiers + .iter() + .all(|m| matches!(m, ContextModifier::All(_) | ContextModifier::AllGlobal)); + if all_are_all { + // Check for explicit AllGlobal - that means grand total + if modifiers + .iter() + .any(|m| matches!(m, ContextModifier::AllGlobal)) + { + return expand_at_to_sql( + measure_col, + agg_fn, + &ContextModifier::AllGlobal, + table_name, + outer_alias, + outer_where, + group_by_cols, + ); + } + + // Accumulate all dimensions to remove + let removed_dims: Vec<&str> = modifiers + .iter() + .filter_map(|m| match m { + ContextModifier::All(dim) => Some(dim.as_str()), + _ => None, + }) + .collect(); + + // Filter group_by_cols to get remaining dimensions + let remaining_cols: Vec<&String> = group_by_cols + .iter() + .filter(|col| { + let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); + !removed_dims.iter().any(|d| d.to_lowercase() == col_name) + }) + .collect(); + + if remaining_cols.is_empty() { + // All dimensions removed = grand total + return expand_at_to_sql( + measure_col, + agg_fn, + &ContextModifier::AllGlobal, + table_name, + outer_alias, + outer_where, + group_by_cols, + ); + } + + // Generate correlation on remaining dimensions only + let outer_ref = outer_alias.unwrap_or(table_name); + let measure_expr = format!("{agg_fn}({measure_col})"); + let where_clauses: Vec<_> = remaining_cols + .iter() + .map(|col| { + let col_name = col.split('.').next_back().unwrap_or(col); + format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + }) + .collect(); + return format!( + "(SELECT {} FROM {} _inner WHERE {})", + measure_expr, + table_name, + where_clauses.join(" AND ") + ); + } + + // Apply modifiers right-to-left + // For now, collect the effects: + // - VISIBLE adds outer WHERE (but SET bypasses it per paper) + // - ALL removes all filters + // - SET changes a dimension and bypasses outer WHERE + // - WHERE adds a filter + + // Check if SET is present - per paper, SET bypasses outer WHERE + let has_set = modifiers + .iter() + .any(|m| matches!(m, ContextModifier::Set(_, _))); + + let mut effective_where: Option = None; + let mut has_all_global = false; + let mut set_conditions: Vec = Vec::new(); + let mut removed_dims: Vec = Vec::new(); + + // Process modifiers (in order, which is right-to-left per paper) + for modifier in modifiers.iter().rev() { + match modifier { + ContextModifier::AllGlobal => { + has_all_global = true; + effective_where = None; + set_conditions.clear(); + } + ContextModifier::All(dim) => { + // ALL dim removes that dimension from context + // Track which dimensions are removed for later filtering + removed_dims.push(dim.to_lowercase()); + } + ContextModifier::Visible => { + // Per paper: SET bypasses outer WHERE, so VISIBLE has no effect when SET is present + if !has_set && !has_all_global { + if let Some(w) = outer_where { + let stripped = strip_at_where_qualifiers(w); + effective_where = Some(qualify_where_for_inner(&stripped)); + } + } + } + ContextModifier::Where(cond) => { + if !has_all_global { + let stripped = strip_at_where_qualifiers(cond); + // Qualify column references with _inner + effective_where = Some(qualify_where_for_inner(&stripped)); + } + } + ContextModifier::Set(dim, expr) => { + // Skip SET if ALL(dim) was already processed (dimension removed from context) + let dim_lower = dim.to_lowercase(); + if !has_all_global && !removed_dims.contains(&dim_lower) { + let outer_ref = outer_alias.unwrap_or(table_name); + let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); + let resolved_expr = resolve_current_in_expr( + expr, + group_by_cols, + outer_where, + Some(outer_ref), + ); + let qualified_expr = if dim.contains('(') { + qualify_where_for_outer(&resolved_expr, outer_ref) + } else { + qualify_outer_reference(&resolved_expr, outer_ref, dim_name) + }; + // For ad hoc dimensions (expressions), qualify column refs inside + let inner_dim = if dim.contains('(') { + qualify_where_for_inner(dim) + } else { + format!("_inner.{dim_name}") + }; + set_conditions.push(format!("{inner_dim} IS NOT DISTINCT FROM {qualified_expr}")); + } + } + } + } + + // Build final SQL + let measure_expr = format!("{agg_fn}({measure_col})"); + + if has_all_global && set_conditions.is_empty() { + // Pure grand total + return format!("(SELECT {measure_expr} FROM {table_name})"); + } + + // Filter group_by_cols to exclude removed dimensions + let remaining_cols: Vec<&String> = group_by_cols + .iter() + .filter(|col| { + let col_lower = col.to_lowercase(); + let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); + // Check both full expression match and simple column match + !removed_dims + .iter() + .any(|d| *d == col_lower || *d == col_name) + }) + .collect(); + + // Build correlation conditions for remaining dimensions + let outer_ref = outer_alias.unwrap_or(table_name); + let correlation_conditions: Vec = remaining_cols + .iter() + .map(|col| { + let col_is_expr = col.contains('('); + if col_is_expr { + // For expression dimensions, qualify column refs inside + let inner_expr = qualify_where_for_inner(col); + format!("{inner_expr} IS NOT DISTINCT FROM {col}") + } else { + let col_name = col.split('.').next_back().unwrap_or(col); + format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + } + }) + .collect(); + + // Combine all conditions: correlation + SET conditions + WHERE + let mut all_conditions: Vec = correlation_conditions; + all_conditions.extend(set_conditions); + if let Some(w) = effective_where { + all_conditions.push(w); + } + + if all_conditions.is_empty() { + format!("(SELECT {measure_expr} FROM {table_name})") + } else { + format!( + "(SELECT {} FROM {} _inner WHERE {})", + measure_expr, + table_name, + all_conditions.join(" AND ") + ) + } +} + +/// Expand AT modifiers for derived measures (pre-expanded expression like "SUM(revenue) - SUM(cost)") +fn expand_modifiers_to_sql_derived( + derived_expr: &str, + modifiers: &[ContextModifier], + table_name: &str, + outer_alias: Option<&str>, + outer_where: Option<&str>, + group_by_cols: &[String], +) -> String { + // For derived measures, we use the pre-expanded expression directly + // The logic is similar to expand_modifiers_to_sql but uses derived_expr instead of AGG(measure) + + if modifiers.is_empty() { + // No modifiers = just use the expression + return format!("(SELECT {derived_expr} FROM {table_name})"); + } + + // Check for AllGlobal (grand total) + if modifiers + .iter() + .any(|m| matches!(m, ContextModifier::AllGlobal)) + { + return format!("(SELECT {derived_expr} FROM {table_name})"); + } + + // Check if all modifiers are ALL (dimension) + let all_are_all = modifiers + .iter() + .all(|m| matches!(m, ContextModifier::All(_))); + if all_are_all { + // Accumulate dimensions to remove + let removed_dims: Vec<&str> = modifiers + .iter() + .filter_map(|m| match m { + ContextModifier::All(dim) => Some(dim.as_str()), + _ => None, + }) + .collect(); + + // Filter group_by_cols to get remaining dimensions + let remaining_cols: Vec<&String> = group_by_cols + .iter() + .filter(|col| { + let col_lower = col.to_lowercase(); + let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); + // Check both full expression match and simple column match + !removed_dims + .iter() + .any(|d| d.to_lowercase() == col_lower || d.to_lowercase() == col_name) + }) + .collect(); + + if remaining_cols.is_empty() { + // All dimensions removed = grand total + return format!("(SELECT {derived_expr} FROM {table_name})"); + } + + // Generate correlation on remaining dimensions + let outer_ref = outer_alias.unwrap_or(table_name); + let where_clauses: Vec<_> = remaining_cols + .iter() + .map(|col| { + let col_is_expr = col.contains('('); + if col_is_expr { + // For expression dimensions, qualify column refs inside + let inner_expr = qualify_where_for_inner(col); + format!("{inner_expr} IS NOT DISTINCT FROM {col}") + } else { + let col_name = col.split('.').next_back().unwrap_or(col); + format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + } + }) + .collect(); + return format!( + "(SELECT {} FROM {} _inner WHERE {})", + derived_expr, + table_name, + where_clauses.join(" AND ") + ); + } + + // For other modifiers (SET, WHERE, VISIBLE), build conditions + // Check if SET is present - per paper, SET bypasses outer WHERE + let has_set = modifiers + .iter() + .any(|m| matches!(m, ContextModifier::Set(_, _))); + + let mut effective_where: Option = None; + let mut has_all_global = false; + let mut removed_dims: Vec = Vec::new(); + + for modifier in modifiers.iter().rev() { + match modifier { + ContextModifier::AllGlobal => { + has_all_global = true; + effective_where = None; + } + ContextModifier::All(dim) => { + removed_dims.push(dim.to_lowercase()); + } + ContextModifier::Visible => { + // Per paper: SET bypasses outer WHERE, so VISIBLE has no effect when SET is present + if !has_set && !has_all_global { + if let Some(w) = outer_where { + let stripped = strip_at_where_qualifiers(w); + effective_where = Some(qualify_where_for_inner(&stripped)); + } + } + } + ContextModifier::Where(cond) => { + if !has_all_global { + let stripped = strip_at_where_qualifiers(cond); + effective_where = Some(qualify_where_for_inner(&stripped)); + } + } + ContextModifier::Set(_, _) => { + // SET doesn't apply directly to derived measures in the same way + // The derived expression already includes the aggregations + } + } + } + + if has_all_global { + return format!("(SELECT {derived_expr} FROM {table_name})"); + } + + // Filter remaining dimensions + let remaining_cols: Vec<&String> = group_by_cols + .iter() + .filter(|col| { + let col_lower = col.to_lowercase(); + let col_name = col.split('.').next_back().unwrap_or(col).to_lowercase(); + // Check both full expression match and simple column match + !removed_dims + .iter() + .any(|d| *d == col_lower || *d == col_name) + }) + .collect(); + + // Build conditions + let outer_ref = outer_alias.unwrap_or(table_name); + let mut all_conditions: Vec = remaining_cols + .iter() + .map(|col| { + let col_is_expr = col.contains('('); + if col_is_expr { + // For expression dimensions, qualify column refs inside + let inner_expr = qualify_where_for_inner(col); + format!("{inner_expr} IS NOT DISTINCT FROM {col}") + } else { + let col_name = col.split('.').next_back().unwrap_or(col); + format!("_inner.{col_name} IS NOT DISTINCT FROM {outer_ref}.{col_name}") + } + }) + .collect(); + + if let Some(w) = effective_where { + all_conditions.push(w); + } + + if all_conditions.is_empty() { + format!("(SELECT {derived_expr} FROM {table_name})") + } else { + format!( + "(SELECT {} FROM {} _inner WHERE {})", + derived_expr, + table_name, + all_conditions.join(" AND ") + ) + } +} + +fn validate_set_expression_requirements( + at_patterns: &[(String, Vec, usize, usize)], + group_by_cols: &[String], + default_qualifier: Option<&str>, +) -> Option { + for (_, modifiers, _, _) in at_patterns { + for modifier in modifiers { + if let ContextModifier::Set(dim, expr) = modifier { + if dim.contains('(') { + continue; + } + let dim_name = dim.split('.').next_back().unwrap_or(dim).trim(); + if expr_mentions_identifier_outside_current(expr, dim_name) + && !dimension_in_group_by(dim, group_by_cols, default_qualifier) + { + return Some(format!( + "AT (SET {dim} = {expr}) references {dim_name}, but the query does not group by {dim_name}. Add {dim_name} to SELECT/GROUP BY or use a constant SET value." + )); + } + } + } + } + + None +} + +fn is_warning_identifier_keyword(ident: &str) -> bool { + [ + "AND", + "OR", + "NOT", + "IN", + "IS", + "NULL", + "TRUE", + "FALSE", + "LIKE", + "BETWEEN", + "EXISTS", + "FROM", + "CASE", + "WHEN", + "THEN", + "ELSE", + "END", + "CAST", + "AS", + "CURRENT_DATE", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + ] + .iter() + .any(|kw| kw.eq_ignore_ascii_case(ident)) +} + +fn is_warning_date_part_keyword(ident: &str) -> bool { + [ + "MICROSECOND", + "MICROSECONDS", + "MILLISECOND", + "MILLISECONDS", + "SECOND", + "SECONDS", + "MINUTE", + "MINUTES", + "HOUR", + "HOURS", + "DAY", + "DAYS", + "DOW", + "DOY", + "WEEK", + "WEEKS", + "MONTH", + "MONTHS", + "QUARTER", + "YEAR", + "YEARS", + "EPOCH", + ] + .iter() + .any(|kw| kw.eq_ignore_ascii_case(ident)) +} + +fn is_warning_typed_literal_keyword(ident: &str) -> bool { + ["DATE", "TIME", "TIMESTAMP", "TIMESTAMPTZ", "INTERVAL"] + .iter() + .any(|kw| kw.eq_ignore_ascii_case(ident)) +} + +fn is_warning_typed_literal_start(ident: &str, bytes: &[u8], after_ws: usize) -> bool { + is_warning_typed_literal_keyword(ident) + && after_ws < bytes.len() + && bytes[after_ws] == b'\'' +} + +fn is_warning_interval_unit_keyword(ident: &str) -> bool { + is_warning_date_part_keyword(ident) +} + +fn skip_quoted_sql(bytes: &[u8], mut i: usize, quote: u8) -> usize { + i += 1; + while i < bytes.len() { + if bytes[i] == quote { + if i + 1 < bytes.len() && bytes[i + 1] == quote { + i += 2; + continue; + } + return i + 1; + } + i += 1; + } + i +} + +fn read_unquoted_identifier(where_clause: &str, mut i: usize) -> (String, usize) { + let bytes = where_clause.as_bytes(); + let start = i; + i += 1; + while i < bytes.len() + && ((bytes[i] as char).is_alphanumeric() || bytes[i] == b'_') + { + i += 1; + } + (where_clause[start..i].to_string(), i) +} + +fn read_quoted_identifier(where_clause: &str, mut i: usize) -> (String, usize) { + let bytes = where_clause.as_bytes(); + let mut token = String::new(); + i += 1; + while i < bytes.len() { + if bytes[i] == b'"' { + if i + 1 < bytes.len() && bytes[i + 1] == b'"' { + token.push('"'); + i += 2; + continue; + } + return (token, i + 1); + } + token.push(bytes[i] as char); + i += 1; + } + (token, i) +} + +fn read_warning_dotted_identifier( + where_clause: &str, + first_token: String, + mut i: usize, +) -> Option<(String, String, usize)> { + let bytes = where_clause.as_bytes(); + let mut parts = vec![first_token]; + loop { + let dot_pos = skip_sql_whitespace(bytes, i); + if dot_pos >= bytes.len() || bytes[dot_pos] != b'.' { + break; + } + let next_start = skip_sql_whitespace(bytes, dot_pos + 1); + if next_start >= bytes.len() { + break; + } + let (part, next_end) = if bytes[next_start] == b'"' { + read_quoted_identifier(where_clause, next_start) + } else if (bytes[next_start] as char).is_alphabetic() || bytes[next_start] == b'_' { + read_unquoted_identifier(where_clause, next_start) + } else { + break; + }; + parts.push(part); + i = next_end; + } + + if parts.len() < 2 { + return None; + } + let name = parts.pop().unwrap(); + Some((parts.join("."), name, i)) +} + +fn skip_sql_whitespace(bytes: &[u8], mut i: usize) -> usize { + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + i +} + +fn skip_warning_typed_literal(where_clause: &str, ident: &str, after_ws: usize) -> usize { + let bytes = where_clause.as_bytes(); + let mut i = skip_quoted_sql(bytes, after_ws, b'\''); + if !ident.eq_ignore_ascii_case("INTERVAL") { + return i; + } + + let unit_start = skip_sql_whitespace(bytes, i); + if unit_start < bytes.len() + && (((bytes[unit_start] as char).is_alphabetic()) || bytes[unit_start] == b'_') + { + let (unit, unit_end) = read_unquoted_identifier(where_clause, unit_start); + if is_warning_interval_unit_keyword(&unit) { + i = unit_end; + } + } + i +} + +fn skip_extract_date_part_prefix(where_clause: &str, open_paren: usize) -> usize { + let bytes = where_clause.as_bytes(); + let mut i = skip_sql_whitespace_and_warning_comments(bytes, open_paren + 1); + if i < bytes.len() && (((bytes[i] as char).is_alphabetic()) || bytes[i] == b'_') { + let (date_part, date_part_end) = read_unquoted_identifier(where_clause, i); + if is_warning_date_part_keyword(&date_part) { + i = date_part_end; + } + } + i = skip_sql_whitespace_and_warning_comments(bytes, i); + if warning_keyword_at(where_clause, i, "FROM") { + i += "FROM".len(); + } + i +} + +fn skip_warning_cast_target(where_clause: &str, mut i: usize) -> usize { + let bytes = where_clause.as_bytes(); + i = skip_sql_whitespace_and_warning_comments(bytes, i); + if i >= bytes.len() { + return i; + } + + if bytes[i] == b'"' { + let (_, after_token) = read_quoted_identifier(where_clause, i); + return skip_warning_cast_type_suffix(where_clause, after_token); + } + + if (bytes[i] as char).is_alphabetic() || bytes[i] == b'_' { + let (_, after_token) = read_unquoted_identifier(where_clause, i); + return skip_warning_cast_type_suffix(where_clause, after_token); + } + + i +} + +fn skip_warning_cast_type_suffix(where_clause: &str, mut i: usize) -> usize { + let bytes = where_clause.as_bytes(); + i = skip_sql_whitespace_and_warning_comments(bytes, i); + if i < bytes.len() && bytes[i] == b'(' { + i = skip_balanced_warning_parens(where_clause, i + 1); + } + skip_sql_whitespace_and_warning_comments(bytes, i) +} + +fn skip_warning_identifier_comment(bytes: &[u8], i: usize) -> Option { + if i + 1 >= bytes.len() { + return None; + } + + if bytes[i] == b'-' && bytes[i + 1] == b'-' { + let mut next = i + 2; + while next < bytes.len() && bytes[next] != b'\n' && bytes[next] != b'\r' { + next += 1; + } + return Some(next); + } + + if bytes[i] == b'/' && bytes[i + 1] == b'*' { + let mut next = i + 2; + while next + 1 < bytes.len() { + if bytes[next] == b'*' && bytes[next + 1] == b'/' { + return Some(next + 2); + } + next += 1; + } + return Some(bytes.len()); + } + + None +} + +fn skip_sql_whitespace_and_warning_comments(bytes: &[u8], mut i: usize) -> usize { + loop { + i = skip_sql_whitespace(bytes, i); + if let Some(next) = skip_warning_identifier_comment(bytes, i) { + i = next; + continue; + } + return i; + } +} + +fn warning_keyword_at(sql: &str, idx: usize, keyword: &str) -> bool { + let bytes = sql.as_bytes(); + let keyword_bytes = keyword.as_bytes(); + if idx + keyword_bytes.len() > bytes.len() { + return false; + } + if idx > 0 && (bytes[idx - 1].is_ascii_alphanumeric() || bytes[idx - 1] == b'_') { + return false; + } + if idx + keyword_bytes.len() < bytes.len() + && (bytes[idx + keyword_bytes.len()].is_ascii_alphanumeric() + || bytes[idx + keyword_bytes.len()] == b'_') + { + return false; + } + bytes[idx..idx + keyword_bytes.len()] + .iter() + .zip(keyword_bytes) + .all(|(left, right)| left.eq_ignore_ascii_case(right)) +} + +fn skip_balanced_warning_parens(sql: &str, mut i: usize) -> usize { + let bytes = sql.as_bytes(); + let mut depth = 0usize; + while i < bytes.len() { + if let Some(next) = skip_warning_identifier_comment(bytes, i) { + i = next; + continue; + } + match bytes[i] { + b'\'' => { + i = skip_quoted_sql(bytes, i, b'\''); + } + b'"' => { + i = skip_quoted_sql(bytes, i, b'"'); + } + b'(' => { + depth += 1; + i += 1; + } + b')' => { + if depth == 0 { + return i + 1; + } + depth -= 1; + i += 1; + } + _ => { + i += 1; + } + } + } + i +} + +fn skip_parenthesized_warning_subquery(where_clause: &str, i: usize) -> Option { + let bytes = where_clause.as_bytes(); + if i >= bytes.len() || bytes[i] != b'(' { + return None; + } + let query_start = skip_sql_whitespace_and_warning_comments(bytes, i + 1); + if warning_keyword_at(where_clause, query_start, "SELECT") + || warning_keyword_at(where_clause, query_start, "WITH") + { + return Some(skip_balanced_warning_parens(where_clause, i + 1)); + } + None +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +struct WarningFilterIdentifier { + qualifier: Option, + name: String, +} + +impl WarningFilterIdentifier { + fn new(qualifier: Option<&str>, name: &str) -> Self { + Self { + qualifier: qualifier.map(normalize_identifier_name), + name: normalize_identifier_name(name), + } + } +} + +fn extract_where_filter_identifiers(where_clause: &str) -> Vec { + let mut identifiers = Vec::new(); + let bytes = where_clause.as_bytes(); + let mut i = 0; + + while i < bytes.len() { + if let Some(delimiter) = dollar_quote_delimiter_at(where_clause, i) { + if let Some(end_offset) = where_clause[i + delimiter.len()..].find(delimiter) { + i += delimiter.len() + end_offset + delimiter.len(); + continue; + } + } + if let Some(next) = skip_warning_identifier_comment(bytes, i) { + i = next; + continue; + } + if let Some(next) = skip_parenthesized_warning_subquery(where_clause, i) { + i = next; + continue; + } + + match bytes[i] { + b'\'' => { + i = skip_quoted_sql(bytes, i, b'\''); + } + b':' if i + 1 < bytes.len() && bytes[i + 1] == b':' => { + i = skip_warning_cast_target(where_clause, i + 2); + } + b'"' => { + let (token, after_token) = read_quoted_identifier(where_clause, i); + if let Some((qualifier, qualified_token, next_end)) = + read_warning_dotted_identifier(where_clause, token.clone(), after_token) + { + let next_after_ws = skip_sql_whitespace(bytes, next_end); + if !is_warning_identifier_keyword(&qualified_token) + && !(next_after_ws < bytes.len() && bytes[next_after_ws] == b'(') + { + identifiers.push(WarningFilterIdentifier::new( + Some(&qualifier), + &qualified_token, + )); + } + i = next_end; + continue; + } + identifiers.push(WarningFilterIdentifier::new(None, &token)); + i = after_token; + } + c if (c as char).is_alphabetic() || c == b'_' => { + let (token, after_token) = read_unquoted_identifier(where_clause, i); + let after_ws = skip_sql_whitespace(bytes, after_token); + + if let Some((qualifier, qualified_token, next_end)) = + read_warning_dotted_identifier(where_clause, token.clone(), after_token) + { + let next_after_ws = skip_sql_whitespace(bytes, next_end); + if !is_warning_identifier_keyword(&qualified_token) + && !is_warning_typed_literal_start(&qualified_token, bytes, next_after_ws) + && !(next_after_ws < bytes.len() && bytes[next_after_ws] == b'(') + { + identifiers.push(WarningFilterIdentifier::new( + Some(&qualifier), + &qualified_token, + )); + } + i = next_end; + continue; + } + + if is_warning_typed_literal_start(&token, bytes, after_ws) { + i = skip_warning_typed_literal(where_clause, &token, after_ws); + continue; + } + if token.eq_ignore_ascii_case("EXTRACT") + && after_ws < bytes.len() + && bytes[after_ws] == b'(' + { + i = skip_extract_date_part_prefix(where_clause, after_ws); + continue; + } + if token.eq_ignore_ascii_case("AS") { + i = skip_warning_cast_target(where_clause, after_ws); + continue; + } + if !is_warning_identifier_keyword(&token) + && !(after_ws < bytes.len() && bytes[after_ws] == b'(') + { + identifiers.push(WarningFilterIdentifier::new(None, &token)); + } + i = after_token; + } + _ => { + i += 1; + } + } + } + + identifiers.sort_by(|left, right| { + left.qualifier + .cmp(&right.qualifier) + .then(left.name.cmp(&right.name)) + }); + identifiers.dedup(); + identifiers +} + +fn warning_filter_matches_source( + identifier: &WarningFilterIdentifier, + source_dims: &HashSet, + source_qualifiers: &HashSet, +) -> bool { + if !source_dims.is_empty() && !source_dims.contains(&identifier.name) { + return false; + } + match identifier.qualifier.as_deref() { + Some(qualifier) => { + source_qualifiers.is_empty() + || source_qualifiers.iter().any(|source| { + qualifier == source + || qualifier + .strip_suffix(source) + .is_some_and(|prefix| prefix.ends_with('.')) + || source + .strip_suffix(qualifier) + .is_some_and(|prefix| prefix.ends_with('.')) + }) + } + None => true, + } +} + +fn dimension_key_in_group_by(ident: &str, group_by_cols: &[String]) -> bool { + group_by_cols.iter().any(|col| { + if let Some((_, dim_name)) = parse_simple_measure_ref(col) { + return dim_name == ident; + } + let simple_name = col.split('.').next_back().unwrap_or(col).trim(); + normalize_dimension_key(simple_name) == ident + }) +} + +fn effective_at_where_filter_identifiers( + modifiers: &[ContextModifier], + source_dims: &HashSet, + source_qualifiers: &HashSet, +) -> Vec { + if modifiers + .iter() + .any(|modifier| matches!(modifier, ContextModifier::AllGlobal)) + { + return Vec::new(); + } + + modifiers + .iter() + .find_map(|modifier| { + if let ContextModifier::Where(condition) = modifier { + Some( + extract_where_filter_identifiers(condition) + .into_iter() + .filter(|identifier| { + warning_filter_matches_source( + identifier, + source_dims, + source_qualifiers, + ) + }) + .map(|identifier| identifier.name) + .collect(), + ) + } else { + None + } + }) + .unwrap_or_default() +} + +fn normalized_warning_expression(sql: &str) -> String { + sql.chars() + .filter(|c| !c.is_whitespace()) + .flat_map(|c| c.to_lowercase()) + .collect() +} + +fn normalized_warning_expression_search_text(sql: &str) -> String { + let bytes = sql.as_bytes(); + let mut result = String::new(); + let mut i = 0; + while i < bytes.len() { + if let Some(delimiter) = dollar_quote_delimiter_at(sql, i) { + if let Some(end_offset) = sql[i + delimiter.len()..].find(delimiter) { + i += delimiter.len() + end_offset + delimiter.len(); + continue; + } + } + if let Some(next) = skip_warning_identifier_comment(bytes, i) { + i = next; + continue; + } + if bytes[i] == b'\'' { + i = skip_quoted_sql(bytes, i, b'\''); + continue; + } + let c = bytes[i] as char; + if !c.is_whitespace() { + result.extend(c.to_lowercase()); + } + i += 1; + } + result +} + +fn warning_expression_in_clause(expr: &str, clause: &str) -> bool { + let needle = normalized_warning_expression(expr); + if needle.is_empty() { + return false; + } + + let haystack = normalized_warning_expression_search_text(clause); + let mut search_from = 0; + while let Some(relative_idx) = haystack[search_from..].find(&needle) { + let idx = search_from + relative_idx; + let before = haystack[..idx].chars().next_back(); + let after = haystack[idx + needle.len()..].chars().next(); + if !before.is_some_and(|c| c.is_alphanumeric() || c == '_') + && !after.is_some_and(|c| c.is_alphanumeric() || c == '_') + { + return true; + } + search_from = idx + needle.len(); + } + + false +} + +fn append_unique_warnings(result: &mut AggregateExpandResult, warnings: Vec) { + for warning in warnings { + if !result.warnings.contains(&warning) { + result.warnings.push(warning); + } + } +} + +#[cfg(test)] +fn warning_for_at_all_ungrouped_where( + measure_name: &str, + modifiers: &[ContextModifier], + outer_where: Option<&str>, + group_by_cols: &[String], + source_dims: &HashSet, +) -> Option { + warning_for_at_all_ungrouped_where_with_qualifiers( + measure_name, + modifiers, + outer_where, + group_by_cols, + source_dims, + &HashSet::new(), + ) +} + +fn warning_for_at_all_ungrouped_where_with_qualifiers( + measure_name: &str, + modifiers: &[ContextModifier], + outer_where: Option<&str>, + group_by_cols: &[String], + source_dims: &HashSet, + source_qualifiers: &HashSet, +) -> Option { + let has_all_global = modifiers + .iter() + .any(|m| matches!(m, ContextModifier::AllGlobal)); + if !has_all_global && !modifiers.iter().any(|m| matches!(m, ContextModifier::All(_))) { + return None; + } + let has_set = modifiers + .iter() + .any(|m| matches!(m, ContextModifier::Set(_, _))); + let visible_is_effective = modifiers.iter().enumerate().any(|(idx, modifier)| { + matches!(modifier, ContextModifier::Visible) + && !has_set + && !has_all_global + && !modifiers + .iter() + .take(idx) + .any(|earlier| matches!(earlier, ContextModifier::Where(_))) + }); + if visible_is_effective { + return None; + } + + let where_clause = outer_where?; + let removed_filter_dims: HashSet = modifiers + .iter() + .filter_map(|modifier| { + if let ContextModifier::All(dim) = modifier { + Some(normalize_dimension_key( + dim.split('.').next_back().unwrap_or(dim).trim(), + )) + } else { + None + } + }) + .collect(); + let encoded_filter_dims: HashSet = modifiers + .iter() + .enumerate() + .filter_map(|(idx, modifier)| { + if let ContextModifier::Set(dim, _) = modifier { + let dim_key = normalize_dimension_key(dim.split('.').next_back().unwrap_or(dim).trim()); + let removed_by_all = + modifiers.iter().enumerate().any(|(all_idx, other)| { + if all_idx == idx { + return false; + } + match other { + ContextModifier::AllGlobal => true, + ContextModifier::All(all_dim) if all_idx > idx => { + normalize_dimension_key( + all_dim.split('.').next_back().unwrap_or(all_dim).trim(), + ) == dim_key + } + _ => false, + } + }); + if removed_by_all { + None + } else { + let mut encoded_dims = vec![dim_key]; + if warning_expression_in_clause(dim, where_clause) { + encoded_dims.extend( + extract_where_filter_identifiers(dim) + .into_iter() + .filter(|identifier| { + warning_filter_matches_source( + identifier, + source_dims, + source_qualifiers, + ) + }) + .map(|identifier| identifier.name), + ); + } + Some(encoded_dims) + } + } else { + None + } + }) + .flatten() + .collect(); + let encoded_filter_dims: HashSet = encoded_filter_dims + .into_iter() + .chain(effective_at_where_filter_identifiers( + modifiers, + source_dims, + source_qualifiers, + )) + .collect(); + let mut ungrouped_filters: Vec = extract_where_filter_identifiers(where_clause) + .into_iter() + .filter(|identifier| { + warning_filter_matches_source(identifier, source_dims, source_qualifiers) + }) + .map(|identifier| identifier.name) + .filter(|ident| { + !dimension_key_in_group_by(ident, group_by_cols) + || has_all_global + || removed_filter_dims.contains(ident) + }) + .filter(|ident| !encoded_filter_dims.contains(ident)) + .collect(); + + ungrouped_filters.sort(); + ungrouped_filters.dedup(); + + if ungrouped_filters.is_empty() { + return None; + } + + Some(format!( + "AT (ALL ...) on AGGREGATE({measure_name}) does not preserve outer WHERE filter(s) on ungrouped dimension(s): {}. Add the filter dimension(s) to SELECT/GROUP BY or use an explicit AT modifier that encodes the intended denominator.", + ungrouped_filters.join(", ") + )) +} + +/// Expand AGGREGATE() with AT modifiers in SQL +pub fn expand_aggregate_with_at(sql: &str) -> AggregateExpandResult { + let cte_expansion = if let Some((body_start, body_end)) = + top_level_parenthesized_query_body_range(sql) + { + let body_sql = &sql[body_start..body_end]; + let body_expansion = expand_cte_queries(body_sql); + if body_expansion.had_aggregate + || body_expansion.sql != body_sql + || !body_expansion.warnings.is_empty() + { + let mut expanded_sql = sql.to_string(); + if body_expansion.sql != body_sql { + expanded_sql.replace_range(body_start..body_end, &body_expansion.sql); + } + CteExpansion { + sql: expanded_sql, + had_aggregate: body_expansion.had_aggregate, + warnings: body_expansion.warnings, + } + } else { + expand_cte_queries(sql) + } + } else { + expand_cte_queries(sql) + }; let mut sql = cte_expansion.sql; let mut had_aggregate = cte_expansion.had_aggregate; + let mut warnings = cte_expansion.warnings; + + if has_measure_at_refs(&sql) { + sql = rewrite_measure_at_refs(&sql); + had_aggregate = true; + } + + if has_implicit_measure_refs(&sql) { + sql = rewrite_implicit_measure_refs(&sql); + had_aggregate = true; + } + + // Check if we need the full expansion path (AT modifiers or non-decomposable measures) + let has_aggregate = has_aggregate_function(&sql); + + // If no AGGREGATE function at all, nothing to do + if !has_aggregate { + return AggregateExpandResult { + had_aggregate, + expanded_sql: sql, + error: None, + warnings, + }; + } + had_aggregate = true; + + let at_patterns = extract_aggregate_with_at_full(&sql); + // Keep full expansion path even without AT to handle non-decomposable measures safely + + // Prefer parser-FFI FROM extraction (supports JOIN aliases when SQL parses there), + // then fall back to string extraction for AGGREGATE/AT syntax that parser-FFI cannot parse. + let context_sql = aggregate_context_sql(&sql); + let mut from_info = extract_from_clause_info(context_sql); + let (primary_table_name, existing_alias) = if let Some(pt) = from_info.primary_table.clone() { + let alias = if pt.has_alias { + Some(pt.effective_name.clone()) + } else { + None + }; + (pt.name, alias) + } else { + let (table_name, alias) = + extract_table_and_alias_from_sql(context_sql).unwrap_or_else(|| ("t".to_string(), None)); + let primary_table = TableInfo { + name: table_name.clone(), + effective_name: alias.clone().unwrap_or_else(|| table_name.clone()), + has_alias: alias.is_some(), + }; + from_info + .tables + .insert(primary_table.effective_name.clone(), primary_table.clone()); + from_info.primary_table = Some(primary_table); + (table_name, alias) + }; + + // Extract outer WHERE clause for VISIBLE semantics + let outer_where = extract_where_clause(context_sql); + let outer_where_ref = outer_where.as_deref(); + + // Extract GROUP BY columns for AT (ALL dim) correlation + let group_by_cols = extract_group_by_columns(context_sql); + + let default_set_qualifier = if let Some(alias) = existing_alias.as_deref() { + Some(alias) + } else if primary_table_name.is_empty() { + None + } else { + Some(primary_table_name.as_str()) + }; + if let Some(error) = validate_set_expression_requirements( + &at_patterns, + &group_by_cols, + default_set_qualifier, + ) { + return AggregateExpandResult { + had_aggregate: true, + expanded_sql: sql.to_string(), + error: Some(error), + warnings: Vec::new(), + }; + } + + // Extract dimension columns from original SQL for implicit GROUP BY + // (must be done before expansion since expanded SQL has SUM() etc) + let original_dim_cols = extract_dimension_columns_from_select(context_sql); + let effective_group_by_cols = if group_by_cols.is_empty() { + original_dim_cols.clone() + } else { + group_by_cols.clone() + }; + + let has_expression_dimensions = original_dim_cols.iter().any(|col| col.contains('(')); + // Check if query needs an explicit outer alias for correlation handling. + let needs_outer_alias = has_expression_dimensions + || at_patterns.iter().any(|(_, modifiers, _, _)| { + modifiers.iter().any(|m| { + matches!(m, ContextModifier::Set(_, _)) + || matches!(m, ContextModifier::All(_)) + || matches!(m, ContextModifier::Visible) + }) + }); + + let mut result_sql = sql; + + // Handle alias for the primary table if needed for correlation + let mut insert_outer_alias = false; + let primary_alias: Option = if needs_outer_alias { + if let Some(ref pt) = from_info.primary_table { + if pt.has_alias { + Some(pt.effective_name.clone()) + } else if insert_primary_table_alias(&result_sql, "_outer").is_some() { + insert_outer_alias = true; + Some("_outer".to_string()) + } else { + None + } + } else { + None + } + } else { + None + }; + + let mut patterns = at_patterns; + patterns.sort_by(|a, b| b.2.cmp(&a.2)); + for (measure_name, modifiers, start, end) in patterns { + let measure_lookup_name = strip_measure_qualifier(&measure_name); + // Look up which view contains this measure (for JOIN support) + let resolved = resolve_measure_source(&measure_lookup_name, &primary_table_name); + let mut measure_group_by_cols = filter_group_by_cols_for_measure( + &effective_group_by_cols, + &resolved.view_group_by_cols, + &resolved.dimension_exprs, + ); + let mut allowed_qualifiers: HashSet = HashSet::new(); + allowed_qualifiers.insert(normalize_identifier_name(&resolved.source_view)); + if let Some(alias) = find_alias_for_view(&from_info, &resolved.source_view) { + allowed_qualifiers.insert(normalize_identifier_name(alias)); + } + if let Some(ref pt) = from_info.primary_table { + if pt.name.eq_ignore_ascii_case(&resolved.source_view) { + if let Some(alias) = primary_alias.as_deref() { + allowed_qualifiers.insert(normalize_identifier_name(alias)); + } + } + } + let source_dims = source_dimension_names(&resolved.source_view); + measure_group_by_cols.retain(|col| { + if let Some((Some(qualifier), dim_name)) = parse_simple_measure_ref(col) { + return allowed_qualifiers.contains(&qualifier) || source_dims.contains(&dim_name); + } + let dim_key = normalize_dimension_key(col); + source_dims.is_empty() + || source_dims.contains(&dim_key) + || resolved.dimension_exprs.contains_key(&dim_key) + || source_dims + .iter() + .any(|dim_name| expr_mentions_identifier(col, dim_name)) + }); + let eval_group_by_cols = + if measure_group_by_cols.is_empty() + && !original_dim_cols.is_empty() + && resolved.source_view.eq_ignore_ascii_case(&primary_table_name) + { + original_dim_cols.clone() + } else { + measure_group_by_cols.clone() + }; + + if let Some(warning) = warning_for_at_all_ungrouped_where_with_qualifiers( + &measure_lookup_name, + &modifiers, + outer_where_ref, + &eval_group_by_cols, + &source_dims, + &allowed_qualifiers, + ) { + if !warnings.contains(&warning) { + warnings.push(warning); + } + } + + // Non-decomposable measures are recomputed from base rows (including AT modifiers) + + // Find the alias for this measure's source view in the FROM clause + // If the source view is the primary table and we added _outer, use _outer + let outer_alias = if let Some(ref pt) = from_info.primary_table { + if pt.name.eq_ignore_ascii_case(&resolved.source_view) { + // Source view is the primary table, use primary_alias (which may be _outer) + primary_alias.clone() + } else { + // Source view is not the primary table, look it up in from_info + find_alias_for_view(&from_info, &resolved.source_view) + .map(|s| s.to_string()) + .or_else(|| primary_alias.clone()) + } + } else { + primary_alias.clone() + }; + let outer_alias_ref = outer_alias.as_deref(); + + let outer_ref_for_eval = outer_alias_ref.or(Some(resolved.source_view.as_str())); + let base_relation_sql = resolved + .base_relation_sql + .clone() + .or_else(|| { + resolved + .base_table + .clone() + .map(|table| format!("SELECT * FROM {table}")) + }) + .unwrap_or_else(|| format!("SELECT * FROM {}", resolved.source_view)); + + let expression_for_eval = resolved + .derived_expr + .clone() + .unwrap_or_else(|| resolved.expression.clone()); + let is_window_measure = + is_window_expression(&expression_for_eval) || is_window_expression(&resolved.expression); + + let expanded = if is_window_measure { + let scalar_eval_sql = expand_non_decomposable_to_sql( + &expression_for_eval, + &base_relation_sql, + outer_ref_for_eval, + outer_where_ref, + &eval_group_by_cols, + &modifiers, + &resolved.dimension_exprs, + ); + let row_eval_sql = match scalar_subquery_to_rows_sql(&scalar_eval_sql, "__window_value") + { + Some(sql) => sql, + None => { + return AggregateExpandResult { + had_aggregate: true, + expanded_sql: result_sql, + error: Some(format!( + "Failed to rewrite window measure {measure_lookup_name} with AT modifiers" + )), + warnings: Vec::new(), + }; + } + }; + let eval_sql = wrap_window_rows_as_single_value(&row_eval_sql, &measure_lookup_name); + if original_dim_cols.is_empty() { + format!("MAX({eval_sql})") + } else { + eval_sql + } + } else if !expression_for_eval.is_empty() { + let eval_sql = expand_non_decomposable_to_sql( + &expression_for_eval, + &base_relation_sql, + outer_ref_for_eval, + outer_where_ref, + &eval_group_by_cols, + &modifiers, + &resolved.dimension_exprs, + ); + if original_dim_cols.is_empty() { + format!("MAX({eval_sql})") + } else { + eval_sql + } + } else { + expand_modifiers_to_sql( + &measure_lookup_name, + &resolved.agg_fn, + &modifiers, + &resolved.source_view, + outer_alias_ref, + outer_where_ref, + &eval_group_by_cols, + ) + }; + result_sql = format!("{}{}{}", &result_sql[..start], expanded, &result_sql[end..]); + } + + // Also expand plain AGGREGATE() calls (without AT modifiers) using text replacement + let mut plain_calls = extract_all_aggregate_calls(&result_sql); + plain_calls.sort_by(|a, b| b.1.cmp(&a.1)); // Sort by position descending + + for (measure_name, start, end) in plain_calls { + let mut replacement_end = end; + let mut use_default_context = false; + let suffix = &result_sql[end..]; + let leading_ws = suffix.len() - suffix.trim_start().len(); + if suffix[leading_ws..].starts_with(DEFAULT_CONTEXT_MARKER) { + use_default_context = true; + replacement_end = end + leading_ws + DEFAULT_CONTEXT_MARKER.len(); + } + + let measure_lookup_name = strip_measure_qualifier(&measure_name); + let resolved = resolve_measure_source(&measure_lookup_name, &primary_table_name); + let mut measure_group_by_cols = filter_group_by_cols_for_measure( + &effective_group_by_cols, + &resolved.view_group_by_cols, + &resolved.dimension_exprs, + ); + let mut allowed_qualifiers: HashSet = HashSet::new(); + allowed_qualifiers.insert(normalize_identifier_name(&resolved.source_view)); + if let Some(alias) = find_alias_for_view(&from_info, &resolved.source_view) { + allowed_qualifiers.insert(normalize_identifier_name(alias)); + } + if let Some(ref pt) = from_info.primary_table { + if pt.name.eq_ignore_ascii_case(&resolved.source_view) { + if let Some(alias) = primary_alias.as_deref() { + allowed_qualifiers.insert(normalize_identifier_name(alias)); + } + } + } + let source_dims = source_dimension_names(&resolved.source_view); + measure_group_by_cols.retain(|col| { + if let Some((Some(qualifier), dim_name)) = parse_simple_measure_ref(col) { + return allowed_qualifiers.contains(&qualifier) || source_dims.contains(&dim_name); + } + let dim_key = normalize_dimension_key(col); + source_dims.is_empty() + || source_dims.contains(&dim_key) + || resolved.dimension_exprs.contains_key(&dim_key) + || source_dims + .iter() + .any(|dim_name| expr_mentions_identifier(col, dim_name)) + }); + let eval_group_by_cols = + if measure_group_by_cols.is_empty() + && !original_dim_cols.is_empty() + && resolved.source_view.eq_ignore_ascii_case(&primary_table_name) + { + original_dim_cols.clone() + } else { + measure_group_by_cols.clone() + }; + + + let outer_alias = if let Some(ref pt) = from_info.primary_table { + if pt.name.eq_ignore_ascii_case(&resolved.source_view) { + primary_alias.clone().or_else(|| { + find_alias_for_view(&from_info, &resolved.source_view).map(|s| s.to_string()) + }) + } else { + find_alias_for_view(&from_info, &resolved.source_view) + .map(|s| s.to_string()) + .or_else(|| primary_alias.clone()) + } + } else { + primary_alias.clone() + }; + let outer_ref_for_eval = outer_alias.as_deref().or(Some(resolved.source_view.as_str())); + let base_relation_sql = resolved + .base_relation_sql + .clone() + .or_else(|| { + resolved + .base_table + .clone() + .map(|table| format!("SELECT * FROM {table}")) + }) + .unwrap_or_else(|| format!("SELECT * FROM {}", resolved.source_view)); + let expression_for_eval = resolved + .derived_expr + .clone() + .unwrap_or_else(|| resolved.expression.clone()); + let is_window_measure = + is_window_expression(&expression_for_eval) || is_window_expression(&resolved.expression); + + let expanded = if is_window_measure { + let _ = use_default_context; + let _ = base_relation_sql; + let _ = outer_ref_for_eval; + let _ = outer_where_ref; + let _ = eval_group_by_cols; + format!("{}({measure_lookup_name})", resolved.agg_fn) + } else if !expression_for_eval.is_empty() { + let eval_sql = if use_default_context { + expand_non_decomposable_default_context( + &expression_for_eval, + &base_relation_sql, + outer_ref_for_eval, + &eval_group_by_cols, + &resolved.dimension_exprs, + ) + } else { + expand_non_decomposable_to_sql( + &expression_for_eval, + &base_relation_sql, + outer_ref_for_eval, + outer_where_ref, + &eval_group_by_cols, + &[], // No modifiers for explicit AGGREGATE() + &resolved.dimension_exprs, + ) + }; + if original_dim_cols.is_empty() { + format!("MAX({eval_sql})") + } else { + eval_sql + } + } else { + format!("{}({measure_lookup_name})", resolved.agg_fn) + }; + result_sql = format!( + "{}{}{}", + &result_sql[..start], + expanded, + &result_sql[replacement_end..] + ); + } + + if insert_outer_alias { + if let Some(updated_sql) = insert_primary_table_alias(&result_sql, "_outer") { + result_sql = updated_sql; + } + } + + // Check if there are still any remaining AGGREGATE calls (shouldn't be, but just in case) + if has_aggregate_function(&result_sql) { + let mut expanded = expand_aggregate(&result_sql); + append_unique_warnings(&mut expanded, warnings); + return expanded; + } + + // If no GROUP BY, add explicit GROUP BY with dimension columns from original SQL + // (GROUP BY ALL doesn't work reliably with scalar subqueries mixed with aggregates) + if !has_group_by_anywhere(&result_sql) && !original_dim_cols.is_empty() { + // Find insertion point: before ORDER BY, LIMIT, HAVING, or at end + let insert_pos = find_group_by_insert_pos(&result_sql); + + result_sql = format!( + "{} GROUP BY {}{}", + result_sql[..insert_pos].trim_end(), + original_dim_cols.join(", "), + if insert_pos < result_sql.len() { + format!(" {}", result_sql[insert_pos..].trim_start()) + } else { + String::new() + } + ); + } + + if let Some(rewritten_sql) = + std::panic::catch_unwind(|| parser_ffi::inline_order_by_subquery_aliases(&result_sql)) + .ok() + .flatten() + { + result_sql = rewritten_sql; + } + + AggregateExpandResult { + had_aggregate, + expanded_sql: result_sql, + error: None, + warnings, + } +} + +// ============================================================================= +// Catalog Functions +// ============================================================================= - if has_measure_at_refs(&sql) { - sql = rewrite_measure_at_refs(&sql); - had_aggregate = true; +pub fn store_measure_view( + view_name: &str, + measures: Vec, + base_query: &str, + base_table: Option, +) { + let view_query = extract_view_query(base_query).unwrap_or_else(|| base_query.to_string()); + let base_relation_sql = extract_base_relation_sql(&view_query); + let dimension_exprs = extract_dimension_exprs_from_query(&view_query); + let group_by_cols = extract_view_group_by_cols(&view_query); + let measure_view = MeasureView { + view_name: view_name.to_string(), + measures, + base_query: base_query.to_string(), + base_table, + base_relation_sql, + dimension_exprs, + group_by_cols, + }; + + let mut views = MEASURE_VIEWS.lock().unwrap(); + remove_measure_view_case_insensitive(&mut views, view_name); + views.insert(view_name.to_string(), measure_view); +} + +pub fn get_measure_view(view_name: &str) -> Option { + let views = MEASURE_VIEWS.lock().unwrap(); + views + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(view_name)) + .map(|(_, view)| view.clone()) +} + +pub fn restore_measure_view(view: MeasureView) { + let mut views = MEASURE_VIEWS.lock().unwrap(); + remove_measure_view_case_insensitive(&mut views, &view.view_name); + views.insert(view.view_name.clone(), view); +} + +pub fn clear_measure_views() { + let mut views = MEASURE_VIEWS.lock().unwrap(); + views.clear(); +} + +pub fn drop_measure_view(view_name: &str) -> bool { + let mut views = MEASURE_VIEWS.lock().unwrap(); + let key = views + .keys() + .find(|k| k.eq_ignore_ascii_case(view_name)) + .cloned(); + if let Some(k) = key { + views.remove(&k); + return true; } + false +} - if has_implicit_measure_refs(&sql) { - sql = rewrite_implicit_measure_refs(&sql); - had_aggregate = true; +pub fn drop_measure_view_from_sql(sql: &str) -> bool { + let name = match extract_drop_view_name(sql) { + Some(name) => name, + None => return false, + }; + drop_measure_view(&name) +} + +pub fn get_measure_aggregation(column_name: &str) -> Option<(String, String)> { + let views = MEASURE_VIEWS.lock().unwrap(); + + for (view_name, view) in views.iter() { + for measure in &view.measures { + if measure.column_name.eq_ignore_ascii_case(column_name) { + if let Some(agg_fn) = extract_aggregation_function(&measure.expression) { + return Some((agg_fn, view_name.clone())); + } + } + } } + None +} - // Check if we need the full expansion path (AT modifiers or non-decomposable measures) - let has_aggregate = has_aggregate_function(&sql); +fn extract_group_by_columns(sql: &str) -> Vec { + if let Ok(info) = parser_ffi::parse_select(sql) { + if info.has_group_by { + if info.group_by_all { + return extract_dimension_columns_from_select(sql); + } + if !info.group_by_cols.is_empty() { + let parser_group_by_cols: Vec = info + .group_by_cols + .into_iter() + .map(|c| c.trim().to_string()) + .filter(|c| !c.is_empty() && !c.chars().all(|ch| ch.is_ascii_digit())) + .collect(); - // If no AGGREGATE function at all, nothing to do - if !has_aggregate { - return AggregateExpandResult { - had_aggregate, - expanded_sql: sql, - error: None, - }; + if !parser_group_by_cols.is_empty() { + return parser_group_by_cols; + } + } + } } - had_aggregate = true; - let at_patterns = extract_aggregate_with_at_full(&sql); - // Keep full expansion path even without AT to handle non-decomposable measures safely + let mut columns = Vec::new(); - // Prefer parser-FFI FROM extraction (supports JOIN aliases when SQL parses there), - // then fall back to string extraction for AGGREGATE/AT syntax that parser-FFI cannot parse. - let mut from_info = extract_from_clause_info_ffi(&sql); - let (primary_table_name, existing_alias) = if let Some(pt) = from_info.primary_table.clone() { - let alias = if pt.has_alias { - Some(pt.effective_name.clone()) - } else { - None - }; - (pt.name, alias) + let query = sql.trim().trim_end_matches(';').trim(); + if let Some(group_by_pos) = find_top_level_keyword(query, "GROUP BY", 0) { + let start = advance_after_group_by(query, group_by_pos) + .unwrap_or_else(|| group_by_pos + "GROUP BY".len()); + let end = find_first_top_level_keyword( + query, + start, + &["ORDER BY", "LIMIT", "HAVING", "QUALIFY", "WINDOW", "UNION", "INTERSECT", "EXCEPT"], + ) + .unwrap_or(query.len()); + + let group_by_content = query[start..end].trim(); + + for part in group_by_content.split(',') { + let col = part.trim(); + if !col.is_empty() && !col.chars().all(|c| c.is_ascii_digit()) { + columns.push(col.to_string()); + } + } + } + + // If no GROUP BY, infer dimension columns from SELECT (non-AGGREGATE columns) + if columns.is_empty() { + columns = extract_dimension_columns_from_select(sql); + } + + columns +} + +/// Check if a SQL expression is a literal constant (integer, float, string, NULL, TRUE, FALSE, +/// or typed literals like DATE '...', TIMESTAMP '...', INTERVAL '...'). +/// These should not be included in GROUP BY since they are not dimension columns. +fn is_literal_constant(expr: &str) -> bool { + let trimmed = expr.trim(); + if trimmed.is_empty() { + return false; + } + + let upper = trimmed.to_uppercase(); + + // NULL, TRUE, FALSE + if upper == "NULL" || upper == "TRUE" || upper == "FALSE" { + return true; + } + + // String literal: starts and ends with single quote + if trimmed.starts_with('\'') && trimmed.ends_with('\'') && trimmed.len() >= 2 { + return true; + } + + // Typed literals: DATE '...', TIMESTAMP '...', INTERVAL '...', TIME '...' + for prefix in &["DATE ", "TIMESTAMP ", "INTERVAL ", "TIME "] { + if upper.starts_with(prefix) { + let rest = trimmed[prefix.len()..].trim(); + if rest.starts_with('\'') && rest.ends_with('\'') { + return true; + } + } + } + + // Numeric: integer or float (possibly negative) + let numeric_part = if trimmed.starts_with('-') || trimmed.starts_with('+') { + &trimmed[1..] } else { - let (table_name, alias) = - extract_table_and_alias_from_sql(&sql).unwrap_or_else(|| ("t".to_string(), None)); - let primary_table = TableInfo { - name: table_name.clone(), - effective_name: alias.clone().unwrap_or_else(|| table_name.clone()), - has_alias: alias.is_some(), - }; - from_info - .tables - .insert(primary_table.effective_name.clone(), primary_table.clone()); - from_info.primary_table = Some(primary_table); - (table_name, alias) + trimmed }; - // Extract outer WHERE clause for VISIBLE semantics - let outer_where = extract_where_clause(&sql); - let outer_where_ref = outer_where.as_deref(); + if !numeric_part.is_empty() { + let mut saw_dot = false; + let is_numeric = numeric_part.chars().all(|c| { + if c == '.' && !saw_dot { + saw_dot = true; + true + } else { + c.is_ascii_digit() + } + }); + // Must have at least one digit (not just "." or "-") + if is_numeric && numeric_part.chars().any(|c| c.is_ascii_digit()) { + return true; + } + } + + false +} + +fn remove_measure_view_case_insensitive( + views: &mut HashMap, + view_name: &str, +) -> bool { + let key = views + .keys() + .find(|name| name.eq_ignore_ascii_case(view_name)) + .cloned(); + if let Some(key) = key { + views.remove(&key); + return true; + } + false +} + +/// Extract non-AGGREGATE columns from SELECT clause to use as implicit GROUP BY columns +fn looks_like_sql_aggregate_expr(expr: &str) -> bool { + // Prefer parser-backed detection when parser FFI is available. + let parsed = std::panic::catch_unwind(|| parser_ffi::parse_expression(expr)) + .ok() + .and_then(|result| result.ok()); + if let Some(info) = parsed { + if info.is_aggregate { + return true; + } + } + + // Fallback heuristic for environments where parser FFI is unavailable. + let upper = expr.to_uppercase(); + [ + "COUNT(", + "COUNT_STAR(", + "SUM(", + "AVG(", + "MIN(", + "MAX(", + "ANY_VALUE(", + "STRING_AGG(", + "ARRAY_AGG(", + "LIST(", + "FIRST(", + "LAST(", + "MEDIAN(", + "MODE(", + "STDDEV(", + "STDDEV_POP(", + "STDDEV_SAMP(", + "VAR_POP(", + "VAR_SAMP(", + "VARIANCE(", + "QUANTILE(", + "QUANTILE_CONT(", + "QUANTILE_DISC(", + "PERCENTILE_CONT(", + "PERCENTILE_DISC(", + "BOOL_AND(", + "BOOL_OR(", + "BIT_AND(", + "BIT_OR(", + "BIT_XOR(", + ] + .iter() + .any(|pattern| upper.contains(pattern)) +} - // Extract GROUP BY columns for AT (ALL dim) correlation - let group_by_cols = extract_group_by_columns(&sql); +fn extract_dimension_columns_from_select(sql: &str) -> Vec { + let mut columns = Vec::new(); - let default_set_qualifier = if let Some(alias) = existing_alias.as_deref() { - Some(alias) - } else if primary_table_name.is_empty() { - None - } else { - Some(primary_table_name.as_str()) + let query = sql.trim().trim_end_matches(';').trim(); + let select_pos = match find_top_level_keyword(query, "SELECT", 0) { + Some(pos) => pos, + None => return columns, }; - if let Some(error) = validate_set_expression_requirements( - &at_patterns, - &group_by_cols, - default_set_qualifier, - ) { - return AggregateExpandResult { - had_aggregate: true, - expanded_sql: sql.to_string(), - error: Some(error), - }; - } - - // Extract dimension columns from original SQL for implicit GROUP BY - // (must be done before expansion since expanded SQL has SUM() etc) - let original_dim_cols = extract_dimension_columns_from_select(&sql); - let effective_group_by_cols = if group_by_cols.is_empty() { - original_dim_cols.clone() - } else { - group_by_cols.clone() + let from_pos = match find_top_level_keyword(query, "FROM", select_pos) { + Some(pos) => pos, + None => return columns, }; - let has_expression_dimensions = original_dim_cols.iter().any(|col| col.contains('(')); - // Check if query needs an explicit outer alias for correlation handling. - let needs_outer_alias = has_expression_dimensions - || at_patterns.iter().any(|(_, modifiers, _, _)| { - modifiers.iter().any(|m| { - matches!(m, ContextModifier::Set(_, _)) - || matches!(m, ContextModifier::All(_)) - || matches!(m, ContextModifier::Visible) - }) - }); + let select_start = select_pos + "SELECT".len(); + if select_start >= from_pos { + return columns; + } - let mut result_sql = sql; + let select_content = &query[select_start..from_pos]; - // Handle alias for the primary table if needed for correlation - let mut insert_outer_alias = false; - let primary_alias: Option = if needs_outer_alias { - if let Some(ref pt) = from_info.primary_table { - if pt.has_alias { - Some(pt.effective_name.clone()) - } else if insert_primary_table_alias(&result_sql, "_outer").is_some() { - insert_outer_alias = true; - Some("_outer".to_string()) - } else { - None + // Split by comma, but be careful about nested parens + let mut depth = 0; + let mut current = String::new(); + let mut items = Vec::new(); + + for c in select_content.chars() { + match c { + '(' => { + depth += 1; + current.push(c); } - } else { - None + ')' => { + depth -= 1; + current.push(c); + } + ',' if depth == 0 => { + items.push(current.trim().to_string()); + current = String::new(); + } + _ => current.push(c), } - } else { - None - }; - - let mut patterns = at_patterns; - patterns.sort_by(|a, b| b.2.cmp(&a.2)); + } + if !current.trim().is_empty() { + items.push(current.trim().to_string()); + } - for (measure_name, modifiers, start, end) in patterns { - let measure_lookup_name = strip_measure_qualifier(&measure_name); - // Look up which view contains this measure (for JOIN support) - let resolved = resolve_measure_source(&measure_lookup_name, &primary_table_name); - let mut measure_group_by_cols = filter_group_by_cols_for_measure( - &effective_group_by_cols, - &resolved.view_group_by_cols, - &resolved.dimension_exprs, - ); - let mut allowed_qualifiers: HashSet = HashSet::new(); - allowed_qualifiers.insert(normalize_identifier_name(&resolved.source_view)); - if let Some(alias) = find_alias_for_view(&from_info, &resolved.source_view) { - allowed_qualifiers.insert(normalize_identifier_name(alias)); - } - if let Some(ref pt) = from_info.primary_table { - if pt.name.eq_ignore_ascii_case(&resolved.source_view) { - if let Some(alias) = primary_alias.as_deref() { - allowed_qualifiers.insert(normalize_identifier_name(alias)); - } - } + // Filter out AGGREGATE() calls, literal constants, and extract column names + for item in items { + if has_aggregate_function(&item) { + continue; } - let source_dims = source_dimension_names(&resolved.source_view); - measure_group_by_cols.retain(|col| { - if let Some((Some(qualifier), dim_name)) = parse_simple_measure_ref(col) { - return allowed_qualifiers.contains(&qualifier) || source_dims.contains(&dim_name); - } - let dim_key = normalize_dimension_key(col); - source_dims.is_empty() - || source_dims.contains(&dim_key) - || resolved.dimension_exprs.contains_key(&dim_key) - || source_dims - .iter() - .any(|dim_name| expr_mentions_identifier(col, dim_name)) - }); - let eval_group_by_cols = - if measure_group_by_cols.is_empty() - && !original_dim_cols.is_empty() - && resolved.source_view.eq_ignore_ascii_case(&primary_table_name) - { - original_dim_cols.clone() + let item_upper = item.to_uppercase(); + // Handle "col AS alias" - use the column name, not alias + let col = if let Some(as_pos) = item_upper.find(" AS ") { + item[..as_pos].trim() } else { - measure_group_by_cols.clone() + item.trim() }; + if looks_like_sql_aggregate_expr(col) { + continue; + } + if is_star_select_expression(col) { + continue; + } + if !col.is_empty() && !is_literal_constant(col) { + columns.push(col.to_string()); + } + } - // Non-decomposable measures are recomputed from base rows (including AT modifiers) + columns +} - // Find the alias for this measure's source view in the FROM clause - // If the source view is the primary table and we added _outer, use _outer - let outer_alias = if let Some(ref pt) = from_info.primary_table { - if pt.name.eq_ignore_ascii_case(&resolved.source_view) { - // Source view is the primary table, use primary_alias (which may be _outer) - primary_alias.clone() - } else { - // Source view is not the primary table, look it up in from_info - find_alias_for_view(&from_info, &resolved.source_view) - .map(|s| s.to_string()) - .or_else(|| primary_alias.clone()) - } - } else { - primary_alias.clone() - }; - let outer_alias_ref = outer_alias.as_deref(); +// ============================================================================= +// Tests +// ============================================================================= - let outer_ref_for_eval = outer_alias_ref.or(Some(resolved.source_view.as_str())); - let base_relation_sql = resolved - .base_relation_sql - .clone() - .or_else(|| { - resolved - .base_table - .clone() - .map(|table| format!("SELECT * FROM {table}")) - }) - .unwrap_or_else(|| format!("SELECT * FROM {}", resolved.source_view)); +#[cfg(test)] +mod tests { + use super::*; + use serial_test::serial; + + #[test] + fn test_has_as_measure() { + assert!(has_as_measure( + "CREATE VIEW foo AS SELECT SUM(x) AS MEASURE revenue FROM t" + )); + assert!(!has_as_measure( + "CREATE VIEW foo AS SELECT SUM(x) AS revenue FROM t" + )); + } + + #[test] + fn test_has_aggregate_function() { + assert!(has_aggregate_function("SELECT AGGREGATE(revenue) FROM foo")); + assert!(has_aggregate_function("SELECT AGGREGATE (revenue) FROM foo")); + assert!(has_aggregate_function("SELECT \"AGGREGATE\"(revenue) FROM foo")); + assert!(has_aggregate_function("SELECT schema.AGGREGATE(revenue) FROM foo")); + assert!(has_aggregate_function( + "SELECT \"schema\".\"AGGREGATE\" (revenue) FROM foo" + )); + assert!(!has_aggregate_function("SELECT $$AGGREGATE(revenue)$$ AS note")); + assert!(!has_aggregate_function( + "SELECT $tag$AGGREGATE(revenue) AT (ALL)$tag$ AS note" + )); + assert!(!has_aggregate_function("SELECT TOTAL_AGGREGATE(revenue) FROM foo")); + assert!(!has_aggregate_function("SELECT \"TOTAL_AGGREGATE\"(revenue) FROM foo")); + assert!(!has_aggregate_function("SELECT myaggregate(revenue) FROM foo")); + assert!(!has_aggregate_function("SELECT SUM(amount) FROM foo")); + } + + #[test] + fn test_is_window_expression_variants() { + assert!(is_window_expression("SUM(amount) OVER (PARTITION BY year)")); + assert!(is_window_expression("SUM(amount)\nOVER\t(\nORDER BY year\n)")); + assert!(is_window_expression("SUM(amount) OVER running_win")); + assert!(is_window_expression("SUM(amount)/*x*/OVER/*y*/running_win")); + assert!(is_window_expression("SUM(amount)OVER(ORDER BY year)")); + + assert!(!is_window_expression("OVER(amount)")); + assert!(!is_window_expression("cover(amount)")); + assert!(!is_window_expression("'SUM(amount) OVER (ORDER BY year)'")); + } + + #[test] + fn test_extract_dimension_columns_ignores_aggregate_with_space() { + let cols = extract_dimension_columns_from_select( + "SELECT region, AGGREGATE (revenue) FROM sales_v", + ); + assert_eq!(cols, vec!["region".to_string()]); - let expression_for_eval = resolved - .derived_expr - .clone() - .unwrap_or_else(|| resolved.expression.clone()); - let is_window_measure = - is_window_expression(&expression_for_eval) || is_window_expression(&resolved.expression); + let cols = extract_dimension_columns_from_select( + "SELECT region, AGGREGATE (revenue) AT (ALL region) FROM sales_v", + ); + assert_eq!(cols, vec!["region".to_string()]); - let expanded = if is_window_measure { - let scalar_eval_sql = expand_non_decomposable_to_sql( - &expression_for_eval, - &base_relation_sql, - outer_ref_for_eval, - outer_where_ref, - &eval_group_by_cols, - &modifiers, - &resolved.dimension_exprs, - ); - let row_eval_sql = match scalar_subquery_to_rows_sql(&scalar_eval_sql, "__window_value") - { - Some(sql) => sql, - None => { - return AggregateExpandResult { - had_aggregate: true, - expanded_sql: result_sql, - error: Some(format!( - "Failed to rewrite window measure {measure_lookup_name} with AT modifiers" - )), - }; - } - }; - let eval_sql = wrap_window_rows_as_single_value(&row_eval_sql, &measure_lookup_name); - if original_dim_cols.is_empty() { - format!("MAX({eval_sql})") - } else { - eval_sql - } - } else if !expression_for_eval.is_empty() { - let eval_sql = expand_non_decomposable_to_sql( - &expression_for_eval, - &base_relation_sql, - outer_ref_for_eval, - outer_where_ref, - &eval_group_by_cols, - &modifiers, - &resolved.dimension_exprs, - ); - if original_dim_cols.is_empty() { - format!("MAX({eval_sql})") - } else { - eval_sql - } - } else { - expand_modifiers_to_sql( - &measure_lookup_name, - &resolved.agg_fn, - &modifiers, - &resolved.source_view, - outer_alias_ref, - outer_where_ref, - &eval_group_by_cols, - ) - }; - result_sql = format!("{}{}{}", &result_sql[..start], expanded, &result_sql[end..]); + let cols = extract_dimension_columns_from_select( + "SELECT AGGREGATE (revenue) FROM sales_v", + ); + assert!(cols.is_empty()); } - // Also expand plain AGGREGATE() calls (without AT modifiers) using text replacement - let mut plain_calls = extract_all_aggregate_calls(&result_sql); - plain_calls.sort_by(|a, b| b.1.cmp(&a.1)); // Sort by position descending + #[test] + fn test_extract_dimension_columns_excludes_standard_aggregates() { + let cols = extract_dimension_columns_from_select( + "SELECT region, COUNT(*) AS c, SUM(amount) AS s, ANY_VALUE(flag) AS f FROM sales GROUP BY ROLLUP(region)", + ); + assert_eq!(cols, vec!["region".to_string()]); + } - for (measure_name, start, end) in plain_calls { - let mut replacement_end = end; - let mut use_default_context = false; - let suffix = &result_sql[end..]; - let leading_ws = suffix.len() - suffix.trim_start().len(); - if suffix[leading_ws..].starts_with(DEFAULT_CONTEXT_MARKER) { - use_default_context = true; - replacement_end = end + leading_ws + DEFAULT_CONTEXT_MARKER.len(); - } + #[test] + fn test_extract_dimension_columns_excludes_star() { + let cols = extract_dimension_columns_from_select( + "SELECT * FROM (SELECT AGGREGATE(revenue) AS revenue FROM sales_v) s", + ); + assert!(cols.is_empty()); - let measure_lookup_name = strip_measure_qualifier(&measure_name); - let resolved = resolve_measure_source(&measure_lookup_name, &primary_table_name); - let mut measure_group_by_cols = filter_group_by_cols_for_measure( - &effective_group_by_cols, - &resolved.view_group_by_cols, - &resolved.dimension_exprs, + let cols = extract_dimension_columns_from_select( + "SELECT s.* FROM (SELECT AGGREGATE(revenue) AS revenue FROM sales_v) s", ); - let mut allowed_qualifiers: HashSet = HashSet::new(); - allowed_qualifiers.insert(normalize_identifier_name(&resolved.source_view)); - if let Some(alias) = find_alias_for_view(&from_info, &resolved.source_view) { - allowed_qualifiers.insert(normalize_identifier_name(alias)); - } - if let Some(ref pt) = from_info.primary_table { - if pt.name.eq_ignore_ascii_case(&resolved.source_view) { - if let Some(alias) = primary_alias.as_deref() { - allowed_qualifiers.insert(normalize_identifier_name(alias)); - } - } - } - let source_dims = source_dimension_names(&resolved.source_view); - measure_group_by_cols.retain(|col| { - if let Some((Some(qualifier), dim_name)) = parse_simple_measure_ref(col) { - return allowed_qualifiers.contains(&qualifier) || source_dims.contains(&dim_name); - } - let dim_key = normalize_dimension_key(col); - source_dims.is_empty() - || source_dims.contains(&dim_key) - || resolved.dimension_exprs.contains_key(&dim_key) - || source_dims - .iter() - .any(|dim_name| expr_mentions_identifier(col, dim_name)) - }); - let eval_group_by_cols = - if measure_group_by_cols.is_empty() - && !original_dim_cols.is_empty() - && resolved.source_view.eq_ignore_ascii_case(&primary_table_name) - { - original_dim_cols.clone() - } else { - measure_group_by_cols.clone() - }; + assert!(cols.is_empty()); + } + #[test] + fn test_extract_dimension_columns_keeps_non_aggregate_suffix() { + let cols = extract_dimension_columns_from_select( + "SELECT region, TOTAL_AGGREGATE(revenue) FROM sales_v", + ); + assert_eq!( + cols, + vec!["region".to_string(), "TOTAL_AGGREGATE(revenue)".to_string()] + ); + } - let outer_alias = if let Some(ref pt) = from_info.primary_table { - if pt.name.eq_ignore_ascii_case(&resolved.source_view) { - primary_alias.clone().or_else(|| { - find_alias_for_view(&from_info, &resolved.source_view).map(|s| s.to_string()) - }) - } else { - find_alias_for_view(&from_info, &resolved.source_view) - .map(|s| s.to_string()) - .or_else(|| primary_alias.clone()) - } - } else { - primary_alias.clone() - }; - let outer_ref_for_eval = outer_alias.as_deref().or(Some(resolved.source_view.as_str())); - let base_relation_sql = resolved - .base_relation_sql - .clone() - .or_else(|| { - resolved - .base_table - .clone() - .map(|table| format!("SELECT * FROM {table}")) - }) - .unwrap_or_else(|| format!("SELECT * FROM {}", resolved.source_view)); - let expression_for_eval = resolved - .derived_expr - .clone() - .unwrap_or_else(|| resolved.expression.clone()); - let is_window_measure = - is_window_expression(&expression_for_eval) || is_window_expression(&resolved.expression); + #[test] + fn test_extract_dimension_columns_ignores_quoted_and_qualified_aggregate() { + let cols = extract_dimension_columns_from_select( + "SELECT region, \"AGGREGATE\"(revenue) FROM sales_v", + ); + assert_eq!(cols, vec!["region".to_string()]); - let expanded = if is_window_measure { - let _ = use_default_context; - let _ = base_relation_sql; - let _ = outer_ref_for_eval; - let _ = outer_where_ref; - let _ = eval_group_by_cols; - format!("{}({measure_lookup_name})", resolved.agg_fn) - } else if !expression_for_eval.is_empty() { - let eval_sql = if use_default_context { - expand_non_decomposable_default_context( - &expression_for_eval, - &base_relation_sql, - outer_ref_for_eval, - &eval_group_by_cols, - &resolved.dimension_exprs, - ) - } else { - expand_non_decomposable_to_sql( - &expression_for_eval, - &base_relation_sql, - outer_ref_for_eval, - outer_where_ref, - &eval_group_by_cols, - &[], // No modifiers for explicit AGGREGATE() - &resolved.dimension_exprs, - ) - }; - if original_dim_cols.is_empty() { - format!("MAX({eval_sql})") - } else { - eval_sql - } - } else { - format!("{}({measure_lookup_name})", resolved.agg_fn) - }; - result_sql = format!( - "{}{}{}", - &result_sql[..start], - expanded, - &result_sql[replacement_end..] + let cols = extract_dimension_columns_from_select( + "SELECT region, schema.AGGREGATE(revenue) FROM sales_v", ); + assert_eq!(cols, vec!["region".to_string()]); } - if insert_outer_alias { - if let Some(updated_sql) = insert_primary_table_alias(&result_sql, "_outer") { - result_sql = updated_sql; - } - } + #[test] + fn test_is_literal_constant() { + // Integers + assert!(is_literal_constant("1000")); + assert!(is_literal_constant("0")); + assert!(is_literal_constant("-1")); + assert!(is_literal_constant("+42")); + + // Floats + assert!(is_literal_constant("3.14")); + assert!(is_literal_constant("-0.5")); + assert!(is_literal_constant("100.0")); + + // String literals + assert!(is_literal_constant("'hello'")); + assert!(is_literal_constant("'it''s a test'")); + + // NULL, TRUE, FALSE (case insensitive) + assert!(is_literal_constant("NULL")); + assert!(is_literal_constant("null")); + assert!(is_literal_constant("TRUE")); + assert!(is_literal_constant("true")); + assert!(is_literal_constant("FALSE")); + assert!(is_literal_constant("false")); + + // Date/time literals + assert!(is_literal_constant("DATE '2023-01-01'")); + assert!(is_literal_constant("TIMESTAMP '2023-01-01 00:00:00'")); + assert!(is_literal_constant("INTERVAL '1 day'")); + assert!(is_literal_constant("TIME '12:00:00'")); - // Check if there are still any remaining AGGREGATE calls (shouldn't be, but just in case) - if has_aggregate_function(&result_sql) { - return expand_aggregate(&result_sql); + // Not constants + assert!(!is_literal_constant("year")); + assert!(!is_literal_constant("region")); + assert!(!is_literal_constant("UPPER(region)")); + assert!(!is_literal_constant("year + 1")); + assert!(!is_literal_constant("")); + assert!(!is_literal_constant("SUM(amount)")); } - // If no GROUP BY, add explicit GROUP BY with dimension columns from original SQL - // (GROUP BY ALL doesn't work reliably with scalar subqueries mixed with aggregates) - if !has_group_by_anywhere(&result_sql) && !original_dim_cols.is_empty() { - // Find insertion point: before ORDER BY, LIMIT, HAVING, or at end - let insert_pos = find_group_by_insert_pos(&result_sql); - - result_sql = format!( - "{} GROUP BY {}{}", - result_sql[..insert_pos].trim_end(), - original_dim_cols.join(", "), - if insert_pos < result_sql.len() { - format!(" {}", result_sql[insert_pos..].trim_start()) - } else { - String::new() - } + #[test] + fn test_extract_dimension_columns_skips_literal_constants() { + // Integer constant only + let cols = extract_dimension_columns_from_select( + "SELECT 1000, AGGREGATE(revenue) FROM sales_v", ); - } - - if let Some(rewritten_sql) = - std::panic::catch_unwind(|| parser_ffi::inline_order_by_subquery_aliases(&result_sql)) - .ok() - .flatten() - { - result_sql = rewritten_sql; - } + assert!(cols.is_empty()); - AggregateExpandResult { - had_aggregate, - expanded_sql: result_sql, - error: None, - } -} + // String literal only + let cols = extract_dimension_columns_from_select( + "SELECT 'hello', AGGREGATE(revenue) FROM sales_v", + ); + assert!(cols.is_empty()); -// ============================================================================= -// Catalog Functions -// ============================================================================= + // Mixed: dimension + constant + let cols = extract_dimension_columns_from_select( + "SELECT year, 1000, AGGREGATE(revenue) FROM sales_v", + ); + assert_eq!(cols, vec!["year".to_string()]); -pub fn store_measure_view( - view_name: &str, - measures: Vec, - base_query: &str, - base_table: Option, -) { - let view_query = extract_view_query(base_query).unwrap_or_else(|| base_query.to_string()); - let base_relation_sql = extract_base_relation_sql(&view_query); - let dimension_exprs = extract_dimension_exprs_from_query(&view_query); - let group_by_cols = extract_view_group_by_cols(&view_query); - let measure_view = MeasureView { - view_name: view_name.to_string(), - measures, - base_query: base_query.to_string(), - base_table, - base_relation_sql, - dimension_exprs, - group_by_cols, - }; + // NULL constant + let cols = extract_dimension_columns_from_select( + "SELECT NULL, AGGREGATE(revenue) FROM sales_v", + ); + assert!(cols.is_empty()); - let mut views = MEASURE_VIEWS.lock().unwrap(); - remove_measure_view_case_insensitive(&mut views, view_name); - views.insert(view_name.to_string(), measure_view); -} + // Float constant + let cols = extract_dimension_columns_from_select( + "SELECT 3.14, AGGREGATE(revenue) FROM sales_v", + ); + assert!(cols.is_empty()); -pub fn get_measure_view(view_name: &str) -> Option { - let views = MEASURE_VIEWS.lock().unwrap(); - views - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(view_name)) - .map(|(_, view)| view.clone()) -} + // Negative integer + let cols = extract_dimension_columns_from_select( + "SELECT -1, AGGREGATE(revenue) FROM sales_v", + ); + assert!(cols.is_empty()); + } -pub fn restore_measure_view(view: MeasureView) { - let mut views = MEASURE_VIEWS.lock().unwrap(); - remove_measure_view_case_insensitive(&mut views, &view.view_name); - views.insert(view.view_name.clone(), view); -} + #[test] + #[serial] + fn test_process_create_view_basic() { + clear_measure_views(); -pub fn clear_measure_views() { - let mut views = MEASURE_VIEWS.lock().unwrap(); - views.clear(); -} + let sql = + "CREATE VIEW orders_summary AS SELECT status, SUM(amount) AS MEASURE revenue FROM orders"; + let result = process_create_view(sql); -pub fn drop_measure_view(view_name: &str) -> bool { - let mut views = MEASURE_VIEWS.lock().unwrap(); - let key = views - .keys() - .find(|k| k.eq_ignore_ascii_case(view_name)) - .cloned(); - if let Some(k) = key { - views.remove(&k); - return true; + assert!(result.is_measure_view); + assert_eq!(result.view_name, Some("orders_summary".to_string())); + assert_eq!(result.measures.len(), 1); + assert_eq!(result.measures[0].column_name, "revenue"); + assert_eq!(result.measures[0].expression, "SUM(amount)"); + assert!(result.clean_sql.contains("AS revenue")); + assert!(!result.clean_sql.contains("AS MEASURE")); } - false -} -pub fn drop_measure_view_from_sql(sql: &str) -> bool { - let name = match extract_drop_view_name(sql) { - Some(name) => name, - None => return false, - }; - drop_measure_view(&name) -} + #[test] + #[serial] + fn test_process_create_view_sanitizes_non_ascii_comments() { + clear_measure_views(); -pub fn get_measure_aggregation(column_name: &str) -> Option<(String, String)> { - let views = MEASURE_VIEWS.lock().unwrap(); + let sql = r#"CREATE VIEW orders_comment AS +SELECT status, -- café +SUM(amount) AS MEASURE revenue /* 東京 */ +FROM orders"#; + let result = std::panic::catch_unwind(|| process_create_view(sql)) + .expect("non-ASCII comments must not panic"); - for (view_name, view) in views.iter() { - for measure in &view.measures { - if measure.column_name.eq_ignore_ascii_case(column_name) { - if let Some(agg_fn) = extract_aggregation_function(&measure.expression) { - return Some((agg_fn, view_name.clone())); - } - } - } + assert!(result.is_measure_view); + assert!(result.error.is_none()); + assert_eq!(result.measures.len(), 1); + assert_eq!(result.measures[0].column_name, "revenue"); + assert!(result.clean_sql.is_ascii()); + assert!(result.clean_sql.contains("-- caf")); + assert!(result.clean_sql.contains("/*")); + assert!(!result.clean_sql.contains("café")); + assert!(!result.clean_sql.contains("東京")); } - None -} -fn extract_group_by_columns(sql: &str) -> Vec { - if let Ok(info) = parser_ffi::parse_select(sql) { - if info.has_group_by { - if info.group_by_all { - return extract_dimension_columns_from_select(sql); - } - if !info.group_by_cols.is_empty() { - let parser_group_by_cols: Vec = info - .group_by_cols - .into_iter() - .map(|c| c.trim().to_string()) - .filter(|c| !c.is_empty() && !c.chars().all(|ch| ch.is_ascii_digit())) - .collect(); + #[test] + fn test_comment_sanitizer_preserves_non_comment_unicode() { + let sql = "SELECT 'café -- literal', \"東京\" /* résumé */ FROM t -- 東京"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); - if !parser_group_by_cols.is_empty() { - return parser_group_by_cols; - } - } - } + assert!(sanitized.contains("'café -- literal'")); + assert!(sanitized.contains("\"東京\"")); + assert!(!sanitized.contains("résumé")); + assert!(!sanitized.ends_with("東京")); + assert_eq!(sanitized.len(), sql.len()); } - let mut columns = Vec::new(); + #[test] + fn test_comment_sanitizer_preserves_dollar_quoted_strings() { + let sql = "SELECT $$-- café$$ AS label, $tag$/* 東京 */$tag$ AS note FROM t -- résumé"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); - let query = sql.trim().trim_end_matches(';').trim(); - if let Some(group_by_pos) = find_top_level_keyword(query, "GROUP BY", 0) { - let start = advance_after_group_by(query, group_by_pos) - .unwrap_or_else(|| group_by_pos + "GROUP BY".len()); - let end = find_first_top_level_keyword( - query, - start, - &["ORDER BY", "LIMIT", "HAVING", "QUALIFY", "WINDOW", "UNION", "INTERSECT", "EXCEPT"], - ) - .unwrap_or(query.len()); + assert!(sanitized.contains("$$-- café$$")); + assert!(sanitized.contains("$tag$/* 東京 */$tag$")); + assert!(!sanitized.ends_with("résumé")); + assert_eq!(sanitized.len(), sql.len()); + } - let group_by_content = query[start..end].trim(); + #[test] + fn test_comment_sanitizer_ignores_dollar_tags_inside_regular_strings() { + let sql = "SELECT '$tag$' AS label FROM t -- $tag$ café"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); - for part in group_by_content.split(',') { - let col = part.trim(); - if !col.is_empty() && !col.chars().all(|c| c.is_ascii_digit()) { - columns.push(col.to_string()); - } - } + assert!(sanitized.contains("'$tag$'")); + assert!(sanitized.contains("-- $tag$ caf")); + assert!(!sanitized.ends_with("café")); + assert!(sanitized.is_ascii()); + assert_eq!(sanitized.len(), sql.len()); } - // If no GROUP BY, infer dimension columns from SELECT (non-AGGREGATE columns) - if columns.is_empty() { - columns = extract_dimension_columns_from_select(sql); + #[test] + fn test_comment_sanitizer_handles_comments_inside_list_brackets() { + let sql = "SELECT [1, /* 東京 */ 2] AS xs FROM t -- résumé"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); + + assert!(sanitized.contains("[1, /*")); + assert!(!sanitized.contains("東京")); + assert!(!sanitized.ends_with("résumé")); + assert!(sanitized.is_ascii()); + assert_eq!(sanitized.len(), sql.len()); } - columns -} + #[test] + fn test_comment_sanitizer_handles_quotes_inside_list_brackets() { + let sql = "SELECT [']'] AS xs FROM t -- 東京"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); -/// Check if a SQL expression is a literal constant (integer, float, string, NULL, TRUE, FALSE, -/// or typed literals like DATE '...', TIMESTAMP '...', INTERVAL '...'). -/// These should not be included in GROUP BY since they are not dimension columns. -fn is_literal_constant(expr: &str) -> bool { - let trimmed = expr.trim(); - if trimmed.is_empty() { - return false; + assert!(sanitized.contains("[']']")); + assert!(!sanitized.ends_with("東京")); + assert!(sanitized.is_ascii()); + assert_eq!(sanitized.len(), sql.len()); } - let upper = trimmed.to_uppercase(); + #[test] + fn test_comment_sanitizer_handles_escape_string_quotes() { + let sql = "SELECT E'can\\'t' AS label FROM sales -- 東京"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); - // NULL, TRUE, FALSE - if upper == "NULL" || upper == "TRUE" || upper == "FALSE" { - return true; + assert!(sanitized.contains("E'can\\'t'")); + assert!(!sanitized.ends_with("東京")); + assert!(sanitized.is_ascii()); + assert_eq!(sanitized.len(), sql.len()); } - // String literal: starts and ends with single quote - if trimmed.starts_with('\'') && trimmed.ends_with('\'') && trimmed.len() >= 2 { - return true; - } + #[test] + fn test_comment_sanitizer_handles_nested_block_comments() { + let sql = "SELECT 1 /* outer /* inner */ 東京 */ FROM t -- résumé"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); - // Typed literals: DATE '...', TIMESTAMP '...', INTERVAL '...', TIME '...' - for prefix in &["DATE ", "TIMESTAMP ", "INTERVAL ", "TIME "] { - if upper.starts_with(prefix) { - let rest = trimmed[prefix.len()..].trim(); - if rest.starts_with('\'') && rest.ends_with('\'') { - return true; - } - } + assert!(sanitized.contains("/* outer /* inner */")); + assert!(!sanitized.contains("東京")); + assert!(!sanitized.ends_with("résumé")); + assert!(sanitized.is_ascii()); + assert_eq!(sanitized.len(), sql.len()); } - // Numeric: integer or float (possibly negative) - let numeric_part = if trimmed.starts_with('-') || trimmed.starts_with('+') { - &trimmed[1..] - } else { - trimmed - }; + #[test] + fn test_comment_sanitizer_preserves_bracket_quoted_identifiers() { + let sql = "SELECT [東京] FROM t -- résumé"; + let sanitized = sanitize_non_ascii_in_sql_comments(sql); - if !numeric_part.is_empty() { - let mut saw_dot = false; - let is_numeric = numeric_part.chars().all(|c| { - if c == '.' && !saw_dot { - saw_dot = true; - true - } else { - c.is_ascii_digit() - } - }); - // Must have at least one digit (not just "." or "-") - if is_numeric && numeric_part.chars().any(|c| c.is_ascii_digit()) { - return true; - } + assert!(sanitized.contains("[東京]")); + assert!(!sanitized.ends_with("résumé")); + assert_eq!(sanitized.len(), sql.len()); } - false -} + #[test] + #[serial] + fn test_process_create_view_case_expression() { + clear_measure_views(); -fn remove_measure_view_case_insensitive( - views: &mut HashMap, - view_name: &str, -) -> bool { - let key = views - .keys() - .find(|name| name.eq_ignore_ascii_case(view_name)) - .cloned(); - if let Some(key) = key { - views.remove(&key); - return true; - } - false -} + let sql = "CREATE VIEW v AS SELECT year, CASE WHEN SUM(x) > 100 THEN 1 ELSE 0 END AS MEASURE flag FROM t GROUP BY year"; + let result = process_create_view(sql); -/// Extract non-AGGREGATE columns from SELECT clause to use as implicit GROUP BY columns -fn looks_like_sql_aggregate_expr(expr: &str) -> bool { - // Prefer parser-backed detection when parser FFI is available. - let parsed = std::panic::catch_unwind(|| parser_ffi::parse_expression(expr)) - .ok() - .and_then(|result| result.ok()); - if let Some(info) = parsed { - if info.is_aggregate { - return true; - } + eprintln!("Result: {result:?}"); + eprintln!("Measures: {:?}", result.measures); + assert!(result.is_measure_view); + assert_eq!(result.measures.len(), 1); + assert_eq!(result.measures[0].column_name, "flag"); + assert_eq!( + result.measures[0].expression, + "CASE WHEN SUM(x) > 100 THEN 1 ELSE 0 END" + ); + + // Now test that AGGREGATE(flag) works + let query_sql = "SELECT year, AGGREGATE(flag) FROM v GROUP BY year"; + let expand_result = std::panic::catch_unwind(|| expand_aggregate(query_sql)) + .unwrap_or_else(|_| expand_aggregate_with_at(query_sql)); + eprintln!("Expand result: {expand_result:?}"); + // This should have expanded AGGREGATE(flag) to something + assert!(expand_result.had_aggregate); } - // Fallback heuristic for environments where parser FFI is unavailable. - let upper = expr.to_uppercase(); - [ - "COUNT(", - "COUNT_STAR(", - "SUM(", - "AVG(", - "MIN(", - "MAX(", - "ANY_VALUE(", - "STRING_AGG(", - "ARRAY_AGG(", - "LIST(", - "FIRST(", - "LAST(", - "MEDIAN(", - "MODE(", - "STDDEV(", - "STDDEV_POP(", - "STDDEV_SAMP(", - "VAR_POP(", - "VAR_SAMP(", - "VARIANCE(", - "QUANTILE(", - "QUANTILE_CONT(", - "QUANTILE_DISC(", - "PERCENTILE_CONT(", - "PERCENTILE_DISC(", - "BOOL_AND(", - "BOOL_OR(", - "BIT_AND(", - "BIT_OR(", - "BIT_XOR(", - ] - .iter() - .any(|pattern| upper.contains(pattern)) -} + #[test] + #[serial] + fn test_process_create_view_derived_measure() { + clear_measure_views(); -fn extract_dimension_columns_from_select(sql: &str) -> Vec { - let mut columns = Vec::new(); + // Create view with base measures and a derived measure + let sql = "CREATE VIEW financials_v AS SELECT year, SUM(revenue) AS MEASURE revenue, SUM(cost) AS MEASURE cost, revenue - cost AS MEASURE profit FROM financials GROUP BY year"; + let result = process_create_view(sql); - let query = sql.trim().trim_end_matches(';').trim(); - let select_pos = match find_top_level_keyword(query, "SELECT", 0) { - Some(pos) => pos, - None => return columns, - }; - let from_pos = match find_top_level_keyword(query, "FROM", select_pos) { - Some(pos) => pos, - None => return columns, - }; + eprintln!("Clean SQL: {}", result.clean_sql); + eprintln!("Measures: {:?}", result.measures); - let select_start = select_pos + "SELECT".len(); - if select_start >= from_pos { - return columns; - } + assert!(result.is_measure_view); + assert_eq!(result.measures.len(), 3); - let select_content = &query[select_start..from_pos]; + // Check base measures + assert_eq!(result.measures[0].column_name, "revenue"); + assert_eq!(result.measures[0].expression, "SUM(revenue)"); + assert_eq!(result.measures[1].column_name, "cost"); + assert_eq!(result.measures[1].expression, "SUM(cost)"); - // Split by comma, but be careful about nested parens - let mut depth = 0; - let mut current = String::new(); - let mut items = Vec::new(); + // Check derived measure + assert_eq!(result.measures[2].column_name, "profit"); + assert_eq!(result.measures[2].expression, "revenue - cost"); - for c in select_content.chars() { - match c { - '(' => { - depth += 1; - current.push(c); - } - ')' => { - depth -= 1; - current.push(c); - } - ',' if depth == 0 => { - items.push(current.trim().to_string()); - current = String::new(); - } - _ => current.push(c), - } - } - if !current.trim().is_empty() { - items.push(current.trim().to_string()); + // Clean SQL should NOT contain the derived measure column + assert!(result.clean_sql.contains("AS revenue")); + assert!(result.clean_sql.contains("AS cost")); + assert!(!result.clean_sql.contains("AS profit")); + assert!(!result.clean_sql.contains("revenue - cost")); } - // Filter out AGGREGATE() calls, literal constants, and extract column names - for item in items { - if has_aggregate_function(&item) { - continue; - } - let item_upper = item.to_uppercase(); - // Handle "col AS alias" - use the column name, not alias - let col = if let Some(as_pos) = item_upper.find(" AS ") { - item[..as_pos].trim() - } else { - item.trim() - }; - if looks_like_sql_aggregate_expr(col) { - continue; - } - if !col.is_empty() && !is_literal_constant(col) { - columns.push(col.to_string()); - } + #[test] + fn test_extract_measures_named_window_measure() { + let sql = r#"CREATE VIEW sales_window_v AS +SELECT + year, + SUM(amount) + OVER win AS MEASURE running_total +FROM sales +WINDOW win AS (ORDER BY year)"#; + let (clean_sql, measures, view_name, _) = extract_measures_from_sql(sql).unwrap(); + + assert_eq!(view_name, Some("sales_window_v".to_string())); + assert_eq!(measures.len(), 1); + assert_eq!(measures[0].column_name, "running_total"); + assert!(!measures[0].is_decomposable); + assert!(!clean_sql.contains("NULL AS running_total")); + assert!(clean_sql.contains("AS running_total")); } - columns -} + #[test] + #[serial] + fn test_process_create_view_without_group_by() { + clear_measure_views(); -// ============================================================================= -// Tests -// ============================================================================= + // Per the paper, views can define measures without GROUP BY + let sql = r#"CREATE VIEW orders_extended AS +SELECT + id, + product, + region, + amount, + SUM(amount) AS MEASURE revenue +FROM orders"#; + let result = process_create_view(sql); -#[cfg(test)] -mod tests { - use super::*; - use serial_test::serial; + eprintln!("is_measure_view: {}", result.is_measure_view); + eprintln!("clean_sql: {}", result.clean_sql); + eprintln!("measures: {:?}", result.measures); - #[test] - fn test_has_as_measure() { - assert!(has_as_measure( - "CREATE VIEW foo AS SELECT SUM(x) AS MEASURE revenue FROM t" - )); - assert!(!has_as_measure( - "CREATE VIEW foo AS SELECT SUM(x) AS revenue FROM t" - )); + assert!(result.is_measure_view); + assert_eq!(result.measures.len(), 1); + assert_eq!(result.measures[0].column_name, "revenue"); + // The clean_sql should NOT contain the measure column at all for no-groupby views + // because it can't be evaluated without a GROUP BY } #[test] - fn test_has_aggregate_function() { - assert!(has_aggregate_function("SELECT AGGREGATE(revenue) FROM foo")); - assert!(has_aggregate_function("SELECT AGGREGATE (revenue) FROM foo")); - assert!(has_aggregate_function("SELECT \"AGGREGATE\"(revenue) FROM foo")); - assert!(has_aggregate_function("SELECT schema.AGGREGATE(revenue) FROM foo")); - assert!(has_aggregate_function( - "SELECT \"schema\".\"AGGREGATE\" (revenue) FROM foo" - )); - assert!(!has_aggregate_function("SELECT $$AGGREGATE(revenue)$$ AS note")); - assert!(!has_aggregate_function( - "SELECT $tag$AGGREGATE(revenue) AT (ALL)$tag$ AS note" + fn test_has_at_syntax() { + assert!(has_at_syntax( + "SELECT AGGREGATE(revenue) AT (ALL status) FROM orders" )); - assert!(!has_aggregate_function("SELECT TOTAL_AGGREGATE(revenue) FROM foo")); - assert!(!has_aggregate_function("SELECT \"TOTAL_AGGREGATE\"(revenue) FROM foo")); - assert!(!has_aggregate_function("SELECT myaggregate(revenue) FROM foo")); - assert!(!has_aggregate_function("SELECT SUM(amount) FROM foo")); + assert!(!has_at_syntax("SELECT AGGREGATE(revenue) FROM orders")); } #[test] - fn test_is_window_expression_variants() { - assert!(is_window_expression("SUM(amount) OVER (PARTITION BY year)")); - assert!(is_window_expression("SUM(amount)\nOVER\t(\nORDER BY year\n)")); - assert!(is_window_expression("SUM(amount) OVER running_win")); - assert!(is_window_expression("SUM(amount)/*x*/OVER/*y*/running_win")); - assert!(is_window_expression("SUM(amount)OVER(ORDER BY year)")); - - assert!(!is_window_expression("OVER(amount)")); - assert!(!is_window_expression("cover(amount)")); - assert!(!is_window_expression("'SUM(amount) OVER (ORDER BY year)'")); + fn test_parse_at_modifier_all() { + let result = parse_at_modifier("ALL status").unwrap(); + assert_eq!(result, ContextModifier::All("status".to_string())); } #[test] - fn test_extract_dimension_columns_ignores_aggregate_with_space() { - let cols = extract_dimension_columns_from_select( - "SELECT region, AGGREGATE (revenue) FROM sales_v", - ); - assert_eq!(cols, vec!["region".to_string()]); - - let cols = extract_dimension_columns_from_select( - "SELECT region, AGGREGATE (revenue) AT (ALL region) FROM sales_v", - ); - assert_eq!(cols, vec!["region".to_string()]); - - let cols = extract_dimension_columns_from_select( - "SELECT AGGREGATE (revenue) FROM sales_v", + fn test_parse_at_modifier_set() { + let result = parse_at_modifier("SET month = month - 1").unwrap(); + assert_eq!( + result, + ContextModifier::Set("month".to_string(), "month - 1".to_string()) ); - assert!(cols.is_empty()); } #[test] - fn test_extract_dimension_columns_excludes_standard_aggregates() { - let cols = extract_dimension_columns_from_select( - "SELECT region, COUNT(*) AS c, SUM(amount) AS s, ANY_VALUE(flag) AS f FROM sales GROUP BY ROLLUP(region)", - ); - assert_eq!(cols, vec!["region".to_string()]); + fn test_parse_at_modifier_where() { + let result = parse_at_modifier("WHERE region = 'US'").unwrap(); + assert_eq!(result, ContextModifier::Where("region = 'US'".to_string())); } #[test] - fn test_extract_dimension_columns_keeps_non_aggregate_suffix() { - let cols = extract_dimension_columns_from_select( - "SELECT region, TOTAL_AGGREGATE(revenue) FROM sales_v", - ); - assert_eq!( - cols, - vec!["region".to_string(), "TOTAL_AGGREGATE(revenue)".to_string()] - ); + fn test_parse_at_modifier_where_strips_qualifier() { + let result = parse_at_modifier("WHERE sales.region = 'US'").unwrap(); + assert_eq!(result, ContextModifier::Where("region = 'US'".to_string())); } #[test] - fn test_extract_dimension_columns_ignores_quoted_and_qualified_aggregate() { - let cols = extract_dimension_columns_from_select( - "SELECT region, \"AGGREGATE\"(revenue) FROM sales_v", - ); - assert_eq!(cols, vec!["region".to_string()]); + fn test_extract_aggregate_with_at() { + let sql = "SELECT status, AGGREGATE(revenue) AT (ALL status) FROM orders"; + let results = extract_aggregate_with_at(sql); - let cols = extract_dimension_columns_from_select( - "SELECT region, schema.AGGREGATE(revenue) FROM sales_v", - ); - assert_eq!(cols, vec!["region".to_string()]); + assert_eq!(results.len(), 1); + assert_eq!(results[0].0, "revenue"); + assert_eq!(results[0].1, ContextModifier::All("status".to_string())); } #[test] - fn test_is_literal_constant() { - // Integers - assert!(is_literal_constant("1000")); - assert!(is_literal_constant("0")); - assert!(is_literal_constant("-1")); - assert!(is_literal_constant("+42")); - - // Floats - assert!(is_literal_constant("3.14")); - assert!(is_literal_constant("-0.5")); - assert!(is_literal_constant("100.0")); - - // String literals - assert!(is_literal_constant("'hello'")); - assert!(is_literal_constant("'it''s a test'")); - - // NULL, TRUE, FALSE (case insensitive) - assert!(is_literal_constant("NULL")); - assert!(is_literal_constant("null")); - assert!(is_literal_constant("TRUE")); - assert!(is_literal_constant("true")); - assert!(is_literal_constant("FALSE")); - assert!(is_literal_constant("false")); - - // Date/time literals - assert!(is_literal_constant("DATE '2023-01-01'")); - assert!(is_literal_constant("TIMESTAMP '2023-01-01 00:00:00'")); - assert!(is_literal_constant("INTERVAL '1 day'")); - assert!(is_literal_constant("TIME '12:00:00'")); + fn test_extract_aggregate_skips_dollar_quoted_literals() { + assert!(extract_all_aggregate_calls( + "SELECT $$AGGREGATE(revenue)$$ AS note" + ) + .is_empty()); + assert!(extract_aggregate_with_at_full( + "SELECT $tag$AGGREGATE(revenue) AT (ALL)$tag$ AS note" + ) + .is_empty()); - // Not constants - assert!(!is_literal_constant("year")); - assert!(!is_literal_constant("region")); - assert!(!is_literal_constant("UPPER(region)")); - assert!(!is_literal_constant("year + 1")); - assert!(!is_literal_constant("")); - assert!(!is_literal_constant("SUM(amount)")); + let calls = extract_all_aggregate_calls( + "SELECT AGGREGATE(revenue), $$AGGREGATE(cost)$$ FROM sales_v", + ); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, "revenue"); } #[test] - fn test_extract_dimension_columns_skips_literal_constants() { - // Integer constant only - let cols = extract_dimension_columns_from_select( - "SELECT 1000, AGGREGATE(revenue) FROM sales_v", + fn test_expand_at_to_sql_all() { + // With no group_by_cols, AT (ALL dim) acts like grand total + let expanded = expand_at_to_sql( + "revenue", + "SUM", + &ContextModifier::All("status".to_string()), + "orders", + None, + None, + &[], ); - assert!(cols.is_empty()); + assert_eq!(expanded, "(SELECT SUM(revenue) FROM orders)"); - // String literal only - let cols = extract_dimension_columns_from_select( - "SELECT 'hello', AGGREGATE(revenue) FROM sales_v", + // With group_by_cols, AT (ALL dim) generates correlated subquery + let expanded2 = expand_at_to_sql( + "revenue", + "SUM", + &ContextModifier::All("region".to_string()), + "sales_v", + Some("_outer"), + None, + &["year".to_string(), "region".to_string()], ); - assert!(cols.is_empty()); + assert_eq!( + expanded2, + "(SELECT SUM(revenue) FROM sales_v _inner WHERE _inner.year IS NOT DISTINCT FROM _outer.year)" + ); + } - // Mixed: dimension + constant - let cols = extract_dimension_columns_from_select( - "SELECT year, 1000, AGGREGATE(revenue) FROM sales_v", + #[test] + fn test_expand_at_to_sql_where() { + let expanded = expand_at_to_sql( + "revenue", + "SUM", + &ContextModifier::Where("orders.region = 'US'".to_string()), + "orders", + None, + None, + &[], ); - assert_eq!(cols, vec!["year".to_string()]); + assert_eq!( + expanded, + "(SELECT SUM(revenue) FROM orders WHERE region = 'US')" + ); + } - // NULL constant - let cols = extract_dimension_columns_from_select( - "SELECT NULL, AGGREGATE(revenue) FROM sales_v", + #[test] + fn test_expand_at_to_sql_set() { + // With outer alias for proper correlation + let expanded = expand_at_to_sql( + "revenue", + "SUM", + &ContextModifier::Set("year".to_string(), "year - 1".to_string()), + "sales_yearly", + Some("_outer"), + None, + &[], + ); + assert_eq!( + expanded, + "(SELECT SUM(revenue) FROM sales_yearly _inner WHERE _inner.year IS NOT DISTINCT FROM _outer.year - 1)" ); - assert!(cols.is_empty()); + } - // Float constant - let cols = extract_dimension_columns_from_select( - "SELECT 3.14, AGGREGATE(revenue) FROM sales_v", + #[test] + fn test_qualify_outer_reference() { + assert_eq!( + qualify_outer_reference("year - 1", "sales", "year"), + "sales.year - 1" ); - assert!(cols.is_empty()); - - // Negative integer - let cols = extract_dimension_columns_from_select( - "SELECT -1, AGGREGATE(revenue) FROM sales_v", + assert_eq!( + qualify_outer_reference("year + month", "t", "year"), + "t.year + month" ); - assert!(cols.is_empty()); } #[test] - #[serial] - fn test_process_create_view_basic() { - clear_measure_views(); + fn test_set_expression_requires_grouped_dimension() { + let sql = + "SELECT region, AGGREGATE(revenue) AT (SET year = year - 1) FROM sales_v"; + let result = expand_aggregate_with_at(sql); + assert!(result.error.is_some()); + let message = result.error.unwrap(); + assert!(message.contains("SET year = year - 1")); + assert!(message.contains("GROUP BY")); + } + #[test] + fn test_set_expression_allows_grouped_dimension() { let sql = - "CREATE VIEW orders_summary AS SELECT status, SUM(amount) AS MEASURE revenue FROM orders"; - let result = process_create_view(sql); + "SELECT year, region, AGGREGATE(revenue) AT (SET year = year - 1) FROM sales_v"; + let result = expand_aggregate_with_at(sql); + assert!(result.error.is_none()); + } - assert!(result.is_measure_view); - assert_eq!(result.view_name, Some("orders_summary".to_string())); - assert_eq!(result.measures.len(), 1); - assert_eq!(result.measures[0].column_name, "revenue"); - assert_eq!(result.measures[0].expression, "SUM(amount)"); - assert!(result.clean_sql.contains("AS revenue")); - assert!(!result.clean_sql.contains("AS MEASURE")); + #[test] + fn test_set_constant_allows_ungrouped_dimension() { + let sql = + "SELECT region, AGGREGATE(revenue) AT (SET year = 2023) FROM sales_v"; + let result = expand_aggregate_with_at(sql); + assert!(result.error.is_none()); } #[test] - #[serial] - fn test_process_create_view_sanitizes_non_ascii_comments() { - clear_measure_views(); + fn test_set_string_literal_allows_ungrouped_dimension() { + let sql = + "SELECT year, AGGREGATE(revenue) AT (SET region = 'region') FROM sales_v"; + let result = expand_aggregate_with_at(sql); + assert!(result.error.is_none()); + } - let sql = r#"CREATE VIEW orders_comment AS -SELECT status, -- café -SUM(amount) AS MEASURE revenue /* 東京 */ -FROM orders"#; - let result = std::panic::catch_unwind(|| process_create_view(sql)) - .expect("non-ASCII comments must not panic"); + #[test] + fn test_set_expression_requires_matching_grouping_alias() { + let sql = "SELECT c.year, AGGREGATE(revenue) AT (SET year = year - 1) \ + FROM orders_v o JOIN calendar_v c ON o.year = c.year \ + GROUP BY c.year"; + let result = expand_aggregate_with_at(sql); + assert!(result.error.is_some()); + let message = result.error.unwrap(); + assert!(message.contains("SET year = year - 1")); + } - assert!(result.is_measure_view); + #[test] + fn test_set_expression_allows_matching_grouping_alias() { + let sql = "SELECT o.year, AGGREGATE(revenue) AT (SET year = year - 1) FROM sales_v o GROUP BY o.year"; + let result = expand_aggregate_with_at(sql); assert!(result.error.is_none()); - assert_eq!(result.measures.len(), 1); - assert_eq!(result.measures[0].column_name, "revenue"); - assert!(result.clean_sql.is_ascii()); - assert!(result.clean_sql.contains("-- caf")); - assert!(result.clean_sql.contains("/*")); - assert!(!result.clean_sql.contains("café")); - assert!(!result.clean_sql.contains("東京")); } #[test] - fn test_comment_sanitizer_preserves_non_comment_unicode() { - let sql = "SELECT 'café -- literal', \"東京\" /* résumé */ FROM t -- 東京"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); - - assert!(sanitized.contains("'café -- literal'")); - assert!(sanitized.contains("\"東京\"")); - assert!(!sanitized.contains("résumé")); - assert!(!sanitized.ends_with("東京")); - assert_eq!(sanitized.len(), sql.len()); + fn test_at_all_warns_for_where_filter_on_ungrouped_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("AT (ALL ...)")); + assert!(warning.contains("year")); } #[test] - fn test_comment_sanitizer_preserves_dollar_quoted_strings() { - let sql = "SELECT $$-- café$$ AS label, $tag$/* 東京 */$tag$ AS note FROM t -- résumé"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); + fn test_at_all_does_not_warn_when_where_filter_dimension_is_grouped() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("year = 2024"), + &["year".to_string(), "region".to_string()], + &source_dims, + ); + assert!(warning.is_none()); + } - assert!(sanitized.contains("$$-- café$$")); - assert!(sanitized.contains("$tag$/* 東京 */$tag$")); - assert!(!sanitized.ends_with("résumé")); - assert_eq!(sanitized.len(), sql.len()); + #[test] + fn test_at_all_warns_for_quoted_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + Some(r#""year" = 2024"#), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); } #[test] - fn test_comment_sanitizer_ignores_dollar_tags_inside_regular_strings() { - let sql = "SELECT '$tag$' AS label FROM t -- $tag$ café"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); + fn test_at_all_warns_for_schema_qualified_source_filter() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let source_qualifiers = HashSet::from([normalize_identifier_name("sales_v")]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("main.sales_v.year = 2024"), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ) + .unwrap(); + assert!(warning.contains("year")); + } - assert!(sanitized.contains("'$tag$'")); - assert!(sanitized.contains("-- $tag$ caf")); - assert!(!sanitized.ends_with("café")); - assert!(sanitized.is_ascii()); - assert_eq!(sanitized.len(), sql.len()); + #[test] + fn test_at_all_top_level_cte_uses_main_select_grouping_context() { + let sql = "WITH noop AS (SELECT 1) \ + SELECT year, region, AGGREGATE(revenue) AT (ALL year) AS region_total \ + FROM sales_v GROUP BY year, region"; + let context_sql = aggregate_context_sql(sql); + let from_info = extract_from_clause_info(context_sql); + + assert!(context_sql.trim_start().starts_with("SELECT year, region")); + assert_eq!(extract_group_by_columns(context_sql), vec!["year", "region"]); + assert_eq!( + from_info.primary_table.map(|table| table.name), + Some("sales_v".to_string()) + ); } #[test] - fn test_comment_sanitizer_handles_comments_inside_list_brackets() { - let sql = "SELECT [1, /* 東京 */ 2] AS xs FROM t -- résumé"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); + fn test_at_all_top_level_cte_warns_for_main_select_where_filter() { + let sql = "WITH noop AS (SELECT 1) \ + SELECT region, AGGREGATE(revenue) AT (ALL region) AS total_revenue \ + FROM sales_v WHERE year = 2024 GROUP BY region"; + let context_sql = aggregate_context_sql(sql); + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + extract_where_clause(context_sql).as_deref(), + &extract_group_by_columns(context_sql), + &source_dims, + ) + .unwrap(); - assert!(sanitized.contains("[1, /*")); - assert!(!sanitized.contains("東京")); - assert!(!sanitized.ends_with("résumé")); - assert!(sanitized.is_ascii()); - assert_eq!(sanitized.len(), sql.len()); + assert!(context_sql.trim_start().starts_with("SELECT region")); + assert!(warning.contains("year")); } #[test] - fn test_comment_sanitizer_handles_quotes_inside_list_brackets() { - let sql = "SELECT [']'] AS xs FROM t -- 東京"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); + fn test_at_all_ctas_with_uses_main_select_grouping_context() { + let sql = "CREATE TABLE out AS WITH noop AS (SELECT 1) \ + SELECT year, region, AGGREGATE(revenue) AT (ALL region) AS region_total \ + FROM sales_v GROUP BY year, region"; + let context_sql = aggregate_context_sql(sql); - assert!(sanitized.contains("[']']")); - assert!(!sanitized.ends_with("東京")); - assert!(sanitized.is_ascii()); - assert_eq!(sanitized.len(), sql.len()); + assert!(context_sql.trim_start().starts_with("SELECT year, region")); + assert!(context_sql.contains("FROM sales_v")); + assert!(context_sql.contains("GROUP BY year, region")); } #[test] - fn test_comment_sanitizer_handles_escape_string_quotes() { - let sql = "SELECT E'can\\'t' AS label FROM sales -- 東京"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); + fn test_at_all_ctas_with_warns_for_main_select_where_filter() { + let sql = "CREATE TABLE out AS WITH noop AS (SELECT 1) \ + SELECT region, AGGREGATE(revenue) AT (ALL region) AS total_revenue \ + FROM sales_v WHERE year = 2024 GROUP BY region"; + let context_sql = aggregate_context_sql(sql); + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + extract_where_clause(context_sql).as_deref(), + &["region".to_string()], + &source_dims, + ) + .unwrap(); - assert!(sanitized.contains("E'can\\'t'")); - assert!(!sanitized.ends_with("東京")); - assert!(sanitized.is_ascii()); - assert_eq!(sanitized.len(), sql.len()); + assert!(context_sql.trim_start().starts_with("SELECT region")); + assert!(warning.contains("year")); } #[test] - fn test_comment_sanitizer_handles_nested_block_comments() { - let sql = "SELECT 1 /* outer /* inner */ 東京 */ FROM t -- résumé"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); + fn test_at_all_parenthesized_insert_warns_for_select_where_filter() { + let sql = "INSERT INTO warning_insert_by_name_target BY NAME ( \ + SELECT region, AGGREGATE(revenue) AT (ALL region) AS total_revenue \ + FROM sales_v WHERE year = 2024 GROUP BY region \ + )"; + let context_sql = aggregate_context_sql(sql); + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + extract_where_clause(context_sql).as_deref(), + &extract_group_by_columns(context_sql), + &source_dims, + ) + .unwrap(); - assert!(sanitized.contains("/* outer /* inner */")); - assert!(!sanitized.contains("東京")); - assert!(!sanitized.ends_with("résumé")); - assert!(sanitized.is_ascii()); - assert_eq!(sanitized.len(), sql.len()); + assert!(context_sql.trim_start().starts_with("SELECT region")); + assert!(warning.contains("year")); } #[test] - fn test_comment_sanitizer_preserves_bracket_quoted_identifiers() { - let sql = "SELECT [東京] FROM t -- résumé"; - let sanitized = sanitize_non_ascii_in_sql_comments(sql); + fn test_at_all_parenthesized_insert_ignores_comment_parens_in_body() { + let sql = "INSERT INTO warning_insert_by_name_target BY NAME ( \ + SELECT region, AGGREGATE(revenue) AT (ALL region) AS total_revenue \ + FROM sales_v /* ) */ WHERE year = 2024 GROUP BY region \ + )"; + let context_sql = aggregate_context_sql(sql); + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let group_by_cols = vec!["region".to_string()]; + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + extract_where_clause(context_sql).as_deref(), + &group_by_cols, + &source_dims, + ) + .unwrap(); - assert!(sanitized.contains("[東京]")); - assert!(!sanitized.ends_with("résumé")); - assert_eq!(sanitized.len(), sql.len()); + assert!(context_sql.contains("WHERE year = 2024")); + assert!(context_sql.contains("GROUP BY region")); + assert!(warning.contains("year")); } #[test] - #[serial] - fn test_process_create_view_case_expression() { - clear_measure_views(); - - let sql = "CREATE VIEW v AS SELECT year, CASE WHEN SUM(x) > 100 THEN 1 ELSE 0 END AS MEASURE flag FROM t GROUP BY year"; - let result = process_create_view(sql); + fn test_at_all_warns_for_date_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("date"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("date = DATE '2023-01-01'"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("date")); + } - eprintln!("Result: {result:?}"); - eprintln!("Measures: {:?}", result.measures); - assert!(result.is_measure_view); - assert_eq!(result.measures.len(), 1); - assert_eq!(result.measures[0].column_name, "flag"); - assert_eq!( - result.measures[0].expression, - "CASE WHEN SUM(x) > 100 THEN 1 ELSE 0 END" + #[test] + fn test_at_all_ignores_where_filter_line_comment_identifiers() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("TRUE -- year"), + &["region".to_string()], + &source_dims, ); - - // Now test that AGGREGATE(flag) works - let query_sql = "SELECT year, AGGREGATE(flag) FROM v GROUP BY year"; - let expand_result = std::panic::catch_unwind(|| expand_aggregate(query_sql)) - .unwrap_or_else(|_| expand_aggregate_with_at(query_sql)); - eprintln!("Expand result: {expand_result:?}"); - // This should have expanded AGGREGATE(flag) to something - assert!(expand_result.had_aggregate); + assert!(warning.is_none()); } #[test] - #[serial] - fn test_process_create_view_derived_measure() { - clear_measure_views(); - - // Create view with base measures and a derived measure - let sql = "CREATE VIEW financials_v AS SELECT year, SUM(revenue) AS MEASURE revenue, SUM(cost) AS MEASURE cost, revenue - cost AS MEASURE profit FROM financials GROUP BY year"; - let result = process_create_view(sql); - - eprintln!("Clean SQL: {}", result.clean_sql); - eprintln!("Measures: {:?}", result.measures); - - assert!(result.is_measure_view); - assert_eq!(result.measures.len(), 3); - - // Check base measures - assert_eq!(result.measures[0].column_name, "revenue"); - assert_eq!(result.measures[0].expression, "SUM(revenue)"); - assert_eq!(result.measures[1].column_name, "cost"); - assert_eq!(result.measures[1].expression, "SUM(cost)"); - - // Check derived measure - assert_eq!(result.measures[2].column_name, "profit"); - assert_eq!(result.measures[2].expression, "revenue - cost"); - - // Clean SQL should NOT contain the derived measure column - assert!(result.clean_sql.contains("AS revenue")); - assert!(result.clean_sql.contains("AS cost")); - assert!(!result.clean_sql.contains("AS profit")); - assert!(!result.clean_sql.contains("revenue - cost")); + fn test_at_all_ignores_where_filter_dollar_quoted_literals() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("$tag$year$tag$ = $tag$year$tag$"), + &["region".to_string()], + &source_dims, + ); + assert!(warning.is_none()); } #[test] - fn test_extract_measures_named_window_measure() { - let sql = r#"CREATE VIEW sales_window_v AS -SELECT - year, - SUM(amount) - OVER win AS MEASURE running_total -FROM sales -WINDOW win AS (ORDER BY year)"#; - let (clean_sql, measures, view_name, _) = extract_measures_from_sql(sql).unwrap(); + fn test_at_all_ignores_at_where_block_comment_identifiers() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Where("1 = 1 /* year */".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); + } - assert_eq!(view_name, Some("sales_window_v".to_string())); - assert_eq!(measures.len(), 1); - assert_eq!(measures[0].column_name, "running_total"); - assert!(!measures[0].is_decomposable); - assert!(!clean_sql.contains("NULL AS running_total")); - assert!(clean_sql.contains("AS running_total")); + #[test] + fn test_at_all_ignores_where_filter_subquery_identifiers() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("EXISTS (/* calendar */ SELECT 1 FROM calendar c WHERE c.year = 2024)"), + &["region".to_string()], + &source_dims, + ); + assert!(warning.is_none()); } #[test] - #[serial] - fn test_process_create_view_without_group_by() { - clear_measure_views(); + fn test_at_all_warns_for_source_qualified_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("s.year = 2024"), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ) + .unwrap(); + assert!(warning.contains("year")); + } - // Per the paper, views can define measures without GROUP BY - let sql = r#"CREATE VIEW orders_extended AS -SELECT - id, - product, - region, - amount, - SUM(amount) AS MEASURE revenue -FROM orders"#; - let result = process_create_view(sql); + #[test] + fn test_at_all_ignores_non_source_qualified_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("c.year = 2024"), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ); + assert!(warning.is_none()); + } - eprintln!("is_measure_view: {}", result.is_measure_view); - eprintln!("clean_sql: {}", result.clean_sql); - eprintln!("measures: {:?}", result.measures); + #[test] + fn test_at_all_ignores_non_source_quoted_qualified_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some(r#"c."year" = 2024"#), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ); + assert!(warning.is_none()); + } - assert!(result.is_measure_view); - assert_eq!(result.measures.len(), 1); - assert_eq!(result.measures[0].column_name, "revenue"); - // The clean_sql should NOT contain the measure column at all for no-groupby views - // because it can't be evaluated without a GROUP BY + #[test] + fn test_at_all_ignores_non_source_three_part_qualified_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("main.calendar.year = 2023"), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ); + assert!(warning.is_none()); } #[test] - fn test_has_at_syntax() { - assert!(has_at_syntax( - "SELECT AGGREGATE(revenue) AT (ALL status) FROM orders" - )); - assert!(!has_at_syntax("SELECT AGGREGATE(revenue) FROM orders")); + fn test_at_all_ignores_interval_unit_as_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + normalize_identifier_name("dt"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("c.dt >= CURRENT_DATE - INTERVAL '1' YEAR"), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ); + assert!(warning.is_none()); } #[test] - fn test_parse_at_modifier_all() { - let result = parse_at_modifier("ALL status").unwrap(); - assert_eq!(result, ContextModifier::All("status".to_string())); + fn test_at_all_interval_unit_does_not_encode_dropped_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + normalize_identifier_name("dt"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Where("dt >= CURRENT_DATE - INTERVAL '1' YEAR".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); } #[test] - fn test_parse_at_modifier_set() { - let result = parse_at_modifier("SET month = month - 1").unwrap(); - assert_eq!( - result, - ContextModifier::Set("month".to_string(), "month - 1".to_string()) + fn test_at_all_ignores_extract_date_part_as_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + normalize_identifier_name("dt"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("EXTRACT(YEAR FROM c.dt) = 2024"), + &["region".to_string()], + &source_dims, + &source_qualifiers, ); + assert!(warning.is_none()); } #[test] - fn test_parse_at_modifier_where() { - let result = parse_at_modifier("WHERE region = 'US'").unwrap(); - assert_eq!(result, ContextModifier::Where("region = 'US'".to_string())); + fn test_at_all_extract_date_part_does_not_encode_dropped_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + normalize_identifier_name("dt"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Where("EXTRACT(YEAR FROM dt) = 2024".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); } #[test] - fn test_parse_at_modifier_where_strips_qualifier() { - let result = parse_at_modifier("WHERE sales.region = 'US'").unwrap(); - assert_eq!(result, ContextModifier::Where("region = 'US'".to_string())); + fn test_at_all_ignores_cast_target_type_as_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("date"), + normalize_identifier_name("region"), + normalize_identifier_name("dt"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("CAST(c.dt AS DATE) = DATE '2023-01-01'"), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ); + assert!(warning.is_none()); } #[test] - fn test_extract_aggregate_with_at() { - let sql = "SELECT status, AGGREGATE(revenue) AT (ALL status) FROM orders"; - let results = extract_aggregate_with_at(sql); - - assert_eq!(results.len(), 1); - assert_eq!(results[0].0, "revenue"); - assert_eq!(results[0].1, ContextModifier::All("status".to_string())); + fn test_at_all_ignores_postgres_cast_target_type_as_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("date"), + normalize_identifier_name("region"), + normalize_identifier_name("dt"), + ]); + let source_qualifiers = HashSet::from([ + normalize_identifier_name("sales_v"), + normalize_identifier_name("s"), + ]); + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "revenue", + &[ContextModifier::All("region".to_string())], + Some("c.dt::DATE = DATE '2023-01-01'"), + &["region".to_string()], + &source_dims, + &source_qualifiers, + ); + assert!(warning.is_none()); } #[test] - fn test_extract_aggregate_skips_dollar_quoted_literals() { - assert!(extract_all_aggregate_calls( - "SELECT $$AGGREGATE(revenue)$$ AS note" - ) - .is_empty()); - assert!(extract_aggregate_with_at_full( - "SELECT $tag$AGGREGATE(revenue) AT (ALL)$tag$ AS note" + fn test_at_all_cast_target_does_not_encode_dropped_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("date"), + normalize_identifier_name("region"), + normalize_identifier_name("dt"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Where("CAST(dt AS DATE) = DATE '2023-01-01'".to_string()), + ], + Some("date = DATE '2023-01-01'"), + &["region".to_string()], + &source_dims, ) - .is_empty()); + .unwrap(); + assert!(warning.contains("date")); + } - let calls = extract_all_aggregate_calls( - "SELECT AGGREGATE(revenue), $$AGGREGATE(cost)$$ FROM sales_v", + #[test] + fn test_at_all_does_not_warn_when_at_where_encodes_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Where("year = 2024".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, ); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].0, "revenue"); + assert!(warning.is_none()); } #[test] - fn test_expand_at_to_sql_all() { - // With no group_by_cols, AT (ALL dim) acts like grand total - let expanded = expand_at_to_sql( + fn test_at_all_warns_when_filter_only_in_overwritten_at_where() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + normalize_identifier_name("category"), + ]); + let warning = warning_for_at_all_ungrouped_where( "revenue", - "SUM", - &ContextModifier::All("status".to_string()), - "orders", - None, - None, - &[], - ); - assert_eq!(expanded, "(SELECT SUM(revenue) FROM orders)"); + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Where("category = 'A'".to_string()), + ContextModifier::Where("year = 2024".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); + } - // With group_by_cols, AT (ALL dim) generates correlated subquery - let expanded2 = expand_at_to_sql( + #[test] + fn test_at_all_does_not_warn_when_filter_in_effective_at_where() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + normalize_identifier_name("category"), + ]); + let warning = warning_for_at_all_ungrouped_where( "revenue", - "SUM", - &ContextModifier::All("region".to_string()), - "sales_v", - Some("_outer"), - None, - &["year".to_string(), "region".to_string()], - ); - assert_eq!( - expanded2, - "(SELECT SUM(revenue) FROM sales_v _inner WHERE _inner.year IS NOT DISTINCT FROM _outer.year)" + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Where("year = 2024".to_string()), + ContextModifier::Where("category = 'A'".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, ); + assert!(warning.is_none()); } #[test] - fn test_expand_at_to_sql_where() { - let expanded = expand_at_to_sql( + fn test_at_all_does_not_warn_when_set_encodes_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( "revenue", - "SUM", - &ContextModifier::Where("orders.region = 'US'".to_string()), - "orders", - None, - None, - &[], - ); - assert_eq!( - expanded, - "(SELECT SUM(revenue) FROM orders WHERE region = 'US')" + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Set("year".to_string(), "2024".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, ); + assert!(warning.is_none()); } #[test] - fn test_expand_at_to_sql_set() { - // With outer alias for proper correlation - let expanded = expand_at_to_sql( + fn test_at_all_does_not_warn_when_expression_set_encodes_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("order_date"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( "revenue", - "SUM", - &ContextModifier::Set("year".to_string(), "year - 1".to_string()), - "sales_yearly", - Some("_outer"), - None, - &[], - ); - assert_eq!( - expanded, - "(SELECT SUM(revenue) FROM sales_yearly _inner WHERE _inner.year IS NOT DISTINCT FROM _outer.year - 1)" + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Set("MONTH(order_date)".to_string(), "2".to_string()), + ], + Some("MONTH(order_date) = 2"), + &["region".to_string()], + &source_dims, ); + assert!(warning.is_none()); } #[test] - fn test_qualify_outer_reference() { - assert_eq!( - qualify_outer_reference("year - 1", "sales", "year"), - "sales.year - 1" - ); - assert_eq!( - qualify_outer_reference("year + month", "t", "year"), - "t.year + month" - ); + fn test_at_all_expression_set_does_not_encode_base_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("order_date"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Set("MONTH(order_date)".to_string(), "2".to_string()), + ], + Some("order_date = DATE '2023-02-10'"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("order_date")); } #[test] - fn test_set_expression_requires_grouped_dimension() { - let sql = - "SELECT region, AGGREGATE(revenue) AT (SET year = year - 1) FROM sales_v"; - let result = expand_aggregate_with_at(sql); - assert!(result.error.is_some()); - let message = result.error.unwrap(); - assert!(message.contains("SET year = year - 1")); - assert!(message.contains("GROUP BY")); + fn test_at_all_expression_set_does_not_match_expression_only_in_comment() { + let source_dims = HashSet::from([ + normalize_identifier_name("order_date"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Set("MONTH(order_date)".to_string(), "2".to_string()), + ], + Some("order_date = DATE '2023-02-10' /* MONTH(order_date) */"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("order_date")); } #[test] - fn test_set_expression_allows_grouped_dimension() { - let sql = - "SELECT year, region, AGGREGATE(revenue) AT (SET year = year - 1) FROM sales_v"; - let result = expand_aggregate_with_at(sql); - assert!(result.error.is_none()); + fn test_at_all_expression_set_does_not_match_expression_only_in_dollar_quote() { + let source_dims = HashSet::from([ + normalize_identifier_name("order_date"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Set("MONTH(order_date)".to_string(), "2".to_string()), + ], + Some("order_date = DATE '2023-02-10' AND $tag$MONTH(order_date)$tag$ = 'x'"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("order_date")); } #[test] - fn test_set_constant_allows_ungrouped_dimension() { - let sql = - "SELECT region, AGGREGATE(revenue) AT (SET year = 2023) FROM sales_v"; - let result = expand_aggregate_with_at(sql); - assert!(result.error.is_none()); + fn test_at_all_preserves_warning_through_fallback_expansion_result() { + let mut result = AggregateExpandResult { + had_aggregate: true, + expanded_sql: "SELECT SUM(revenue) FROM sales_v".to_string(), + error: None, + warnings: vec!["existing warning".to_string()], + }; + + append_unique_warnings( + &mut result, + vec![ + "AT (ALL ...) dropped year".to_string(), + "existing warning".to_string(), + ], + ); + + assert_eq!(result.warnings.len(), 2); + assert!( + result + .warnings + .contains(&"AT (ALL ...) dropped year".to_string()) + ); } #[test] - fn test_set_string_literal_allows_ungrouped_dimension() { - let sql = - "SELECT year, AGGREGATE(revenue) AT (SET region = 'region') FROM sales_v"; - let result = expand_aggregate_with_at(sql); - assert!(result.error.is_none()); + fn test_at_all_does_not_warn_when_visible_preserves_outer_where() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Visible, + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ); + assert!(warning.is_none()); } #[test] - fn test_set_expression_requires_matching_grouping_alias() { - let sql = "SELECT c.year, AGGREGATE(revenue) AT (SET year = year - 1) \ - FROM orders_v o JOIN calendar_v c ON o.year = c.year \ - GROUP BY c.year"; - let result = expand_aggregate_with_at(sql); - assert!(result.error.is_some()); - let message = result.error.unwrap(); - assert!(message.contains("SET year = year - 1")); + fn test_at_all_warns_when_visible_is_bypassed_by_set() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ + ContextModifier::All("region".to_string()), + ContextModifier::Visible, + ContextModifier::Set("category".to_string(), "CURRENT(category)".to_string()), + ], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); } #[test] - fn test_set_expression_allows_matching_grouping_alias() { - let sql = "SELECT o.year, AGGREGATE(revenue) AT (SET year = year - 1) FROM sales_v o GROUP BY o.year"; - let result = expand_aggregate_with_at(sql); - assert!(result.error.is_none()); + fn test_at_all_warns_when_grouped_filter_dimension_is_removed() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::All("year".to_string())], + Some("year = 2024"), + &["year".to_string(), "region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); + } + + #[test] + fn test_at_all_global_warns_for_where_filter_dimension() { + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + let warning = warning_for_at_all_ungrouped_where( + "revenue", + &[ContextModifier::AllGlobal], + Some("year = 2024"), + &["region".to_string()], + &source_dims, + ) + .unwrap(); + assert!(warning.contains("year")); } #[test] @@ -7700,6 +9761,10 @@ FROM orders"#; extract_where_clause(sql3), Some("a = 1 AND b = 2".to_string()) ); + + let sql4 = "SELECT $tag$WHERE year = 2023$tag$ AS note, region, \ + AGGREGATE(revenue) AT (ALL region) FROM sales_v GROUP BY region"; + assert_eq!(extract_where_clause(sql4), None); } #[test] @@ -8031,6 +10096,17 @@ GROUP BY s.year"; assert_eq!(find_alias_for_view(&info, "nonexistent"), None); } + #[test] + fn test_extract_from_clause_info_string_fallback_join_aliases() { + let sql = "SELECT r.region, AGGREGATE(refunds) AT (ALL region) \ + FROM fact_orders_v o JOIN fact_returns_v r ON o.id = r.order_id"; + let info = extract_from_clause_info(sql); + + assert_eq!(info.primary_table.as_ref().unwrap().name, "fact_orders_v"); + assert_eq!(find_alias_for_view(&info, "fact_orders_v"), Some("o")); + assert_eq!(find_alias_for_view(&info, "fact_returns_v"), Some("r")); + } + #[test] #[serial] fn test_expand_aggregate_with_join_measure_from_first_table() { @@ -8084,6 +10160,33 @@ GROUP BY s.year"; assert!(result.expanded_sql.contains("FROM orders_v")); } + #[test] + fn test_at_all_warns_for_joined_measure_alias_where_filter() { + let sql = "SELECT r.region, AGGREGATE(refunds) AT (ALL region) AS all_region_refunds \ + FROM fact_orders_v o JOIN fact_returns_v r ON o.id = r.order_id"; + let from_info = extract_from_clause_info(sql); + let mut source_qualifiers = HashSet::from([normalize_identifier_name("fact_returns_v")]); + if let Some(alias) = find_alias_for_view(&from_info, "fact_returns_v") { + source_qualifiers.insert(normalize_identifier_name(alias)); + } + let source_dims = HashSet::from([ + normalize_identifier_name("year"), + normalize_identifier_name("region"), + ]); + + let warning = warning_for_at_all_ungrouped_where_with_qualifiers( + "refunds", + &[ContextModifier::All("region".to_string())], + Some("r.year = 2023"), + &["r.region".to_string()], + &source_dims, + &source_qualifiers, + ) + .unwrap(); + + assert!(warning.contains("year")); + } + #[test] fn test_parse_at_modifier_ad_hoc_dimension() { // Ad hoc dimension with function expression