diff --git a/sidemantic/adapters/lookml.py b/sidemantic/adapters/lookml.py index cdc2b7d3..34d07989 100644 --- a/sidemantic/adapters/lookml.py +++ b/sidemantic/adapters/lookml.py @@ -35,7 +35,12 @@ class LookMLAdapter(BaseAdapter): """ def parse(self, source: str | Path) -> SemanticGraph: - """Parse LookML files into semantic graph. + """Parse LookML files into a semantic graph. + + Auto-registration to an active SemanticLayer is suppressed while the graph is + built, so a tableless view that only gets its default table after refinements/ + extends are resolved is not validated mid-parse (which would raise inside a + ``with SemanticLayer():`` block). The finalized models are registered once. Args: source: Path to .lkml file or directory @@ -43,6 +48,40 @@ def parse(self, source: str | Path) -> SemanticGraph: Returns: Semantic graph with imported models """ + from sidemantic.core.registry import get_current_layer, set_current_layer + + prev_layer = get_current_layer() + set_current_layer(None) + try: + graph = self._build_graph(source) + finally: + set_current_layer(prev_layer) + # Defer auto-registration until models are complete (tables defaulted). Skip ONLY + # intentional non-queryable templates -- abstract extension:required bases and + # unsupported derived tables, marked in meta -- which add_model's validation would + # reject; mirrors loaders._is_registerable_model. A merely-broken tableless view + # (e.g. extends:[missing], left unresolved) is NOT skipped, so add_model still + # surfaces the real "no table/sql" error instead of silently dropping it. Strip + # survivors' relationships pointing at a skipped template (e.g. an explore join to + # it) so the active layer is not left with a dangling relationship validation flags. + if prev_layer is not None: + skipped = { + name + for name, m in graph.models.items() + if not (m.table or m.sql or getattr(m, "dax", None) or getattr(m, "source_uri", None)) + and (m.meta or {}).get("lookml_template") + } + for model in graph.models.values(): + if model.name in skipped or model.name in prev_layer.graph.models: + continue + rels = getattr(model, "relationships", None) + if rels: + model.relationships = [r for r in rels if r.name not in skipped] + prev_layer.add_model(model) + return graph + + def _build_graph(self, source: str | Path) -> SemanticGraph: + """Build the semantic graph from LookML files (see :meth:`parse`).""" graph = SemanticGraph() source_path = Path(source) @@ -58,17 +97,50 @@ def parse(self, source: str | Path) -> SemanticGraph: for lkml_file in lkml_files: self._parse_views_from_file(lkml_file, graph, refinements) + # Snapshot abstract / unsupported-derived_table flags BEFORE refinement merge: + # merge_model REPLACES a base view's meta when the refinement carries metadata, + # which would otherwise drop these markers. + abstract_pre = {n for n, m in graph.models.items() if (m.meta or {}).get("extension_required")} + unsupported_pre = {n for n, m in graph.models.items() if (m.meta or {}).get("unsupported_derived_table")} + # Apply refinements: merge each refinement into its base view from sidemantic.core.inheritance import merge_model, resolve_model_inheritance + refinement_abstract: set[str] = set() + refinement_unsupported_dt: set[str] = set() for refinement in refinements: base_name = refinement.name.lstrip("+") + # Record flags from EACH refinement's own meta: a later refinement's merge + # can replace the base meta and drop a flag an earlier refinement added. + rmeta = refinement.meta or {} + if rmeta.get("extension_required"): + refinement_abstract.add(base_name) + if rmeta.get("unsupported_derived_table"): + refinement_unsupported_dt.add(base_name) if base_name in graph.models: # Create a copy with the base name for merging refinement_for_merge = refinement.model_copy(update={"name": base_name}) merged = merge_model(refinement_for_merge, graph.models[base_name]) graph.models[base_name] = merged + # Union the pre-merge snapshot, every refinement's own flags, and the post-merge + # state (so a flag added by ANY refinement is caught even if a later refinement + # replaced the meta). Resolve this BEFORE extends so a concrete child that only + # INHERITS abstractness is not treated as abstract. Record extends parents too, + # so descendants of an unsupported derived table are detectable after resolution + # clears `extends`. + abstract_views = ( + abstract_pre + | refinement_abstract + | {n for n, m in graph.models.items() if (m.meta or {}).get("extension_required")} + ) + unsupported_dt_views = ( + unsupported_pre + | refinement_unsupported_dt + | {n for n, m in graph.models.items() if (m.meta or {}).get("unsupported_derived_table")} + ) + extends_parent = {n: m.extends for n, m in graph.models.items() if m.extends} + # Resolve extends chains. Pre-filter to models whose full chain # is present so one broken/missing parent doesn't block valid ones. def _chain_resolvable(name: str, visited: set[str] | None = None) -> bool: @@ -92,10 +164,85 @@ def _chain_resolvable(name: str, visited: set[str] | None = None) -> bool: resolved.update(unresolvable) graph.models = resolved + def _extends_chain_has(name: str, flagset: set[str]) -> bool: + """True if name or any of its extends-ancestors is in flagset.""" + seen: set[str] = set() + cur: str | None = name + while cur is not None and cur not in seen: + if cur in flagset: + return True + seen.add(cur) + cur = extends_parent.get(cur) + return False + + # Apply the implicit "table = view name" default AFTER refinements and extends + # are resolved, so a view whose fields/name come from a refinement or whose + # parent was tableless still gets its OWN name as the table (Looker's behavior). + # Skip abstract views, still-unresolved-extends, and views that are (or extend) an + # unsupported derived table. Do NOT require parsed fields: Looker defaults the table + # for an ordinary fieldless view (`view: orders {}`, or one with only adapter-ignored + # fields) too, so leaving it tableless would wrongly fail CLI validation/registration. + # Abstractness is NOT inherited through extends (a concrete child of an abstract base + # gets its own table), but an unsupported derived table IS inherited by descendants. + def _apply_default_tables(): + for model_name, model in graph.models.items(): + if ( + model.table is None + and model.sql is None + and not model.extends + and model_name not in abstract_views + and not _extends_chain_has(model_name, unsupported_dt_views) + and not (model.meta or {}).get("unsupported_derived_table") + ): + model.table = model_name + + _apply_default_tables() + # Second pass: parse explores and add relationships for lkml_file in lkml_files: self._parse_explores_from_file(lkml_file, graph) + # Re-apply the default: explores can add segments (sql_always_where / + # always_filter) to an otherwise-fieldless view, which only now makes it + # eligible for the implicit table default. + _apply_default_tables() + + # Re-assert non-queryable markers on models intentionally left tableless. A + # refinement can OVERWRITE a view's meta (dropping an `extension_required` flag an + # earlier refinement added) even though abstractness is tracked in side-sets, so + # without this the loader (which keys off final meta) would try to register and + # reject them. Set the flag definitively now, after all merges. Also stamp a + # PARSER-OWNED `lookml_template` marker so registration skips (here and in the + # loader) key off a sidemantic-internal flag, not the public `extension_required`/ + # `unsupported_derived_table` keys -- a native/other-format model that happens to + # carry those user-facing keys must still surface its missing-source error. + for model_name, model in graph.models.items(): + if model.table is not None or model.sql is not None: + continue + is_template = False + if model_name in abstract_views: + model.meta = {**(model.meta or {}), "extension_required": True, "lookml_template": True} + is_template = True + elif _extends_chain_has(model_name, unsupported_dt_views) or (model.meta or {}).get( + "unsupported_derived_table" + ): + model.meta = {**(model.meta or {}), "unsupported_derived_table": True, "lookml_template": True} + is_template = True + if is_template: + # Stamp this template's GRAPH-LEVEL measures (time_comparison/conversion, which + # add_model auto-registers into graph.metrics) with a parser-owned provenance + # marker. The loader drops orphaned graph metrics by this marker -- proving they + # came from a dropped template -- instead of guessing from the base ref, so a + # same-named standalone metric from another file (no marker) is never dropped. + # Mark BOTH the model's (possibly refinement-reconstructed) metric AND the graph- + # registered object (auto-registered pre-refinement, so a different instance). + for mt in model.metrics or []: + if mt.type in ("time_comparison", "conversion"): + mt.meta = {**(mt.meta or {}), "_lookml_template_metric": True} + gm = graph.metrics.get(mt.name) + if gm is not None: + gm.meta = {**(gm.meta or {}), "_lookml_template_metric": True} + # Rebuild adjacency graph now that relationships have been added graph.build_adjacency() @@ -1260,6 +1407,17 @@ def _sub(mm): elif isinstance(extends_list, str): extends = extends_list + # A LookML view with no sql_table_name/derived_table implicitly uses a table + # named after the view (Looker's default). A view with a derived_table the + # adapter cannot turn into SQL must NOT get such a default; mark it so the + # post-merge pass skips it, and RETAIN the raw derived_table dict so export can + # re-emit it -- a derived_table with no `sql` keeps the view unsupported on + # re-import instead of being defaulted to a physical table named after the view. + unsupported_derived_table = bool(view_def.get("derived_table")) and sql is None + if unsupported_derived_table: + model_meta["unsupported_derived_table"] = True + model_meta["_unsupported_derived_table_raw"] = view_def["derived_table"] + # Build kwargs conditionally so that unset scalars don't appear in # model_fields_set. This matters for refinements: merge_model treats # every field in model_fields_set as an explicit child override, so @@ -1321,8 +1479,76 @@ def _parse_dimension(self, dim_def: dict, dimension_sql_lookup: dict[str, str] | if sql: sql = sql.replace("${TABLE}", "{model}") + # Recover the granularity of the collision-export form: a standalone time + # dimension we emit as `type: date_time sql: DATE_TRUNC('', ...)` whose + # name carries a LookML timeframe suffix for that grain (e.g. started_date with + # DATE_TRUNC('day', ...), started_minute with DATE_TRUNC('minute', ...) -- see + # _export_collision_time_dim). Guard tightly so a hand-written DATE_TRUNC dim is + # NOT hijacked: the grain must be one sidemantic supports, the name's trailing + # `_` must be a LookML timeframe mapping to that exact grain, AND the dim + # must carry no other properties (the collision form emits only name/type/sql). + # This keeps a hand-written `created` (DATE_TRUNC('day', ...)) on the categorical + # path (no rename to created_date), preserves a hand-written `created_date` with + # `hidden`/`label`/etc. as a plain dimension (props not dropped), and never copies + # a dialect grain like 'isoweek' into Dimension.granularity (a validation error). + granularity = None + recovered_timeframe = None + if ( + dim_type in ("date", "date_time", "datetime", "time") + and sql + and not (set(dim_def) & self._PRESERVED_DIM_PROPS) + ): + bucket = self._MINUTE_BUCKET_RE.match(sql) + tm = self._DATE_TRUNC_RE.match(sql) + if bucket: + # A collision-export minute15/minute30 bucket: recover minute grain + the + # exact timeframe. KEEP the full bucket expression as the dim's sql (do NOT + # reduce to the raw column) so QUERIES bucket every 15/30 minutes; the + # generator's DATE_TRUNC('minute', ...) wrap is idempotent on an already + # minute-aligned bucket. Re-export detects this bucket to avoid re-wrapping. + tf = f"minute{bucket.group(2)}" + if name.endswith("_" + tf): + sidemantic_type = "time" + granularity = "minute" + recovered_timeframe = tf + elif tm: + grain = tm.group(1).lower() + if grain in self._SUBSECOND_TF and name.endswith("_" + grain): + # A collision-export sub-second DATE_TRUNC (millisecond/microsecond): + # sidemantic has no such grain, so store the nearest (second) + the + # exact timeframe in meta, KEEPING the sub-second DATE_TRUNC sql so the + # exported precision round-trips (queries run at second, sidemantic's max). + sidemantic_type = "time" + granularity = "second" + recovered_timeframe = grain + # Match the LONGEST known timeframe suffix, not just text after the last + # underscore -- multi-word timeframes like "time_of_day" must round-trip + # (started_time_of_day, not a bogus "day" suffix lookup). + elif grain in self._SUPPORTED_GRAINS: + matched_tf = None + for tf, g in self._TIME_GRANULARITY_TIMEFRAMES.items(): + if name.endswith("_" + tf) and g == grain and (not matched_tf or len(tf) > len(matched_tf)): + matched_tf = tf + if matched_tf: + sidemantic_type = "time" + granularity = grain + # STORE the timeframe unless it is one whose exact semantics live in a + # special SQL form (minute15/30 bucket, sub-second DATE_TRUNC) that a PLAIN + # DATE_TRUNC has already lost -- recording those would make the next export + # WIDEN a 1-minute dim into a 15-min bucket. `time_of_day` is NOT in that + # set: its canonical export IS this plain hour DATE_TRUNC named + # `*_time_of_day`, so it is recoverable only by name -- record it, else the + # next export loses the timeframe and renames the field to `*_hour`. + if matched_tf not in self._EXACT_FORM_TF: + recovered_timeframe = matched_tf + # Build meta dict from LookML-specific display properties meta = {} + if recovered_timeframe: + # Remember the exact LookML timeframe so the NEXT export strips this suffix + # (e.g. _time_of_day) instead of re-deriving a wrong one and renaming the + # field -- keeps repeated collision round-trips stable. + meta["lookml_timeframe"] = recovered_timeframe if dim_def.get("hidden") in ("yes", True): meta["hidden"] = True if dim_def.get("group_label"): @@ -1338,6 +1564,7 @@ def _parse_dimension(self, dim_def: dict, dimension_sql_lookup: dict[str, str] | name=name, type=sidemantic_type, sql=sql, + granularity=granularity, description=dim_def.get("description"), label=dim_def.get("label"), value_format_name=dim_def.get("value_format_name"), @@ -1396,7 +1623,9 @@ def _parse_dimension_group( # Timeframes that truncate a timestamp to a coarser time grain. These keep # type="time" with a Sidemantic granularity so they behave as time dimensions. _TIME_GRANULARITY_TIMEFRAMES = { - "time": "hour", + # Looker's "time" timeframe keeps full timestamp precision (to the second); + # truncating to the hour silently collapses sub-hour rows. + "time": "second", "time_of_day": "hour", "hour": "hour", "minute": "minute", @@ -1456,7 +1685,10 @@ def _build_timeframe_dimension( label = dim_group_def.get("label") description = dim_group_def.get("description") - # Time-truncation timeframes -> time dimension with granularity. + # Time-truncation timeframes -> time dimension with granularity. Remember the + # original LookML timeframe so export can round-trip it exactly: several + # timeframes (e.g. "time" and "second") map to the same sidemantic granularity + # and would otherwise collapse to one on export. granularity = self._TIME_GRANULARITY_TIMEFRAMES.get(timeframe) if granularity is not None: return Dimension( @@ -1466,6 +1698,7 @@ def _build_timeframe_dimension( granularity=granularity, label=label, description=description, + meta={"lookml_timeframe": timeframe}, ) # Fiscal quarter/year truncations honoring fiscal_month_offset. The base @@ -2738,6 +2971,145 @@ def _fold_filters_into_aggregate(cls, agg_sql: str, filters: list[str], model: M return None return f"{func}(CASE WHEN {conds} THEN {arg} END)" + # DATE_TRUNC('', ) emitted for collision time dimensions, and matched + # on import to recover the grain. The grain is the first capture; expr is the rest. + _DATE_TRUNC_RE = re.compile(r"(?is)^\s*DATE_TRUNC\(\s*'(\w+)'\s*,\s*(.+)\)\s*$") + # minute15 / minute30 bucket every N minutes -- sidemantic has no such grain (it + # stores them as `minute` + meta), and DATE_TRUNC('minute', ...) loses the bucket. + # The collision export emits the expression below; this regex recovers (src, N). + _MINUTE_BUCKET_TF = {"minute15": 15, "minute30": 30} + # Sub-second LookML timeframes finer than sidemantic's `second` grain but valid as + # DATE_TRUNC units; the collision export truncates at these and import recovers them. + _SUBSECOND_TF = frozenset({"millisecond", "microsecond"}) + # LookML timeframes that map MANY-to-one onto a coarser/finer sidemantic grain (15/30-min + # buckets -> minute, sub-second -> second, time-of-day extraction -> hour). A native dim's + # NAME suffix must NOT infer these on export: `created_minute15` at MINUTE grain is 1-minute + # data, not 15-minute buckets, so emitting `[minute15]` would silently re-bucket. They only + # round-trip when preserved in meta['lookml_timeframe'] (the import path). + _INEXACT_NAME_TF = frozenset(_MINUTE_BUCKET_TF) | _SUBSECOND_TF | {"time_of_day"} + # Inexact timeframes whose exact semantics live in a SPECIAL SQL FORM (15/30-min bucket + # expression, sub-second DATE_TRUNC unit). If one reaches import recovery as a PLAIN + # DATE_TRUNC, that exact form was lost, so its name suffix must NOT be recorded (recording + # `minute15` on a plain minute DATE_TRUNC would re-bucket 1-minute data). `time_of_day` is + # deliberately EXCLUDED: it has no finer SQL form -- its canonical collision export IS a plain + # hour DATE_TRUNC named `*_time_of_day`, so it is recoverable (and only recoverable) by name. + _EXACT_FORM_TF = frozenset(_MINUTE_BUCKET_TF) | _SUBSECOND_TF + # Canonical EXACT LookML timeframe for each sidemantic grain (inverse of the common + # cases in _TIME_GRANULARITY_TIMEFRAMES). Used to give a suffixless collision time dim + # a recoverable name on export (e.g. `started` at hour grain -> `started_hour`). + _GRAIN_TO_TIMEFRAME = { + "second": "second", + "minute": "minute", + "hour": "hour", + "day": "date", + "week": "week", + "month": "month", + "quarter": "quarter", + "year": "year", + } + _MINUTE_BUCKET_RE = re.compile( + r"(?is)^\s*DATE_TRUNC\('hour',\s*(.+?)\)\s*\+\s*INTERVAL '1 minute'\s*\*\s*" + r"CAST\(FLOOR\(EXTRACT\(MINUTE FROM .+?\)\s*/\s*(15|30)\)\s*\*\s*(?:15|30)\s*AS INTEGER\)\s*$" + ) + + @staticmethod + def _minute_bucket_sql(src: str, n: int) -> str: + """An N-minute bucket truncation of ``src`` (N = 15 or 30). + + Uses portable ``FLOOR(x / n) * n`` (not DuckDB-only ``//``) so the exported SQL + runs on Postgres/BigQuery/Snowflake too. + """ + return ( + f"DATE_TRUNC('hour', {src}) + INTERVAL '1 minute' * " + f"CAST(FLOOR(EXTRACT(MINUTE FROM {src}) / {n}) * {n} AS INTEGER)" + ) + + # Grains sidemantic's Dimension.granularity accepts; used to guard DATE_TRUNC recovery. + _SUPPORTED_GRAINS = frozenset({"second", "minute", "hour", "day", "week", "month", "quarter", "year"}) + # Dimension-level properties the collision-export form never emits (it writes only + # name/type/sql). Their presence marks a HAND-WRITTEN DATE_TRUNC dimension that must + # not be reclassified into a time dimension (which would drop these on round-trip). + _PRESERVED_DIM_PROPS = frozenset( + { + "hidden", + "label", + "group_label", + "description", + "order_by_field", + "value_format", + "value_format_name", + "tags", + "can_filter", + "primary_key", + "drill_fields", + } + ) + + @classmethod + def _export_collision_time_dim(cls, dim, group_sql: str | None, used_names: set | None = None) -> dict: + """Export a same-prefix time dimension (different source) as a standalone dim. + + Such a dimension cannot share its base name's dimension_group, so it is written + as a plain ``type: date_time`` dimension preserving its exact field name, with + the granularity baked into ``DATE_TRUNC`` so re-import recovers it (see + ``_parse_dimension``). If the source SQL is already a DATE_TRUNC at this grain it + is reused verbatim, keeping repeated round-trips stable (no nested truncation). + """ + src = (group_sql if group_sql is not None else dim.sql) or "" + src = src.replace("{model}", "${TABLE}") + grain = (dim.granularity or "").lower() + tf = (dim.meta or {}).get("lookml_timeframe") + # Compute the exported SQL for each class of timeframe: + n = cls._MINUTE_BUCKET_TF.get(tf) + if n is not None: + # minute15/minute30 (stored as `minute` grain + meta): emit an explicit N-minute + # bucket so Looker buckets correctly, not every minute. Reuse the dim's sql if it + # is ALREADY that bucket (recovered on a prior import) to avoid nesting. + bm = cls._MINUTE_BUCKET_RE.match(src) + sql = src if (bm and int(bm.group(2)) == n) else cls._minute_bucket_sql(src, n) + else: + # millisecond/microsecond are finer than the stored `second` grain but ARE valid + # DATE_TRUNC units, so truncate at the LookML timeframe to keep sub-second precision. + if tf in cls._SUBSECOND_TF: + grain = tf + m = cls._DATE_TRUNC_RE.match(src) + sql = src if (m and m.group(1).lower() == grain) else f"DATE_TRUNC('{grain}', {src})" + + # A standalone dim is only recoverable as a TIME dim if its name ends in a LookML + # timeframe suffix that re-import maps to this grain (see _parse_dimension). Determine + # that trailing `_` (prefer the STORED lookml_timeframe -- e.g. minute15/millisecond + # -- so bucket/sub-second forms round-trip; else a name suffix matching the grain; else + # synthesize the grain's canonical timeframe for a suffixless name like `started`, which + # would otherwise re-import categorical). `stem` is the name without that suffix. + name = dim.name + tf_suffix = None + if tf and name.endswith("_" + tf): + tf_suffix = tf + elif grain in cls._SUBSECOND_TF and name.endswith("_" + grain): + tf_suffix = grain + else: + for t, g in cls._TIME_GRANULARITY_TIMEFRAMES.items(): + if name.endswith("_" + t) and g == grain and (tf_suffix is None or len(t) > len(tf_suffix)): + tf_suffix = t + if tf_suffix is not None: + stem = name[: -(len(tf_suffix) + 1)] + else: + tf_suffix = tf or (grain if grain in cls._SUBSECOND_TF else cls._GRAIN_TO_TIMEFRAME.get(grain, "date")) + stem = name + name = f"{stem}_{tf_suffix}" + # Guarantee uniqueness against every other emitted field (a sibling standalone, a + # dimension_group's generated `_` field, a regular dimension/measure). This + # trio (`started` + `started_hour` + `started_date`, all different SQL) is otherwise + # unrepresentable; insert the disambiguator into the STEM (`started_2_hour`, NOT + # `started_hour_2`) so the trailing timeframe -- and thus time-recoverability on + # re-import -- is preserved. Valid + lossless. + if used_names is not None and name in used_names: + i = 2 + while f"{stem}_{i}_{tf_suffix}" in used_names: + i += 1 + name = f"{stem}_{i}_{tf_suffix}" + return {"name": name, "type": "date_time", "sql": sql} + def _export_view(self, model: Model, graph: SemanticGraph) -> dict: """Export model to LookML view definition. @@ -2750,10 +3122,25 @@ def _export_view(self, model: Model, graph: SemanticGraph) -> dict: """ view = {"name": model.name} + # Re-emit `extension: required` for an abstract (tableless) base so a round-trip keeps + # it non-queryable; otherwise it re-imports as an ordinary tableless view and the + # implicit-table default assigns it a physical table named after the view. Only for a + # genuinely TABLELESS view: a concrete child INHERITS `extension_required` in meta via + # inheritance but has its own table, so emitting the marker would wrongly make the + # usable child abstract on round-trip. + if (model.meta or {}).get("extension_required") and not model.table and not model.sql: + view["extension"] = "required" + if model.sql: view["derived_table"] = {"sql": model.sql} elif model.table: view["sql_table_name"] = model.table + elif (model.meta or {}).get("unsupported_derived_table"): + # Re-emit the unsupported derived_table (no `sql`) so re-import keeps it marked + # unsupported instead of defaulting it to a physical table. Prefer the retained + # raw dict; fall back to a minimal non-sql marker. + raw = (model.meta or {}).get("_unsupported_derived_table_raw") + view["derived_table"] = raw if isinstance(raw, dict) and raw else {"sql_trigger_value": "select 1"} if model.description: view["description"] = model.description @@ -2816,55 +3203,161 @@ def _export_view(self, model: Model, graph: SemanticGraph) -> dict: # Group time dimensions by base name time_dims = [d for d in model.dimensions if d.type == "time" and d.granularity] if time_dims: - # Group by base name and collect all timeframes - from collections import defaultdict + # Group by (base name, source SQL): same-prefix time dimensions backed by + # DIFFERENT columns are not one dimension_group, and merging them would + # rewire a field to the wrong source on round-trip. + from collections import Counter - base_name_groups = defaultdict(list) + base_name_groups: dict[tuple[str, str | None], list] = {} for dim in time_dims: - # Extract base name (remove _date, _week, etc suffix) + # Extract the base name by stripping the timeframe suffix. For imported + # dims, use the EXACT stored LookML timeframe (covers every mapped + # timeframe, e.g. millisecond/microsecond, not just the common few); + # for native dims fall back to the common suffix list. base_name = dim.name - for suffix in ["_date", "_week", "_month", "_quarter", "_year", "_time", "_hour"]: - if dim.name.endswith(suffix): - base_name = dim.name[: -len(suffix)] - break - base_name_groups[base_name].append(dim) + stored_tf = (dim.meta or {}).get("lookml_timeframe") + if stored_tf and dim.name.endswith("_" + stored_tf): + base_name = dim.name[: -(len(stored_tf) + 1)] + else: + # Native dims (no stored timeframe): strip any known LookML truncation + # timeframe suffix, LONGEST first so e.g. `_minute15`/`_time_of_day`/ + # `_millisecond` win over `_minute`/`_time`/`_second` (else the field + # re-imports renamed, e.g. created_minute15 -> created_minute15_minute). + for tf in sorted(self._TIME_GRANULARITY_TIMEFRAMES, key=len, reverse=True): + if dim.name.endswith("_" + tf): + base_name = dim.name[: -(len(tf) + 1)] + break + base_name_groups.setdefault((base_name, dim.sql), []).append(dim) + + # When one base name spans multiple source SQLs, only ONE can be the + # dimension_group (a second `dimension_group: ` is illegal LookML); + # the rest are emitted as standalone DATE_TRUNC dimensions below so their + # field names round-trip exactly instead of being renamed. + base_name_counts = Counter(bn for (bn, _) in base_name_groups) + assigned_names: set[str] = set() + + # Map granularity to timeframe. Used as a fallback for dimensions not + # imported from LookML (which carry no meta['lookml_timeframe']); a + # second-grain dimension maps to the LookML `second` timeframe so native + # `*_second` fields round-trip instead of collapsing to `time`. + granularity_mapping = { + "second": "second", + "minute": "minute", + "hour": "hour", + "day": "date", + "week": "week", + "month": "month", + "quarter": "quarter", + "year": "year", + } dimension_groups = [] - for base_name, dims in base_name_groups.items(): - # Map granularity to timeframe - granularity_mapping = { - "hour": "time", - "day": "date", - "week": "week", - "month": "month", - "quarter": "quarter", - "year": "year", - } + # Same-prefix time dimensions backed by a DIFFERENT source column can't all + # be dimension_groups: a second `dimension_group: ` is illegal LookML, + # and renaming it (started_2) would rename its fields (started_minute -> + # started_2_minute) and break every reference. The first source keeps the + # dimension_group; the rest are emitted as standalone dimensions that + # preserve the exact field name (granularity baked into DATE_TRUNC so the + # importer recovers it losslessly). + collision_dims = [] + # Field names already taken by fields NOT produced by this time-dim pass (regular + # dimensions + measures). Time dims get their emitted names added below as each + # dimension_group (base + generated `_` fields) and collision standalone + # is built, so a collision standalone never duplicates a group-generated field, + # another standalone, or a regular field (see _export_collision_time_dim). + _time_names = {d.name for d in time_dims} + used_names: set[str] = {d.name for d in model.dimensions if d.name not in _time_names} + used_names |= {m.name for m in model.metrics} + # Choose the dimension_group winner within a colliding base so field names map to + # the RIGHT source: (1) DEPRIORITIZE a suffixless representative -- a suffixless + # `started` winning the group generates `started_` (e.g. started_hour) from + # ITS OWN sql, mis-mapping the field that an existing sibling `started_hour` should + # own; letting the suffixed sibling win means the group's generated `started_hour` + # comes from that dim's source (and the suffixless one goes standalone, renamed into + # its stem, e.g. started_2_hour). (2) Prefer a clean (non-DATE_TRUNC) source so + # repeated round-trips don't nest truncations. (3) SQL order for stability. + ordered_groups = sorted( + base_name_groups.items(), + key=lambda kv: ( + kv[0][0], + 1 if any(d.name == kv[0][0] for d in kv[1]) else 0, + 1 if self._DATE_TRUNC_RE.match(kv[0][1] or "") else 0, + kv[0][1] or "", + ), + ) - # Collect all timeframes for this base name + def _group_timeframes(dims): + # De-duplicated LookML timeframes for a dimension_group's dims. LookML's + # "time" and "second" both import to second granularity, so dedup avoids + # emitting [time, time] and dropping a field on re-import. timeframes = [] - sql = None + seen_timeframes = set() for dim in dims: - timeframe = granularity_mapping.get(dim.granularity, "date") - timeframes.append(timeframe) - if dim.sql and not sql: - sql = dim.sql + # Prefer the original LookML timeframe captured at import (so + # "time"/"second"/etc. round-trip distinctly). + timeframe = (dim.meta or {}).get("lookml_timeframe") + if timeframe is None: + # Native (non-import) dim: derive from the field-name suffix (longest + # known timeframe) so the EXACT name round-trips (created_hour -> [hour]) + # -- but ONLY when that suffix is an EXACT one-to-one match for the grain. + # Skip the inexact/bucketing timeframes (minute15/30, milli/microsecond, + # time_of_day): their coarse mapping equals the grain, so a native + # created_minute15 at MINUTE grain would wrongly export [minute15]. Also + # skip a suffix that CONTRADICTS the grain. Otherwise fall back to grain. + for tf in sorted(self._TIME_GRANULARITY_TIMEFRAMES, key=len, reverse=True): + if ( + tf not in self._INEXACT_NAME_TF + and dim.name.endswith("_" + tf) + and self._TIME_GRANULARITY_TIMEFRAMES[tf] == dim.granularity + ): + timeframe = tf + break + if timeframe is None: + timeframe = granularity_mapping.get(dim.granularity, "date") + if timeframe not in seen_timeframes: + seen_timeframes.add(timeframe) + timeframes.append(timeframe) + return timeframes + + # PRE-SEED used_names with every field ANY dimension_group will generate BEFORE + # emitting collision standalones. The group winner is the FIRST entry per base; a + # standalone processed earlier must not pick a disambiguated name (`started_2_hour`) + # that a LATER group (`started_2`) also generates. Without this pre-pass the + # incremental seeding misses future groups' outputs. + _seeded_bases: set[str] = set() + for (base_name, _group_sql), dims in ordered_groups: + if base_name in _seeded_bases: + continue + _seeded_bases.add(base_name) + used_names.add(base_name) + used_names.update(f"{base_name}_{tf}" for tf in _group_timeframes(dims)) + + for (base_name, group_sql), dims in ordered_groups: + if base_name_counts[base_name] > 1 and base_name in assigned_names: + for dim in dims: + cdim = self._export_collision_time_dim(dim, group_sql, used_names) + used_names.add(cdim["name"]) + collision_dims.append(cdim) + continue + group_name = base_name + assigned_names.add(group_name) dim_group_def = { - "name": base_name, + "name": group_name, "type": "time", - "timeframes": timeframes, + "timeframes": _group_timeframes(dims), } - if sql: - sql = sql.replace("{model}", "${TABLE}") - dim_group_def["sql"] = sql + if group_sql: + dim_group_def["sql"] = group_sql.replace("{model}", "${TABLE}") dimension_groups.append(dim_group_def) if dimension_groups: view["dimension_groups"] = dimension_groups + if collision_dims: + view.setdefault("dimensions", []).extend(collision_dims) # Export measures from sidemantic.sql.aggregation_detection import sql_has_aggregate as _sql_has_aggregate diff --git a/sidemantic/loaders.py b/sidemantic/loaders.py index e17df1f0..97feb865 100644 --- a/sidemantic/loaders.py +++ b/sidemantic/loaders.py @@ -13,6 +13,60 @@ from sidemantic.core.semantic_layer import SemanticLayer +def _is_registerable_model(model) -> bool: + """True unless ``model`` is an INTENTIONAL non-queryable template that should be + skipped rather than registered/validated. + + Only LookML's ``extension: required`` abstract bases and unsupported derived tables + (kept in the parsed graph as tableless templates) are skipped -- ``add_model`` would + reject them and abort loading an otherwise-valid project. The skip keys off the + PARSER-OWNED ``lookml_template`` marker (set by the LookML adapter), NOT the public + ``extension_required``/``unsupported_derived_table`` keys, so a native or other-format + tableless model that merely carries those user-facing keys still surfaces its real + missing-source validation error. Any OTHER tableless model (e.g. an erroneous Hex view + missing its base) is likewise left in. + """ + if model.table or model.sql or getattr(model, "dax", None) or getattr(model, "source_uri", None): + return True + return not (model.meta or {}).get("lookml_template") + + +def _drop_non_registerable_models(all_models: dict, all_metrics: dict | None = None) -> dict: + """Filter to registerable models AND strip any relationship targeting a dropped one. + + An explore may have already added a relationship pointing at a now-skipped template + (e.g. an `extension: required` join target); leaving it would dangle and fail + ``sidemantic validate``. Only relationships to models THIS function drops are removed + -- a relationship to a never-defined model is a real error that validation must still + surface, so it is left intact. Returns the filtered dict (mutating survivors' rels). + + When ``all_metrics`` is given, also drop graph-level metrics contributed by a dropped + model -- ``SemanticGraph.add_model()`` auto-registers a model's ``time_comparison`` / + ``conversion`` measures into the graph, so a ``period_over_period`` measure on an + ``extension: required`` template would otherwise linger as a graph metric whose base + model is gone, breaking ``compile``/``info``. A metric still defined on a SURVIVING + model is kept. + """ + kept = {name: model for name, model in all_models.items() if _is_registerable_model(model)} + dropped = set(all_models) - set(kept) + for model in kept.values(): + rels = getattr(model, "relationships", None) + if rels: + model.relationships = [r for r in rels if r.name not in dropped] + if all_metrics is not None and dropped: + # add_model auto-registers a model's GRAPH-LEVEL measures (time_comparison/conversion) + # into the graph; a dropped template's such measure would linger in all_metrics with + # its base model gone. Drop by PROVENANCE: the LookML parser stamps a template's graph + # measures with `_lookml_template_metric` (surviving even when a refinement reconstructs + # the Metric object). That proves the metric came from a dropped template -- unlike a + # base-ref heuristic, it never drops a same-named STANDALONE metric from another file + # (no marker) nor mistakes an unqualified base that merely exists on a surviving model. + for mn in list(all_metrics): + if (all_metrics[mn].meta or {}).get("_lookml_template_metric"): + all_metrics.pop(mn, None) + return kept + + def load_from_directory( layer: "SemanticLayer", directory: str | Path, @@ -331,10 +385,16 @@ def load_from_directory( # table pair. _apply_snowflake_pending_relationships(all_models, all_pending_relationships) + # Drop non-queryable template models (LookML `extension: required` abstract bases / + # unsupported derived tables) BEFORE inference + registration, so FK inference never + # targets a model that won't be registered (which would leave a dangling relationship) + # and add_model never rejects a template that was never meant to be queried. + all_models = _drop_non_registerable_models(all_models, all_metrics) + # Infer cross-model relationships based on naming conventions _infer_relationships(all_models) - # Add all models to the layer (now with relationships) + # Add all models to the layer (now with relationships). for model in all_models.values(): if model.name not in layer.graph.models: layer.add_model(model) @@ -420,6 +480,8 @@ def _load_sml_directory(layer: "SemanticLayer", directory: Path, all_models: dic if not hasattr(model, "_source_file"): model._source_file = str(directory) all_models.update(graph.models) + # Drop non-queryable templates (+ their dangling rels) before inference + registration. + all_models = _drop_non_registerable_models(all_models) _infer_relationships(all_models) for model in all_models.values(): if model.name not in layer.graph.models: diff --git a/tests/adapters/lookml/test_advanced_measure_types.py b/tests/adapters/lookml/test_advanced_measure_types.py index ef04d832..d61070fd 100644 --- a/tests/adapters/lookml/test_advanced_measure_types.py +++ b/tests/adapters/lookml/test_advanced_measure_types.py @@ -129,7 +129,7 @@ class TestDimensionGroupTimeframes: def test_time_truncation_timeframes_are_time(self, graph): model = graph.get_model("events_calendar") for tf, gran in [ - ("occurred_time", "hour"), + ("occurred_time", "second"), ("occurred_date", "day"), ("occurred_week", "week"), ("occurred_month", "month"), diff --git a/tests/adapters/lookml/test_edge_cases.py b/tests/adapters/lookml/test_edge_cases.py index 655b931d..a3195bb1 100644 --- a/tests/adapters/lookml/test_edge_cases.py +++ b/tests/adapters/lookml/test_edge_cases.py @@ -3145,6 +3145,1528 @@ def test_lookml_self_referential_dimension_terminates(): assert selfd.sql.count("+ 1") <= 2 +def test_lookml_view_with_unsupported_derived_table_not_defaulted(): + """A view with a derived_table the adapter can't read must NOT get a default physical table.""" + adapter = LookMLAdapter() + model = adapter._parse_view( + { + "name": "ndt_unknown", + # derived_table with no sql / explore_source the adapter understands + "derived_table": {"persist_for": "1 hour"}, + "dimensions": [{"name": "id", "type": "number", "sql": "${TABLE}.id"}], + } + ) + assert model.table is None # not fabricated as a physical table named after the view + + +def test_lookml_time_timeframe_keeps_second_precision(): + """Looker's "time" timeframe keeps to-the-second precision, not hour.""" + graph = _parse_lkml( + """ +view: v { + sql_table_name: t ;; + dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } + dimension_group: created { + type: time + timeframes: [time, date, minute] + sql: ${TABLE}.created_at ;; + } +} +""" + ) + model = graph.get_model("v") + assert model.get_dimension("created_time").granularity == "second" + assert model.get_dimension("created_minute").granularity == "minute" + assert model.get_dimension("created_date").granularity == "day" + + +def test_lookml_native_suffix_contradicting_grain_uses_grain(): + """A native dim whose name suffix CONTRADICTS its granularity preserves the GRAIN, not the name. + + created_time at hour granularity must export as [hour] (-> created_hour), not [time] + (which imports as second), so queries keep grouping by hour. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="created_time", type="time", granularity="hour", sql="ts"), # suffix `time` != hour + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + grains = { + d.name: d.granularity for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time" + } + assert grains == {"created_hour": "hour"} # grain preserved (not silently downgraded to second) + + +def test_lookml_native_minute15_name_does_not_widen_grain(): + """A native created_minute15 at MINUTE grain must not export [minute15] (15-min buckets). + + The coarse mapping of minute15 equals `minute`, so the old name-suffix inference emitted + [minute15] -- silently re-bucketing 1-minute data into 15-minute intervals. Inexact + suffixes are now excluded from name inference, so it exports [minute] (true grain). + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="created_minute15", type="time", granularity="minute", sql="ts"), # no meta + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + txt = open(out).read() + assert "minute15" not in txt # no silent grain widening + assert "minute" in txt + # meta-preserved minute15 still round-trips as a 15-min bucket (import path unaffected) + graph2 = SemanticGraph() + graph2.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension( + name="created_minute15", + type="time", + granularity="minute", + sql="ts", + meta={"lookml_timeframe": "minute15"}, + ), + ], + ) + ) + out2 = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph2, out2) + assert "minute15" in open(out2).read() + + +def test_lookml_native_inexact_timeframe_suffix_preserves_grain_not_name(): + """NATIVE inexact-suffix time dims (no meta) preserve the GRAIN, even if the name changes. + + minute15 / minute30 / millisecond / microsecond / time_of_day map MANY-to-one onto a + coarser-or-finer sidemantic grain, so a native created_minute15 at MINUTE grain must NOT + export [minute15] (which buckets into 15-min intervals -- a silent grain change). It + exports at the true grain instead, so the field re-imports renamed (created_minute) but + with the CORRECT queryable grain. Exact name round-trip for these needs preserved + meta['lookml_timeframe'] (the import path), covered separately. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + for name, grain in [ + ("created_minute15", "minute"), + ("created_millisecond", "second"), + ("created_time_of_day", "hour"), + ]: + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name=name, type="time", granularity=grain, sql="ts"), # no meta + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + grains = [d.granularity for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time"] + assert grains == [grain], f"{name}: grain not preserved -> {grains}" + + +def test_lookml_uncommon_timeframe_suffix_roundtrips(): + """Imported timeframes like millisecond/microsecond round-trip (base derived from stored timeframe).""" + import tempfile + + graph = _parse_lkml( + """ +view: v { + sql_table_name: t ;; + dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } + dimension_group: created { + type: time + timeframes: [millisecond, microsecond, date] + sql: ${TABLE}.created_at ;; + } +} +""" + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + names = {d.name for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time"} + assert {"created_millisecond", "created_microsecond"} <= names + assert not any(n.endswith("_millisecond_millisecond") or n.endswith("_microsecond_microsecond") for n in names) + + +def test_lookml_time_grain_roundtrip(): + """time/hour/minute/date grains round-trip through export without grain or suffix corruption.""" + import tempfile + + graph = _parse_lkml( + """ +view: v { + sql_table_name: t ;; + dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } + dimension_group: created { + type: time + timeframes: [time, hour, minute, date] + sql: ${TABLE}.created_at ;; + } +} +""" + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + grains = { + d.name: d.granularity for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time" + } + assert grains == { + "created_time": "second", + "created_hour": "hour", + "created_minute": "minute", + "created_date": "day", + } + + +def test_lookml_native_second_grain_roundtrips(): + """A second-grain dimension not imported from LookML (no meta) exports as `second`, not `time`.""" + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="created_second", type="time", granularity="second", sql="{model}.created_at"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + names = {d.name for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time"} + assert "created_second" in names + + +def test_lookml_same_prefix_time_dims_different_sources_not_merged(): + """Same-prefix time dims backed by different columns must not merge to one group (wrong source).""" + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started_date", type="time", granularity="day", sql="{model}.started_at"), + Dimension(name="started_second", type="time", granularity="second", sql="{model}.other_at"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + time_dims = [d for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time"] + sources = {d.sql for d in time_dims} + # Each field keeps its own source column (no rewiring to the other group's SQL). + assert any("started_at" in (s or "") for s in sources) + assert any("other_at" in (s or "") for s in sources) + # Split groups get suffix-free, collision-free names (no started_date_date etc.). + assert not any(d.name.endswith("_date_date") or d.name.endswith("_hour_hour") for d in time_dims) + + +def test_lookml_collision_subsecond_timeframe_preserves_precision(): + """A millisecond/microsecond collision dim exports DATE_TRUNC at that grain (not second). + + sidemantic stores them as `second`, so the exported SQL must use the LookML timeframe + grain to keep sub-second precision, and re-import recovers second grain + the timeframe. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension( + name="started_date", type="time", granularity="day", sql="a", meta={"lookml_timeframe": "date"} + ), + Dimension( + name="started_millisecond", + type="time", + granularity="second", + sql="b", + meta={"lookml_timeframe": "millisecond"}, + ), + ], + ) + ) + adapter = LookMLAdapter() + out = tempfile.mktemp(suffix=".lkml") + adapter.export(graph, out) + assert "DATE_TRUNC('millisecond'" in open(out).read() # sub-second precision kept, not 'second' + g = adapter.parse(Path(out)) + ms = g.get_model("v").get_dimension("started_millisecond") + assert ms.granularity == "second" and (ms.meta or {}).get("lookml_timeframe") == "millisecond" + # second round-trip stable (no nested DATE_TRUNC) + out2 = tempfile.mktemp(suffix=".lkml") + adapter.export(g, out2) + assert "DATE_TRUNC('millisecond', DATE_TRUNC" not in open(out2).read() + + +def test_lookml_collision_disambiguation_reserves_future_group_fields(): + """Disambiguating a standalone must avoid names a LATER dimension_group will generate. + + `started` + `started_hour` (base `started`) plus `started_2_hour` (base `started_2`): the + standalone must not disambiguate to `started_2_hour` because the `started_2` group also + generates it. used_names is pre-seeded with every group's generated fields, so it skips to + `started_3_hour` -- no duplicate LookML field across the whole view. + """ + import re + import tempfile + from collections import Counter + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started", type="time", granularity="hour", sql="a"), + Dimension(name="started_hour", type="time", granularity="hour", sql="b"), + Dimension(name="started_2_hour", type="time", granularity="hour", sql="c"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + text = open(out).read() + names = re.findall(r"\n dimension: (\w+) \{", text) + for base, tfs in re.findall(r"dimension_group: (\w+) \{[^}]*?timeframes:\s*\[([^\]]*)\]", text, re.S): + names += [f"{base}_{tf.strip()}" for tf in tfs.split(",")] + dups = {n: c for n, c in Counter(names).items() if c > 1} + assert not dups, f"duplicate field names: {dups}" + + +def test_lookml_minute_bucket_collision_field_deduplicated(): + """A minute15/30 collision dim must go through the same uniqueness logic (no duplicate names). + + The minute-bucket path previously returned early, bypassing used_names dedup, so a + suffixless `started` (minute15) colliding with an existing `started_minute15` could emit + two `started_minute15` fields. It now disambiguates (`started_2_minute15`), still recoverable. + """ + import re + import tempfile + from collections import Counter + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension( + name="started", type="time", granularity="minute", sql="a", meta={"lookml_timeframe": "minute15"} + ), + Dimension( + name="started_minute15", + type="time", + granularity="minute", + sql="b", + meta={"lookml_timeframe": "minute15"}, + ), + Dimension( + name="started_date", type="time", granularity="day", sql="c", meta={"lookml_timeframe": "date"} + ), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + text = open(out).read() + names = re.findall(r"\n dimension: (\w+) \{", text) + for base, tfs in re.findall(r"dimension_group: (\w+) \{[^}]*?timeframes:\s*\[([^\]]*)\]", text, re.S): + names += [f"{base}_{tf.strip()}" for tf in tfs.split(",")] + dups = {n: c for n, c in Counter(names).items() if c > 1} + assert not dups, f"duplicate field names: {dups}" + # the disambiguated field is still time-recoverable (ends in a real timeframe suffix) + g2 = LookMLAdapter().parse(Path(out)) + assert all(d.type == "time" for d in g2.get_model("ev").dimensions if d.name.startswith("started")) + + +def test_lookml_collision_disambiguated_field_stays_time_recoverable(): + """A disambiguated collision field must stay TIME-recoverable, not become categorical. + + The unrepresentable trio (`started` + `started_hour` + `started_date`, all different SQL) + forces one field to be renamed for uniqueness. The disambiguator goes into the STEM + (`started_2_hour`, not `started_hour_2`) so the trailing timeframe is preserved and the + field re-imports as time@hour, not a categorical dim. Stable across round-trips. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started", type="time", granularity="hour", sql="a"), + Dimension(name="started_hour", type="time", granularity="hour", sql="b"), + Dimension(name="started_date", type="time", granularity="day", sql="c"), + ], + ) + ) + adapter = LookMLAdapter() + g = graph + for _ in range(2): + out = tempfile.mktemp(suffix=".lkml") + adapter.export(g, out) + g = adapter.parse(Path(out)) + kinds = {d.name: d.type for d in g.get_model("ev").dimensions if d.name != "id"} + # all three time sources stay TIME (none degraded to categorical), 3 distinct names + assert all(t == "time" for t in kinds.values()), kinds + assert len(kinds) == 3, kinds + + +def test_lookml_suffixless_collision_no_dimension_group_field_duplicate(): + """A suffixless dim must not win the group slot and GENERATE a name a sibling standalone owns. + + `started` (hour) + `started_hour` (hour, diff SQL) + `started_date`: if suffixless `started` + won the dimension_group, the group would generate `started_hour` AND a standalone + `started_hour` would exist -> a runtime field collision. A suffixed sibling wins the group + slot instead, so every generated field name (group fields + standalones) is unique. + """ + import re + import tempfile + from collections import Counter + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started", type="time", granularity="hour", sql="a"), + Dimension(name="started_hour", type="time", granularity="hour", sql="b"), + Dimension(name="started_date", type="time", granularity="day", sql="c"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + text = open(out).read() + generated = re.findall(r"\n dimension: (\w+) \{", text) # standalones + for base, tfs in re.findall(r"dimension_group: (\w+) \{[^}]*?timeframes:\s*\[([^\]]*)\]", text, re.S): + generated += [f"{base}_{tf.strip()}" for tf in tfs.split(",")] # group-generated fields + dups = {n: c for n, c in Counter(generated).items() if c > 1} + assert not dups, f"duplicate generated field names: {dups}" + + +def test_lookml_suffixless_collision_group_field_maps_to_owning_source(): + """The generated group field must map to the source that ORIGINALLY owned that field name. + + `started` (hour, source A) + `started_hour` (hour, source B): the `dimension_group` slot must + go to the suffixed `started_hour` so the group's generated `started_hour` field comes from + source B (its own source). If the suffixless `started` won the slot, the group would generate + `started_hour` from source A while the real `started_hour` got renamed to `started_2_hour` -- + so a round-tripped query for `started_hour` would silently read the WRONG column. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started", type="time", granularity="hour", sql="src_suffixless"), + Dimension(name="started_hour", type="time", granularity="hour", sql="src_started_hour"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + reloaded = LookMLAdapter().parse(out).get_model("ev") + by_name = {d.name: d.sql for d in reloaded.dimensions} + # started_hour must still resolve to its own source, not the suffixless dim's source. + assert "src_started_hour" in (by_name.get("started_hour") or ""), by_name + assert "src_suffixless" not in (by_name.get("started_hour") or ""), by_name + # The suffixless dim's source survives under a distinct, time-recoverable name. + assert any("src_suffixless" in (s or "") for n, s in by_name.items() if n != "started_hour"), by_name + + +def test_lookml_suffixless_collision_synth_name_avoids_duplicate(): + """Synthesizing a recoverable suffix must not duplicate an existing field name. + + `started` (hour) alongside an EXISTING `started_hour` and a `started_date` source must not + emit two `started_hour` blocks; the suffixless one keeps its name (no data-losing dup). + """ + import re + import tempfile + from collections import Counter + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started", type="time", granularity="hour", sql="a"), + Dimension(name="started_hour", type="time", granularity="hour", sql="b"), # already exists + Dimension(name="started_date", type="time", granularity="day", sql="c"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + names = re.findall(r"dimension(?:_group)?: (\w+)", open(out).read()) + dups = {n: c for n, c in Counter(names).items() if c > 1} + assert not dups, f"duplicate field names emitted: {dups}" + + +def test_lookml_suffixless_collision_time_dim_stays_time(): + """A suffixless collision time dim must remain a TIME dim across round-trips, not categorical. + + `started` (hour grain) colliding with `started_date` (day, different SQL) can't keep its + bare name as a recoverable standalone, so the collision export appends the grain timeframe + (-> `started_hour`) so re-import restores time granularity instead of dropping to a + categorical dim (which would break time queries/filters on the field). + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started", type="time", granularity="hour", sql="created_at"), # suffixless + Dimension(name="started_date", type="time", granularity="day", sql="updated_at"), + ], + ) + ) + adapter = LookMLAdapter() + g = graph + for _ in range(2): + out = tempfile.mktemp(suffix=".lkml") + adapter.export(g, out) + g = adapter.parse(Path(out)) + times = {d.name: d.granularity for d in g.get_model("ev").dimensions if d.type == "time"} + # the suffixless dim is recoverable as time@hour (renamed started -> started_hour) + assert times.get("started_hour") == "hour", times + assert times.get("started_date") == "day", times + + +def test_lookml_native_collision_inexact_name_not_widened_on_import(): + """A NATIVE collision dim named *_minute15 (no meta) must not gain false minute15 meta. + + Without meta the collision exporter writes a plain DATE_TRUNC('minute', ...), so import + must recover grain=minute with NO lookml_timeframe (recording 'minute15' would make the + next export widen the 1-minute dim into a 15-minute bucket). Stable across round-trips. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started_date", type="time", granularity="day", sql="created_at"), + # same prefix, DIFFERENT sql -> collision; NO meta + Dimension(name="started_minute15", type="time", granularity="minute", sql="updated_at"), + ], + ) + ) + adapter = LookMLAdapter() + out = tempfile.mktemp(suffix=".lkml") + adapter.export(graph, out) + assert "DATE_TRUNC('minute'" in open(out).read() # plain minute trunc, not a 15-min bucket + g = adapter.parse(Path(out)) + m15 = g.get_model("ev").get_dimension("started_minute15") + assert m15.granularity == "minute" + assert not (m15.meta or {}).get("lookml_timeframe") # NO false minute15 meta + # stable: second round-trip still minute, still no bucket + out2 = tempfile.mktemp(suffix=".lkml") + adapter.export(g, out2) + assert "FLOOR(" not in open(out2).read() # never widened into a bucket + m15b = adapter.parse(Path(out2)).get_model("ev").get_dimension("started_minute15") + assert m15b.granularity == "minute" and not (m15b.meta or {}).get("lookml_timeframe") + + +def test_lookml_collision_minute15_bucket_roundtrips(): + """A minute15 collision dim exports a faithful 15-min bucket (not minute) and round-trips. + + sidemantic has no 15-min grain (stores minute15 as `minute` + meta), so the standalone + collision form must emit an explicit N-minute bucket that the importer recovers back to + grain=minute + meta['lookml_timeframe']='minute15', stable across repeated round-trips. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension( + name="started_date", type="time", granularity="day", sql="a", meta={"lookml_timeframe": "date"} + ), + Dimension( + name="started_minute15", + type="time", + granularity="minute", + sql="b", + meta={"lookml_timeframe": "minute15"}, + ), + ], + ) + ) + adapter = LookMLAdapter() + out = tempfile.mktemp(suffix=".lkml") + adapter.export(graph, out) + text = open(out).read() + # Faithful 15-minute bucket via PORTABLE FLOOR (not DuckDB-only //), not DATE_TRUNC('minute'). + assert "FLOOR(" in text and " / 15) * 15" in text and "//" not in text + g = adapter.parse(Path(out)) + m15 = g.get_model("v").get_dimension("started_minute15") + assert m15.granularity == "minute" and (m15.meta or {}).get("lookml_timeframe") == "minute15" + # The recovered dim KEEPS the bucket expression (so QUERIES bucket 15-min, not minute). + assert "FLOOR(" in (m15.sql or "") + # second round-trip stays stable (no nested bucket) + out2 = tempfile.mktemp(suffix=".lkml") + adapter.export(g, out2) + assert "FLOOR(EXTRACT(MINUTE FROM DATE_TRUNC('hour'" not in open(out2).read() # no FLOOR(...bucket...) + m15b = adapter.parse(Path(out2)).get_model("v").get_dimension("started_minute15") + assert m15b.granularity == "minute" and (m15b.meta or {}).get("lookml_timeframe") == "minute15" + + +def test_lookml_collision_minute15_query_buckets_by_15_minutes(): + """A round-tripped minute15 collision dim must GROUP queries by 15-minute buckets, not minute.""" + import tempfile + + import duckdb + + from sidemantic import Dimension, Metric, Model, SemanticLayer + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="v", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension( + name="started_date", type="time", granularity="day", sql="a", meta={"lookml_timeframe": "date"} + ), + Dimension( + name="started_minute15", + type="time", + granularity="minute", + sql="ts", + meta={"lookml_timeframe": "minute15"}, + ), + ], + metrics=[Metric(name="cnt", agg="count")], + ) + ) + adapter = LookMLAdapter() + out = tempfile.mktemp(suffix=".lkml") + adapter.export(graph, out) + g = adapter.parse(Path(out)) + layer = SemanticLayer(auto_register=False) + for m in g.models.values(): + layer.add_model(m) + sql = layer.compile(dimensions=["v.started_minute15"], metrics=["v.cnt"]) + con = duckdb.connect() + con.execute( + "create table v as select 1 id, TIMESTAMP '2024-01-01 10:07' a, TIMESTAMP '2024-01-01 10:07' ts " + "union all select 2, TIMESTAMP '2024-01-01 10:11', TIMESTAMP '2024-01-01 10:11' " + "union all select 3, TIMESTAMP '2024-01-01 10:22', TIMESTAMP '2024-01-01 10:22'" + ) + import datetime + + rows = sorted(con.execute(sql).fetchall()) + assert rows == [(datetime.datetime(2024, 1, 1, 10, 0), 2), (datetime.datetime(2024, 1, 1, 10, 15), 1)] + + +def test_lookml_collision_time_dim_multiword_timeframe_suffix_recovered(): + """A collision standalone dim with a MULTI-WORD timeframe suffix (time_of_day) recovers. + + The grain suffix is matched against the longest known LookML timeframe, not just the + text after the last underscore, so started_time_of_day round-trips as a time dimension + (gran=hour), not a renamed categorical. + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension( + name="started_date", type="time", granularity="day", sql="a", meta={"lookml_timeframe": "date"} + ), + Dimension( + name="started_time_of_day", + type="time", + granularity="hour", + sql="b", + meta={"lookml_timeframe": "time_of_day"}, + ), + ], + ) + ) + adapter = LookMLAdapter() + # Repeated round-trips must be STABLE: the recovered timeframe is stored in meta so + # the next export strips the _time_of_day suffix instead of renaming the field. + g = graph + for _ in range(3): + out = tempfile.mktemp(suffix=".lkml") + adapter.export(g, out) + g = adapter.parse(Path(out)) + grains = {d.name: d.granularity for d in g.get_model("v").dimensions if d.type == "time"} + assert grains == {"started_date": "day", "started_time_of_day": "hour"} + + +def test_lookml_collision_time_of_day_recovers_timeframe_meta_and_survives_sibling_drop(): + """A collision-exported `time_of_day` standalone must recover its timeframe into meta. + + time_of_day has no finer SQL form -- its collision export is a PLAIN hour DATE_TRUNC named + `*_time_of_day`. On import the recovery guard must still store `lookml_timeframe=time_of_day` + (it was wrongly grouped with the exact-form minute-bucket / sub-second timeframes and dropped). + Without the meta, a later export WITHOUT a colliding sibling loses the timeframe and renames the + field to `*_hour`. This differs from the multiword test above, where the collision itself masks + the loss via name preservation. + """ + import re + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + adapter = LookMLAdapter() + graph = SemanticGraph() + graph.add_model( + Model( + name="ev", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension( + name="started", type="time", granularity="hour", sql="a", meta={"lookml_timeframe": "time_of_day"} + ), + Dimension(name="started_time_of_day", type="time", granularity="hour", sql="b"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + adapter.export(graph, out) + reimported = adapter.parse(Path(out)).get_model("ev") + tf = {d.name: (d.meta or {}).get("lookml_timeframe") for d in reimported.dimensions} + assert tf.get("started_time_of_day") == "time_of_day", tf # recovered, not dropped + + # Drop the colliding sibling so the time_of_day field exports ALONE: it must keep its name, + # NOT fall back to the hour timeframe and rename to started_hour. + reimported.dimensions = [d for d in reimported.dimensions if d.name != "started_hour"] + solo = SemanticGraph() + solo.add_model(reimported) + out2 = tempfile.mktemp(suffix=".lkml") + adapter.export(solo, out2) + text2 = open(out2).read() + names = re.findall(r"\n dimension: (\w+) \{", text2) + for base, tfs in re.findall(r"dimension_group: (\w+) \{[^}]*?timeframes:\s*\[([^\]]*)\]", text2, re.S): + names += [f"{base}_{x.strip()}" for x in tfs.split(",")] + assert "started_time_of_day" in names, names + assert "started_hour" not in names, names # no bogus rename + + +def test_lookml_same_prefix_time_dims_roundtrip_names_and_grains_losslessly(): + """Collision time dims must round-trip with EXACT names AND granularities. + + Two same-prefix time dims on different sources can't both be dimension_groups, so + the extra one is exported as a standalone DATE_TRUNC dimension. Both the field name + and the granularity must survive re-import (no started_2_minute rename, no grain + loss), and a second round-trip must be stable (no nested DATE_TRUNC). + """ + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id", primary_key=True), + Dimension(name="started_hour", type="time", granularity="hour", sql="started_at"), + Dimension(name="started_minute", type="time", granularity="minute", sql="completed_at"), + ], + ) + ) + adapter = LookMLAdapter() + out1 = tempfile.mktemp(suffix=".lkml") + adapter.export(graph, out1) + g2 = adapter.parse(Path(out1)) + grains = {d.name: d.granularity for d in g2.get_model("v").dimensions if d.type == "time"} + assert grains == {"started_hour": "hour", "started_minute": "minute"} + + # Second round-trip is stable and never nests DATE_TRUNC. + out2 = tempfile.mktemp(suffix=".lkml") + adapter.export(g2, out2) + assert "DATE_TRUNC('minute', DATE_TRUNC" not in open(out2).read() + assert "DATE_TRUNC('hour', DATE_TRUNC" not in open(out2).read() + grains2 = {d.name: d.granularity for d in adapter.parse(Path(out2)).get_model("v").dimensions if d.type == "time"} + assert grains2 == {"started_hour": "hour", "started_minute": "minute"} + + +def test_lookml_handwritten_date_trunc_dimension_not_hijacked(): + """A hand-written DATE_TRUNC dimension must keep its name/type, not be turned into a time group. + + The DATE_TRUNC granularity recovery is only for the collision-export form (name ends + in _); an arbitrary-named dim like `created` stays categorical (no rename to + created_date), and an unsupported dialect grain like 'isoweek' must not crash parsing. + """ + import tempfile + + # Unsupported grain must not raise a granularity validation error. + graph = _parse_lkml( + """ +view: v { + sql_table_name: t ;; + dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } + dimension: w { type: date_time sql: DATE_TRUNC('isoweek', ${TABLE}.created_at) ;; } +} +""" + ) + w = graph.get_model("v").get_dimension("w") + assert w.type == "categorical" and w.granularity is None + + # Arbitrary-named DATE_TRUNC dim keeps its exact name across a round-trip. + graph2 = _parse_lkml( + """ +view: v { + sql_table_name: t ;; + dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } + dimension: created { type: date_time sql: DATE_TRUNC('day', ${TABLE}.created_at) ;; } +} +""" + ) + assert graph2.get_model("v").get_dimension("created").type == "categorical" + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph2, out) + names = {d.name for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions} + assert "created" in names and "created_date" not in names + + # A hand-written dim whose NAME does carry a timeframe suffix (created_date) but also + # has dimension-level properties (hidden/label) is NOT the collision-export form, so + # it stays a plain dimension and those properties survive the round-trip. + graph3 = _parse_lkml( + """ +view: v { + sql_table_name: t ;; + dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } + dimension: created_date { type: date_time hidden: yes label: "Created" sql: DATE_TRUNC('day', ${TABLE}.created_at) ;; } +} +""" + ) + assert graph3.get_model("v").get_dimension("created_date").type == "categorical" + out3 = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph3, out3) + text3 = open(out3).read() + assert "created_date" in {d.name for d in LookMLAdapter().parse(Path(out3)).get_model("v").dimensions} + assert "hidden: yes" in text3 and "Created" in text3 # properties preserved + + +def test_lookml_split_time_group_names_avoid_existing_bases(): + """A synthetic split-group name must not collide with an existing time-group base.""" + import re + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="started_date", type="time", granularity="day", sql="{model}.a"), + Dimension(name="started_hour", type="time", granularity="hour", sql="{model}.b"), + Dimension(name="started_2_hour", type="time", granularity="hour", sql="{model}.c"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + names = re.findall(r"dimension_group: (\w+)", open(out).read()) + assert len(names) == len(set(names)), f"colliding dimension_group names: {names}" + + +def test_lookml_native_time_named_second_grain_roundtrips(): + """A native second-grain dim named *_time must export as `time` so the name round-trips.""" + import tempfile + + from sidemantic import Dimension, Model + from sidemantic.core.semantic_graph import SemanticGraph + + graph = SemanticGraph() + graph.add_model( + Model( + name="v", + table="t", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id"), + Dimension(name="created_time", type="time", granularity="second", sql="{model}.created_at"), + ], + ) + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + names = {d.name for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time"} + assert "created_time" in names # not renamed to created_second + + +def test_lookml_time_and_second_timeframes_no_duplicate_export(): + """A group with both `time` and `second` (both -> second grain) must not emit duplicate timeframes.""" + import re + import tempfile + + graph = _parse_lkml( + """ +view: v { + sql_table_name: t ;; + dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } + dimension_group: created { + type: time + timeframes: [time, second, hour] + sql: ${TABLE}.created_at ;; + } +} +""" + ) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + text = open(out).read() + timeframes = re.search(r"timeframes:\s*\[([^\]]*)\]", text).group(1) + parts = [p.strip() for p in timeframes.split(",")] + assert len(parts) == len(set(parts)), f"duplicate timeframes exported: {parts}" + # Both `time` and `second` are preserved (no collapse/drop) via the stored + # original timeframe, so re-import keeps every member. + names = {d.name for d in LookMLAdapter().parse(Path(out)).get_model("v").dimensions if d.type == "time"} + assert {"created_time", "created_second", "created_hour"} <= names + + +def test_lookml_view_without_table_defaults_to_view_name(): + """A view with no sql_table_name/derived_table should default its table to the view name.""" + from sidemantic import SemanticLayer + + graph = _parse_lkml( + "view: just_fields { " + "dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "measure: c { type: count } }" + ) + model = graph.get_model("just_fields") + assert model.table == "just_fields" + # And it must be a valid, queryable model (previously raised ModelValidationError). + layer = SemanticLayer() + layer.add_model(model) + assert "just_fields" in layer.compile(metrics=["just_fields.c"]) + + +def test_lookml_parse_tableless_view_inside_layer_context(): + """Parsing a tableless view inside a `with SemanticLayer()` block must not crash. + + Auto-registration fires during Model construction; the default table is applied + after parse, so parsing must suppress registration until models are complete. + """ + import tempfile + + from sidemantic import SemanticLayer + + with tempfile.NamedTemporaryFile(mode="w", suffix=".lkml", delete=False) as f: + f.write( + "view: just_fields { dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } measure: c { type: count } }" + ) + f.flush() + path = Path(f.name) + + with SemanticLayer() as layer: + LookMLAdapter().parse(path) + # The model is registered to the context layer with its defaulted table. + assert layer.graph.get_model("just_fields").table == "just_fields" + + +def test_lookml_fieldless_ordinary_view_gets_default_table(): + """An ordinary fieldless view (or one with only adapter-ignored fields) still defaults. + + Looker defaults the table name to the view name regardless of parsed fields, so a + `view: orders {}` must not be left tableless (which would fail CLI load/registration). + Abstract/unsupported templates are still skipped; this only covers ordinary views. + """ + import tempfile + + from sidemantic import SemanticLayer + from sidemantic.loaders import load_from_directory + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write("view: orders {}\nview: just_set { set: foo { fields: [a, b] } }\n") + + layer = SemanticLayer() + load_from_directory(layer, d) # must not raise the no-table ModelValidationError + assert layer.graph.get_model("orders").table == "orders" + assert layer.graph.get_model("just_set").table == "just_set" + + +def test_lookml_abstract_base_not_registered_inside_layer_context(): + """An abstract (extension: required) base must NOT be registered inside a `with` layer. + + It's intentionally tableless; deferred registration must skip it (not raise) while the + concrete child still registers with its defaulted table. + """ + import tempfile + + from sidemantic import SemanticLayer + + with tempfile.NamedTemporaryFile(mode="w", suffix=".lkml", delete=False) as f: + f.write( + "view: base { extension: required dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: x { type: number sql: ${TABLE}.x ;; } } " + "view: orders { extends: [base] dimension: y { type: number sql: ${TABLE}.y ;; } }" + ) + f.flush() + path = Path(f.name) + + with SemanticLayer() as layer: + LookMLAdapter().parse(path) # must not raise ModelValidationError + assert "base" not in layer.graph.models # abstract base skipped + assert layer.graph.get_model("orders").table == "orders" # concrete child registered + + +def test_lookml_broken_tableless_view_surfaces_error_inside_layer_context(): + """A genuinely-broken tableless view (unresolved extends) must NOT be silently skipped. + + Deferred registration skips only INTENTIONAL templates (extension:required / unsupported + derived tables). A `view: child { extends: [missing] }` stays tableless because its parent + can't resolve -- that's a real error, so add_model must still raise rather than drop it. + """ + import tempfile + + import pytest + + from sidemantic import SemanticLayer + from sidemantic.validation import ModelValidationError + + with tempfile.NamedTemporaryFile(mode="w", suffix=".lkml", delete=False) as f: + f.write( + "view: child { extends: [missing] dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } }" + ) + f.flush() + path = Path(f.name) + + with pytest.raises(ModelValidationError): + with SemanticLayer(): + LookMLAdapter().parse(path) + + +def test_lookml_export_unsupported_derived_table_stays_tableless_on_reimport(): + """An unsupported derived_table must round-trip as tableless, not gain a physical table. + + _export_view re-emits the (retained) derived_table with no `sql`, so re-import keeps it + marked unsupported instead of the implicit-table default assigning `table = `. + """ + import tempfile + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + 'view: pdt { derived_table: { sql_trigger_value: "SELECT max(id) FROM orders" ;; persist_for: "24 hours" } ' + " dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } }" + ) + graph = LookMLAdapter().parse(Path(d)) + assert graph.get_model("pdt").table is None + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + text = open(out).read() + assert "derived_table" in text and "sql_trigger_value" in text + reimported = LookMLAdapter().parse(Path(out)) + m = reimported.get_model("pdt") + assert m.table is None # stays tableless, not defaulted to 'pdt' + assert (m.meta or {}).get("unsupported_derived_table") + + +def test_lookml_export_abstract_view_stays_tableless_on_reimport(): + """An abstract (extension: required) base must round-trip as tableless, not gain a table. + + _export_view must re-emit `extension: required` so the re-imported base stays non-queryable + instead of the implicit-table default assigning it a physical table named after the view. + """ + import tempfile + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + "view: base { extension: required " + " dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + " dimension: x { type: number sql: ${TABLE}.x ;; } } " + "view: orders { extends: [base] dimension: y { type: number sql: ${TABLE}.y ;; } }" + ) + graph = LookMLAdapter().parse(Path(d)) + out = tempfile.mktemp(suffix=".lkml") + LookMLAdapter().export(graph, out) + text = open(out).read() + import re + + assert "extension: required" in re.search(r"view: base \{.*?\n\}", text, re.S).group(0) # base abstract + # The concrete child INHERITS extension_required in meta but has its own table; it must NOT + # re-emit the marker (that would make the usable child abstract on round-trip). + assert "extension: required" not in re.search(r"view: orders \{.*?\n\}", text, re.S).group(0) + reimported = LookMLAdapter().parse(Path(out)) + assert reimported.get_model("base").table is None # base stays tableless, not defaulted + assert reimported.get_model("orders").table == "orders" # child stays usable + + +def test_lookml_abstract_base_does_not_break_directory_load(): + """The CLI path (load_from_directory) must skip the abstract base, not abort the project.""" + import tempfile + + from sidemantic import SemanticLayer + from sidemantic.loaders import load_from_directory + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + "view: base { extension: required dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: x { type: number sql: ${TABLE}.x ;; } } " + "view: orders { extends: [base] dimension: y { type: number sql: ${TABLE}.y ;; } }" + ) + + layer = SemanticLayer() + load_from_directory(layer, d) # must not raise ModelValidationError on `base` + assert "base" not in layer.graph.models # non-queryable abstract base skipped + assert layer.graph.get_model("orders").table == "orders" + + +def test_lookml_drop_metrics_uses_provenance_marker_not_base_ref(): + """Orphan-metric drop keys off the parser-owned `_lookml_template_metric` PROVENANCE marker. + + A metric the LookML parser stamped as coming from a template is dropped; a same-named + STANDALONE metric from another file (no marker) is preserved -- regardless of whether the + base ref is qualified or unqualified, and regardless of object identity (refinements + reconstruct the object but the marker is re-stamped on the graph-registered instance). + """ + from sidemantic import Metric, Model + from sidemantic.loaders import _drop_non_registerable_models + + def pop(bm, template_metric=False): + meta = {"_lookml_template_metric": True} if template_metric else None + return Metric( + name="pop", + type="time_comparison", + base_metric=bm, + comparison_type="yoy", + calculation="difference", + meta=meta, + ) + + template = Model(name="base", meta={"lookml_template": True}) + orders = Model(name="orders", table="orders", metrics=[Metric(name="total", agg="sum", sql="amt")]) + + # A standalone `pop` (NO provenance marker) is preserved, even with an UNqualified base that + # matches a surviving model's `total` -- and even qualified. + standalone = pop("total") + metrics = {"pop": standalone} + _drop_non_registerable_models({"base": template, "orders": orders}, metrics) + assert metrics.get("pop") is standalone + + # A template's `pop` (marked) is dropped -- unqualified base, same-named surviving `total` + # present: the marker (not the base ref) decides. + metrics2 = {"pop": pop("total", template_metric=True)} + _drop_non_registerable_models({"base": template, "orders": orders}, metrics2) + assert "pop" not in metrics2 + + +def test_lookml_dropped_template_metric_dropped_despite_samename_local_measure(): + """A dropped template's graph metric is removed even if a surviving model has a same-named measure. + + The orphan check matches graph-level metric types (time_comparison/conversion), not bare + names: a surviving `orders.pop` SIMPLE measure must not keep the template's `pop` + period_over_period alive (which would expose a metric whose base model is gone). + """ + import tempfile + + from sidemantic import SemanticLayer + from sidemantic.loaders import load_from_directory + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + "view: base { extension: required " + " dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + " dimension: amt { type: number sql: ${TABLE}.amt ;; } " + " measure: total { type: sum sql: ${amt} ;; } " + " measure: pop { type: period_over_period based_on: total period: year kind: difference } } " + "view: orders { sql_table_name: orders ;; " + " dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + " measure: pop { type: count } }" + ) + + layer = SemanticLayer() + load_from_directory(layer, d) + assert "base" not in layer.graph.models + assert "pop" not in layer.graph.metrics # template graph metric dropped despite orders.pop + assert layer.graph.get_model("orders").get_metric("pop") is not None # local measure survives + + +def test_lookml_dropped_template_graph_metric_not_orphaned(): + """A graph metric (period_over_period) on a dropped template must not linger after CLI load. + + add_model auto-registers time_comparison/conversion measures as graph metrics; when the + only model carrying it is a skipped extension:required template, the metric must be + dropped too, else compile/info expose a metric whose base model is missing. + """ + import tempfile + + from sidemantic import SemanticLayer + from sidemantic.loaders import load_from_directory + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + "view: base { extension: required " + " dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + " dimension: amt { type: number sql: ${TABLE}.amt ;; } " + " measure: total { type: sum sql: ${amt} ;; } " + " measure: pop { type: period_over_period based_on: total period: year kind: difference } }" + ) + + layer = SemanticLayer() + load_from_directory(layer, d) # must not raise + assert "base" not in layer.graph.models # template skipped + assert "pop" not in layer.graph.metrics # orphaned graph metric dropped too + + +def test_lookml_registerability_keys_off_parser_marker_not_user_meta(): + """A tableless model with user `extension_required` meta but no parser marker must surface error. + + The skip keys off the parser-owned `lookml_template` marker so a native/other-format + model that merely carries the public `extension_required` key is still registered (and + its missing-source error surfaced), not silently dropped. + """ + from types import SimpleNamespace + + from sidemantic.loaders import _is_registerable_model + + user_meta = SimpleNamespace(table=None, sql=None, dax=None, source_uri=None, meta={"extension_required": True}) + assert _is_registerable_model(user_meta) is True # surfaces error, not dropped + template = SimpleNamespace( + table=None, sql=None, dax=None, source_uri=None, meta={"extension_required": True, "lookml_template": True} + ) + assert _is_registerable_model(template) is False # genuine LookML template skipped + + +def test_lookml_abstract_marker_survives_later_refinement_meta_overwrite(): + """A refinement overwriting meta must not strip the abstract marker the loader keys off. + + `+base { extension: required }` then `+base { label: ... }` leaves base tableless but + its final meta only has the label; the marker is re-asserted so the CLI loader still + skips it instead of raising the no-table validation error. + """ + import tempfile + + from sidemantic import SemanticLayer + from sidemantic.loaders import load_from_directory + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + "view: base { dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: x { type: number sql: ${TABLE}.x ;; } } " + "view: +base { extension: required } " + 'view: +base { label: "Base" } ' + "view: orders { extends: [base] dimension: y { type: number sql: ${TABLE}.y ;; } }" + ) + + layer = SemanticLayer() + load_from_directory(layer, d) # must not raise even though `extension_required` was overwritten + assert "base" not in layer.graph.models + assert layer.graph.get_model("orders").table == "orders" + + +def test_lookml_fk_inference_skips_abstract_template_no_dangling_rel(): + """FK inference must not target a skipped abstract template, leaving a dangling relationship.""" + import tempfile + + from sidemantic import SemanticLayer + from sidemantic.loaders import load_from_directory + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + "view: customer { extension: required dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } } " + "view: customers { extends: [customer] sql_table_name: customers ;; " + "dimension: name { type: string sql: ${TABLE}.name ;; } } " + "view: orders { sql_table_name: orders ;; dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: customer_id { type: number sql: ${TABLE}.customer_id ;; } }" + ) + + layer = SemanticLayer() + load_from_directory(layer, d) + assert "customer" not in layer.graph.models # abstract template not registered + # No model carries a relationship pointing at the skipped template. + for model in layer.graph.models.values(): + for rel in getattr(model, "relationships", []) or []: + assert rel.name in layer.graph.models, f"dangling relationship to {rel.name}" + + +def test_lookml_explore_join_to_template_no_dangling_rel(): + """An explore join to a skipped template must not leave a dangling relationship.""" + import tempfile + + from sidemantic import SemanticLayer + from sidemantic.loaders import load_from_directory + + d = tempfile.mkdtemp() + with open(Path(d) / "v.lkml", "w") as f: + f.write( + "view: customer { extension: required dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } } " + "view: orders { sql_table_name: orders ;; dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: customer_id { type: number sql: ${TABLE}.customer_id ;; } } " + "explore: orders { join: customer { sql_on: ${orders.customer_id} = ${customer.id} ;; relationship: many_to_one } }" + ) + + layer = SemanticLayer() + load_from_directory(layer, d) + assert "customer" not in layer.graph.models + for model in layer.graph.models.values(): + for rel in getattr(model, "relationships", []) or []: + assert rel.name in layer.graph.models, f"dangling relationship to {rel.name}" + + +def test_lookml_active_layer_explore_join_to_template_no_dangling_rel(): + """Same as above but via the ACTIVE-layer parse() path (with SemanticLayer()).""" + import tempfile + + from sidemantic import SemanticLayer + + with tempfile.NamedTemporaryFile(mode="w", suffix=".lkml", delete=False) as f: + f.write( + "view: customer { extension: required dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } } " + "view: orders { sql_table_name: orders ;; dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: customer_id { type: number sql: ${TABLE}.customer_id ;; } } " + "explore: orders { join: customer { sql_on: ${orders.customer_id} = ${customer.id} ;; relationship: many_to_one } }" + ) + f.flush() + path = Path(f.name) + + with SemanticLayer() as layer: + LookMLAdapter().parse(path) + assert "customer" not in layer.graph.models + for model in layer.graph.models.values(): + for rel in getattr(model, "relationships", []) or []: + assert rel.name in layer.graph.models, f"dangling relationship to {rel.name}" + + +def test_lookml_extends_tableless_view_keeps_own_table(): + """A child extending a tableless base must default to its OWN name, not inherit the base's default.""" + graph = _parse_lkml( + "view: base { dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: x { type: number sql: ${TABLE}.x ;; } } " + "view: child { extends: [base] dimension: y { type: number sql: ${TABLE}.y ;; } }" + ) + assert graph.get_model("base").table == "base" + assert graph.get_model("child").table == "child" + + +def test_lookml_refinement_adds_fields_then_defaults_table(): + """A fieldless base whose fields come from a +refinement still defaults its table after merge.""" + graph = _parse_lkml( + "view: orders {} view: +orders { dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } }" + ) + assert graph.get_model("orders").table == "orders" + + +def test_lookml_filter_only_view_gets_default_table(): + """A view that declares only LookML `filter` fields (-> segments) still defaults its table. + + Such a view is not 'fieldless' -- without the default it fails validation (no table/sql). + """ + graph = _parse_lkml("view: ff { filter: recent { type: yesno sql: ${TABLE}.x ;; } }") + model = graph.get_model("ff") + assert model.table == "ff" + assert [s.name for s in model.segments] == ["recent"] + + +def test_lookml_concrete_child_of_abstract_view_gets_table(): + """A concrete view extending an extension:required base must default to its OWN name, not be treated as abstract.""" + graph = _parse_lkml( + "view: base { extension: required " + "dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } " + "dimension: x { type: number sql: ${TABLE}.x ;; } } " + "view: orders { extends: [base] dimension: y { type: number sql: ${TABLE}.y ;; } }" + ) + assert graph.get_model("base").table is None # abstract base stays table-less + assert graph.get_model("orders").table == "orders" # concrete child defaults to its name + + +def test_lookml_unsupported_derived_table_marker_survives_refinement(): + """The unsupported-derived_table marker must survive a refinement that carries meta (e.g. label).""" + graph = _parse_lkml( + "view: ndt { derived_table: { sql_trigger_value: SELECT CURRENT_DATE() ;; } " + "dimension: id { type: number sql: ${TABLE}.id ;; } } " + 'view: +ndt { label: "NDT View" }' + ) + ndt = graph.get_model("ndt") + # Not defaulted to a physical table even though the refinement replaced its meta. + assert ndt.table is None + assert ndt.sql is None + + +def test_lookml_child_inheriting_unsupported_derived_table_not_defaulted(): + """A child extending a parent with an unsupported derived_table must stay table-less too.""" + graph = _parse_lkml( + "view: base_ndt { derived_table: { sql_trigger_value: SELECT 1 ;; } " + "dimension: id { type: number sql: ${TABLE}.id ;; } } " + "view: child_ndt { extends: [base_ndt] }" + ) + # Inherited the unsupported-derived_table marker via extends -> not defaulted. + assert graph.get_model("child_ndt").table is None + + +def test_lookml_refinement_added_extension_required_stays_abstract(): + """A +refinement that adds extension: required must keep the refined view tableless.""" + graph = _parse_lkml( + "view: base { dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } } " + "view: +base { extension: required }" + ) + assert graph.get_model("base").table is None + + +def test_lookml_abstract_flag_survives_later_refinement(): + """extension: required added by one refinement survives a later refinement that replaces meta.""" + graph = _parse_lkml( + "view: base { dimension: id { primary_key: yes type: number sql: ${TABLE}.id ;; } } " + "view: +base { extension: required } " + 'view: +base { label: "Base" }' + ) + assert graph.get_model("base").table is None + + +def test_lookml_child_of_unsupported_dt_with_own_meta_not_defaulted(): + """A child extending an unsupported derived_table base must stay tableless even with its own meta.""" + graph = _parse_lkml( + "view: base { derived_table: { sql_trigger_value: SELECT 1 ;; } " + "dimension: id { type: number sql: ${TABLE}.id ;; } } " + 'view: child { extends: [base] label: "Child" }' + ) + assert graph.get_model("child").table is None + + def test_lookml_export_unmapped_aggregations_not_count(): """Unmapped aggregations / complex types must not be silently exported as COUNT.""" import tempfile diff --git a/tests/adapters/lookml/test_fixtures.py b/tests/adapters/lookml/test_fixtures.py index 766da461..01caf966 100644 --- a/tests/adapters/lookml/test_fixtures.py +++ b/tests/adapters/lookml/test_fixtures.py @@ -69,7 +69,7 @@ def test_redshift_etl_errors_time_dimensions(self): model = self.graph.get_model("redshift_etl_errors") assert model.get_dimension("error_time") is not None assert model.get_dimension("error_time").type == "time" - assert model.get_dimension("error_time").granularity == "hour" + assert model.get_dimension("error_time").granularity == "second" assert model.get_dimension("error_date") is not None assert model.get_dimension("error_date").type == "time" assert model.get_dimension("error_date").granularity == "day" @@ -332,7 +332,7 @@ def test_created_time_group(self): assert model.get_dimension("created_month") is not None assert model.get_dimension("created_year") is not None assert model.get_dimension("created_time") is not None - assert model.get_dimension("created_time").granularity == "hour" + assert model.get_dimension("created_time").granularity == "second" def test_shipped_time_group(self): model = self.graph.get_model("order_items") diff --git a/tests/adapters/lookml/test_parsing.py b/tests/adapters/lookml/test_parsing.py index e8beb23b..30c3b62e 100644 --- a/tests/adapters/lookml/test_parsing.py +++ b/tests/adapters/lookml/test_parsing.py @@ -119,7 +119,8 @@ def test_lookml_adapter_ecommerce(): # Test dimension group with multiple timeframes created_time = orders.get_dimension("created_time") assert created_time is not None - assert created_time.granularity == "hour" + # Looker's "time" timeframe keeps to-the-second precision. + assert created_time.granularity == "second" created_date = orders.get_dimension("created_date") assert created_date is not None diff --git a/tests/adapters/test_added_fixture_coverage.py b/tests/adapters/test_added_fixture_coverage.py index 8fb423ab..ea08d3d5 100644 --- a/tests/adapters/test_added_fixture_coverage.py +++ b/tests/adapters/test_added_fixture_coverage.py @@ -227,7 +227,8 @@ "tests/fixtures/lookml/ga360_ga_block.view.lkml", "tests/fixtures/lookml/lkml_model_all_fields.model.lkml", "tests/fixtures/lookml/lkml_parameter_join.model.lkml", - "tests/fixtures/lookml/node_lookml_refinement_merging.model.lkml", + # node_lookml_refinement_merging now compiles: its `deep_merging` view has a + # dimension and (per Looker semantics) defaults its table to the view name. "tests/fixtures/lookml/node_lookml_refinement_sequencing.model.lkml", "tests/fixtures/lookml/pylookml_aggregate_tables.model.lkml", "tests/fixtures/lookml/pylookml_manifest.lkml", diff --git a/tests/adapters/test_fixture_functionality_contracts.py b/tests/adapters/test_fixture_functionality_contracts.py index 60e20953..71613933 100644 --- a/tests/adapters/test_fixture_functionality_contracts.py +++ b/tests/adapters/test_fixture_functionality_contracts.py @@ -147,7 +147,10 @@ "RillAdapter", "ThoughtSpotAdapter", }, - "source_fragments_without_fields": {"AtScaleSMLAdapter", "MalloyAdapter", "RillAdapter"}, + # LookML: a fieldless view now gets its implicit "table = view name" default (Looker + # behavior), so a refinement-fragment fixture with field-less views is a source fragment + # without fields rather than a tableless semantic-only model. + "source_fragments_without_fields": {"AtScaleSMLAdapter", "LookMLAdapter", "MalloyAdapter", "RillAdapter"}, "semantic_only_no_sources": { "AtScaleSMLAdapter", "HolisticsAdapter",