diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548b40f22..e2728d46e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,22 @@ jobs: matrix: python-version: ["3.10", "3.12"] + # FalkorDB is the graph backend; without a reachable instance the conftest + # session fixture skips every graph-dependent test, so CI would go green + # without exercising the engine at all. Run it as a service container — the + # fixtures connect to localhost:6379 by default (overridable via FALKORDB_*). + services: + falkordb: + image: falkordb/falkordb:latest + ports: + - 6379:6379 + # falkordb is redis-based, so redis-cli ping gates readiness before tests. + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + steps: - uses: actions/checkout@v6 with: diff --git a/graphify/affected.py b/graphify/affected.py index 4fc7b8fbf..dea0e9646 100644 --- a/graphify/affected.py +++ b/graphify/affected.py @@ -6,8 +6,6 @@ from typing import Iterable import unicodedata -import networkx as nx - DEFAULT_AFFECTED_RELATIONS = ( "calls", @@ -50,16 +48,16 @@ def _format_location(data: dict) -> str: return str(source_file) +def _normalize_label(label: str) -> str: + return unicodedata.normalize("NFC", label).casefold() + + def _bare_name(label: str) -> str: - """Lowercased label with the callable decoration (trailing "()") removed.""" + """Normalized label with the callable decoration (trailing "()") removed.""" label = _normalize_label(label) return label[:-2] if label.endswith("()") else label -def _normalize_label(label: str) -> str: - return unicodedata.normalize("NFC", label).casefold() - - def _prefer_file_node( graph: nx.Graph, node_ids: list[str], @@ -95,11 +93,37 @@ def _prefer_file_node( return None -def resolve_seed(graph: nx.Graph, query: str) -> str | None: +def resolve_seed(graph, query: str) -> str | None: # A trailing path separator must not change a source-file match — serve's # _find_node tokenizes the path (which drops it), so strip it here for parity # (otherwise `affected "src/x.ts/"` returned None while `explain` resolved it). query = query.rstrip("/\\") or query + # FalkorDB-native: resolve via scoped queries (no full-graph load). + if hasattr(graph, "find_node_ids"): + if graph.has_node(query): + return query + # Mirror the in-memory tier order: exact label -> bare callable name -> + # exact source_file -> substring (#1353). Without the bare-name tier a + # query like "foo" failed to resolve to "foo()" on the native path. + for kwargs in ({"label": query}, {"label_bare": _bare_name(query)}): + matches = graph.find_node_ids(limit=2, **kwargs) + if len(matches) == 1: + return str(matches[0]) + # source_file tier: when several nodes share the file, prefer the + # file-level node, exactly as the in-memory path does — otherwise a + # path query is ambiguous natively but resolves in memory. + src_matches = [str(m) for m in graph.find_node_ids(source_file=query, limit=50)] + if len(src_matches) == 1: + return src_matches[0] + if src_matches: + preferred = _prefer_file_node(graph, src_matches, query) + if preferred is not None: + return preferred + matches = graph.find_node_ids(limit=2, label_contains=query) + if len(matches) == 1: + return str(matches[0]) + return None + # In-memory fallback (nx / MemGraph) — normalized + bare-name matching (#1353). if query in graph: return query query_lower = _normalize_label(query) @@ -150,6 +174,47 @@ def affected_nodes( depth: int = 2, ) -> list[AffectedHit]: relation_set = set(relations) + + # FalkorDB-native: one batched reverse-edge query per BFS level (depth queries + # total) instead of a per-node loop over an in-memory copy of the graph. + if hasattr(graph, "incoming_edges"): + seen = {seed} + hits: list[AffectedHit] = [] + frontier = [seed] + # #1669: seed the reverse walk with the root's own member nodes (one + # outward `method`/`contains` hop), mirroring the in-memory path. A caller + # can bind to a class's method node rather than the class node itself, so + # those callers are otherwise unreachable. Seeds only — not reported as + # hits, and `method`/`contains` stay out of the relation-filtered walk. + if hasattr(graph, "member_nodes"): + for member in sorted(str(m) for m in graph.member_nodes(seed)): + if member not in seen: + seen.add(member) + frontier.append(member) + for d in range(depth): + if not frontier: + break + rows = graph.incoming_edges(frontier, relation_set) + nxt: list[str] = [] + for _fid, src, relation, via_file, via_loc in sorted( + rows, key=lambda r: (str(r[0]), str(r[1]), str(r[2])) + ): + src = str(src) + if src in seen: + continue + seen.add(src) + # Location comes from the SAME edge whose relation passed the + # filter, so relation and site stay consistent (#BUG1). + hits.append(AffectedHit( + src, d + 1, str(relation), + via_file=str(via_file or "") or None, + via_location=str(via_loc or "") or None, + )) + nxt.append(src) + frontier = nxt + return hits + + # In-memory fallback (nx / MemGraph) seen = {seed} queue: deque[tuple[str, int]] = deque([(seed, 0)]) hits: list[AffectedHit] = [] @@ -206,7 +271,6 @@ def affected_nodes( ) hits.append(hit) queue.append((source, current_depth + 1)) - return hits @@ -223,8 +287,19 @@ def format_affected( return f"No unique node match for {query}" hits = affected_nodes(graph, seed, relations=relation_list, depth=depth) + + # Fetch attrs for seed + all hits in one batched query (native) or per-node (fallback). + ids = [seed] + [h.node_id for h in hits] + if hasattr(graph, "node_attrs_batch"): + attrs = graph.node_attrs_batch(ids) + else: + attrs = {nid: dict(graph.nodes[nid]) for nid in ids if nid in graph} + + def _label(nid): + return str(attrs.get(nid, {}).get("label") or nid) + lines = [ - f"Affected nodes for {_node_label(graph, seed)}", + f"Affected nodes for {_label(seed)}", f"Relations: {', '.join(relation_list)}", f"Depth: {depth}", ] @@ -233,7 +308,7 @@ def format_affected( return "\n".join(lines) for hit in hits: - data = graph.nodes[hit.node_id] + data = attrs.get(hit.node_id, {}) if hit.via_location: # The relation SITE in this node's file (call/import/reference line), # labeled by [via_relation] so it's never mistaken for a def line. @@ -241,32 +316,40 @@ def format_affected( else: location = _format_location(data) # honest fallback: the node's own def line lines.append( - f"- {_node_label(graph, hit.node_id)} [{hit.via_relation}] {location}" + f"- {_label(hit.node_id)} [{hit.via_relation}] {location}" ) return "\n".join(lines) -def load_graph(path: Path) -> nx.Graph: - import json - from networkx.readwrite import json_graph - - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise RuntimeError( - f"Cannot read graph file {path}: {exc}. " - "Re-run 'graphify extract' to regenerate it." - ) from exc - # Force directed so stored caller→callee direction survives the round-trip; - # mirrors serve.py and __main__.py (#1174). - raw = {**raw, "directed": True} - # Normalize the edge key: graphify's `extract` output uses "edges" while - # networkx's node_link_data default is "links". Without this, an edges-keyed - # graph.json raises an uncaught KeyError: 'links' here — every other loader - # (__main__.py) already normalizes this (#738; same class as #1198). - if "links" not in raw and "edges" in raw: - raw = dict(raw, links=raw["edges"]) - try: - return json_graph.node_link_graph(raw, edges="links") - except TypeError: - return json_graph.node_link_graph(raw) +def connect_graph(path: Path): + """Open a connection to the FalkorDB-backed graph for the output dir containing + `path`. Cache-free: returns a store handle (a connection), it does NOT load the + graph into memory. + + `path` is the legacy graph.json location (e.g. graphify-out/graph.json); we + use its parent directory to locate the FalkorDB pointer and open the store. + """ + from .store import open_store + + resolved = Path(path) + out_dir = resolved.parent if resolved.suffix else resolved + store = open_store(out_dir, create=False) + if store.number_of_nodes() == 0: + # Back-compat: the store is empty but a node-link graph.json may still + # hold the graph (a pre-FalkorDB project, or a `--no-cluster` run that + # only wrote JSON). Import it on first use, exactly as serve._connect_graph + # does, so `affected`/`god-nodes` stay usable on the same graphs `query` + # and `explain` can already read. + from .serve import _import_graph_json_into_store + + gj = resolved if resolved.suffix == ".json" else (out_dir / "graph.json") + _import_graph_json_into_store(gj, store) + # open_store(create=False) hands back a store even when no graph was built + # (it derives a name from the pointer/root rather than erroring), so guard the + # empty case here — matching serve._connect_graph — so `graphify affected` + # tells the user to build instead of silently reporting nothing. + if store.number_of_nodes() == 0: + raise FileNotFoundError( + f"No graph found for {out_dir} (FalkorDB graph empty). Re-run /graphify to build." + ) + return store diff --git a/graphify/analyze.py b/graphify/analyze.py index 0707e2be7..08dc9e76a 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -1,7 +1,6 @@ """Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions.""" from __future__ import annotations from pathlib import Path -import networkx as nx from graphify.build import edge_data @@ -60,35 +59,37 @@ def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: return {n: cid for cid, nodes in communities.items() for n in nodes} -def _is_file_node(G: nx.Graph, node_id: str) -> bool: - """ - Return True if this node is a file-level hub node (e.g. 'client', 'models') - or an AST method stub (e.g. '.auth_flow()', '.__init__()'). - - These are synthetic nodes created by the AST extractor and should be excluded - from god nodes, surprising connections, and knowledge gap reporting. - """ - attrs = G.nodes[node_id] - label = attrs.get("label", "") +def _is_file_node_attrs(label: str, source_file: str, degree: int) -> bool: + """Attrs-based core of _is_file_node (works on a query row, no graph access).""" if not label: return False # File-level hub: label matches the actual source filename — bare basename OR # the directory-qualified form the #2032 disambiguation pass may assign. - source_file = attrs.get("source_file", "") + # Imported lazily: build.py imports analyze helpers, so a module-level import + # would close a cycle. if source_file: from graphify.build import _is_file_node_label if _is_file_node_label(label, source_file): return True - # Method stub: AST extractor labels methods as '.method_name()' if label.startswith(".") and label.endswith("()"): return True - # Module-level function stub: labeled 'function_name()' - only has a contains edge - # These are real functions but structurally isolated by definition; not a gap worth flagging - if label.endswith("()") and G.degree(node_id) <= 1: + if label.endswith("()") and degree <= 1: return True return False +def _is_file_node(G: nx.Graph, node_id: str) -> bool: + """ + Return True if this node is a file-level hub node (e.g. 'client', 'models') + or an AST method stub (e.g. '.auth_flow()', '.__init__()'). + + These are synthetic nodes created by the AST extractor and should be excluded + from god nodes, surprising connections, and knowledge gap reporting. + """ + attrs = G.nodes[node_id] + return _is_file_node_attrs(attrs.get("label", ""), attrs.get("source_file", ""), G.degree(node_id)) + + _JSON_NOISE_LABELS: frozenset[str] = frozenset({ "start", "end", "name", "id", "type", "properties", "value", "key", "data", "items", "title", "description", "version", @@ -97,13 +98,16 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: }) -def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: - attrs = G.nodes[node_id] - src = (attrs.get("source_file") or "").lower() +def _is_json_key_node_attrs(label: str, source_file: str) -> bool: + src = (source_file or "").lower() if not src.endswith(".json"): return False - label = (attrs.get("label") or "").strip().lower() - return label in _JSON_NOISE_LABELS + return (label or "").strip().lower() in _JSON_NOISE_LABELS + + +def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: + attrs = G.nodes[node_id] + return _is_json_key_node_attrs(attrs.get("label") or "", attrs.get("source_file") or "") def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: @@ -112,19 +116,27 @@ def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: File-level hub nodes are excluded: they accumulate import/contains edges mechanically and don't represent meaningful architectural abstractions. """ - degree = dict(G.degree()) - sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True) + # FalkorDB-native path: rank by degree in-engine and return only top candidates + # (no full-graph load). Filter the small candidate set in Python with the same + # heuristics. Fall back to the in-memory path for non-store graphs (MemGraph/nx). + if hasattr(G, "top_degree_nodes"): + cands = G.top_degree_nodes(limit=max(top_n * 50, 500)) + else: + degree = dict(G.degree()) + order = sorted(degree.items(), key=lambda x: x[1], reverse=True) + cands = [ + {"id": nid, "label": G.nodes[nid].get("label", ""), + "source_file": G.nodes[nid].get("source_file", ""), "degree": deg} + for nid, deg in order[: max(top_n * 50, 500)] + ] result = [] - for node_id, deg in sorted_nodes: - if _is_file_node(G, node_id) or _is_concept_node(G, node_id) or _is_json_key_node(G, node_id): + for c in cands: + label, src, deg = c.get("label", "") or "", c.get("source_file", "") or "", c["degree"] + if _is_file_node_attrs(label, src, deg) or _is_concept_node_attrs(src) or _is_json_key_node_attrs(label, src): continue - if G.nodes[node_id].get("label", "") in _BUILTIN_NOISE_LABELS: + if label in _BUILTIN_NOISE_LABELS: continue - result.append({ - "id": node_id, - "label": G.nodes[node_id].get("label", node_id), - "degree": deg, - }) + result.append({"id": c["id"], "label": label or c["id"], "degree": deg}) if len(result) >= top_n: break return result @@ -171,12 +183,14 @@ def _is_concept_node(G: nx.Graph, node_id: str) -> bool: - Empty source_file - source_file doesn't look like a real file path (no extension) """ - data = G.nodes[node_id] - source = data.get("source_file", "") - if not source: + return _is_concept_node_attrs(G.nodes[node_id].get("source_file", "")) + + +def _is_concept_node_attrs(source_file: str) -> bool: + if not source_file: return True # Has no file extension → probably a concept label, not a real file - if "." not in source.split("/")[-1]: + if "." not in source_file.split("/")[-1]: return True return False @@ -303,25 +317,15 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: u_source = G.nodes[u].get("source_file", "") v_source = G.nodes[v].get("source_file", "") - if not u_source or not v_source or u_source == v_source: continue score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source, degrees) - src_id = data.get("_src", u) - if src_id not in G.nodes: - src_id = u - tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: - tgt_id = v candidates.append({ "_score": score, - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), - "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), - ], + "source": G.nodes[u].get("label", u), + "target": G.nodes[v].get("label", v), + "source_files": [u_source, v_source], "confidence": data.get("confidence", "EXTRACTED"), "relation": relation, "why": "; ".join(reasons) if reasons else "cross-file semantic connection", @@ -355,7 +359,7 @@ def _cross_community_surprises( return [] if G.number_of_nodes() > 5000: return [] - betweenness = nx.edge_betweenness_centrality(G) + betweenness = G.edge_betweenness() top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] result = [] for (u, v), score in top_edges: @@ -388,21 +392,11 @@ def _cross_community_surprises( relation = data.get("relation", "") if relation in ("imports", "imports_from", "contains", "method"): continue - # This edge crosses community boundaries - interesting confidence = data.get("confidence", "EXTRACTED") - src_id = data.get("_src", u) - if src_id not in G.nodes: - src_id = u - tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: - tgt_id = v surprises.append({ - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), - "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), - ], + "source": G.nodes[u].get("label", u), + "target": G.nodes[v].get("label", v), + "source_files": [G.nodes[u].get("source_file", ""), G.nodes[v].get("source_file", "")], "confidence": confidence, "relation": relation, "note": f"Bridges community {cid_u} → community {cid_v}", @@ -454,10 +448,16 @@ def suggest_questions( "why": f"Edge tagged AMBIGUOUS (relation: {relation}) - confidence is low.", }) - # 2. Bridge nodes (high betweenness) → cross-cutting concern questions - if G.number_of_edges() > 0: - k = min(100, G.number_of_nodes()) if G.number_of_nodes() > 1000 else None - betweenness = nx.betweenness_centrality(G, k=k, seed=42) + # 2. Bridge nodes (high betweenness) → cross-cutting concern questions. + # Exact betweenness is ~O(V·E); cap at 5000 nodes (same guard as + # _cross_community_surprises) so report/cluster-only stays fast on large + # repos. Bridge-node questions are supplementary; the ambiguous-edge and + # community questions below are unaffected. + if G.number_of_edges() > 0 and G.number_of_nodes() <= 5000: + # Node betweenness via FalkorDB's built-in algo.betweenness (exact, whole + # graph). The old nx path sampled k sources with seed=42 on large graphs; + # the built-in computes exactly, so rankings are at least as accurate. + betweenness = G.node_betweenness() # Top bridge nodes that are NOT file-level hubs bridges = sorted( [(n, s) for n, s in betweenness.items() @@ -666,9 +666,9 @@ def _endpoint_source_file(node_id: str) -> str: src_file = attrs.get("source_file", "") return src_file if isinstance(src_file, str) else "" - # Step 1: Build a directed file-level graph from import/re-export edges. + # Step 1: Build a directed file-level edge list from import/re-export edges. # IMPORTANT: resolve endpoints using source_file only; never infer from label/id. - file_graph = nx.DiGraph() + file_edges: set[tuple[str, str]] = set() for u, v, data in G.edges(data=True): rel = data.get("relation", "") @@ -687,36 +687,42 @@ def _endpoint_source_file(node_id: str) -> str: u_file = _endpoint_source_file(u) v_file = _endpoint_source_file(v) - # Works for both DiGraph and Graph inputs: - # orient edge from edge.source_file endpoint to the opposite endpoint. + # Orient edge from edge.source_file endpoint to the opposite endpoint. if u_file == src_file_attr: tgt_file = v_file elif v_file == src_file_attr: tgt_file = u_file else: - # Fallback: if source endpoint cannot be matched exactly, - # still treat edge.source_file as source and pick the opposite endpoint - # only if one endpoint has a real source_file. tgt_file = v_file if v_file and v_file != src_file_attr else u_file if not tgt_file: continue - file_graph.add_edge(src_file_attr, tgt_file) + file_edges.add((src_file_attr, tgt_file)) - if not file_graph.edges(): + if not file_edges: return [] - # Step 2: Find simple cycles, bounded by length. - # Pass length_bound so networkx prunes during enumeration rather than - # enumerating all elementary cycles and post-filtering — avoids exponential - # blowup on dense graphs with many long cycles (#1196). + # Step 2: Find simple cycles, bounded by length, via the simpleCycles UDF + # (max_cycle_length bounds enumeration in-engine, like nx's length_bound #1196). + # Self-loops (a file importing itself) are length-1 cycles; the UDF only + # enumerates multi-node cycles, so we surface self-loops here directly. cycles: list[list[str]] = [] - for cycle in nx.simple_cycles(file_graph, length_bound=max_cycle_length): + self_loops = sorted({a for a, b in file_edges if a == b}) + multi_edges = [(a, b) for a, b in file_edges if a != b] + for f in self_loops: + cycles.append([f]) + # Prefer the backend's own enumeration (FalkorDB runs it as a server-side + # UDF); otherwise fall back to the pure-Python one. The fallback keeps this + # working for a plain NetworkX graph passed in by a library caller, which has + # no .simple_cycles() method (networkx exposes it as a module function). + _cycles = getattr(G, "simple_cycles", None) + if _cycles is None: + from graphify.store import simple_cycles_from_edges as _cycles + for cycle in _cycles(sorted(multi_edges), max_cycle_length): if len(cycle) <= max_cycle_length: cycles.append(cycle) if len(cycles) >= top_n * 10: - # Stop early to avoid combinatorial explosion break # Step 3: Sort by length (shortest = tightest coupling), then deduplicate. diff --git a/graphify/benchmark.py b/graphify/benchmark.py index cf2bc6ba2..66fa7cec6 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -1,10 +1,11 @@ """Token-reduction benchmark - measures how much context graphify saves vs naive full-corpus approach.""" from __future__ import annotations import sys -import networkx as nx +from pathlib import Path from graphify.build import edge_data from graphify.serve import _query_terms +from graphify.store import open_store from graphify.paths import default_graph_json as _default_graph_json @@ -96,12 +97,22 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ - graph_path = graph_path or _default_graph_json() - # Size-cap check + links/edges normalization + node-link parse. A raw - # --no-cluster graph stores edges under "edges" and used to KeyError - # here (#2212). - from graphify.paths import load_node_link_graph - G = load_node_link_graph(graph_path) + # FalkorDB-only: graph_path is the legacy graph.json location; its parent dir + # holds the pointer to the actual graph, so no file parsing (and no "edges" + # vs "links" key normalization, #2212) is involved. + resolved = Path(graph_path or _default_graph_json()) + out_dir = resolved.parent if resolved.suffix else resolved + G = open_store(out_dir, create=False) + if G.number_of_nodes() == 0: + # Back-compat, matching serve._connect_graph and affected.connect_graph: + # an empty store may still have a node-link graph.json beside it (a + # pre-FalkorDB project, or a `--no-cluster` run that only wrote JSON). + # Import it so `graphify benchmark` works on the same graphs the read + # commands already accept. + from graphify.serve import _import_graph_json_into_store + + gj = resolved if resolved.suffix == ".json" else (out_dir / "graph.json") + _import_graph_json_into_store(gj, G) if corpus_words is None: # Rough estimate: each node label is ~3 words, plus source context diff --git a/graphify/build.py b/graphify/build.py index 1f840bbc0..9e6efd3dd 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -28,7 +28,7 @@ import sys import unicodedata from pathlib import Path -import networkx as nx +from .store import GraphStore, DEFAULT_URI, graph_name_for from .ids import make_id, normalize_id as _normalize_id from .paths import default_graph_json as _default_graph_json from .validate import validate_extraction @@ -74,6 +74,14 @@ } +def _search_norm_label(label: str) -> str: + """Diacritic-stripped, lowercased label for in-engine search prefiltering. + Must match serve._strip_diacritics(label).lower() so Cypher CONTAINS prefilters + agree with the Python scoring tiers.""" + nfkd = unicodedata.normalize("NFKD", label or "") + return "".join(c for c in nfkd if not unicodedata.combining(c)).lower() + + # Hyperedge member lists are canonically keyed `nodes` (see graphify/llm.py # extraction spec), but LLM/subagent drift and externally-supplied graph.json # sometimes emit `members` or `node_ids`. _normalize_hyperedge_members folds @@ -304,25 +312,30 @@ def _infer_merge_root(graph_path: Path) -> str | None: return None -def edge_data(G: nx.Graph, u: str, v: str) -> dict: +def edge_data(G, u: str, v: str) -> dict: """Return one edge attribute dict for (u, v), tolerating MultiGraph. - For MultiGraph/MultiDiGraph there can be multiple parallel edges; - this returns the first one (sufficient for callers that only need - relation/confidence for rendering). Fixes #796. + FalkorDB stores parallel edges natively; ``G[u][v]`` returns the first + edge's attributes, which is sufficient for callers that only need + relation/confidence for rendering. Fixes #796. """ - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return next(iter(raw.values()), {}) - return raw + return G[u][v] -def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: - """Return every edge attribute dict for (u, v); always a list.""" - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return list(raw.values()) - return [raw] +def edge_datas(G, u: str, v: str) -> list[dict]: + """Return every edge attribute dict for (u, v); always a list. + + Stores expose ``edge_attrs_all`` so parallel links (e.g. a `references` and + a `calls` edge between the same pair) all survive; ``G[u][v]`` alone returns + only the first match, which let `path` print a relation the traversed pair + doesn't actually carry (#2074). + """ + getter = getattr(G, "edge_attrs_all", None) + if getter is not None: + datas = getter(u, v) + if datas: + return datas + return [G[u][v]] def dedupe_nodes(nodes: list[dict]) -> list[dict]: @@ -348,13 +361,13 @@ def dedupe_edges(edges: list[dict]) -> list[dict]: """Collapse exact parallel edges by ``(source, target, relation)``, keeping the first occurrence. - The clustered build path runs edges through a NetworkX ``DiGraph``, which - collapses parallel edges automatically. The ``--no-cluster`` and incremental - ``update`` write paths bypass NetworkX and concatenate edge lists raw, so - duplicates accumulate and edge counts become non-deterministic across build - modes / repeated updates (#1317). Deduping on the connectivity identity is - zero-signal-loss and restores idempotency. Callers that intentionally keep - parallel edges (multigraph output) must not use this. + The clustered build path runs edges through the GraphStore dedup (collapses + parallel edges). The ``--no-cluster`` and incremental ``update`` write paths + concatenate edge lists raw, so duplicates accumulate and edge counts become + non-deterministic across build modes / repeated updates (#1317). Deduping on + the connectivity identity is zero-signal-loss and restores idempotency. + Callers that intentionally keep parallel edges (multigraph output) must not + use this. """ seen: set[tuple] = set() out: list[dict] = [] @@ -536,13 +549,25 @@ def _doc_twin_remap(nodes: list) -> dict[str, str]: return remap -def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: - """Build a NetworkX graph from an extraction dict. - - directed=True produces a DiGraph that preserves edge direction (source→target). - directed=False (default) produces an undirected Graph for backward compatibility. +def build_from_json( + extraction: dict, + *, + directed: bool = False, + root: str | Path | None = None, + store: GraphStore | None = None, + graph_name: str = "graphify", + uri: str = DEFAULT_URI, +) -> GraphStore: + """Build a FalkorDB-backed graph (GraphStore) from an extraction dict. + + Edges are always stored in their native source→target orientation. When + ``directed`` is False (default) a reverse-direction duplicate of the same + node pair + relation is collapsed to the first-seen edge, matching the old + undirected nx.Graph semantics; when True both directions are kept. root: if given, absolute source_file paths from semantic subagents are made relative to root so all nodes share a consistent path key (#932). + store: build into this GraphStore (cleared first); otherwise a new one is + created for ``graph_name`` at ``uri``. """ _root = str(Path(root).resolve()) if root else None # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. @@ -670,13 +695,16 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if isinstance(he, dict) and isinstance(he.get("nodes"), list): he["nodes"] = [_doc_remap.get(n, n) for n in he["nodes"]] - G: nx.Graph = nx.DiGraph() if directed else nx.Graph() + # Collect node attributes in extraction order (last write wins per id), the + # same idempotent-overwrite semantics nx.add_node had. + node_attrs: dict[str, dict] = {} + node_order: list[str] = [] for node in extraction.get("nodes", []): # Skip dict nodes with a missing or non-hashable id (e.g. a list emitted - # by a buggy LLM extraction) so NetworkX add_node never raises - # TypeError: unhashable type. Non-dict nodes are deliberately left to - # raise as before, so callers that probe build for shape errors (e.g. - # the multigraph diagnostic) still observe the malformed shape. + # by a buggy LLM extraction) so the store never receives an unusable key. + # Non-dict nodes are deliberately left to raise as before, so callers that + # probe build for shape errors (e.g. the multigraph diagnostic) still + # observe the malformed shape. if isinstance(node, dict): if "id" not in node: continue @@ -691,8 +719,15 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat continue if "source_file" in node: node["source_file"] = _norm_source_file(node["source_file"], _root) - G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) - node_set = set(G.nodes()) + nid = node["id"] + if nid not in node_attrs: + node_order.append(nid) + attrs = {k: v for k, v in node.items() if k != "id"} + # Stored searchable label so query/explain can prefilter candidates in + # the engine (Cypher CONTAINS) instead of scanning every node in Python. + attrs["norm_label"] = _search_norm_label(str(attrs.get("label", ""))) + node_attrs[nid] = attrs + node_set = set(node_attrs) # #1145 (extended): merge LLM ghost-duplicate nodes into AST canonical nodes. # Original bug: AST uses parent-qualified IDs (mingpt_bpe_get_pairs) while LLM @@ -716,7 +751,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # with CPython's per-process string-hash seed (#1753) — the same reason the # edge-iteration loop further down sorts on purpose. for nid in sorted(node_set): - attrs = G.nodes[nid] + attrs = node_attrs[nid] label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) if not label or not sf: @@ -735,7 +770,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if is_ast: # Two AST nodes on the same key (same file, same label) is an # ambiguous collision. - if key in _loc_nodes and G.nodes[_loc_nodes[key]].get("_origin") == "ast": + if key in _loc_nodes and node_attrs[_loc_nodes[key]].get("_origin") == "ast": _loc_collisions.add(key) # AST-origin nodes always overwrite a prior non-AST entry. _loc_nodes[key] = nid @@ -747,7 +782,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Pass 2: find ghosts — non-AST nodes that have an AST canonical twin. for nid in sorted(node_set): - attrs = G.nodes[nid] + attrs = node_attrs[nid] if attrs.get("_origin") == "ast": continue # AST nodes are never ghosts label = str(attrs.get("label", "")).strip() @@ -765,9 +800,9 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat ast_id = _loc_nodes.get(key) if ast_id is not None: _ghost_remap[sem_id] = ast_id - # Remove ghost nodes from the graph; edges will be re-pointed via norm_to_id. + # Remove ghost nodes; edges will be re-pointed via norm_to_id. for ghost_id in _ghost_remap: - G.remove_node(ghost_id) + node_attrs.pop(ghost_id, None) node_set.discard(ghost_id) # Normalized ID map: lets edges survive when the LLM generates IDs with @@ -810,7 +845,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat from graphify.extractors.base import _file_stem as _fs _alias_candidates: dict[str, set[str]] = {} for nid in node_set: - attrs = G.nodes[nid] + attrs = node_attrs[nid] sf = attrs.get("source_file") if not sf: continue @@ -833,6 +868,9 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat for alias_key, candidates in _alias_candidates.items(): if len(candidates) == 1: norm_to_id.setdefault(alias_key, next(iter(candidates))) + edge_items: list[tuple[str, str, dict]] = [] + seen_exact: set[tuple] = set() # (src, tgt, relation) — collapse exact directed dups + seen_pairs: dict[tuple, tuple[str, str]] = {} # (unordered pair, relation) -> first src/tgt # Iterate edges in a deterministic order. The graph is undirected and stores # direction in _src/_tgt; when two edges collapse onto the same node pair the # last write wins, so an unstable iteration order flips _src/_tgt run-to-run @@ -907,8 +945,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # flags and leaves query results with no file reference (#1279). if not attrs.get("source_file"): attrs["source_file"] = ( - G.nodes[src].get("source_file") - or G.nodes[tgt].get("source_file") + node_attrs[src].get("source_file") + or node_attrs[tgt].get("source_file") or "" ) if "source_file" in attrs: @@ -920,8 +958,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Python `import time` must not bind to a `time.ts`, #1749). _edge_rel = attrs.get("relation") if _edge_rel in ("calls", "imports", "imports_from", "references"): - src_ext = Path(G.nodes[src].get("source_file") or "").suffix.lower() - tgt_ext = Path(G.nodes[tgt].get("source_file") or "").suffix.lower() + src_ext = Path(node_attrs[src].get("source_file") or "").suffix.lower() + tgt_ext = Path(node_attrs[tgt].get("source_file") or "").suffix.lower() src_fam = _EDGE_LANG_FAMILY.get(src_ext) tgt_fam = _EDGE_LANG_FAMILY.get(tgt_ext) if _edge_rel == "calls": @@ -948,24 +986,47 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # program structure rather than import-resolution artifacts. if src == tgt and _edge_rel in ("imports", "imports_from", "re_exports"): continue - # Preserve original edge direction - undirected graphs lose it otherwise, - # causing display functions to show edges backwards. - attrs["_src"] = src - attrs["_tgt"] = tgt - # When the graph is undirected and the same node pair appears twice with - # the same relation but opposite directions (e.g. a `calls` b and b `calls` a), - # nx.Graph collapses them into one edge. The deterministic sort above means - # the lexicographically-later direction would systematically overwrite the - # earlier one's _src/_tgt, silently flipping the surviving edge's caller - # and callee. First-seen direction wins instead — drop the redundant - # reverse-direction duplicate so the original direction is preserved (#1061). - if not G.is_directed() and G.has_edge(src, tgt): - existing = edge_data(G, src, tgt) - if existing.get("relation") == attrs.get("relation") and ( - existing.get("_src") == tgt and existing.get("_tgt") == src - ): + # Edges are stored DIRECTED in their native source→target orientation, so + # direction survives without the old _src/_tgt markers. Collapse exact + # directed duplicates always (same as nx.DiGraph / the old MERGE upsert) + # so the fresh CREATE path doesn't emit duplicate edges. + exact_key = (src, tgt, attrs.get("relation")) + if exact_key in seen_exact: + continue + seen_exact.add(exact_key) + # When the graph is undirected (default) and the same node pair appears + # again with the same relation in the opposite direction (a `calls` b and + # b `calls` a), the deterministic sort above means the lexicographically- + # later direction would systematically win and silently flip the surviving + # edge's caller and callee. First-seen direction wins instead — drop the + # redundant reverse-direction duplicate (#1061). + if not directed: + pair_key = (frozenset((src, tgt)), attrs.get("relation")) + if pair_key in seen_pairs: continue - G.add_edge(src, tgt, **attrs) + seen_pairs[pair_key] = (src, tgt) + edge_items.append((src, tgt, attrs)) + + # Runs LAST, after the alias-competition above (which relies on file-node + # labels still being bare basenames): give colliding-basename file nodes a + # directory-qualified display label so lookup/discovery can disambiguate them + # (#2032). Labels only — ids and edges are untouched. Applied to node_attrs + # before the write, since the store is populated in one shot below. + for _nid, _new_label in _file_label_reassignments( + [(nid, a.get("label"), a.get("source_file")) for nid, a in node_attrs.items()] + ).items(): + node_attrs[_nid]["label"] = _new_label + # Keep the stored search key in sync with the relabel, or query/explain + # would prefilter on the pre-disambiguation label. + node_attrs[_nid]["norm_label"] = _search_norm_label(str(_new_label)) + + if store is None: + store = GraphStore(graph_name=graph_name, uri=uri, directed=True) + store.clear() + # Fresh build into a cleared graph: ids are unique and there are no existing + # edges, so CREATE (no MERGE existence-check) is correct and much faster. + store.add_nodes_from([(nid, node_attrs[nid]) for nid in node_order if nid in node_attrs], fresh=True) + store.add_edges_from(edge_items, fresh=True) hyperedges = extraction.get("hyperedges", []) if hyperedges: # Relativize hyperedge source_file the same way nodes and edges are @@ -1007,13 +1068,24 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat he["nodes"] = valid_members kept_hyperedges.append(he) if kept_hyperedges: - G.graph["hyperedges"] = kept_hyperedges - # Runs LAST, after the alias-competition above (which relies on file-node - # labels still being bare basenames): give colliding-basename file nodes a - # directory-qualified display label so lookup/discovery can disambiguate - # them (#2032). Labels only — ids and edges are untouched. - _disambiguate_file_node_labels(G) - return G + store.graph["hyperedges"] = kept_hyperedges + store.save_meta() + # Record the scan root on the graph itself. This is the FalkorDB replacement + # for the committed `graphify-out/.graphify_root` marker: a later build_merge + # called WITHOUT root (the skill's --update runbook does exactly that) reads + # it back to relativize absolute prune_sources against the stored relative + # source_file keys. Without it those paths never match and a deleted file's + # nodes survive as ghosts (#1571). + if _root: + store.graph["scan_root"] = _root + store.save_meta() + # Warm + persist the hub-threshold (p99 degree) so query traversals don't + # recompute it on every invocation. + try: + store._hub_threshold() + except Exception: + pass + return store def build( @@ -1023,7 +1095,10 @@ def build( dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: + store: GraphStore | None = None, + graph_name: str = "graphify", + uri: str = DEFAULT_URI, +) -> GraphStore: """Merge multiple extraction results into one graph. directed=True produces a DiGraph that preserves edge direction (source→target). @@ -1039,7 +1114,6 @@ def build( from duplicate records of the same source entity. Genuine cross-file ID collisions remain isolated and are reported. """ - from graphify.dedup import deduplicate_entities combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} for ext in extractions: combined["nodes"].extend(ext.get("nodes", [])) @@ -1048,6 +1122,9 @@ def build( combined["input_tokens"] += ext.get("input_tokens", 0) combined["output_tokens"] += ext.get("output_tokens", 0) if dedup and combined["nodes"]: + # Imported lazily so dedup=False callers don't require the datasketch dep. + from graphify.dedup import deduplicate_entities + # Fold legacy node field aliases before dedup (#2194): dedup runs BEFORE # build_from_json and keys on `label`, so a `name`/`path` alias node # would be invisible to it and only label-dedup one build later, after @@ -1059,7 +1136,9 @@ def build( combined["nodes"], combined["edges"], communities={}, dedup_llm_backend=dedup_llm_backend, ) - return build_from_json(combined, directed=directed, root=root) + return build_from_json( + combined, directed=directed, root=root, store=store, graph_name=graph_name, uri=uri + ) def _norm_label(label: str | None) -> str: @@ -1257,15 +1336,16 @@ def _dropped(item: dict) -> bool: def build_merge( new_chunks: list[dict], - graph_path: str | Path | None = None, + graph_name: str = "graphify", prune_sources: list[str] | None = None, *, + uri: str = DEFAULT_URI, directed: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: - """Load existing graph.json, merge new chunks into it, and save back. +) -> GraphStore: + """Merge new chunks into the existing FalkorDB graph and persist. Re-extracted files REPLACE their prior contribution: any source_file present in new_chunks is dropped from the loaded graph before merging, so a changed @@ -1274,38 +1354,38 @@ def build_merge( Safe to call repeatedly. root: if given, absolute source_file paths in new_chunks are made relative (#932). """ - graph_path = Path(graph_path if graph_path is not None else _default_graph_json()) - _loaded = _load_existing_graph(graph_path) - if _loaded is not None: - existing_nodes, existing_edges, existing_hyperedges = _loaded - had_graph = True - else: - existing_nodes = [] - existing_edges = [] - existing_hyperedges = [] - had_graph = False - - # Effective root for relativizing absolute source_file / prune paths back to the - # stored relative source_file keys. When the caller passes root we use it; - # otherwise fall back to the graph's recorded scan root, so absolute - # prune_sources and new-chunk paths still match even when a caller omits root - # (#1571 — the skill's --update runbook calls build_merge without root, so - # absolute deleted-file paths never matched the relative node keys and their - # nodes survived as ghosts). + store = GraphStore(graph_name=graph_name, uri=uri, directed=True) + # Pull the existing graph back out of FalkorDB as an extraction-shaped chunk. + # Edges are stored in true direction, so (u, v) are the real source/target. + existing_nodes = [dict(attrs, id=nid) for nid, attrs in store.nodes(data=True)] + existing_edges = [] + for u, v, a in store.edges(data=True): + e = {k: val for k, val in a.items()} + e["source"], e["target"] = u, v + existing_edges.append(e) + existing_hyperedges = list(store.graph.get("hyperedges", []) or []) + had_graph = bool(existing_nodes) + + # Effective root for relativizing absolute source_file / prune paths back to + # the stored relative source_file keys. When the caller passes root we use it; + # otherwise fall back to the scan root the graph recorded at build time — the + # FalkorDB replacement for the `.graphify_root` marker. Without that fallback + # a root-less call (the skill's --update runbook) never matches absolute + # prune_sources and the deleted files' nodes survive as ghosts (#1571). _eff_root = ( str(Path(root).resolve()) if root is not None - else _infer_merge_root(graph_path) + else (store.graph.get("scan_root") or None) ) - # Re-extracted files REPLACE their prior contribution. Every source_file - # present in new_chunks is dropped from the loaded base before merging, so a - # CHANGED file's stale nodes/edges don't accumulate across incremental - # updates. Without this, build() merges old+new for the same file and only - # exact-duplicate edges collapse — edges/nodes that disappeared from the new - # version survive forever. Brand-new files aren't in base, so this is a no-op - # for them; genuinely deleted files are still handled via prune_sources. - # Matched in both raw and _norm_source_file form because new_chunks may carry - # absolute win32 paths while the stored graph keeps relative posix (#1007). + # Re-extracted files REPLACE their prior contribution (#1344/#1007). Every + # source_file present in new_chunks is dropped from the loaded base before + # merging, so a CHANGED file's stale nodes/edges don't accumulate across + # incremental updates. Without this, build() merges old+new for the same file + # and only exact-duplicate edges collapse — edges/nodes that disappeared from + # the new version survive forever. Brand-new files aren't in base (no-op); + # genuinely deleted files are still handled via prune_sources. Matched in both + # raw and _norm_source_file form because new_chunks may carry absolute win32 + # paths while the stored graph keeps relative posix. _replace_root = _eff_root new_sources: set[str] = set() for ch in new_chunks: @@ -1327,7 +1407,10 @@ def _kept(item: dict) -> bool: base = [{"nodes": existing_nodes, "edges": existing_edges}] if had_graph else [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) + G = build( + all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, + root=root, store=store, graph_name=graph_name, uri=uri, + ) # Prune set for deleted source files — both the raw form (matches nodes that # kept absolute source_file) and the normalised relative form (matches nodes @@ -1434,7 +1517,7 @@ def _prune_match(sf: "str | None") -> bool: # Safety check: refuse to shrink the graph silently (#479) # Skip when dedup or prune_sources is active — shrinkage is intentional there. - if graph_path.exists() and not dedup and not prune_sources: + if existing_nodes and not dedup and not prune_sources: existing_n = len(existing_nodes) new_n = G.number_of_nodes() if new_n < existing_n: @@ -1446,20 +1529,26 @@ def _prune_match(sf: "str | None") -> bool: return G -def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: - """Return a copy of G with all node IDs prefixed with repo_tag::. +def prefix_graph_for_global(G, repo_tag: str, target: GraphStore) -> GraphStore: + """Copy G's nodes/edges into `target` with all node IDs prefixed repo_tag::. - Labels are preserved unchanged (for display). A 'local_id' attribute - is added to each node so the original ID can be recovered. Edges are - rewritten to match the new prefixed IDs. The 'repo' attribute is set - on every node. + Labels are preserved unchanged (for display). A 'local_id' attribute is added + so the original ID can be recovered, and 'repo' is set on every node. Edges + are rewritten to the prefixed IDs. """ - relabel = {n: f"{repo_tag}::{n}" for n in G.nodes} - H = nx.relabel_nodes(G, relabel, copy=True) - for node, data in H.nodes(data=True): - data["repo"] = repo_tag - data.setdefault("local_id", node.split("::", 1)[1]) - return H + def _pfx(n: str) -> str: + return f"{repo_tag}::{n}" + + nodes = [] + for nid, data in G.nodes(data=True): + attrs = dict(data) + attrs["repo"] = repo_tag + attrs.setdefault("local_id", nid) + nodes.append((_pfx(nid), attrs)) + edges = [(_pfx(u), _pfx(v), dict(a)) for u, v, a in G.edges(data=True)] + target.add_nodes_from(nodes) + target.add_edges_from(edges) + return target def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": @@ -1490,8 +1579,6 @@ def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": return unique -def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int: - """Remove all nodes tagged with repo_tag from G in-place. Returns count removed.""" - to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag] - G.remove_nodes_from(to_remove) - return len(to_remove) +def prune_repo_from_graph(G: GraphStore, repo_tag: str) -> int: + """Remove all nodes tagged with repo_tag from G. Returns count removed.""" + return G.prune_repo(repo_tag) diff --git a/graphify/callflow_html.py b/graphify/callflow_html.py index dabb2997c..dd15f60f6 100644 --- a/graphify/callflow_html.py +++ b/graphify/callflow_html.py @@ -220,34 +220,32 @@ def normalize_edge(raw: dict, index: int) -> dict | None: def _node_link_payload(data: dict) -> tuple[list, list] | None: - """Read current graphify graph.json via NetworkX's node-link parser.""" + """Parse a node-link graph.json dict directly (no NetworkX).""" if not isinstance(data.get("nodes"), list): return None - if not isinstance(data.get("links"), list) and not isinstance(data.get("edges"), list): - return None - - try: - # Shared loader normalizes the raw writer's "edges" key to "links" - # before parsing; without it an edges-keyed payload raised - # KeyError: 'links' and this function silently returned None even - # though the shape check above accepts "edges" (#2212). - from graphify.paths import load_node_link_graph - - graph = load_node_link_graph(data) - except Exception: + # Accept the raw writer's "edges" key as well as node-link "links"; without + # the fallback an edges-keyed payload silently returned None even though the + # shape check accepts it (#2212). Parsed inline — no NetworkX loader needed. + links = data.get("links") + if not isinstance(links, list): + links = data.get("edges") + if not isinstance(links, list): return None nodes = [] - for node_id, attrs in graph.nodes(data=True): - node = dict(attrs) - node["id"] = node_id - nodes.append(node) + for n in data["nodes"]: + if not isinstance(n, dict): + continue + nodes.append(dict(n)) edges = [] - for index, (source, target, attrs) in enumerate(graph.edges(data=True), 1): - edge = dict(attrs) - edge["source"] = edge.get("_src", edge.get("source", source)) - edge["target"] = edge.get("_tgt", edge.get("target", target)) + for index, e in enumerate(links, 1): + if not isinstance(e, dict): + continue + edge = dict(e) + # Edges are stored in native direction; _src/_tgt fall back to source/target. + edge["source"] = edge.get("_src", edge.get("source")) + edge["target"] = edge.get("_tgt", edge.get("target")) edge.setdefault("id", f"edge_{index}") edges.append(edge) return nodes, edges diff --git a/graphify/cli.py b/graphify/cli.py index bd8f12bca..6922746e5 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -411,7 +411,7 @@ def _enforce_graph_size_cap_or_exit(gp: Path) -> None: Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. Use this from ``__main__.py`` subcommands that already use the ``print + - sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, + sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._connect_graph``, ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, ``global_graph``, ``watch``, ``export``) call the security helper directly and let the ``ValueError`` propagate. @@ -853,9 +853,8 @@ def dispatch_command(cmd: str) -> None: if len(sys.argv) < 3: print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) sys.exit(1) - from graphify.serve import _query_graph_text + from graphify.serve import _query_graph_text, _connect_graph from graphify.security import sanitize_label - from networkx.readwrite import json_graph from graphify import querylog question = sys.argv[2] @@ -892,53 +891,16 @@ def dispatch_command(cmd: str) -> None: else: i += 1 gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print(f"error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - try: - import json as _json - import networkx as _nx - - _raw = _json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # `query` deliberately keeps the graph undirected (unlike `path` / - # `explain`, which force directed=True): BFS/DFS here must explore - # both callers and callees of the seed node to build useful - # context, and forcing a DiGraph would make G.neighbors() return - # successors only, silently dropping every caller-side result for - # a seed with no outgoing edges. Direction is instead preserved - # per-edge below (mirrors graphify/build.py's _src/_tgt pattern) - # so the *rendering* stays correct without narrowing traversal. - _raw = dict( - _raw, - links=[ - {**link, "_src": link.get("source"), "_tgt": link.get("target")} - for link in _raw.get("links", []) - ], - ) - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(_raw.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` to get path-qualified IDs " - "(fixes same-name-file collisions).", - file=sys.stderr, - ) - except Exception: - pass - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) + # FalkorDB-only: the graph lives in the engine, located via the + # falkordb.json pointer next to graph_path; the legacy graph.json artifact + # need not exist, so there is no file-existence/suffix/size check here. + # Traversal direction is not narrowed the way the old undirected + # node_link_graph load was: the store keeps each edge's native + # source→target orientation, and _query_graph_text walks both incident + # directions, so caller-side results for a seed with no outgoing edges are + # still reachable. _connect_graph also emits the pre-#1504 legacy-node-ID + # nudge that the old json loader printed here. + G = _connect_graph(graph_path) import time as _time _t0 = _time.perf_counter() _mode = "dfs" if use_dfs else "bfs" @@ -966,7 +928,7 @@ def dispatch_command(cmd: str) -> None: if len(sys.argv) < 3: print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) sys.exit(1) - from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph + from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, connect_graph query = sys.argv[2] graph_path = _default_graph_path() depth = 2 @@ -1002,17 +964,15 @@ def dispatch_command(cmd: str) -> None: i += 1 else: i += 1 + # FalkorDB-only: the graph lives in the engine, located via the + # falkordb.json pointer next to graph_path — the legacy graph.json file + # need not exist. connect_graph resolves the pointer and guards the + # empty/missing case (matches the `query` command). gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) try: - graph = load_graph(gp) + graph = connect_graph(gp) except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) + print(f"error: could not connect to graph: {exc}", file=sys.stderr) sys.exit(1) print( format_affected( @@ -1027,7 +987,7 @@ def dispatch_command(cmd: str) -> None: # README-advertised capability, but never a CLI subcommand — `graphify # god_nodes` fell through to "unknown command" (#2004). Wire it as a # read-only graph query, mirroring `affected`. - from graphify.affected import load_graph + from graphify.affected import connect_graph from graphify.analyze import god_nodes as _god_nodes from graphify.security import sanitize_label as _sanitize_label graph_path = _default_graph_path() @@ -1059,16 +1019,13 @@ def dispatch_command(cmd: str) -> None: else: i += 1 gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) + # FalkorDB-only: the graph lives in the engine, located via the + # falkordb.json pointer next to graph_path, so the legacy graph.json file + # need not exist (matches the `affected` command). try: - G = load_graph(gp) + G = connect_graph(gp) except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) + print(f"error: could not connect to graph: {exc}", file=sys.stderr) sys.exit(1) gods = _god_nodes(G, top_n=top_n) if as_json: @@ -1172,9 +1129,7 @@ def dispatch_command(cmd: str) -> None: file=sys.stderr, ) sys.exit(1) - from graphify.serve import _pick_scored_endpoint, _score_nodes - from networkx.readwrite import json_graph - import networkx as _nx + from graphify.serve import _pick_scored_endpoint, _score_nodes, _connect_graph source_label = sys.argv[2] target_label = sys.argv[3] @@ -1184,24 +1139,12 @@ def dispatch_command(cmd: str) -> None: if a == "--graph" and i + 1 < len(args): graph_path = args[i + 1] gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee - # direction, and multigraph so exact-pair parallel links (e.g. a - # `references` and a `calls` edge between the same two nodes) survive load - # instead of being silently collapsed last-writer-wins — otherwise the - # printed relation could be one the traversed pair doesn't actually - # carry (#2074). Local to this read; serve's shared graph is untouched. - _raw = {**_raw, "directed": True, "multigraph": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) + # FalkorDB-only: the graph lives in the engine (no graph.json file needed). + # The store keeps each edge's native caller→callee orientation and its + # parallel links, so the #2074 "directed + multigraph" load flags have no + # analogue here — direction comes from has_directed_edge and every + # parallel relation from edge_datas/edge_attrs_all. + G = _connect_graph(graph_path) src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) if not src_scored: @@ -1237,17 +1180,11 @@ def dispatch_command(cmd: str) -> None: f"(top score {_top:g}, runner-up {_runner:g})", file=sys.stderr, ) - # Deterministic shortest path (#2074): to_undirected(as_view=True) - # iterates neighbors via a hash-seeded set union, so among equal-length - # paths BFS returned an arbitrary route that varied per process. Build a - # sorted, materialized undirected graph so neighbor order — and thus the - # chosen path — is canonical for a given graph.json. - _und = _nx.Graph() - _und.add_nodes_from(sorted(G.nodes)) - _und.add_edges_from(sorted((min(u, v), max(u, v)) for u, v in G.edges())) - try: - path_nodes = _nx.shortest_path(_und, src_nid, tgt_nid) - except (_nx.NetworkXNoPath, _nx.NodeNotFound): + # Deterministic shortest path (#2074): the store's BFS already walks a + # sorted adjacency, so among equal-length paths the chosen route is + # canonical for a given graph — no sorted scratch copy needed here. + path_nodes = G.shortest_path(src_nid, tgt_nid) + if not path_nodes: print(f"No path found between '{source_label}' and '{target_label}'.") sys.exit(0) hops = len(path_nodes) - 1 @@ -1259,7 +1196,7 @@ def dispatch_command(cmd: str) -> None: # direction — never a fabricated `calls` (#2074). A pair may carry # several parallel relations; show all, and fall back to an honest # "related" when the stored edge has no relation. - if G.has_edge(u, v): + if G.has_directed_edge(u, v): datas = edge_datas(G, u, v) forward = True else: @@ -1289,8 +1226,7 @@ def dispatch_command(cmd: str) -> None: if len(sys.argv) < 3: print('Usage: graphify explain "" [--graph path]', file=sys.stderr) sys.exit(1) - from graphify.serve import _find_node - from networkx.readwrite import json_graph + from graphify.serve import _find_node, _connect_graph label = sys.argv[2] graph_path = _default_graph_path() @@ -1299,25 +1235,33 @@ def dispatch_command(cmd: str) -> None: if a == "--graph" and i + 1 < len(args): graph_path = args[i + 1] gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) + G = _connect_graph(graph_path) matches = _find_node(G, label) if not matches: print(f"No node matching '{label}' found.") sys.exit(0) nid = matches[0] - d = G.nodes[nid] + if hasattr(G, "node_connections"): + # FalkorDB-native: node detail + connections in two scoped queries + # instead of walking successors/predecessors node-by-node. + d, deg = G.node_detail(nid) + connections = [ + (c["dir"], c["label"], c["relation"], c["confidence"], c["degree"], + c["source_file"], c["source_location"]) + for c in G.node_connections(nid) + ] + else: + from graphify.build import edge_data + d = G.nodes[nid] + deg = G.degree(nid) + _raw_conns = [("out", nb, edge_data(G, nid, nb)) for nb in G.successors(nid)] + _raw_conns += [("in", nb, edge_data(G, nb, nid)) for nb in G.predecessors(nid)] + connections = [ + (dr, G.nodes[nb].get("label", nb), e.get("relation", ""), + e.get("confidence", ""), G.degree(nb), + e.get("source_file") or "", e.get("source_location") or "") + for dr, nb, e in _raw_conns + ] print(f"Node: {d.get('label', nid)}") print(f" ID: {nid}") print( @@ -1349,27 +1293,17 @@ def dispatch_command(cmd: str) -> None: print(_line) except Exception: pass - print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data - connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) - for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) - for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) + print(f" Degree: {deg}") if connections: print(f"\nConnections ({len(connections)}):") - connections.sort(key=lambda c: G.degree(c[1]), reverse=True) - for direction, nb, edata in connections[:20]: - rel = edata.get("relation", "") - conf = edata.get("confidence", "") + connections.sort(key=lambda c: c[4], reverse=True) + for direction, nb_label, rel, conf, _deg, sfile, loc in connections[:20]: arrow = "-->" if direction == "out" else "<--" # Append the edge's location — the actual call/import/reference # SITE (in the caller's file for an incoming call), not a def # line (#BUG1). Labeled by [rel] so the meaning is unambiguous. - loc = edata.get("source_location") or "" - sfile = edata.get("source_file") or "" at = f" {sfile}:{loc}" if loc else "" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]{at}") + print(f" {arrow} {nb_label} [{rel}] [{conf}]{at}") if len(connections) > 20: remainder = connections[20:] print(f" ... and {len(remainder)} more") @@ -1378,9 +1312,8 @@ def dispatch_command(cmd: str) -> None: # connections by direction + file so their shape is visible # without falling back to a repo-wide grep. by_file: dict[tuple[str, str], int] = {} - for direction, _nb, edata in remainder: - sfile = edata.get("source_file") or "(unknown file)" - key = (direction, sfile) + for direction, _nb_label, _rel, _conf, _deg, sfile, _loc in remainder: + key = (direction, sfile or "(unknown file)") by_file[key] = by_file.get(key, 0) + 1 # Count desc, then (direction, file) so equal-count groups have a # byte-stable order (not the degree-derived insertion order). @@ -1605,15 +1538,11 @@ def dispatch_command(cmd: str) -> None: i_arg += 1 if watch_path is None: watch_path = Path(".") + # FalkorDB-only: graph_json locates the falkordb.json pointer (its parent + # dir); the legacy graph.json artifact need not exist. _connect_graph + # resolves the pointer and errors with "run /graphify" if no graph is built. graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): - print( - f"error: no graph found at {graph_json} — run /graphify first", - file=sys.stderr, - ) - sys.exit(1) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json + from graphify.serve import _connect_graph from graphify.cluster import cluster, score_all, remap_communities_to_previous from graphify.analyze import ( god_nodes, @@ -1624,29 +1553,13 @@ def dispatch_command(cmd: str) -> None: from graphify.export import to_json, to_html stages = _StageTimer(co_timing) - print("Loading existing graph...") - # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. - # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the - # graph.html render below falls back to the community-aggregation view - # (node_limit=5000) when over the cap. - from graphify.security import check_graph_file_size_cap as _check_cap - _over_cap = False - try: - _check_cap(graph_json) - except ValueError: - _over_cap = True - try: - _over_cap_bytes = graph_json.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) - _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) + print("Connecting to existing graph...") + G = _connect_graph(str(graph_json)) + # No graph.json file-size cap in FalkorDB mode (the graph lives in the + # engine). Preserve the #1019 huge-graph guard by node count: very large + # graphs still fall back to the community-aggregation view (node_limit=5000) + # in the graph.html render below. + _over_cap = G.number_of_nodes() > 5000 print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") stages.mark("load") print("Re-clustering...") @@ -1657,9 +1570,9 @@ def dispatch_command(cmd: str) -> None: # labels follow raw cid index and become misaligned whenever the # graph has changed between labeling and cluster-only (#1027). previous_node_community = { - n["id"]: n["community"] - for n in _raw.get("nodes", []) - if n.get("community") is not None and n.get("id") is not None + nid: attrs["community"] + for nid, attrs in G.nodes(data=True) + if attrs.get("community") is not None } if previous_node_community: communities = remap_communities_to_previous(communities, previous_node_community) @@ -2021,43 +1934,26 @@ def dispatch_command(cmd: str) -> None: # the merge so a human can investigate. _MERGE_MAX_BYTES = 50 * 1024 * 1024 _MERGE_MAX_NODES = 100_000 - import networkx as _nx - from networkx.readwrite import json_graph as _jg - def _load_graph(p: str): - path_obj = Path(p) - try: - size = path_obj.stat().st_size - except OSError as exc: - raise RuntimeError(f"cannot stat {p}: {exc}") from exc - if size > _MERGE_MAX_BYTES: - raise RuntimeError( - f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" - ) - data = json.loads(path_obj.read_text(encoding="utf-8")) - # A committed raw (--no-cluster) graph stores edges under "edges"; - # parse via the shared links/edges-normalizing loader (#2212). - from graphify.paths import load_node_link_graph as _lnlg - return _lnlg(data), data + # Pure-JSON node-link handling, no NetworkX: this merges graph.json + # artifacts, not the live store. load_node_link enforces the byte cap and + # normalizes a raw --no-cluster graph's "edges" key to "links" (#2212). + from graphify.graphjson import load_node_link, merge_node_link, node_count try: - G_cur, _ = _load_graph(_current_path) - G_oth, _ = _load_graph(_other_path) + cur = load_node_link(_current_path, max_bytes=_MERGE_MAX_BYTES) + oth = load_node_link(_other_path, max_bytes=_MERGE_MAX_BYTES) except Exception as exc: print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge - merged = _nx.compose(G_cur, G_oth) - if merged.number_of_nodes() > _MERGE_MAX_NODES: + merged = merge_node_link([cur, oth]) + if node_count(merged) > _MERGE_MAX_NODES: print( - f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " + f"[graphify merge-driver] merged graph has {node_count(merged)} nodes, " f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", file=sys.stderr, ) sys.exit(1) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) from graphify.paths import write_json_atomic - write_json_atomic(_current_path, out_data, indent=2) + write_json_atomic(_current_path, merged, indent=2) sys.exit(0) elif cmd == "merge-graphs": @@ -2079,40 +1975,22 @@ def _load_graph(p: str): file=sys.stderr, ) sys.exit(1) - import networkx as _nx - from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix, distinct_repo_tags as _repo_tags - graphs = [] + # Pure-JSON node-link handling, no NetworkX: these are graph.json + # artifacts, not the live store. load_node_link normalizes an "edges"-keyed + # graph to "links" (#738/#2212). The old nx.compose type-normalization + # (#1606: directed vs undirected, multi vs simple) has no analogue — plain + # dicts carry no graph type to conflict on. + from graphify.build import distinct_repo_tags as _repo_tags + from graphify.graphjson import ( + load_node_link, merge_node_link, prefix_node_link, node_count, edge_count, + ) + datas = [] for gp in graph_paths: if not gp.exists(): print(f"error: not found: {gp}", file=sys.stderr) sys.exit(1) _enforce_graph_size_cap_or_exit(gp) - data = json.loads(gp.read_text(encoding="utf-8")) - # Normalize edges/links key before loading — graphify writes "links" - # via node_link_data but older runs may have used "edges" (#738). - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = _jg.node_link_graph(data, edges="links") - except TypeError: - G = _jg.node_link_graph(data) - graphs.append(G) - # nx.compose requires all graphs to be the same type. When input graphs - # come from different sources (e.g. an AST-only run vs a full LLM run) one - # may be a MultiGraph and another a Graph. Normalise everything to Graph - # (the graphify default) by converting MultiGraphs with nx.Graph(). - def _to_simple(g: "_nx.Graph") -> "_nx.Graph": - # nx.compose requires every graph to be the same type. Inputs may - # disagree on BOTH axes — directed vs undirected, and multi vs simple - # — because per-repo graph.json files are written by different extract - # paths at different times. Normalise everything to a plain undirected - # Graph (the merged cross-repo view is undirected anyway), which covers - # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input - # crashed compose with "All graphs must be directed or undirected" (#1606). - if type(g) is not _nx.Graph: - return _nx.Graph(g) - return g + datas.append(load_node_link(gp)) # Unique repo tag per graph. The bare `graphify-out/..` dir name is not # unique across inputs (src/graphify-out and frontend/src/graphify-out both # → "src"), which collides same-stem node ids and silently merges unrelated @@ -2121,18 +1999,14 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": naive_tags = [gp.parent.parent.name for gp in graph_paths] if len(set(naive_tags)) != len(naive_tags): print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") - merged = _nx.Graph() - for G, repo_tag in zip(graphs, repo_tags): - prefixed = _to_simple(_prefix(G, repo_tag)) - merged = _nx.compose(merged, prefixed) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) + merged = merge_node_link( + [prefix_node_link(data, repo_tag) for data, repo_tag in zip(datas, repo_tags)], + directed=False, # the combined cross-repo view is undirected (#1606) + ) out_path.parent.mkdir(parents=True, exist_ok=True) from graphify.paths import write_json_atomic as _wja - _wja(out_path, out_data, indent=2) - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") + _wja(out_path, merged, indent=2) + print(f"Merged {len(datas)} graphs -> {node_count(merged)} nodes, {edge_count(merged)} edges") print(f"Written to: {out_path}") elif cmd == "clone": @@ -2307,39 +2181,14 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print(f"callflow HTML written - open in any browser: {out}") sys.exit(0) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json as _bfj - from graphify.security import check_graph_file_size_cap as _check_cap + from graphify.serve import _connect_graph - # Solution 3 (#1019): for the HTML view, an oversized graph.json should - # not be a hard error. Detect the over-cap condition here and fall back - # to the community-aggregation view (node_limit=5000) below instead of - # exiting 1. All other subcommands keep the hard cap. - _over_cap = False - try: - _check_cap(graph_path) - except ValueError as _cap_err: - if subcmd == "html": - _over_cap = True - try: - _over_cap_bytes = graph_path.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - else: - print(f"error: {_cap_err}", file=sys.stderr) - sys.exit(1) - _raw = json.loads(graph_path.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = _jg.node_link_graph(_raw, edges="links") - except TypeError: - G = _jg.node_link_graph(_raw) + G = _connect_graph(str(graph_path)) + # No graph.json file-size cap in FalkorDB mode (the graph lives in the + # engine). Preserve the #1019 html guard by node count: for the html view, + # very large graphs fall back to the community-aggregation view + # (node_limit=5000) below instead of rendering an unusable full graph. + _over_cap = subcmd == "html" and G.number_of_nodes() > 5000 # Load optional analysis/labels communities: dict[int, list[str]] = {} @@ -3480,6 +3329,10 @@ def _invalidate_file_manifest_for_db_graph() -> None: from graphify.export import to_json as _to_json from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising dedup_backend = backend if dedup_llm else None + # Bind every build/merge/export step to the one FalkorDB graph for this + # output dir (open_store writes the pointer file so query/serve/etc. find it). + from graphify.store import open_store as _open_store + _store = _open_store(graphify_out, create=True) if incremental_mode: # Prune everything the current scan no longer covers: genuinely # deleted manifest rows, excluded-but-alive manifest rows (#1908), @@ -3491,14 +3344,15 @@ def _invalidate_file_manifest_for_db_graph() -> None: _prune_sources.append(_src) G = _build_merge( [merged], - graph_path=existing_graph_path, + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=_prune_sources or None, dedup=True, dedup_llm_backend=dedup_backend, root=target, ) else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) + G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target, store=_store) stages.mark("build") if G.number_of_nodes() == 0: print( diff --git a/graphify/cluster.py b/graphify/cluster.py index 682210700..d553ef5c8 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -1,80 +1,19 @@ -"""Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores.""" +"""Community detection on the FalkorDB-backed GraphStore. Runs Louvain via a +server-side JS UDF (graphify_algos.louvain), splits oversized/low-cohesion +communities, and returns stable community IDs with cohesion scores.""" from __future__ import annotations -import contextlib -import inspect -import io -import json -import sys -import networkx as nx -def _suppress_output(): - """Context manager to suppress stdout/stderr during library calls. +def _partition(G, resolution: float = 1.0) -> dict[str, int]: + """Run Louvain community detection. Returns {node_id: community_id}. - graspologic's leiden() emits ANSI escape sequences (progress bars, - colored warnings) that corrupt PowerShell 5.1's scroll buffer on - Windows (see issue #19). Redirecting stdout/stderr to devnull during - the call prevents this without losing any graphify output. - """ - return contextlib.redirect_stdout(io.StringIO()) - - -def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: - """Run community detection. Returns {node_id: community_id}. - - Tries Leiden (graspologic) first — best quality. - Falls back to Louvain (built into networkx) if graspologic is not installed. + Delegates to the server-side `graphify_algos.louvain` UDF (deterministic: + sorted node order, no RNG). `G` is a GraphStore or a subgraph view, both of + which expose `louvain_partition(resolution)`. - resolution > 1.0 → more, smaller communities. - resolution < 1.0 → fewer, larger communities. - - Output from graspologic is suppressed to prevent ANSI escape codes - from corrupting terminal scroll buffers on Windows PowerShell 5.1. + resolution > 1.0 → more, smaller communities; < 1.0 → fewer, larger. """ - stable = nx.Graph() - stable.add_nodes_from(sorted(G.nodes(), key=str)) - edge_rows = sorted( - G.edges(data=True), - key=lambda row: ( - str(row[0]), - str(row[1]), - json.dumps(row[2], sort_keys=True, ensure_ascii=False, default=str), - ), - ) - for src, tgt, attrs in edge_rows: - stable.add_edge(src, tgt, **attrs) - - try: - from graspologic.partition import leiden - lsig = inspect.signature(leiden).parameters - kwargs: dict = {} - if "random_seed" in lsig: - kwargs["random_seed"] = 42 - if "trials" in lsig: - kwargs["trials"] = 1 - if "resolution" in lsig: - kwargs["resolution"] = resolution - # Suppress graspologic output to prevent ANSI escape codes from - # corrupting PowerShell 5.1 scroll buffer (issue #19) - old_stderr = sys.stderr - try: - sys.stderr = io.StringIO() - with _suppress_output(): - result = leiden(stable, **kwargs) - finally: - sys.stderr = old_stderr - return result - except ImportError: - pass - - # Fallback: networkx louvain (available since networkx 2.7). - # Inspect kwargs to stay compatible across NetworkX versions — max_level - # was added in a later release and prevents hangs on large sparse graphs. - kwargs: dict = {"seed": 42, "threshold": 1e-4, "resolution": resolution} - if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: - kwargs["max_level"] = 10 - communities = nx.community.louvain_communities(stable, **kwargs) - return {node: cid for cid, nodes in enumerate(communities) for node in nodes} + return G.louvain_partition(resolution=resolution) _MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split @@ -159,21 +98,27 @@ def cluster( if G.number_of_edges() == 0: return {i: [n] for i, n in enumerate(sorted(G.nodes))} + # Materialize the full degree map in one streamed pass. Per-node G.degree(n) + # against the FalkorDB store is a round-trip each, so the comprehensions below + # would otherwise issue O(N) queries (twice over); one dict() keeps it to a + # single scan, matching the old in-memory nx cost. + degmap = dict(G.degree()) + # Compute hub exclusion set before removing anything so degree is based on full graph hub_nodes: set[str] = set() if exclude_hubs_percentile is not None: - degrees = sorted(d for _, d in G.degree()) + degrees = sorted(degmap.values()) if degrees: idx = max(0, int(len(degrees) * exclude_hubs_percentile / 100) - 1) threshold = degrees[idx] - hub_nodes = {n for n, d in G.degree() if d > threshold} + hub_nodes = {n for n, d in degmap.items() if d > threshold} # Leiden warns and drops isolates - handle them separately # Also exclude hub nodes from partitioning so they don't pull unrelated # subsystems into the same community excluded = hub_nodes - isolates = [n for n in G.nodes() if G.degree(n) == 0 and n not in excluded] - connected_nodes = [n for n in G.nodes() if G.degree(n) > 0 and n not in excluded] + isolates = [n for n, d in degmap.items() if d == 0 and n not in excluded] + connected_nodes = [n for n, d in degmap.items() if d > 0 and n not in excluded] connected = G.subgraph(connected_nodes) raw: dict[int, list[str]] = {} diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index fcb9a11cf..05d794e5c 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -9,8 +9,6 @@ from pathlib import Path from typing import Any -import networkx as nx - _SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") _TYPE_TUPLE_RE = re.compile(r"set\[tuple\[(?P[^\]]+)\]\]") @@ -234,7 +232,7 @@ def diagnose_extraction( post_build_node_count: int | None = None try: graph_input = deepcopy(extraction) - graph: nx.Graph = build_from_json(graph_input, directed=directed, root=root) + graph = build_from_json(graph_input, directed=directed, root=root) graph_type = type(graph).__name__ post_build_edge_count = graph.number_of_edges() post_build_node_count = graph.number_of_nodes() diff --git a/graphify/export.py b/graphify/export.py index e1f2caa99..5a0d30d69 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -11,8 +11,6 @@ from collections import Counter from datetime import date from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph from graphify.security import sanitize_label from graphify.analyze import _node_community_map from graphify.build import edge_data @@ -287,30 +285,45 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, return False node_community = _node_community_map(communities) + # FalkorDB is the source of truth: persist community ids onto the stored + # nodes. graph.json is now a derived export artifact built from the store. + if communities and hasattr(G, "set_communities"): + G.set_communities(communities) + _labels: dict[int, str] = {int(k): v for k, v in (community_labels or {}).items()} - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - for node in data["nodes"]: - cid = node_community.get(node["id"]) - node["community"] = cid + nodes = [] + for nid, attrs in G.nodes(data=True): + nd = {k: v for k, v in attrs.items() if not k.startswith("_")} + # Keep _origin (AST provenance): incremental rebuild reads it back to + # evict stale AST symbols removed from a surviving file (#1116). + if isinstance(attrs.get("_origin"), str): + nd["_origin"] = attrs["_origin"] + nd["id"] = nid + cid = node_community.get(nid) + nd["community"] = cid + # Carry the community name into the export (#1305) so query/MCP show + # names, not bare ids. if cid is not None and _labels: - node["community_name"] = _labels.get(cid, f"Community {cid}") - node["norm_label"] = _strip_diacritics(node.get("label", "")).lower() - for link in data["links"]: - if "confidence_score" not in link: - conf = link.get("confidence", "EXTRACTED") - link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) - # Restore original edge direction. Undirected NetworkX storage may - # canonicalize endpoint order, flipping `calls` and other directional - # edges in graph.json. The build path stashes the true endpoints in - # _src/_tgt for exactly this purpose (#563). - true_src = link.pop("_src", None) - true_tgt = link.pop("_tgt", None) - if true_src is not None and true_tgt is not None: - link["source"] = true_src - link["target"] = true_tgt + nd["community_name"] = _labels.get(cid, f"Community {cid}") + nd["norm_label"] = _strip_diacritics(nd.get("label", "")).lower() + nodes.append(nd) + # Edges are stored in native source→target direction, so no _src/_tgt restore. + links = [] + for u, v, attrs in G.edges(data=True): + ld = {k: val for k, val in attrs.items() if not k.startswith("_")} + ld["source"] = u + ld["target"] = v + if "confidence_score" not in ld: + conf = ld.get("confidence", "EXTRACTED") + ld["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) + links.append(ld) + data = { + "directed": True, + "multigraph": False, + "graph": {}, + "nodes": nodes, + "links": links, + } data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) commit = built_at_commit if built_at_commit is not None else _git_head() if commit: @@ -970,24 +983,12 @@ def to_graphml( Community IDs are written as a node attribute so Gephi can colour by community. Edge confidence (EXTRACTED/INFERRED/AMBIGUOUS) is preserved as an edge attribute. """ - H = G.copy() node_community = _node_community_map(communities) - for node_id in H.nodes(): - H.nodes[node_id]["community"] = node_community.get(node_id, -1) - # Drop internal markers (e.g. the AST-provenance "_origin" tag, #1116, and - # the "_src"/"_tgt" direction markers) — they are persistence/runtime details, - # not graph data, and should not leak into the exported file. - for _, attrs in H.nodes(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - for _, _, attrs in H.edges(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - # nx.write_graphml only accepts scalar attribute values: None raises, and a - # dict/list value (e.g. a per-node `metadata` dict, or the graph-level - # `hyperedges` list set by attach_hyperedges()) raises - # "GraphML does not support type as data values" (#1831). - # Coerce None -> "" and non-scalars -> a JSON string, across all three scopes. + + # GraphML only accepts scalar attribute values: None has no representation, + # and a dict/list value (e.g. a per-node `metadata` dict, or the graph-level + # `hyperedges` list set by attach_hyperedges()) is not expressible (#1831). + # Coerce None -> "" and non-scalars -> a JSON string. def _graphml_safe(val): if val is None: return "" @@ -998,22 +999,71 @@ def _graphml_safe(val): except (TypeError, ValueError): return str(val) - for key, val in list(H.graph.items()): - H.graph[key] = _graphml_safe(val) - for node_id in H.nodes(): - for key, val in list(H.nodes[node_id].items()): - H.nodes[node_id][key] = _graphml_safe(val) - for u, v in H.edges(): - for key, val in list(H.edges[u, v].items()): - H.edges[u, v][key] = _graphml_safe(val) - - # Write atomically: a mid-serialization error otherwise leaves a 0-byte + # Collect nodes/edges (dropping internal "_"-prefixed markers) and the set of + # attribute keys, so we can declare GraphML elements up front. + node_rows = [] + node_keys: dict[str, str] = {} # attr name -> graphml type + for nid, attrs in G.nodes(data=True): + clean = {k: _graphml_safe(v) for k, v in attrs.items() + if not k.startswith("_") and k != "id"} + clean["community"] = node_community.get(nid, -1) + node_rows.append((nid, clean)) + for k, v in clean.items(): + node_keys.setdefault(k, _graphml_type(v)) + + edge_rows = [] + edge_keys: dict[str, str] = {} + for u, v, attrs in G.edges(data=True): + clean = {k: _graphml_safe(val) for k, val in attrs.items() if not k.startswith("_")} + edge_rows.append((u, v, clean)) + for k, val in clean.items(): + edge_keys.setdefault(k, _graphml_type(val)) + + def esc(x): + return _html.escape(str(x), quote=True) + + lines = [ + '', + '', + ] + # Graph-level attributes (notably the `hyperedges` list attached by + # attach_hyperedges) are part of the export too — dropping them would lose + # the hypergraph layer entirely. Same scalar coercion as nodes/edges (#1831). + graph_rows = { + k: _graphml_safe(v) for k, v in getattr(G, "graph", {}).items() + if not k.startswith("_") + } + for name, gtype in node_keys.items(): + lines.append(f' ') + for name, gtype in edge_keys.items(): + lines.append(f' ') + for name, val in graph_rows.items(): + lines.append( + f' ' + ) + lines.append(' ') + for name, val in graph_rows.items(): + lines.append(f' {esc(val)}') + for nid, attrs in node_rows: + lines.append(f' ') + for k, v in attrs.items(): + lines.append(f' {esc(v)}') + lines.append(" ") + for i, (u, v, attrs) in enumerate(edge_rows): + lines.append(f' ') + for k, val in attrs.items(): + lines.append(f' {esc(val)}') + lines.append(" ") + lines.append(" ") + lines.append("") + # Write atomically: a mid-serialization error otherwise leaves a truncated # .graphml on disk that downstream tooling mistakes for a completed export # (#1831). Write to a sibling temp file, then replace on success. out = Path(output_path) tmp = out.with_name(out.name + ".tmp") try: - nx.write_graphml(H, str(tmp)) + tmp.write_text("\n".join(lines), encoding="utf-8") os.replace(str(tmp), str(out)) finally: if tmp.exists(): @@ -1023,6 +1073,77 @@ def _graphml_safe(val): pass +def _graphml_type(value) -> str: + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "long" + if isinstance(value, float): + return "double" + return "string" + + +def _spring_layout(G, seed: int = 42, iterations: int = 60) -> dict: + """Deterministic Fruchterman-Reingold layout (replaces nx.spring_layout). + + Pure-Python, seeded for reproducibility. Returns {node_id: (x, y)} in roughly + [-1, 1]. Good enough for the static SVG overview; not performance-critical. + """ + import math + import random as _random + + nodes = list(G.nodes()) + n = len(nodes) + if n == 0: + return {} + rng = _random.Random(seed) + pos = {nid: [rng.uniform(-1, 1), rng.uniform(-1, 1)] for nid in nodes} + if n == 1: + return {nodes[0]: (0.0, 0.0)} + adj = {nid: set() for nid in nodes} + for u, v in G.edges(): + if u in adj and v in adj: + adj[u].add(v) + adj[v].add(u) + k = math.sqrt(1.0 / n) + t = 0.1 + for _ in range(iterations): + disp = {nid: [0.0, 0.0] for nid in nodes} + for i in range(n): + a = nodes[i] + for j in range(i + 1, n): + b = nodes[j] + dx = pos[a][0] - pos[b][0] + dy = pos[a][1] - pos[b][1] + dist = math.hypot(dx, dy) or 0.01 + rep = (k * k) / dist + ux, uy = dx / dist, dy / dist + disp[a][0] += ux * rep; disp[a][1] += uy * rep + disp[b][0] -= ux * rep; disp[b][1] -= uy * rep + for a in nodes: + for b in adj[a]: + if a >= b: + continue + dx = pos[a][0] - pos[b][0] + dy = pos[a][1] - pos[b][1] + dist = math.hypot(dx, dy) or 0.01 + att = (dist * dist) / k + ux, uy = dx / dist, dy / dist + disp[a][0] -= ux * att; disp[a][1] -= uy * att + disp[b][0] += ux * att; disp[b][1] += uy * att + for nid in nodes: + dlen = math.hypot(*disp[nid]) or 0.01 + pos[nid][0] += (disp[nid][0] / dlen) * min(dlen, t) + pos[nid][1] += (disp[nid][1] / dlen) * min(dlen, t) + t = max(t * 0.95, 0.01) + return {nid: (p[0], p[1]) for nid, p in pos.items()} + + +# Above this node count, to_svg renders a top-degree overview instead of the full +# graph: a full-graph SVG is an unreadable hairball and the pure-Python force +# layout is O(n^2) per iteration, so the full graph would be impractically slow. +_SVG_MAX_NODES = 600 + def to_svg( G: nx.Graph, communities: dict[int, list[str]], @@ -1045,19 +1166,33 @@ def to_svg( except ImportError as e: raise ImportError("matplotlib not installed. Run: pip install matplotlib") from e + # For large graphs, render a top-degree overview built as an in-memory + # MemGraph (keeps the full node/edge/degree drawing API) so the SVG stays + # readable and the O(n^2) layout stays bounded. + if G.number_of_nodes() > _SVG_MAX_NODES and hasattr(G, "top_degree_nodes"): + from graphify.store import MemGraph + _top = [d["id"] for d in G.top_degree_nodes(limit=_SVG_MAX_NODES)] + _top_set = set(_top) + _attrs = G.node_attrs_batch(_top) + _nodes = [(nid, _attrs.get(nid, {})) for nid in _top] + _edges = [(u, v, d) for u, v, d in G.edges(nbunch=_top, data=True) + if u in _top_set and v in _top_set] + G = MemGraph(_nodes, _edges, directed=G.is_directed()) + node_community = _node_community_map(communities) fig, ax = plt.subplots(figsize=figsize, facecolor="#1a1a2e") ax.set_facecolor("#1a1a2e") ax.axis("off") - pos = nx.spring_layout(G, seed=42, k=2.0 / (G.number_of_nodes() ** 0.5 + 1)) + pos = _spring_layout(G, seed=42) degree = dict(G.degree()) max_deg = max(degree.values(), default=1) or 1 - node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in G.nodes()] - node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in G.nodes()] + nodes_list = list(G.nodes()) + node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in nodes_list] + node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in nodes_list] # Draw edges - dashed for non-EXTRACTED for u, v, data in G.edges(data=True): @@ -1069,11 +1204,13 @@ def to_svg( ax.plot([x0, x1], [y0, y1], color="#aaaaaa", linewidth=0.8, linestyle=style, alpha=alpha, zorder=1) - nx.draw_networkx_nodes(G, pos, ax=ax, node_color=node_colors, - node_size=node_sizes, alpha=0.9) - nx.draw_networkx_labels(G, pos, ax=ax, - labels={n: G.nodes[n].get("label", n) for n in G.nodes()}, - font_size=7, font_color="white") + xs = [pos[n][0] for n in nodes_list] + ys = [pos[n][1] for n in nodes_list] + ax.scatter(xs, ys, s=node_sizes, c=node_colors, alpha=0.9, zorder=2, edgecolors="none") + for n in nodes_list: + x, y = pos[n] + ax.annotate(str(G.nodes[n].get("label", n)), (x, y), fontsize=7, + color="white", ha="center", va="center", zorder=3) # Legend if community_labels: diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py index 14c47f0d5..5a4a0449c 100644 --- a/graphify/exporters/graphdb.py +++ b/graphify/exporters/graphdb.py @@ -2,7 +2,6 @@ from __future__ import annotations from graphify.analyze import _node_community_map -import networkx as nx import re diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py index 59c0e52e3..04fad59fc 100644 --- a/graphify/exporters/html.py +++ b/graphify/exporters/html.py @@ -6,7 +6,6 @@ import html as _html from graphify.analyze import _node_community_map import json -import networkx as nx from graphify.security import sanitize_label @@ -348,20 +347,26 @@ def to_html( if node_limit is not None: # Build aggregated community meta-graph from collections import Counter as _Counter - import networkx as _nx + from graphify.store import MemGraph print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - meta = _nx.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) + meta_nodes = [ + (str(cid), {"label": (community_labels or {}).get(cid, f"Community {cid}")}) + for cid, members in communities.items() + ] edge_counts = _Counter() for u, v in G.edges(): cu, cv = node_to_community.get(u), node_to_community.get(v) if cu is not None and cv is not None and cu != cv: edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, - relation=f"{w} cross-community edges", confidence="AGGREGATED") + meta_edges = [ + (str(cu), str(cv), {"weight": w, "relation": f"{w} cross-community edges", + "confidence": "AGGREGATED"}) + for (cu, cv), w in edge_counts.items() + ] + # MemGraph, not nx.Graph: the meta-graph feeds the same rendering API + # the store-backed graph uses, so the viz path stays NetworkX-free. + meta = MemGraph(meta_nodes, meta_edges) if meta.number_of_nodes() <= 1: print("Single community - aggregated view not useful. Skipping graph.html.") return diff --git a/graphify/global_graph.py b/graphify/global_graph.py index eddd0c92a..ab9e419e7 100644 --- a/graphify/global_graph.py +++ b/graphify/global_graph.py @@ -4,12 +4,12 @@ import sys from datetime import datetime, timezone from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph as _jg + +from .store import GraphStore, DEFAULT_URI, open_store _GLOBAL_DIR = Path.home() / ".graphify" -_GLOBAL_GRAPH = _GLOBAL_DIR / "global-graph.json" _GLOBAL_MANIFEST = _GLOBAL_DIR / "global-manifest.json" +_GLOBAL_NAME = "graphify_global" def _load_manifest() -> dict: @@ -46,33 +46,18 @@ def _save_manifest(manifest: dict) -> None: write_json_atomic(_GLOBAL_MANIFEST, manifest, indent=2) -def _load_global_graph() -> nx.Graph: - if _GLOBAL_GRAPH.exists(): - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(_GLOBAL_GRAPH) - data = json.loads(_GLOBAL_GRAPH.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - return _jg.node_link_graph(data, edges="links") - except TypeError: - return _jg.node_link_graph(data) - return nx.Graph() - - -def _save_global_graph(G: nx.Graph) -> None: - _GLOBAL_DIR.mkdir(parents=True, exist_ok=True) - try: - data = _jg.node_link_data(G, edges="links") - except TypeError: - data = _jg.node_link_data(G) - from graphify.paths import write_json_atomic - write_json_atomic(_GLOBAL_GRAPH, data, indent=2) +def _load_global_graph(uri: str = DEFAULT_URI) -> GraphStore: + """The global graph is a dedicated named FalkorDB graph.""" + return GraphStore(graph_name=_GLOBAL_NAME, uri=uri) -def _file_hash(path: Path) -> str: +def _store_content_hash(G) -> str: + """Stable content hash of a store's nodes + edges (replaces file hashing).""" h = hashlib.sha256() - h.update(path.read_bytes()) + for nid, attrs in G.nodes(data=True): + h.update(json.dumps([nid, attrs], sort_keys=True, default=str).encode("utf-8")) + for u, v, attrs in G.edges(data=True): + h.update(json.dumps([u, v, attrs], sort_keys=True, default=str).encode("utf-8")) return h.hexdigest()[:16] @@ -82,13 +67,17 @@ def global_add(source_path: Path, repo_tag: str) -> dict: Returns a summary dict with keys: repo_tag, nodes_added, nodes_removed, skipped. Skipped=True means the source graph hasn't changed since last add. """ - from graphify.build import prefix_graph_for_global, prune_repo_from_graph + from graphify.build import prune_repo_from_graph - if not source_path.exists(): - raise FileNotFoundError(f"graph not found: {source_path}") + # Load source graph from its FalkorDB store (source_path is the legacy + # graph.json location; its parent dir holds the FalkorDB pointer). + out_dir = source_path.parent if source_path.suffix else source_path + src_G = open_store(out_dir, create=False) + if src_G.number_of_nodes() == 0: + raise FileNotFoundError(f"graph not found for: {source_path}") manifest = _load_manifest() - src_hash = _file_hash(source_path) + src_hash = _store_content_hash(src_G) existing = manifest["repos"].get(repo_tag, {}) existing_path = existing.get("source_path", "") @@ -102,20 +91,6 @@ def global_add(source_path: Path, repo_tag: str) -> dict: if existing.get("source_hash") == src_hash: return {"repo_tag": repo_tag, "nodes_added": 0, "nodes_removed": 0, "skipped": True} - # Load source graph - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(source_path) - data = json.loads(source_path.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - src_G = _jg.node_link_graph(data, edges="links") - except TypeError: - src_G = _jg.node_link_graph(data) - - # Prefix IDs for cross-project isolation - prefixed = prefix_graph_for_global(src_G, repo_tag) - # Load global graph and prune stale nodes for this repo G = _load_global_graph() removed = prune_repo_from_graph(G, repo_tag) @@ -126,31 +101,40 @@ def global_add(source_path: Path, repo_tag: str) -> dict: for n, d in G.nodes(data=True) if not d.get("source_file") and d.get("label") } - # Map each deduplicated external onto the existing global node so that - # edges incident to it can be rewired instead of dropped. - remap = {} - for node, data in prefixed.nodes(data=True): + # Prefix source IDs for cross-project isolation. External-library nodes + # (no source_file) that already exist in the global graph by label are + # remapped onto the existing global node so incident edges are rewired + # instead of dropped — preserves cross-repo connectivity. + remap: dict[str, str] = {} + prefixed_nodes = [] + for nid, data in src_G.nodes(data=True): + pid = f"{repo_tag}::{nid}" if not data.get("source_file") and data.get("label") in external_labels: - remap[node] = external_labels[data["label"]] - - # Compose: add prefixed nodes (except deduplicated externals) into global graph - for node, data in prefixed.nodes(data=True): - if node not in remap: - G.add_node(node, **data) - for u, v, data in prefixed.edges(data=True): - u = remap.get(u, u) - v = remap.get(v, v) - if u != v: # don't introduce self-loops via remapping - G.add_edge(u, v, **data) - - added = prefixed.number_of_nodes() - len(remap) - _save_global_graph(G) - + remap[pid] = external_labels[data["label"]] + continue + attrs = dict(data) + attrs["repo"] = repo_tag + attrs.setdefault("local_id", nid) + prefixed_nodes.append((pid, attrs)) + prefixed_edges = [] + n_src_edges = 0 + for u, v, data in src_G.edges(data=True): + n_src_edges += 1 + pu = remap.get(f"{repo_tag}::{u}", f"{repo_tag}::{u}") + pv = remap.get(f"{repo_tag}::{v}", f"{repo_tag}::{v}") + if pu == pv: # don't introduce self-loops via remapping + continue + prefixed_edges.append((pu, pv, dict(data))) + + G.add_nodes_from(prefixed_nodes) + G.add_edges_from(prefixed_edges) + + added = len(prefixed_nodes) manifest["repos"][repo_tag] = { "added_at": datetime.now(timezone.utc).isoformat(), "source_path": str(source_path.resolve()), "node_count": added, - "edge_count": prefixed.number_of_edges(), + "edge_count": n_src_edges, "source_hash": src_hash, } _save_manifest(manifest) @@ -168,7 +152,6 @@ def global_remove(repo_tag: str) -> int: G = _load_global_graph() removed = prune_repo_from_graph(G, repo_tag) - _save_global_graph(G) del manifest["repos"][repo_tag] _save_manifest(manifest) @@ -180,5 +163,6 @@ def global_list() -> dict: return _load_manifest().get("repos", {}) -def global_path() -> Path: - return _GLOBAL_GRAPH +def global_path() -> str: + """Name of the global FalkorDB graph (replaces the old global-graph.json path).""" + return _GLOBAL_NAME diff --git a/graphify/graphjson.py b/graphify/graphjson.py new file mode 100644 index 000000000..30ee05800 --- /dev/null +++ b/graphify/graphjson.py @@ -0,0 +1,124 @@ +"""Plain-dict helpers for the node-link graph.json export artifact. + +FalkorDB is the source of truth, but graphify still emits a ``graph.json`` +node-link snapshot (and the git merge-driver / ``merge-graphs`` commands operate +on those artifacts). These helpers union and prefix node-link dicts directly, +without NetworkX. + +A node-link dict looks like:: + + {"directed": true, "multigraph": false, "graph": {}, + "nodes": [{"id": ..., ...}, ...], + "links": [{"source": ..., "target": ..., "relation": ...}, ...]} +""" +from __future__ import annotations + +import json +from pathlib import Path + + +def load_node_link(path, *, max_bytes: int | None = None) -> dict: + """Read a node-link graph.json, normalizing the edges key to ``links``.""" + p = Path(path) + if max_bytes is not None: + size = p.stat().st_size + if size > max_bytes: + raise RuntimeError(f"graph.json {p} is {size} bytes, exceeds {max_bytes}-byte cap") + data = json.loads(p.read_text(encoding="utf-8")) + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + data.setdefault("nodes", []) + data.setdefault("links", []) + return data + + +def _edge_key(e: dict): + return (e.get("source"), e.get("target"), e.get("relation")) + + +def merge_node_link(graphs: list[dict], *, directed: bool | None = None) -> dict: + """Union several node-link dicts. Later graphs win on node/edge attr conflicts. + + ``directed`` sets the merged graph's flag. None (the default) inherits the + first input's, so merging two versions of the same graph round-trips. The + cross-repo `merge-graphs` view passes False: per-repo graphs are written by + different extract paths and may disagree on directedness, and the combined + view is undirected — what the old nx.compose path produced by normalizing + every input to a plain Graph (#1606). + """ + nodes: dict = {} + edges: dict = {} + for g in graphs: + for n in g.get("nodes", []): + nid = n.get("id") + if nid is None: + continue + nodes[nid] = {**nodes.get(nid, {}), **n} + for e in g.get("links", []): + edges[_edge_key(e)] = e + if directed is None: + directed = bool(graphs[0].get("directed", True)) if graphs else True + return { + "directed": directed, + "multigraph": False, + "graph": {}, + "nodes": list(nodes.values()), + "links": list(edges.values()), + } + + +def prefix_node_link(data: dict, repo_tag: str) -> dict: + """Prefix every node id with ``repo_tag::`` and rewrite edge endpoints. + + Mirrors build.prefix_graph_for_global for the JSON artifact: sets ``repo`` and + ``local_id`` on each node so the original id is recoverable. + """ + def pfx(x): + return f"{repo_tag}::{x}" + + nodes = [] + for n in data.get("nodes", []): + nid = n.get("id") + nn = dict(n) + nn["id"] = pfx(nid) + nn["repo"] = repo_tag + nn.setdefault("local_id", nid) + nodes.append(nn) + links = [] + for e in data.get("links", []): + ee = dict(e) + ee["source"] = pfx(e.get("source")) + ee["target"] = pfx(e.get("target")) + links.append(ee) + return {"directed": True, "multigraph": False, "graph": {}, "nodes": nodes, "links": links} + + +def to_node_link(G) -> dict: + """Build a node-link dict from a GraphStore/MemGraph (drops internal `_` keys).""" + nodes = [] + for nid, attrs in G.nodes(data=True): + nd = {k: v for k, v in attrs.items() if not k.startswith("_")} + nd["id"] = nid + nodes.append(nd) + links = [] + for u, v, attrs in G.edges(data=True): + ld = {k: val for k, val in attrs.items() if not k.startswith("_")} + ld["source"] = u + ld["target"] = v + links.append(ld) + return { + "directed": True, + "multigraph": False, + "graph": {}, + "nodes": nodes, + "links": links, + "hyperedges": getattr(G, "graph", {}).get("hyperedges", []), + } + + +def node_count(data: dict) -> int: + return len(data.get("nodes", [])) + + +def edge_count(data: dict) -> int: + return len(data.get("links", [])) diff --git a/graphify/hooks.py b/graphify/hooks.py index b121b6896..f541e23d5 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -279,9 +279,9 @@ def _detached_launch(rebuild_body: str) -> str: # Auto-rebuilds the knowledge graph after each commit (code files only, no LLM needed). # Installed by: graphify hook install -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. +# Deterministic clustering: string-keyed set/dict iteration is randomized +# per-process by PYTHONHASHSEED, so community assignments churn run-to-run. +# Pinning it makes graphify-out reproducible. export PYTHONHASHSEED=0 # Git for Windows/MSYS hooks can inherit fragile pipe handles from GUI clients @@ -334,9 +334,9 @@ def _detached_launch(rebuild_body: str) -> str: # Auto-rebuilds the knowledge graph (code only) when switching branches. # Installed by: graphify hook install -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. +# Deterministic clustering: string-keyed set/dict iteration is randomized +# per-process by PYTHONHASHSEED, so community assignments churn run-to-run. +# Pinning it makes graphify-out reproducible. export PYTHONHASHSEED=0 # Git for Windows/MSYS hooks can inherit fragile pipe handles from GUI clients diff --git a/graphify/multigraph_compat.py b/graphify/multigraph_compat.py deleted file mode 100644 index 7ac62e275..000000000 --- a/graphify/multigraph_compat.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Runtime compatibility probe for Graphify MultiDiGraph mode. - -Verifies that the current NetworkX runtime supports the behaviors a future -opt-in --multigraph build will rely on. The probe is BEHAVIOR-based, not -version-based — both NX 3.4.2 (Py 3.10 lane) and NX 3.6.1+ (Py 3.11+ lane) -pass. The probe result is cached for the process lifetime via lru_cache. - -No call sites added yet; downstream multigraph PRs will gate on -require_multigraph_capabilities() before enabling MDG mode. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from functools import lru_cache -import sys -from typing import Any - -import networkx as nx -from networkx.readwrite import json_graph - - -@dataclass(frozen=True) -class CapabilityCheck: - name: str - ok: bool - detail: str - - -@dataclass(frozen=True) -class MultigraphCapabilityResult: - python_version: str - networkx_version: str - checks: tuple[CapabilityCheck, ...] - - @property - def ok(self) -> bool: - return all(check.ok for check in self.checks) - - @property - def failed(self) -> tuple[CapabilityCheck, ...]: - return tuple(check for check in self.checks if not check.ok) - - def error_message(self) -> str: - if self.ok: - return ( - "Graphify MultiDiGraph capability probe passed " - f"(Python {self.python_version}, NetworkX {self.networkx_version})." - ) - failed = "; ".join(f"{check.name}: {check.detail}" for check in self.failed) - return ( - "error: --multigraph requires NetworkX keyed MultiDiGraph node-link " - "round-trip support. " - f"Detected Python {self.python_version}, NetworkX {self.networkx_version}. " - f"Failed capability check(s): {failed}. " - "Default simple graph mode remains available." - ) - - -def _check(name: str, func: Callable[[], bool | str]) -> CapabilityCheck: - try: - detail = func() - except Exception as exc: - return CapabilityCheck(name, False, f"{type(exc).__name__}: {exc}") - if detail is True: - return CapabilityCheck(name, True, "ok") - if isinstance(detail, str): - return CapabilityCheck(name, False, detail) - return CapabilityCheck(name, False, f"unexpected result {detail!r}") - - -def _build_probe_graph() -> nx.MultiDiGraph: - graph = nx.MultiDiGraph() - graph.add_node("a", label="A") - graph.add_node("b", label="B") - graph.add_edge("a", "b", key="calls:a.py:L1", relation="calls", source_file="a.py") - graph.add_edge("a", "b", key="imports:a.py:L2", relation="imports", source_file="a.py") - return graph - - -def _probe_keyed_parallel_edges() -> bool | str: - graph = _build_probe_graph() - if not graph.is_multigraph() or not graph.is_directed(): - return f"probe graph type was {type(graph).__name__}" - if graph.number_of_edges("a", "b") != 2: - return f"expected 2 keyed parallel edges, got {graph.number_of_edges('a', 'b')}" - keys = set(graph["a"]["b"].keys()) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if keys != expected: - return f"expected keys {sorted(expected)}, got {sorted(keys)}" - return True - - -def _probe_node_link_round_trip() -> bool | str: - graph = _build_probe_graph() - data = json_graph.node_link_data(graph, edges="links") - if data.get("multigraph") is not True: - return f"serialized multigraph flag was {data.get('multigraph')!r}" - if data.get("directed") is not True: - return f"serialized directed flag was {data.get('directed')!r}" - links = data.get("links") - if not isinstance(links, list) or len(links) != 2: - length = 0 if not isinstance(links, list) else len(links) - return f"serialized links length was {length}" - serialized_keys: set[str] = set() - for edge in links: - if isinstance(edge, dict): - edge_key = edge.get("key") - if isinstance(edge_key, str): - serialized_keys.add(edge_key) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if serialized_keys != expected: - return f"serialized keys {sorted(serialized_keys)} did not match {sorted(expected)}" - loaded = json_graph.node_link_graph(data, edges="links") - if not isinstance(loaded, nx.MultiDiGraph): - return f"round-trip graph type was {type(loaded).__name__}" - if loaded.number_of_edges("a", "b") != 2: - return f"round-trip edge count was {loaded.number_of_edges('a', 'b')}" - loaded_keys = set(loaded["a"]["b"].keys()) - if loaded_keys != expected: - return f"round-trip keys {sorted(loaded_keys)} did not match {sorted(expected)}" - return True - - -def _probe_duplicate_key_overwrite_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("x", "y", key="same", marker="first") - graph.add_edge("x", "y", key="same", marker="second") - edges = list(graph.edges(keys=True, data=True)) - if len(edges) != 1: - return f"expected one edge after duplicate-key add, got {len(edges)}" - if edges[0][3].get("marker") != "second": - return f"expected second attr overwrite, got {edges[0][3].get('marker')!r}" - return True - - -def _probe_reserved_key_attr_rejected() -> bool | str: - """Verify the Python language guarantee that NetworkX add_edge inherits. - - Python forbids passing the same keyword argument twice — once explicitly - and once via **kwargs. This probe confirms that protection still applies - to nx.MultiDiGraph.add_edge: a future loader that builds attrs from JSON - will be reliably protected from accidentally setting `key` via attrs while - also passing `key=` explicitly. - - The probe always passes on any Python 3.x version. Its purpose is to - document the invariant explicitly in the probe suite so that if a future - Python version relaxes this rule (extremely unlikely), the probe surfaces - the regression. - """ - graph = nx.MultiDiGraph() - attrs: dict[str, Any] = {"key": "attr-key", "relation": "calls"} - try: - graph.add_edge("a", "b", key="schema-key", **attrs) - except TypeError: - return True - return "add_edge accepted duplicate key keyword and attr; loader must not rely on this" - - -def _probe_remove_edges_from_two_tuple_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("a", "b", key="one") - graph.add_edge("a", "b", key="two") - graph.remove_edges_from([("a", "b")]) - remaining = graph.number_of_edges("a", "b") - if remaining != 1: - return f"expected one remaining edge after two-tuple removal, got {remaining}" - return True - - -def _probe_to_undirected_preserves_multigraph_type() -> bool | str: - graph = _build_probe_graph() - undirected = graph.to_undirected() - undirected_view = graph.to_undirected(as_view=True) - if not isinstance(undirected, nx.MultiGraph): - return f"to_undirected() returned {type(undirected).__name__}" - if not isinstance(undirected_view, nx.MultiGraph): - return f"to_undirected(as_view=True) returned {type(undirected_view).__name__}" - return True - - -@lru_cache(maxsize=1) -def probe_multigraph_capabilities() -> MultigraphCapabilityResult: - checks = ( - _check("keyed_parallel_edges", _probe_keyed_parallel_edges), - _check("node_link_edges_links_round_trip", _probe_node_link_round_trip), - _check("duplicate_key_overwrite_semantics", _probe_duplicate_key_overwrite_semantics), - _check("reserved_key_attr_rejected", _probe_reserved_key_attr_rejected), - _check( - "remove_edges_from_two_tuple_semantics", - _probe_remove_edges_from_two_tuple_semantics, - ), - _check( - "to_undirected_preserves_multigraph_type", - _probe_to_undirected_preserves_multigraph_type, - ), - ) - return MultigraphCapabilityResult( - python_version=( - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ), - networkx_version=nx.__version__, - checks=checks, - ) - - -def require_multigraph_capabilities() -> MultigraphCapabilityResult: - result = probe_multigraph_capabilities() - if not result.ok: - raise RuntimeError(result.error_message()) - return result diff --git a/graphify/report.py b/graphify/report.py index 248bce9a1..f8fbadd7e 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -2,7 +2,6 @@ from __future__ import annotations import re from datetime import date -import networkx as nx def _safe_community_name(label: str) -> str: diff --git a/graphify/serve.py b/graphify/serve.py index f32a91673..bb90b2c59 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -7,8 +7,7 @@ from array import array from pathlib import Path from typing import NamedTuple -import networkx as nx -from networkx.readwrite import json_graph +from .store import open_store, MemGraph from graphify.security import sanitize_label, check_graph_file_size_cap from graphify.build import edge_data, edge_datas from graphify.paths import default_graph_json as _default_graph_json @@ -19,22 +18,35 @@ _jieba = None -def _load_graph(graph_path: str) -> nx.Graph: +def _connect_graph(graph_path: str): + """Open a connection to the FalkorDB-backed graph for the output dir containing + `graph_path`. Cache-free: this returns a store handle (a connection), it does + NOT load the graph into memory. + + `graph_path` is the legacy graph.json location; its parent directory holds the + FalkorDB pointer (falkordb.json). Errors out if no graph has been built yet. + """ try: resolved = Path(graph_path).resolve() - if resolved.suffix != ".json": - raise ValueError(f"Graph path must be a .json file, got: {graph_path!r}") - if not resolved.exists(): - raise FileNotFoundError(f"Graph file not found: {resolved}") - check_graph_file_size_cap(resolved) - safe = resolved - data = json.loads(safe.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - data = {**data, "directed": True} + out_dir = resolved.parent if resolved.suffix else resolved + store = open_store(out_dir, create=False) + if store.number_of_nodes() == 0: + # Back-compat: the FalkorDB store is empty but a node-link graph.json + # may still hold the graph — an existing pre-FalkorDB project, or a + # `--no-cluster` run that only wrote JSON. Import it into the store on + # first use so query/path/explain/export work without forcing a + # rebuild, preserving the previous workflow. + gj = resolved if resolved.suffix == ".json" else (out_dir / "graph.json") + _import_graph_json_into_store(gj, store) + if store.number_of_nodes() == 0: + raise FileNotFoundError( + f"No graph found for {out_dir} (FalkorDB graph empty). Re-run /graphify to build." + ) + # Nudge when the graph still uses pre-#1504 node IDs, as the old json + # loader did. Sampled via a scoped file-level query so this stays cheap. try: from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(data.get("nodes", [])): + if _legacy(_file_level_sample(store)): print( "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " "rebuild with `graphify extract --force` for path-qualified IDs.", @@ -42,27 +54,84 @@ def _load_graph(graph_path: str) -> nx.Graph: ) except Exception: pass - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) # Attach the work-memory overlay (derived sidecar next to graph.json) so # the query/MCP read surface can annotate NODE lines display-only. Empty # when no sidecar exists, leaving un-annotated output byte-identical. + # The `_` prefix keeps save_meta from persisting it into the graph. try: from graphify.reflect import load_learning_overlay as _llo - G.graph["_learning_overlay"] = _llo(resolved) + store.graph["_learning_overlay"] = _llo(resolved) except Exception: - G.graph["_learning_overlay"] = {} - return G - except json.JSONDecodeError as exc: - print(f"error: graph.json is corrupted ({exc}). Re-run /graphify to rebuild.", file=sys.stderr) - sys.exit(1) + store.graph["_learning_overlay"] = {} + return store except (ValueError, FileNotFoundError) as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1) +def _file_level_sample(store, limit: int = 300) -> list[dict]: + """Up to `limit` file-level nodes (``source_location == 'L1'``) as plain dicts. + + ``graph_has_legacy_ids`` only inspects file-level nodes, so fetching just + those keeps the pre-#1504 ID check to one scoped query rather than streaming + the whole graph on every connect. + """ + rows = getattr(store, "_rows", None) + if rows is not None: + try: + return [ + {"id": r[0], "source_file": r[1], "source_location": r[2]} + for r in rows( + "MATCH (n:Entity) WHERE n.source_location = 'L1' " + "RETURN n.id, n.source_file, n.source_location LIMIT $lim", + {"lim": limit}, + ) + ] + except Exception: + pass + sample: list[dict] = [] + for nid, attrs in store.nodes(data=True): + if str(attrs.get("source_location") or "") != "L1": + continue + sample.append({**attrs, "id": nid}) + if len(sample) >= limit: + break + return sample + + +def _import_graph_json_into_store(gj_path: Path, store) -> bool: + """One-time import of a node-link ``graph.json`` into an empty FalkorDB store. + + Back-compat for projects built before the FalkorDB backend (and for + ``--no-cluster`` output, which writes only JSON): rather than erroring with + "graph empty", load the JSON so existing graphs stay queryable without a + rebuild. Returns True if any nodes were imported. + """ + if not gj_path.exists(): + return False + # This is the one path that still parses an arbitrary graph.json, so it keeps + # the file-size cap the FalkorDB loaders no longer need (#F4). Raised, not + # swallowed, so an oversized file is refused loudly rather than silently + # importing nothing and reporting "graph empty". + check_graph_file_size_cap(gj_path) + try: + data = json.loads(gj_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + nodes = data.get("nodes") or [] + if not nodes: + return False + edges = data.get("links", data.get("edges", [])) + from graphify.build import build_from_json + build_from_json({"nodes": nodes, "edges": edges}, store=store) + print( + f"[graphify] Imported {len(nodes)} nodes from {gj_path.name} into FalkorDB " + "(one-time migration from a pre-FalkorDB / --no-cluster graph).", + file=sys.stderr, + ) + return True + + def _communities_from_graph(G: nx.Graph) -> dict[int, list[str]]: """Reconstruct community dict from community property stored on nodes.""" communities: dict[int, list[str]] = {} @@ -202,16 +271,17 @@ def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]: N = G.number_of_nodes() or 1 uncached = [t for t in terms if t not in cache] if uncached: - df: dict[str, int] = {t: 0 for t in uncached} - for _, data in G.nodes(data=True): - norm_label = ( - data.get("norm_label") or _strip_diacritics(data.get("label") or "") - ).lower() - for t in uncached: - if t in norm_label: - df[t] += 1 + if hasattr(G, "doc_freqs"): + df = G.doc_freqs(uncached) # in-engine count per term, no full scan + else: + df = {t: 0 for t in uncached} + for _, data in G.nodes(data=True): + norm_label = (data.get("norm_label") or _strip_diacritics(data.get("label") or "")).lower() + for t in uncached: + if t in norm_label: + df[t] += 1 for t in uncached: - cache[t] = math.log(1 + N / (1 + df[t])) + cache[t] = math.log(1 + N / (1 + df.get(t, 0))) return {t: cache.get(t, math.log(1 + N)) for t in terms} @@ -389,15 +459,38 @@ def _score_query( # Weight the full-query bonus by the rarest constituent term so a specific # multi-word label still outweighs common-token noise; floor at 1.0. joined_w = max((idf.get(t, 1.0) for t in norm_terms), default=1.0) - # Trigram prefilter: score only nodes whose text could match a term, falling - # back to the whole graph when the index isn't selective. The result is - # identical either way — the per-node scoring below is unchanged and a - # non-candidate node always scores 0. (IDF above stays a whole-graph statistic.) - candidate_ids = _trigram_candidates(G, norm_terms + ([joined] if joined else [])) - node_iter = ( - G.nodes(data=True) if candidate_ids is None - else ((nid, G.nodes[nid]) for nid in candidate_ids) - ) + # Candidate prefilter: score only nodes whose text could match a term. The + # result is identical to scanning everything — the per-node scoring below is + # unchanged and a non-candidate node always scores 0. (IDF above stays a + # whole-graph statistic.) Both branches yield (nid, attrs) pairs so the + # scoring loop is backend-agnostic. + _label_by_id: dict[str, str] | None = None + if hasattr(G, "search_nodes"): + # FalkorDB-native: a node scores > 0 only if some term matches its label + # or source, so prefilter those candidates in-engine instead of scanning + # every node in Python. Caching the labels here also keeps the tie-break + # sort below from issuing a per-node query. + _seen: set = set() + _cands: list[tuple[str, dict]] = [] + for _c in G.search_nodes(norm_terms + ([joined] if joined else [])): + if _c["id"] in _seen: + continue + _seen.add(_c["id"]) + _cands.append((_c["id"], { + "label": _c["label"], + "norm_label": _c["norm_label"] or _strip_diacritics(_c["label"]).lower(), + "source_file": _c["source_file"], + })) + node_iter = iter(_cands) + _label_by_id = {nid: attrs["label"] for nid, attrs in _cands} + else: + # Trigram prefilter, falling back to the whole graph when the index isn't + # selective. + candidate_ids = _trigram_candidates(G, norm_terms + ([joined] if joined else [])) + node_iter = ( + G.nodes(data=True) if candidate_ids is None + else ((nid, G.nodes[nid]) for nid in candidate_ids) + ) # Per-token best tracking, only when the caller (the query path) wants the # seed metadata. The key tuple is the full multi-key tie-break # (`(-singleton_score, -degree, label_len, nid)`), so `min` over the @@ -511,7 +604,14 @@ def _score_query( scored.append((score, nid)) # Sort by score desc; break ties toward the shorter label so a concise exact # match beats a longer superset that happens to share the same score. - scored.sort(key=lambda s: (-s[0], len(G.nodes[s[1]].get("label") or s[1]), s[1])) + # Label lookup for the tie-break: prefer the prefilter's cached labels so the + # store path doesn't issue a per-node query for every tied candidate. + def _label_of(nid: str) -> str: + if _label_by_id is not None: + return _label_by_id.get(nid) or nid + return G.nodes[nid].get("label") or nid + + scored.sort(key=lambda s: (-s[0], len(_label_of(s[1])), s[1])) best_seed_by_term: dict[str, str] = {} if collect_per_term_seeds and best_by_term: best_seed_by_term = {t: nid for t, (_key, nid) in best_by_term.items()} @@ -720,25 +820,36 @@ def _resolve_context_filters(question: str, explicit_filters: list[str] | None = return [], None +def _confidence_distribution(G) -> tuple[dict, int]: + """(counts_by_confidence, total_edges). Uses the store's in-engine aggregation + when available so graph_stats/audit never stream every edge into Python; falls + back to an nx/MemGraph edge scan otherwise.""" + if hasattr(G, "confidence_counts"): + counts = G.confidence_counts() + else: + counts: dict = {} + for _, _, d in G.edges(data=True): + k = d.get("confidence", "EXTRACTED") + counts[k] = counts.get(k, 0) + 1 + return counts, (sum(counts.values()) or 1) + + def _filter_graph_by_context(G: nx.Graph, context_filters: list[str] | None) -> nx.Graph: filters = set(_normalize_context_filters(context_filters)) if not filters: return G - H = G.__class__() - H.add_nodes_from(G.nodes(data=True)) - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - for u, v, key, data in G.edges(keys=True, data=True): - if data.get("context") in filters: - H.add_edge(u, v, key=key, **data) - else: - for u, v, data in G.edges(data=True): - if data.get("context") in filters: - H.add_edge(u, v, **data) - return H - - -def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: - # Compute hub threshold: nodes above this degree are not expanded as transit. + nodes = list(G.nodes(data=True)) + edges = [(u, v, data) for u, v, data in G.edges(data=True) if data.get("context") in filters] + return MemGraph(nodes, edges, directed=G.is_directed()) + + +def _bfs(G: nx.Graph, start_nodes: list[str], depth: int, contexts: list[str] | None = None) -> tuple[set[str], list[tuple]]: + # FalkorDB-native: traverse in the engine, one query per level (no graph load). + # `contexts` scopes the walk to matching edge-contexts in-engine, so context + # filtering never copies the graph into Python. + if hasattr(G, "bfs"): + return G.bfs(start_nodes, depth, contexts=contexts) + # In-memory fallback (context-filtered MemGraph / nx). Compute hub threshold: # p99 of degree distribution, floored at 50 to avoid over-blocking small graphs. degrees = [G.degree(n) for n in G.nodes()] if degrees: @@ -767,7 +878,9 @@ def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], lis return visited, edges_seen -def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: +def _dfs(G: nx.Graph, start_nodes: list[str], depth: int, contexts: list[str] | None = None) -> tuple[set[str], list[tuple]]: + if hasattr(G, "dfs"): + return G.dfs(start_nodes, depth, contexts=contexts) degrees = [G.degree(n) for n in G.nodes()] if degrees: degrees_sorted = sorted(degrees) @@ -805,36 +918,65 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu # Empty when no sidecar exists, so un-annotated output stays byte-identical. overlay = getattr(G, "graph", {}).get("_learning_overlay", {}) or {} seed_set = set(seeds or []) - seed_hits = [n for n in (seeds or []) if n in nodes] + total_nodes = len(nodes) + + # FalkorDB-native: fetch node attrs+degree and edge attrs in batched queries + # instead of ~per-element scoped lookups. Fallback for MemGraph/nx. + if hasattr(G, "subgraph_render_data"): + # Only fetch attrs for as many nodes as the char budget can possibly + # render (each NODE line is >~20 chars). Without this cap a hub query + # whose BFS reaches tens of thousands of nodes would pull every node's + # attributes out of the engine only to discard all but the top slice. + render_limit = max(50, char_budget // 20) + nattrs, eattrs, total_nodes = G.subgraph_render_data( + nodes, [(u, v) for u, v in edges], + limit=render_limit, seeds=list(seeds or []), + ) + kept = set(nattrs) + def _n(nid): + return nattrs.get(nid, {}) + def _deg(nid): + return nattrs.get(nid, {}).get("degree", 0) + def _e(u, v): + return eattrs.get((u, v), {}) + else: + kept = set(nodes) + def _n(nid): + return G.nodes[nid] + def _deg(nid): + return G.degree(nid) + def _e(u, v): + return G[u][v] + + seed_hits = [n for n in (seeds or []) if n in kept] # Rank non-seed nodes by hop distance from the seeds so the node that answers # the query (a direct hit or its close neighbors) survives the budget cut # instead of being pushed past it by incidental high-degree hubs (#BUG2). BFS # discovery order was discarded upstream (_bfs returns a set), so recompute - # layers here over BOTH edge directions. Deterministic: neighbor iteration is - # insertion-ordered and the sort key ends in str(n) (no hash-order). - def _adj(n): - if G.is_directed(): - yield from G.successors(n) - yield from G.predecessors(n) - else: - yield from G.neighbors(n) + # layers here from the subgraph's own edge list — it already covers both + # directions and costs no per-node neighbor query against the store. + # Deterministic: adjacency is insertion-ordered and the sort key ends in str(n). + _adj_map: dict[str, list[str]] = {} + for _u, _v in edges: + _adj_map.setdefault(_u, []).append(_v) + _adj_map.setdefault(_v, []).append(_u) dist: dict[str, int] = {n: 0 for n in seed_hits} frontier, hop = seed_hits, 0 while frontier: hop += 1 nxt = [] for n in frontier: - for nb in _adj(n): - if nb in nodes and nb not in dist: + for nb in _adj_map.get(n, ()): + if nb in kept and nb not in dist: dist[nb] = hop nxt.append(nb) frontier = nxt ordered = seed_hits + sorted( - nodes - seed_set, - key=lambda n: (dist.get(n, 1 << 30), -G.degree(n), str(n)), + kept - seed_set, + key=lambda n: (dist.get(n, 1 << 30), -_deg(n), str(n)), ) for nid in ordered: - d = G.nodes[nid] + d = _n(nid) # Every LLM-derived field passes through sanitize_label before being # concatenated into MCP tool output (F-010): an attacker who controls a # corpus document can otherwise inject ANSI escapes, fake graphify-out @@ -857,21 +999,20 @@ def _adj(n): ) lines.append(line) for u, v in edges: - if u in nodes and v in nodes: - raw = G[u][v] - d = next(iter(raw.values()), {}) if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)) else raw + if u in kept and v in kept: + d = _e(u, v) # (u, v) is BFS/DFS visit order, not necessarily the true edge - # direction: on an undirected graph G.neighbors() walks callers - # and callees alike, so a caller->callee edge renders backwards - # whenever the callee is visited first. _src/_tgt (stashed on the - # edge data by the `query` CLI loader) carry the real direction; - # fall back to (u, v) for graphs/edges that don't set them. + # direction: traversal walks callers and callees alike, so a + # caller->callee edge renders backwards whenever the callee is + # visited first. _src/_tgt carry the stored orientation (set from + # startNode(r) by the batched fetch); fall back to (u, v) for + # graphs/edges that don't set them. src = d.get("_src", u) tgt = d.get("_tgt", v) # Guard against a stray/dangling _src/_tgt (hand-edited or adversarial # graph.json): only trust them when they name exactly this edge's - # endpoints, else fall back to (u, v). Without this, G.nodes[src] - # would KeyError on an unknown id (#2080 review). + # endpoints, else fall back to (u, v). Without this, _n(src) would + # render an empty label for an unknown id (#2080 review). if {src, tgt} != {u, v}: src, tgt = u, v context = d.get("context") @@ -885,13 +1026,16 @@ def _adj(n): if _loc else "" ) line = ( - f"EDGE {sanitize_label(G.nodes[src].get('label', src))} " + f"EDGE {sanitize_label(_n(src).get('label', src))} " f"--{sanitize_label(str(d.get('relation', '')))} " f"[{sanitize_label(str(d.get('confidence', '')))}{context_suffix}]--> " - f"{sanitize_label(G.nodes[tgt].get('label', tgt))}{at_suffix}" + f"{sanitize_label(_n(tgt).get('label', tgt))}{at_suffix}" ) lines.append(line) output = "\n".join(lines) + # shown_nodes = NODE lines that survive the char budget; cut_count counts + # against the FULL subgraph (total_nodes), so nodes dropped by the render + # limit are reported as cut too — never silently omitted. if len(output) > char_budget: cut_at = output[:char_budget].rfind("\n") cut_at = cut_at if cut_at > 0 else char_budget @@ -902,9 +1046,15 @@ def _adj(n): if seed_hits: seed_block_end = sum(len(lines[i]) + 1 for i in range(len(seed_hits))) - 1 cut_at = max(cut_at, min(seed_block_end, len(output))) - total_nodes = sum(1 for l in lines if l.startswith("NODE ")) - shown_nodes = output[:cut_at].count("\nNODE ") + (1 if output.startswith("NODE ") else 0) - cut_count = total_nodes - shown_nodes + output = output[:cut_at] + shown_nodes = output.count("\nNODE ") + (1 if output.startswith("NODE ") else 0) + else: + shown_nodes = sum(1 for l in lines if l.startswith("NODE ")) + # total_nodes is the FULL subgraph count, so nodes dropped by the render + # limit before any line was built are reported as cut too. That also means + # cut_count can be > 0 on output that fit the char budget. + cut_count = total_nodes - shown_nodes + if cut_count > 0: # Prominent notice at the TOP so a truncated answer can never be mistaken # for a complete one — silence used to read as absence (#BUG2). The # notice + end marker sit OUTSIDE char_budget by design (two bounded @@ -915,7 +1065,7 @@ def _adj(n): f"{cut_count} cut nodes — raise the token budget (CLI: --budget) or " f"narrow the query (e.g. context_filter=['call'], or get_node for a " f"specific symbol).\n\n" - + output[:cut_at] + + output + f"\n... (truncated — {cut_count} more nodes cut by ~{token_budget}-token budget." f" Narrow with context_filter=['call'] or use get_node for a specific symbol)" ) @@ -969,8 +1119,17 @@ def _query_graph_text( if not start_nodes: return "No matching nodes found." resolved_filters, filter_source = _resolve_context_filters(question, context_filters) - traversal_graph = _filter_graph_by_context(G, resolved_filters) - nodes, edges = _dfs(traversal_graph, start_nodes, depth) if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth) + if resolved_filters and hasattr(G, "bfs"): + # Store-backed: apply the context filter inside the traversal query and + # render against the live store — never materialize a filtered copy. + traversal_graph = G + contexts = resolved_filters + else: + # nx/MemGraph fallback: build the filtered in-memory view as before. + traversal_graph = _filter_graph_by_context(G, resolved_filters) + contexts = None + nodes, edges = (_dfs(traversal_graph, start_nodes, depth, contexts) + if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth, contexts)) header_parts = [ f"Traversal: {mode.upper()} depth={depth}", f"Start: {[G.nodes[n].get('label', n) for n in start_nodes]}", @@ -1006,13 +1165,29 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: exact: list[str] = [] prefix: list[str] = [] substring: list[str] = [] - # Trigram prefilter (graph-iteration order preserved so exact/prefix/substring - # ordering — and thus matches[0] — is byte-identical to the full scan). - candidate_ids = _trigram_candidates(G, [term, norm_query]) - node_iter = ( - G.nodes(data=True) if candidate_ids is None - else ((nid, G.nodes[nid]) for nid in candidate_ids) - ) + # Candidate prefilter; tiering below runs in Python over (nid, attrs) pairs. + if hasattr(G, "search_nodes"): + # FalkorDB-native: prefilter in-engine, no full scan. + seen: set[str] = set() + _cands: list[tuple[str, dict]] = [] + for c in G.search_nodes([term, norm_query]): + if c["id"] in seen: + continue + seen.add(c["id"]) + _cands.append((c["id"], { + "label": c["label"], + "norm_label": c["norm_label"] or _strip_diacritics(c["label"]).lower(), + "source_file": c["source_file"], + })) + node_iter = iter(_cands) + else: + # Trigram prefilter (graph-iteration order preserved so exact/prefix/substring + # ordering — and thus matches[0] — is byte-identical to the full scan). + candidate_ids = _trigram_candidates(G, [term, norm_query]) + node_iter = ( + G.nodes(data=True) if candidate_ids is None + else ((nid, G.nodes[nid]) for nid in candidate_ids) + ) for nid, d in node_iter: norm_label = d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower() bare_label = norm_label.rstrip("()") @@ -1037,11 +1212,14 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: prefix.append(nid) elif term in norm_label or term in label_tokens or norm_query in norm_label: substring.append(nid) - + # Sort within each tier by node id so the result is deterministic and + # independent of the backend's row order (FalkorDB scan order vs. in-memory + # iteration). Without this, an ambiguous label that matches several nodes in + # the same tier could surface a different top match per backend/run (#1175). if source_exact: query_basename = _strip_diacritics(Path(label).name).lower() preferred = [] - for nid in source_exact: + for nid in sorted(source_exact): if str(G.nodes[nid].get("source_location", "")) != "L1": continue # File-node label is the bare basename OR a directory-qualified form @@ -1049,10 +1227,13 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: lbl = _strip_diacritics(str(G.nodes[nid].get("label") or "")).lower() if lbl == query_basename or lbl.endswith("/" + query_basename): preferred.append(nid) + source_exact = sorted(source_exact) + # A single unambiguous file-node match is promoted ahead of the rest; + # the remainder stays id-sorted so the tail is still backend-stable. if len(preferred) == 1: source_exact = preferred + [nid for nid in source_exact if nid != preferred[0]] - return source_exact + exact + prefix + substring + return source_exact + sorted(exact) + sorted(prefix) + sorted(substring) def _filter_blank_stdin() -> None: @@ -1105,8 +1286,9 @@ def _build_server(graph_path: str): All graph query tools and resources are registered here over a single ``mcp.server.Server`` instance; the caller picks the transport (stdio or - Streamable HTTP) and runs it. Hot-reload of graph.json works the same way - regardless of transport, since reloads happen inside the tool handlers. + Streamable HTTP) and runs it. Hot-reload (reconnecting to the store when the + graph.json change-sentinel is rewritten on disk) works the same way regardless + of transport, since the reconnect happens inside the tool handlers. """ import threading @@ -1128,34 +1310,45 @@ def _build_server(graph_path: str): _ctx_lock = threading.Lock() _ctx_cache: dict[str, dict] = {} + def _ctx_key(G): + """Cache-validity key for a connected store. + + The store is a live connection, so node/edge reads are never stale and + there is no file mtime to watch (a FalkorDB-only project may have no + graph.json at all). Only the DERIVED `communities` map can go stale, so + key on the graph's size: a rebuild underneath us changes it and forces a + recompute. Two cheap count queries, never a full-graph read. + """ + return (G.number_of_nodes(), G.number_of_edges()) + def _load_ctx(path: str): - """Return (G, communities) for a graph.json path, reusing a cached - context until the file's (mtime, size) changes and then transparently - rebuilding it. Unlike ``_load_graph`` it never exits the process on a - missing/corrupt file — it raises, so a bad project_path surfaces as a - tool error instead of killing a server that is happily serving other - projects.""" - try: - s = Path(path).stat() - key = (s.st_mtime_ns, s.st_size) - except FileNotFoundError: - raise FileNotFoundError(f"graph.json not found: {path}") + """Return (G, communities) for a graph path, reusing a cached context + until the underlying graph changes and then transparently rebuilding it. + Unlike ``_connect_graph`` it never exits the process on a missing/empty + graph — it raises, so a bad project_path surfaces as a tool error instead + of killing a server that is happily serving other projects.""" ent = _ctx_cache.get(path) - if ent is not None and ent["key"] == key: + if ent is not None and ent["key"] == _ctx_key(ent["G"]): return ent["G"], ent["communities"] with _ctx_lock: ent = _ctx_cache.get(path) - if ent is not None and ent["key"] == key: + if ent is not None and ent["key"] == _ctx_key(ent["G"]): return ent["G"], ent["communities"] # another thread built it try: - new_G = _load_graph(path) - except SystemExit as e: # _load_graph exits on missing/corrupt file - raise RuntimeError(f"could not load graph.json at {path}") from e + new_G = _connect_graph(path) + except SystemExit as e: # _connect_graph exits on missing/empty graph + # Phrased as not-found because that is what it is: no graph has + # been built for that project. Raised (not exited) so a bad + # project_path is one tool error, not a dead server. + raise RuntimeError(f"graph not found for {path} (no graph built there)") from e # Warm the trigram index before exposing the graph so the first query # against it is fast (same rationale as the original startup warm-up). - _get_trigram_index(new_G) + # Skipped for store-backed graphs: they prefilter with an in-engine + # search_nodes query, and building the index would read every node. + if not hasattr(new_G, "search_nodes"): + _get_trigram_index(new_G) comm = _communities_from_graph(new_G) - _ctx_cache[path] = {"key": key, "G": new_G, "communities": comm} + _ctx_cache[path] = {"key": _ctx_key(new_G), "G": new_G, "communities": comm} return new_G, comm def _resolve_graph_path(project_path) -> str: @@ -1178,7 +1371,7 @@ def _resolve_graph_path(project_path) -> str: except (FileNotFoundError, RuntimeError): # No default graph at startup → run as a pure multi-project server. Tools # then require project_path; a call without one gets a clear error rather - # than the process refusing to start (which is what _load_graph would do). + # than the process refusing to start (which is what _connect_graph would do). G, communities = None, {} def _select_graph(project_path) -> None: @@ -1361,12 +1554,18 @@ def _tool_query_graph(arguments: dict) -> str: return result def _tool_get_node(arguments: dict) -> str: - label = arguments["label"].lower() - matches = [(nid, d) for nid, d in G.nodes(data=True) - if label in (d.get("label") or "").lower() or label == nid.lower()] + label = arguments["label"] + # Use the candidate prefilter (in-engine for the store) instead of scanning + # every node, then fetch just the matched node's attrs+degree. + matches = _find_node(G, label) if not matches: return f"No node matching '{label}' found." - nid, d = matches[0] + nid = matches[0] + if hasattr(G, "node_detail"): + detail = G.node_detail(nid) + d, deg = detail if detail else ({}, 0) + else: + d, deg = G.nodes[nid], G.degree(nid) # Sanitise every LLM-derived field before concatenation (F-010). return "\n".join([ f"Node: {sanitize_label(d.get('label', nid))}", @@ -1374,7 +1573,7 @@ def _tool_get_node(arguments: dict) -> str: f" Source: {sanitize_label(str(d.get('source_file', '')))} {sanitize_label(str(d.get('source_location', '')))}", f" Type: {sanitize_label(str(d.get('file_type', '')))}", f" Community: {sanitize_label(str(d.get('community_name') or d.get('community', '')))}", - f" Degree: {G.degree(nid)}", + f" Degree: {deg}", ]) def _tool_get_neighbors(arguments: dict) -> str: @@ -1443,15 +1642,14 @@ def _tool_god_nodes(arguments: dict) -> str: return "\n".join(lines) def _tool_graph_stats(_: dict) -> str: - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 + confs, total = _confidence_distribution(G) return ( f"Nodes: {G.number_of_nodes()}\n" f"Edges: {G.number_of_edges()}\n" f"Communities: {len(communities)}\n" - f"EXTRACTED: {round(confs.count('EXTRACTED')/total*100)}%\n" - f"INFERRED: {round(confs.count('INFERRED')/total*100)}%\n" - f"AMBIGUOUS: {round(confs.count('AMBIGUOUS')/total*100)}%\n" + f"EXTRACTED: {round(confs.get('EXTRACTED', 0)/total*100)}%\n" + f"INFERRED: {round(confs.get('INFERRED', 0)/total*100)}%\n" + f"AMBIGUOUS: {round(confs.get('AMBIGUOUS', 0)/total*100)}%\n" ) def _tool_shortest_path(arguments: dict) -> str: @@ -1486,16 +1684,12 @@ def _tool_shortest_path(arguments: dict) -> str: f"(top score {top:g}, runner-up {runner:g})" ) max_hops = int(arguments.get("max_hops", 8)) - try: - # Deterministic path (#2074): the hash-seeded undirected view picked an - # arbitrary route among equal-length paths. Build a sorted, materialized - # undirected graph so the chosen path is canonical. Serve's shared G is - # left untouched (its degree feeds query-seed tie-breaks). - _und = nx.Graph() - _und.add_nodes_from(sorted(G.nodes)) - _und.add_edges_from(sorted((min(u, v), max(u, v)) for u, v in G.edges())) - path_nodes = nx.shortest_path(_und, src_nid, tgt_nid) - except (nx.NetworkXNoPath, nx.NodeNotFound): + # Undirected shortest path (works regardless of query src/tgt order). + # Deterministic (#2074): the store's BFS walks a sorted adjacency, so the + # route chosen among equal-length paths is canonical — no sorted scratch + # copy needed, and serve's shared G is left untouched. + path_nodes = G.shortest_path(src_nid, tgt_nid, max_hops=max_hops) + if not path_nodes: return f"No path found between '{G.nodes[src_nid].get('label', src_nid)}' and '{G.nodes[tgt_nid].get('label', tgt_nid)}'." hops = len(path_nodes) - 1 if hops > max_hops: @@ -1505,7 +1699,8 @@ def _tool_shortest_path(arguments: dict) -> str: u, v = path_nodes[i], path_nodes[i + 1] # Report the actual stored relation(s), never a fabricated `calls`; # fall back to an honest "related" when the edge has no relation (#2074). - if G.has_edge(u, v): + # has_directed_edge, not has_edge: the store keeps native orientation. + if G.has_directed_edge(u, v): datas = edge_datas(G, u, v) forward = True else: @@ -1668,13 +1863,13 @@ async def read_resource(uri: AnyUrl) -> str: except Exception as exc: return f"Could not compute surprising connections: {exc}" if uri_str == "graphify://audit": - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 + confs, total = _confidence_distribution(G) + e, i, a = confs.get("EXTRACTED", 0), confs.get("INFERRED", 0), confs.get("AMBIGUOUS", 0) return ( f"Total edges: {total}\n" - f"EXTRACTED: {confs.count('EXTRACTED')} ({round(confs.count('EXTRACTED')/total*100)}%)\n" - f"INFERRED: {confs.count('INFERRED')} ({round(confs.count('INFERRED')/total*100)}%)\n" - f"AMBIGUOUS: {confs.count('AMBIGUOUS')} ({round(confs.count('AMBIGUOUS')/total*100)}%)\n" + f"EXTRACTED: {e} ({round(e/total*100)}%)\n" + f"INFERRED: {i} ({round(i/total*100)}%)\n" + f"AMBIGUOUS: {a} ({round(a/total*100)}%)\n" ) if uri_str == "graphify://questions": try: diff --git a/graphify/skills/agents/references/query.md b/graphify/skills/agents/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/agents/references/query.md +++ b/graphify/skills/agents/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/amp/references/query.md +++ b/graphify/skills/amp/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/claude/references/query.md b/graphify/skills/claude/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/claude/references/query.md +++ b/graphify/skills/claude/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/claw/references/query.md +++ b/graphify/skills/claw/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/codex/references/query.md +++ b/graphify/skills/codex/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/copilot/references/query.md +++ b/graphify/skills/copilot/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/droid/references/query.md +++ b/graphify/skills/droid/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/kilo/references/query.md +++ b/graphify/skills/kilo/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/kiro/references/query.md +++ b/graphify/skills/kiro/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/opencode/references/query.md +++ b/graphify/skills/opencode/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/pi/references/query.md +++ b/graphify/skills/pi/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/trae/references/query.md +++ b/graphify/skills/trae/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/vscode/references/query.md +++ b/graphify/skills/vscode/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md index 56565eb78..a22fc620b 100644 --- a/graphify/skills/windows/references/query.md +++ b/graphify/skills/windows/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 3632fd412..4912b032f 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/store.py b/graphify/store.py new file mode 100644 index 000000000..7ca9c1d64 --- /dev/null +++ b/graphify/store.py @@ -0,0 +1,1468 @@ +"""FalkorDB-backed graph store — the single replacement for NetworkX. + +`GraphStore` wraps a named FalkorDB graph and exposes a NetworkX-shaped API so +the rest of graphify can treat it almost exactly like the `nx.Graph` it used to +pass around. FalkorDB is the source of truth and the compute engine: + +* **Writes / mutation** go straight to FalkorDB (batched `UNWIND ... MERGE`). +* **Algorithms** run server-side — built-in `algo.*` where they exist, and the + JS UDFs in ``graphify/udfs/graphify_algos.js`` (louvain, edge betweenness, + simple cycles) where they don't. +* **Reads** (the `G.nodes[id]`, `G.edges(data=True)`, `G.degree(n)` patterns used + in ~200 call sites) are served from a read cache that is bulk-loaded from + FalkorDB once via Cypher and invalidated on mutation. This keeps the hot + analysis loops fast (in-memory, like the old nx graph) without reimplementing + any algorithm in Python. + +Schema +------ +Every node has the base label ``:Entity`` plus a sanitized ``file_type`` label; +``id`` is the key and a unique index on ``:Entity(id)`` (created at init) makes +MERGE/MATCH fast. Edges are stored DIRECTED in original orientation with the +sanitized relation as the Cypher type and the raw relation kept as the +``relation`` property. Parallel edges are native, so the old ``_src``/``_tgt`` +markers are gone. Graph-level metadata (e.g. ``hyperedges``) lives on a singleton +``:GraphMeta`` node and is mirrored to the in-memory ``.graph`` dict. +""" +from __future__ import annotations + +import hashlib +import json +import re +import threading +from pathlib import Path +from urllib.parse import urlparse + +_UDF_LIB = "graphify_algos" +_UDF_SRC = Path(__file__).parent / "udfs" / "graphify_algos.js" +_udf_loaded_servers: set[str] = set() +_udf_lock = threading.Lock() + +DEFAULT_URI = "falkordb://localhost:6379" +_META_KEY = "__graphmeta__" + +# FalkorDB caps rows returned per query at RESULTSET_SIZE (default 10000), which +# silently truncates bulk reads on large graphs. We raise it at connect time so +# row-returning procedures (e.g. algo.betweenness) aren't truncated, and ALSO +# paginate the cache-load below at a page size safely under the default cap so +# the read cache is complete even if the config raise is rejected. +_RESULTSET_SIZE = 1_000_000_000 +_PAGE = 9000 +# Default read TIMEOUT is 1000ms, which the server-side UDFs (Louvain, edge +# betweenness) blow past on large graphs. Raise the server defaults at connect +# and also pass this per-query timeout on heavy READ calls (not writes — FalkorDB +# disallows timeouts on write queries). +_QUERY_TIMEOUT_MS = 600_000 +_server_configured: set[str] = set() + + +def _safe_label(label: str) -> str: + sanitized = re.sub(r"[^A-Za-z0-9_]", "", label or "") + return sanitized if sanitized else "Entity" + + +def _safe_rel(relation: str) -> str: + return re.sub(r"[^A-Z0-9_]", "_", (relation or "").upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" + + +def _scalar_props(data: dict) -> dict: + return { + k: v + for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + + +def graph_name_for(path: str | Path) -> str: + """Stable, key-safe FalkorDB graph name derived from a directory path.""" + resolved = str(Path(path).resolve()) + base = _safe_label(Path(resolved).name) or "graph" + digest = hashlib.sha1(resolved.encode("utf-8")).hexdigest()[:8] + return f"graphify_{base}_{digest}" + + +def simple_cycles_from_edges(edges, max_len: int = 5) -> list[list[str]]: + """Directed simple cycles up to ``max_len``, over an explicit edge list. + + Pure-Python counterpart of ``GraphStore.simple_cycles`` (which runs the + FalkorDB UDF). Depends only on the edge list, not on the graph object, so + callers work against a store, a MemGraph, or a plain NetworkX graph handed + in by a library user. Each cycle is returned once, rotated to start at its + smallest node, so the output is canonical and order-independent. + """ + adj: dict = {} + for u, v in edges: + adj.setdefault(str(u), []).append(str(v)) + for succs in adj.values(): + succs.sort() + seen: set[tuple] = set() + out: list[list[str]] = [] + + def _walk(start: str, node: str, path: list[str]) -> None: + for nxt in adj.get(node, ()): + if nxt == start and len(path) > 1: + lo = path.index(min(path)) + canon = tuple(path[lo:] + path[:lo]) + if canon not in seen: + seen.add(canon) + out.append(list(canon)) + elif nxt not in path and len(path) < max_len and nxt > start: + # `nxt > start` keeps each cycle to the single traversal that + # begins at its smallest member — no duplicate rotations. + _walk(start, nxt, path + [nxt]) + + for n in sorted(adj): + _walk(n, n, [n]) + return out + + +def pointer_path(out_dir: str | Path = "graphify-out") -> Path: + """Path of the FalkorDB pointer file that records the graph name + URI.""" + return Path(out_dir) / "falkordb.json" + + +def open_store( + out_dir: str | Path = "graphify-out", + uri: str = DEFAULT_URI, + *, + create: bool = True, + directed: bool = True, +) -> "GraphStore": + """Open the GraphStore for an output directory. + + The graph name + URI are recorded in ``/falkordb.json`` (the FalkorDB + replacement for the old ``graph.json`` location). When absent, the name is + derived deterministically from the project root (the parent of ``out_dir``). + """ + p = pointer_path(out_dir) + name = None + if p.exists(): + try: + cfg = json.loads(p.read_text(encoding="utf-8")) + name = cfg.get("graph_name") + uri = cfg.get("uri", uri) + except Exception: + name = None + if not name: + # Standard layout is /graphify-out, and the name is derived from + # the PROJECT dir so it stays stable if the output dir is recreated. For a + # non-standard dir (an explicit --graph elsewhere) derive from the dir + # itself: deriving from its parent would give every sibling graph dir the + # same name, silently pointing two different graphs at one FalkorDB graph. + from graphify.paths import GRAPHIFY_OUT as _OUT + + resolved_out = Path(out_dir).resolve() + root = resolved_out.parent if resolved_out.name == Path(_OUT).name else resolved_out + name = graph_name_for(root) + if create: + Path(out_dir).mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps({"graph_name": name, "uri": uri}), encoding="utf-8") + return GraphStore(graph_name=name, uri=uri, directed=directed) + + +def _connect_lite(dbfile: str): + """Embedded FalkorDB Lite (redislite): run the engine in-process from an RDB + file — no external server. Opt-in via the GRAPHIFY_FALKORDB_LITE env var.""" + # redis 8.0.0 crashes enabling maintenance-notifications on the host-less + # unix-socket connection redislite uses; it is irrelevant for an embedded + # engine, so neutralize it. + import redis.connection as _rc + for _name in ("AbstractConnection", "Connection", "UnixDomainSocketConnection"): + _cls = getattr(_rc, _name, None) + if _cls and hasattr(_cls, "activate_maint_notifications_handling_if_enabled"): + _cls.activate_maint_notifications_handling_if_enabled = lambda self, **kw: None + import redislite + return redislite.FalkorDB(dbfilename=dbfile) + + +def _connect(uri: str, user: str | None = None, password: str | None = None): + import os + # Embedded FalkorDB Lite is selected by (in priority order) the + # GRAPHIFY_FALKORDB_LITE env var pointing at an RDB file, or a + # `falkordb-lite://` / `lite://` URI. Default server path otherwise. + _lite = os.environ.get("GRAPHIFY_FALKORDB_LITE") + if _lite: + return _connect_lite(_lite) + parsed = urlparse(uri if "://" in uri else f"redis://{uri}") + if parsed.scheme in ("falkordb-lite", "lite"): + return _connect_lite(parsed.path or None) + + try: + from falkordb import FalkorDB + except ImportError as e: # pragma: no cover + raise ImportError("falkordb SDK not installed. Run: pip install falkordb") from e + host = parsed.hostname or "localhost" + port = parsed.port or 6379 + connect_user = parsed.username or (user if password else None) + connect_password = parsed.password or (password or None) + db = FalkorDB(host=host, port=port, username=connect_user, password=connect_password) + try: + db.connection.ping() + return db + except Exception as exc: + # Server unreachable. For the local default, transparently fall back to + # the embedded engine (FalkorDB Lite) so graphify works with zero setup, + # like the pre-FalkorDB version. An explicitly-configured remote host that + # is down is a real error and must surface, not silently use a local DB. + if host not in ("localhost", "127.0.0.1", "::1"): + raise + try: + return _connect_lite(_default_lite_dbfile()) + except ImportError: + raise ConnectionError( + f"Could not reach a FalkorDB server at {host}:{port}, and the embedded " + "engine (FalkorDB Lite) is not installed. Either start a FalkorDB server " + "or install the embedded engine: pip install falkordblite (Python >= 3.12)." + ) from exc + + +def _default_lite_dbfile() -> str: + """Stable on-disk location for the auto-fallback embedded engine. One shared + RDB holds every project's graph (each project uses a unique graph name, so + they coexist); lives outside any repo, like a server's datadir. Override with + GRAPHIFY_LITE_DBFILE.""" + import os + override = os.environ.get("GRAPHIFY_LITE_DBFILE") + if override: + return override + base = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share") + d = Path(base) / "graphify" + d.mkdir(parents=True, exist_ok=True) + return str(d / "falkordb_lite.rdb") + + +# --------------------------------------------------------------------------- +# NetworkX-shaped views over the read cache +# --------------------------------------------------------------------------- +class _NodeView: + """Mimics nx ``G.nodes``: iterable, callable(data=), subscriptable [id].""" + + def __init__(self, store: "GraphStore"): + self._s = store + + def __call__(self, data=False): + self._s._ensure_cache() + items = [(nid, self._s._ncache[nid]) for nid in self._s._nsorted] + if data: + return iter(items) + return iter([nid for nid, _ in items]) + + def __iter__(self): + self._s._ensure_cache() + return iter(self._s._nsorted) + + def __getitem__(self, node_id): + self._s._ensure_cache() + return self._s._ncache[node_id] + + def __contains__(self, node_id): + self._s._ensure_cache() + return node_id in self._s._ncache + + def __len__(self): + self._s._ensure_cache() + return len(self._s._ncache) + + def get(self, node_id, default=None): + self._s._ensure_cache() + return self._s._ncache.get(node_id, default) + + +class _EdgeView: + """Mimics nx ``G.edges``: callable(nbunch=None, data=) and iterable.""" + + def __init__(self, store: "GraphStore"): + self._s = store + + def __call__(self, nbunch=None, data=False): + self._s._ensure_cache() + if nbunch is None: + rows = self._s._esorted + else: + if isinstance(nbunch, (str, bytes)): + wanted = {nbunch} + else: + wanted = set(nbunch) + rows = [e for e in self._s._esorted if e[0] in wanted or e[1] in wanted] + if data: + return iter([(u, v, dict(a)) for u, v, a in rows]) + return iter([(u, v) for u, v, _ in rows]) + + def __iter__(self): + self._s._ensure_cache() + return iter([(u, v) for u, v, _ in self._s._esorted]) + + def __getitem__(self, pair): + """nx-style ``G.edges[u, v]`` -> edge attrs (first match, either direction).""" + u, v = pair + self._s._ensure_cache() + for a, b, attrs in self._s._esorted: + if (a == u and b == v) or (a == v and b == u): + return attrs + raise KeyError(pair) + + +class _DegreeView: + """Mimics nx ``G.degree``: callable() -> items, callable(n) -> int.""" + + def __init__(self, store: "GraphStore"): + self._s = store + + def __call__(self, nbunch=None): + degs = self._s._degree_map() + if nbunch is None: + return list(degs.items()) + if isinstance(nbunch, (str, bytes)): + return degs.get(nbunch, 0) + return [(n, degs.get(n, 0)) for n in nbunch] + + def __getitem__(self, node_id): + return self._s._degree_map().get(node_id, 0) + + +class _SubgraphView: + """Lightweight subgraph supporting the operations cohesion scoring needs.""" + + def __init__(self, store: "GraphStore", node_ids): + self._s = store + self._ids = set(node_ids) + + def number_of_nodes(self): + return len(self._ids) + + def number_of_edges(self): + self._s._ensure_cache() + return sum(1 for u, v, _ in self._s._esorted if u in self._ids and v in self._ids) + + def is_directed(self): + return self._s.directed + + def to_undirected(self, as_view: bool = False): + return self + + def neighbors(self, node_id): + self._s._ensure_cache() + out = set() + for u, v, _ in self._s._esorted: + if u not in self._ids or v not in self._ids: + continue + if u == node_id: + out.add(v) + elif v == node_id: + out.add(u) + return iter(sorted(out)) + + def louvain_partition(self, resolution: float = 1.0) -> dict: + self._s._ensure_cache() + edges = [ + [u, v, float(a.get("weight", 1.0) or 1.0)] + for u, v, a in self._s._esorted + if u in self._ids and v in self._ids + ] + return self._s.run_louvain(edges, resolution) + + def nodes(self, data=False): + self._s._ensure_cache() + ids = sorted(self._ids) + if data: + return iter([(n, self._s._ncache.get(n, {})) for n in ids]) + return iter(ids) + + def edges(self, data=False): + self._s._ensure_cache() + rows = [(u, v, a) for u, v, a in self._s._esorted if u in self._ids and v in self._ids] + if data: + return iter([(u, v, dict(a)) for u, v, a in rows]) + return iter([(u, v) for u, v, _ in rows]) + + +class MemGraph: + """In-memory read-only graph with the same view API as GraphStore. + + Used for transient derived graphs (e.g. context-filtered subgraphs) that + must not touch FalkorDB. Built from explicit node/edge lists. + """ + + def __init__(self, nodes, edges, directed: bool = True): + # nodes: iterable of (id, attrs); edges: iterable of (u, v, attrs) + self._ncache = {nid: dict(a) for nid, a in nodes} + self._nsorted = sorted(self._ncache.keys(), key=str) + self._esorted = [(u, v, dict(a)) for u, v, a in edges] + self._esorted.sort(key=lambda e: (str(e[0]), str(e[1]), json.dumps(e[2], sort_keys=True, default=str))) + self.directed = directed + self._degmap = None + self.graph: dict = {} + + def _ensure_cache(self): + return + + def _degree_map(self): + if self._degmap is None: + degs = {nid: 0 for nid in self._ncache} + for u, v, _ in self._esorted: + degs[u] = degs.get(u, 0) + 1 + degs[v] = degs.get(v, 0) + 1 + self._degmap = degs + return self._degmap + + # views + nodes = property(lambda self: _NodeView(self)) + edges = property(lambda self: _EdgeView(self)) + degree = property(lambda self: _DegreeView(self)) + + def is_directed(self): + return self.directed + + def is_multigraph(self): + return False + + def to_undirected(self, as_view: bool = False): + return self + + def number_of_nodes(self): + return len(self._ncache) + + def number_of_edges(self): + return len(self._esorted) + + def __contains__(self, node_id): + return node_id in self._ncache + + def __iter__(self): + return iter(self._nsorted) + + def __len__(self): + return len(self._ncache) + + def __getitem__(self, node_id): + adj: dict = {} + for u, v, a in self._esorted: + if u == node_id: + adj.setdefault(v, a) + elif v == node_id: + adj.setdefault(u, a) + return adj + + def has_node(self, node_id): + return node_id in self._ncache + + def has_edge(self, u, v): + for a, b, _ in self._esorted: + if (a == u and b == v) or (a == v and b == u): + return True + return False + + def has_directed_edge(self, u, v): + return any(a == u and b == v for a, b, _ in self._esorted) + + def simple_cycles(self, edges, max_len: int = 5): + return simple_cycles_from_edges(edges, max_len) + + def edge_attrs_all(self, u, v) -> list[dict]: + """Every stored edge between u and v, forward-preferred. + + ``self.edges[u, v]`` returns only the first match, which collapses + parallel links (a `references` and a `calls` edge between the same pair) + and lets `path` print a relation the traversed pair doesn't carry + (#2074). Callers that must report all relations use this instead. + """ + fwd = [attrs for a, b, attrs in self._esorted if a == u and b == v] + if fwd: + return fwd + return [attrs for a, b, attrs in self._esorted if a == v and b == u] + + def neighbors(self, node_id): + out = set() + for u, v, _ in self._esorted: + if u == node_id: + out.add(v) + elif v == node_id: + out.add(u) + return iter(sorted(out)) + + def successors(self, node_id): + return iter(sorted({v for u, v, _ in self._esorted if u == node_id})) + + def predecessors(self, node_id): + return iter(sorted({u for u, v, _ in self._esorted if v == node_id})) + + def in_edges(self, node_id, data=False): + rows = [(u, v, a) for u, v, a in self._esorted if v == node_id] + if data: + return [(u, v, dict(a)) for u, v, a in rows] + return [(u, v) for u, v, _ in rows] + + def in_degree(self, node_id) -> int: + return sum(1 for _u, v, _a in self._esorted if v == node_id) + + def out_degree(self, node_id) -> int: + return sum(1 for u, _v, _a in self._esorted if u == node_id) + + def subgraph(self, node_ids): + return _SubgraphView(self, node_ids) + + def shortest_path(self, src, tgt, max_hops: int = 8): + from collections import deque + + if src not in self._ncache or tgt not in self._ncache: + return None + adj: dict = {} + for u, v, _ in self._esorted: + adj.setdefault(u, set()).add(v) + adj.setdefault(v, set()).add(u) + prev = {src: None} + q = deque([(src, 0)]) + while q: + node, d = q.popleft() + if node == tgt: + path = [] + while node is not None: + path.append(node) + node = prev[node] + return list(reversed(path)) + if d >= max_hops: + continue + for nb in sorted(adj.get(node, ())): + if nb not in prev: + prev[nb] = node + q.append((nb, d + 1)) + return None + + +# --------------------------------------------------------------------------- +# Cache-free views for GraphStore: every access is a live FalkorDB query or a +# paginated stream. The graph is NEVER materialized in Python — the store holds +# no node/edge data, only a connection. +# --------------------------------------------------------------------------- +class _SNodeView: + def __init__(self, s): + self._s = s + + def __call__(self, data=False): + return self._s._iter_nodes(data=data) + + def __iter__(self): + return self._s._iter_nodes(data=False) + + def __getitem__(self, nid): + return self._s._node_attrs(nid) # raises KeyError if absent + + def __contains__(self, nid): + return self._s.has_node(nid) + + def __len__(self): + return self._s.number_of_nodes() + + def get(self, nid, default=None): + try: + return self._s._node_attrs(nid) + except KeyError: + return default + + +class _SEdgeView: + def __init__(self, s): + self._s = s + + def __call__(self, nbunch=None, data=False): + return self._s._iter_edges(nbunch=nbunch, data=data) + + def __iter__(self): + return self._s._iter_edges(nbunch=None, data=False) + + def __getitem__(self, pair): + u, v = pair + d = self._s._edge_attrs(u, v) + if d is None: + raise KeyError(pair) + return d + + +class _SDegreeView: + def __init__(self, s): + self._s = s + + def __call__(self, nbunch=None): + if nbunch is None: + return list(self._s._iter_degrees()) + if isinstance(nbunch, (str, bytes)): + return self._s._degree_one(nbunch) + return [(n, self._s._degree_one(n)) for n in nbunch] + + def __getitem__(self, nid): + return self._s._degree_one(nid) + + +class _SStoreSubgraph: + """Subgraph restricted to a node-id set; all ops run as scoped queries.""" + + def __init__(self, store, node_ids): + self._s = store + self._ids = list(node_ids) + + def number_of_nodes(self): + return len(self._ids) + + def number_of_edges(self): + if not self._ids: + return 0 + return self._s._rows( + "MATCH (a:Entity)-[r]-(b:Entity) WHERE a.id IN $ids AND b.id IN $ids " + "AND id(a) < id(b) RETURN count(r)", + {"ids": self._ids}, timeout=_QUERY_TIMEOUT_MS, + )[0][0] + + def louvain_partition(self, resolution: float = 1.0) -> dict: + if not self._ids: + return {} + edges = self._s._rows( + "MATCH (a:Entity)-[r]->(b:Entity) WHERE a.id IN $ids AND b.id IN $ids " + "RETURN a.id, b.id, coalesce(r.weight, 1.0)", + {"ids": self._ids}, timeout=_QUERY_TIMEOUT_MS, + ) + return self._s.run_louvain([[str(u), str(v), float(w)] for u, v, w in edges], resolution) + + def is_directed(self): + return self._s.directed + + def to_undirected(self, as_view: bool = False): + return self + + +class GraphStore: + """A named FalkorDB graph with a NetworkX-shaped API. + + Holds NO graph data in Python — the graph lives in FalkorDB. Every read is a + scoped query or a paginated stream; nothing is cached/materialized. + """ + + def __init__( + self, + graph_name: str = "graphify", + uri: str = DEFAULT_URI, + *, + user: str | None = None, + password: str | None = None, + directed: bool = True, + ): + self.graph_name = graph_name + self.uri = uri + self.directed = directed + self._db = _connect(uri, user, password) + self._g = self._db.select_graph(graph_name) + p = urlparse(uri if "://" in uri else "redis://" + uri) + self._server_key = f"{p.hostname}:{p.port or 6379}" + self.graph: dict = {} # graph-level metadata only (hyperedges) — NOT the graph + self._ensure_resultset_size() + self._ensure_schema() + # UDFs (louvain/edgeBetweenness/simpleCycles) are loaded lazily by the + # algorithm wrappers that need them — not here — so interactive commands + # (query/path/explain/affected) don't pay a UDF reload on every process. + self._load_meta() + + # ---- views (nx-compatible, cache-free: each access is a live query) ----- + @property + def nodes(self): + return _SNodeView(self) + + @property + def edges(self): + return _SEdgeView(self) + + @property + def degree(self): + return _SDegreeView(self) + + def in_degree(self, node_id) -> int: + """Number of incoming edges (nx-compatible scoped count).""" + return int(self._rows( + "MATCH (:Entity)-[r]->(:Entity {id:$id}) RETURN count(r)", {"id": node_id})[0][0]) + + def out_degree(self, node_id) -> int: + """Number of outgoing edges (nx-compatible scoped count).""" + return int(self._rows( + "MATCH (:Entity {id:$id})-[r]->(:Entity) RETURN count(r)", {"id": node_id})[0][0]) + + # ---- streaming + scoped read helpers (no materialization) --------------- + def _stream(self, match_return: str, order_by: str, params: dict | None = None): + """Yield rows page-by-page (only one page in memory at a time).""" + skip = 0 + while True: + rows = self._rows( + f"{match_return} ORDER BY {order_by} SKIP {skip} LIMIT {_PAGE}", + params, timeout=_QUERY_TIMEOUT_MS, + ) + for r in rows: + yield r + if len(rows) < _PAGE: + break + skip += _PAGE + + def _iter_nodes(self, data=False): + for r in self._stream("MATCH (n:Entity) RETURN n.id, properties(n)", "id(n)"): + if data: + d = dict(r[1]); d.pop(_META_KEY, None) + yield (r[0], d) + else: + yield r[0] + + def _iter_edges(self, nbunch=None, data=False): + if nbunch is None: + gen = self._stream("MATCH (a:Entity)-[r]->(b:Entity) RETURN a.id, b.id, properties(r)", "id(r)") + else: + ids = [nbunch] if isinstance(nbunch, (str, bytes)) else list(nbunch) + gen = self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) WHERE a.id IN $ids OR b.id IN $ids " + "RETURN a.id, b.id, properties(r)", "id(r)", {"ids": ids}, + ) + for r in gen: + if data: + yield (r[0], r[1], dict(r[2])) + else: + yield (r[0], r[1]) + + def _iter_degrees(self): + for r in self._stream("MATCH (n:Entity) RETURN n.id, size((n)--())", "id(n)"): + yield (r[0], int(r[1])) + + def _node_attrs(self, nid): + rows = self._rows("MATCH (n:Entity {id:$id}) RETURN properties(n)", {"id": nid}, timeout=_QUERY_TIMEOUT_MS) + if not rows: + raise KeyError(nid) + d = dict(rows[0][0]); d.pop(_META_KEY, None) + return d + + def _edge_attrs(self, u, v): + # Prefer the forward u->v edge; fall back to a reverse v->u edge so the + # lookup still succeeds in the undirected model (but reports the right + # direction's relation when both exist). + rows = self._rows( + "MATCH (a:Entity {id:$u})-[r]->(b:Entity {id:$v}) RETURN properties(r) LIMIT 1", + {"u": u, "v": v}, timeout=_QUERY_TIMEOUT_MS, + ) + if not rows: + rows = self._rows( + "MATCH (a:Entity {id:$u})<-[r]-(b:Entity {id:$v}) RETURN properties(r) LIMIT 1", + {"u": u, "v": v}, timeout=_QUERY_TIMEOUT_MS, + ) + return dict(rows[0][0]) if rows else None + + def _degree_one(self, nid): + rows = self._rows( + "MATCH (n:Entity {id:$id}) OPTIONAL MATCH (n)-[r]-() RETURN count(r)", + {"id": nid}, timeout=_QUERY_TIMEOUT_MS, + ) + return rows[0][0] if rows else 0 + + # ---- low-level helpers -------------------------------------------------- + def query(self, cypher: str, params: dict | None = None, timeout: int | None = None): + if timeout is not None: + return self._g.query(cypher, params or {}, timeout=timeout) + return self._g.query(cypher, params or {}) + + def _rows(self, cypher: str, params: dict | None = None, timeout: int | None = None) -> list: + return self.query(cypher, params, timeout=timeout).result_set + + def _ensure_resultset_size(self) -> None: + """Raise the row cap and query timeouts so large-graph reads aren't + silently truncated (RESULTSET_SIZE) or killed (TIMEOUT).""" + with _udf_lock: + if self._server_key in _server_configured: + return + for key, val in ( + ("RESULTSET_SIZE", _RESULTSET_SIZE), + ("TIMEOUT_DEFAULT", _QUERY_TIMEOUT_MS), + ("TIMEOUT_MAX", _QUERY_TIMEOUT_MS), + ): + try: + self._db.connection.execute_command("GRAPH.CONFIG", "SET", key, str(val)) + except Exception: + pass # managed server may forbid it; per-query timeout + pagination cover us + _server_configured.add(self._server_key) + + def _ensure_schema(self) -> None: + try: + self._g.query("CREATE INDEX FOR (n:Entity) ON (n.id)") + except Exception: + pass + + def _ensure_udfs(self) -> None: + with _udf_lock: + if self._server_key in _udf_loaded_servers: + return + src = _UDF_SRC.read_text() + try: + self._db.connection.execute_command("GRAPH.UDF", "LOAD", "REPLACE", _UDF_LIB, src) + except Exception: + try: + self._db.connection.execute_command("GRAPH.UDF", "LOAD", _UDF_LIB, src) + except Exception: + pass + _udf_loaded_servers.add(self._server_key) + + def _invalidate(self) -> None: + # No cache to invalidate — the store holds no graph data. Kept as a no-op + # so mutation methods don't need to special-case it. + return + + # ---- graph-level metadata ---------------------------------------------- + def _load_meta(self) -> None: + try: + rows = self._rows("MATCH (m:GraphMeta {key:$k}) RETURN m.data", {"k": _META_KEY}) + if rows and rows[0][0]: + self.graph = json.loads(rows[0][0]) + except Exception: + self.graph = {} + + def save_meta(self) -> None: + """Persist the in-memory `.graph` dict (hyperedges etc.) to FalkorDB.""" + payload = json.dumps({k: v for k, v in self.graph.items() if not k.startswith("_")}, default=str) + self._g.query( + "MERGE (m:GraphMeta {key:$k}) SET m.data = $d", {"k": _META_KEY, "d": payload} + ) + + # ---- lifecycle ---------------------------------------------------------- + def clear(self) -> None: + try: + self._g.delete() + except Exception: + pass + self._g = self._db.select_graph(self.graph_name) + self.graph = {} + self._invalidate() + self._ensure_schema() + + def __getitem__(self, node_id): + """Adjacency access: ``G[u]`` -> {neighbor_id: edge_attrs}. + + Lists neighbors reachable in EITHER direction (the undirected nx.Graph + model graphify uses), but when a pair is connected both ways the FORWARD + (u->m) edge wins, so ``G[u][v]`` reports the u->v relation rather than an + arbitrary direction's attributes. + """ + rows = self._rows( + "MATCH (n:Entity {id:$id})-[r]-(m:Entity) " + "RETURN m.id, properties(r), startNode(r).id = $id AS fwd ORDER BY fwd DESC", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + adj: dict = {} + for mid, props, _fwd in rows: + adj.setdefault(mid, dict(props)) + return adj + + def __contains__(self, node_id): + return self.has_node(node_id) + + def __iter__(self): + return self._iter_nodes(data=False) + + def __len__(self): + return self.number_of_nodes() + + def is_directed(self) -> bool: + return self.directed + + def is_multigraph(self) -> bool: + return False + + def to_undirected(self, as_view: bool = False): + return self + + def copy(self): + return self + + # ---- construction (batched) -------------------------------------------- + _BATCH = 1000 + + def add_nodes_from(self, rows, *, fresh: bool = False) -> None: + """Bulk-upsert nodes. When ``fresh`` (graph known empty/just cleared, ids + unique) use CREATE — it skips MERGE's existence check and is several times + faster on a from-scratch build.""" + by_label: dict[str, list[dict]] = {} + materialized = [] + for item in rows: + if isinstance(item, tuple): + node_id, attrs = item + else: + node_id, attrs = item, {} + props = _scalar_props(attrs) + props["id"] = node_id + # _origin (AST provenance) is the one underscore-marker that must + # survive into the store: incremental rebuild reads it back from + # graph.json to evict stale AST symbols removed from a surviving + # file (#1116). Other _-prefixed keys stay dropped. + if isinstance(attrs.get("_origin"), str): + props["_origin"] = attrs["_origin"] + ftype = _safe_label(str(attrs.get("file_type", "Entity")).capitalize()) + by_label.setdefault(ftype, []).append(props) + materialized.append(1) + for ftype, batch in by_label.items(): + verb = f"CREATE (n:Entity:{ftype})" if fresh else f"MERGE (n:Entity {{id: row.id}}) SET n:{ftype}" + for i in range(0, len(batch), self._BATCH): + chunk = batch[i : i + self._BATCH] + self._g.query(f"UNWIND $rows AS row {verb} SET n += row", {"rows": chunk}) + if materialized: + self._invalidate() + + def add_node(self, node_id: str, **attrs) -> None: + self.add_nodes_from([(node_id, attrs)]) + + def add_edges_from(self, rows, *, fresh: bool = False) -> None: + """Bulk-upsert edges. When ``fresh`` use CREATE instead of MERGE (no + edge-existence check) — endpoints must already exist.""" + by_rel: dict[str, list[dict]] = {} + for u, v, attrs in rows: + rel = _safe_rel(str(attrs.get("relation") or "")) + props = _scalar_props(attrs) + # The Cypher relationship TYPE always needs a value (_safe_rel falls + # back to RELATED_TO), but the `relation` PROPERTY must stay absent + # when the edge genuinely carried none — otherwise a reader can't + # distinguish "no relation" from a real one and prints a fabricated + # label instead of the honest "related" fallback (#2074). + if attrs.get("relation"): + props.setdefault("relation", attrs["relation"]) + by_rel.setdefault(rel, []).append({"src": u, "tgt": v, "props": props}) + for rel, batch in by_rel.items(): + verb = "CREATE" if fresh else "MERGE" + for i in range(0, len(batch), self._BATCH): + chunk = batch[i : i + self._BATCH] + self._g.query( + f"UNWIND $rows AS row MATCH (a:Entity {{id: row.src}}), (b:Entity {{id: row.tgt}}) " + f"{verb} (a)-[r:{rel}]->(b) SET r += row.props", + {"rows": chunk}, + ) + self._invalidate() + + def add_edge(self, u: str, v: str, **attrs) -> None: + self.add_edges_from([(u, v, attrs)]) + + # ---- mutation ----------------------------------------------------------- + def remove_nodes(self, node_ids) -> None: + ids = list(node_ids) + if not ids: + return + self._g.query("UNWIND $ids AS nid MATCH (n:Entity {id: nid}) DETACH DELETE n", {"ids": ids}) + self._invalidate() + + remove_nodes_from = remove_nodes + + def remove_node(self, node_id: str) -> None: + self.remove_nodes([node_id]) + + def remove_edges(self, pairs) -> None: + rows = [{"src": u, "tgt": v} for u, v in pairs] + if not rows: + return + # DIRECTED delete: pairs come from edges() in native source->target + # order, so only the src->tgt edge is removed. An undirected `-[r]-` + # here would also delete a reciprocal tgt->src edge whose own + # source_file may not be pruned (over-deletion during incremental prune). + self._g.query( + "UNWIND $rows AS row MATCH (a:Entity {id: row.src})-[r]->(b:Entity {id: row.tgt}) DELETE r", + {"rows": rows}, + ) + self._invalidate() + + remove_edges_from = remove_edges + + def remove_edge(self, u: str, v: str) -> None: + self.remove_edges([(u, v)]) + + def has_node(self, node_id: str) -> bool: + return bool(self._rows("MATCH (n:Entity {id:$id}) RETURN 1 LIMIT 1", {"id": node_id})) + + def has_edge(self, u: str, v: str) -> bool: + return bool(self._rows( + "MATCH (a:Entity {id:$u})-[]-(b:Entity {id:$v}) RETURN 1 LIMIT 1", {"u": u, "v": v})) + + def has_directed_edge(self, u: str, v: str) -> bool: + """True only if an edge is stored in the u -> v orientation.""" + return bool(self._rows( + "MATCH (a:Entity {id:$u})-[]->(b:Entity {id:$v}) RETURN 1 LIMIT 1", {"u": u, "v": v})) + + def edge_attrs_all(self, u: str, v: str) -> list[dict]: + """Every stored edge between u and v, forward-preferred. + + ``_edge_attrs`` uses LIMIT 1, which collapses parallel links (a + `references` and a `calls` edge between the same pair) and lets `path` + print a relation the traversed pair doesn't carry (#2074). Callers that + must report all relations use this instead. + """ + rows = self._rows( + "MATCH (a:Entity {id:$u})-[r]->(b:Entity {id:$v}) RETURN properties(r)", + {"u": u, "v": v}, timeout=_QUERY_TIMEOUT_MS, + ) + if not rows: + rows = self._rows( + "MATCH (a:Entity {id:$u})<-[r]-(b:Entity {id:$v}) RETURN properties(r)", + {"u": u, "v": v}, timeout=_QUERY_TIMEOUT_MS, + ) + return [dict(r[0]) for r in rows] + + def number_of_nodes(self) -> int: + # count() is uncapped and cheap — independent of the read cache. + return self._rows("MATCH (n:Entity) RETURN count(n)")[0][0] + + def number_of_edges(self) -> int: + return self._rows("MATCH (:Entity)-[r]->(:Entity) RETURN count(r)")[0][0] + + def neighbors(self, node_id: str): + rows = self._rows( + "MATCH (n:Entity {id:$id})-[]-(m:Entity) RETURN DISTINCT m.id", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + return iter(sorted(str(r[0]) for r in rows)) + + def in_edges(self, node_id: str, data: bool = False): + rows = self._rows( + "MATCH (s:Entity)-[r]->(n:Entity {id:$id}) RETURN s.id, properties(r)", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + rows.sort(key=lambda r: str(r[0])) + if data: + return [(str(s), node_id, dict(p)) for s, p in rows] + return [(str(s), node_id) for s, _ in rows] + + def successors(self, node_id: str): + """Outgoing neighbors (directed), sorted.""" + rows = self._rows("MATCH (n:Entity {id:$id})-[]->(m:Entity) RETURN DISTINCT m.id", {"id": node_id}) + return iter(sorted(str(r[0]) for r in rows)) + + def predecessors(self, node_id: str): + """Incoming neighbors (directed), sorted.""" + rows = self._rows("MATCH (m:Entity)-[]->(n:Entity {id:$id}) RETURN DISTINCT m.id", {"id": node_id}) + return iter(sorted(str(r[0]) for r in rows)) + + def subgraph(self, node_ids): + return _SStoreSubgraph(self, node_ids) + + def set_communities(self, communities: dict) -> None: + """Persist community ids onto nodes: {cid: [node_ids]} -> n.community.""" + rows = [] + for cid, node_ids in communities.items(): + for nid in node_ids: + rows.append({"id": nid, "c": int(cid)}) + if not rows: + return + for i in range(0, len(rows), self._BATCH): + chunk = rows[i : i + self._BATCH] + self._g.query( + "UNWIND $rows AS row MATCH (n:Entity {id: row.id}) SET n.community = row.c", + {"rows": chunk}, + ) + self._invalidate() + + def prune_repo(self, repo_tag: str) -> int: + before = self.number_of_nodes() + self._g.query("MATCH (n:Entity {repo: $t}) DETACH DELETE n", {"t": repo_tag}) + self._invalidate() + return before - self.number_of_nodes() + + # ---- algorithms --------------------------------------------------------- + def shortest_path(self, src: str, tgt: str, max_hops: int = 8): + """Undirected shortest path via level-batched BFS in the engine — one + query per BFS level expanding the whole frontier (not the whole graph). + Avoids algo.SPpaths (pathologically slow undirected) and never loads the + graph into Python.""" + if src == tgt: + return [src] if self.has_node(src) else None + prev = {src: None} + frontier = [src] + for _ in range(int(max_hops)): + if not frontier: + break + rows = self._rows( + "UNWIND $f AS fid MATCH (n:Entity {id: fid})-[]-(m:Entity) RETURN fid, m.id", + {"f": frontier}, timeout=_QUERY_TIMEOUT_MS, + ) + nxt = [] + for fid, mid in sorted(rows, key=lambda r: (str(r[0]), str(r[1]))): + if mid in prev: + continue + prev[mid] = fid + if mid == tgt: + path = [mid] + while path[-1] is not None: + path.append(prev[path[-1]]) + return list(reversed(path[:-1])) + nxt.append(mid) + frontier = nxt + return None + + _RENDER_COLS = "n.label, n.source_file, n.source_location, n.community" + + def subgraph_render_data(self, node_ids, edge_pairs, *, limit=None, seeds=None): + """Fetch node attrs+degree and edge attrs for a subgraph, for rendering + query/path results. Returns ``(nodes, eattrs, total)`` where ``total`` is + the full node count. + + When ``limit`` is set and the subgraph is larger, only the top-``limit`` + nodes by degree (plus ``seeds``, always kept) are fetched — the engine + ranks+truncates so a hub query never pulls attributes for tens of + thousands of nodes just to discard all but the slice the caller's token + budget can show. Edges are fetched only among the kept nodes.""" + ids = list(node_ids) + total = len(ids) + nodes: dict = {} + + def _absorb(rows): + for r in rows: + nodes[r[0]] = {"label": r[1], "source_file": r[2], "source_location": r[3], + "community": r[4], "degree": int(r[5])} + + if ids and (limit is None or total <= limit): + _absorb(self._rows( + f"UNWIND $ids AS i MATCH (n:Entity {{id: i}}) " + f"RETURN i, {self._RENDER_COLS}, size((n)--())", + {"ids": ids}, timeout=_QUERY_TIMEOUT_MS, + )) + elif ids: + # Seeds are always kept (rendered first); the rest are ranked by + # degree in-engine and truncated to fill out the limit. + id_set = set(ids) + seed_list = [s for s in (seeds or []) if s in id_set] + if seed_list: + _absorb(self._rows( + f"UNWIND $ids AS i MATCH (n:Entity {{id: i}}) " + f"RETURN i, {self._RENDER_COLS}, size((n)--())", + {"ids": seed_list}, timeout=_QUERY_TIMEOUT_MS, + )) + k = max(0, limit - len(nodes)) + others = [i for i in ids if i not in nodes] + if k and others: + _absorb(self._rows( + f"UNWIND $ids AS i MATCH (n:Entity {{id: i}}) " + f"WITH n, size((n)--()) AS deg ORDER BY deg DESC LIMIT $k " + f"RETURN n.id, {self._RENDER_COLS}, deg", + {"ids": others, "k": k}, timeout=_QUERY_TIMEOUT_MS, + )) + + eattrs: dict = {} + pairs = [[u, v] for u, v in edge_pairs if u in nodes and v in nodes] + if pairs: + # startNode(r) recovers the STORED orientation: the pair (p[0], p[1]) + # is traversal visit order, so rendering it as-is prints a + # caller→callee edge backwards whenever the callee was visited first. + # r.source_file/source_location are the relation SITE, which the + # renderer cites instead of a def line (#BUG1). + for u, v, rel, conf, ctx, src, sf, loc in self._rows( + "UNWIND $pairs AS p MATCH (a:Entity {id: p[0]})-[r]-(b:Entity {id: p[1]}) " + "RETURN p[0], p[1], r.relation, r.confidence, r.context, " + "startNode(r).id, r.source_file, r.source_location", + {"pairs": pairs}, timeout=_QUERY_TIMEOUT_MS, + ): + eattrs.setdefault((u, v), { + "relation": rel, "confidence": conf, "context": ctx, + "_src": src, "_tgt": (v if src == u else u), + "source_file": sf, "source_location": loc, + }) + return nodes, eattrs, total + + def node_detail(self, node_id: str): + """(attrs, degree) for one node, in one scoped query. None if absent.""" + rows = self._rows( + "MATCH (n:Entity {id:$id}) RETURN properties(n), size((n)--())", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + if not rows: + return None + attrs = dict(rows[0][0]) + attrs.pop(_META_KEY, None) + return attrs, int(rows[0][1]) + + def node_connections(self, node_id: str) -> list[dict]: + """All edges incident to a node (both directions) with neighbor label + + degree + relation/confidence — one scoped query, no full-graph load. + + Also returns each edge's own source_file/source_location: that is the + call/import/reference SITE (in the caller's file for an incoming call), + which `explain` prints per connection (#BUG1). Reading it from the edge + — not the neighbor node — is what makes it a site rather than a def line. + """ + rows = self._rows( + "MATCH (n:Entity {id:$id})-[r]->(m:Entity) " + "RETURN 'out' AS dir, m.id AS mid, m.label AS mlabel, r.relation AS rel, " + "r.confidence AS conf, size((m)--()) AS mdeg, " + "r.source_file AS rsf, r.source_location AS rloc " + "UNION ALL " + "MATCH (n:Entity {id:$id})<-[r]-(m:Entity) " + "RETURN 'in' AS dir, m.id AS mid, m.label AS mlabel, r.relation AS rel, " + "r.confidence AS conf, size((m)--()) AS mdeg, " + "r.source_file AS rsf, r.source_location AS rloc", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + return [ + {"dir": r[0], "id": r[1], "label": r[2] or r[1], "relation": r[3] or "", + "confidence": r[4] or "", "degree": int(r[5]), + "source_file": r[6] or "", "source_location": r[7] or ""} + for r in rows + ] + + def search_nodes(self, terms) -> list[dict]: + """Candidate nodes whose norm_label / id / source_file contains any term. + This is the in-engine prefilter for query/explain — only nodes that could + score > 0 are returned, so Python scores a small set, never the whole graph.""" + terms = [t for t in terms if t] + if not terms: + return [] + conds, params = [], {} + for i, t in enumerate(terms): + params[f"t{i}"] = t + # coalesce: graphs built without a stored norm_label (e.g. test fixtures) + # still match on the lowercased label. + conds.append( + f"(coalesce(n.norm_label, toLower(n.label)) CONTAINS $t{i} " + f"OR toLower(n.id) CONTAINS $t{i} OR toLower(n.source_file) CONTAINS $t{i})" + ) + rows = self._rows( + f"MATCH (n:Entity) WHERE {' OR '.join(conds)} " + "RETURN n.id, n.label, n.norm_label, n.source_file", + params, timeout=_QUERY_TIMEOUT_MS, + ) + return [ + {"id": r[0], "label": r[1] or "", "norm_label": r[2] or "", "source_file": r[3] or ""} + for r in rows + ] + + def doc_freqs(self, terms) -> dict: + """{term: number of nodes whose norm_label contains term} — for IDF, in-engine.""" + out = {} + for t in terms: + if not t: + continue + out[t] = self._rows( + "MATCH (n:Entity) WHERE n.norm_label CONTAINS $t RETURN count(n)", + {"t": t}, timeout=_QUERY_TIMEOUT_MS, + )[0][0] + return out + + def confidence_counts(self) -> dict: + """{confidence_label: edge_count} over all edges — one in-engine aggregation + (so graph_stats/audit never stream every edge into Python).""" + rows = self._rows( + "MATCH (:Entity)-[r]->(:Entity) " + "RETURN coalesce(r.confidence, 'EXTRACTED'), count(r)", + timeout=_QUERY_TIMEOUT_MS, + ) + return {str(k): int(v) for k, v in rows} + + def find_node_ids(self, *, label=None, source_file=None, label_contains=None, label_bare=None, limit=None) -> list: + """Scoped node lookup by exact label / exact source_file / bare callable + name / label substring.""" + if label is not None: + q, v = "MATCH (n:Entity) WHERE toLower(n.label) = $v RETURN n.id", label.lower() + elif source_file is not None: + q, v = "MATCH (n:Entity) WHERE toLower(n.source_file) = $v RETURN n.id", source_file.lower() + elif label_bare is not None: + # Bare callable name: a query like "foo" matches a node labelled + # "foo" OR "foo()" (the "()" decoration callables carry). Mirrors the + # in-memory path's _bare_name tier so `affected foo` resolves natively. + q, v = ( + "MATCH (n:Entity) WHERE toLower(n.label) = $v OR toLower(n.label) = $v + '()' RETURN n.id", + label_bare.lower(), + ) + else: + q, v = "MATCH (n:Entity) WHERE toLower(n.label) CONTAINS $v RETURN n.id", label_contains.lower() + if limit: + q += f" LIMIT {int(limit)}" + return [r[0] for r in self._rows(q, {"v": v}, timeout=_QUERY_TIMEOUT_MS)] + + def incoming_edges(self, frontier_ids, relations) -> list: + """All (frontier_id, src_id, relation, source_file, source_location) + incoming edges for a frontier whose relation is in `relations` — one query + for the whole frontier level. + + The edge's own source_file/source_location is the call/import/reference + SITE in the SOURCE's file, which `affected` reports instead of the node's + definition line (#BUG1).""" + if not frontier_ids: + return [] + return self._rows( + "UNWIND $f AS fid MATCH (src:Entity)-[r]->(n:Entity {id: fid}) " + "WHERE r.relation IN $rels " + "RETURN fid, src.id, r.relation, r.source_file, r.source_location", + {"f": list(frontier_ids), "rels": list(relations)}, + timeout=_QUERY_TIMEOUT_MS, + ) + + def member_nodes(self, node_id: str) -> list: + """Node ids one outward `method`/`contains` hop from `node_id`. + + A caller can bind to a class's method node rather than the class node + itself, so those callers are unreachable from the class in a reverse walk + (#1669/#1634). `affected` uses these as extra BFS seeds only.""" + return [ + r[0] for r in self._rows( + "MATCH (n:Entity {id:$id})-[r]->(m:Entity) " + "WHERE r.relation IN ['method','contains'] RETURN m.id", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + ] + + def node_attrs_batch(self, ids) -> dict: + """Fetch properties for many node ids in one query.""" + ids = list(ids) + if not ids: + return {} + rows = self._rows( + "UNWIND $ids AS i MATCH (n:Entity {id: i}) RETURN i, properties(n)", + {"ids": ids}, + timeout=_QUERY_TIMEOUT_MS, + ) + return {r[0]: dict(r[1]) for r in rows} + + def top_degree_nodes(self, limit: int = 500) -> list[dict]: + """Top-N nodes by (undirected) degree, computed in-engine. Returns only + `limit` rows with the attrs god_nodes needs — no full-graph load.""" + rows = self._rows( + "MATCH (n:Entity) WITH n, size((n)--()) AS deg " + "RETURN n.id, n.label, n.source_file, deg ORDER BY deg DESC LIMIT $lim", + {"lim": int(limit)}, + timeout=_QUERY_TIMEOUT_MS, + ) + return [ + {"id": r[0], "label": r[1] or "", "source_file": r[2] or "", "degree": int(r[3])} + for r in rows + ] + + def stream_full_edges(self): + """Stream edges with endpoint label/source/degree + relation/confidence so + surprise analysis needs no per-edge node lookups. Page-at-a-time, no load.""" + for r in self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) " + "WITH a, b, r, size((a)--()) AS da, size((b)--()) AS db " + "RETURN a.id, a.label, a.source_file, da, b.id, b.label, b.source_file, db, " + "r.relation, r.confidence", "id(r)", + ): + yield { + "u": r[0], "u_label": r[1] or "", "u_source": r[2] or "", "u_deg": int(r[3]), + "v": r[4], "v_label": r[5] or "", "v_source": r[6] or "", "v_deg": int(r[7]), + "relation": r[8] or "", "confidence": r[9] or "EXTRACTED", + } + + def node_betweenness(self) -> dict: + # Scope to :Entity so the internal :GraphMeta bookkeeping node (which has + # no id and no edges) never leaks in as a phantom "None" key. + rows = self._rows("CALL algo.betweenness() YIELD node, score WHERE node:Entity RETURN node.id, score", timeout=_QUERY_TIMEOUT_MS) + return {str(nid): float(s) for nid, s in rows} + + def _all_weighted_edges(self): + # Transient marshaling for the Louvain UDF param (the UDF can't read edges + # itself). Streamed page-by-page; not retained as a cache. + return [ + [r[0], r[1], float(r[2] if r[2] is not None else 1.0)] + for r in self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) RETURN a.id, b.id, coalesce(r.weight, 1.0)", "id(r)") + ] + + def run_louvain(self, edges, resolution: float = 1.0) -> dict: + """Run the Louvain UDF over an explicit weighted edge list.""" + if not edges: + return {} + self._ensure_udfs() + rows = self._rows(f"RETURN {_UDF_LIB}.louvain($e, $res)", {"e": edges, "res": resolution}, timeout=_QUERY_TIMEOUT_MS) + return {str(k): int(v) for k, v in dict(rows[0][0]).items()} + + def louvain_partition(self, resolution: float = 1.0) -> dict: + edges = self._all_weighted_edges() + if not edges: + return {nid: i for i, nid in enumerate(self.nodes)} + return self.run_louvain(edges, resolution) + + def edge_betweenness(self) -> dict: + edges = [[r[0], r[1]] for r in self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) RETURN a.id, b.id", "id(r)")] + if not edges: + return {} + self._ensure_udfs() + res = self._rows(f"RETURN {_UDF_LIB}.edgeBetweenness($e)", {"e": edges}, timeout=_QUERY_TIMEOUT_MS) + out: dict = {} + for key, score in dict(res[0][0]).items(): + u, v = key.split("\t", 1) + out[(u, v)] = float(score) + return out + + def simple_cycles(self, edges, max_len: int = 5): + payload = [[str(u), str(v), 1.0] for u, v in edges] + if not payload: + return [] + self._ensure_udfs() + rows = self._rows(f"RETURN {_UDF_LIB}.simpleCycles($e, $n)", {"e": payload, "n": int(max_len)}, timeout=_QUERY_TIMEOUT_MS) + return [[str(x) for x in cyc] for cyc in rows[0][0]] + + # ---- traversal (hub-capped, level-batched in the engine) ---------------- + def _hub_threshold(self) -> int: + # Graph-level constant (p99 of the degree distribution). Computing it scans + # every node's degree (~0.4s), so cache it in graph metadata and reuse it + # across query invocations. Cleared automatically on rebuild (clear() drops + # the GraphMeta node); SET-only ops like community labels don't change it. + cached = self.graph.get("hub_threshold") + if cached is not None: + return int(cached) + rows = self._rows( + "MATCH (n:Entity) WITH size((n)--()) AS d RETURN percentileDisc(d, 0.99)", + timeout=_QUERY_TIMEOUT_MS, + ) + p99 = rows[0][0] if rows and rows[0][0] is not None else 0 + hub = max(50, int(p99)) + self.graph["hub_threshold"] = hub + try: + self.save_meta() + except Exception: + pass + return hub + + def bfs(self, seeds, depth, hub_threshold=None, contexts=None): + return self._traverse(seeds, depth, hub_threshold, contexts) + + def dfs(self, seeds, depth, hub_threshold=None, contexts=None): + # Depth-bounded neighborhood is gathered the same way; results are + # relevance-ranked downstream, so DFS vs BFS ordering does not matter here. + return self._traverse(seeds, depth, hub_threshold, contexts) + + def _traverse(self, seeds, depth, hub_threshold, contexts=None): + """Hub-capped depth-bounded traversal — one query per level expanding the + whole frontier in the engine (never loads the graph). When ``contexts`` is + given, only edges whose ``context`` matches are followed (in-engine, so the + graph is never copied into Python for context filtering).""" + seeds = list(seeds) + if hub_threshold is None: + hub_threshold = self._hub_threshold() + contexts = list(contexts) if contexts else None + edge_match = "MATCH (n)-[r]-(m:Entity)" + edge_where = " WHERE r.context IN $ctx" if contexts else "" + seed_set = set(seeds) + visited = set(seeds) + edges_seen: list = [] + frontier = seeds + for _ in range(depth): + if not frontier: + break + params = {"f": frontier, "seeds": seeds, "hub": hub_threshold} + if contexts: + params["ctx"] = contexts + rows = self._rows( + "UNWIND $f AS fid MATCH (n:Entity {id: fid}) " + "WITH fid, n, size((n)--()) AS deg " + "WHERE fid IN $seeds OR deg < $hub " + f"{edge_match}{edge_where} RETURN DISTINCT fid, m.id", + params, + timeout=_QUERY_TIMEOUT_MS, + ) + # Defer the visited update to END-OF-LEVEL (mirrors v8 _bfs). Marking + # a node visited mid-level would drop every parent->child edge except + # the first when a node has multiple parents in the same BFS level + # (e.g. the diamond A->B, A->C, B->D, C->D loses C->D), thinning the + # subgraph the user sees. Record an edge to each newly-discovered node + # from EVERY same-level parent, but enqueue the node only once. + nxt = [] + newly: set = set() + for fid, mid in sorted(rows, key=lambda r: (str(r[0]), str(r[1]))): + if mid not in visited: + edges_seen.append((fid, mid)) + if mid not in newly: + newly.add(mid) + nxt.append(mid) + visited |= newly + frontier = nxt + return visited, edges_seen diff --git a/graphify/udfs/graphify_algos.js b/graphify/udfs/graphify_algos.js new file mode 100644 index 000000000..5376eb7a8 --- /dev/null +++ b/graphify/udfs/graphify_algos.js @@ -0,0 +1,344 @@ +// graphify_algos — FalkorDB server-side UDF library. +// +// These replace the NetworkX algorithms graphify used to run in-process, +// for the ones FalkorDB has no built-in equivalent of: +// - louvain : community detection (was graspologic Leiden / nx.community.louvain) +// - edgeBetweenness : edge betweenness (was nx.edge_betweenness_centrality) +// - simpleCycles : directed simple cycles (was nx.simple_cycles) +// +// DESIGN: these are PURE-COMPUTE functions. They do NOT read the graph through +// the UDF graph API — that API (as of FalkorDB 4.18) does not expose edge +// endpoints via iterateEdges, and getNeighbors() ignores its direction config +// and never surfaces edge weights. Instead the Python GraphStore pulls the edge +// list (with weights) via ordinary Cypher and passes it in as an argument. That +// keeps weights/direction correct and makes these functions trivially testable. +// +// DETERMINISM: every function processes nodes/edges in a fixed sorted order and +// uses no randomness, so results are byte-stable across runs (graphify pins +// seed=42 everywhere; here we get the same guarantee from sorted iteration). +// +// Edge argument shape (all functions): an array of [u, v, w] triples, where u/v +// are node-id strings and w is an optional numeric weight (defaults to 1). + +// ---------------------------------------------------------------------------- +// louvain(edges, resolution) -> { nodeId: communityId(int) } +// +// Standard Louvain modularity maximization on the UNDIRECTED, WEIGHTED graph. +// Two-phase: (1) greedy local moving, (2) community aggregation, repeated until +// modularity stops improving. The returned community ids are arbitrary integers; +// the Python cluster() wrapper re-indexes them to size-stable ids. +// +// Node move order is the sorted node order (deterministic). Ties in modularity +// gain are broken toward the smallest community id, again for determinism. +// ---------------------------------------------------------------------------- +function louvain(edges, resolution) { + var gamma = (resolution === undefined || resolution === null) ? 1.0 : resolution; + + // Collect a sorted, de-duplicated node list -> integer index. + var idSet = {}; + for (var i = 0; i < edges.length; i++) { + idSet[edges[i][0]] = true; + idSet[edges[i][1]] = true; + } + var ids = Object.keys(idSet).sort(); + var index = {}; + for (var n = 0; n < ids.length; n++) index[ids[n]] = n; + var N = ids.length; + if (N === 0) return {}; + + // Build a weighted undirected adjacency as an aggregated edge map. + // adj[i] = { j: weight } summed across parallel/both-direction edges. + // Self-loops are kept (they contribute to a node's degree twice, as usual). + var adj = new Array(N); + for (var a = 0; a < N; a++) adj[a] = {}; + var totalW = 0.0; // m (sum of all edge weights, each undirected edge once) + for (var e = 0; e < edges.length; e++) { + var u = index[edges[e][0]]; + var v = index[edges[e][1]]; + var w = edges[e][2]; + if (w === undefined || w === null || isNaN(w)) w = 1.0; + if (u === v) { + adj[u][u] = (adj[u][u] || 0) + w; // self-loop + totalW += w; + continue; + } + adj[u][v] = (adj[u][v] || 0) + w; + adj[v][u] = (adj[v][u] || 0) + w; + totalW += w; + } + if (totalW === 0) { + // No edges: every node is its own community. + var solo = {}; + for (var s = 0; s < N; s++) solo[ids[s]] = s; + return solo; + } + + // Weighted degree k_i for each node (self-loops count twice). + function degrees(adjacency, count) { + var k = new Array(count); + for (var p = 0; p < count; p++) { + var sum = 0.0; + var row = adjacency[p]; + for (var q in row) { + sum += (parseInt(q, 10) === p) ? 2 * row[q] : row[q]; + } + k[p] = sum; + } + return k; + } + + // One Louvain level: greedy local moving on `adjacency` (size `count`), + // total weight `m`. Returns { comm: int[], improved: bool }. + function oneLevel(adjacency, count, m) { + var k = degrees(adjacency, count); + var comm = new Array(count); + var sigmaTot = new Array(count); // total degree of community c + for (var i = 0; i < count; i++) { comm[i] = i; sigmaTot[i] = k[i]; } + var twoM = 2 * m; + + var improvedAny = false; + var moved = true; + var guard = 0; + while (moved && guard < 100) { + moved = false; + guard++; + for (var node = 0; node < count; node++) { + var nodeComm = comm[node]; + // Weight from `node` into each neighboring community. + var wToComm = {}; + var row = adjacency[node]; + var selfLoop = row[node] || 0; + for (var nb in row) { + var nbi = parseInt(nb, 10); + if (nbi === node) continue; + var c = comm[nbi]; + wToComm[c] = (wToComm[c] || 0) + row[nb]; + } + // Remove node from its current community. + sigmaTot[nodeComm] -= k[node]; + var wToOwn = wToComm[nodeComm] || 0; + + // Evaluate candidate communities: current + all neighbor communities. + // Deterministic: consider community ids in ascending order. + var candidates = {}; + candidates[nodeComm] = true; + for (var cc in wToComm) candidates[cc] = true; + var candList = Object.keys(candidates).map(function (x) { return parseInt(x, 10); }).sort(function (p, q) { return p - q; }); + + var bestComm = nodeComm; + // Gain of staying (relative baseline) — start from current community. + var bestGain = wToOwn - gamma * sigmaTot[nodeComm] * k[node] / twoM; + for (var ci = 0; ci < candList.length; ci++) { + var cand = candList[ci]; + var wln = wToComm[cand] || 0; + var gain = wln - gamma * sigmaTot[cand] * k[node] / twoM; + if (gain > bestGain + 1e-12) { + bestGain = gain; + bestComm = cand; + } + } + // Insert node into the chosen community. + sigmaTot[bestComm] += k[node]; + comm[node] = bestComm; + if (bestComm !== nodeComm) { moved = true; improvedAny = true; } + } + } + return { comm: comm, improved: improvedAny }; + } + + // Aggregate communities into a smaller graph for the next level. + function aggregate(adjacency, count, comm) { + // Renumber communities to a dense 0..K-1 range (deterministic by first appearance in sorted node order). + var remap = {}; + var K = 0; + for (var i = 0; i < count; i++) { + var c = comm[i]; + if (remap[c] === undefined) { remap[c] = K++; } + } + var dense = new Array(count); + for (var j = 0; j < count; j++) dense[j] = remap[comm[j]]; + var newAdj = new Array(K); + for (var a = 0; a < K; a++) newAdj[a] = {}; + for (var p = 0; p < count; p++) { + var cp = dense[p]; + var row = adjacency[p]; + for (var q in row) { + var cq = dense[parseInt(q, 10)]; + newAdj[cp][cq] = (newAdj[cp][cq] || 0) + row[q]; + } + } + return { adj: newAdj, count: K, dense: dense }; + } + + // node -> current community id at the original-node granularity. + var nodeComm = new Array(N); + for (var t = 0; t < N; t++) nodeComm[t] = t; + + var curAdj = adj; + var curCount = N; + var pass = 0; + while (pass < 50) { + pass++; + var level = oneLevel(curAdj, curCount, totalW); + if (!level.improved) break; + var agg = aggregate(curAdj, curCount, level.comm); + // Fold this level's mapping back onto original nodes. + for (var orig = 0; orig < N; orig++) { + nodeComm[orig] = agg.dense[nodeComm[orig]]; + } + curAdj = agg.adj; + curCount = agg.count; + if (agg.count === curCount && !level.improved) break; + } + + var out = {}; + for (var z = 0; z < N; z++) out[ids[z]] = nodeComm[z]; + return out; +} + +// ---------------------------------------------------------------------------- +// edgeBetweenness(edges) -> { "u\tv": score } +// +// Brandes' algorithm for edge betweenness on the UNDIRECTED, UNWEIGHTED graph +// (matches nx.edge_betweenness_centrality default weight=None). Normalized by +// 2/(n(n-1)) like networkx's undirected default. Edge keys are "u\tv" with u= 0; si--) { + var ww = S[si]; + var preds = P[ww]; + for (var pi = 0; pi < preds.length; pi++) { + var pv = preds[pi]; + var c = (sigma[pv] / sigma[ww]) * (1 + delta[ww]); + var key = ekey(pv, ww); + eb[key] = (eb[key] || 0) + c; + delta[pv] += c; + } + } + } + + // Undirected: each shortest path counted from both endpoints -> divide by 2. + // Normalize like networkx undirected default: scale = 2/(n(n-1)). + var scale = (N > 1) ? (2.0 / (N * (N - 1))) : 1.0; + var out = {}; + for (var key2 in eb) { + var parts = key2.split(","); + var a2 = parseInt(parts[0], 10), b2 = parseInt(parts[1], 10); + var val = (eb[key2] / 2.0) * scale; + out[ids[a2] + "\t" + ids[b2]] = val; + } + return out; +} + +// ---------------------------------------------------------------------------- +// simpleCycles(edges, maxLen) -> [[id, id, ...], ...] +// +// Enumerate simple cycles in a DIRECTED graph, bounded by maxLen (number of +// nodes in the cycle). Edges are treated as directed u->v (weight ignored). +// Replaces nx.simple_cycles for import-cycle detection; the Python side already +// dedupes rotations and caps the result, so here we just enumerate (with a hard +// safety cap) starting each search from the lexicographically smallest member. +// ---------------------------------------------------------------------------- +function simpleCycles(edges, maxLen) { + var limit = (maxLen === undefined || maxLen === null) ? 5 : maxLen; + var SAFETY = 5000; + + var idSet = {}; + for (var i = 0; i < edges.length; i++) { idSet[edges[i][0]] = true; idSet[edges[i][1]] = true; } + var ids = Object.keys(idSet).sort(); + var index = {}; + for (var n = 0; n < ids.length; n++) index[ids[n]] = n; + var N = ids.length; + + var succ = new Array(N); + for (var a = 0; a < N; a++) succ[a] = {}; + for (var e = 0; e < edges.length; e++) { + var u = index[edges[e][0]], v = index[edges[e][1]]; + if (u === v) continue; // self-loop is not a multi-node cycle + succ[u][v] = true; + } + var out = []; + + // For each start node (ascending), DFS for cycles returning to start, only + // visiting nodes > start so each cycle is found once from its min member. + for (var start = 0; start < N && out.length < SAFETY; start++) { + var stack = [start]; + var onStack = {}; + onStack[start] = true; + + function dfs(node) { + if (out.length >= SAFETY) return; + var nbrs = Object.keys(succ[node]).map(function (x) { return parseInt(x, 10); }).sort(function (p, q) { return p - q; }); + for (var ni = 0; ni < nbrs.length; ni++) { + var nb = nbrs[ni]; + if (nb === start && stack.length >= 2) { + if (stack.length <= limit) { + var cyc = []; + for (var si = 0; si < stack.length; si++) cyc.push(ids[stack[si]]); + out.push(cyc); + } + continue; + } + if (nb < start || onStack[nb]) continue; + if (stack.length >= limit) continue; + stack.push(nb); + onStack[nb] = true; + dfs(nb); + stack.pop(); + delete onStack[nb]; + } + } + dfs(start); + } + return out; +} + +falkor.register('louvain', louvain); +falkor.register('edgeBetweenness', edgeBetweenness); +falkor.register('simpleCycles', simpleCycles); diff --git a/graphify/watch.py b/graphify/watch.py index 45bd97b66..a547a99c3 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -676,6 +676,10 @@ def _canonical_topology_for_compare(graph_data: dict) -> dict: n.pop("community", None) n.pop("community_name", None) n.pop("norm_label", None) + # _origin is AST provenance (kept in graph.json for #1116 eviction), + # not topology — exclude it so an unchanged graph compares equal + # regardless of which serializer wrote each side. + n.pop("_origin", None) norm_nodes.append(n) canonical["nodes"] = sorted( norm_nodes, @@ -720,13 +724,8 @@ def _canonical_topology_for_compare(graph_data: dict) -> dict: def _topology_from_graph(G) -> dict: - from networkx.readwrite import json_graph - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - return data + from graphify.graphjson import to_node_link + return to_node_link(G) def _check_shrink( @@ -1206,7 +1205,10 @@ def _add_deleted_source(path: Path) -> None: "total_words": detected.get("total_words", 0), } - G = build_from_json(result) + # Build into the FalkorDB graph bound to this output dir so query/serve + # (which load via the pointer file) see the rebuilt graph. + from graphify.store import open_store as _open_store + G = build_from_json(result, store=_open_store(out, create=True)) candidate_topology = _topology_from_graph(G) if existing_graph_data: try: diff --git a/graphify/wiki.py b/graphify/wiki.py index cb9c6cf35..5ab4d2b49 100644 --- a/graphify/wiki.py +++ b/graphify/wiki.py @@ -4,7 +4,6 @@ from collections import Counter from pathlib import Path from urllib.parse import quote -import networkx as nx from graphify.build import edge_data diff --git a/pyproject.toml b/pyproject.toml index 68dfbe048..0e0ea33cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,12 @@ license-files = ["LICENSE", "LICENSE-MIT", "NOTICE"] keywords = ["claude", "claude-code", "codex", "opencode", "kilo", "cursor", "gemini", "aider", "kiro", "pi", "devin", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] requires-python = ">=3.10" dependencies = [ - "networkx>=3.4", + "falkordb>=1.0", + # Embedded FalkorDB engine — lets graphify run with zero setup (no separate + # server) by auto-falling back to it when no local server is reachable. + # Only available on Python >= 3.12; on older interpreters a running FalkorDB + # server is required. + "falkordblite; python_version >= '3.12'", "numpy>=1.21", "rapidfuzz>=3.0", "tree-sitter>=0.23.0,<0.26", @@ -54,11 +59,18 @@ Issues = "https://github.com/Graphify-Labs/graphify/issues" # CVE-2026-48818 / CVE-2026-54283 fixes (both resolved by 1.3.1) (#1391, #1396). mcp = ["mcp", "starlette>=1.3.1"] neo4j = ["neo4j"] -falkordb = ["falkordb"] +# falkordb is a core dependency; this extra exists so the documented +# `graphifyy[falkordb]` install command resolves instead of erroring on an +# unknown extra. +falkordb = ["falkordb>=1.0"] pdf = ["pypdf>=6.12.0", "markdownify"] watch = ["watchdog"] svg = ["matplotlib", "numpy>=2.0; python_version >= '3.13'"] leiden = ["graspologic; python_version < '3.13'"] +# FalkorDB Lite: embedded in-process engine (redislite + bundled falkordb module), +# no external server. Requires Python >= 3.12. Select with a falkordb-lite:// URI +# or the GRAPHIFY_FALKORDB_LITE env var. +lite = ["falkordblite; python_version >= '3.12'"] office = ["python-docx", "openpyxl"] google = ["openpyxl"] postgres = ["psycopg[binary]"] @@ -121,7 +133,11 @@ include-package-data = false # under graphify/skills//references/, and the always-on injection blocks # under graphify/always_on/. There is no graphify/skills//SKILL.md in the # repo, so no SKILL.md glob is needed here. -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-agents.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md", "skills/*/references/*.md", "always_on/*.md"] +# "udfs/*.js" ships the FalkorDB server-side algorithms (Louvain, edge +# betweenness, simple cycles) that store.py loads at runtime via +# Path(__file__).parent / "udfs" — without it, wheel/sdist installs hit a +# FileNotFoundError when clustering/analyze run (works only from a checkout). +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-agents.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md", "skills/*/references/*.md", "always_on/*.md", "udfs/*.js"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/conftest.py b/tests/conftest.py index 33dc81a0a..59e903617 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from pathlib import Path from typing import Any @@ -23,6 +24,129 @@ def _sandbox_home(tmp_path_factory, monkeypatch): monkeypatch.setattr(Path, "home", classmethod(lambda cls: home)) return home +# -------------------------------------------------------------------------- +# FalkorDB-backed graph fixtures (NetworkX replacement). +# Every test that needs a graph gets a fresh, uniquely-named GraphStore bound +# to a running FalkorDB. Skips cleanly when none is reachable. +# docker run -d -p 6379:6379 falkordb/falkordb:latest +# Overridable via FALKORDB_URI / FALKORDB_HOST / FALKORDB_PORT. +# -------------------------------------------------------------------------- +def _falkordb_uri() -> str: + if os.environ.get("FALKORDB_URI"): + return os.environ["FALKORDB_URI"] + host = os.environ.get("FALKORDB_HOST", "localhost") + port = os.environ.get("FALKORDB_PORT", "6379") + return f"falkordb://{host}:{port}" + + +@pytest.fixture(scope="session") +def falkordb_uri() -> str: + return _falkordb_uri() + + +@pytest.fixture(scope="session") +def _require_falkordb(falkordb_uri): + """Skip only tests that actually need a live FalkorDB. NOT autouse — the + DB-backed fixtures (store/seed_graph/make_store) depend on it, so DB tests + skip when no engine is reachable while DB-free tests (graphjson, _minhash, + detect, security, ...) still run, instead of the whole suite vanishing. + Session-scoped so the ping happens once.""" + pytest.importorskip("falkordb") + from graphify.store import _connect + + try: + _connect(falkordb_uri).connection.ping() + except Exception as e: # pragma: no cover - environment dependent + pytest.skip(f"no FalkorDB reachable at {falkordb_uri} ({e})") + + +_store_counter = {"n": 0} + + +@pytest.fixture() +def store(falkordb_uri, _require_falkordb): + """A fresh, empty, uniquely-named GraphStore; cleaned up after the test.""" + from graphify.store import GraphStore + + _store_counter["n"] += 1 + gs = GraphStore(graph_name=f"pytest_{os.getpid()}_{_store_counter['n']}", uri=falkordb_uri) + gs.clear() + try: + yield gs + finally: + try: + gs.clear() + except Exception: + pass + + +@pytest.fixture() +def seed_graph(falkordb_uri, _require_falkordb): + """Seed a FalkorDB graph for an output dir from node-link data, writing the + pointer file so the CLI/serve loaders find it (replaces writing graph.json). + + Usage: seed_graph(tmp_path, nodes=[...], links=[...]) -> GraphStore. + Each node dict needs an ``id``; each link dict ``source``/``target``. + """ + from graphify.store import open_store + + created = [] + + def _seed(out_dir, nodes=(), links=(), *, write_artifact=True): + store = open_store(out_dir, uri=falkordb_uri, create=True) + store.clear() + store.add_nodes_from([ + (n["id"], {k: v for k, v in n.items() if k != "id"}) for n in nodes + ]) + store.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in links + ]) + created.append(store) + if write_artifact: + import json as _json + from pathlib import Path as _Path + (_Path(out_dir) / "graph.json").write_text( + _json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": list(nodes), "links": list(links)}), + encoding="utf-8", + ) + return store + + try: + yield _seed + finally: + for s in created: + try: + s.clear() + except Exception: + pass + + +@pytest.fixture() +def make_store(falkordb_uri, _require_falkordb): + """Factory for tests needing more than one graph (e.g. graph_diff).""" + from graphify.store import GraphStore + + created = [] + + def _make(): + _store_counter["n"] += 1 + gs = GraphStore(graph_name=f"pytest_{os.getpid()}_{_store_counter['n']}", uri=falkordb_uri) + gs.clear() + created.append(gs) + return gs + + try: + yield _make + finally: + for gs in created: + try: + gs.clear() + except Exception: + pass + + _ANALYZE_WARNING_FILTERS = ( "ignore:Tensorflow not installed; ParametricUMAP will be unavailable:ImportWarning:umap", "ignore:Please import `random` from the `scipy\\.sparse` namespace.*:" diff --git a/tests/nxcompat.py b/tests/nxcompat.py new file mode 100644 index 000000000..0eb95e51e --- /dev/null +++ b/tests/nxcompat.py @@ -0,0 +1,120 @@ +"""Minimal networkx-compatible shim for the test suite. + +The FalkorDB solution ships no networkx dependency (the package has zero runtime +networkx imports). The suite still builds small in-memory graph fixtures with the +`nx.Graph()` / `nx.DiGraph()` + add_node/add_edge idiom, so this module provides a +drop-in `nx`: a mutable builder whose read API is delegated to the in-house +`MemGraph` (which already exposes the full nx-style view API). Tests import it as +`from tests import nxcompat as nx` — no networkx install required. +""" +from graphify.store import MemGraph + + +class _MutableGraph: + """nx-like mutable graph; reads delegate to a lazily (re)built MemGraph.""" + + def __init__(self, directed: bool): + self._nodes: dict = {} + self._edges: list = [] + self._directed = directed + self.graph: dict = {} + self._mg = None + + # --- construction (networkx API) --- + def add_node(self, nid, **attrs): + self._nodes.setdefault(nid, {}).update(attrs) + self._mg = None + + def add_nodes_from(self, items): + for it in items: + if isinstance(it, (tuple, list)): + self.add_node(it[0], **(it[1] if len(it) > 1 and it[1] else {})) + else: + self.add_node(it) + + def add_edge(self, u, v, **attrs): + self._nodes.setdefault(u, {}) + self._nodes.setdefault(v, {}) + self._edges.append((u, v, attrs)) + self._mg = None + + def add_edges_from(self, items): + for it in items: + attrs = it[2] if len(it) > 2 and it[2] else {} + self.add_edge(it[0], it[1], **attrs) + + def is_directed(self): + return self._directed + + # --- read API: build a MemGraph (sharing the .graph dict) and delegate --- + def _build(self) -> MemGraph: + if self._mg is None: + self._mg = MemGraph(list(self._nodes.items()), self._edges, directed=self._directed) + self._mg.graph = self.graph + return self._mg + + def __getattr__(self, name): + # Only reached for names not set on the instance (nodes/edges/degree/ + # neighbors/subgraph/number_of_*/has_*/shortest_path/...). + return getattr(self._build(), name) + + def __contains__(self, x): + return x in self._build() + + def __iter__(self): + return iter(self._build()) + + def __len__(self): + return len(self._build()) + + def __getitem__(self, key): + return self._build()[key] + + +class Graph(_MutableGraph): + def __init__(self): + super().__init__(directed=False) + + +class DiGraph(_MutableGraph): + def __init__(self): + super().__init__(directed=True) + + +def node_link_data(G, edges: str = "links") -> dict: + """Serialize to networkx's node-link dict shape (the graph.json on-disk form). + + Stands in for ``networkx.readwrite.json_graph.node_link_data`` so tests can + write a graph.json fixture without importing networkx. ``edges`` names the + key the link list lands under, matching the nx keyword of the same name. + """ + return { + "directed": G.is_directed(), + "multigraph": False, + "graph": dict(G.graph), + "nodes": [{**data, "id": nid} for nid, data in G.nodes(data=True)], + edges: [ + {**data, "source": u, "target": v} for u, v, data in G.edges(data=True) + ], + } + + +def relabel_nodes(G, mapping): + """Return a new graph with node ids remapped via `mapping` (unmapped ids kept).""" + out = DiGraph() if G.is_directed() else Graph() + for nid, data in G.nodes(data=True): + out.add_node(mapping.get(nid, nid), **data) + for u, v, data in G.edges(data=True): + out.add_edge(mapping.get(u, u), mapping.get(v, v), **data) + return out + + +def complete_graph(n): + """Undirected graph on nodes 0..n-1 with every pair connected (nx semantics).""" + out = Graph() + for i in range(n): + out.add_node(i) + for i in range(n): + for j in range(i + 1, n): + out.add_edge(i, j) + return out diff --git a/tests/test_affected_cli.py b/tests/test_affected_cli.py index ca608b6b3..d18d4285a 100644 --- a/tests/test_affected_cli.py +++ b/tests/test_affected_cli.py @@ -2,28 +2,29 @@ import json -import networkx as nx -from networkx.readwrite import json_graph +import pytest -import graphify.__main__ as mainmod +from tests import nxcompat as nx +from tests.nxcompat import node_link_data +import graphify.__main__ as mainmod -def _write_graph(tmp_path): - graph = nx.DiGraph() - graph.add_node("target", label="Foo", source_file="pkg/foo.py", source_location="L1") - graph.add_node("caller", label="X()", source_file="app.py", source_location="L4") - graph.add_node("barrel", label="__init__.py", source_file="pkg/__init__.py", source_location=None) - graph.add_node("consumer", label="app.py", source_file="app.py", source_location=None) - graph.add_edge("caller", "target", relation="calls", context="call", confidence="EXTRACTED") - graph.add_edge("barrel", "target", relation="re_exports", context="export", confidence="EXTRACTED") - graph.add_edge("consumer", "target", relation="imports", context="import", confidence="EXTRACTED") +_NODES = [ + {"id": "target", "label": "Foo", "source_file": "pkg/foo.py", "source_location": "L1", "file_type": "code"}, + {"id": "caller", "label": "X()", "source_file": "app.py", "source_location": "L4", "file_type": "code"}, + {"id": "barrel", "label": "__init__.py", "source_file": "pkg/__init__.py", "file_type": "code"}, + {"id": "consumer", "label": "app.py", "source_file": "app.py", "file_type": "code"}, +] +_LINKS = [ + {"source": "caller", "target": "target", "relation": "calls", "context": "call", "confidence": "EXTRACTED"}, + {"source": "barrel", "target": "target", "relation": "re_exports", "context": "export", "confidence": "EXTRACTED"}, + {"source": "consumer", "target": "target", "relation": "imports", "context": "import", "confidence": "EXTRACTED"}, +] + + +def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links")), encoding="utf-8") - return graph_path - - -def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -43,8 +44,9 @@ def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, caps assert "imports" in out -def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -60,38 +62,19 @@ def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_ assert "__init__.py" not in out -def test_affected_cli_forces_directed_on_undirected_graph(monkeypatch, tmp_path, capsys): - """A graph persisted with directed=false must still recover caller->callee - direction (#1174): affected on the callee returns the caller, not the callee - or nothing. Without forcing directed=True, node_link_graph builds an - undirected Graph, predecessors() collapses, and the reverse traversal breaks. - """ - graph = nx.DiGraph() - graph.add_node("A", label="caller_fn", source_file="a.py", source_location="L1") - graph.add_node("B", label="callee_fn", source_file="b.py", source_location="L2") - graph.add_edge("A", "B", relation="calls", context="call", confidence="EXTRACTED") - - data = json_graph.node_link_data(graph, edges="links") - # Persist as undirected on disk to reproduce the bug condition. - data["directed"] = False - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(data), encoding="utf-8") - +def test_affected_cli_unbuilt_graph_exits_with_build_hint(monkeypatch, tmp_path): + """On a never-built graph, `affected` must exit non-zero (telling the user to + build) rather than silently reporting an empty result. Guards the + connect_graph empty-graph check (parity with query/path/explain).""" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, "argv", - ["graphify", "affected", "B", "--relation", "calls", "--graph", str(graph_path)], + ["graphify", "affected", "Foo", "--graph", str(tmp_path / "graph.json")], ) - - mainmod.main() - - out = capsys.readouterr().out - # A (the caller) is affected by a change to B (the callee). - assert "caller_fn" in out - assert "calls" in out - # B is the query node, not an affected node, and the result is not empty. - assert "No affected nodes found." not in out + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code != 0 def test_affected_cli_loads_edges_keyed_graph(monkeypatch, tmp_path, capsys): @@ -104,7 +87,7 @@ def test_affected_cli_loads_edges_keyed_graph(monkeypatch, tmp_path, capsys): graph.add_edge("caller", "target", relation="calls", context="call", confidence="EXTRACTED") # Emulate graphify extract output: top-level "edges" key instead of "links". - data = json_graph.node_link_data(graph, edges="links") + data = node_link_data(graph, edges="links") data["edges"] = data.pop("links") graph_path = tmp_path / "graph.json" graph_path.write_text(json.dumps(data), encoding="utf-8") @@ -252,7 +235,7 @@ def test_affected_cli_source_file_path_uses_file_level_node(monkeypatch, tmp_pat confidence="EXTRACTED", ) graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links")), encoding="utf-8") + graph_path.write_text(json.dumps(node_link_data(graph, edges="links")), encoding="utf-8") monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( @@ -284,7 +267,7 @@ def _write_callsite_graph(tmp_path): confidence="EXTRACTED", source_file="apollo_pipeline_status.py", source_location="L158") gp = tmp_path / "graph.json" - gp.write_text(json.dumps(json_graph.node_link_data(g, edges="links")), encoding="utf-8") + gp.write_text(json.dumps(node_link_data(g, edges="links")), encoding="utf-8") return gp @@ -306,7 +289,7 @@ def test_affected_falls_back_to_def_line_when_edge_has_no_location(monkeypatch, g.add_node("t", label="target()", source_file="b.py", source_location="L5") g.add_edge("loader", "t", relation="calls", confidence="INFERRED") # no source_location gp = tmp_path / "graph.json" - gp.write_text(json.dumps(json_graph.node_link_data(g, edges="links")), encoding="utf-8") + gp.write_text(json.dumps(node_link_data(g, edges="links")), encoding="utf-8") monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr(mainmod.sys, "argv", ["graphify", "affected", "target", "--graph", str(gp)]) mainmod.main() diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 7bff432cf..d8178efbc 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -1,6 +1,6 @@ """Tests for analyze.py.""" import json -import networkx as nx +from tests import nxcompat as nx import pytest from pathlib import Path from graphify.build import build_from_json @@ -60,9 +60,9 @@ def test_surprising_connections_excludes_concept_nodes(): assert "Abstract Concept" not in labels -def test_surprising_connections_single_file_uses_community_bridges(): +def test_surprising_connections_single_file_uses_community_bridges(store): """Single-file graph: should return cross-community edges, not empty list.""" - G = nx.Graph() + G = store # Build a graph with 2 clear communities + 1 bridge edge for i in range(5): G.add_node(f"a{i}", label=f"A{i}", file_type="code", source_file="single.py", @@ -625,46 +625,39 @@ def _make_file_node(path: str) -> tuple[str, dict]: return nid, {"label": Path(path).name, "source_file": path, "file_type": "code"} -def _make_cycle_graph_directed() -> nx.DiGraph: - G = nx.DiGraph() - +def _make_cycle_graph_directed(store): a_id, a = _make_file_node("src/a.ts") b_id, b = _make_file_node("src/b.ts") c_id, c = _make_file_node("src/c.ts") d_id, d = _make_file_node("src/d.ts") ext_id = _make_id("react") - G.add_node(a_id, **a) - G.add_node(b_id, **b) - G.add_node(c_id, **c) - G.add_node(d_id, **d) - # External-like node (no source_file): must be skipped safely. - G.add_node(ext_id, label="react", file_type="code") - - # 2-cycle: a <-> b - G.add_edge(a_id, b_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - G.add_edge(b_id, a_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - - # 3-cycle: b -> c -> d -> b - G.add_edge(b_id, c_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - G.add_edge(c_id, d_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - G.add_edge(d_id, b_id, relation="imports_from", source_file="src/d.ts", confidence="EXTRACTED") - - # Self-loop: c imports itself - G.add_edge(c_id, c_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - - # Mixed edge types: must not bleed into cycle graph - G.add_edge(a_id, ext_id, relation="calls", source_file="src/a.ts", confidence="INFERRED") - G.add_edge(a_id, ext_id, relation="contains", source_file="src/a.ts", confidence="EXTRACTED") - - # Edge whose target has no source_file: must be skipped, no garbage label fallback - G.add_edge(a_id, ext_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - - return G - - -def test_find_import_cycles_returns_structured_records(): - G = _make_cycle_graph_directed() + store.add_nodes_from([ + (a_id, a), (b_id, b), (c_id, c), (d_id, d), + # External-like node (no source_file): must be skipped safely. + (ext_id, {"label": "react", "file_type": "code"}), + ]) + store.add_edges_from([ + # 2-cycle: a <-> b + (a_id, b_id, {"relation": "imports_from", "source_file": "src/a.ts", "confidence": "EXTRACTED"}), + (b_id, a_id, {"relation": "imports_from", "source_file": "src/b.ts", "confidence": "EXTRACTED"}), + # 3-cycle: b -> c -> d -> b + (b_id, c_id, {"relation": "imports_from", "source_file": "src/b.ts", "confidence": "EXTRACTED"}), + (c_id, d_id, {"relation": "imports_from", "source_file": "src/c.ts", "confidence": "EXTRACTED"}), + (d_id, b_id, {"relation": "imports_from", "source_file": "src/d.ts", "confidence": "EXTRACTED"}), + # Self-loop: c imports itself + (c_id, c_id, {"relation": "imports_from", "source_file": "src/c.ts", "confidence": "EXTRACTED"}), + # Mixed edge types: must not bleed into cycle graph + (a_id, ext_id, {"relation": "calls", "source_file": "src/a.ts", "confidence": "INFERRED"}), + (a_id, ext_id, {"relation": "contains", "source_file": "src/a.ts", "confidence": "EXTRACTED"}), + # Edge whose target has no source_file: must be skipped, no garbage fallback. + (a_id, ext_id, {"relation": "imports_from", "source_file": "src/a.ts", "confidence": "EXTRACTED"}), + ]) + return store + + +def test_find_import_cycles_returns_structured_records(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) assert isinstance(cycles, list) assert cycles @@ -674,40 +667,39 @@ def test_find_import_cycles_returns_structured_records(): assert "why" in cycles[0] -def test_find_import_cycles_detects_2_and_3_cycles(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_detects_2_and_3_cycles(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) cycle_sets = [set(c["cycle"]) for c in cycles] assert any({"src/a.ts", "src/b.ts"}.issubset(s) for s in cycle_sets) assert any({"src/b.ts", "src/c.ts", "src/d.ts"}.issubset(s) for s in cycle_sets) -def test_find_import_cycles_includes_self_loop_cycle(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_includes_self_loop_cycle(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) assert any(c["cycle"] == ["src/c.ts"] and c["length"] == 1 for c in cycles) -def test_find_import_cycles_respects_max_cycle_length(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_respects_max_cycle_length(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G, max_cycle_length=2) assert all(c["length"] <= 2 for c in cycles) -def test_find_import_cycles_skips_nodes_without_source_file(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_skips_nodes_without_source_file(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) flat = " ".join(" ".join(c["cycle"]) for c in cycles) assert "react" not in flat -def test_find_import_cycles_handles_undirected_graph_input(): - Gd = _make_cycle_graph_directed() - Gu = nx.Graph() - Gu.add_nodes_from(Gd.nodes(data=True)) - Gu.add_edges_from(Gd.edges(data=True)) - cycles = find_import_cycles(Gu) - assert cycles # should still resolve orientation via edge.source_file +def test_find_import_cycles_handles_undirected_graph_input(store): + # Edges are stored directed natively; orientation is still resolved via + # edge.source_file, so cycle detection works regardless. + G = _make_cycle_graph_directed(store) + cycles = find_import_cycles(G) + assert cycles def test_find_import_cycles_ignores_non_import_relations(): @@ -726,8 +718,8 @@ def test_find_import_cycles_empty_graph(): assert find_import_cycles(nx.DiGraph()) == [] -def test_find_import_cycles_no_cycles(): - G = nx.DiGraph() +def test_find_import_cycles_no_cycles(store): + G = store x_id, x = _make_file_node("x.ts") y_id, y = _make_file_node("y.ts") G.add_node(x_id, **x) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index b5751adcc..67ffca2ae 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,9 +1,8 @@ """Tests for graphify/benchmark.py.""" from __future__ import annotations import json -import pytest -import networkx as nx -from networkx.readwrite import json_graph +from pathlib import Path +from tests import nxcompat as nx from graphify.benchmark import run_benchmark, print_benchmark, _query_subgraph_tokens, _SAMPLE_QUESTIONS, _safe, _hr @@ -23,8 +22,16 @@ def _make_graph() -> nx.Graph: def _write_graph(G: nx.Graph, path) -> None: - data = json_graph.node_link_data(G, edges="links") - path.write_text(json.dumps(data)) + """Seed the FalkorDB graph for path's output dir from an nx fixture graph.""" + from graphify.store import open_store + + store = open_store(Path(path).parent, create=True) + store.clear() + store.add_nodes_from([(n, dict(d, file_type=d.get("file_type", "code"))) for n, d in G.nodes(data=True)]) + store.add_edges_from([(u, v, dict(d)) for u, v, d in G.edges(data=True)]) + path.write_text(json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": [{"id": n, **d} for n, d in G.nodes(data=True)], + "links": []})) # --- _query_subgraph_tokens --- @@ -170,14 +177,3 @@ def test_print_benchmark_survives_cp1252_stdout(tmp_path, monkeypatch, capsys): # ASCII fallbacks must be present, fancy glyphs must not. assert "─" not in written assert "→" not in written - - -def test_run_benchmark_rejects_oversized_graph(monkeypatch, tmp_path): - """#F4: run_benchmark must refuse to read a graph.json that exceeds - the size cap before parsing it into memory.""" - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - run_benchmark(str(graph_file)) diff --git a/tests/test_build.py b/tests/test_build.py index 0851ba75e..c009f87bd 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,52 +1,9 @@ import json from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph -from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges, dedupe_nodes +from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas FIXTURES = Path(__file__).parent / "fixtures" - -def test_dedupe_edges_collapses_exact_parallels(): - # #1317: --no-cluster / incremental update concatenate edge lists raw. - edges = [ - {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, - {"source": "a", "target": "b", "relation": "calls", "source_location": "L9"}, # dup - {"source": "a", "target": "b", "relation": "imports"}, # different relation: kept - {"source": "b", "target": "c", "relation": "calls"}, - ] - out = dedupe_edges(edges) - keys = [(e["source"], e["target"], e["relation"]) for e in out] - assert keys == [("a", "b", "calls"), ("a", "b", "imports"), ("b", "c", "calls")] - # first occurrence wins (keeps L1, not L9) - assert out[0]["source_location"] == "L1" - - -def test_dedupe_edges_is_idempotent(): - edges = [ - {"source": "a", "target": "b", "relation": "calls"}, - {"source": "a", "target": "b", "relation": "calls"}, - ] - once = dedupe_edges(edges) - twice = dedupe_edges(once + edges) # simulate a second `update` re-concatenating - assert len(once) == 1 - assert len(twice) == 1 - - -def test_dedupe_nodes_collapses_by_id_last_wins(): - # #1327: a shared module anchor is emitted once per importing file; the - # --no-cluster raw writer must collapse same-id node dicts (#1317). - nodes = [ - {"id": "foundation", "label": "Foundation", "type": "module", "source_file": "A.swift"}, - {"id": "akit", "label": "AKit", "file_type": "code"}, - {"id": "foundation", "label": "Foundation", "type": "module", "source_file": "B.swift"}, - ] - out = dedupe_nodes(nodes) - ids = [n["id"] for n in out] - assert ids == ["foundation", "akit"] # first-appearance order - # last writer wins on attributes - assert next(n for n in out if n["id"] == "foundation")["source_file"] == "B.swift" - def load_extraction(): return json.loads((FIXTURES / "extraction.json").read_text()) @@ -135,6 +92,18 @@ def test_legacy_edge_from_to_canonicalized(): assert G.number_of_edges() == 1 +def test_edge_missing_source_file_backfills_from_endpoints(): + """An edge with no source_file must not crash the build; it backfills from + its endpoint nodes (regression: build_from_json referenced an undefined + `G.nodes`, raising NameError on any source_file-less edge).""" + ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"}], + "edges": [{"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED"}], + "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + assert G.number_of_edges() == 1 + assert edge_data(G, "n1", "n2")["source_file"] == "a.py" + def test_legacy_node_name_path_aliases_folded(): """#2194: nodes carrying `name`/`path` instead of `label`/`source_file` must be canonicalized before validation, not enter the graph as label-less @@ -289,23 +258,6 @@ def test_source_file_backslash_normalized(): assert sources == {"src/middleware/auth.py"} -def test_edge_missing_source_file_backfilled_from_node(): - """#1279: a semantic/LLM edge lacking source_file must inherit it from its - source node rather than reach graph.json with no file reference.""" - extraction = { - "nodes": [ - {"id": "n1", "label": "A", "file_type": "concept", "source_file": "docs/a.md"}, - {"id": "n2", "label": "B", "file_type": "concept", "source_file": "docs/b.md"}, - ], - # No source_file on the edge (as LLM output sometimes omits it). - "edges": [{"source": "n1", "target": "n2", "relation": "relates_to", "confidence": "INFERRED"}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(extraction) - sf = edge_data(G, "n1", "n2").get("source_file") - assert sf == "docs/a.md" # backfilled from the source node - - def test_build_merges_multiple_extractions(): ext1 = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}], "edges": [], "input_tokens": 0, "output_tokens": 0} @@ -388,6 +340,7 @@ def test_file_type_synonym_mapping(): assert G.nodes["n3"]["file_type"] == "concept" + def test_ghost_merge_unique_located_node_still_merges(): """#1145 ghost-merge: a semantic ghost collapses into the single AST node sharing its (basename, label), and edges re-point to the AST node.""" @@ -546,8 +499,9 @@ def test_build_merge_preserves_call_edge_direction(tmp_path): assert saved_calls[0]["source"] == truth_src assert saved_calls[0]["target"] == truth_tgt - # Now simulate `--update` with no new chunks — load + re-save. - G2 = build_merge([], graph_path, dedup=False) + # Now simulate `--update` with no new chunks — reconnect to the same + # FalkorDB graph (build_merge takes a graph_name, not a graph.json path). + G2 = build_merge([], G1.graph_name, dedup=False) assert to_json(G2, communities, str(graph_path), force=True) # The calls edge must still go a -> b, not b -> a. @@ -600,13 +554,11 @@ def test_build_from_json_preserves_first_direction_on_bidirectional_pair(tmp_pat "output_tokens": 0, } G = build_from_json(extraction) - # Only one undirected edge between the pair survives, but its stored - # direction must be the first-seen one (a_handler -> z_emitter), not the - # lexicographically-later one (z_emitter -> a_handler). + # Only one edge between the pair survives, stored in the first-seen direction + # (a_handler -> z_emitter). FalkorDB stores edges directed natively, so the + # surviving edge orientation itself encodes direction (no _src/_tgt markers). assert G.number_of_edges() == 1 - data = edge_data(G, "a_handler", "z_emitter") - assert data["_src"] == "a_handler" - assert data["_tgt"] == "z_emitter" + assert ("a_handler", "z_emitter") in list(G.edges()) graph_path = tmp_path / "graph.json" assert to_json(G, {}, str(graph_path), force=True) @@ -624,90 +576,28 @@ def test_build_from_json_preserves_first_direction_on_bidirectional_pair(tmp_pat ) -# Regression tests for #796 — edge_data / edge_datas helpers must tolerate -# MultiGraph and MultiDiGraph, which networkx's node_link_graph() produces -# whenever the loaded JSON has multigraph: true. Plain G.edges[u, v] crashes -# on those with `ValueError: not enough values to unpack (expected 3, got 2)`. +# edge_data / edge_datas helpers operate on the FalkorDB-backed GraphStore. +# (The old MultiGraph variants are gone: FalkorDB stores parallel edges natively, +# so there is no MultiGraph wrapper to tolerate.) -def test_edge_data_simple_graph(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - d = edge_data(G, "a", "b") +def test_edge_data_simple_graph(store): + store.add_nodes_from([("a", {"file_type": "code"}), ("b", {"file_type": "code"})]) + store.add_edge("a", "b", relation="calls", confidence="EXTRACTED") + d = edge_data(store, "a", "b") assert isinstance(d, dict) assert d["relation"] == "calls" assert d["confidence"] == "EXTRACTED" -def test_edge_datas_simple_graph_returns_singleton_list(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - ds = edge_datas(G, "a", "b") +def test_edge_datas_simple_graph_returns_singleton_list(store): + store.add_nodes_from([("a", {"file_type": "code"}), ("b", {"file_type": "code"})]) + store.add_edge("a", "b", relation="calls", confidence="EXTRACTED") + ds = edge_datas(store, "a", "b") assert isinstance(ds, list) assert len(ds) == 1 assert ds[0]["relation"] == "calls" -def test_edge_data_multigraph_with_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - # First parallel edge wins; should be one of the two attribute dicts above. - assert d.get("relation") in ("calls", "references") - - -def test_edge_datas_multigraph_returns_all_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - ds = edge_datas(G, "a", "b") - assert isinstance(ds, list) - assert len(ds) == 2 - relations = {e.get("relation") for e in ds} - assert relations == {"calls", "references"} - - -def test_edge_data_multidigraph(): - G = nx.MultiDiGraph() - G.add_edge("a", "b", relation="calls") - G.add_edge("a", "b", relation="imports") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "imports") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 - - -def test_edge_data_node_link_multigraph_roundtrip(): - """A node_link JSON with multigraph: true must load as MultiGraph and the - helpers must operate on it without raising the 3-tuple unpack ValueError.""" - data = { - "directed": False, - "multigraph": True, - "graph": {}, - "nodes": [ - {"id": "a", "label": "A"}, - {"id": "b", "label": "B"}, - ], - "links": [ - {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "a", "target": "b", "relation": "references", "confidence": "INFERRED"}, - ], - } - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) - assert isinstance(G, nx.MultiGraph) - # Plain G.edges[u, v] would raise here; the helper must not. - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "references") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 - - def test_build_from_json_relativizes_absolute_source_file(tmp_path): """Semantic subagents emit absolute source_file paths; build_from_json must relativize them to root so MCP traversal works correctly (#932).""" @@ -850,16 +740,13 @@ def test_build_from_json_relative_source_file_unchanged(tmp_path): assert G.nodes["src_foo_bar"]["source_file"] == "src/foo.py" -def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): +def test_build_merge_prune_absolute_paths_match_relative_nodes(store, tmp_path): """#1007: manifest stores absolute paths, graph nodes store relative paths. prune_sources with absolute paths must still remove the right nodes and edges.""" - import networkx as nx - root = tmp_path / "corpus" root.mkdir() - graph_path = tmp_path / "graph.json" - # Simulate a graph with relative source_file paths (as built normally) + # Build a graph with relative source_file paths into the store. chunk = {"nodes": [ {"id": "n1", "label": "login", "file_type": "code", "source_file": "module_a/auth.py"}, {"id": "n2", "label": "format_date", "file_type": "code", "source_file": "module_b/utils.py"}, @@ -867,12 +754,11 @@ def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", "source_file": "module_b/utils.py", "weight": 1.0}, ]} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + build([chunk], dedup=False, store=store) # prune_sources from manifest — absolute paths (what detect_incremental emits) deleted_abs = [str(root / "module_b" / "utils.py")] - G1 = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) + G1 = build_merge([], store.graph_name, prune_sources=deleted_abs, dedup=False, root=root) node_labels = {d["label"] for _, d in G1.nodes(data=True)} assert "format_date" not in node_labels, "stale node from deleted file should be pruned" @@ -881,39 +767,33 @@ def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): assert G1.number_of_edges() == 0, "edge from deleted source_file should be pruned" -def test_build_merge_prune_windows_backslash_paths(tmp_path): +def test_build_merge_prune_windows_backslash_paths(store, tmp_path): """#1007: prune_sources with Windows-style backslash absolute paths must still match.""" - import networkx as nx - root = tmp_path / "corpus" root.mkdir() - graph_path = tmp_path / "graph.json" chunk = {"nodes": [ {"id": "n1", "label": "parse_date", "file_type": "code", "source_file": "module_b/utils.py"}, ], "edges": []} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + build([chunk], dedup=False, store=store) # Simulate Windows manifest path with backslashes win_path = str(root / "module_b" / "utils.py").replace("/", "\\") - G1 = build_merge([], graph_path, prune_sources=[win_path], dedup=False, root=root) + G1 = build_merge([], store.graph_name, prune_sources=[win_path], dedup=False, root=root) node_labels = {d["label"] for _, d in G1.nodes(data=True)} assert "parse_date" not in node_labels, "node should be pruned even with backslash path" + def test_build_merge_replaces_changed_file_stale_edges(tmp_path): """Re-extracting a CHANGED file must REPLACE its prior nodes/edges, not accumulate them. build_merge previously only grew the graph, so an edge that disappeared from a file's new version survived forever (only exact-duplicate edges collapsed). The new-chunk source_file may be an absolute win32 path while the stored graph keeps relative posix — both forms must match.""" - import networkx as nx - root = tmp_path / "corpus" root.mkdir() - graph_path = tmp_path / "graph.json" # First build: changed.md contributed A, B and edge A->B; keep.md is unrelated. chunk0 = {"nodes": [ @@ -927,7 +807,6 @@ def test_build_merge_replaces_changed_file_stale_edges(tmp_path): "source_file": "keep.md", "weight": 1.0}, ]} G0 = build([chunk0], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") # changed.md edited: re-extraction now yields A, C and edge A->C (B dropped). # source_file arrives as an absolute win32-style path (as detect emits on Windows). @@ -939,7 +818,7 @@ def test_build_merge_replaces_changed_file_stale_edges(tmp_path): {"source": "A", "target": "C", "relation": "references", "confidence": "EXTRACTED", "source_file": abs_changed, "weight": 1.0}, ]} - G1 = build_merge([new_chunk], graph_path, dedup=False, root=root) + G1 = build_merge([new_chunk], G0.graph_name, dedup=False, root=root) labels = {d["label"] for _, d in G1.nodes(data=True)} edges = {(u, v) for u, v in G1.edges()} @@ -963,11 +842,7 @@ def test_build_merge_root_collapses_convention_drift(tmp_path): that file) instead of accumulating a duplicate. Without root, a drifted relative base (e.g. a bare basename from a different run) mismatches and the graph duplicates. Engine is unchanged — this pins the prompt/root contract.""" - import networkx as nx - root = tmp_path - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) # Stored graph: nested project-relative convention + a STALE node for the same # file that the re-extraction no longer emits. @@ -978,8 +853,6 @@ def test_build_merge_root_collapses_convention_drift(tmp_path): "source_file": "docs/wiki/overview.md"}, ], "edges": []} G0 = build([stored], dedup=False) - saved = json.dumps(nx.node_link_data(G0, edges="edges")) - graph_path.write_text(saved, encoding="utf-8") # BUG: --update drifted to a bare basename and no root was passed. Different # base -> source_file replace misses -> stale + duplicate both survive. @@ -987,17 +860,17 @@ def test_build_merge_root_collapses_convention_drift(tmp_path): {"id": "overview_overview", "label": "Overview", "file_type": "document", "source_file": "overview.md"}, ], "edges": []} - G_bug = build_merge([drift], graph_path, dedup=False) + G_bug = build_merge([drift], G0.graph_name, dedup=False) assert G_bug.number_of_nodes() == 3, "mismatched base must NOT replace -> stale+dup remain" # FIX: subagent emits the verbatim path; caller passes root (the build root). - graph_path.write_text(saved, encoding="utf-8") + build([stored], dedup=False, store=G0) # reset to the stored state abs_overview = str(root / "docs" / "wiki" / "overview.md") fixed = {"nodes": [ {"id": "wiki_overview_overview", "label": "Overview", "file_type": "document", "source_file": abs_overview}, ], "edges": []} - G_ok = build_merge([fixed], graph_path, prune_sources=None, dedup=False, root=root) + G_ok = build_merge([fixed], G0.graph_name, prune_sources=None, dedup=False, root=root) assert G_ok.number_of_nodes() == 1, "verbatim path + root must collapse to one node" # #1504 re-keys the author-chosen short ids to the canonical full-path stem. assert "docs_wiki_overview_stale" not in G_ok, "stale node for the re-extracted file must be dropped" @@ -1005,16 +878,11 @@ def test_build_merge_root_collapses_convention_drift(tmp_path): "new chunk must be canonicalized to the stored relative base" -def test_build_merge_rejects_oversized_existing_graph(monkeypatch, tmp_path): - """#F4: build_merge must refuse to read an existing graph.json that - exceeds the size cap, rather than json.loads-ing it into memory.""" - import pytest - - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps({"nodes": [], "links": []}), encoding="utf-8") - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - build_merge([], graph_path, dedup=False) +# The #F4 counterpart of this file's oversized-graph test now lives in +# test_query_cli.py: build_merge reads its base from FalkorDB and never parses a +# graph.json, so there is no file to cap here. The one path that still parses an +# arbitrary graph.json — serve._import_graph_json_into_store, the pre-FalkorDB +# back-compat import — enforces the cap, and that is what the query-CLI test pins. def test_build_from_json_skips_non_hashable_node_id(): diff --git a/tests/test_build_merge_hyperedges_and_prune.py b/tests/test_build_merge_hyperedges_and_prune.py index d3629a73c..5bd8b4015 100644 --- a/tests/test_build_merge_hyperedges_and_prune.py +++ b/tests/test_build_merge_hyperedges_and_prune.py @@ -18,15 +18,27 @@ import pytest -from graphify.build import build_merge, _infer_merge_root - - -def _write_graph(graph_path: Path, nodes, edges, hyperedges) -> None: - """Write a graph.json in the shape to_json emits (top-level hyperedges).""" - graph_path.write_text( - json.dumps({"nodes": nodes, "edges": edges, "hyperedges": hyperedges}), - encoding="utf-8", +from graphify.build import build_merge, build_from_json, _infer_merge_root + + +def _seed(make_store, nodes, edges, hyperedges, *, root=None, scan_root=None): + """Seed a FalkorDB graph the way a prior build would have left it. + + Replaces the old `_write_graph` graph.json fixture: build_merge now reads its + base from the store, so the "existing graph" has to live there. `root` + normalizes source_file the way a rooted build would; `scan_root` records the + root WITHOUT normalizing, which models a graph built with a root whose + semantic nodes still carry absolute source_file paths (#932/#2012). + """ + st = make_store() + build_from_json( + {"nodes": nodes, "edges": edges, "hyperedges": hyperedges}, + store=st, root=root, ) + if scan_root is not None: + st.graph["scan_root"] = str(Path(scan_root).resolve()) + st.save_meta() + return st def _he_ids(G) -> set[str]: @@ -35,10 +47,9 @@ def _he_ids(G) -> set[str]: # ── #1574: hyperedge preservation ───────────────────────────────────────────── -def _seed_two_file_graph(tmp_path): +def _seed_two_file_graph(tmp_path, make_store): root = tmp_path / "corpus" root.mkdir() - graph_path = tmp_path / "graph.json" nodes = [ {"id": "a1", "label": "a1", "file_type": "document", "source_file": "a.md"}, {"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}, @@ -48,19 +59,19 @@ def _seed_two_file_graph(tmp_path): {"id": "he_b", "label": "flow B", "source_file": "b.md", "nodes": ["b1"]}, {"id": "he_global", "label": "cross-file flow", "nodes": ["a1", "b1"]}, # no source_file ] - _write_graph(graph_path, nodes, [], hyperedges) - return root, graph_path + # scan_root recorded so the root-less calls below can still relativize (#1571). + return root, _seed(make_store, nodes, [], hyperedges, root=root, scan_root=root) -def test_update_preserves_hyperedges_of_unchanged_files(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) +def test_update_preserves_hyperedges_of_unchanged_files(tmp_path, make_store): + root, st = _seed_two_file_graph(tmp_path, make_store) # Re-extract only b.md, with a fresh hyperedge for it. new_chunk = { "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], "edges": [], "hyperedges": [{"id": "he_b_v2", "label": "flow B v2", "source_file": "b.md", "nodes": ["b1"]}], } - G = build_merge([new_chunk], graph_path, dedup=False, root=root) + G = build_merge([new_chunk], st.graph_name, dedup=False, root=root) ids = _he_ids(G) assert "he_a" in ids # unchanged file's hyperedge preserved (the bug) assert "he_global" in ids # source_file-less hyperedge preserved @@ -68,24 +79,24 @@ def test_update_preserves_hyperedges_of_unchanged_files(tmp_path): assert "he_b" not in ids # re-extracted file's OLD hyperedge replaced -def test_update_without_root_still_preserves_hyperedges(tmp_path): +def test_update_without_root_still_preserves_hyperedges(tmp_path, make_store): """The runbook omits root; the fallback root must not break preservation.""" - root, graph_path = _seed_two_file_graph(tmp_path) + root, st = _seed_two_file_graph(tmp_path, make_store) new_chunk = { "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], "edges": [], "hyperedges": [{"id": "he_b_v2", "source_file": "b.md", "nodes": ["b1"]}], } - G = build_merge([new_chunk], graph_path, dedup=False) # no root + G = build_merge([new_chunk], st.graph_name, dedup=False) # no root ids = _he_ids(G) assert {"he_a", "he_global", "he_b_v2"} <= ids assert "he_b" not in ids -def test_deleted_file_hyperedges_are_pruned(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) +def test_deleted_file_hyperedges_are_pruned(tmp_path, make_store): + root, st = _seed_two_file_graph(tmp_path, make_store) deleted_abs = [str(root / "a.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) + G = build_merge([], st.graph_name, prune_sources=deleted_abs, dedup=False, root=root) ids = _he_ids(G) assert "he_a" not in ids # deleted file's hyperedge pruned assert "he_b" in ids # untouched file's hyperedge kept @@ -96,41 +107,45 @@ def test_deleted_file_hyperedges_are_pruned(tmp_path): # ── #1571: root-less prune (absolute deleted paths vs relative node keys) ────── -def test_prune_without_root_removes_ghost_nodes_via_grandparent_fallback(tmp_path): +def test_prune_without_root_removes_ghost_nodes_via_recorded_scan_root(tmp_path, make_store): + """#1571, store edition: the graph records its scan root at build time (the + FalkorDB replacement for the `.graphify_root` marker), so a runbook-style + root-less call can still relativize an absolute prune path.""" root = tmp_path / "corpus" (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" nodes = [ {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, ] - _write_graph(graph_path, nodes, [], []) + st = _seed(make_store, nodes, [], [], root=root) # Runbook-style call: absolute prune path, NO root passed. deleted_abs = [str(root / "HANDOFF.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False) + G = build_merge([], st.graph_name, prune_sources=deleted_abs, dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "handoff" not in labels, "deleted file's ghost node must be pruned without root" assert "keep" in labels -def test_prune_without_root_uses_graphify_root_marker(tmp_path): - # graph.json not under a /graphify-out layout, so grandparent wouldn't - # help — the committed .graphify_root marker must be honored instead. +def test_prune_without_root_uses_recorded_root_outside_graphify_out_layout(tmp_path, make_store): + """A scan root that is NOT the output dir's parent must still be honored. + _infer_merge_root still reads the committed `.graphify_root` marker for the + file-based callers; the store carries the same value as `scan_root`.""" out = tmp_path / "out" out.mkdir() graph_path = out / "graph.json" real_root = tmp_path / "elsewhere" / "repo" real_root.mkdir(parents=True) (out / ".graphify_root").write_text(str(real_root), encoding="utf-8") - nodes = [{"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}] - _write_graph(graph_path, nodes, [], []) + graph_path.write_text("{}", encoding="utf-8") assert _infer_merge_root(graph_path) == str(real_root.resolve()) - G = build_merge([], graph_path, prune_sources=[str(real_root / "HANDOFF.md")], dedup=False) + nodes = [{"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}] + st = _seed(make_store, nodes, [], [], root=real_root) + G = build_merge([], st.graph_name, prune_sources=[str(real_root / "HANDOFF.md")], dedup=False) assert "handoff" not in {d["label"] for _, d in G.nodes(data=True)} @pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics") -def test_prune_matches_across_symlinked_root(tmp_path): +def test_prune_matches_across_symlinked_root(tmp_path, make_store): """A symlinked scan root (macOS /var -> /private/var, symlinked home/worktree) makes the absolute prune path and the resolved root differ by prefix. The prune must still match — lexical relative_to fails, so normalization resolves both @@ -139,27 +154,24 @@ def test_prune_matches_across_symlinked_root(tmp_path): (real / "graphify-out").mkdir(parents=True) link = tmp_path / "link" os.symlink(real, link) - graph_path = real / "graphify-out" / "graph.json" - _write_graph(graph_path, [ + st = _seed(make_store, [ {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, - ], [], []) + ], [], [], root=real) # prune path addressed via the SYMLINK, root resolved to the real dir - G = build_merge([], graph_path=graph_path, + G = build_merge([], st.graph_name, prune_sources=[str(link / "HANDOFF.md")], root=str(real), dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "handoff" not in labels and "keep" in labels -def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): +def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path, make_store): """#1796: a file present in BOTH new_chunks (re-extracted) and prune_sources must be REPLACED, not deleted. The old edit-workflow passed the changed file in prune_sources; combined with dedup keeping a same-label node, that used to silently delete the freshly re-extracted concept. Replace wins over delete.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, + st = _seed( + make_store, nodes=[ {"id": "foo_widget_cache", "label": "Widget Cache Design", "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, @@ -168,6 +180,7 @@ def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): ], edges=[], hyperedges=[], + root=tmp_path, ) # foo.md edited: same-label node re-extracted (new content/line) new_chunk = {"nodes": [ @@ -175,19 +188,17 @@ def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} ], "edges": []} - G = build_merge([new_chunk], graph_path=str(graph_path), + G = build_merge([new_chunk], st.graph_name, prune_sources=["docs/foo.md"], root=str(tmp_path)) labels = {G.nodes[n].get("label") for n in G.nodes()} assert "Widget Cache Design" in labels, "re-extracted node was wrongly pruned" -def test_genuine_deletion_still_prunes(tmp_path): +def test_genuine_deletion_still_prunes(tmp_path, make_store): """#1796 guard must not break real deletions: a file in prune_sources but NOT in new_chunks is still removed.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, + st = _seed( + make_store, nodes=[ {"id": "foo_widget_cache", "label": "Widget Cache Design", "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, @@ -196,13 +207,14 @@ def test_genuine_deletion_still_prunes(tmp_path): ], edges=[], hyperedges=[], + root=tmp_path, ) new_chunk = {"nodes": [ {"id": "foo_widget_cache", "label": "Widget Cache Design", "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} ], "edges": []} # bar.md genuinely deleted (not re-extracted) - G = build_merge([new_chunk], graph_path=str(graph_path), + G = build_merge([new_chunk], st.graph_name, prune_sources=["docs/bar.md"], root=str(tmp_path)) labels = {G.nodes[n].get("label") for n in G.nodes()} assert "Other" not in labels, "genuinely deleted file's node should be pruned" @@ -211,7 +223,7 @@ def test_genuine_deletion_still_prunes(tmp_path): # ── #2012: form-insensitive prune (absolute node vs relative prune, and back) ── -def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): +def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path, make_store): """#2012: a node whose source_file survived in ABSOLUTE form must still be pruned when the deletion is expressed relative to root. The runbook calls build_merge WITHOUT root, so build() does not re-normalize the node's stored @@ -221,7 +233,6 @@ def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): now normalizes the node side too + an absolute-identity fallback.""" root = tmp_path / "corpus" (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" nodes = [ # gone.py's node kept an ABSOLUTE source_file (a semantic subagent wrote # it that way, #932); keep.py's is relative. @@ -233,34 +244,35 @@ def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): {"source": "g1", "target": "k1", "type": "calls", "source_file": str(root / "gone.py")}, ] - _write_graph(graph_path, nodes, edges, []) + # scan_root (not root=) so the absolute source_file is NOT re-normalized — + # that mismatched-form survival is exactly what #2012 is about. + st = _seed(make_store, nodes, edges, [], scan_root=root) # Runbook-style: NO root passed (eff_root inferred from the graphify-out # grandparent), so build() leaves the absolute node form intact. Deletion is # expressed RELATIVE — a third form vs the stored absolute node. - G = build_merge([], graph_path, prune_sources=["gone.py"], dedup=False) + G = build_merge([], st.graph_name, prune_sources=["gone.py"], dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "gone" not in labels, "absolute-stored node not pruned by relative delete (#2012)" assert "keep" in labels assert G.number_of_edges() == 0, "edge from the deleted file must be pruned too (#2012)" -def test_prune_reextracted_absolute_node_not_deleted(tmp_path): +def test_prune_reextracted_absolute_node_not_deleted(tmp_path, make_store): """#1796 protection must hold in absolute-identity space too: a file present in BOTH new_chunks and prune_sources (in mismatched forms) is REPLACED, not deleted — the #2012 form-insensitive match must not resurrect the delete for a re-extracted file.""" root = tmp_path / "corpus" (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" - _write_graph(graph_path, [ + st = _seed(make_store, [ {"id": "g1", "label": "gone", "file_type": "code", "source_file": str(root / "mod.py")}, - ], [], []) + ], [], [], scan_root=root) # Re-extracted with a RELATIVE source_file; prune lists it RELATIVE too. # No root passed (runbook), so the stored absolute node is not re-normalized. new_chunk = {"nodes": [ {"id": "g1", "label": "gone", "file_type": "code", "source_file": "mod.py"}, ], "edges": []} - G = build_merge([new_chunk], graph_path, prune_sources=["mod.py"], dedup=False) + G = build_merge([new_chunk], st.graph_name, prune_sources=["mod.py"], dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "gone" in labels, "re-extracted file wrongly pruned across mismatched forms (#2012/#1796)" diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 6173a61aa..969faf463 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -16,13 +16,21 @@ FIXTURES = Path(__file__).parent / "fixtures" +_REPO_ROOT = str(Path(__file__).resolve().parent.parent) + + def _run(args: list[str], cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess: + # Run the CLI as a subprocess from `cwd`. Ensure the repo root is importable + # so `python -m graphify` works even in a non-pip-installed source checkout. + run_env = {**os.environ, **(env or {})} + existing = run_env.get("PYTHONPATH", "") + run_env["PYTHONPATH"] = _REPO_ROOT + (os.pathsep + existing if existing else "") return subprocess.run( [PYTHON, "-m", "graphify"] + args, cwd=cwd, capture_output=True, text=True, - env=env, + env=run_env, ) @@ -36,8 +44,13 @@ def _make_graph(tmp_path: Path) -> Path: from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.export import to_json + from graphify.store import open_store - G = build_from_json(extraction) + # Build into the FalkorDB graph bound to this output dir (writes the pointer + # file) so the CLI subprocess loads the same graph. + G = open_store(out, create=True) + G.clear() + build_from_json(extraction, store=G) communities = cluster(G) cohesion = score_all(G, communities) gods = god_nodes(G) @@ -139,6 +152,23 @@ def test_export_graphml_creates_file(tmp_path): assert gml.stat().st_size > 0 content = gml.read_text() assert " elements" + + +def test_export_svg_creates_file(tmp_path): + pytest.importorskip("matplotlib") + _make_graph(tmp_path) + r = _run(["export", "svg"], tmp_path) + assert r.returncode == 0, r.stderr + svg = tmp_path / "graphify-out" / "graph.svg" + assert svg.exists() + assert svg.stat().st_size > 0 + assert "= 4, "fixture must have enough nodes to form 2+ communities" + # Tag every node with an out-of-band community id (on the FalkorDB store, + # which is what cluster-only reads as the prior assignment) and write a + # labels file keyed on those ids. After cluster-only, at least one of those + # sentinel ids must survive (= remap succeeded by node overlap). Without the + # remap, Leiden returns small ints (0, 1, ...) and the sentinel keys vanish. + store = open_store(out, create=False) + node_ids = sorted(n for n in store.nodes) + assert len(node_ids) >= 4, "fixture must have enough nodes to form 2+ communities" sentinel_a, sentinel_b = 4242, 9999 - half = len(nodes) // 2 - for i, n in enumerate(nodes): - n["community"] = sentinel_a if i < half else sentinel_b - graph_json.write_text(json.dumps(g), encoding="utf-8") + half = len(node_ids) // 2 + store.set_communities({sentinel_a: node_ids[:half], sentinel_b: node_ids[half:]}) labels_json.write_text( json.dumps({str(sentinel_a): "First Group", str(sentinel_b): "Second Group"}), encoding="utf-8", diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 21fd2ca3a..2ae30ae00 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,6 +1,6 @@ import json import sys -import networkx as nx +from tests import nxcompat as nx from pathlib import Path from graphify.build import build_from_json from graphify.cluster import cluster, cohesion_score, remap_communities_to_previous, score_all diff --git a/tests/test_confidence.py b/tests/test_confidence.py index 299548aca..8ffccb64d 100644 --- a/tests/test_confidence.py +++ b/tests/test_confidence.py @@ -3,7 +3,7 @@ import tempfile from pathlib import Path -import networkx as nx +from tests import nxcompat as nx from graphify.build import build_from_json from graphify.cluster import cluster, score_all diff --git a/tests/test_connect_fallback.py b/tests/test_connect_fallback.py new file mode 100644 index 000000000..f8660e751 --- /dev/null +++ b/tests/test_connect_fallback.py @@ -0,0 +1,60 @@ +"""Unit tests for the embedded-engine auto-fallback in store._connect. + +These mock the FalkorDB client so they run without a live server (and without +the embedded engine, which needs Python >= 3.12) — they verify the dispatch +logic only.""" +from __future__ import annotations + +import falkordb +import pytest + +import graphify.store as st + + +class _FakeConn: + def __init__(self, ok: bool): + self._ok = ok + + def ping(self): + if not self._ok: + raise ConnectionError("Connection refused") + return True + + +class _FakeDB: + def __init__(self, ok: bool = True, **kwargs): + self.connection = _FakeConn(ok) + + +def test_uses_server_when_reachable(monkeypatch): + fake = _FakeDB(ok=True) + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: fake) + monkeypatch.setattr(st, "_connect_lite", lambda dbfile: pytest.fail("must not fall back")) + assert st._connect("falkordb://localhost:6379") is fake + + +def test_local_server_down_falls_back_to_lite(monkeypatch): + sentinel = object() + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: _FakeDB(ok=False)) + monkeypatch.setattr(st, "_connect_lite", lambda dbfile: sentinel) + assert st._connect("falkordb://localhost:6379") is sentinel + + +def test_remote_server_down_raises(monkeypatch): + """A connection failure to an explicitly-configured remote host must surface, + not silently switch to a local empty embedded DB.""" + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: _FakeDB(ok=False)) + monkeypatch.setattr(st, "_connect_lite", lambda dbfile: pytest.fail("no fallback for remote")) + with pytest.raises(ConnectionError): + st._connect("falkordb://db.internal.example.com:6379") + + +def test_local_down_without_lite_gives_helpful_error(monkeypatch): + def _no_lite(dbfile): + raise ImportError("redislite not installed") + + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: _FakeDB(ok=False)) + monkeypatch.setattr(st, "_connect_lite", _no_lite) + with pytest.raises(ConnectionError) as exc: + st._connect("falkordb://localhost:6379") + assert "falkordblite" in str(exc.value) diff --git a/tests/test_corrupt_graph_json.py b/tests/test_corrupt_graph_json.py index 7f2258c38..00eb4e657 100644 --- a/tests/test_corrupt_graph_json.py +++ b/tests/test_corrupt_graph_json.py @@ -1,17 +1,23 @@ """Corrupt graph.json produces an actionable error, not a raw traceback (#1536/#1537). -Three load paths call json.loads on graph.json — build_merge (`--update`), -affected.load_graph (`graphify prs`), and diagnostics._read_json_file -(`graphify diagnose`). A truncated / invalid file (incomplete write, power loss, -manual edit) must raise a clear RuntimeError with recovery guidance at each. +With the FalkorDB backend the graph lives in the engine, so most former +json.loads callers no longer touch graph.json at all: `build_merge` reads its +base from the store, and `affected.connect_graph` opens a store connection. Two +paths still parse the file and so still need the guarantee: + + * ``diagnostics._read_json_file`` (``graphify diagnose``) + * ``serve._import_graph_json_into_store`` — the one-time back-compat import of + a pre-FalkorDB / ``--no-cluster`` graph.json. + +A truncated / invalid file (incomplete write, power loss, manual edit) must +produce a clear, actionable error at each — never a raw traceback. """ from __future__ import annotations import pytest -from graphify.build import build_merge -from graphify.affected import load_graph from graphify.diagnostics import _read_json_file +from graphify.serve import _connect_graph, _import_graph_json_into_store _CORRUPT = '{"nodes": [{"id": "a", "labe' # truncated mid-object @@ -22,32 +28,39 @@ def _corrupt(tmp_path): return p -def test_build_merge_corrupt_graph_raises_runtimeerror(tmp_path): +def test_diagnostics_read_corrupt_raises_runtimeerror(tmp_path): p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot read .*incremental merge|rebuild"): - build_merge([], graph_path=p, dedup=False) - + with pytest.raises(RuntimeError, match=r"Cannot parse|corrupted"): + _read_json_file(p) -def test_affected_load_graph_corrupt_raises_runtimeerror(tmp_path): - p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot read graph file|regenerate"): - load_graph(p) +def test_backcompat_import_of_corrupt_json_reports_no_graph(monkeypatch, tmp_path, capsys, _require_falkordb): + """A corrupt graph.json must not surface as a JSONDecodeError traceback. -def test_diagnostics_read_corrupt_raises_runtimeerror(tmp_path): + The import declines the file, so the caller falls through to the ordinary + "no graph built" guidance — actionable, not a stack trace (#1536/#1537). + """ p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot parse|corrupted"): - _read_json_file(p) + with pytest.raises(SystemExit): + _connect_graph(str(p)) + err = capsys.readouterr().err + assert "No graph found" in err + assert "Re-run /graphify" in err -def test_valid_graph_still_loads(tmp_path): +def test_valid_graph_still_imports(tmp_path, _require_falkordb): """Happy path unchanged: a well-formed graph.json loads without raising.""" p = tmp_path / "graph.json" p.write_text( '{"nodes": [{"id": "a", "label": "a", "file_type": "code"}], "edges": []}', encoding="utf-8", ) - # none of these should raise - load_graph(p) _read_json_file(p) - build_merge([], graph_path=p, dedup=False) + G = _connect_graph(str(p)) + assert G.number_of_nodes() == 1 + + +def test_import_helper_declines_corrupt_file(tmp_path, store): + """The import helper itself returns False rather than raising, so a corrupt + file can never abort a connect with a decode traceback.""" + assert _import_graph_json_into_store(_corrupt(tmp_path), store) is False diff --git a/tests/test_explain_cli.py b/tests/test_explain_cli.py index 94ba22b9c..5e431970f 100644 --- a/tests/test_explain_cli.py +++ b/tests/test_explain_cli.py @@ -4,30 +4,34 @@ import graphify.__main__ as mainmod +_NODES = [ + {"id": "validate", "label": "validateSanitySession()", "source_file": "server/sanity-validate-session.ts", "community": 0, "file_type": "code"}, + {"id": "create_patch", "label": "createPatchHandler()", "source_file": "server/create-patch-handler.ts", "community": 0, "file_type": "code"}, + {"id": "create_edit", "label": "createEditHandler()", "source_file": "server/create-edit-handler.ts", "community": 0, "file_type": "code"}, + {"id": "stable_stringify", "label": "stableStringify()", "source_file": "shared/stringify.ts", "community": 0, "file_type": "code"}, +] +_LINKS = [ + {"source": "create_patch", "target": "validate", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "create_edit", "target": "validate", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "validate", "target": "stable_stringify", "relation": "calls", "confidence": "EXTRACTED"}, +] + + def _write_graph(tmp_path): - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "validate", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", "community": 0}, - {"id": "create_patch", "label": "createPatchHandler()", - "source_file": "server/create-patch-handler.ts", "community": 0}, - {"id": "create_edit", "label": "createEditHandler()", - "source_file": "server/create-edit-handler.ts", "community": 0}, - {"id": "stable_stringify", "label": "stableStringify()", - "source_file": "shared/stringify.ts", "community": 0}, - ], - "links": [ - {"source": "create_patch", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "create_edit", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "validate", "target": "stable_stringify", - "relation": "calls", "confidence": "EXTRACTED"}, - ], - } + """Seed the FalkorDB graph for tmp_path so `explain` (which loads from the + store) finds it.""" + from graphify.store import open_store + + store = open_store(tmp_path, create=True) + store.clear() + store.add_nodes_from([(n["id"], {k: v for k, v in n.items() if k != "id"}) for n in _NODES]) + store.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in _LINKS + ]) p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) + p.write_text(json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": _NODES, "links": _LINKS})) return p @@ -77,7 +81,11 @@ def test_explain_source_file_path_prefers_file_level_node(monkeypatch, tmp_path, out = _run(monkeypatch, p, source_file, capsys) assert "Node: route.ts" in out - assert "ID: example_route" in out + # Importing a pre-FalkorDB graph.json runs the #1504 semantic re-key, which + # re-derives each node id from its full repo-relative source_file — so the + # authored short id canonicalizes to the path-qualified form. The point of + # this test is which NODE wins (the file node, not the symbol), not its id. + assert "ID: app_api_example_route" in out assert f"Source: {source_file} L1" in out assert "Node: GET()" not in out @@ -223,7 +231,12 @@ def test_explain_grouping_boundary_at_exactly_21_vs_20_connections(monkeypatch, connections (one past the cutoff) must show the grouped section with exactly one grouped entry; one node at exactly 20 (at the cutoff, not past it) must show neither.""" - p21 = _write_high_degree_graph(tmp_path, n_callers=21, files=["lib/only.py"]) + # Each graph gets its OWN directory: the FalkorDB store is keyed by output + # dir and imported once, so rewriting graph.json in a shared tmp_path would + # leave the first graph in place and silently compare the wrong one. + (tmp_path / "at21").mkdir() + (tmp_path / "at20").mkdir() + p21 = _write_high_degree_graph(tmp_path / "at21", n_callers=21, files=["lib/only.py"]) out21 = _run(monkeypatch, p21, "hub", capsys) assert "Grouped by file:" in out21 assert "<-- lib/only.py: 1 connection" in out21 @@ -232,7 +245,7 @@ def test_explain_grouping_boundary_at_exactly_21_vs_20_connections(monkeypatch, ] assert len(grouped_lines21) == 1 # exactly one grouped entry, not zero, not more - p20 = _write_high_degree_graph(tmp_path, n_callers=20, files=["lib/only.py"]) + p20 = _write_high_degree_graph(tmp_path / "at20", n_callers=20, files=["lib/only.py"]) out20 = _run(monkeypatch, p20, "hub", capsys) assert "Grouped by file:" not in out20 assert "more" not in out20 diff --git a/tests/test_export.py b/tests/test_export.py index 28a4707a7..d5ac61539 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -104,14 +104,27 @@ def test_to_graphml_tolerates_dict_and_list_attribute_values(): metadata, or the graph-level hyperedges list) used to crash the whole export. to_graphml must JSON-serialize them across graph/node/edge scopes (#1831).""" import networkx as nx - G = make_graph() - communities = cluster(G) - a_node = next(iter(G.nodes())) - G.nodes[a_node]["metadata"] = {"kind": "file", "size": 12} - G.nodes[a_node]["tags"] = ["x", "y"] - if G.number_of_edges(): - u, v = next(iter(G.edges())) - G.edges[u, v]["ctx"] = {"k": "v"} + + from graphify.store import MemGraph + + # Built as a MemGraph rather than a FalkorDB-backed store: the engine only + # stores scalar properties, so a dict/list attribute can only ever reach + # to_graphml through the in-memory graph (the aggregated-view and library + # paths). Attributes are set at construction because a store's node view + # hands back a snapshot — assigning into it would not persist. + a_node, b_node = "n_meta", "n_other" + G = MemGraph( + [ + (a_node, {"label": "A", "source_file": "a.py", + "metadata": {"kind": "file", "size": 12}, "tags": ["x", "y"]}), + (b_node, {"label": "B", "source_file": "b.py"}), + ], + [(a_node, b_node, {"relation": "calls", "confidence": "EXTRACTED", + "ctx": {"k": "v"}})], + ) + # A literal mapping rather than cluster(): community detection runs in the + # engine, and this test is about GraphML serialization, not clustering. + communities = {0: [a_node, b_node]} G.graph["hyperedges"] = [{"nodes": [a_node], "label": "h"}] with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) / "graph.graphml" diff --git a/tests/test_extract.py b/tests/test_extract.py index e0a880b89..0314f02ee 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2309,19 +2309,22 @@ def test_extract_json_import_and_extends_targets_are_real_nodes(tmp_path): extracted = extract([package_json, tsconfig], cache_root=tmp_path, parallel=False) graph = build_from_json(extracted, directed=True) + # Edges are stored in their native source->target orientation, so the (u, v) + # yielded by edges() IS the direction — the old _src/_tgt markers that the + # undirected NetworkX storage needed no longer exist. import_targets = { - graph.nodes[data["_tgt"]]["label"] - for _, _, data in graph.edges(data=True) + graph.nodes[v]["label"] + for _, v, data in graph.edges(data=True) if data.get("relation") == "imports" } extends_targets = { - graph.nodes[data["_tgt"]]["label"] - for _, _, data in graph.edges(data=True) + graph.nodes[v]["label"] + for _, v, data in graph.edges(data=True) if data.get("relation") == "extends" } self_loops = [ - data for _, _, data in graph.edges(data=True) - if data.get("relation") in {"imports", "extends"} and data["_src"] == data["_tgt"] + data for u, v, data in graph.edges(data=True) + if data.get("relation") in {"imports", "extends"} and u == v ] assert self_loops == [] assert {"left-pad", "bats"} <= import_targets diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index b1c369a5e..48e3ca92f 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -664,6 +664,22 @@ def _sem_doc_count(g): {"id": "h1", "label": "Shared", "nodes": ["doc_readme_a", "doc_readme_b"], "relation": "participate_in", "source_file": "README.md"}) graph_path.write_text(json.dumps(graph)) + # The FalkorDB store — not graph.json — is the incremental baseline, so the + # committed semantic layer has to be injected there too. graph.json alone is + # a derived artifact and the rebuild would never see these nodes. + from graphify.store import open_store as _open_store + + _st = _open_store(graphify_out, create=False) + _st.add_nodes_from([ + (n["id"], {k: v for k, v in n.items() if k != "id"}) + for n in graph["nodes"] if n["id"].startswith("doc_readme_") + ]) + _st.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in graph["edges"] if str(e.get("source", "")).startswith("doc_readme_") + ]) + _st.graph["hyperedges"] = graph["hyperedges"] + _st.save_meta() (graphify_out / ".graphify_semantic_marker").write_text( json.dumps({"output_tokens": 1})) diff --git a/tests/test_global_graph.py b/tests/test_global_graph.py index b91a1ebe7..1014b4680 100644 --- a/tests/test_global_graph.py +++ b/tests/test_global_graph.py @@ -1,209 +1,153 @@ """Tests for the global graph infrastructure (graphify/global_graph.py), prefix/prune helpers in graphify/build.py, and the cross-repo guard in -graphify/dedup.py.""" +graphify/dedup.py. FalkorDB-backed.""" from __future__ import annotations import json import pytest -import networkx as nx from unittest.mock import patch -# ── helpers ────────────────────────────────────────────────────────────────── - -def _make_graph(nodes, edges=None): - """Build a simple nx.Graph from node dicts.""" - G = nx.Graph() - for n in nodes: - nid = n["id"] - G.add_node(nid, **{k: v for k, v in n.items() if k != "id"}) - for e in (edges or []): - G.add_edge( - e["source"], - e["target"], - **{k: v for k, v in e.items() if k not in ("source", "target")}, - ) - return G - - -def _graph_to_json(G, path): - from networkx.readwrite import json_graph as jg - try: - data = jg.node_link_data(G, edges="links") - except TypeError: - data = jg.node_link_data(G) - path.write_text(json.dumps(data), encoding="utf-8") - - # ── build.py helpers ────────────────────────────────────────────────────────── -def test_prefix_graph_preserves_label(): +def test_prefix_graph_preserves_label(store, make_store): from graphify.build import prefix_graph_for_global - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - H = prefix_graph_for_global(G, "repoA") + store.add_node("userservice", label="UserService", source_file="src/user.py", file_type="code") + target = make_store() + H = prefix_graph_for_global(store, "repoA", target) assert "repoA::userservice" in H.nodes assert "userservice" not in H.nodes assert H.nodes["repoA::userservice"]["label"] == "UserService" -def test_prefix_graph_sets_repo_and_local_id(): +def test_prefix_graph_sets_repo_and_local_id(store, make_store): from graphify.build import prefix_graph_for_global - G = _make_graph([{"id": "userservice", "label": "UserService"}]) - H = prefix_graph_for_global(G, "repoA") + store.add_node("userservice", label="UserService", file_type="code") + target = make_store() + H = prefix_graph_for_global(store, "repoA", target) data = H.nodes["repoA::userservice"] assert data["repo"] == "repoA" assert data["local_id"] == "userservice" -def test_prefix_graph_rewrites_edges(): +def test_prefix_graph_rewrites_edges(store, make_store): from graphify.build import prefix_graph_for_global - G = _make_graph( - [{"id": "a", "label": "A"}, {"id": "b", "label": "B"}], - [{"source": "a", "target": "b"}], - ) - H = prefix_graph_for_global(G, "repo1") + store.add_nodes_from([("a", {"label": "A", "file_type": "code"}), ("b", {"label": "B", "file_type": "code"})]) + store.add_edge("a", "b", relation="calls") + target = make_store() + H = prefix_graph_for_global(store, "repo1", target) assert H.has_edge("repo1::a", "repo1::b") assert not H.has_edge("a", "b") -def test_prune_repo_removes_correct_nodes(): +def test_prune_repo_removes_correct_nodes(store): from graphify.build import prune_repo_from_graph - G = nx.Graph() - G.add_node("repoA::userservice", repo="repoA", label="UserService") - G.add_node("repoB::userservice", repo="repoB", label="UserService") - G.add_node("repoA::auth", repo="repoA", label="Auth") - removed = prune_repo_from_graph(G, "repoA") + store.add_nodes_from([ + ("repoA::userservice", {"repo": "repoA", "label": "UserService", "file_type": "code"}), + ("repoB::userservice", {"repo": "repoB", "label": "UserService", "file_type": "code"}), + ("repoA::auth", {"repo": "repoA", "label": "Auth", "file_type": "code"}), + ]) + removed = prune_repo_from_graph(store, "repoA") assert removed == 2 - assert "repoB::userservice" in G.nodes - assert "repoA::userservice" not in G.nodes - assert "repoA::auth" not in G.nodes + assert "repoB::userservice" in store.nodes + assert "repoA::userservice" not in store.nodes + assert "repoA::auth" not in store.nodes -def test_prune_repo_returns_zero_if_not_present(): +def test_prune_repo_returns_zero_if_not_present(store): from graphify.build import prune_repo_from_graph - G = nx.Graph() - G.add_node("repoA::x", repo="repoA") - removed = prune_repo_from_graph(G, "repoB") + store.add_node("repoA::x", repo="repoA", file_type="code") + removed = prune_repo_from_graph(store, "repoB") assert removed == 0 - assert G.number_of_nodes() == 1 + assert store.number_of_nodes() == 1 # ── global_graph.py ─────────────────────────────────────────────────────────── -def test_global_add_creates_global_graph(tmp_path): - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G, src_graph) - +@pytest.fixture() +def global_env(tmp_path, falkordb_uri): + """Isolate the global graph to a unique FalkorDB name + temp manifest dir.""" + import os + from graphify.store import GraphStore global_dir = tmp_path / ".graphify" + name = f"pytest_global_{os.getpid()}_{id(tmp_path)}" with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - result = global_add(src_graph, "repoA") + patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"), \ + patch("graphify.global_graph._GLOBAL_NAME", name): + gs = GraphStore(graph_name=name, uri=falkordb_uri) + gs.clear() + try: + yield global_dir + finally: + try: + gs.clear() + except Exception: + pass + + +def test_global_add_creates_global_graph(tmp_path, seed_graph, global_env): + seed_graph(tmp_path, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add + result = global_add(tmp_path / "graph.json", "repoA") assert result["skipped"] is False assert result["nodes_added"] > 0 - manifest_path = global_dir / "global-manifest.json" - assert manifest_path.exists() - manifest = json.loads(manifest_path.read_text()) + manifest = json.loads((global_env / "global-manifest.json").read_text()) assert "repoA" in manifest["repos"] -def test_global_add_skip_on_unchanged_hash(tmp_path): - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G, src_graph) - - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - global_add(src_graph, "repoA") - result2 = global_add(src_graph, "repoA") - +def test_global_add_skip_on_unchanged_hash(tmp_path, seed_graph, global_env): + seed_graph(tmp_path, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add + global_add(tmp_path / "graph.json", "repoA") + result2 = global_add(tmp_path / "graph.json", "repoA") assert result2["skipped"] is True -def test_global_add_two_repos_no_collision(tmp_path): - g1 = tmp_path / "graph1.json" - g2 = tmp_path / "graph2.json" - G1 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - G2 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G1, g1) - _graph_to_json(G2, g2) - - global_dir = tmp_path / ".graphify" - global_graph_path = global_dir / "global-graph.json" - global_manifest_path = global_dir / "global-manifest.json" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_graph_path), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_manifest_path): - from graphify.global_graph import global_add, _load_global_graph - global_add(g1, "repoA") - global_add(g2, "repoB") - G = _load_global_graph() - +def test_global_add_two_repos_no_collision(tmp_path, seed_graph, global_env): + d1 = tmp_path / "r1" + d2 = tmp_path / "r2" + d1.mkdir(); d2.mkdir() + seed_graph(d1, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + seed_graph(d2, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add, _load_global_graph + global_add(d1 / "graph.json", "repoA") + global_add(d2 / "graph.json", "repoB") + G = _load_global_graph() assert "repoA::userservice" in G.nodes assert "repoB::userservice" in G.nodes assert G.number_of_nodes() == 2 # no silent merge -def test_global_remove(tmp_path): - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G, src_graph) - - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add, global_remove - global_add(src_graph, "repoA") - removed = global_remove("repoA") - +def test_global_remove(tmp_path, seed_graph, global_env): + seed_graph(tmp_path, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add, global_remove, global_list + global_add(tmp_path / "graph.json", "repoA") + removed = global_remove("repoA") assert removed > 0 - # manifest should no longer list repoA - need to re-patch for list call - global_dir2 = global_dir # same dir - with patch("graphify.global_graph._GLOBAL_DIR", global_dir2), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir2 / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir2 / "global-manifest.json"): - from graphify.global_graph import global_list - repos = global_list() - assert "repoA" not in repos + assert "repoA" not in global_list() -def test_global_remove_unknown_tag_raises(tmp_path): - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_remove - with pytest.raises(KeyError): - global_remove("nonexistent") - +def test_global_remove_unknown_tag_raises(tmp_path, global_env): + from graphify.global_graph import global_remove + with pytest.raises(KeyError): + global_remove("nonexistent") -def test_global_add_collision_warning(tmp_path, capsys): - g1 = tmp_path / "graph1.json" - g2 = tmp_path / "graph2.json" - G = _make_graph([{"id": "x", "label": "X", "source_file": "x.py"}]) - _graph_to_json(G, g1) - _graph_to_json(G, g2) - - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - global_add(g1, "myrepo") - global_add(g2, "myrepo") # different source path, same tag +def test_global_add_collision_warning(tmp_path, seed_graph, global_env, capsys): + d1 = tmp_path / "r1" + d2 = tmp_path / "r2" + d1.mkdir(); d2.mkdir() + seed_graph(d1, [{"id": "x", "label": "X", "source_file": "x.py", "file_type": "code"}]) + seed_graph(d2, [{"id": "x", "label": "X", "source_file": "x.py", "file_type": "code"}]) + from graphify.global_graph import global_add + global_add(d1 / "graph.json", "myrepo") + global_add(d2 / "graph.json", "myrepo") # different source path, same tag captured = capsys.readouterr() assert "warning" in captured.err.lower() or "warning" in captured.out.lower() -# ── dedup guard ─────────────────────────────────────────────────────────────── +# ── dedup guard (needs datasketch) ───────────────────────────────────────────── def test_dedup_raises_on_cross_repo_nodes(): from graphify.dedup import deduplicate_entities @@ -221,8 +165,8 @@ def test_dedup_ok_with_single_repo(): {"id": "repoA::userservice", "label": "UserService", "repo": "repoA"}, {"id": "repoA::auth", "label": "Auth", "repo": "repoA"}, ] - result_nodes, result_edges = deduplicate_entities(nodes, [], communities={}) - assert len(result_nodes) == 2 # no false merge + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 2 def test_dedup_ok_with_no_repo_attr(): @@ -231,109 +175,20 @@ def test_dedup_ok_with_no_repo_attr(): {"id": "userservice", "label": "UserService"}, {"id": "auth", "label": "Auth"}, ] - result_nodes, result_edges = deduplicate_entities(nodes, [], communities={}) + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) assert len(result_nodes) == 2 -# ── merge-graphs prefix ─────────────────────────────────────────────────────── +# ── merge-graphs prefix (plain-dict node-link, no NetworkX) ──────────────────── -def test_merge_graphs_prefixes_ids(tmp_path): - """merge-graphs should prefix node IDs with repo name to avoid silent collision.""" - from graphify.build import prefix_graph_for_global - from networkx.readwrite import json_graph as jg - - # Two graphs with same node ID - G1 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - G2 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - - repo1 = tmp_path / "repo1" / "graphify-out" - repo2 = tmp_path / "repo2" / "graphify-out" - repo1.mkdir(parents=True) - repo2.mkdir(parents=True) - - g1_path = repo1 / "graph.json" - g2_path = repo2 / "graph.json" - _graph_to_json(G1, g1_path) - _graph_to_json(G2, g2_path) - - # Simulate what merge-graphs now does (prefix before compose) - graphs = [] - graph_paths = [g1_path, g2_path] - for gp in graph_paths: - data = json.loads(gp.read_text()) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = jg.node_link_graph(data, edges="links") - except TypeError: - G = jg.node_link_graph(data) - repo_tag = gp.parent.parent.name - graphs.append(prefix_graph_for_global(G, repo_tag)) - - merged = nx.Graph() - for G in graphs: - merged = nx.compose(merged, G) - - assert "repo1::userservice" in merged.nodes - assert "repo2::userservice" in merged.nodes - assert merged.number_of_nodes() == 2 # no silent collapse - - -def test_global_add_rewires_edges_to_deduplicated_externals(tmp_path): - """Edges incident to an external node that gets deduplicated against an - already-present external must be rewired to the existing node, not dropped.""" - g1 = tmp_path / "graph1.json" - g2 = tmp_path / "graph2.json" - GA = _make_graph( - [ - {"id": "moda", "label": "ModA", "source_file": "src/a.py"}, - {"id": "requests", "label": "requests"}, - ], - [{"source": "moda", "target": "requests", "relation": "imports"}], - ) - GB = _make_graph( - [ - {"id": "modb", "label": "ModB", "source_file": "src/b.py"}, - {"id": "requests", "label": "requests"}, - ], - [{"source": "modb", "target": "requests", "relation": "imports"}], - ) - _graph_to_json(GA, g1) - _graph_to_json(GB, g2) +def test_merge_graphs_prefixes_ids(): + """merge-graphs prefixes node IDs with repo name to avoid silent collision.""" + from graphify.graphjson import prefix_node_link, merge_node_link, node_count - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add, _load_global_graph - global_add(g1, "repoA") - global_add(g2, "repoB") - G = _load_global_graph() - - # repoB's external "requests" was deduplicated against repoA's - assert "repoA::requests" in G.nodes - assert "repoB::requests" not in G.nodes - # repoA's edge is untouched - assert G.has_edge("repoA::moda", "repoA::requests") - # repoB's edge must be rewired to the existing external node, not dropped - assert G.has_edge("repoB::modb", "repoA::requests") - assert G.edges["repoB::modb", "repoA::requests"]["relation"] == "imports" - - -def test_global_add_rejects_oversized_source_graph(monkeypatch, tmp_path): - """#F4: global_add must refuse to read a source graph.json that - exceeds the size cap, rather than json.loads-ing it into memory.""" - import pytest - - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "x", "label": "X", "source_file": "src/x.py"}]) - _graph_to_json(G, src_graph) - - global_dir = tmp_path / ".graphify" - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - with pytest.raises(ValueError, match="exceeds"): - global_add(src_graph, "repoA") + g1 = {"nodes": [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}], "links": []} + g2 = {"nodes": [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}], "links": []} + merged = merge_node_link([prefix_node_link(g1, "repo1"), prefix_node_link(g2, "repo2")]) + ids = {n["id"] for n in merged["nodes"]} + assert "repo1::userservice" in ids + assert "repo2::userservice" in ids + assert node_count(merged) == 2 # no silent collapse diff --git a/tests/test_god_nodes_cli.py b/tests/test_god_nodes_cli.py index 28bacc9b1..c450147a9 100644 --- a/tests/test_god_nodes_cli.py +++ b/tests/test_god_nodes_cli.py @@ -73,4 +73,9 @@ def test_god_nodes_cli_missing_graph_errors(monkeypatch, tmp_path, capsys): with pytest.raises(SystemExit) as exc: _run(monkeypatch, ["graphify", "god-nodes", "--graph", str(tmp_path / "nope.json")]) assert exc.value.code == 1 - assert "graph file not found" in capsys.readouterr().err + # FalkorDB-backed: the graph lives in the engine, so a missing graph.json is + # not itself the error — an unbuilt graph is, and it says so (same wording as + # `affected`, which shares connect_graph). + err = capsys.readouterr().err + assert "No graph found" in err + assert "Re-run /graphify" in err diff --git a/tests/test_hypergraph.py b/tests/test_hypergraph.py index 20e79e679..a532206e9 100644 --- a/tests/test_hypergraph.py +++ b/tests/test_hypergraph.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.build import build_from_json diff --git a/tests/test_labeling.py b/tests/test_labeling.py index bb7bc3c9d..4e1d204b6 100644 --- a/tests/test_labeling.py +++ b/tests/test_labeling.py @@ -6,7 +6,7 @@ import json import sys -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.llm import label_communities, generate_community_labels diff --git a/tests/test_multigraph_compat.py b/tests/test_multigraph_compat.py deleted file mode 100644 index 36902e691..000000000 --- a/tests/test_multigraph_compat.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -import networkx as nx - -from graphify.multigraph_compat import ( - CapabilityCheck, - MultigraphCapabilityResult, - probe_multigraph_capabilities, - require_multigraph_capabilities, -) - - -def test_probe_multigraph_capabilities_passes_current_runtime() -> None: - result = probe_multigraph_capabilities() - - assert result.ok, result.error_message() - assert result.python_version - assert result.networkx_version - assert {check.name for check in result.checks} == { - "keyed_parallel_edges", - "node_link_edges_links_round_trip", - "duplicate_key_overwrite_semantics", - "reserved_key_attr_rejected", - "remove_edges_from_two_tuple_semantics", - "to_undirected_preserves_multigraph_type", - } - - -def test_require_multigraph_capabilities_returns_result() -> None: - result = require_multigraph_capabilities() - - assert result.ok - - -def test_failure_message_is_actionable() -> None: - result = MultigraphCapabilityResult( - python_version="3.10.0", - networkx_version="0.0", - checks=(CapabilityCheck("node_link_edges_links_round_trip", False, "boom"),), - ) - - message = result.error_message() - - assert "--multigraph requires NetworkX keyed MultiDiGraph node-link" in message - assert "Default simple graph mode remains available" in message - assert "node_link_edges_links_round_trip: boom" in message - - -def test_networkx_duplicate_key_overwrite_trap_is_real() -> None: - graph = nx.MultiDiGraph() - - graph.add_edge("a", "b", key="same", relation="first") - graph.add_edge("a", "b", key="same", relation="second") - - assert graph.number_of_edges("a", "b") == 1 - assert graph["a"]["b"]["same"]["relation"] == "second" diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py index 8c39b8e23..ab43bdc6c 100644 --- a/tests/test_multigraph_diagnostics.py +++ b/tests/test_multigraph_diagnostics.py @@ -99,8 +99,11 @@ def test_diagnose_extraction_categorizes_same_endpoint_collapse() -> None: assert summary["same_endpoint_group_count"] == 1 assert summary["relation_variant_groups"] == 1 assert summary["source_location_variant_groups"] == 1 - assert summary["post_build_graph_type"] == "DiGraph" - assert summary["post_build_edge_count"] == 2 + assert summary["post_build_graph_type"] == "GraphStore" + # FalkorDB stores parallel edges natively (one per relation type), so + # same-endpoint edges that differ by relation are preserved rather than + # collapsed the way nx.DiGraph used to (which silently dropped one). + assert summary["post_build_edge_count"] == 3 def test_diagnose_extraction_accepts_node_link_links_key() -> None: @@ -193,7 +196,7 @@ def test_diagnose_extraction_defaults_raw_inputs_to_directed(tmp_path: Path) -> summary = diagnose_file(graph_path) assert summary["effective_directed"] is True - assert summary["post_build_graph_type"] == "DiGraph" + assert summary["post_build_graph_type"] == "GraphStore" def test_diagnose_file_reads_json_and_formats_report(tmp_path: Path) -> None: @@ -302,7 +305,7 @@ def test_diagnose_file_defaults_to_json_directed_flag(tmp_path: Path) -> None: summary = diagnose_file(graph_path) assert summary["effective_directed"] is False - assert summary["post_build_graph_type"] == "Graph" + assert summary["post_build_graph_type"] == "GraphStore" def test_diagnose_file_explicit_directed_override(tmp_path: Path) -> None: @@ -314,7 +317,7 @@ def test_diagnose_file_explicit_directed_override(tmp_path: Path) -> None: summary = diagnose_file(graph_path, directed=True) assert summary["effective_directed"] is True - assert summary["post_build_graph_type"] == "DiGraph" + assert summary["post_build_graph_type"] == "GraphStore" def test_scan_producer_suppression_sites_reports_missing_file(tmp_path: Path) -> None: @@ -360,7 +363,7 @@ def test_diagnose_multigraph_cli_undirected_override(monkeypatch, tmp_path: Path out = capsys.readouterr().out assert "effective_directed: False" in out - assert "post_build_graph_type: Graph" in out + assert "post_build_graph_type: GraphStore" in out def test_diagnose_multigraph_cli_max_examples_zero(monkeypatch, tmp_path: Path, capsys) -> None: diff --git a/tests/test_obsidian_filename_cap.py b/tests/test_obsidian_filename_cap.py index 2d7f1c85d..2000aba53 100644 --- a/tests/test_obsidian_filename_cap.py +++ b/tests/test_obsidian_filename_cap.py @@ -1,7 +1,7 @@ """Regression tests for issue #1094: to_obsidian / to_canvas must cap filenames to stay under the 255-byte filesystem limit, instead of crashing with OSError ENAMETOOLONG on long node labels.""" -import networkx as nx +from tests import nxcompat as nx from graphify.export import to_obsidian, to_canvas diff --git a/tests/test_path_cli.py b/tests/test_path_cli.py index 2584ce4b2..676116c57 100644 --- a/tests/test_path_cli.py +++ b/tests/test_path_cli.py @@ -5,29 +5,18 @@ import os import subprocess import sys -import networkx as nx import pytest -from networkx.readwrite import json_graph import graphify.__main__ as mainmod - -def _write_graph(tmp_path): - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "create_patch", "label": "createPatchHandler()", - "source_file": "server/create-patch-handler.ts", "community": 0}, - {"id": "validate", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", "community": 0}, - ], - "links": [ - {"source": "create_patch", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - ], - } - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - return p +_NODES = [ + {"id": "create_patch", "label": "createPatchHandler()", + "source_file": "server/create-patch-handler.ts", "community": 0, "file_type": "code"}, + {"id": "validate", "label": "validateSanitySession()", + "source_file": "server/sanity-validate-session.ts", "community": 0, "file_type": "code"}, +] +_LINKS = [ + {"source": "create_patch", "target": "validate", "relation": "calls", "confidence": "EXTRACTED"}, +] def _run(monkeypatch, graph_path, src, tgt, capsys): @@ -38,15 +27,17 @@ def _run(monkeypatch, graph_path, src, tgt, capsys): return capsys.readouterr().out -def test_forward_arrow(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) +def test_forward_arrow(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + p = tmp_path / "graph.json" out = _run(monkeypatch, p, "createPatchHandler", "validateSanitySession", capsys) assert "Shortest path (1 hops):" in out assert "createPatchHandler() --calls [EXTRACTED]--> validateSanitySession()" in out -def test_reverse_arrow(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) +def test_reverse_arrow(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + p = tmp_path / "graph.json" out = _run(monkeypatch, p, "validateSanitySession", "createPatchHandler", capsys) assert "Shortest path (1 hops):" in out assert "validateSanitySession() <--calls [EXTRACTED]-- createPatchHandler()" in out diff --git a/tests/test_prs.py b/tests/test_prs.py index 4acc343e6..c40a3ddbf 100644 --- a/tests/test_prs.py +++ b/tests/test_prs.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta, timezone from unittest.mock import patch, MagicMock -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.prs import ( diff --git a/tests/test_query_cli.py b/tests/test_query_cli.py index 0db4e6fa8..47cf6e139 100644 --- a/tests/test_query_cli.py +++ b/tests/test_query_cli.py @@ -1,28 +1,22 @@ """Tests for graphify query CLI context filtering.""" from __future__ import annotations -import json - -import networkx as nx -from networkx.readwrite import json_graph - import graphify.__main__ as mainmod - -def _write_graph(tmp_path): - G = nx.Graph() - G.add_node("n1", label="extract", source_file="extract.py", source_location="L10", community=0) - G.add_node("n2", label="cluster", source_file="cluster.py", source_location="L5", community=0) - G.add_node("n3", label="build", source_file="build.py", source_location="L1", community=1) - G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", context="call") - G.add_edge("n2", "n3", relation="imports", confidence="EXTRACTED", context="import") - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(G, edges="links"))) - return graph_path +_NODES = [ + {"id": "n1", "label": "extract", "source_file": "extract.py", "source_location": "L10", "community": 0, "file_type": "code"}, + {"id": "n2", "label": "cluster", "source_file": "cluster.py", "source_location": "L5", "community": 0, "file_type": "code"}, + {"id": "n3", "label": "build", "source_file": "build.py", "source_location": "L1", "community": 1, "file_type": "code"}, +] +_LINKS = [ + {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", "context": "call"}, + {"source": "n2", "target": "n3", "relation": "imports", "confidence": "EXTRACTED", "context": "import"}, +] -def test_query_cli_explicit_context_filter(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +def test_query_cli_explicit_context_filter(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -36,8 +30,9 @@ def test_query_cli_explicit_context_filter(monkeypatch, tmp_path, capsys): assert "build" not in out -def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -51,22 +46,17 @@ def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys): assert "build" not in out -def _write_calls_graph(tmp_path): - """A single directed `calls` edge on an (on-disk) undirected graph.json, +_CALLS_NODES = [ + {"id": "caller", "label": "caller_fn", "source_file": "a.py", "source_location": "L1", "community": 0}, + {"id": "callee", "label": "callee_fn", "source_file": "b.py", "source_location": "L1", "community": 1}, +] +_CALLS_LINKS = [ + {"source": "caller", "target": "callee", "relation": "calls", + "confidence": "EXTRACTED", "context": "call"}, +] - the standard `graphify extract`/`update` output shape (`"directed": - false`, direction implied only by each link's source/target). - """ - G = nx.Graph() - G.add_node("caller", label="caller_fn", source_file="a.py", source_location="L1", community=0) - G.add_node("callee", label="callee_fn", source_file="b.py", source_location="L1", community=1) - G.add_edge("caller", "callee", relation="calls", confidence="EXTRACTED", context="call") - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(G, edges="links"))) - return graph_path - -def test_query_cli_preserves_calls_direction_when_seeded_on_callee(monkeypatch, tmp_path, capsys): +def test_query_cli_preserves_calls_direction_when_seeded_on_callee(monkeypatch, tmp_path, capsys, seed_graph): """`graphify query` must render `calls` edges caller->callee regardless of which endpoint the query term matches first. @@ -78,7 +68,8 @@ def test_query_cli_preserves_calls_direction_when_seeded_on_callee(monkeypatch, "callee_fn --calls--> caller_fn". graph.json's `source`/`target` for this edge stay correct on disk either way; only the query rendering was wrong. """ - graph_path = _write_calls_graph(tmp_path) + seed_graph(tmp_path, _CALLS_NODES, _CALLS_LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -91,9 +82,10 @@ def test_query_cli_preserves_calls_direction_when_seeded_on_callee(monkeypatch, assert "callee_fn --calls" not in out -def test_query_cli_preserves_calls_direction_when_seeded_on_caller(monkeypatch, tmp_path, capsys): +def test_query_cli_preserves_calls_direction_when_seeded_on_caller(monkeypatch, tmp_path, capsys, seed_graph): """Same edge, seeded from the caller side — must stay correct too.""" - graph_path = _write_calls_graph(tmp_path) + seed_graph(tmp_path, _CALLS_NODES, _CALLS_LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -106,11 +98,22 @@ def test_query_cli_preserves_calls_direction_when_seeded_on_caller(monkeypatch, assert "callee_fn --calls" not in out -def test_query_cli_rejects_oversized_graph(monkeypatch, tmp_path, capsys): - """#F4: query CLI must refuse to parse a graph.json that exceeds the cap.""" +def test_query_cli_rejects_oversized_graph(monkeypatch, tmp_path, capsys, _require_falkordb): + """#F4: the graph.json back-compat import must refuse a file over the cap. + + No seed_graph here on purpose: with an empty store, `query` falls back to + importing graph.json, which is the one path that still parses an arbitrary + file and so must still enforce the size cap. + """ + import json + import pytest - graph_path = _write_graph(tmp_path) + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps( + {"directed": True, "multigraph": False, "graph": {}, + "nodes": _NODES, "links": _LINKS} + )) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) monkeypatch.setattr( diff --git a/tests/test_semantic_similarity.py b/tests/test_semantic_similarity.py index 55d9cce34..9f93ca701 100644 --- a/tests/test_semantic_similarity.py +++ b/tests/test_semantic_similarity.py @@ -1,5 +1,5 @@ """Tests for semantically_similar_to edge support.""" -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.build import build_from_json from graphify.analyze import surprising_connections, _surprise_score diff --git a/tests/test_serve.py b/tests/test_serve.py index fdaa2753c..fd1ade0dd 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -1,8 +1,7 @@ """Tests for serve.py - MCP graph query helpers (no mcp package required).""" import json import pytest -import networkx as nx -from networkx.readwrite import json_graph +from tests import nxcompat as nx from graphify.serve import ( _communities_from_graph, @@ -25,8 +24,8 @@ _query_graph_text, _resolve_context_filters, _subgraph_to_text, + _connect_graph, _cut_lines_to_budget, - _load_graph, _community_header, _search_tokens, ) @@ -582,85 +581,82 @@ def test_query_graph_text_heuristic_context_filter_changes_traversal(): assert "build" not in text -# --- _load_graph --- +# --- _connect_graph --- + +_LOAD_NODES = [ + {"id": "n1", "label": "extract", "source_file": "extract.py", "source_location": "L10", "community": 0, "file_type": "code"}, + {"id": "n2", "label": "cluster", "source_file": "cluster.py", "source_location": "L5", "community": 0, "file_type": "code"}, + {"id": "n3", "label": "build", "source_file": "build.py", "source_location": "L1", "community": 1, "file_type": "code"}, +] +_LOAD_LINKS = [{"source": "n1", "target": "n2", "relation": "calls", "confidence": "INFERRED"}] -def test_load_graph_roundtrip(tmp_path): - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") - p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - G2 = _load_graph(str(p)) - assert G2.number_of_nodes() == G.number_of_nodes() - assert G2.number_of_edges() == G.number_of_edges() + +def test_load_graph_roundtrip(tmp_path, seed_graph): + seed_graph(tmp_path, _LOAD_NODES, _LOAD_LINKS) + G2 = _connect_graph(str(tmp_path / "graph.json")) + assert G2.number_of_nodes() == 3 + assert G2.number_of_edges() == 1 def test_load_graph_missing_file(tmp_path): graphify_dir = tmp_path / "graphify-out" graphify_dir.mkdir() with pytest.raises(SystemExit): - _load_graph(str(graphify_dir / "nonexistent.json")) - - -def test_load_graph_corrupted_json_prints_recovery_message(tmp_path, capsys): - """json.JSONDecodeError is a ValueError subclass, so its except clause - must be checked before the bare (ValueError, FileNotFoundError) clause, - or the corrupted-graph recovery hint is unreachable (#2005).""" - p = tmp_path / "graph.json" - p.write_text("{not valid json") - with pytest.raises(SystemExit): - _load_graph(str(p)) - err = capsys.readouterr().err - assert "graph.json is corrupted" in err - assert "Re-run /graphify to rebuild" in err + _connect_graph(str(graphify_dir / "nonexistent.json")) -def test_load_graph_generic_value_error_message_unchanged(tmp_path, capsys): - """A non-decode ValueError (e.g. a non-.json path) must still print the - generic error, not the corrupted-graph hint — pins the except-clause - order from #2005 so a future refactor can't collapse them back.""" - p = tmp_path / "graph.txt" - p.write_text("not a graph") - with pytest.raises(SystemExit): - _load_graph(str(p)) - err = capsys.readouterr().err - assert "must be a .json file" in err - assert "corrupted" not in err +# The #2005 corrupted-JSON / non-.json-suffix messages have no analogue on the +# FalkorDB path: _connect_graph resolves a pointer to the engine rather than +# parsing a file, so there is no decode step to distinguish from a generic +# ValueError. A malformed graph.json now only matters on the back-compat import +# path, which reports "graph empty" and is covered below. -def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path, capsys): - # #F4: oversized graph.json must fail fast (SystemExit) with a clear error. - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") +def test_connect_graph_rejects_oversized_backcompat_json(monkeypatch, tmp_path, capsys, _require_falkordb): + """#F4: the pre-FalkorDB graph.json import is the one path that still parses + an arbitrary file, so it must fail fast on an oversized one.""" p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) + p.write_text(json.dumps({ + "directed": True, "multigraph": False, "graph": {}, + "nodes": [{"id": "a", "label": "alpha"}], "links": [], + })) monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) with pytest.raises(SystemExit): - _load_graph(str(p)) + _connect_graph(str(p)) err = capsys.readouterr().err assert "exceeds" in err assert "byte cap" in err -def test_load_graph_accepts_under_cap(monkeypatch, tmp_path): - # Verifies the cap path does not regress the normal load. - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") +def test_connect_graph_imports_backcompat_json_under_cap(monkeypatch, tmp_path, _require_falkordb): + """Verifies the cap path does not regress the normal back-compat import.""" p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - # Cap well above the actual file size — load proceeds. + p.write_text(json.dumps({ + "directed": True, "multigraph": False, "graph": {}, + "nodes": [{"id": "a", "label": "alpha"}, {"id": "b", "label": "beta"}], + "links": [{"source": "a", "target": "b", "relation": "calls"}], + })) monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 10 * 1024 * 1024) - G2 = _load_graph(str(p)) - assert G2.number_of_nodes() == G.number_of_nodes() + G = _connect_graph(str(p)) + assert G.number_of_nodes() == 2 # --- #874: MCP hot-reload --- def _write_graph(path, nodes: list[str]) -> None: - """Write a minimal graph.json with the given node IDs.""" - G = nx.DiGraph() - for n in nodes: - G.add_node(n, label=n, community=0) - data = json_graph.node_link_data(G, edges="links") - path.write_text(json.dumps(data), encoding="utf-8") + """Seed the FalkorDB graph for path's output dir with the given node IDs.""" + from pathlib import Path as _Path + from graphify.store import open_store + + store = open_store(_Path(path).parent, create=True) + store.clear() + store.add_nodes_from([(n, {"label": n, "community": 0, "file_type": "code"}) for n in nodes]) + # Also keep a graph.json artifact so file-stat based tests still work. + import json as _json + _Path(path).write_text( + _json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": [{"id": n, "label": n, "community": 0} for n in nodes], "links": []}), + encoding="utf-8", + ) def test_maybe_reload_detects_graph_change(tmp_path): @@ -673,15 +669,15 @@ def test_maybe_reload_detects_graph_change(tmp_path): graph_path = out / "graph.json" _write_graph(graph_path, ["alpha", "beta"]) - # Bootstrap _load_graph + _communities_from_graph to verify the reload path - G1 = _load_graph(str(graph_path)) + # Bootstrap _connect_graph + _communities_from_graph to verify the reload path + G1 = _connect_graph(str(graph_path)) assert set(G1.nodes()) == {"alpha", "beta"} # Simulate file changing (bump mtime by touching) time.sleep(0.01) _write_graph(graph_path, ["alpha", "beta", "gamma"]) - G2 = _load_graph(str(graph_path)) + G2 = _connect_graph(str(graph_path)) assert "gamma" in G2.nodes() @@ -921,7 +917,7 @@ def test_query_seeds_from_identifier_not_noise(): def test_query_graph_text_parameter_type_context_filter_changes_traversal(): - import networkx as nx + from tests import nxcompat as nx from graphify.serve import _query_graph_text graph = nx.Graph() @@ -939,7 +935,7 @@ def test_query_graph_text_parameter_type_context_filter_changes_traversal(): def test_query_graph_text_context_filter_aliases_resolve(): - import networkx as nx + from tests import nxcompat as nx from graphify.serve import _normalize_context_filters assert _normalize_context_filters(["param"]) == ["parameter_type"] diff --git a/tests/test_serve_http.py b/tests/test_serve_http.py index 7d542901b..f6194d155 100644 --- a/tests/test_serve_http.py +++ b/tests/test_serve_http.py @@ -47,6 +47,19 @@ def _graph_file(tmp_path: Path) -> str: + """Seed the FalkorDB graph for tmp_path so the server's loader finds it.""" + from graphify.store import open_store + + store = open_store(tmp_path, create=True) + store.clear() + store.add_nodes_from([ + (n["id"], dict({k: v for k, v in n.items() if k != "id"}, file_type="code")) + for n in SAMPLE_GRAPH["nodes"] + ]) + store.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in SAMPLE_GRAPH["edges"] + ]) p = tmp_path / "graph.json" p.write_text(json.dumps(SAMPLE_GRAPH), encoding="utf-8") return str(p) diff --git a/tests/test_src_layout_import_resolution.py b/tests/test_src_layout_import_resolution.py index beeb6b95b..e25cca8b0 100644 --- a/tests/test_src_layout_import_resolution.py +++ b/tests/test_src_layout_import_resolution.py @@ -63,7 +63,7 @@ def test_resolve_python_module_path_walks_up_to_src_package_root(tmp_path): ) -def test_import_edges_identical_from_root_or_src(tmp_path): +def test_import_edges_identical_from_root_or_src(tmp_path, make_store): """Headline (#2072): the same project yields the same import edges whether scanned from the repo root or from src/ (modulo the `src_` id prefix).""" direct = tmp_path / "direct" @@ -73,8 +73,13 @@ def test_import_edges_identical_from_root_or_src(tmp_path): dpaths = [direct / r for r in _FILES] npaths = [nested / "src" / r for r in _FILES] - dG = build_from_json(extract(dpaths, cache_root=tmp_path / "cd", root=direct, parallel=False), root=str(direct)) - nG = build_from_json(extract(npaths, cache_root=tmp_path / "cn", root=nested, parallel=False), root=str(nested)) + # Distinct stores: build_from_json without an explicit store targets the + # default graph name, so both builds would land in ONE FalkorDB graph and the + # second would silently overwrite the first (comparing a graph with itself). + dG = build_from_json(extract(dpaths, cache_root=tmp_path / "cd", root=direct, parallel=False), + root=str(direct), store=make_store()) + nG = build_from_json(extract(npaths, cache_root=tmp_path / "cn", root=nested, parallel=False), + root=str(nested), store=make_store()) d_edges = _import_edges(dG) # strip the `src_` prefix the nested layout adds to every id. diff --git a/tests/test_store_edge_semantics.py b/tests/test_store_edge_semantics.py new file mode 100644 index 000000000..ba9a7a22f --- /dev/null +++ b/tests/test_store_edge_semantics.py @@ -0,0 +1,86 @@ +"""Regression tests for GraphStore edge-direction and seed-resolution semantics +surfaced by the PR #1379 review.""" +from __future__ import annotations + +import json + +from graphify import affected as aff +from graphify.serve import _import_graph_json_into_store + + +def test_edge_attr_lookup_prefers_forward_direction(store): + """With reciprocal directed edges, ``G[u][v]`` must report the u->v edge's + attributes, not an arbitrary direction (regression: undirected ``-[r]-`` + returned the reverse edge's relation).""" + store.add_nodes_from([("a", {"label": "A"}), ("b", {"label": "B"})]) + store.add_edges_from([("a", "b", {"relation": "ab"}), ("b", "a", {"relation": "ba"})]) + assert store["a"]["b"]["relation"] == "ab" + assert store["b"]["a"]["relation"] == "ba" + + +def test_edge_attr_lookup_falls_back_to_reverse(store): + """A single v->u edge is still found when queried as (u, v): the undirected + graph model must not lose connectivity just because direction is preferred.""" + store.add_nodes_from([("a", {"label": "A"}), ("b", {"label": "B"})]) + store.add_edges_from([("b", "a", {"relation": "ba"})]) + assert store["a"]["b"]["relation"] == "ba" + + +def test_remove_edges_is_directed(store): + """Removing (u, v) must delete only the u->v edge, not a reciprocal v->u + edge (regression: undirected delete nuked both directions, over-pruning + edges whose own source_file was not deleted).""" + store.add_nodes_from([("a", {"label": "A"}), ("b", {"label": "B"})]) + store.add_edges_from([("a", "b", {"relation": "ab"}), ("b", "a", {"relation": "ba"})]) + store.remove_edges([("a", "b")]) + remaining = sorted((u, v, d.get("relation")) for u, v, d in store.edges(data=True)) + assert remaining == [("b", "a", "ba")] + + +def test_traverse_keeps_all_same_level_parent_edges(store): + """A node reached from multiple parents in the same BFS level must keep an + edge from EVERY parent (regression: marking visited mid-level dropped all but + the first, thinning ~⅓ of edges in the subgraph `query` shows the user). + + Diamond: A->B, A->C, B->D, C->D ; from A at depth 2 both B->D and C->D survive. + """ + store.add_nodes_from([(x, {"label": x}) for x in "ABCD"]) + store.add_edges_from([("A", "B", {}), ("A", "C", {}), ("B", "D", {}), ("C", "D", {})]) + visited, edges = store._traverse(["A"], depth=2, hub_threshold=10 ** 9) + assert set(visited) == set("ABCD") + und = {frozenset((u, v)) for u, v in edges} + assert frozenset(("B", "D")) in und + assert frozenset(("C", "D")) in und + assert len(edges) == 4 + + +def test_traverse_dedups_parallel_edges(store): + """Parallel edges between the same pair must not double-count in the walk.""" + store.add_nodes_from([("A", {"label": "A"}), ("B", {"label": "B"})]) + store.add_edges_from([("A", "B", {"relation": "r1"}), ("A", "B", {"relation": "r2"})]) + _visited, edges = store._traverse(["A"], depth=1, hub_threshold=10 ** 9) + assert edges == [("A", "B")] + + +def test_resolve_seed_native_bare_name(store): + """`affected foo` must resolve to a uniquely-matching callable node ``foo()`` + on the native store path, matching the in-memory _bare_name tier.""" + store.add_nodes_from([("n1", {"label": "foo()"}), ("n2", {"label": "fooExtra()"})]) + assert aff.resolve_seed(store, "foo") == "n1" + + +def test_legacy_graph_json_imports_into_empty_store(store, tmp_path): + """A node-link graph.json (pre-FalkorDB project / --no-cluster output) must + import into an empty store so it stays queryable without a rebuild.""" + gj = tmp_path / "graph.json" + gj.write_text(json.dumps({ + "nodes": [{"id": "a", "label": "A", "source_file": "a.py"}, + {"id": "b", "label": "B", "source_file": "b.py"}], + "links": [{"source": "a", "target": "b", "relation": "calls"}], + })) + assert store.number_of_nodes() == 0 + assert _import_graph_json_into_store(gj, store) is True + assert store.number_of_nodes() == 2 + assert store.number_of_edges() == 1 + # Missing / empty JSON is a no-op, not an error. + assert _import_graph_json_into_store(tmp_path / "missing.json", store) is False diff --git a/tests/test_wiki.py b/tests/test_wiki.py index 289a0a2b4..c478554a0 100644 --- a/tests/test_wiki.py +++ b/tests/test_wiki.py @@ -3,7 +3,7 @@ import urllib.parse import pytest from pathlib import Path -import networkx as nx +from tests import nxcompat as nx from graphify.wiki import to_wiki, _index_md, _community_article, _god_node_article _MD_LINK = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") diff --git a/tools/skillgen/expected/graphify__skills__agents__references__query.md b/tools/skillgen/expected/graphify__skills__agents__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__query.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__query.md b/tools/skillgen/expected/graphify__skills__amp__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__query.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__query.md b/tools/skillgen/expected/graphify__skills__claude__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__query.md b/tools/skillgen/expected/graphify__skills__claw__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__query.md b/tools/skillgen/expected/graphify__skills__codex__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__query.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__query.md b/tools/skillgen/expected/graphify__skills__copilot__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__query.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__query.md b/tools/skillgen/expected/graphify__skills__droid__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__query.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__query.md b/tools/skillgen/expected/graphify__skills__kilo__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__query.md b/tools/skillgen/expected/graphify__skills__kiro__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__query.md b/tools/skillgen/expected/graphify__skills__opencode__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__query.md b/tools/skillgen/expected/graphify__skills__pi__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__query.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__query.md b/tools/skillgen/expected/graphify__skills__trae__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__query.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__query.md b/tools/skillgen/expected/graphify__skills__vscode__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__query.md b/tools/skillgen/expected/graphify__skills__windows__references__query.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__query.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/fragments/references/query/default.md b/tools/skillgen/fragments/references/query/default.md index 56565eb78..a22fc620b 100644 --- a/tools/skillgen/fragments/references/query/default.md +++ b/tools/skillgen/fragments/references/query/default.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -68,7 +69,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. 2. Run the appropriate traversal from each starting node. @@ -78,13 +79,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -153,7 +151,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -195,13 +193,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -222,22 +217,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -263,13 +256,10 @@ If the CLI is unavailable, run it inline: ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -293,7 +283,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 3632fd412..4912b032f 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -101,17 +101,20 @@ deleted = list(incremental.get('deleted_files', [])) # now that replace — not the dedup pass — reconciles changed files). prune = list(deleted) or None -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir; edge direction (calls, implements, imports) is stored natively. # Pass root= so prune_sources (absolute paths from detect_incremental) are # relativized to match the graph's relative source_file values; without it # nothing is pruned and stale nodes accumulate on every update (#1361). # directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else # False. Without it a --directed --update silently rebuilds undirected and collapses # reciprocal A<->B edges (#1392). +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=prune, root='INPUT_PATH', directed=IS_DIRECTED, @@ -122,9 +125,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -171,26 +174,32 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + # directed=IS_DIRECTED so the scratch graph is built the same way build_merge + # built G_new; otherwise the diff reports phantom edge changes (#1392). + G_old = build_from_json(old_extract, directed=IS_DIRECTED, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/uv.lock b/uv.lock index 671c304f9..e82d847ae 100644 --- a/uv.lock +++ b/uv.lock @@ -6,11 +6,14 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version < '3.11'", ] @@ -548,11 +551,14 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, @@ -946,6 +952,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/90/02/1cf9cef72228ca2b2776b4ed0cb2645298ddcc57c1fe2c545cd46bc11eae/falkordb-1.6.1-py3-none-any.whl", hash = "sha256:cf51caeb433c04db303de5967a0c2590675fcc0354e80997870b1e0497d30c34", size = 37802, upload-time = "2026-04-28T13:24:45.677Z" }, ] +[[package]] +name = "falkordblite" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "falkordb", marker = "python_full_version >= '3.12'" }, + { name = "psutil", marker = "python_full_version >= '3.12'" }, + { name = "redis", marker = "python_full_version >= '3.12'" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/03/afbb6e0f03c302aa9b64a38e1a2c43664f86921f0dcbf03ec9e31da06ac6/falkordblite-0.10.0.tar.gz", hash = "sha256:65a72abafd30711f699c15571df6959edb8901605053ce940ccdd837832e709b", size = 23675620, upload-time = "2026-05-02T13:12:29.429Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/43/39d8cf13964784447676d24f0cefa3bdc99c10e647c71e6a4172d302dcac/falkordblite-0.10.0-cp312-cp312-macosx_10_13_x86_64.macosx_15_0_arm64.whl", hash = "sha256:741fda166170513db1815d5369870e47d44da2e9b85320fddbc50c88a7338d51", size = 17752268, upload-time = "2026-05-02T13:11:57.906Z" }, + { url = "https://files.pythonhosted.org/packages/61/17/3ec180ca7c2e79a0c3e9912ac04b446e733745cd944845fe4306be016efd/falkordblite-0.10.0-cp312-cp312-manylinux_2_39_aarch64.whl", hash = "sha256:bfe1f47ae03decdc0ad111ec82606bac82ee6cdd7fea04cbcff1283608cc2d2e", size = 34579293, upload-time = "2026-05-02T13:12:01.601Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5e/d343c5249bd24c6614e8c1b718a73bb9f68beb2cf90464bc51c9436d25af/falkordblite-0.10.0-cp312-cp312-manylinux_2_39_x86_64.whl", hash = "sha256:ee9659bd0c7cdf0c2532977f2ec8bf1d1ab01cb3b6776e50c67046481f3162c7", size = 36154066, upload-time = "2026-05-02T13:12:05.435Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1e/18612dc9e75e5c02a281a49d97b56b5504eb5907f971c68e8a5801033f36/falkordblite-0.10.0-cp313-cp313-macosx_10_13_x86_64.macosx_15_0_arm64.whl", hash = "sha256:54a051cbc0bb25b30e65f46ddb1b63149a80bb7004c97615d39c491d492a6d9b", size = 17752240, upload-time = "2026-05-02T13:12:08.837Z" }, + { url = "https://files.pythonhosted.org/packages/29/b3/63f9b1168c4d1abbee4418a0a165496f57cd4b93a261cb481272ce353890/falkordblite-0.10.0-cp313-cp313-manylinux_2_39_aarch64.whl", hash = "sha256:ce1bd408ebaafc3dbdf39e860888382aae69de2b3c949dadfd132779130b99b2", size = 34579294, upload-time = "2026-05-02T13:12:12.069Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cf/d792cf46292f7756f96727916e77ebf9821371e22bff65ed8e90db6b80b9/falkordblite-0.10.0-cp313-cp313-manylinux_2_39_x86_64.whl", hash = "sha256:7d52a3665f6de30cbffc5809a1c6d722492c95bf8dae4a7ee108ca58da939b25", size = 36154069, upload-time = "2026-05-02T13:12:15.742Z" }, + { url = "https://files.pythonhosted.org/packages/ab/57/eb2ac5824f7831fcb550468cc504a944adfee1b0c5a09b4bc684b0217976/falkordblite-0.10.0-cp314-cp314-macosx_10_15_x86_64.macosx_15_0_arm64.whl", hash = "sha256:5eff806e09902b36f34a7c058de0dcf237f0ae047d2b39c0423a5d6a52193341", size = 17752257, upload-time = "2026-05-02T13:12:18.896Z" }, + { url = "https://files.pythonhosted.org/packages/61/13/76932987b43286443088be9d1ebdbc6999ac084d5c40dbfe4a4704aa264b/falkordblite-0.10.0-cp314-cp314-manylinux_2_39_aarch64.whl", hash = "sha256:6cd10bb1b0e35946e0e4c06bd0c16298fe6df70e8e18bc055db0d60359c7271c", size = 34579275, upload-time = "2026-05-02T13:12:22.181Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a6/6eba83964a0633aeee698a1fbd23b954ce39ee4cfd7eced73f52d95ca89a/falkordblite-0.10.0-cp314-cp314-manylinux_2_39_x86_64.whl", hash = "sha256:b36665ebd9e2d5e38b401b34f7ba6f0b22e5b88cbc817971c8c1d71a8ad305f7", size = 36154066, upload-time = "2026-05-02T13:12:26.001Z" }, +] + [[package]] name = "faster-whisper" version = "1.2.1" @@ -1093,8 +1122,8 @@ name = "graphifyy" version = "0.9.28" source = { editable = "." } dependencies = [ - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "falkordb" }, + { name = "falkordblite", marker = "python_full_version >= '3.12'" }, { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "rapidfuzz" }, @@ -1181,6 +1210,9 @@ kimi = [ leiden = [ { name = "graspologic", marker = "python_full_version < '3.13'" }, ] +lite = [ + { name = "falkordblite", marker = "python_full_version >= '3.12'" }, +] mcp = [ { name = "mcp" }, { name = "starlette" }, @@ -1252,8 +1284,11 @@ requires-dist = [ { name = "anthropic", marker = "extra == 'anthropic'" }, { name = "boto3", marker = "extra == 'all'" }, { name = "boto3", marker = "extra == 'bedrock'" }, + { name = "falkordb", specifier = ">=1.0" }, { name = "falkordb", marker = "extra == 'all'" }, - { name = "falkordb", marker = "extra == 'falkordb'" }, + { name = "falkordb", marker = "extra == 'falkordb'", specifier = ">=1.0" }, + { name = "falkordblite", marker = "python_full_version >= '3.12'" }, + { name = "falkordblite", marker = "python_full_version >= '3.12' and extra == 'lite'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'all'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'video'" }, { name = "graspologic", marker = "python_full_version < '3.13' and extra == 'all'" }, @@ -1268,7 +1303,6 @@ requires-dist = [ { name = "mcp", marker = "extra == 'mcp'" }, { name = "neo4j", marker = "extra == 'all'" }, { name = "neo4j", marker = "extra == 'neo4j'" }, - { name = "networkx", specifier = ">=3.4" }, { name = "numpy", specifier = ">=1.21" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'all'", specifier = ">=2.0" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'svg'", specifier = ">=2.0" }, @@ -1331,7 +1365,7 @@ requires-dist = [ { name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" }, { name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" }, ] -provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "all"] +provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "lite", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "all"] [package.metadata.requires-dev] dev = [ @@ -2244,15 +2278,12 @@ name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ @@ -2315,9 +2346,12 @@ name = "numpy" version = "1.26.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version < '3.11'", ] sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } @@ -2591,9 +2625,12 @@ name = "pandas" version = "3.0.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, @@ -2922,6 +2959,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/95/608f665226bca68b736b79e457fded9a2a38c4f4379a4a7614303d9db3bc/protobuf-7.34.1-py3-none-any.whl", hash = "sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11", size = 170715, upload-time = "2026-03-20T17:34:45.384Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "psycopg" version = "3.3.4" @@ -3914,9 +3979,12 @@ name = "scikit-learn" version = "1.8.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "joblib", marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, @@ -4028,9 +4096,12 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" },