From 23be8700a19ffa12cb102378d5d53713bd46a120 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 09:30:10 +0300 Subject: [PATCH 01/17] feat: FalkorDB backend (server + embedded Lite), replacing NetworkX Replace NetworkX with FalkorDB as the sole graph store, via a cache-free GraphStore facade: the graph lives in the engine and is never loaded into Python. All traversal/query/analysis run as scoped or in-engine Cypher; algorithms FalkorDB lacks (Louvain, edge-betweenness, simple-cycles) are server-side JS UDFs (graphify/udfs/graphify_algos.js). multigraph_compat is removed (native parallel edges). Query-path optimizations: - truncate-before-fetch rendering: rank by degree + LIMIT in-engine, fetch attrs only for the nodes that fit the token budget (hub query 7-14s -> ~1s) - in-engine edge-context filtering (no MemGraph copy of the whole graph) - hub-degree threshold computed once and cached in graph metadata Audit fixes: MCP tools get_node / graph_stats / audit no longer scan all nodes/edges (use search_nodes + a confidence_counts aggregation). NetworkX fully removed (not a dependency): the package has zero runtime nx imports; the test suite builds fixtures via tests/nxcompat.py, a drop-in nx shim backed by the in-house MemGraph. FalkorDB Lite: embedded in-process engine (redislite + bundled falkordb module), no external server. Select with a `falkordb-lite://` URI or the GRAPHIFY_FALKORDB_LITE env var; install via the new `lite` extra (Python >=3.12). Warm performance matches the server (no TCP hop) at comparable memory. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/__main__.py | 200 +-- graphify/affected.py | 106 +- graphify/analyze.py | 148 +- graphify/benchmark.py | 13 +- graphify/build.py | 242 ++-- graphify/callflow_html.py | 36 +- graphify/cluster.py | 81 +- graphify/diagnostics.py | 4 +- graphify/export.py | 212 ++- graphify/global_graph.py | 113 +- graphify/graphjson.py | 114 ++ graphify/hooks.py | 12 +- graphify/multigraph_compat.py | 212 --- graphify/report.py | 1 - graphify/serve.py | 263 ++-- graphify/skills/amp/references/query.md | 52 +- graphify/skills/amp/references/update.md | 31 +- graphify/skills/claude/references/update.md | 31 +- graphify/skills/claw/references/query.md | 52 +- graphify/skills/claw/references/update.md | 31 +- graphify/skills/codex/references/query.md | 52 +- graphify/skills/codex/references/update.md | 31 +- graphify/skills/copilot/references/query.md | 52 +- graphify/skills/copilot/references/update.md | 31 +- graphify/skills/droid/references/query.md | 52 +- graphify/skills/droid/references/update.md | 31 +- graphify/skills/kilo/references/query.md | 52 +- graphify/skills/kilo/references/update.md | 31 +- graphify/skills/kiro/references/query.md | 52 +- graphify/skills/kiro/references/update.md | 31 +- graphify/skills/opencode/references/query.md | 52 +- graphify/skills/opencode/references/update.md | 31 +- graphify/skills/pi/references/query.md | 52 +- graphify/skills/pi/references/update.md | 31 +- graphify/skills/trae/references/query.md | 52 +- graphify/skills/trae/references/update.md | 31 +- graphify/skills/vscode/references/query.md | 52 +- graphify/skills/vscode/references/update.md | 31 +- graphify/skills/windows/references/query.md | 52 +- graphify/skills/windows/references/update.md | 31 +- graphify/store.py | 1251 +++++++++++++++++ graphify/udfs/graphify_algos.js | 344 +++++ graphify/watch.py | 14 +- graphify/wiki.py | 1 - pyproject.toml | 7 +- tests/conftest.py | 120 ++ tests/nxcompat.py | 102 ++ tests/test_affected_cli.py | 41 +- tests/test_analyze.py | 100 +- tests/test_benchmark.py | 28 +- tests/test_build.py | 130 +- tests/test_cli_export.py | 41 +- tests/test_cluster.py | 2 +- tests/test_confidence.py | 2 +- tests/test_explain_cli.py | 48 +- tests/test_global_graph.py | 306 ++-- tests/test_hypergraph.py | 2 +- tests/test_labeling.py | 2 +- tests/test_multigraph_compat.py | 56 - tests/test_multigraph_diagnostics.py | 15 +- tests/test_obsidian_filename_cap.py | 2 +- tests/test_path_cli.py | 40 +- tests/test_prs.py | 2 +- tests/test_query_cli.py | 54 +- tests/test_semantic_similarity.py | 2 +- tests/test_serve.py | 74 +- tests/test_serve_http.py | 13 + tests/test_wiki.py | 2 +- ...raphify__skills__amp__references__query.md | 52 +- ...aphify__skills__amp__references__update.md | 31 +- ...ify__skills__claude__references__update.md | 31 +- ...aphify__skills__claw__references__query.md | 52 +- ...phify__skills__claw__references__update.md | 31 +- ...phify__skills__codex__references__query.md | 52 +- ...hify__skills__codex__references__update.md | 31 +- ...ify__skills__copilot__references__query.md | 52 +- ...fy__skills__copilot__references__update.md | 31 +- ...phify__skills__droid__references__query.md | 52 +- ...hify__skills__droid__references__update.md | 31 +- ...aphify__skills__kilo__references__query.md | 52 +- ...phify__skills__kilo__references__update.md | 31 +- ...aphify__skills__kiro__references__query.md | 52 +- ...phify__skills__kiro__references__update.md | 31 +- ...fy__skills__opencode__references__query.md | 52 +- ...y__skills__opencode__references__update.md | 31 +- ...graphify__skills__pi__references__query.md | 52 +- ...raphify__skills__pi__references__update.md | 31 +- ...aphify__skills__trae__references__query.md | 52 +- ...phify__skills__trae__references__update.md | 31 +- ...hify__skills__vscode__references__query.md | 52 +- ...ify__skills__vscode__references__update.md | 31 +- ...ify__skills__windows__references__query.md | 52 +- ...fy__skills__windows__references__update.md | 31 +- .../fragments/references/query/cli-inline.md | 52 +- .../fragments/references/shared/update.md | 31 +- 95 files changed, 4056 insertions(+), 2639 deletions(-) create mode 100644 graphify/graphjson.py delete mode 100644 graphify/multigraph_compat.py create mode 100644 graphify/store.py create mode 100644 graphify/udfs/graphify_algos.js create mode 100644 tests/nxcompat.py delete mode 100644 tests/test_multigraph_compat.py diff --git a/graphify/__main__.py b/graphify/__main__.py index 7e4102a5c..f4d2de93b 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2511,9 +2511,8 @@ def main() -> None: if len(sys.argv) < 3: print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) sys.exit(1) - from graphify.serve import _query_graph_text + from graphify.serve import _query_graph_text, _load_graph from graphify.security import sanitize_label - from networkx.readwrite import json_graph from graphify import querylog question = sys.argv[2] @@ -2550,27 +2549,7 @@ def main() -> None: else: i += 1 gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print(f"error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - try: - import json as _json - import networkx as _nx - - _raw = _json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) + G = _load_graph(graph_path) import time as _time _t0 = _time.perf_counter() _mode = "dfs" if use_dfs else "bfs" @@ -2681,9 +2660,7 @@ def main() -> None: file=sys.stderr, ) sys.exit(1) - from graphify.serve import _score_nodes - from networkx.readwrite import json_graph - import networkx as _nx + from graphify.serve import _score_nodes, _load_graph source_label = sys.argv[2] target_label = sys.argv[3] @@ -2693,19 +2670,7 @@ def main() -> None: if a == "--graph" and i + 1 < len(args): graph_path = args[i + 1] gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) + G = _load_graph(graph_path) src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) if not src_scored: @@ -2734,9 +2699,8 @@ def main() -> None: f"(top score {_top:g}, runner-up {_runner:g})", file=sys.stderr, ) - try: - path_nodes = _nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) - except (_nx.NetworkXNoPath, _nx.NodeNotFound): + path_nodes = G.shortest_path(src_nid, tgt_nid) + if not path_nodes: print(f"No path found between '{source_label}' and '{target_label}'.") sys.exit(0) hops = len(path_nodes) - 1 @@ -2744,8 +2708,8 @@ def main() -> None: from graphify.build import edge_data for i in range(len(path_nodes) - 1): u, v = path_nodes[i], path_nodes[i + 1] - # Check which direction the stored edge points. - if G.has_edge(u, v): + # Check which direction the stored edge points (edges are directed). + if G.has_directed_edge(u, v): edata = edge_data(G, u, v) forward = True else: @@ -2773,8 +2737,7 @@ def main() -> None: if len(sys.argv) < 3: print('Usage: graphify explain "" [--graph path]', file=sys.stderr) sys.exit(1) - from graphify.serve import _find_node - from networkx.readwrite import json_graph + from graphify.serve import _find_node, _load_graph label = sys.argv[2] graph_path = _default_graph_path() @@ -2783,25 +2746,25 @@ def main() -> None: if a == "--graph" and i + 1 < len(args): graph_path = args[i + 1] gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) + G = _load_graph(graph_path) matches = _find_node(G, label) if not matches: print(f"No node matching '{label}' found.") sys.exit(0) nid = matches[0] - d = G.nodes[nid] + if hasattr(G, "node_connections"): + # FalkorDB-native: node detail + connections in two scoped queries. + d, deg = G.node_detail(nid) + conns = G.node_connections(nid) + connections = [(c["dir"], c["label"], c["relation"], c["confidence"], c["degree"]) for c in conns] + else: + from graphify.build import edge_data + d = G.nodes[nid] + deg = G.degree(nid) + conns_raw = [("out", nb, edge_data(G, nid, nb)) for nb in G.successors(nid)] + conns_raw += [("in", nb, edge_data(G, nb, nid)) for nb in G.predecessors(nid)] + connections = [(dr, G.nodes[nb].get("label", nb), e.get("relation", ""), + e.get("confidence", ""), G.degree(nb)) for dr, nb, e in conns_raw] print(f"Node: {d.get('label', nid)}") print(f" ID: {nid}") print( @@ -2809,21 +2772,13 @@ def main() -> None: ) print(f" Type: {d.get('file_type', '')}") print(f" Community: {d.get('community', '')}") - print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data - connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) - for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) - for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) + print(f" Degree: {deg}") if connections: print(f"\nConnections ({len(connections)}):") - connections.sort(key=lambda c: G.degree(c[1]), reverse=True) - for direction, nb, edata in connections[:20]: - rel = edata.get("relation", "") - conf = edata.get("confidence", "") + connections.sort(key=lambda c: c[4], reverse=True) + for direction, nb_label, rel, conf, _deg in connections[:20]: arrow = "-->" if direction == "out" else "<--" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]") + print(f" {arrow} {nb_label} [{rel}] [{conf}]") if len(connections) > 20: print(f" ... and {len(connections) - 20} more") from graphify import querylog @@ -3023,8 +2978,7 @@ def main() -> None: file=sys.stderr, ) sys.exit(1) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json + from graphify.serve import _load_graph from graphify.cluster import cluster, score_all, remap_communities_to_previous from graphify.analyze import ( god_nodes, @@ -3035,10 +2989,7 @@ def main() -> None: from graphify.export import to_json, to_html print("Loading existing graph...") - _enforce_graph_size_cap_or_exit(graph_json) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) - _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) + G = _load_graph(str(graph_json)) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") print("Re-clustering...") communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) @@ -3048,9 +2999,9 @@ def main() -> None: # labels follow raw cid index and become misaligned whenever the # graph has changed between labeling and cluster-only (#1027). previous_node_community = { - n["id"]: n["community"] - for n in _raw.get("nodes", []) - if n.get("community") is not None and n.get("id") is not None + nid: attrs["community"] + for nid, attrs in G.nodes(data=True) + if attrs.get("community") is not None } if previous_node_community: communities = remap_communities_to_previous(communities, previous_node_community) @@ -3254,42 +3205,22 @@ def main() -> None: # the merge so a human can investigate. _MERGE_MAX_BYTES = 50 * 1024 * 1024 _MERGE_MAX_NODES = 100_000 - import networkx as _nx - from networkx.readwrite import json_graph as _jg - def _load_graph(p: str): - path_obj = Path(p) - try: - size = path_obj.stat().st_size - except OSError as exc: - raise RuntimeError(f"cannot stat {p}: {exc}") from exc - if size > _MERGE_MAX_BYTES: - raise RuntimeError( - f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" - ) - data = json.loads(path_obj.read_text(encoding="utf-8")) - try: - return _jg.node_link_graph(data, edges="links"), data - except TypeError: - return _jg.node_link_graph(data), data + from graphify.graphjson import load_node_link, merge_node_link, node_count try: - G_cur, _ = _load_graph(_current_path) - G_oth, _ = _load_graph(_other_path) + cur = load_node_link(_current_path, max_bytes=_MERGE_MAX_BYTES) + oth = load_node_link(_other_path, max_bytes=_MERGE_MAX_BYTES) except Exception as exc: print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge - merged = _nx.compose(G_cur, G_oth) - if merged.number_of_nodes() > _MERGE_MAX_NODES: + merged = merge_node_link([cur, oth]) + if node_count(merged) > _MERGE_MAX_NODES: print( - f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " + f"[graphify merge-driver] merged graph has {node_count(merged)} nodes, " f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", file=sys.stderr, ) sys.exit(1) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - Path(_current_path).write_text(json.dumps(out_data, indent=2), encoding="utf-8") + Path(_current_path).write_text(json.dumps(merged, indent=2), encoding="utf-8") sys.exit(0) elif cmd == "merge-graphs": @@ -3311,37 +3242,19 @@ def _load_graph(p: str): file=sys.stderr, ) sys.exit(1) - import networkx as _nx - from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix - graphs = [] + from graphify.graphjson import load_node_link, merge_node_link, prefix_node_link, node_count, edge_count + prefixed_graphs = [] for gp in graph_paths: if not gp.exists(): print(f"error: not found: {gp}", file=sys.stderr) sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - data = json.loads(gp.read_text(encoding="utf-8")) - # Normalize edges/links key before loading — graphify writes "links" - # via node_link_data but older runs may have used "edges" (#738). - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = _jg.node_link_graph(data, edges="links") - except TypeError: - G = _jg.node_link_graph(data) - graphs.append(G) - merged = _nx.Graph() - for G, gp in zip(graphs, graph_paths): + data = load_node_link(gp) repo_tag = gp.parent.parent.name # graphify-out/../ → repo dir name - prefixed = _prefix(G, repo_tag) - merged = _nx.compose(merged, prefixed) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) + prefixed_graphs.append(prefix_node_link(data, repo_tag)) + merged = merge_node_link(prefixed_graphs) out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") + out_path.write_text(json.dumps(merged, indent=2), encoding="utf-8") + print(f"Merged {len(graph_paths)} graphs -> {node_count(merged)} nodes, {edge_count(merged)} edges") print(f"Written to: {out_path}") elif cmd == "clone": @@ -3516,17 +3429,9 @@ def _load_graph(p: str): print(f"callflow HTML written - open in any browser: {out}") sys.exit(0) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json as _bfj + from graphify.serve import _load_graph - _enforce_graph_size_cap_or_exit(graph_path) - _raw = json.loads(graph_path.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = _jg.node_link_graph(_raw, edges="links") - except TypeError: - G = _jg.node_link_graph(_raw) + G = _load_graph(str(graph_path)) # Load optional analysis/labels communities: dict[int, list[str]] = {} @@ -4233,17 +4138,22 @@ def _progress(idx: int, total: int, _result: dict) -> None: from graphify.export import to_json as _to_json from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising dedup_backend = backend if dedup_llm else None + # Bind every build/merge/export step to the one FalkorDB graph for this + # output dir (open_store writes the pointer file so query/serve/etc. find it). + from graphify.store import open_store as _open_store + _store = _open_store(graphify_out, create=True) if incremental_mode: G = _build_merge( [merged], - graph_path=existing_graph_path, + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted_files or None, dedup=True, dedup_llm_backend=dedup_backend, root=target, ) else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) + G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target, store=_store) if G.number_of_nodes() == 0: print( "[graphify extract] graph is empty — extraction produced no nodes. " diff --git a/graphify/affected.py b/graphify/affected.py index 109eaa95e..eae6d9dba 100644 --- a/graphify/affected.py +++ b/graphify/affected.py @@ -5,8 +5,6 @@ from pathlib import Path from typing import Iterable -import networkx as nx - DEFAULT_AFFECTED_RELATIONS = ( "calls", @@ -43,32 +41,26 @@ def _format_location(data: dict) -> str: return str(source_file) -def resolve_seed(graph: nx.Graph, query: str) -> str | None: +def resolve_seed(graph, query: str) -> str | None: + # FalkorDB-native: resolve via scoped queries (no full-graph load). + if hasattr(graph, "find_node_ids"): + if graph.has_node(query): + return query + for kwargs in ({"label": query}, {"source_file": query}, {"label_contains": query}): + matches = graph.find_node_ids(limit=2, **kwargs) + if len(matches) == 1: + return str(matches[0]) + return None + # In-memory fallback (nx / MemGraph) if query in graph: return query query_lower = query.lower() - exact_label_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) - if str(data.get("label", "")).lower() == query_lower - ] - if len(exact_label_matches) == 1: - return exact_label_matches[0] - exact_source_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) - if str(data.get("source_file", "")).lower() == query_lower - ] - if len(exact_source_matches) == 1: - return exact_source_matches[0] - contains_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) - if query_lower in str(data.get("label", "")).lower() - ] - if len(contains_matches) == 1: - return contains_matches[0] - return None + for field in ("label", "source_file"): + exact = [str(n) for n, d in graph.nodes(data=True) if str(d.get(field, "")).lower() == query_lower] + if len(exact) == 1: + return exact[0] + contains = [str(n) for n, d in graph.nodes(data=True) if query_lower in str(d.get("label", "")).lower()] + return contains[0] if len(contains) == 1 else None def affected_nodes( @@ -79,10 +71,32 @@ def affected_nodes( depth: int = 2, ) -> list[AffectedHit]: relation_set = set(relations) + + # FalkorDB-native: one batched reverse-edge query per BFS level (depth queries + # total) instead of a per-node loop over an in-memory copy of the graph. + if hasattr(graph, "incoming_edges"): + seen = {seed} + hits: list[AffectedHit] = [] + frontier = [seed] + for d in range(depth): + if not frontier: + break + rows = graph.incoming_edges(frontier, relation_set) + nxt: list[str] = [] + for _fid, src, relation in sorted(rows, key=lambda r: (str(r[0]), str(r[1]), str(r[2]))): + src = str(src) + if src in seen: + continue + seen.add(src) + hits.append(AffectedHit(src, d + 1, str(relation))) + nxt.append(src) + frontier = nxt + return hits + + # In-memory fallback (nx / MemGraph) seen = {seed} queue: deque[tuple[str, int]] = deque([(seed, 0)]) - hits: list[AffectedHit] = [] - + hits = [] while queue: current, current_depth = queue.popleft() if current_depth >= depth: @@ -103,10 +117,8 @@ def affected_nodes( if source in seen: continue seen.add(source) - hit = AffectedHit(source, current_depth + 1, relation) - hits.append(hit) + hits.append(AffectedHit(source, current_depth + 1, relation)) queue.append((source, current_depth + 1)) - return hits @@ -123,8 +135,19 @@ def format_affected( return f"No unique node match for {query}" hits = affected_nodes(graph, seed, relations=relation_list, depth=depth) + + # Fetch attrs for seed + all hits in one batched query (native) or per-node (fallback). + ids = [seed] + [h.node_id for h in hits] + if hasattr(graph, "node_attrs_batch"): + attrs = graph.node_attrs_batch(ids) + else: + attrs = {nid: dict(graph.nodes[nid]) for nid in ids if nid in graph} + + def _label(nid): + return str(attrs.get(nid, {}).get("label") or nid) + lines = [ - f"Affected nodes for {_node_label(graph, seed)}", + f"Affected nodes for {_label(seed)}", f"Relations: {', '.join(relation_list)}", f"Depth: {depth}", ] @@ -133,19 +156,20 @@ def format_affected( return "\n".join(lines) for hit in hits: - data = graph.nodes[hit.node_id] + data = attrs.get(hit.node_id, {}) lines.append( - f"- {_node_label(graph, hit.node_id)} [{hit.via_relation}] {_format_location(data)}" + f"- {_label(hit.node_id)} [{hit.via_relation}] {_format_location(data)}" ) return "\n".join(lines) -def load_graph(path: Path) -> nx.Graph: - import json - from networkx.readwrite import json_graph +def load_graph(path: Path): + """Open the FalkorDB-backed graph for the output dir containing `path`. + + `path` is the legacy graph.json location (e.g. graphify-out/graph.json); we + use its parent directory to locate the FalkorDB pointer and open the store. + """ + from .store import open_store - raw = json.loads(path.read_text(encoding="utf-8")) - try: - return json_graph.node_link_graph(raw, edges="links") - except TypeError: - return json_graph.node_link_graph(raw) + out_dir = Path(path).parent if Path(path).suffix else Path(path) + return open_store(out_dir, create=False) diff --git a/graphify/analyze.py b/graphify/analyze.py index 5f28179d5..2eafc1b9f 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -1,7 +1,6 @@ """Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions.""" from __future__ import annotations from pathlib import Path -import networkx as nx from graphify.build import edge_data @@ -47,34 +46,33 @@ def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: return {n: cid for cid, nodes in communities.items() for n in nodes} -def _is_file_node(G: nx.Graph, node_id: str) -> bool: - """ - Return True if this node is a file-level hub node (e.g. 'client', 'models') - or an AST method stub (e.g. '.auth_flow()', '.__init__()'). - - These are synthetic nodes created by the AST extractor and should be excluded - from god nodes, surprising connections, and knowledge gap reporting. - """ - attrs = G.nodes[node_id] - label = attrs.get("label", "") +def _is_file_node_attrs(label: str, source_file: str, degree: int) -> bool: + """Attrs-based core of _is_file_node (works on a query row, no graph access).""" if not label: return False - # File-level hub: label matches the actual source filename (not just any label ending in .py) - source_file = attrs.get("source_file", "") if source_file: from pathlib import Path as _Path if label == _Path(source_file).name: return True - # Method stub: AST extractor labels methods as '.method_name()' if label.startswith(".") and label.endswith("()"): return True - # Module-level function stub: labeled 'function_name()' - only has a contains edge - # These are real functions but structurally isolated by definition; not a gap worth flagging - if label.endswith("()") and G.degree(node_id) <= 1: + if label.endswith("()") and degree <= 1: return True return False +def _is_file_node(G: nx.Graph, node_id: str) -> bool: + """ + Return True if this node is a file-level hub node (e.g. 'client', 'models') + or an AST method stub (e.g. '.auth_flow()', '.__init__()'). + + These are synthetic nodes created by the AST extractor and should be excluded + from god nodes, surprising connections, and knowledge gap reporting. + """ + attrs = G.nodes[node_id] + return _is_file_node_attrs(attrs.get("label", ""), attrs.get("source_file", ""), G.degree(node_id)) + + _JSON_NOISE_LABELS: frozenset[str] = frozenset({ "start", "end", "name", "id", "type", "properties", "value", "key", "data", "items", "title", "description", "version", @@ -83,13 +81,16 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: }) -def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: - attrs = G.nodes[node_id] - src = (attrs.get("source_file") or "").lower() +def _is_json_key_node_attrs(label: str, source_file: str) -> bool: + src = (source_file or "").lower() if not src.endswith(".json"): return False - label = (attrs.get("label") or "").strip().lower() - return label in _JSON_NOISE_LABELS + return (label or "").strip().lower() in _JSON_NOISE_LABELS + + +def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: + attrs = G.nodes[node_id] + return _is_json_key_node_attrs(attrs.get("label") or "", attrs.get("source_file") or "") def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: @@ -98,19 +99,27 @@ def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: File-level hub nodes are excluded: they accumulate import/contains edges mechanically and don't represent meaningful architectural abstractions. """ - degree = dict(G.degree()) - sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True) + # FalkorDB-native path: rank by degree in-engine and return only top candidates + # (no full-graph load). Filter the small candidate set in Python with the same + # heuristics. Fall back to the in-memory path for non-store graphs (MemGraph/nx). + if hasattr(G, "top_degree_nodes"): + cands = G.top_degree_nodes(limit=max(top_n * 50, 500)) + else: + degree = dict(G.degree()) + order = sorted(degree.items(), key=lambda x: x[1], reverse=True) + cands = [ + {"id": nid, "label": G.nodes[nid].get("label", ""), + "source_file": G.nodes[nid].get("source_file", ""), "degree": deg} + for nid, deg in order[: max(top_n * 50, 500)] + ] result = [] - for node_id, deg in sorted_nodes: - if _is_file_node(G, node_id) or _is_concept_node(G, node_id) or _is_json_key_node(G, node_id): + for c in cands: + label, src, deg = c.get("label", "") or "", c.get("source_file", "") or "", c["degree"] + if _is_file_node_attrs(label, src, deg) or _is_concept_node_attrs(src) or _is_json_key_node_attrs(label, src): continue - if G.nodes[node_id].get("label", "") in _BUILTIN_NOISE_LABELS: + if label in _BUILTIN_NOISE_LABELS: continue - result.append({ - "id": node_id, - "label": G.nodes[node_id].get("label", node_id), - "degree": deg, - }) + result.append({"id": c["id"], "label": label or c["id"], "degree": deg}) if len(result) >= top_n: break return result @@ -157,12 +166,14 @@ def _is_concept_node(G: nx.Graph, node_id: str) -> bool: - Empty source_file - source_file doesn't look like a real file path (no extension) """ - data = G.nodes[node_id] - source = data.get("source_file", "") - if not source: + return _is_concept_node_attrs(G.nodes[node_id].get("source_file", "")) + + +def _is_concept_node_attrs(source_file: str) -> bool: + if not source_file: return True # Has no file extension → probably a concept label, not a real file - if "." not in source.split("/")[-1]: + if "." not in source_file.split("/")[-1]: return True return False @@ -289,25 +300,15 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: u_source = G.nodes[u].get("source_file", "") v_source = G.nodes[v].get("source_file", "") - if not u_source or not v_source or u_source == v_source: continue score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source, degrees) - src_id = data.get("_src", u) - if src_id not in G.nodes: - src_id = u - tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: - tgt_id = v candidates.append({ "_score": score, - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), - "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), - ], + "source": G.nodes[u].get("label", u), + "target": G.nodes[v].get("label", v), + "source_files": [u_source, v_source], "confidence": data.get("confidence", "EXTRACTED"), "relation": relation, "why": "; ".join(reasons) if reasons else "cross-file semantic connection", @@ -341,7 +342,7 @@ def _cross_community_surprises( return [] if G.number_of_nodes() > 5000: return [] - betweenness = nx.edge_betweenness_centrality(G) + betweenness = G.edge_betweenness() top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] result = [] for (u, v), score in top_edges: @@ -374,21 +375,11 @@ def _cross_community_surprises( relation = data.get("relation", "") if relation in ("imports", "imports_from", "contains", "method"): continue - # This edge crosses community boundaries - interesting confidence = data.get("confidence", "EXTRACTED") - src_id = data.get("_src", u) - if src_id not in G.nodes: - src_id = u - tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: - tgt_id = v surprises.append({ - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), - "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), - ], + "source": G.nodes[u].get("label", u), + "target": G.nodes[v].get("label", v), + "source_files": [G.nodes[u].get("source_file", ""), G.nodes[v].get("source_file", "")], "confidence": confidence, "relation": relation, "note": f"Bridges community {cid_u} → community {cid_v}", @@ -442,8 +433,10 @@ def suggest_questions( # 2. Bridge nodes (high betweenness) → cross-cutting concern questions if G.number_of_edges() > 0: - k = min(100, G.number_of_nodes()) if G.number_of_nodes() > 1000 else None - betweenness = nx.betweenness_centrality(G, k=k, seed=42) + # Node betweenness via FalkorDB's built-in algo.betweenness (exact, whole + # graph). The old nx path sampled k sources with seed=42 on large graphs; + # the built-in computes exactly, so rankings are at least as accurate. + betweenness = G.node_betweenness() # Top bridge nodes that are NOT file-level hubs bridges = sorted( [(n, s) for n, s in betweenness.items() @@ -649,9 +642,9 @@ def _endpoint_source_file(node_id: str) -> str: src_file = attrs.get("source_file", "") return src_file if isinstance(src_file, str) else "" - # Step 1: Build a directed file-level graph from import/re-export edges. + # Step 1: Build a directed file-level edge list from import/re-export edges. # IMPORTANT: resolve endpoints using source_file only; never infer from label/id. - file_graph = nx.DiGraph() + file_edges: set[tuple[str, str]] = set() for u, v, data in G.edges(data=True): rel = data.get("relation", "") @@ -665,33 +658,34 @@ def _endpoint_source_file(node_id: str) -> str: u_file = _endpoint_source_file(u) v_file = _endpoint_source_file(v) - # Works for both DiGraph and Graph inputs: - # orient edge from edge.source_file endpoint to the opposite endpoint. + # Orient edge from edge.source_file endpoint to the opposite endpoint. if u_file == src_file_attr: tgt_file = v_file elif v_file == src_file_attr: tgt_file = u_file else: - # Fallback: if source endpoint cannot be matched exactly, - # still treat edge.source_file as source and pick the opposite endpoint - # only if one endpoint has a real source_file. tgt_file = v_file if v_file and v_file != src_file_attr else u_file if not tgt_file: continue - file_graph.add_edge(src_file_attr, tgt_file) + file_edges.add((src_file_attr, tgt_file)) - if not file_graph.edges(): + if not file_edges: return [] - # Step 2: Find simple cycles, bounded by length. + # Step 2: Find simple cycles, bounded by length, via the simpleCycles UDF. + # Self-loops (a file importing itself) are length-1 cycles; the UDF only + # enumerates multi-node cycles, so we surface self-loops here directly. cycles: list[list[str]] = [] - for cycle in nx.simple_cycles(file_graph): + self_loops = sorted({a for a, b in file_edges if a == b}) + multi_edges = [(a, b) for a, b in file_edges if a != b] + for f in self_loops: + cycles.append([f]) + for cycle in G.simple_cycles(sorted(multi_edges), max_cycle_length): if len(cycle) <= max_cycle_length: cycles.append(cycle) if len(cycles) >= top_n * 10: - # Stop early to avoid combinatorial explosion break # Step 3: Sort by length (shortest = tightest coupling), then deduplicate. diff --git a/graphify/benchmark.py b/graphify/benchmark.py index eabade292..341d15860 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -3,11 +3,10 @@ import json import sys from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph from graphify.build import edge_data from graphify.serve import _query_terms +from graphify.store import open_store _CHARS_PER_TOKEN = 4 # standard approximation @@ -98,13 +97,9 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(Path(graph_path)) - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) + resolved = Path(graph_path) + out_dir = resolved.parent if resolved.suffix else resolved + G = open_store(out_dir, create=False) if corpus_words is None: # Rough estimate: each node label is ~3 words, plus source context diff --git a/graphify/build.py b/graphify/build.py index 1e040420e..61cec4fb1 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -27,7 +27,7 @@ import sys import unicodedata from pathlib import Path -import networkx as nx +from .store import GraphStore, DEFAULT_URI, graph_name_for from .validate import validate_extraction @@ -51,6 +51,14 @@ } +def _search_norm_label(label: str) -> str: + """Diacritic-stripped, lowercased label for in-engine search prefiltering. + Must match serve._strip_diacritics(label).lower() so Cypher CONTAINS prefilters + agree with the Python scoring tiers.""" + nfkd = unicodedata.normalize("NFKD", label or "") + return "".join(c for c in nfkd if not unicodedata.combining(c)).lower() + + def _normalize_id(s: str) -> str: r"""Normalize an ID string the same way extract._make_id does. @@ -83,34 +91,40 @@ def _norm_source_file(p: str | None, root: str | None = None) -> str | None: return p -def edge_data(G: nx.Graph, u: str, v: str) -> dict: - """Return one edge attribute dict for (u, v), tolerating MultiGraph. +def edge_data(G, u: str, v: str) -> dict: + """Return one edge attribute dict for (u, v). - For MultiGraph/MultiDiGraph there can be multiple parallel edges; - this returns the first one (sufficient for callers that only need - relation/confidence for rendering). Fixes #796. + FalkorDB stores parallel edges natively; ``G[u][v]`` returns the first + edge's attributes, which is sufficient for callers that only need + relation/confidence for rendering. Fixes #796. """ - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return next(iter(raw.values()), {}) - return raw - + return G[u][v] -def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: - """Return every edge attribute dict for (u, v); always a list.""" - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return list(raw.values()) - return [raw] +def edge_datas(G, u: str, v: str) -> list[dict]: + """Return edge attribute dict(s) for (u, v); always a list.""" + return [G[u][v]] -def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: - """Build a NetworkX graph from an extraction dict. - directed=True produces a DiGraph that preserves edge direction (source→target). - directed=False (default) produces an undirected Graph for backward compatibility. +def build_from_json( + extraction: dict, + *, + directed: bool = False, + root: str | Path | None = None, + store: GraphStore | None = None, + graph_name: str = "graphify", + uri: str = DEFAULT_URI, +) -> GraphStore: + """Build a FalkorDB-backed graph (GraphStore) from an extraction dict. + + Edges are always stored in their native source→target orientation. When + ``directed`` is False (default) a reverse-direction duplicate of the same + node pair + relation is collapsed to the first-seen edge, matching the old + undirected nx.Graph semantics; when True both directions are kept. root: if given, absolute source_file paths from semantic subagents are made relative to root so all nodes share a consistent path key (#932). + store: build into this GraphStore (cleared first); otherwise a new one is + created for ``graph_name`` at ``uri``. """ _root = str(Path(root).resolve()) if root else None # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. @@ -150,12 +164,22 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat real_errors = [e for e in errors if "does not match any node id" not in e] if real_errors: print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr) - G: nx.Graph = nx.DiGraph() if directed else nx.Graph() + # Collect node attributes in extraction order (last write wins per id), the + # same idempotent-overwrite semantics nx.add_node had. + node_attrs: dict[str, dict] = {} + node_order: list[str] = [] for node in extraction.get("nodes", []): if "source_file" in node: node["source_file"] = _norm_source_file(node["source_file"], _root) - G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) - node_set = set(G.nodes()) + nid = node["id"] + if nid not in node_attrs: + node_order.append(nid) + attrs = {k: v for k, v in node.items() if k != "id"} + # Stored searchable label so query/explain can prefilter candidates in + # the engine (Cypher CONTAINS) instead of scanning every node in Python. + attrs["norm_label"] = _search_norm_label(str(attrs.get("label", ""))) + node_attrs[nid] = attrs + node_set = set(node_attrs) # #1145: merge semantic ghost-duplicate nodes into AST nodes. # When AST and semantic extractors emit different IDs for the same symbol @@ -166,7 +190,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat _loc_nodes: dict[tuple[str, str], str] = {} # (basename, label) -> AST node id _noloc_nodes: dict[tuple[str, str], str] = {} # (basename, label) -> semantic node id for nid in node_set: - attrs = G.nodes[nid] + attrs = node_attrs[nid] label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) basename = Path(sf).name if sf else "" @@ -175,7 +199,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if attrs.get("source_location"): _loc_nodes[(basename, label)] = nid for nid in node_set: - attrs = G.nodes[nid] + attrs = node_attrs[nid] label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) basename = Path(sf).name if sf else "" @@ -190,9 +214,9 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat ast_id = _loc_nodes.get(key) if ast_id is not None: _ghost_remap[sem_id] = ast_id - # Remove ghost nodes from the graph; edges will be re-pointed via norm_to_id. + # Remove ghost nodes; edges will be re-pointed via norm_to_id. for ghost_id in _ghost_remap: - G.remove_node(ghost_id) + node_attrs.pop(ghost_id, None) node_set.discard(ghost_id) # Normalized ID map: lets edges survive when the LLM generates IDs with @@ -203,6 +227,9 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat for ghost_id, canonical_id in _ghost_remap.items(): norm_to_id[_normalize_id(ghost_id)] = canonical_id norm_to_id[ghost_id] = canonical_id + edge_items: list[tuple[str, str, dict]] = [] + seen_exact: set[tuple] = set() # (src, tgt, relation) — collapse exact directed dups + seen_pairs: dict[tuple, tuple[str, str]] = {} # (unordered pair, relation) -> first src/tgt # Iterate edges in a deterministic order. The graph is undirected and stores # direction in _src/_tgt; when two edges collapse onto the same node pair the # last write wins, so an unstable iteration order flips _src/_tgt run-to-run @@ -245,32 +272,46 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".hpp": "cpp", ".rb": "rb", ".php": "php", ".cs": "cs", ".swift": "swift", ".lua": "lua", } - src_ext = Path(G.nodes[src].get("source_file") or "").suffix.lower() - tgt_ext = Path(G.nodes[tgt].get("source_file") or "").suffix.lower() + src_ext = Path(node_attrs[src].get("source_file") or "").suffix.lower() + tgt_ext = Path(node_attrs[tgt].get("source_file") or "").suffix.lower() if src_ext and tgt_ext and _LANG_FAMILY.get(src_ext) != _LANG_FAMILY.get(tgt_ext): continue - # Preserve original edge direction - undirected graphs lose it otherwise, - # causing display functions to show edges backwards. - attrs["_src"] = src - attrs["_tgt"] = tgt - # When the graph is undirected and the same node pair appears twice with - # the same relation but opposite directions (e.g. a `calls` b and b `calls` a), - # nx.Graph collapses them into one edge. The deterministic sort above means - # the lexicographically-later direction would systematically overwrite the - # earlier one's _src/_tgt, silently flipping the surviving edge's caller - # and callee. First-seen direction wins instead — drop the redundant - # reverse-direction duplicate so the original direction is preserved (#1061). - if not G.is_directed() and G.has_edge(src, tgt): - existing = edge_data(G, src, tgt) - if existing.get("relation") == attrs.get("relation") and ( - existing.get("_src") == tgt and existing.get("_tgt") == src - ): + # Edges are stored DIRECTED in their native source→target orientation, so + # direction survives without the old _src/_tgt markers. When the graph is + # undirected (default) and the same node pair appears again with the same + # relation (in either direction), collapse to the first-seen edge so the + # original direction wins, matching the old nx.Graph behaviour (#1061). + # Collapse exact directed duplicates always (same as nx.DiGraph / the old + # MERGE upsert) so the fresh CREATE path doesn't emit duplicate edges. + exact_key = (src, tgt, attrs.get("relation")) + if exact_key in seen_exact: + continue + seen_exact.add(exact_key) + if not directed: + pair_key = (frozenset((src, tgt)), attrs.get("relation")) + if pair_key in seen_pairs: continue - G.add_edge(src, tgt, **attrs) + seen_pairs[pair_key] = (src, tgt) + edge_items.append((src, tgt, attrs)) + + if store is None: + store = GraphStore(graph_name=graph_name, uri=uri, directed=True) + store.clear() + # Fresh build into a cleared graph: ids are unique and there are no existing + # edges, so CREATE (no MERGE existence-check) is correct and much faster. + store.add_nodes_from([(nid, node_attrs[nid]) for nid in node_order if nid in node_attrs], fresh=True) + store.add_edges_from(edge_items, fresh=True) hyperedges = extraction.get("hyperedges", []) if hyperedges: - G.graph["hyperedges"] = hyperedges - return G + store.graph["hyperedges"] = hyperedges + store.save_meta() + # Warm + persist the hub-threshold (p99 degree) so query traversals don't + # recompute it on every invocation. + try: + store._hub_threshold() + except Exception: + pass + return store def build( @@ -280,7 +321,10 @@ def build( dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: + store: GraphStore | None = None, + graph_name: str = "graphify", + uri: str = DEFAULT_URI, +) -> GraphStore: """Merge multiple extraction results into one graph. directed=True produces a DiGraph that preserves edge direction (source→target). @@ -295,7 +339,6 @@ def build( results before semantic results so semantic labels take precedence, or reverse the order if you prefer AST source_location precision to win. """ - from graphify.dedup import deduplicate_entities combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} for ext in extractions: combined["nodes"].extend(ext.get("nodes", [])) @@ -304,11 +347,16 @@ def build( combined["input_tokens"] += ext.get("input_tokens", 0) combined["output_tokens"] += ext.get("output_tokens", 0) if dedup and combined["nodes"]: + # Imported lazily so dedup=False callers don't require the datasketch dep. + from graphify.dedup import deduplicate_entities + combined["nodes"], combined["edges"] = deduplicate_entities( combined["nodes"], combined["edges"], communities={}, dedup_llm_backend=dedup_llm_backend, ) - return build_from_json(combined, directed=directed, root=root) + return build_from_json( + combined, directed=directed, root=root, store=store, graph_name=graph_name, uri=uri + ) def _norm_label(label: str) -> str: @@ -365,42 +413,38 @@ def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dic def build_merge( new_chunks: list[dict], - graph_path: str | Path = "graphify-out/graph.json", + graph_name: str = "graphify", prune_sources: list[str] | None = None, *, + uri: str = DEFAULT_URI, directed: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: - """Load existing graph.json, merge new chunks into it, and save back. +) -> GraphStore: + """Merge new chunks into the existing FalkorDB graph and persist. Never replaces - only grows (or prunes deleted-file nodes via prune_sources). - Safe to call repeatedly: existing nodes and edges are preserved. - root: if given, absolute source_file paths in new_chunks are made relative (#932). + Safe to call repeatedly: existing nodes and edges are preserved (pulled back + out of the store, combined with the new chunks, de-duplicated as a whole, and + rebuilt). root: absolute source_file paths in new_chunks are made relative (#932). """ - graph_path = Path(graph_path) - if graph_path.exists(): - # Read JSON directly instead of going through node_link_graph(). - # The latter rebuilds an undirected nx.Graph and then enumerating - # edges() yields endpoints based on node insertion order, which - # silently flips directional edges (e.g. `calls`) when the callee - # was inserted before the caller. The _src/_tgt direction-preserving - # attrs are popped before saving in export.py, so going through the - # NetworkX round-trip loses direction permanently (#760). - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(graph_path) - data = json.loads(graph_path.read_text(encoding="utf-8")) - links_key = "links" if "links" in data else "edges" - existing_nodes = list(data.get("nodes", [])) - existing_edges = list(data.get(links_key, [])) - base = [{"nodes": existing_nodes, "edges": existing_edges}] - else: - existing_nodes = [] - base = [] + store = GraphStore(graph_name=graph_name, uri=uri, directed=True) + # Pull the existing graph back out of FalkorDB as an extraction-shaped chunk. + # Edges are stored in true direction, so (u, v) are the real source/target. + existing_nodes = [dict(attrs, id=nid) for nid, attrs in store.nodes(data=True)] + existing_edges = [] + for u, v, a in store.edges(data=True): + e = {k: val for k, val in a.items()} + e["source"], e["target"] = u, v + existing_edges.append(e) + base = [{"nodes": existing_nodes, "edges": existing_edges}] if existing_nodes else [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) + G = build( + all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, + root=root, store=store, graph_name=graph_name, uri=uri, + ) # Prune nodes and edges from deleted source files if prune_sources: @@ -452,7 +496,7 @@ def build_merge( # Safety check: refuse to shrink the graph silently (#479) # Skip when dedup or prune_sources is active — shrinkage is intentional there. - if graph_path.exists() and not dedup and not prune_sources: + if existing_nodes and not dedup and not prune_sources: existing_n = len(existing_nodes) new_n = G.number_of_nodes() if new_n < existing_n: @@ -464,24 +508,28 @@ def build_merge( return G -def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: - """Return a copy of G with all node IDs prefixed with repo_tag::. +def prefix_graph_for_global(G, repo_tag: str, target: GraphStore) -> GraphStore: + """Copy G's nodes/edges into `target` with all node IDs prefixed repo_tag::. - Labels are preserved unchanged (for display). A 'local_id' attribute - is added to each node so the original ID can be recovered. Edges are - rewritten to match the new prefixed IDs. The 'repo' attribute is set - on every node. + Labels are preserved unchanged (for display). A 'local_id' attribute is added + so the original ID can be recovered, and 'repo' is set on every node. Edges + are rewritten to the prefixed IDs. """ - relabel = {n: f"{repo_tag}::{n}" for n in G.nodes} - H = nx.relabel_nodes(G, relabel, copy=True) - for node, data in H.nodes(data=True): - data["repo"] = repo_tag - data.setdefault("local_id", node.split("::", 1)[1]) - return H - - -def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int: - """Remove all nodes tagged with repo_tag from G in-place. Returns count removed.""" - to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag] - G.remove_nodes_from(to_remove) - return len(to_remove) + def _pfx(n: str) -> str: + return f"{repo_tag}::{n}" + + nodes = [] + for nid, data in G.nodes(data=True): + attrs = dict(data) + attrs["repo"] = repo_tag + attrs.setdefault("local_id", nid) + nodes.append((_pfx(nid), attrs)) + edges = [(_pfx(u), _pfx(v), dict(a)) for u, v, a in G.edges(data=True)] + target.add_nodes_from(nodes) + target.add_edges_from(edges) + return target + + +def prune_repo_from_graph(G: GraphStore, repo_tag: str) -> int: + """Remove all nodes tagged with repo_tag from G. Returns count removed.""" + return G.prune_repo(repo_tag) diff --git a/graphify/callflow_html.py b/graphify/callflow_html.py index 6195adb98..f0b837d50 100644 --- a/graphify/callflow_html.py +++ b/graphify/callflow_html.py @@ -218,33 +218,29 @@ def normalize_edge(raw: dict, index: int) -> dict | None: def _node_link_payload(data: dict) -> tuple[list, list] | None: - """Read current graphify graph.json via NetworkX's node-link parser.""" + """Parse a node-link graph.json dict directly (no NetworkX).""" if not isinstance(data.get("nodes"), list): return None - if not isinstance(data.get("links"), list) and not isinstance(data.get("edges"), list): - return None - - try: - from networkx.readwrite import json_graph - - try: - graph = json_graph.node_link_graph(data, edges="links") - except TypeError: - graph = json_graph.node_link_graph(data) - except Exception: + links = data.get("links") + if not isinstance(links, list): + links = data.get("edges") + if not isinstance(links, list): return None nodes = [] - for node_id, attrs in graph.nodes(data=True): - node = dict(attrs) - node["id"] = node_id - nodes.append(node) + for n in data["nodes"]: + if not isinstance(n, dict): + continue + nodes.append(dict(n)) edges = [] - for index, (source, target, attrs) in enumerate(graph.edges(data=True), 1): - edge = dict(attrs) - edge["source"] = edge.get("_src", edge.get("source", source)) - edge["target"] = edge.get("_tgt", edge.get("target", target)) + for index, e in enumerate(links, 1): + if not isinstance(e, dict): + continue + edge = dict(e) + # Edges are stored in native direction; _src/_tgt fall back to source/target. + edge["source"] = edge.get("_src", edge.get("source")) + edge["target"] = edge.get("_tgt", edge.get("target")) edge.setdefault("id", f"edge_{index}") edges.append(edge) return nodes, edges diff --git a/graphify/cluster.py b/graphify/cluster.py index 82f063d21..e9d7b452d 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -1,80 +1,19 @@ -"""Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores.""" +"""Community detection on the FalkorDB-backed GraphStore. Runs Louvain via a +server-side JS UDF (graphify_algos.louvain), splits oversized/low-cohesion +communities, and returns stable community IDs with cohesion scores.""" from __future__ import annotations -import contextlib -import inspect -import io -import json -import sys -import networkx as nx -def _suppress_output(): - """Context manager to suppress stdout/stderr during library calls. +def _partition(G, resolution: float = 1.0) -> dict[str, int]: + """Run Louvain community detection. Returns {node_id: community_id}. - graspologic's leiden() emits ANSI escape sequences (progress bars, - colored warnings) that corrupt PowerShell 5.1's scroll buffer on - Windows (see issue #19). Redirecting stdout/stderr to devnull during - the call prevents this without losing any graphify output. - """ - return contextlib.redirect_stdout(io.StringIO()) - - -def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: - """Run community detection. Returns {node_id: community_id}. - - Tries Leiden (graspologic) first — best quality. - Falls back to Louvain (built into networkx) if graspologic is not installed. + Delegates to the server-side `graphify_algos.louvain` UDF (deterministic: + sorted node order, no RNG). `G` is a GraphStore or a subgraph view, both of + which expose `louvain_partition(resolution)`. - resolution > 1.0 → more, smaller communities. - resolution < 1.0 → fewer, larger communities. - - Output from graspologic is suppressed to prevent ANSI escape codes - from corrupting terminal scroll buffers on Windows PowerShell 5.1. + resolution > 1.0 → more, smaller communities; < 1.0 → fewer, larger. """ - stable = nx.Graph() - stable.add_nodes_from(sorted(G.nodes(), key=str)) - edge_rows = sorted( - G.edges(data=True), - key=lambda row: ( - str(row[0]), - str(row[1]), - json.dumps(row[2], sort_keys=True, ensure_ascii=False, default=str), - ), - ) - for src, tgt, attrs in edge_rows: - stable.add_edge(src, tgt, **attrs) - - try: - from graspologic.partition import leiden - lsig = inspect.signature(leiden).parameters - kwargs: dict = {} - if "random_seed" in lsig: - kwargs["random_seed"] = 42 - if "trials" in lsig: - kwargs["trials"] = 1 - if "resolution" in lsig: - kwargs["resolution"] = resolution - # Suppress graspologic output to prevent ANSI escape codes from - # corrupting PowerShell 5.1 scroll buffer (issue #19) - old_stderr = sys.stderr - try: - sys.stderr = io.StringIO() - with _suppress_output(): - result = leiden(stable, **kwargs) - finally: - sys.stderr = old_stderr - return result - except ImportError: - pass - - # Fallback: networkx louvain (available since networkx 2.7). - # Inspect kwargs to stay compatible across NetworkX versions — max_level - # was added in a later release and prevents hangs on large sparse graphs. - kwargs: dict = {"seed": 42, "threshold": 1e-4, "resolution": resolution} - if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: - kwargs["max_level"] = 10 - communities = nx.community.louvain_communities(stable, **kwargs) - return {node: cid for cid, nodes in enumerate(communities) for node in nodes} + return G.louvain_partition(resolution=resolution) _MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index 4d8abe294..aa35d9026 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -9,8 +9,6 @@ from pathlib import Path from typing import Any -import networkx as nx - _SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") _TYPE_TUPLE_RE = re.compile(r"set\[tuple\[(?P[^\]]+)\]\]") @@ -226,7 +224,7 @@ def diagnose_extraction( post_build_node_count: int | None = None try: graph_input = deepcopy(extraction) - graph: nx.Graph = build_from_json(graph_input, directed=directed, root=root) + graph = build_from_json(graph_input, directed=directed, root=root) graph_type = type(graph).__name__ post_build_edge_count = graph.number_of_edges() post_build_node_count = graph.number_of_nodes() diff --git a/graphify/export.py b/graphify/export.py index 05b3fea23..2cf6cb3e7 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -10,8 +10,6 @@ from collections import Counter from datetime import date from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph from graphify.security import sanitize_label from graphify.analyze import _node_community_map from graphify.build import edge_data @@ -503,26 +501,35 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, pass # unreadable existing file — proceed with write node_community = _node_community_map(communities) - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - for node in data["nodes"]: - node["community"] = node_community.get(node["id"]) - node["norm_label"] = _strip_diacritics(node.get("label", "")).lower() - for link in data["links"]: - if "confidence_score" not in link: - conf = link.get("confidence", "EXTRACTED") - link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) - # Restore original edge direction. Undirected NetworkX storage may - # canonicalize endpoint order, flipping `calls` and other directional - # edges in graph.json. The build path stashes the true endpoints in - # _src/_tgt for exactly this purpose (#563). - true_src = link.pop("_src", None) - true_tgt = link.pop("_tgt", None) - if true_src is not None and true_tgt is not None: - link["source"] = true_src - link["target"] = true_tgt + # FalkorDB is the source of truth: persist community ids onto the stored + # nodes. graph.json is now a derived export artifact built from the store. + if communities and hasattr(G, "set_communities"): + G.set_communities(communities) + + nodes = [] + for nid, attrs in G.nodes(data=True): + nd = {k: v for k, v in attrs.items() if not k.startswith("_")} + nd["id"] = nid + nd["community"] = node_community.get(nid) + nd["norm_label"] = _strip_diacritics(nd.get("label", "")).lower() + nodes.append(nd) + # Edges are stored in native source→target direction, so no _src/_tgt restore. + links = [] + for u, v, attrs in G.edges(data=True): + ld = {k: val for k, val in attrs.items() if not k.startswith("_")} + ld["source"] = u + ld["target"] = v + if "confidence_score" not in ld: + conf = ld.get("confidence", "EXTRACTED") + ld["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) + links.append(ld) + data = { + "directed": True, + "multigraph": False, + "graph": {}, + "nodes": nodes, + "links": links, + } data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) commit = built_at_commit if built_at_commit is not None else _git_head() if commit: @@ -644,20 +651,24 @@ def to_html( if node_limit is not None: # Build aggregated community meta-graph from collections import Counter as _Counter - import networkx as _nx + from graphify.store import MemGraph print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - meta = _nx.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) + meta_nodes = [ + (str(cid), {"label": (community_labels or {}).get(cid, f"Community {cid}")}) + for cid, members in communities.items() + ] edge_counts = _Counter() for u, v in G.edges(): cu, cv = node_to_community.get(u), node_to_community.get(v) if cu is not None and cv is not None and cu != cv: edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, - relation=f"{w} cross-community edges", confidence="AGGREGATED") + meta_edges = [ + (str(cu), str(cv), {"weight": w, "relation": f"{w} cross-community edges", + "confidence": "AGGREGATED"}) + for (cu, cv), w in edge_counts.items() + ] + meta = MemGraph(meta_nodes, meta_edges) if meta.number_of_nodes() <= 1: print("Single community - aggregated view not useful. Skipping graph.html.") return @@ -1415,20 +1426,118 @@ def to_graphml( Community IDs are written as a node attribute so Gephi can colour by community. Edge confidence (EXTRACTED/INFERRED/AMBIGUOUS) is preserved as an edge attribute. """ - H = G.copy() node_community = _node_community_map(communities) - for node_id in H.nodes(): - H.nodes[node_id]["community"] = node_community.get(node_id, -1) - # Drop internal markers (e.g. the AST-provenance "_origin" tag, #1116, and - # the "_src"/"_tgt" direction markers) — they are persistence/runtime details, - # not graph data, and should not leak into the exported file. - for _, attrs in H.nodes(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - for _, _, attrs in H.edges(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - nx.write_graphml(H, output_path) + + # Collect nodes/edges (dropping internal "_"-prefixed markers) and the set of + # attribute keys, so we can declare GraphML elements up front. + node_rows = [] + node_keys: dict[str, str] = {} # attr name -> graphml type + for nid, attrs in G.nodes(data=True): + clean = {k: v for k, v in attrs.items() if not k.startswith("_") and k != "id"} + clean["community"] = node_community.get(nid, -1) + node_rows.append((nid, clean)) + for k, v in clean.items(): + node_keys.setdefault(k, _graphml_type(v)) + + edge_rows = [] + edge_keys: dict[str, str] = {} + for u, v, attrs in G.edges(data=True): + clean = {k: val for k, val in attrs.items() if not k.startswith("_")} + edge_rows.append((u, v, clean)) + for k, val in clean.items(): + edge_keys.setdefault(k, _graphml_type(val)) + + def esc(x): + return _html.escape(str(x), quote=True) + + lines = [ + '', + '', + ] + for name, gtype in node_keys.items(): + lines.append(f' ') + for name, gtype in edge_keys.items(): + lines.append(f' ') + lines.append(' ') + for nid, attrs in node_rows: + lines.append(f' ') + for k, v in attrs.items(): + lines.append(f' {esc(v)}') + lines.append(" ") + for i, (u, v, attrs) in enumerate(edge_rows): + lines.append(f' ') + for k, val in attrs.items(): + lines.append(f' {esc(val)}') + lines.append(" ") + lines.append(" ") + lines.append("") + Path(output_path).write_text("\n".join(lines), encoding="utf-8") + + +def _graphml_type(value) -> str: + if isinstance(value, bool): + return "boolean" + if isinstance(value, int): + return "long" + if isinstance(value, float): + return "double" + return "string" + + +def _spring_layout(G, seed: int = 42, iterations: int = 60) -> dict: + """Deterministic Fruchterman-Reingold layout (replaces nx.spring_layout). + + Pure-Python, seeded for reproducibility. Returns {node_id: (x, y)} in roughly + [-1, 1]. Good enough for the static SVG overview; not performance-critical. + """ + import math + import random as _random + + nodes = list(G.nodes()) + n = len(nodes) + if n == 0: + return {} + rng = _random.Random(seed) + pos = {nid: [rng.uniform(-1, 1), rng.uniform(-1, 1)] for nid in nodes} + if n == 1: + return {nodes[0]: (0.0, 0.0)} + adj = {nid: set() for nid in nodes} + for u, v in G.edges(): + if u in adj and v in adj: + adj[u].add(v) + adj[v].add(u) + k = math.sqrt(1.0 / n) + t = 0.1 + for _ in range(iterations): + disp = {nid: [0.0, 0.0] for nid in nodes} + for i in range(n): + a = nodes[i] + for j in range(i + 1, n): + b = nodes[j] + dx = pos[a][0] - pos[b][0] + dy = pos[a][1] - pos[b][1] + dist = math.hypot(dx, dy) or 0.01 + rep = (k * k) / dist + ux, uy = dx / dist, dy / dist + disp[a][0] += ux * rep; disp[a][1] += uy * rep + disp[b][0] -= ux * rep; disp[b][1] -= uy * rep + for a in nodes: + for b in adj[a]: + if a >= b: + continue + dx = pos[a][0] - pos[b][0] + dy = pos[a][1] - pos[b][1] + dist = math.hypot(dx, dy) or 0.01 + att = (dist * dist) / k + ux, uy = dx / dist, dy / dist + disp[a][0] -= ux * att; disp[a][1] -= uy * att + disp[b][0] += ux * att; disp[b][1] += uy * att + for nid in nodes: + dlen = math.hypot(*disp[nid]) or 0.01 + pos[nid][0] += (disp[nid][0] / dlen) * min(dlen, t) + pos[nid][1] += (disp[nid][1] / dlen) * min(dlen, t) + t = max(t * 0.95, 0.01) + return {nid: (p[0], p[1]) for nid, p in pos.items()} def to_svg( @@ -1459,13 +1568,14 @@ def to_svg( ax.set_facecolor("#1a1a2e") ax.axis("off") - pos = nx.spring_layout(G, seed=42, k=2.0 / (G.number_of_nodes() ** 0.5 + 1)) + pos = _spring_layout(G, seed=42) degree = dict(G.degree()) max_deg = max(degree.values(), default=1) or 1 - node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in G.nodes()] - node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in G.nodes()] + nodes_list = list(G.nodes()) + node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in nodes_list] + node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in nodes_list] # Draw edges - dashed for non-EXTRACTED for u, v, data in G.edges(data=True): @@ -1477,11 +1587,13 @@ def to_svg( ax.plot([x0, x1], [y0, y1], color="#aaaaaa", linewidth=0.8, linestyle=style, alpha=alpha, zorder=1) - nx.draw_networkx_nodes(G, pos, ax=ax, node_color=node_colors, - node_size=node_sizes, alpha=0.9) - nx.draw_networkx_labels(G, pos, ax=ax, - labels={n: G.nodes[n].get("label", n) for n in G.nodes()}, - font_size=7, font_color="white") + xs = [pos[n][0] for n in nodes_list] + ys = [pos[n][1] for n in nodes_list] + ax.scatter(xs, ys, s=node_sizes, c=node_colors, alpha=0.9, zorder=2, edgecolors="none") + for n in nodes_list: + x, y = pos[n] + ax.annotate(str(G.nodes[n].get("label", n)), (x, y), fontsize=7, + color="white", ha="center", va="center", zorder=3) # Legend if community_labels: diff --git a/graphify/global_graph.py b/graphify/global_graph.py index c6310f94b..38b163849 100644 --- a/graphify/global_graph.py +++ b/graphify/global_graph.py @@ -4,12 +4,12 @@ import sys from datetime import datetime, timezone from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph as _jg + +from .store import GraphStore, DEFAULT_URI, open_store _GLOBAL_DIR = Path.home() / ".graphify" -_GLOBAL_GRAPH = _GLOBAL_DIR / "global-graph.json" _GLOBAL_MANIFEST = _GLOBAL_DIR / "global-manifest.json" +_GLOBAL_NAME = "graphify_global" def _load_manifest() -> dict: @@ -26,32 +26,18 @@ def _save_manifest(manifest: dict) -> None: _GLOBAL_MANIFEST.write_text(json.dumps(manifest, indent=2), encoding="utf-8") -def _load_global_graph() -> nx.Graph: - if _GLOBAL_GRAPH.exists(): - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(_GLOBAL_GRAPH) - data = json.loads(_GLOBAL_GRAPH.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - return _jg.node_link_graph(data, edges="links") - except TypeError: - return _jg.node_link_graph(data) - return nx.Graph() - - -def _save_global_graph(G: nx.Graph) -> None: - _GLOBAL_DIR.mkdir(parents=True, exist_ok=True) - try: - data = _jg.node_link_data(G, edges="links") - except TypeError: - data = _jg.node_link_data(G) - _GLOBAL_GRAPH.write_text(json.dumps(data, indent=2), encoding="utf-8") +def _load_global_graph(uri: str = DEFAULT_URI) -> GraphStore: + """The global graph is a dedicated named FalkorDB graph.""" + return GraphStore(graph_name=_GLOBAL_NAME, uri=uri) -def _file_hash(path: Path) -> str: +def _store_content_hash(G) -> str: + """Stable content hash of a store's nodes + edges (replaces file hashing).""" h = hashlib.sha256() - h.update(path.read_bytes()) + for nid, attrs in G.nodes(data=True): + h.update(json.dumps([nid, attrs], sort_keys=True, default=str).encode("utf-8")) + for u, v, attrs in G.edges(data=True): + h.update(json.dumps([u, v, attrs], sort_keys=True, default=str).encode("utf-8")) return h.hexdigest()[:16] @@ -61,13 +47,17 @@ def global_add(source_path: Path, repo_tag: str) -> dict: Returns a summary dict with keys: repo_tag, nodes_added, nodes_removed, skipped. Skipped=True means the source graph hasn't changed since last add. """ - from graphify.build import prefix_graph_for_global, prune_repo_from_graph + from graphify.build import prune_repo_from_graph - if not source_path.exists(): - raise FileNotFoundError(f"graph not found: {source_path}") + # Load source graph from its FalkorDB store (source_path is the legacy + # graph.json location; its parent dir holds the FalkorDB pointer). + out_dir = source_path.parent if source_path.suffix else source_path + src_G = open_store(out_dir, create=False) + if src_G.number_of_nodes() == 0: + raise FileNotFoundError(f"graph not found for: {source_path}") manifest = _load_manifest() - src_hash = _file_hash(source_path) + src_hash = _store_content_hash(src_G) existing = manifest["repos"].get(repo_tag, {}) existing_path = existing.get("source_path", "") @@ -81,20 +71,6 @@ def global_add(source_path: Path, repo_tag: str) -> dict: if existing.get("source_hash") == src_hash: return {"repo_tag": repo_tag, "nodes_added": 0, "nodes_removed": 0, "skipped": True} - # Load source graph - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(source_path) - data = json.loads(source_path.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - src_G = _jg.node_link_graph(data, edges="links") - except TypeError: - src_G = _jg.node_link_graph(data) - - # Prefix IDs for cross-project isolation - prefixed = prefix_graph_for_global(src_G, repo_tag) - # Load global graph and prune stale nodes for this repo G = _load_global_graph() removed = prune_repo_from_graph(G, repo_tag) @@ -105,27 +81,38 @@ def global_add(source_path: Path, repo_tag: str) -> dict: for n, d in G.nodes(data=True) if not d.get("source_file") and d.get("label") } - nodes_to_skip = set() - for node, data in prefixed.nodes(data=True): + # Prefix source IDs for cross-project isolation, skipping external dups. + prefixed_nodes = [] + edge_skip: set[str] = set() + n_prefixed = 0 + for nid, data in src_G.nodes(data=True): + n_prefixed += 1 + pid = f"{repo_tag}::{nid}" if not data.get("source_file") and data.get("label") in external_labels: - nodes_to_skip.add(node) - - # Compose: add prefixed nodes (except deduplicated externals) into global graph - for node, data in prefixed.nodes(data=True): - if node not in nodes_to_skip: - G.add_node(node, **data) - for u, v, data in prefixed.edges(data=True): - if u not in nodes_to_skip and v not in nodes_to_skip: - G.add_edge(u, v, **data) - - added = prefixed.number_of_nodes() - len(nodes_to_skip) - _save_global_graph(G) - + edge_skip.add(pid) + continue + attrs = dict(data) + attrs["repo"] = repo_tag + attrs.setdefault("local_id", nid) + prefixed_nodes.append((pid, attrs)) + prefixed_edges = [] + n_src_edges = 0 + for u, v, data in src_G.edges(data=True): + n_src_edges += 1 + pu, pv = f"{repo_tag}::{u}", f"{repo_tag}::{v}" + if pu in edge_skip or pv in edge_skip: + continue + prefixed_edges.append((pu, pv, dict(data))) + + G.add_nodes_from(prefixed_nodes) + G.add_edges_from(prefixed_edges) + + added = len(prefixed_nodes) manifest["repos"][repo_tag] = { "added_at": datetime.now(timezone.utc).isoformat(), "source_path": str(source_path.resolve()), "node_count": added, - "edge_count": prefixed.number_of_edges(), + "edge_count": n_src_edges, "source_hash": src_hash, } _save_manifest(manifest) @@ -143,7 +130,6 @@ def global_remove(repo_tag: str) -> int: G = _load_global_graph() removed = prune_repo_from_graph(G, repo_tag) - _save_global_graph(G) del manifest["repos"][repo_tag] _save_manifest(manifest) @@ -155,5 +141,6 @@ def global_list() -> dict: return _load_manifest().get("repos", {}) -def global_path() -> Path: - return _GLOBAL_GRAPH +def global_path() -> str: + """Name of the global FalkorDB graph (replaces the old global-graph.json path).""" + return _GLOBAL_NAME diff --git a/graphify/graphjson.py b/graphify/graphjson.py new file mode 100644 index 000000000..56bed0cb2 --- /dev/null +++ b/graphify/graphjson.py @@ -0,0 +1,114 @@ +"""Plain-dict helpers for the node-link graph.json export artifact. + +FalkorDB is the source of truth, but graphify still emits a ``graph.json`` +node-link snapshot (and the git merge-driver / ``merge-graphs`` commands operate +on those artifacts). These helpers union and prefix node-link dicts directly, +without NetworkX. + +A node-link dict looks like:: + + {"directed": true, "multigraph": false, "graph": {}, + "nodes": [{"id": ..., ...}, ...], + "links": [{"source": ..., "target": ..., "relation": ...}, ...]} +""" +from __future__ import annotations + +import json +from pathlib import Path + + +def load_node_link(path, *, max_bytes: int | None = None) -> dict: + """Read a node-link graph.json, normalizing the edges key to ``links``.""" + p = Path(path) + if max_bytes is not None: + size = p.stat().st_size + if size > max_bytes: + raise RuntimeError(f"graph.json {p} is {size} bytes, exceeds {max_bytes}-byte cap") + data = json.loads(p.read_text(encoding="utf-8")) + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + data.setdefault("nodes", []) + data.setdefault("links", []) + return data + + +def _edge_key(e: dict): + return (e.get("source"), e.get("target"), e.get("relation")) + + +def merge_node_link(graphs: list[dict]) -> dict: + """Union several node-link dicts. Later graphs win on node/edge attr conflicts.""" + nodes: dict = {} + edges: dict = {} + for g in graphs: + for n in g.get("nodes", []): + nid = n.get("id") + if nid is None: + continue + nodes[nid] = {**nodes.get(nid, {}), **n} + for e in g.get("links", []): + edges[_edge_key(e)] = e + return { + "directed": True, + "multigraph": False, + "graph": {}, + "nodes": list(nodes.values()), + "links": list(edges.values()), + } + + +def prefix_node_link(data: dict, repo_tag: str) -> dict: + """Prefix every node id with ``repo_tag::`` and rewrite edge endpoints. + + Mirrors build.prefix_graph_for_global for the JSON artifact: sets ``repo`` and + ``local_id`` on each node so the original id is recoverable. + """ + def pfx(x): + return f"{repo_tag}::{x}" + + nodes = [] + for n in data.get("nodes", []): + nid = n.get("id") + nn = dict(n) + nn["id"] = pfx(nid) + nn["repo"] = repo_tag + nn.setdefault("local_id", nid) + nodes.append(nn) + links = [] + for e in data.get("links", []): + ee = dict(e) + ee["source"] = pfx(e.get("source")) + ee["target"] = pfx(e.get("target")) + links.append(ee) + return {"directed": True, "multigraph": False, "graph": {}, "nodes": nodes, "links": links} + + +def to_node_link(G) -> dict: + """Build a node-link dict from a GraphStore/MemGraph (drops internal `_` keys).""" + nodes = [] + for nid, attrs in G.nodes(data=True): + nd = {k: v for k, v in attrs.items() if not k.startswith("_")} + nd["id"] = nid + nodes.append(nd) + links = [] + for u, v, attrs in G.edges(data=True): + ld = {k: val for k, val in attrs.items() if not k.startswith("_")} + ld["source"] = u + ld["target"] = v + links.append(ld) + return { + "directed": True, + "multigraph": False, + "graph": {}, + "nodes": nodes, + "links": links, + "hyperedges": getattr(G, "graph", {}).get("hyperedges", []), + } + + +def node_count(data: dict) -> int: + return len(data.get("nodes", [])) + + +def edge_count(data: dict) -> int: + return len(data.get("links", [])) diff --git a/graphify/hooks.py b/graphify/hooks.py index d704eea20..b0f7cf32e 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -183,9 +183,9 @@ def _detached_launch(rebuild_body: str) -> str: # Auto-rebuilds the knowledge graph after each commit (code files only, no LLM needed). # Installed by: graphify hook install -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. +# Deterministic clustering: string-keyed set/dict iteration is randomized +# per-process by PYTHONHASHSEED, so community assignments churn run-to-run. +# Pinning it makes graphify-out reproducible. export PYTHONHASHSEED=0 # Skip during rebase/merge/cherry-pick to avoid blocking --continue with unstaged changes @@ -228,9 +228,9 @@ def _detached_launch(rebuild_body: str) -> str: # Auto-rebuilds the knowledge graph (code only) when switching branches. # Installed by: graphify hook install -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. +# Deterministic clustering: string-keyed set/dict iteration is randomized +# per-process by PYTHONHASHSEED, so community assignments churn run-to-run. +# Pinning it makes graphify-out reproducible. export PYTHONHASHSEED=0 PREV_HEAD=$1 diff --git a/graphify/multigraph_compat.py b/graphify/multigraph_compat.py deleted file mode 100644 index 7ac62e275..000000000 --- a/graphify/multigraph_compat.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Runtime compatibility probe for Graphify MultiDiGraph mode. - -Verifies that the current NetworkX runtime supports the behaviors a future -opt-in --multigraph build will rely on. The probe is BEHAVIOR-based, not -version-based — both NX 3.4.2 (Py 3.10 lane) and NX 3.6.1+ (Py 3.11+ lane) -pass. The probe result is cached for the process lifetime via lru_cache. - -No call sites added yet; downstream multigraph PRs will gate on -require_multigraph_capabilities() before enabling MDG mode. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from functools import lru_cache -import sys -from typing import Any - -import networkx as nx -from networkx.readwrite import json_graph - - -@dataclass(frozen=True) -class CapabilityCheck: - name: str - ok: bool - detail: str - - -@dataclass(frozen=True) -class MultigraphCapabilityResult: - python_version: str - networkx_version: str - checks: tuple[CapabilityCheck, ...] - - @property - def ok(self) -> bool: - return all(check.ok for check in self.checks) - - @property - def failed(self) -> tuple[CapabilityCheck, ...]: - return tuple(check for check in self.checks if not check.ok) - - def error_message(self) -> str: - if self.ok: - return ( - "Graphify MultiDiGraph capability probe passed " - f"(Python {self.python_version}, NetworkX {self.networkx_version})." - ) - failed = "; ".join(f"{check.name}: {check.detail}" for check in self.failed) - return ( - "error: --multigraph requires NetworkX keyed MultiDiGraph node-link " - "round-trip support. " - f"Detected Python {self.python_version}, NetworkX {self.networkx_version}. " - f"Failed capability check(s): {failed}. " - "Default simple graph mode remains available." - ) - - -def _check(name: str, func: Callable[[], bool | str]) -> CapabilityCheck: - try: - detail = func() - except Exception as exc: - return CapabilityCheck(name, False, f"{type(exc).__name__}: {exc}") - if detail is True: - return CapabilityCheck(name, True, "ok") - if isinstance(detail, str): - return CapabilityCheck(name, False, detail) - return CapabilityCheck(name, False, f"unexpected result {detail!r}") - - -def _build_probe_graph() -> nx.MultiDiGraph: - graph = nx.MultiDiGraph() - graph.add_node("a", label="A") - graph.add_node("b", label="B") - graph.add_edge("a", "b", key="calls:a.py:L1", relation="calls", source_file="a.py") - graph.add_edge("a", "b", key="imports:a.py:L2", relation="imports", source_file="a.py") - return graph - - -def _probe_keyed_parallel_edges() -> bool | str: - graph = _build_probe_graph() - if not graph.is_multigraph() or not graph.is_directed(): - return f"probe graph type was {type(graph).__name__}" - if graph.number_of_edges("a", "b") != 2: - return f"expected 2 keyed parallel edges, got {graph.number_of_edges('a', 'b')}" - keys = set(graph["a"]["b"].keys()) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if keys != expected: - return f"expected keys {sorted(expected)}, got {sorted(keys)}" - return True - - -def _probe_node_link_round_trip() -> bool | str: - graph = _build_probe_graph() - data = json_graph.node_link_data(graph, edges="links") - if data.get("multigraph") is not True: - return f"serialized multigraph flag was {data.get('multigraph')!r}" - if data.get("directed") is not True: - return f"serialized directed flag was {data.get('directed')!r}" - links = data.get("links") - if not isinstance(links, list) or len(links) != 2: - length = 0 if not isinstance(links, list) else len(links) - return f"serialized links length was {length}" - serialized_keys: set[str] = set() - for edge in links: - if isinstance(edge, dict): - edge_key = edge.get("key") - if isinstance(edge_key, str): - serialized_keys.add(edge_key) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if serialized_keys != expected: - return f"serialized keys {sorted(serialized_keys)} did not match {sorted(expected)}" - loaded = json_graph.node_link_graph(data, edges="links") - if not isinstance(loaded, nx.MultiDiGraph): - return f"round-trip graph type was {type(loaded).__name__}" - if loaded.number_of_edges("a", "b") != 2: - return f"round-trip edge count was {loaded.number_of_edges('a', 'b')}" - loaded_keys = set(loaded["a"]["b"].keys()) - if loaded_keys != expected: - return f"round-trip keys {sorted(loaded_keys)} did not match {sorted(expected)}" - return True - - -def _probe_duplicate_key_overwrite_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("x", "y", key="same", marker="first") - graph.add_edge("x", "y", key="same", marker="second") - edges = list(graph.edges(keys=True, data=True)) - if len(edges) != 1: - return f"expected one edge after duplicate-key add, got {len(edges)}" - if edges[0][3].get("marker") != "second": - return f"expected second attr overwrite, got {edges[0][3].get('marker')!r}" - return True - - -def _probe_reserved_key_attr_rejected() -> bool | str: - """Verify the Python language guarantee that NetworkX add_edge inherits. - - Python forbids passing the same keyword argument twice — once explicitly - and once via **kwargs. This probe confirms that protection still applies - to nx.MultiDiGraph.add_edge: a future loader that builds attrs from JSON - will be reliably protected from accidentally setting `key` via attrs while - also passing `key=` explicitly. - - The probe always passes on any Python 3.x version. Its purpose is to - document the invariant explicitly in the probe suite so that if a future - Python version relaxes this rule (extremely unlikely), the probe surfaces - the regression. - """ - graph = nx.MultiDiGraph() - attrs: dict[str, Any] = {"key": "attr-key", "relation": "calls"} - try: - graph.add_edge("a", "b", key="schema-key", **attrs) - except TypeError: - return True - return "add_edge accepted duplicate key keyword and attr; loader must not rely on this" - - -def _probe_remove_edges_from_two_tuple_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("a", "b", key="one") - graph.add_edge("a", "b", key="two") - graph.remove_edges_from([("a", "b")]) - remaining = graph.number_of_edges("a", "b") - if remaining != 1: - return f"expected one remaining edge after two-tuple removal, got {remaining}" - return True - - -def _probe_to_undirected_preserves_multigraph_type() -> bool | str: - graph = _build_probe_graph() - undirected = graph.to_undirected() - undirected_view = graph.to_undirected(as_view=True) - if not isinstance(undirected, nx.MultiGraph): - return f"to_undirected() returned {type(undirected).__name__}" - if not isinstance(undirected_view, nx.MultiGraph): - return f"to_undirected(as_view=True) returned {type(undirected_view).__name__}" - return True - - -@lru_cache(maxsize=1) -def probe_multigraph_capabilities() -> MultigraphCapabilityResult: - checks = ( - _check("keyed_parallel_edges", _probe_keyed_parallel_edges), - _check("node_link_edges_links_round_trip", _probe_node_link_round_trip), - _check("duplicate_key_overwrite_semantics", _probe_duplicate_key_overwrite_semantics), - _check("reserved_key_attr_rejected", _probe_reserved_key_attr_rejected), - _check( - "remove_edges_from_two_tuple_semantics", - _probe_remove_edges_from_two_tuple_semantics, - ), - _check( - "to_undirected_preserves_multigraph_type", - _probe_to_undirected_preserves_multigraph_type, - ), - ) - return MultigraphCapabilityResult( - python_version=( - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ), - networkx_version=nx.__version__, - checks=checks, - ) - - -def require_multigraph_capabilities() -> MultigraphCapabilityResult: - result = probe_multigraph_capabilities() - if not result.ok: - raise RuntimeError(result.error_message()) - return result diff --git a/graphify/report.py b/graphify/report.py index f0210897d..14190e35a 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -2,7 +2,6 @@ from __future__ import annotations import re from datetime import date -import networkx as nx def _safe_community_name(label: str) -> str: diff --git a/graphify/serve.py b/graphify/serve.py index a349f511b..3a682bedb 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -5,8 +5,7 @@ import re import sys from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph +from .store import open_store, MemGraph from graphify.security import sanitize_label, check_graph_file_size_cap from graphify.build import edge_data @@ -16,29 +15,24 @@ _jieba = None -def _load_graph(graph_path: str) -> nx.Graph: +def _load_graph(graph_path: str): + """Open the FalkorDB-backed graph for the output dir containing `graph_path`. + + `graph_path` is the legacy graph.json location; its parent directory holds the + FalkorDB pointer (falkordb.json). Errors out if no graph has been built yet. + """ try: resolved = Path(graph_path).resolve() - if resolved.suffix != ".json": - raise ValueError(f"Graph path must be a .json file, got: {graph_path!r}") - if not resolved.exists(): - raise FileNotFoundError(f"Graph file not found: {resolved}") - check_graph_file_size_cap(resolved) - safe = resolved - data = json.loads(safe.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - data = {**data, "directed": True} - try: - return json_graph.node_link_graph(data, edges="links") - except TypeError: - return json_graph.node_link_graph(data) + out_dir = resolved.parent if resolved.suffix else resolved + store = open_store(out_dir, create=False) + if store.number_of_nodes() == 0: + raise FileNotFoundError( + f"No graph found for {out_dir} (FalkorDB graph empty). Re-run /graphify to build." + ) + return store except (ValueError, FileNotFoundError) as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1) - except json.JSONDecodeError as exc: - print(f"error: graph.json is corrupted ({exc}). Re-run /graphify to rebuild.", file=sys.stderr) - sys.exit(1) def _communities_from_graph(G: nx.Graph) -> dict[int, list[str]]: @@ -119,16 +113,17 @@ def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]: N = G.number_of_nodes() or 1 uncached = [t for t in terms if t not in cache] if uncached: - df: dict[str, int] = {t: 0 for t in uncached} - for _, data in G.nodes(data=True): - norm_label = ( - data.get("norm_label") or _strip_diacritics(data.get("label") or "") - ).lower() - for t in uncached: - if t in norm_label: - df[t] += 1 + if hasattr(G, "doc_freqs"): + df = G.doc_freqs(uncached) # in-engine count per term, no full scan + else: + df = {t: 0 for t in uncached} + for _, data in G.nodes(data=True): + norm_label = (data.get("norm_label") or _strip_diacritics(data.get("label") or "")).lower() + for t in uncached: + if t in norm_label: + df[t] += 1 for t in uncached: - cache[t] = math.log(1 + N / (1 + df[t])) + cache[t] = math.log(1 + N / (1 + df.get(t, 0))) return {t: cache.get(t, math.log(1 + N)) for t in terms} @@ -141,16 +136,36 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: # Weight the full-query bonus by the rarest constituent term so a specific # multi-word label still outweighs common-token noise; floor at 1.0. joined_w = max((idf.get(t, 1.0) for t in norm_terms), default=1.0) - for nid, data in G.nodes(data=True): - norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() + # FalkorDB-native: a node scores > 0 only if some term matches its label or + # source, so prefilter those candidates in-engine instead of scanning every + # node in Python. Non-matching nodes would contribute 0 anyway — equivalent. + if hasattr(G, "search_nodes"): + _seen: set = set() + _items = [] + for _c in G.search_nodes(norm_terms + ([joined] if joined else [])): + if _c["id"] in _seen: + continue + _seen.add(_c["id"]) + _items.append((_c["id"], _c["label"], + _c["norm_label"] or _strip_diacritics(_c["label"]).lower(), + _c["source_file"])) + else: + _items = [ + (nid, data.get("label") or "", + data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower(), + data.get("source_file") or "") + for nid, data in G.nodes(data=True) + ] + _label_by_id = {it[0]: it[1] for it in _items} + for nid, _label, norm_label, _src in _items: bare_label = norm_label.rstrip("()") # Tokenized form of the label (punctuation stripped, same transform as the # query). norm_label may still carry punctuation like ':' or '-', which a # tokenized query can never equal; comparing token-joined forms on both # sides makes "uoce: dehumidifier driver" match query "uoce dehumidifier # driver". - label_tokens = " ".join(_search_tokens(data.get("label") or "")) - source = (data.get("source_file") or "").lower() + label_tokens = " ".join(_search_tokens(_label)) + source = (_src or "").lower() score = 0.0 # Full-query tier: a multi-word query that equals (or prefixes) the whole # label must dominate the per-token bag-of-words sums below, so `path`/ @@ -184,7 +199,7 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: scored.append((score, nid)) # Sort by score desc; break ties toward the shorter label so a concise exact # match beats a longer superset that happens to share the same score. - scored.sort(key=lambda s: (-s[0], len(G.nodes[s[1]].get("label") or s[1]), s[1])) + scored.sort(key=lambda s: (-s[0], len(_label_by_id.get(s[1]) or s[1]), s[1])) return scored @@ -293,25 +308,36 @@ def _resolve_context_filters(question: str, explicit_filters: list[str] | None = return [], None +def _confidence_distribution(G) -> tuple[dict, int]: + """(counts_by_confidence, total_edges). Uses the store's in-engine aggregation + when available so graph_stats/audit never stream every edge into Python; falls + back to an nx/MemGraph edge scan otherwise.""" + if hasattr(G, "confidence_counts"): + counts = G.confidence_counts() + else: + counts: dict = {} + for _, _, d in G.edges(data=True): + k = d.get("confidence", "EXTRACTED") + counts[k] = counts.get(k, 0) + 1 + return counts, (sum(counts.values()) or 1) + + def _filter_graph_by_context(G: nx.Graph, context_filters: list[str] | None) -> nx.Graph: filters = set(_normalize_context_filters(context_filters)) if not filters: return G - H = G.__class__() - H.add_nodes_from(G.nodes(data=True)) - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - for u, v, key, data in G.edges(keys=True, data=True): - if data.get("context") in filters: - H.add_edge(u, v, key=key, **data) - else: - for u, v, data in G.edges(data=True): - if data.get("context") in filters: - H.add_edge(u, v, **data) - return H - - -def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: - # Compute hub threshold: nodes above this degree are not expanded as transit. + nodes = list(G.nodes(data=True)) + edges = [(u, v, data) for u, v, data in G.edges(data=True) if data.get("context") in filters] + return MemGraph(nodes, edges, directed=G.is_directed()) + + +def _bfs(G: nx.Graph, start_nodes: list[str], depth: int, contexts: list[str] | None = None) -> tuple[set[str], list[tuple]]: + # FalkorDB-native: traverse in the engine, one query per level (no graph load). + # `contexts` scopes the walk to matching edge-contexts in-engine, so context + # filtering never copies the graph into Python. + if hasattr(G, "bfs"): + return G.bfs(start_nodes, depth, contexts=contexts) + # In-memory fallback (context-filtered MemGraph / nx). Compute hub threshold: # p99 of degree distribution, floored at 50 to avoid over-blocking small graphs. degrees = [G.degree(n) for n in G.nodes()] if degrees: @@ -340,7 +366,9 @@ def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], lis return visited, edges_seen -def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: +def _dfs(G: nx.Graph, start_nodes: list[str], depth: int, contexts: list[str] | None = None) -> tuple[set[str], list[tuple]]: + if hasattr(G, "dfs"): + return G.dfs(start_nodes, depth, contexts=contexts) degrees = [G.degree(n) for n in G.nodes()] if degrees: degrees_sorted = sorted(degrees) @@ -375,10 +403,40 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu char_budget = token_budget * 3 lines = [] seed_set = set(seeds or []) - ordered = [n for n in (seeds or []) if n in nodes] + \ - sorted(nodes - seed_set, key=lambda n: G.degree(n), reverse=True) + total_nodes = len(nodes) + + # FalkorDB-native: fetch node attrs+degree and edge attrs in batched queries + # instead of ~per-element scoped lookups. Fallback for MemGraph/nx. + if hasattr(G, "subgraph_render_data"): + # Only fetch attrs for as many nodes as the char budget can possibly + # render (each NODE line is >~20 chars). Without this cap a hub query + # whose BFS reaches tens of thousands of nodes would pull every node's + # attributes out of the engine only to discard all but the top slice. + render_limit = max(50, char_budget // 20) + nattrs, eattrs, total_nodes = G.subgraph_render_data( + nodes, [(u, v) for u, v in edges], + limit=render_limit, seeds=list(seeds or []), + ) + kept = set(nattrs) + def _n(nid): + return nattrs.get(nid, {}) + def _deg(nid): + return nattrs.get(nid, {}).get("degree", 0) + def _e(u, v): + return eattrs.get((u, v), {}) + else: + kept = set(nodes) + def _n(nid): + return G.nodes[nid] + def _deg(nid): + return G.degree(nid) + def _e(u, v): + return G[u][v] + + ordered = [n for n in (seeds or []) if n in kept] + \ + sorted(kept - seed_set, key=_deg, reverse=True) for nid in ordered: - d = G.nodes[nid] + d = _n(nid) # Every LLM-derived field passes through sanitize_label before being # concatenated into MCP tool output (F-010): an attacker who controls a # corpus document can otherwise inject ANSI escapes, fake graphify-out @@ -392,27 +450,32 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu ) lines.append(line) for u, v in edges: - if u in nodes and v in nodes: - raw = G[u][v] - d = next(iter(raw.values()), {}) if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)) else raw + if u in kept and v in kept: + d = _e(u, v) context = d.get("context") context_suffix = f" context={sanitize_label(str(context))}" if context else "" line = ( - f"EDGE {sanitize_label(G.nodes[u].get('label', u))} " + f"EDGE {sanitize_label(_n(u).get('label', u))} " f"--{sanitize_label(str(d.get('relation', '')))} " f"[{sanitize_label(str(d.get('confidence', '')))}{context_suffix}]--> " - f"{sanitize_label(G.nodes[v].get('label', v))}" + f"{sanitize_label(_n(v).get('label', v))}" ) lines.append(line) output = "\n".join(lines) + # shown_nodes = NODE lines that survive the char budget; cut_count counts + # against the FULL subgraph (total_nodes), so nodes dropped by the render + # limit are reported as cut too — never silently omitted. if len(output) > char_budget: cut_at = output[:char_budget].rfind("\n") cut_at = cut_at if cut_at > 0 else char_budget - total_nodes = sum(1 for l in lines if l.startswith("NODE ")) - shown_nodes = output[:cut_at].count("\nNODE ") + (1 if output.startswith("NODE ") else 0) - cut_count = total_nodes - shown_nodes + output = output[:cut_at] + shown_nodes = output.count("\nNODE ") + (1 if output.startswith("NODE ") else 0) + else: + shown_nodes = sum(1 for l in lines if l.startswith("NODE ")) + cut_count = total_nodes - shown_nodes + if cut_count > 0: output = ( - output[:cut_at] + output + f"\n... (truncated — {cut_count} more nodes cut by ~{token_budget}-token budget." f" Narrow with context_filter=['call'] or use get_node for a specific symbol)" ) @@ -434,8 +497,17 @@ def _query_graph_text( if not start_nodes: return "No matching nodes found." resolved_filters, filter_source = _resolve_context_filters(question, context_filters) - traversal_graph = _filter_graph_by_context(G, resolved_filters) - nodes, edges = _dfs(traversal_graph, start_nodes, depth) if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth) + if resolved_filters and hasattr(G, "bfs"): + # Store-backed: apply the context filter inside the traversal query and + # render against the live store — never materialize a filtered copy. + traversal_graph = G + contexts = resolved_filters + else: + # nx/MemGraph fallback: build the filtered in-memory view as before. + traversal_graph = _filter_graph_by_context(G, resolved_filters) + contexts = None + nodes, edges = (_dfs(traversal_graph, start_nodes, depth, contexts) + if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth, contexts)) header_parts = [ f"Traversal: {mode.upper()} depth={depth}", f"Start: {[G.nodes[n].get('label', n) for n in start_nodes]}", @@ -444,7 +516,7 @@ def _query_graph_text( header_parts.append(f"Context: {', '.join(resolved_filters)} ({filter_source})") header_parts.append(f"{len(nodes)} nodes found") header = " | ".join(header_parts) + "\n\n" - return header + _subgraph_to_text(traversal_graph, nodes, edges, token_budget) + return header + _subgraph_to_text(traversal_graph, nodes, edges, token_budget, seeds=start_nodes) def _find_node(G: nx.Graph, label: str) -> list[str]: @@ -456,11 +528,24 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: term = " ".join(_search_tokens(label)) if not term: return [] + # FalkorDB-native: prefilter candidates in-engine (no full scan); tier in Python. + if hasattr(G, "search_nodes"): + seen: set[str] = set() + rows = [] + for c in G.search_nodes([term]): + if c["id"] in seen: + continue + seen.add(c["id"]) + rows.append((c["id"], c["norm_label"] or _strip_diacritics(c["label"]).lower())) + else: + rows = [ + (nid, d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower()) + for nid, d in G.nodes(data=True) + ] exact: list[str] = [] prefix: list[str] = [] substring: list[str] = [] - for nid, d in G.nodes(data=True): - norm_label = d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower() + for nid, norm_label in rows: bare_label = norm_label.rstrip("()") nid_lower = nid.lower() if term == norm_label or term == bare_label or term == nid_lower: @@ -713,12 +798,18 @@ def _tool_query_graph(arguments: dict) -> str: return result def _tool_get_node(arguments: dict) -> str: - label = arguments["label"].lower() - matches = [(nid, d) for nid, d in G.nodes(data=True) - if label in (d.get("label") or "").lower() or label == nid.lower()] + label = arguments["label"] + # Use the candidate prefilter (in-engine for the store) instead of scanning + # every node, then fetch just the matched node's attrs+degree. + matches = _find_node(G, label) if not matches: return f"No node matching '{label}' found." - nid, d = matches[0] + nid = matches[0] + if hasattr(G, "node_detail"): + detail = G.node_detail(nid) + d, deg = detail if detail else ({}, 0) + else: + d, deg = G.nodes[nid], G.degree(nid) # Sanitise every LLM-derived field before concatenation (F-010). return "\n".join([ f"Node: {sanitize_label(d.get('label', nid))}", @@ -726,7 +817,7 @@ def _tool_get_node(arguments: dict) -> str: f" Source: {sanitize_label(str(d.get('source_file', '')))} {sanitize_label(str(d.get('source_location', '')))}", f" Type: {sanitize_label(str(d.get('file_type', '')))}", f" Community: {sanitize_label(str(d.get('community', '')))}", - f" Degree: {G.degree(nid)}", + f" Degree: {deg}", ]) def _tool_get_neighbors(arguments: dict) -> str: @@ -780,15 +871,14 @@ def _tool_god_nodes(arguments: dict) -> str: return "\n".join(lines) def _tool_graph_stats(_: dict) -> str: - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 + confs, total = _confidence_distribution(G) return ( f"Nodes: {G.number_of_nodes()}\n" f"Edges: {G.number_of_edges()}\n" f"Communities: {len(communities)}\n" - f"EXTRACTED: {round(confs.count('EXTRACTED')/total*100)}%\n" - f"INFERRED: {round(confs.count('INFERRED')/total*100)}%\n" - f"AMBIGUOUS: {round(confs.count('AMBIGUOUS')/total*100)}%\n" + f"EXTRACTED: {round(confs.get('EXTRACTED', 0)/total*100)}%\n" + f"INFERRED: {round(confs.get('INFERRED', 0)/total*100)}%\n" + f"AMBIGUOUS: {round(confs.get('AMBIGUOUS', 0)/total*100)}%\n" ) def _tool_shortest_path(arguments: dict) -> str: @@ -817,10 +907,9 @@ def _tool_shortest_path(arguments: dict) -> str: f"(top score {top:g}, runner-up {runner:g})" ) max_hops = int(arguments.get("max_hops", 8)) - try: - # Use undirected view for path-finding (works regardless of query src/tgt order) - path_nodes = nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) - except (nx.NetworkXNoPath, nx.NodeNotFound): + # Undirected shortest path (works regardless of query src/tgt order). + path_nodes = G.shortest_path(src_nid, tgt_nid, max_hops=max_hops) + if not path_nodes: return f"No path found between '{G.nodes[src_nid].get('label', src_nid)}' and '{G.nodes[tgt_nid].get('label', tgt_nid)}'." hops = len(path_nodes) - 1 if hops > max_hops: @@ -828,7 +917,7 @@ def _tool_shortest_path(arguments: dict) -> str: segments = [] for i in range(len(path_nodes) - 1): u, v = path_nodes[i], path_nodes[i + 1] - if G.has_edge(u, v): + if G.has_directed_edge(u, v): edata = edge_data(G, u, v) forward = True else: @@ -990,13 +1079,13 @@ async def read_resource(uri: AnyUrl) -> str: except Exception as exc: return f"Could not compute surprising connections: {exc}" if uri_str == "graphify://audit": - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 + confs, total = _confidence_distribution(G) + e, i, a = confs.get("EXTRACTED", 0), confs.get("INFERRED", 0), confs.get("AMBIGUOUS", 0) return ( f"Total edges: {total}\n" - f"EXTRACTED: {confs.count('EXTRACTED')} ({round(confs.count('EXTRACTED')/total*100)}%)\n" - f"INFERRED: {confs.count('INFERRED')} ({round(confs.count('INFERRED')/total*100)}%)\n" - f"AMBIGUOUS: {confs.count('AMBIGUOUS')} ({round(confs.count('AMBIGUOUS')/total*100)}%)\n" + f"EXTRACTED: {e} ({round(e/total*100)}%)\n" + f"INFERRED: {i} ({round(i/total*100)}%)\n" + f"AMBIGUOUS: {a} ({round(a/total*100)}%)\n" ) if uri_str == "graphify://questions": try: diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/amp/references/query.md +++ b/graphify/skills/amp/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/claw/references/query.md +++ b/graphify/skills/claw/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/codex/references/query.md +++ b/graphify/skills/codex/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/copilot/references/query.md +++ b/graphify/skills/copilot/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/droid/references/query.md +++ b/graphify/skills/droid/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/kilo/references/query.md +++ b/graphify/skills/kilo/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/kiro/references/query.md +++ b/graphify/skills/kiro/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/opencode/references/query.md +++ b/graphify/skills/opencode/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/pi/references/query.md +++ b/graphify/skills/pi/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/trae/references/query.md +++ b/graphify/skills/trae/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/vscode/references/query.md +++ b/graphify/skills/vscode/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md index 25b826f8e..304447582 100644 --- a/graphify/skills/windows/references/query.md +++ b/graphify/skills/windows/references/query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index f53974a66..66084ca1c 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/graphify/store.py b/graphify/store.py new file mode 100644 index 000000000..04bc6ec4a --- /dev/null +++ b/graphify/store.py @@ -0,0 +1,1251 @@ +"""FalkorDB-backed graph store — the single replacement for NetworkX. + +`GraphStore` wraps a named FalkorDB graph and exposes a NetworkX-shaped API so +the rest of graphify can treat it almost exactly like the `nx.Graph` it used to +pass around. FalkorDB is the source of truth and the compute engine: + +* **Writes / mutation** go straight to FalkorDB (batched `UNWIND ... MERGE`). +* **Algorithms** run server-side — built-in `algo.*` where they exist, and the + JS UDFs in ``graphify/udfs/graphify_algos.js`` (louvain, edge betweenness, + simple cycles) where they don't. +* **Reads** (the `G.nodes[id]`, `G.edges(data=True)`, `G.degree(n)` patterns used + in ~200 call sites) are served from a read cache that is bulk-loaded from + FalkorDB once via Cypher and invalidated on mutation. This keeps the hot + analysis loops fast (in-memory, like the old nx graph) without reimplementing + any algorithm in Python. + +Schema +------ +Every node has the base label ``:Entity`` plus a sanitized ``file_type`` label; +``id`` is the key and a unique index on ``:Entity(id)`` (created at init) makes +MERGE/MATCH fast. Edges are stored DIRECTED in original orientation with the +sanitized relation as the Cypher type and the raw relation kept as the +``relation`` property. Parallel edges are native, so the old ``_src``/``_tgt`` +markers are gone. Graph-level metadata (e.g. ``hyperedges``) lives on a singleton +``:GraphMeta`` node and is mirrored to the in-memory ``.graph`` dict. +""" +from __future__ import annotations + +import hashlib +import json +import re +import threading +from pathlib import Path +from urllib.parse import urlparse + +_UDF_LIB = "graphify_algos" +_UDF_SRC = Path(__file__).parent / "udfs" / "graphify_algos.js" +_udf_loaded_servers: set[str] = set() +_udf_lock = threading.Lock() + +DEFAULT_URI = "falkordb://localhost:6379" +_META_KEY = "__graphmeta__" + +# FalkorDB caps rows returned per query at RESULTSET_SIZE (default 10000), which +# silently truncates bulk reads on large graphs. We raise it at connect time so +# row-returning procedures (e.g. algo.betweenness) aren't truncated, and ALSO +# paginate the cache-load below at a page size safely under the default cap so +# the read cache is complete even if the config raise is rejected. +_RESULTSET_SIZE = 1_000_000_000 +_PAGE = 9000 +# Default read TIMEOUT is 1000ms, which the server-side UDFs (Louvain, edge +# betweenness) blow past on large graphs. Raise the server defaults at connect +# and also pass this per-query timeout on heavy READ calls (not writes — FalkorDB +# disallows timeouts on write queries). +_QUERY_TIMEOUT_MS = 600_000 +_server_configured: set[str] = set() + + +def _safe_label(label: str) -> str: + sanitized = re.sub(r"[^A-Za-z0-9_]", "", label or "") + return sanitized if sanitized else "Entity" + + +def _safe_rel(relation: str) -> str: + return re.sub(r"[^A-Z0-9_]", "_", (relation or "").upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" + + +def _scalar_props(data: dict) -> dict: + return { + k: v + for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + + +def graph_name_for(path: str | Path) -> str: + """Stable, key-safe FalkorDB graph name derived from a directory path.""" + resolved = str(Path(path).resolve()) + base = _safe_label(Path(resolved).name) or "graph" + digest = hashlib.sha1(resolved.encode("utf-8")).hexdigest()[:8] + return f"graphify_{base}_{digest}" + + +def pointer_path(out_dir: str | Path = "graphify-out") -> Path: + """Path of the FalkorDB pointer file that records the graph name + URI.""" + return Path(out_dir) / "falkordb.json" + + +def open_store( + out_dir: str | Path = "graphify-out", + uri: str = DEFAULT_URI, + *, + create: bool = True, + directed: bool = True, +) -> "GraphStore": + """Open the GraphStore for an output directory. + + The graph name + URI are recorded in ``/falkordb.json`` (the FalkorDB + replacement for the old ``graph.json`` location). When absent, the name is + derived deterministically from the project root (the parent of ``out_dir``). + """ + p = pointer_path(out_dir) + name = None + if p.exists(): + try: + cfg = json.loads(p.read_text(encoding="utf-8")) + name = cfg.get("graph_name") + uri = cfg.get("uri", uri) + except Exception: + name = None + if not name: + root = Path(out_dir).resolve().parent + name = graph_name_for(root) + if create: + Path(out_dir).mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps({"graph_name": name, "uri": uri}), encoding="utf-8") + return GraphStore(graph_name=name, uri=uri, directed=directed) + + +def _connect_lite(dbfile: str): + """Embedded FalkorDB Lite (redislite): run the engine in-process from an RDB + file — no external server. Opt-in via the GRAPHIFY_FALKORDB_LITE env var.""" + # redis 8.0.0 crashes enabling maintenance-notifications on the host-less + # unix-socket connection redislite uses; it is irrelevant for an embedded + # engine, so neutralize it. + import redis.connection as _rc + for _name in ("AbstractConnection", "Connection", "UnixDomainSocketConnection"): + _cls = getattr(_rc, _name, None) + if _cls and hasattr(_cls, "activate_maint_notifications_handling_if_enabled"): + _cls.activate_maint_notifications_handling_if_enabled = lambda self, **kw: None + import redislite + return redislite.FalkorDB(dbfilename=dbfile) + + +def _connect(uri: str, user: str | None = None, password: str | None = None): + import os + # Embedded FalkorDB Lite is selected by (in priority order) the + # GRAPHIFY_FALKORDB_LITE env var pointing at an RDB file, or a + # `falkordb-lite://` / `lite://` URI. Default server path otherwise. + _lite = os.environ.get("GRAPHIFY_FALKORDB_LITE") + if _lite: + return _connect_lite(_lite) + parsed = urlparse(uri if "://" in uri else f"redis://{uri}") + if parsed.scheme in ("falkordb-lite", "lite"): + return _connect_lite(parsed.path or None) + + try: + from falkordb import FalkorDB + except ImportError as e: # pragma: no cover + raise ImportError("falkordb SDK not installed. Run: pip install falkordb") from e + connect_user = parsed.username or (user if password else None) + connect_password = parsed.password or (password or None) + return FalkorDB( + host=parsed.hostname or "localhost", + port=parsed.port or 6379, + username=connect_user, + password=connect_password, + ) + + +# --------------------------------------------------------------------------- +# NetworkX-shaped views over the read cache +# --------------------------------------------------------------------------- +class _NodeView: + """Mimics nx ``G.nodes``: iterable, callable(data=), subscriptable [id].""" + + def __init__(self, store: "GraphStore"): + self._s = store + + def __call__(self, data=False): + self._s._ensure_cache() + items = [(nid, self._s._ncache[nid]) for nid in self._s._nsorted] + if data: + return iter(items) + return iter([nid for nid, _ in items]) + + def __iter__(self): + self._s._ensure_cache() + return iter(self._s._nsorted) + + def __getitem__(self, node_id): + self._s._ensure_cache() + return self._s._ncache[node_id] + + def __contains__(self, node_id): + self._s._ensure_cache() + return node_id in self._s._ncache + + def __len__(self): + self._s._ensure_cache() + return len(self._s._ncache) + + def get(self, node_id, default=None): + self._s._ensure_cache() + return self._s._ncache.get(node_id, default) + + +class _EdgeView: + """Mimics nx ``G.edges``: callable(nbunch=None, data=) and iterable.""" + + def __init__(self, store: "GraphStore"): + self._s = store + + def __call__(self, nbunch=None, data=False): + self._s._ensure_cache() + if nbunch is None: + rows = self._s._esorted + else: + if isinstance(nbunch, (str, bytes)): + wanted = {nbunch} + else: + wanted = set(nbunch) + rows = [e for e in self._s._esorted if e[0] in wanted or e[1] in wanted] + if data: + return iter([(u, v, dict(a)) for u, v, a in rows]) + return iter([(u, v) for u, v, _ in rows]) + + def __iter__(self): + self._s._ensure_cache() + return iter([(u, v) for u, v, _ in self._s._esorted]) + + def __getitem__(self, pair): + """nx-style ``G.edges[u, v]`` -> edge attrs (first match, either direction).""" + u, v = pair + self._s._ensure_cache() + for a, b, attrs in self._s._esorted: + if (a == u and b == v) or (a == v and b == u): + return attrs + raise KeyError(pair) + + +class _DegreeView: + """Mimics nx ``G.degree``: callable() -> items, callable(n) -> int.""" + + def __init__(self, store: "GraphStore"): + self._s = store + + def __call__(self, nbunch=None): + degs = self._s._degree_map() + if nbunch is None: + return list(degs.items()) + if isinstance(nbunch, (str, bytes)): + return degs.get(nbunch, 0) + return [(n, degs.get(n, 0)) for n in nbunch] + + def __getitem__(self, node_id): + return self._s._degree_map().get(node_id, 0) + + +class _SubgraphView: + """Lightweight subgraph supporting the operations cohesion scoring needs.""" + + def __init__(self, store: "GraphStore", node_ids): + self._s = store + self._ids = set(node_ids) + + def number_of_nodes(self): + return len(self._ids) + + def number_of_edges(self): + self._s._ensure_cache() + return sum(1 for u, v, _ in self._s._esorted if u in self._ids and v in self._ids) + + def is_directed(self): + return self._s.directed + + def to_undirected(self, as_view: bool = False): + return self + + def neighbors(self, node_id): + self._s._ensure_cache() + out = set() + for u, v, _ in self._s._esorted: + if u not in self._ids or v not in self._ids: + continue + if u == node_id: + out.add(v) + elif v == node_id: + out.add(u) + return iter(sorted(out)) + + def louvain_partition(self, resolution: float = 1.0) -> dict: + self._s._ensure_cache() + edges = [ + [u, v, float(a.get("weight", 1.0) or 1.0)] + for u, v, a in self._s._esorted + if u in self._ids and v in self._ids + ] + return self._s.run_louvain(edges, resolution) + + def nodes(self, data=False): + self._s._ensure_cache() + ids = sorted(self._ids) + if data: + return iter([(n, self._s._ncache.get(n, {})) for n in ids]) + return iter(ids) + + def edges(self, data=False): + self._s._ensure_cache() + rows = [(u, v, a) for u, v, a in self._s._esorted if u in self._ids and v in self._ids] + if data: + return iter([(u, v, dict(a)) for u, v, a in rows]) + return iter([(u, v) for u, v, _ in rows]) + + +class MemGraph: + """In-memory read-only graph with the same view API as GraphStore. + + Used for transient derived graphs (e.g. context-filtered subgraphs) that + must not touch FalkorDB. Built from explicit node/edge lists. + """ + + def __init__(self, nodes, edges, directed: bool = True): + # nodes: iterable of (id, attrs); edges: iterable of (u, v, attrs) + self._ncache = {nid: dict(a) for nid, a in nodes} + self._nsorted = sorted(self._ncache.keys(), key=str) + self._esorted = [(u, v, dict(a)) for u, v, a in edges] + self._esorted.sort(key=lambda e: (str(e[0]), str(e[1]), json.dumps(e[2], sort_keys=True, default=str))) + self.directed = directed + self._degmap = None + self.graph: dict = {} + + def _ensure_cache(self): + return + + def _degree_map(self): + if self._degmap is None: + degs = {nid: 0 for nid in self._ncache} + for u, v, _ in self._esorted: + degs[u] = degs.get(u, 0) + 1 + degs[v] = degs.get(v, 0) + 1 + self._degmap = degs + return self._degmap + + # views + nodes = property(lambda self: _NodeView(self)) + edges = property(lambda self: _EdgeView(self)) + degree = property(lambda self: _DegreeView(self)) + + def is_directed(self): + return self.directed + + def is_multigraph(self): + return False + + def to_undirected(self, as_view: bool = False): + return self + + def number_of_nodes(self): + return len(self._ncache) + + def number_of_edges(self): + return len(self._esorted) + + def __contains__(self, node_id): + return node_id in self._ncache + + def __iter__(self): + return iter(self._nsorted) + + def __len__(self): + return len(self._ncache) + + def __getitem__(self, node_id): + adj: dict = {} + for u, v, a in self._esorted: + if u == node_id: + adj.setdefault(v, a) + elif v == node_id: + adj.setdefault(u, a) + return adj + + def has_node(self, node_id): + return node_id in self._ncache + + def has_edge(self, u, v): + for a, b, _ in self._esorted: + if (a == u and b == v) or (a == v and b == u): + return True + return False + + def has_directed_edge(self, u, v): + return any(a == u and b == v for a, b, _ in self._esorted) + + def neighbors(self, node_id): + out = set() + for u, v, _ in self._esorted: + if u == node_id: + out.add(v) + elif v == node_id: + out.add(u) + return iter(sorted(out)) + + def successors(self, node_id): + return iter(sorted({v for u, v, _ in self._esorted if u == node_id})) + + def predecessors(self, node_id): + return iter(sorted({u for u, v, _ in self._esorted if v == node_id})) + + def in_edges(self, node_id, data=False): + rows = [(u, v, a) for u, v, a in self._esorted if v == node_id] + if data: + return [(u, v, dict(a)) for u, v, a in rows] + return [(u, v) for u, v, _ in rows] + + def subgraph(self, node_ids): + return _SubgraphView(self, node_ids) + + def shortest_path(self, src, tgt, max_hops: int = 8): + from collections import deque + + if src not in self._ncache or tgt not in self._ncache: + return None + adj: dict = {} + for u, v, _ in self._esorted: + adj.setdefault(u, set()).add(v) + adj.setdefault(v, set()).add(u) + prev = {src: None} + q = deque([(src, 0)]) + while q: + node, d = q.popleft() + if node == tgt: + path = [] + while node is not None: + path.append(node) + node = prev[node] + return list(reversed(path)) + if d >= max_hops: + continue + for nb in sorted(adj.get(node, ())): + if nb not in prev: + prev[nb] = node + q.append((nb, d + 1)) + return None + + +# --------------------------------------------------------------------------- +# Cache-free views for GraphStore: every access is a live FalkorDB query or a +# paginated stream. The graph is NEVER materialized in Python — the store holds +# no node/edge data, only a connection. +# --------------------------------------------------------------------------- +class _SNodeView: + def __init__(self, s): + self._s = s + + def __call__(self, data=False): + return self._s._iter_nodes(data=data) + + def __iter__(self): + return self._s._iter_nodes(data=False) + + def __getitem__(self, nid): + return self._s._node_attrs(nid) # raises KeyError if absent + + def __contains__(self, nid): + return self._s.has_node(nid) + + def __len__(self): + return self._s.number_of_nodes() + + def get(self, nid, default=None): + try: + return self._s._node_attrs(nid) + except KeyError: + return default + + +class _SEdgeView: + def __init__(self, s): + self._s = s + + def __call__(self, nbunch=None, data=False): + return self._s._iter_edges(nbunch=nbunch, data=data) + + def __iter__(self): + return self._s._iter_edges(nbunch=None, data=False) + + def __getitem__(self, pair): + u, v = pair + d = self._s._edge_attrs(u, v) + if d is None: + raise KeyError(pair) + return d + + +class _SDegreeView: + def __init__(self, s): + self._s = s + + def __call__(self, nbunch=None): + if nbunch is None: + return list(self._s._iter_degrees()) + if isinstance(nbunch, (str, bytes)): + return self._s._degree_one(nbunch) + return [(n, self._s._degree_one(n)) for n in nbunch] + + def __getitem__(self, nid): + return self._s._degree_one(nid) + + +class _SStoreSubgraph: + """Subgraph restricted to a node-id set; all ops run as scoped queries.""" + + def __init__(self, store, node_ids): + self._s = store + self._ids = list(node_ids) + + def number_of_nodes(self): + return len(self._ids) + + def number_of_edges(self): + if not self._ids: + return 0 + return self._s._rows( + "MATCH (a:Entity)-[r]-(b:Entity) WHERE a.id IN $ids AND b.id IN $ids " + "AND id(a) < id(b) RETURN count(r)", + {"ids": self._ids}, timeout=_QUERY_TIMEOUT_MS, + )[0][0] + + def louvain_partition(self, resolution: float = 1.0) -> dict: + if not self._ids: + return {} + edges = self._s._rows( + "MATCH (a:Entity)-[r]->(b:Entity) WHERE a.id IN $ids AND b.id IN $ids " + "RETURN a.id, b.id, coalesce(r.weight, 1.0)", + {"ids": self._ids}, timeout=_QUERY_TIMEOUT_MS, + ) + return self._s.run_louvain([[str(u), str(v), float(w)] for u, v, w in edges], resolution) + + def is_directed(self): + return self._s.directed + + def to_undirected(self, as_view: bool = False): + return self + + +class GraphStore: + """A named FalkorDB graph with a NetworkX-shaped API. + + Holds NO graph data in Python — the graph lives in FalkorDB. Every read is a + scoped query or a paginated stream; nothing is cached/materialized. + """ + + def __init__( + self, + graph_name: str = "graphify", + uri: str = DEFAULT_URI, + *, + user: str | None = None, + password: str | None = None, + directed: bool = True, + ): + self.graph_name = graph_name + self.uri = uri + self.directed = directed + self._db = _connect(uri, user, password) + self._g = self._db.select_graph(graph_name) + p = urlparse(uri if "://" in uri else "redis://" + uri) + self._server_key = f"{p.hostname}:{p.port or 6379}" + self.graph: dict = {} # graph-level metadata only (hyperedges) — NOT the graph + self._ensure_resultset_size() + self._ensure_schema() + self._ensure_udfs() + self._load_meta() + + # ---- views (nx-compatible, cache-free: each access is a live query) ----- + @property + def nodes(self): + return _SNodeView(self) + + @property + def edges(self): + return _SEdgeView(self) + + @property + def degree(self): + return _SDegreeView(self) + + # ---- streaming + scoped read helpers (no materialization) --------------- + def _stream(self, match_return: str, order_by: str, params: dict | None = None): + """Yield rows page-by-page (only one page in memory at a time).""" + skip = 0 + while True: + rows = self._rows( + f"{match_return} ORDER BY {order_by} SKIP {skip} LIMIT {_PAGE}", + params, timeout=_QUERY_TIMEOUT_MS, + ) + for r in rows: + yield r + if len(rows) < _PAGE: + break + skip += _PAGE + + def _iter_nodes(self, data=False): + for r in self._stream("MATCH (n:Entity) RETURN n.id, properties(n)", "id(n)"): + if data: + d = dict(r[1]); d.pop(_META_KEY, None) + yield (r[0], d) + else: + yield r[0] + + def _iter_edges(self, nbunch=None, data=False): + if nbunch is None: + gen = self._stream("MATCH (a:Entity)-[r]->(b:Entity) RETURN a.id, b.id, properties(r)", "id(r)") + else: + ids = [nbunch] if isinstance(nbunch, (str, bytes)) else list(nbunch) + gen = self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) WHERE a.id IN $ids OR b.id IN $ids " + "RETURN a.id, b.id, properties(r)", "id(r)", {"ids": ids}, + ) + for r in gen: + if data: + yield (r[0], r[1], dict(r[2])) + else: + yield (r[0], r[1]) + + def _iter_degrees(self): + for r in self._stream("MATCH (n:Entity) RETURN n.id, size((n)--())", "id(n)"): + yield (r[0], int(r[1])) + + def _node_attrs(self, nid): + rows = self._rows("MATCH (n:Entity {id:$id}) RETURN properties(n)", {"id": nid}, timeout=_QUERY_TIMEOUT_MS) + if not rows: + raise KeyError(nid) + d = dict(rows[0][0]); d.pop(_META_KEY, None) + return d + + def _edge_attrs(self, u, v): + rows = self._rows( + "MATCH (a:Entity {id:$u})-[r]-(b:Entity {id:$v}) RETURN properties(r) LIMIT 1", + {"u": u, "v": v}, timeout=_QUERY_TIMEOUT_MS, + ) + return dict(rows[0][0]) if rows else None + + def _degree_one(self, nid): + rows = self._rows( + "MATCH (n:Entity {id:$id}) OPTIONAL MATCH (n)-[r]-() RETURN count(r)", + {"id": nid}, timeout=_QUERY_TIMEOUT_MS, + ) + return rows[0][0] if rows else 0 + + # ---- low-level helpers -------------------------------------------------- + def query(self, cypher: str, params: dict | None = None, timeout: int | None = None): + if timeout is not None: + return self._g.query(cypher, params or {}, timeout=timeout) + return self._g.query(cypher, params or {}) + + def _rows(self, cypher: str, params: dict | None = None, timeout: int | None = None) -> list: + return self.query(cypher, params, timeout=timeout).result_set + + def _ensure_resultset_size(self) -> None: + """Raise the row cap and query timeouts so large-graph reads aren't + silently truncated (RESULTSET_SIZE) or killed (TIMEOUT).""" + with _udf_lock: + if self._server_key in _server_configured: + return + for key, val in ( + ("RESULTSET_SIZE", _RESULTSET_SIZE), + ("TIMEOUT_DEFAULT", _QUERY_TIMEOUT_MS), + ("TIMEOUT_MAX", _QUERY_TIMEOUT_MS), + ): + try: + self._db.connection.execute_command("GRAPH.CONFIG", "SET", key, str(val)) + except Exception: + pass # managed server may forbid it; per-query timeout + pagination cover us + _server_configured.add(self._server_key) + + def _ensure_schema(self) -> None: + try: + self._g.query("CREATE INDEX FOR (n:Entity) ON (n.id)") + except Exception: + pass + + def _ensure_udfs(self) -> None: + with _udf_lock: + if self._server_key in _udf_loaded_servers: + return + src = _UDF_SRC.read_text() + try: + self._db.connection.execute_command("GRAPH.UDF", "LOAD", "REPLACE", _UDF_LIB, src) + except Exception: + try: + self._db.connection.execute_command("GRAPH.UDF", "LOAD", _UDF_LIB, src) + except Exception: + pass + _udf_loaded_servers.add(self._server_key) + + def _invalidate(self) -> None: + # No cache to invalidate — the store holds no graph data. Kept as a no-op + # so mutation methods don't need to special-case it. + return + + # ---- graph-level metadata ---------------------------------------------- + def _load_meta(self) -> None: + try: + rows = self._rows("MATCH (m:GraphMeta {key:$k}) RETURN m.data", {"k": _META_KEY}) + if rows and rows[0][0]: + self.graph = json.loads(rows[0][0]) + except Exception: + self.graph = {} + + def save_meta(self) -> None: + """Persist the in-memory `.graph` dict (hyperedges etc.) to FalkorDB.""" + payload = json.dumps({k: v for k, v in self.graph.items() if not k.startswith("_")}, default=str) + self._g.query( + "MERGE (m:GraphMeta {key:$k}) SET m.data = $d", {"k": _META_KEY, "d": payload} + ) + + # ---- lifecycle ---------------------------------------------------------- + def clear(self) -> None: + try: + self._g.delete() + except Exception: + pass + self._g = self._db.select_graph(self.graph_name) + self.graph = {} + self._invalidate() + self._ensure_schema() + + def __getitem__(self, node_id): + """Adjacency access: ``G[u]`` -> {neighbor_id: edge_attrs} (both directions), scoped.""" + rows = self._rows( + "MATCH (n:Entity {id:$id})-[r]-(m:Entity) RETURN m.id, properties(r)", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + adj: dict = {} + for mid, props in rows: + adj.setdefault(mid, dict(props)) + return adj + + def __contains__(self, node_id): + return self.has_node(node_id) + + def __iter__(self): + return self._iter_nodes(data=False) + + def __len__(self): + return self.number_of_nodes() + + def is_directed(self) -> bool: + return self.directed + + def is_multigraph(self) -> bool: + return False + + def to_undirected(self, as_view: bool = False): + return self + + def copy(self): + return self + + # ---- construction (batched) -------------------------------------------- + _BATCH = 1000 + + def add_nodes_from(self, rows, *, fresh: bool = False) -> None: + """Bulk-upsert nodes. When ``fresh`` (graph known empty/just cleared, ids + unique) use CREATE — it skips MERGE's existence check and is several times + faster on a from-scratch build.""" + by_label: dict[str, list[dict]] = {} + materialized = [] + for item in rows: + if isinstance(item, tuple): + node_id, attrs = item + else: + node_id, attrs = item, {} + props = _scalar_props(attrs) + props["id"] = node_id + ftype = _safe_label(str(attrs.get("file_type", "Entity")).capitalize()) + by_label.setdefault(ftype, []).append(props) + materialized.append(1) + for ftype, batch in by_label.items(): + verb = f"CREATE (n:Entity:{ftype})" if fresh else f"MERGE (n:Entity {{id: row.id}}) SET n:{ftype}" + for i in range(0, len(batch), self._BATCH): + chunk = batch[i : i + self._BATCH] + self._g.query(f"UNWIND $rows AS row {verb} SET n += row", {"rows": chunk}) + if materialized: + self._invalidate() + + def add_node(self, node_id: str, **attrs) -> None: + self.add_nodes_from([(node_id, attrs)]) + + def add_edges_from(self, rows, *, fresh: bool = False) -> None: + """Bulk-upsert edges. When ``fresh`` use CREATE instead of MERGE (no + edge-existence check) — endpoints must already exist.""" + by_rel: dict[str, list[dict]] = {} + for u, v, attrs in rows: + rel = _safe_rel(str(attrs.get("relation", "RELATED_TO"))) + props = _scalar_props(attrs) + props.setdefault("relation", attrs.get("relation", rel)) + by_rel.setdefault(rel, []).append({"src": u, "tgt": v, "props": props}) + for rel, batch in by_rel.items(): + verb = "CREATE" if fresh else "MERGE" + for i in range(0, len(batch), self._BATCH): + chunk = batch[i : i + self._BATCH] + self._g.query( + f"UNWIND $rows AS row MATCH (a:Entity {{id: row.src}}), (b:Entity {{id: row.tgt}}) " + f"{verb} (a)-[r:{rel}]->(b) SET r += row.props", + {"rows": chunk}, + ) + self._invalidate() + + def add_edge(self, u: str, v: str, **attrs) -> None: + self.add_edges_from([(u, v, attrs)]) + + # ---- mutation ----------------------------------------------------------- + def remove_nodes(self, node_ids) -> None: + ids = list(node_ids) + if not ids: + return + self._g.query("UNWIND $ids AS nid MATCH (n:Entity {id: nid}) DETACH DELETE n", {"ids": ids}) + self._invalidate() + + remove_nodes_from = remove_nodes + + def remove_node(self, node_id: str) -> None: + self.remove_nodes([node_id]) + + def remove_edges(self, pairs) -> None: + rows = [{"src": u, "tgt": v} for u, v in pairs] + if not rows: + return + self._g.query( + "UNWIND $rows AS row MATCH (a:Entity {id: row.src})-[r]-(b:Entity {id: row.tgt}) DELETE r", + {"rows": rows}, + ) + self._invalidate() + + remove_edges_from = remove_edges + + def remove_edge(self, u: str, v: str) -> None: + self.remove_edges([(u, v)]) + + def has_node(self, node_id: str) -> bool: + return bool(self._rows("MATCH (n:Entity {id:$id}) RETURN 1 LIMIT 1", {"id": node_id})) + + def has_edge(self, u: str, v: str) -> bool: + return bool(self._rows( + "MATCH (a:Entity {id:$u})-[]-(b:Entity {id:$v}) RETURN 1 LIMIT 1", {"u": u, "v": v})) + + def has_directed_edge(self, u: str, v: str) -> bool: + """True only if an edge is stored in the u -> v orientation.""" + return bool(self._rows( + "MATCH (a:Entity {id:$u})-[]->(b:Entity {id:$v}) RETURN 1 LIMIT 1", {"u": u, "v": v})) + + def number_of_nodes(self) -> int: + # count() is uncapped and cheap — independent of the read cache. + return self._rows("MATCH (n:Entity) RETURN count(n)")[0][0] + + def number_of_edges(self) -> int: + return self._rows("MATCH (:Entity)-[r]->(:Entity) RETURN count(r)")[0][0] + + def neighbors(self, node_id: str): + rows = self._rows( + "MATCH (n:Entity {id:$id})-[]-(m:Entity) RETURN DISTINCT m.id", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + return iter(sorted(str(r[0]) for r in rows)) + + def in_edges(self, node_id: str, data: bool = False): + rows = self._rows( + "MATCH (s:Entity)-[r]->(n:Entity {id:$id}) RETURN s.id, properties(r)", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + rows.sort(key=lambda r: str(r[0])) + if data: + return [(str(s), node_id, dict(p)) for s, p in rows] + return [(str(s), node_id) for s, _ in rows] + + def successors(self, node_id: str): + """Outgoing neighbors (directed), sorted.""" + rows = self._rows("MATCH (n:Entity {id:$id})-[]->(m:Entity) RETURN DISTINCT m.id", {"id": node_id}) + return iter(sorted(str(r[0]) for r in rows)) + + def predecessors(self, node_id: str): + """Incoming neighbors (directed), sorted.""" + rows = self._rows("MATCH (m:Entity)-[]->(n:Entity {id:$id}) RETURN DISTINCT m.id", {"id": node_id}) + return iter(sorted(str(r[0]) for r in rows)) + + def subgraph(self, node_ids): + return _SStoreSubgraph(self, node_ids) + + def set_communities(self, communities: dict) -> None: + """Persist community ids onto nodes: {cid: [node_ids]} -> n.community.""" + rows = [] + for cid, node_ids in communities.items(): + for nid in node_ids: + rows.append({"id": nid, "c": int(cid)}) + if not rows: + return + for i in range(0, len(rows), self._BATCH): + chunk = rows[i : i + self._BATCH] + self._g.query( + "UNWIND $rows AS row MATCH (n:Entity {id: row.id}) SET n.community = row.c", + {"rows": chunk}, + ) + self._invalidate() + + def prune_repo(self, repo_tag: str) -> int: + before = self.number_of_nodes() + self._g.query("MATCH (n:Entity {repo: $t}) DETACH DELETE n", {"t": repo_tag}) + self._invalidate() + return before - self.number_of_nodes() + + # ---- algorithms --------------------------------------------------------- + def shortest_path(self, src: str, tgt: str, max_hops: int = 8): + """Undirected shortest path via level-batched BFS in the engine — one + query per BFS level expanding the whole frontier (not the whole graph). + Avoids algo.SPpaths (pathologically slow undirected) and never loads the + graph into Python.""" + if src == tgt: + return [src] if self.has_node(src) else None + prev = {src: None} + frontier = [src] + for _ in range(int(max_hops)): + if not frontier: + break + rows = self._rows( + "UNWIND $f AS fid MATCH (n:Entity {id: fid})-[]-(m:Entity) RETURN fid, m.id", + {"f": frontier}, timeout=_QUERY_TIMEOUT_MS, + ) + nxt = [] + for fid, mid in sorted(rows, key=lambda r: (str(r[0]), str(r[1]))): + if mid in prev: + continue + prev[mid] = fid + if mid == tgt: + path = [mid] + while path[-1] is not None: + path.append(prev[path[-1]]) + return list(reversed(path[:-1])) + nxt.append(mid) + frontier = nxt + return None + + _RENDER_COLS = "n.label, n.source_file, n.source_location, n.community" + + def subgraph_render_data(self, node_ids, edge_pairs, *, limit=None, seeds=None): + """Fetch node attrs+degree and edge attrs for a subgraph, for rendering + query/path results. Returns ``(nodes, eattrs, total)`` where ``total`` is + the full node count. + + When ``limit`` is set and the subgraph is larger, only the top-``limit`` + nodes by degree (plus ``seeds``, always kept) are fetched — the engine + ranks+truncates so a hub query never pulls attributes for tens of + thousands of nodes just to discard all but the slice the caller's token + budget can show. Edges are fetched only among the kept nodes.""" + ids = list(node_ids) + total = len(ids) + nodes: dict = {} + + def _absorb(rows): + for r in rows: + nodes[r[0]] = {"label": r[1], "source_file": r[2], "source_location": r[3], + "community": r[4], "degree": int(r[5])} + + if ids and (limit is None or total <= limit): + _absorb(self._rows( + f"UNWIND $ids AS i MATCH (n:Entity {{id: i}}) " + f"RETURN i, {self._RENDER_COLS}, size((n)--())", + {"ids": ids}, timeout=_QUERY_TIMEOUT_MS, + )) + elif ids: + # Seeds are always kept (rendered first); the rest are ranked by + # degree in-engine and truncated to fill out the limit. + id_set = set(ids) + seed_list = [s for s in (seeds or []) if s in id_set] + if seed_list: + _absorb(self._rows( + f"UNWIND $ids AS i MATCH (n:Entity {{id: i}}) " + f"RETURN i, {self._RENDER_COLS}, size((n)--())", + {"ids": seed_list}, timeout=_QUERY_TIMEOUT_MS, + )) + k = max(0, limit - len(nodes)) + others = [i for i in ids if i not in nodes] + if k and others: + _absorb(self._rows( + f"UNWIND $ids AS i MATCH (n:Entity {{id: i}}) " + f"WITH n, size((n)--()) AS deg ORDER BY deg DESC LIMIT $k " + f"RETURN n.id, {self._RENDER_COLS}, deg", + {"ids": others, "k": k}, timeout=_QUERY_TIMEOUT_MS, + )) + + eattrs: dict = {} + pairs = [[u, v] for u, v in edge_pairs if u in nodes and v in nodes] + if pairs: + for u, v, rel, conf, ctx in self._rows( + "UNWIND $pairs AS p MATCH (a:Entity {id: p[0]})-[r]-(b:Entity {id: p[1]}) " + "RETURN p[0], p[1], r.relation, r.confidence, r.context", + {"pairs": pairs}, timeout=_QUERY_TIMEOUT_MS, + ): + eattrs.setdefault((u, v), {"relation": rel, "confidence": conf, "context": ctx}) + return nodes, eattrs, total + + def node_detail(self, node_id: str): + """(attrs, degree) for one node, in one scoped query. None if absent.""" + rows = self._rows( + "MATCH (n:Entity {id:$id}) RETURN properties(n), size((n)--())", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + if not rows: + return None + attrs = dict(rows[0][0]) + attrs.pop(_META_KEY, None) + return attrs, int(rows[0][1]) + + def node_connections(self, node_id: str) -> list[dict]: + """All edges incident to a node (both directions) with neighbor label + + degree + relation/confidence — one scoped query, no full-graph load.""" + rows = self._rows( + "MATCH (n:Entity {id:$id})-[r]->(m:Entity) " + "RETURN 'out' AS dir, m.id AS mid, m.label AS mlabel, r.relation AS rel, " + "r.confidence AS conf, size((m)--()) AS mdeg " + "UNION ALL " + "MATCH (n:Entity {id:$id})<-[r]-(m:Entity) " + "RETURN 'in' AS dir, m.id AS mid, m.label AS mlabel, r.relation AS rel, " + "r.confidence AS conf, size((m)--()) AS mdeg", + {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, + ) + return [ + {"dir": r[0], "id": r[1], "label": r[2] or r[1], "relation": r[3] or "", + "confidence": r[4] or "", "degree": int(r[5])} + for r in rows + ] + + def search_nodes(self, terms) -> list[dict]: + """Candidate nodes whose norm_label / id / source_file contains any term. + This is the in-engine prefilter for query/explain — only nodes that could + score > 0 are returned, so Python scores a small set, never the whole graph.""" + terms = [t for t in terms if t] + if not terms: + return [] + conds, params = [], {} + for i, t in enumerate(terms): + params[f"t{i}"] = t + # coalesce: graphs built without a stored norm_label (e.g. test fixtures) + # still match on the lowercased label. + conds.append( + f"(coalesce(n.norm_label, toLower(n.label)) CONTAINS $t{i} " + f"OR toLower(n.id) CONTAINS $t{i} OR toLower(n.source_file) CONTAINS $t{i})" + ) + rows = self._rows( + f"MATCH (n:Entity) WHERE {' OR '.join(conds)} " + "RETURN n.id, n.label, n.norm_label, n.source_file", + params, timeout=_QUERY_TIMEOUT_MS, + ) + return [ + {"id": r[0], "label": r[1] or "", "norm_label": r[2] or "", "source_file": r[3] or ""} + for r in rows + ] + + def doc_freqs(self, terms) -> dict: + """{term: number of nodes whose norm_label contains term} — for IDF, in-engine.""" + out = {} + for t in terms: + if not t: + continue + out[t] = self._rows( + "MATCH (n:Entity) WHERE n.norm_label CONTAINS $t RETURN count(n)", + {"t": t}, timeout=_QUERY_TIMEOUT_MS, + )[0][0] + return out + + def confidence_counts(self) -> dict: + """{confidence_label: edge_count} over all edges — one in-engine aggregation + (so graph_stats/audit never stream every edge into Python).""" + rows = self._rows( + "MATCH (:Entity)-[r]->(:Entity) " + "RETURN coalesce(r.confidence, 'EXTRACTED'), count(r)", + timeout=_QUERY_TIMEOUT_MS, + ) + return {str(k): int(v) for k, v in rows} + + def find_node_ids(self, *, label=None, source_file=None, label_contains=None, limit=None) -> list: + """Scoped node lookup by exact label / exact source_file / label substring.""" + if label is not None: + q, v = "MATCH (n:Entity) WHERE toLower(n.label) = $v RETURN n.id", label.lower() + elif source_file is not None: + q, v = "MATCH (n:Entity) WHERE toLower(n.source_file) = $v RETURN n.id", source_file.lower() + else: + q, v = "MATCH (n:Entity) WHERE toLower(n.label) CONTAINS $v RETURN n.id", label_contains.lower() + if limit: + q += f" LIMIT {int(limit)}" + return [r[0] for r in self._rows(q, {"v": v}, timeout=_QUERY_TIMEOUT_MS)] + + def incoming_edges(self, frontier_ids, relations) -> list: + """All (frontier_id, src_id, relation) incoming edges for a frontier whose + relation is in `relations` — one query for the whole frontier level.""" + if not frontier_ids: + return [] + return self._rows( + "UNWIND $f AS fid MATCH (src:Entity)-[r]->(n:Entity {id: fid}) " + "WHERE r.relation IN $rels RETURN fid, src.id, r.relation", + {"f": list(frontier_ids), "rels": list(relations)}, + timeout=_QUERY_TIMEOUT_MS, + ) + + def node_attrs_batch(self, ids) -> dict: + """Fetch properties for many node ids in one query.""" + ids = list(ids) + if not ids: + return {} + rows = self._rows( + "UNWIND $ids AS i MATCH (n:Entity {id: i}) RETURN i, properties(n)", + {"ids": ids}, + timeout=_QUERY_TIMEOUT_MS, + ) + return {r[0]: dict(r[1]) for r in rows} + + def top_degree_nodes(self, limit: int = 500) -> list[dict]: + """Top-N nodes by (undirected) degree, computed in-engine. Returns only + `limit` rows with the attrs god_nodes needs — no full-graph load.""" + rows = self._rows( + "MATCH (n:Entity) WITH n, size((n)--()) AS deg " + "RETURN n.id, n.label, n.source_file, deg ORDER BY deg DESC LIMIT $lim", + {"lim": int(limit)}, + timeout=_QUERY_TIMEOUT_MS, + ) + return [ + {"id": r[0], "label": r[1] or "", "source_file": r[2] or "", "degree": int(r[3])} + for r in rows + ] + + def stream_full_edges(self): + """Stream edges with endpoint label/source/degree + relation/confidence so + surprise analysis needs no per-edge node lookups. Page-at-a-time, no load.""" + for r in self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) " + "WITH a, b, r, size((a)--()) AS da, size((b)--()) AS db " + "RETURN a.id, a.label, a.source_file, da, b.id, b.label, b.source_file, db, " + "r.relation, r.confidence", "id(r)", + ): + yield { + "u": r[0], "u_label": r[1] or "", "u_source": r[2] or "", "u_deg": int(r[3]), + "v": r[4], "v_label": r[5] or "", "v_source": r[6] or "", "v_deg": int(r[7]), + "relation": r[8] or "", "confidence": r[9] or "EXTRACTED", + } + + def node_betweenness(self) -> dict: + # Scope to :Entity so the internal :GraphMeta bookkeeping node (which has + # no id and no edges) never leaks in as a phantom "None" key. + rows = self._rows("CALL algo.betweenness() YIELD node, score WHERE node:Entity RETURN node.id, score", timeout=_QUERY_TIMEOUT_MS) + return {str(nid): float(s) for nid, s in rows} + + def _all_weighted_edges(self): + # Transient marshaling for the Louvain UDF param (the UDF can't read edges + # itself). Streamed page-by-page; not retained as a cache. + return [ + [r[0], r[1], float(r[2] if r[2] is not None else 1.0)] + for r in self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) RETURN a.id, b.id, coalesce(r.weight, 1.0)", "id(r)") + ] + + def run_louvain(self, edges, resolution: float = 1.0) -> dict: + """Run the Louvain UDF over an explicit weighted edge list.""" + if not edges: + return {} + rows = self._rows(f"RETURN {_UDF_LIB}.louvain($e, $res)", {"e": edges, "res": resolution}, timeout=_QUERY_TIMEOUT_MS) + return {str(k): int(v) for k, v in dict(rows[0][0]).items()} + + def louvain_partition(self, resolution: float = 1.0) -> dict: + edges = self._all_weighted_edges() + if not edges: + return {nid: i for i, nid in enumerate(self.nodes)} + return self.run_louvain(edges, resolution) + + def edge_betweenness(self) -> dict: + edges = [[r[0], r[1]] for r in self._stream( + "MATCH (a:Entity)-[r]->(b:Entity) RETURN a.id, b.id", "id(r)")] + if not edges: + return {} + res = self._rows(f"RETURN {_UDF_LIB}.edgeBetweenness($e)", {"e": edges}, timeout=_QUERY_TIMEOUT_MS) + out: dict = {} + for key, score in dict(res[0][0]).items(): + u, v = key.split("\t", 1) + out[(u, v)] = float(score) + return out + + def simple_cycles(self, edges, max_len: int = 5): + payload = [[str(u), str(v), 1.0] for u, v in edges] + if not payload: + return [] + rows = self._rows(f"RETURN {_UDF_LIB}.simpleCycles($e, $n)", {"e": payload, "n": int(max_len)}, timeout=_QUERY_TIMEOUT_MS) + return [[str(x) for x in cyc] for cyc in rows[0][0]] + + # ---- traversal (hub-capped, level-batched in the engine) ---------------- + def _hub_threshold(self) -> int: + # Graph-level constant (p99 of the degree distribution). Computing it scans + # every node's degree (~0.4s), so cache it in graph metadata and reuse it + # across query invocations. Cleared automatically on rebuild (clear() drops + # the GraphMeta node); SET-only ops like community labels don't change it. + cached = self.graph.get("hub_threshold") + if cached is not None: + return int(cached) + rows = self._rows( + "MATCH (n:Entity) WITH size((n)--()) AS d RETURN percentileDisc(d, 0.99)", + timeout=_QUERY_TIMEOUT_MS, + ) + p99 = rows[0][0] if rows and rows[0][0] is not None else 0 + hub = max(50, int(p99)) + self.graph["hub_threshold"] = hub + try: + self.save_meta() + except Exception: + pass + return hub + + def bfs(self, seeds, depth, hub_threshold=None, contexts=None): + return self._traverse(seeds, depth, hub_threshold, contexts) + + def dfs(self, seeds, depth, hub_threshold=None, contexts=None): + # Depth-bounded neighborhood is gathered the same way; results are + # relevance-ranked downstream, so DFS vs BFS ordering does not matter here. + return self._traverse(seeds, depth, hub_threshold, contexts) + + def _traverse(self, seeds, depth, hub_threshold, contexts=None): + """Hub-capped depth-bounded traversal — one query per level expanding the + whole frontier in the engine (never loads the graph). When ``contexts`` is + given, only edges whose ``context`` matches are followed (in-engine, so the + graph is never copied into Python for context filtering).""" + seeds = list(seeds) + if hub_threshold is None: + hub_threshold = self._hub_threshold() + contexts = list(contexts) if contexts else None + edge_match = "MATCH (n)-[r]-(m:Entity)" + edge_where = " WHERE r.context IN $ctx" if contexts else "" + seed_set = set(seeds) + visited = set(seeds) + edges_seen: list = [] + frontier = seeds + for _ in range(depth): + if not frontier: + break + params = {"f": frontier, "seeds": seeds, "hub": hub_threshold} + if contexts: + params["ctx"] = contexts + rows = self._rows( + "UNWIND $f AS fid MATCH (n:Entity {id: fid}) " + "WITH fid, n, size((n)--()) AS deg " + "WHERE fid IN $seeds OR deg < $hub " + f"{edge_match}{edge_where} RETURN fid, m.id", + params, + timeout=_QUERY_TIMEOUT_MS, + ) + nxt = [] + for fid, mid in sorted(rows, key=lambda r: (str(r[0]), str(r[1]))): + if mid not in visited: + visited.add(mid) + edges_seen.append((fid, mid)) + nxt.append(mid) + frontier = nxt + return visited, edges_seen diff --git a/graphify/udfs/graphify_algos.js b/graphify/udfs/graphify_algos.js new file mode 100644 index 000000000..5376eb7a8 --- /dev/null +++ b/graphify/udfs/graphify_algos.js @@ -0,0 +1,344 @@ +// graphify_algos — FalkorDB server-side UDF library. +// +// These replace the NetworkX algorithms graphify used to run in-process, +// for the ones FalkorDB has no built-in equivalent of: +// - louvain : community detection (was graspologic Leiden / nx.community.louvain) +// - edgeBetweenness : edge betweenness (was nx.edge_betweenness_centrality) +// - simpleCycles : directed simple cycles (was nx.simple_cycles) +// +// DESIGN: these are PURE-COMPUTE functions. They do NOT read the graph through +// the UDF graph API — that API (as of FalkorDB 4.18) does not expose edge +// endpoints via iterateEdges, and getNeighbors() ignores its direction config +// and never surfaces edge weights. Instead the Python GraphStore pulls the edge +// list (with weights) via ordinary Cypher and passes it in as an argument. That +// keeps weights/direction correct and makes these functions trivially testable. +// +// DETERMINISM: every function processes nodes/edges in a fixed sorted order and +// uses no randomness, so results are byte-stable across runs (graphify pins +// seed=42 everywhere; here we get the same guarantee from sorted iteration). +// +// Edge argument shape (all functions): an array of [u, v, w] triples, where u/v +// are node-id strings and w is an optional numeric weight (defaults to 1). + +// ---------------------------------------------------------------------------- +// louvain(edges, resolution) -> { nodeId: communityId(int) } +// +// Standard Louvain modularity maximization on the UNDIRECTED, WEIGHTED graph. +// Two-phase: (1) greedy local moving, (2) community aggregation, repeated until +// modularity stops improving. The returned community ids are arbitrary integers; +// the Python cluster() wrapper re-indexes them to size-stable ids. +// +// Node move order is the sorted node order (deterministic). Ties in modularity +// gain are broken toward the smallest community id, again for determinism. +// ---------------------------------------------------------------------------- +function louvain(edges, resolution) { + var gamma = (resolution === undefined || resolution === null) ? 1.0 : resolution; + + // Collect a sorted, de-duplicated node list -> integer index. + var idSet = {}; + for (var i = 0; i < edges.length; i++) { + idSet[edges[i][0]] = true; + idSet[edges[i][1]] = true; + } + var ids = Object.keys(idSet).sort(); + var index = {}; + for (var n = 0; n < ids.length; n++) index[ids[n]] = n; + var N = ids.length; + if (N === 0) return {}; + + // Build a weighted undirected adjacency as an aggregated edge map. + // adj[i] = { j: weight } summed across parallel/both-direction edges. + // Self-loops are kept (they contribute to a node's degree twice, as usual). + var adj = new Array(N); + for (var a = 0; a < N; a++) adj[a] = {}; + var totalW = 0.0; // m (sum of all edge weights, each undirected edge once) + for (var e = 0; e < edges.length; e++) { + var u = index[edges[e][0]]; + var v = index[edges[e][1]]; + var w = edges[e][2]; + if (w === undefined || w === null || isNaN(w)) w = 1.0; + if (u === v) { + adj[u][u] = (adj[u][u] || 0) + w; // self-loop + totalW += w; + continue; + } + adj[u][v] = (adj[u][v] || 0) + w; + adj[v][u] = (adj[v][u] || 0) + w; + totalW += w; + } + if (totalW === 0) { + // No edges: every node is its own community. + var solo = {}; + for (var s = 0; s < N; s++) solo[ids[s]] = s; + return solo; + } + + // Weighted degree k_i for each node (self-loops count twice). + function degrees(adjacency, count) { + var k = new Array(count); + for (var p = 0; p < count; p++) { + var sum = 0.0; + var row = adjacency[p]; + for (var q in row) { + sum += (parseInt(q, 10) === p) ? 2 * row[q] : row[q]; + } + k[p] = sum; + } + return k; + } + + // One Louvain level: greedy local moving on `adjacency` (size `count`), + // total weight `m`. Returns { comm: int[], improved: bool }. + function oneLevel(adjacency, count, m) { + var k = degrees(adjacency, count); + var comm = new Array(count); + var sigmaTot = new Array(count); // total degree of community c + for (var i = 0; i < count; i++) { comm[i] = i; sigmaTot[i] = k[i]; } + var twoM = 2 * m; + + var improvedAny = false; + var moved = true; + var guard = 0; + while (moved && guard < 100) { + moved = false; + guard++; + for (var node = 0; node < count; node++) { + var nodeComm = comm[node]; + // Weight from `node` into each neighboring community. + var wToComm = {}; + var row = adjacency[node]; + var selfLoop = row[node] || 0; + for (var nb in row) { + var nbi = parseInt(nb, 10); + if (nbi === node) continue; + var c = comm[nbi]; + wToComm[c] = (wToComm[c] || 0) + row[nb]; + } + // Remove node from its current community. + sigmaTot[nodeComm] -= k[node]; + var wToOwn = wToComm[nodeComm] || 0; + + // Evaluate candidate communities: current + all neighbor communities. + // Deterministic: consider community ids in ascending order. + var candidates = {}; + candidates[nodeComm] = true; + for (var cc in wToComm) candidates[cc] = true; + var candList = Object.keys(candidates).map(function (x) { return parseInt(x, 10); }).sort(function (p, q) { return p - q; }); + + var bestComm = nodeComm; + // Gain of staying (relative baseline) — start from current community. + var bestGain = wToOwn - gamma * sigmaTot[nodeComm] * k[node] / twoM; + for (var ci = 0; ci < candList.length; ci++) { + var cand = candList[ci]; + var wln = wToComm[cand] || 0; + var gain = wln - gamma * sigmaTot[cand] * k[node] / twoM; + if (gain > bestGain + 1e-12) { + bestGain = gain; + bestComm = cand; + } + } + // Insert node into the chosen community. + sigmaTot[bestComm] += k[node]; + comm[node] = bestComm; + if (bestComm !== nodeComm) { moved = true; improvedAny = true; } + } + } + return { comm: comm, improved: improvedAny }; + } + + // Aggregate communities into a smaller graph for the next level. + function aggregate(adjacency, count, comm) { + // Renumber communities to a dense 0..K-1 range (deterministic by first appearance in sorted node order). + var remap = {}; + var K = 0; + for (var i = 0; i < count; i++) { + var c = comm[i]; + if (remap[c] === undefined) { remap[c] = K++; } + } + var dense = new Array(count); + for (var j = 0; j < count; j++) dense[j] = remap[comm[j]]; + var newAdj = new Array(K); + for (var a = 0; a < K; a++) newAdj[a] = {}; + for (var p = 0; p < count; p++) { + var cp = dense[p]; + var row = adjacency[p]; + for (var q in row) { + var cq = dense[parseInt(q, 10)]; + newAdj[cp][cq] = (newAdj[cp][cq] || 0) + row[q]; + } + } + return { adj: newAdj, count: K, dense: dense }; + } + + // node -> current community id at the original-node granularity. + var nodeComm = new Array(N); + for (var t = 0; t < N; t++) nodeComm[t] = t; + + var curAdj = adj; + var curCount = N; + var pass = 0; + while (pass < 50) { + pass++; + var level = oneLevel(curAdj, curCount, totalW); + if (!level.improved) break; + var agg = aggregate(curAdj, curCount, level.comm); + // Fold this level's mapping back onto original nodes. + for (var orig = 0; orig < N; orig++) { + nodeComm[orig] = agg.dense[nodeComm[orig]]; + } + curAdj = agg.adj; + curCount = agg.count; + if (agg.count === curCount && !level.improved) break; + } + + var out = {}; + for (var z = 0; z < N; z++) out[ids[z]] = nodeComm[z]; + return out; +} + +// ---------------------------------------------------------------------------- +// edgeBetweenness(edges) -> { "u\tv": score } +// +// Brandes' algorithm for edge betweenness on the UNDIRECTED, UNWEIGHTED graph +// (matches nx.edge_betweenness_centrality default weight=None). Normalized by +// 2/(n(n-1)) like networkx's undirected default. Edge keys are "u\tv" with u= 0; si--) { + var ww = S[si]; + var preds = P[ww]; + for (var pi = 0; pi < preds.length; pi++) { + var pv = preds[pi]; + var c = (sigma[pv] / sigma[ww]) * (1 + delta[ww]); + var key = ekey(pv, ww); + eb[key] = (eb[key] || 0) + c; + delta[pv] += c; + } + } + } + + // Undirected: each shortest path counted from both endpoints -> divide by 2. + // Normalize like networkx undirected default: scale = 2/(n(n-1)). + var scale = (N > 1) ? (2.0 / (N * (N - 1))) : 1.0; + var out = {}; + for (var key2 in eb) { + var parts = key2.split(","); + var a2 = parseInt(parts[0], 10), b2 = parseInt(parts[1], 10); + var val = (eb[key2] / 2.0) * scale; + out[ids[a2] + "\t" + ids[b2]] = val; + } + return out; +} + +// ---------------------------------------------------------------------------- +// simpleCycles(edges, maxLen) -> [[id, id, ...], ...] +// +// Enumerate simple cycles in a DIRECTED graph, bounded by maxLen (number of +// nodes in the cycle). Edges are treated as directed u->v (weight ignored). +// Replaces nx.simple_cycles for import-cycle detection; the Python side already +// dedupes rotations and caps the result, so here we just enumerate (with a hard +// safety cap) starting each search from the lexicographically smallest member. +// ---------------------------------------------------------------------------- +function simpleCycles(edges, maxLen) { + var limit = (maxLen === undefined || maxLen === null) ? 5 : maxLen; + var SAFETY = 5000; + + var idSet = {}; + for (var i = 0; i < edges.length; i++) { idSet[edges[i][0]] = true; idSet[edges[i][1]] = true; } + var ids = Object.keys(idSet).sort(); + var index = {}; + for (var n = 0; n < ids.length; n++) index[ids[n]] = n; + var N = ids.length; + + var succ = new Array(N); + for (var a = 0; a < N; a++) succ[a] = {}; + for (var e = 0; e < edges.length; e++) { + var u = index[edges[e][0]], v = index[edges[e][1]]; + if (u === v) continue; // self-loop is not a multi-node cycle + succ[u][v] = true; + } + var out = []; + + // For each start node (ascending), DFS for cycles returning to start, only + // visiting nodes > start so each cycle is found once from its min member. + for (var start = 0; start < N && out.length < SAFETY; start++) { + var stack = [start]; + var onStack = {}; + onStack[start] = true; + + function dfs(node) { + if (out.length >= SAFETY) return; + var nbrs = Object.keys(succ[node]).map(function (x) { return parseInt(x, 10); }).sort(function (p, q) { return p - q; }); + for (var ni = 0; ni < nbrs.length; ni++) { + var nb = nbrs[ni]; + if (nb === start && stack.length >= 2) { + if (stack.length <= limit) { + var cyc = []; + for (var si = 0; si < stack.length; si++) cyc.push(ids[stack[si]]); + out.push(cyc); + } + continue; + } + if (nb < start || onStack[nb]) continue; + if (stack.length >= limit) continue; + stack.push(nb); + onStack[nb] = true; + dfs(nb); + stack.pop(); + delete onStack[nb]; + } + } + dfs(start); + } + return out; +} + +falkor.register('louvain', louvain); +falkor.register('edgeBetweenness', edgeBetweenness); +falkor.register('simpleCycles', simpleCycles); diff --git a/graphify/watch.py b/graphify/watch.py index 712f9fc65..fc397cb1e 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -308,13 +308,8 @@ def _canonical_topology_for_compare(graph_data: dict) -> dict: def _topology_from_graph(G) -> dict: - from networkx.readwrite import json_graph - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - return data + from graphify.graphjson import to_node_link + return to_node_link(G) def _check_shrink( @@ -635,7 +630,10 @@ def _rebuild_code( "total_words": detected.get("total_words", 0), } - G = build_from_json(result) + # Build into the FalkorDB graph bound to this output dir so query/serve + # (which load via the pointer file) see the rebuilt graph. + from graphify.store import open_store as _open_store + G = build_from_json(result, store=_open_store(out, create=True)) candidate_topology = _topology_from_graph(G) if existing_graph_data: try: diff --git a/graphify/wiki.py b/graphify/wiki.py index eb662317f..b1b84dd8f 100644 --- a/graphify/wiki.py +++ b/graphify/wiki.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections import Counter from pathlib import Path -import networkx as nx from graphify.build import edge_data diff --git a/pyproject.toml b/pyproject.toml index d498dc2d4..e21643027 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ license = { file = "LICENSE" } keywords = ["claude", "claude-code", "codex", "opencode", "kilo", "cursor", "gemini", "aider", "kiro", "pi", "devin", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] requires-python = ">=3.10" dependencies = [ - "networkx>=3.4", + "falkordb>=1.0", "datasketch>=1.6", "rapidfuzz>=3.0", "tree-sitter>=0.23.0", @@ -50,11 +50,14 @@ Issues = "https://github.com/safishamsi/graphify/issues" [project.optional-dependencies] mcp = ["mcp"] neo4j = ["neo4j"] -falkordb = ["falkordb"] pdf = ["pypdf", "markdownify"] watch = ["watchdog"] svg = ["matplotlib", "numpy>=2.0; python_version >= '3.13'"] leiden = ["graspologic; python_version < '3.13'"] +# FalkorDB Lite: embedded in-process engine (redislite + bundled falkordb module), +# no external server. Requires Python >= 3.12. Select with a falkordb-lite:// URI +# or the GRAPHIFY_FALKORDB_LITE env var. +lite = ["falkordblite; python_version >= '3.12'"] office = ["python-docx", "openpyxl"] google = ["openpyxl"] postgres = ["psycopg[binary]"] diff --git a/tests/conftest.py b/tests/conftest.py index 835ff5e52..0e94d8986 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,129 @@ from __future__ import annotations +import os from typing import Any import pytest + +# -------------------------------------------------------------------------- +# FalkorDB-backed graph fixtures (NetworkX replacement). +# Every test that needs a graph gets a fresh, uniquely-named GraphStore bound +# to a running FalkorDB. Skips cleanly when none is reachable. +# docker run -d -p 6379:6379 falkordb/falkordb:latest +# Overridable via FALKORDB_URI / FALKORDB_HOST / FALKORDB_PORT. +# -------------------------------------------------------------------------- +def _falkordb_uri() -> str: + if os.environ.get("FALKORDB_URI"): + return os.environ["FALKORDB_URI"] + host = os.environ.get("FALKORDB_HOST", "localhost") + port = os.environ.get("FALKORDB_PORT", "6379") + return f"falkordb://{host}:{port}" + + +@pytest.fixture(scope="session") +def falkordb_uri() -> str: + return _falkordb_uri() + + +@pytest.fixture(scope="session", autouse=True) +def _require_falkordb(falkordb_uri): + pytest.importorskip("falkordb") + from graphify.store import _connect + + try: + _connect(falkordb_uri).connection.ping() + except Exception as e: # pragma: no cover - environment dependent + pytest.skip(f"no FalkorDB reachable at {falkordb_uri} ({e})") + + +_store_counter = {"n": 0} + + +@pytest.fixture() +def store(falkordb_uri): + """A fresh, empty, uniquely-named GraphStore; cleaned up after the test.""" + from graphify.store import GraphStore + + _store_counter["n"] += 1 + gs = GraphStore(graph_name=f"pytest_{os.getpid()}_{_store_counter['n']}", uri=falkordb_uri) + gs.clear() + try: + yield gs + finally: + try: + gs.clear() + except Exception: + pass + + +@pytest.fixture() +def seed_graph(falkordb_uri): + """Seed a FalkorDB graph for an output dir from node-link data, writing the + pointer file so the CLI/serve loaders find it (replaces writing graph.json). + + Usage: seed_graph(tmp_path, nodes=[...], links=[...]) -> GraphStore. + Each node dict needs an ``id``; each link dict ``source``/``target``. + """ + from graphify.store import open_store + + created = [] + + def _seed(out_dir, nodes=(), links=(), *, write_artifact=True): + store = open_store(out_dir, uri=falkordb_uri, create=True) + store.clear() + store.add_nodes_from([ + (n["id"], {k: v for k, v in n.items() if k != "id"}) for n in nodes + ]) + store.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in links + ]) + created.append(store) + if write_artifact: + import json as _json + from pathlib import Path as _Path + (_Path(out_dir) / "graph.json").write_text( + _json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": list(nodes), "links": list(links)}), + encoding="utf-8", + ) + return store + + try: + yield _seed + finally: + for s in created: + try: + s.clear() + except Exception: + pass + + +@pytest.fixture() +def make_store(falkordb_uri): + """Factory for tests needing more than one graph (e.g. graph_diff).""" + from graphify.store import GraphStore + + created = [] + + def _make(): + _store_counter["n"] += 1 + gs = GraphStore(graph_name=f"pytest_{os.getpid()}_{_store_counter['n']}", uri=falkordb_uri) + gs.clear() + created.append(gs) + return gs + + try: + yield _make + finally: + for gs in created: + try: + gs.clear() + except Exception: + pass + + _ANALYZE_WARNING_FILTERS = ( "ignore:Tensorflow not installed; ParametricUMAP will be unavailable:ImportWarning:umap", "ignore:Please import `random` from the `scipy\\.sparse` namespace.*:" diff --git a/tests/nxcompat.py b/tests/nxcompat.py new file mode 100644 index 000000000..82111bc9a --- /dev/null +++ b/tests/nxcompat.py @@ -0,0 +1,102 @@ +"""Minimal networkx-compatible shim for the test suite. + +The FalkorDB solution ships no networkx dependency (the package has zero runtime +networkx imports). The suite still builds small in-memory graph fixtures with the +`nx.Graph()` / `nx.DiGraph()` + add_node/add_edge idiom, so this module provides a +drop-in `nx`: a mutable builder whose read API is delegated to the in-house +`MemGraph` (which already exposes the full nx-style view API). Tests import it as +`from tests import nxcompat as nx` — no networkx install required. +""" +from graphify.store import MemGraph + + +class _MutableGraph: + """nx-like mutable graph; reads delegate to a lazily (re)built MemGraph.""" + + def __init__(self, directed: bool): + self._nodes: dict = {} + self._edges: list = [] + self._directed = directed + self.graph: dict = {} + self._mg = None + + # --- construction (networkx API) --- + def add_node(self, nid, **attrs): + self._nodes.setdefault(nid, {}).update(attrs) + self._mg = None + + def add_nodes_from(self, items): + for it in items: + if isinstance(it, (tuple, list)): + self.add_node(it[0], **(it[1] if len(it) > 1 and it[1] else {})) + else: + self.add_node(it) + + def add_edge(self, u, v, **attrs): + self._nodes.setdefault(u, {}) + self._nodes.setdefault(v, {}) + self._edges.append((u, v, attrs)) + self._mg = None + + def add_edges_from(self, items): + for it in items: + attrs = it[2] if len(it) > 2 and it[2] else {} + self.add_edge(it[0], it[1], **attrs) + + def is_directed(self): + return self._directed + + # --- read API: build a MemGraph (sharing the .graph dict) and delegate --- + def _build(self) -> MemGraph: + if self._mg is None: + self._mg = MemGraph(list(self._nodes.items()), self._edges, directed=self._directed) + self._mg.graph = self.graph + return self._mg + + def __getattr__(self, name): + # Only reached for names not set on the instance (nodes/edges/degree/ + # neighbors/subgraph/number_of_*/has_*/shortest_path/...). + return getattr(self._build(), name) + + def __contains__(self, x): + return x in self._build() + + def __iter__(self): + return iter(self._build()) + + def __len__(self): + return len(self._build()) + + def __getitem__(self, key): + return self._build()[key] + + +class Graph(_MutableGraph): + def __init__(self): + super().__init__(directed=False) + + +class DiGraph(_MutableGraph): + def __init__(self): + super().__init__(directed=True) + + +def relabel_nodes(G, mapping): + """Return a new graph with node ids remapped via `mapping` (unmapped ids kept).""" + out = DiGraph() if G.is_directed() else Graph() + for nid, data in G.nodes(data=True): + out.add_node(mapping.get(nid, nid), **data) + for u, v, data in G.edges(data=True): + out.add_edge(mapping.get(u, u), mapping.get(v, v), **data) + return out + + +def complete_graph(n): + """Undirected graph on nodes 0..n-1 with every pair connected (nx semantics).""" + out = Graph() + for i in range(n): + out.add_node(i) + for i in range(n): + for j in range(i + 1, n): + out.add_edge(i, j) + return out diff --git a/tests/test_affected_cli.py b/tests/test_affected_cli.py index 65a3be8ca..cd03fdac6 100644 --- a/tests/test_affected_cli.py +++ b/tests/test_affected_cli.py @@ -1,29 +1,23 @@ from __future__ import annotations -import json - -import networkx as nx -from networkx.readwrite import json_graph - import graphify.__main__ as mainmod - -def _write_graph(tmp_path): - graph = nx.DiGraph() - graph.add_node("target", label="Foo", source_file="pkg/foo.py", source_location="L1") - graph.add_node("caller", label="X()", source_file="app.py", source_location="L4") - graph.add_node("barrel", label="__init__.py", source_file="pkg/__init__.py", source_location=None) - graph.add_node("consumer", label="app.py", source_file="app.py", source_location=None) - graph.add_edge("caller", "target", relation="calls", context="call", confidence="EXTRACTED") - graph.add_edge("barrel", "target", relation="re_exports", context="export", confidence="EXTRACTED") - graph.add_edge("consumer", "target", relation="imports", context="import", confidence="EXTRACTED") +_NODES = [ + {"id": "target", "label": "Foo", "source_file": "pkg/foo.py", "source_location": "L1", "file_type": "code"}, + {"id": "caller", "label": "X()", "source_file": "app.py", "source_location": "L4", "file_type": "code"}, + {"id": "barrel", "label": "__init__.py", "source_file": "pkg/__init__.py", "file_type": "code"}, + {"id": "consumer", "label": "app.py", "source_file": "app.py", "file_type": "code"}, +] +_LINKS = [ + {"source": "caller", "target": "target", "relation": "calls", "context": "call", "confidence": "EXTRACTED"}, + {"source": "barrel", "target": "target", "relation": "re_exports", "context": "export", "confidence": "EXTRACTED"}, + {"source": "consumer", "target": "target", "relation": "imports", "context": "import", "confidence": "EXTRACTED"}, +] + + +def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links")), encoding="utf-8") - return graph_path - - -def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -43,8 +37,9 @@ def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, caps assert "imports" in out -def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, diff --git a/tests/test_analyze.py b/tests/test_analyze.py index ecf1555d3..39658ba7f 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -1,6 +1,6 @@ """Tests for analyze.py.""" import json -import networkx as nx +from tests import nxcompat as nx import pytest from pathlib import Path from graphify.build import build_from_json @@ -60,9 +60,9 @@ def test_surprising_connections_excludes_concept_nodes(): assert "Abstract Concept" not in labels -def test_surprising_connections_single_file_uses_community_bridges(): +def test_surprising_connections_single_file_uses_community_bridges(store): """Single-file graph: should return cross-community edges, not empty list.""" - G = nx.Graph() + G = store # Build a graph with 2 clear communities + 1 bridge edge for i in range(5): G.add_node(f"a{i}", label=f"A{i}", file_type="code", source_file="single.py", @@ -612,46 +612,39 @@ def _make_file_node(path: str) -> tuple[str, dict]: return nid, {"label": Path(path).name, "source_file": path, "file_type": "code"} -def _make_cycle_graph_directed() -> nx.DiGraph: - G = nx.DiGraph() - +def _make_cycle_graph_directed(store): a_id, a = _make_file_node("src/a.ts") b_id, b = _make_file_node("src/b.ts") c_id, c = _make_file_node("src/c.ts") d_id, d = _make_file_node("src/d.ts") ext_id = _make_id("react") - G.add_node(a_id, **a) - G.add_node(b_id, **b) - G.add_node(c_id, **c) - G.add_node(d_id, **d) - # External-like node (no source_file): must be skipped safely. - G.add_node(ext_id, label="react", file_type="code") - - # 2-cycle: a <-> b - G.add_edge(a_id, b_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - G.add_edge(b_id, a_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - - # 3-cycle: b -> c -> d -> b - G.add_edge(b_id, c_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - G.add_edge(c_id, d_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - G.add_edge(d_id, b_id, relation="imports_from", source_file="src/d.ts", confidence="EXTRACTED") - - # Self-loop: c imports itself - G.add_edge(c_id, c_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - - # Mixed edge types: must not bleed into cycle graph - G.add_edge(a_id, ext_id, relation="calls", source_file="src/a.ts", confidence="INFERRED") - G.add_edge(a_id, ext_id, relation="contains", source_file="src/a.ts", confidence="EXTRACTED") - - # Edge whose target has no source_file: must be skipped, no garbage label fallback - G.add_edge(a_id, ext_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - - return G - - -def test_find_import_cycles_returns_structured_records(): - G = _make_cycle_graph_directed() + store.add_nodes_from([ + (a_id, a), (b_id, b), (c_id, c), (d_id, d), + # External-like node (no source_file): must be skipped safely. + (ext_id, {"label": "react", "file_type": "code"}), + ]) + store.add_edges_from([ + # 2-cycle: a <-> b + (a_id, b_id, {"relation": "imports_from", "source_file": "src/a.ts", "confidence": "EXTRACTED"}), + (b_id, a_id, {"relation": "imports_from", "source_file": "src/b.ts", "confidence": "EXTRACTED"}), + # 3-cycle: b -> c -> d -> b + (b_id, c_id, {"relation": "imports_from", "source_file": "src/b.ts", "confidence": "EXTRACTED"}), + (c_id, d_id, {"relation": "imports_from", "source_file": "src/c.ts", "confidence": "EXTRACTED"}), + (d_id, b_id, {"relation": "imports_from", "source_file": "src/d.ts", "confidence": "EXTRACTED"}), + # Self-loop: c imports itself + (c_id, c_id, {"relation": "imports_from", "source_file": "src/c.ts", "confidence": "EXTRACTED"}), + # Mixed edge types: must not bleed into cycle graph + (a_id, ext_id, {"relation": "calls", "source_file": "src/a.ts", "confidence": "INFERRED"}), + (a_id, ext_id, {"relation": "contains", "source_file": "src/a.ts", "confidence": "EXTRACTED"}), + # Edge whose target has no source_file: must be skipped, no garbage fallback. + (a_id, ext_id, {"relation": "imports_from", "source_file": "src/a.ts", "confidence": "EXTRACTED"}), + ]) + return store + + +def test_find_import_cycles_returns_structured_records(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) assert isinstance(cycles, list) assert cycles @@ -661,40 +654,39 @@ def test_find_import_cycles_returns_structured_records(): assert "why" in cycles[0] -def test_find_import_cycles_detects_2_and_3_cycles(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_detects_2_and_3_cycles(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) cycle_sets = [set(c["cycle"]) for c in cycles] assert any({"src/a.ts", "src/b.ts"}.issubset(s) for s in cycle_sets) assert any({"src/b.ts", "src/c.ts", "src/d.ts"}.issubset(s) for s in cycle_sets) -def test_find_import_cycles_includes_self_loop_cycle(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_includes_self_loop_cycle(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) assert any(c["cycle"] == ["src/c.ts"] and c["length"] == 1 for c in cycles) -def test_find_import_cycles_respects_max_cycle_length(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_respects_max_cycle_length(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G, max_cycle_length=2) assert all(c["length"] <= 2 for c in cycles) -def test_find_import_cycles_skips_nodes_without_source_file(): - G = _make_cycle_graph_directed() +def test_find_import_cycles_skips_nodes_without_source_file(store): + G = _make_cycle_graph_directed(store) cycles = find_import_cycles(G) flat = " ".join(" ".join(c["cycle"]) for c in cycles) assert "react" not in flat -def test_find_import_cycles_handles_undirected_graph_input(): - Gd = _make_cycle_graph_directed() - Gu = nx.Graph() - Gu.add_nodes_from(Gd.nodes(data=True)) - Gu.add_edges_from(Gd.edges(data=True)) - cycles = find_import_cycles(Gu) - assert cycles # should still resolve orientation via edge.source_file +def test_find_import_cycles_handles_undirected_graph_input(store): + # Edges are stored directed natively; orientation is still resolved via + # edge.source_file, so cycle detection works regardless. + G = _make_cycle_graph_directed(store) + cycles = find_import_cycles(G) + assert cycles def test_find_import_cycles_ignores_non_import_relations(): @@ -713,8 +705,8 @@ def test_find_import_cycles_empty_graph(): assert find_import_cycles(nx.DiGraph()) == [] -def test_find_import_cycles_no_cycles(): - G = nx.DiGraph() +def test_find_import_cycles_no_cycles(store): + G = store x_id, x = _make_file_node("x.ts") y_id, y = _make_file_node("y.ts") G.add_node(x_id, **x) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index b5751adcc..67ffca2ae 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,9 +1,8 @@ """Tests for graphify/benchmark.py.""" from __future__ import annotations import json -import pytest -import networkx as nx -from networkx.readwrite import json_graph +from pathlib import Path +from tests import nxcompat as nx from graphify.benchmark import run_benchmark, print_benchmark, _query_subgraph_tokens, _SAMPLE_QUESTIONS, _safe, _hr @@ -23,8 +22,16 @@ def _make_graph() -> nx.Graph: def _write_graph(G: nx.Graph, path) -> None: - data = json_graph.node_link_data(G, edges="links") - path.write_text(json.dumps(data)) + """Seed the FalkorDB graph for path's output dir from an nx fixture graph.""" + from graphify.store import open_store + + store = open_store(Path(path).parent, create=True) + store.clear() + store.add_nodes_from([(n, dict(d, file_type=d.get("file_type", "code"))) for n, d in G.nodes(data=True)]) + store.add_edges_from([(u, v, dict(d)) for u, v, d in G.edges(data=True)]) + path.write_text(json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": [{"id": n, **d} for n, d in G.nodes(data=True)], + "links": []})) # --- _query_subgraph_tokens --- @@ -170,14 +177,3 @@ def test_print_benchmark_survives_cp1252_stdout(tmp_path, monkeypatch, capsys): # ASCII fallbacks must be present, fancy glyphs must not. assert "─" not in written assert "→" not in written - - -def test_run_benchmark_rejects_oversized_graph(monkeypatch, tmp_path): - """#F4: run_benchmark must refuse to read a graph.json that exceeds - the size cap before parsing it into memory.""" - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - run_benchmark(str(graph_file)) diff --git a/tests/test_build.py b/tests/test_build.py index 9be6c1289..6e8e07581 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,7 +1,5 @@ import json from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas FIXTURES = Path(__file__).parent / "fixtures" @@ -251,13 +249,11 @@ def test_build_from_json_preserves_first_direction_on_bidirectional_pair(tmp_pat "output_tokens": 0, } G = build_from_json(extraction) - # Only one undirected edge between the pair survives, but its stored - # direction must be the first-seen one (a_handler -> z_emitter), not the - # lexicographically-later one (z_emitter -> a_handler). + # Only one edge between the pair survives, stored in the first-seen direction + # (a_handler -> z_emitter). FalkorDB stores edges directed natively, so the + # surviving edge orientation itself encodes direction (no _src/_tgt markers). assert G.number_of_edges() == 1 - data = edge_data(G, "a_handler", "z_emitter") - assert data["_src"] == "a_handler" - assert data["_tgt"] == "z_emitter" + assert ("a_handler", "z_emitter") in list(G.edges()) graph_path = tmp_path / "graph.json" assert to_json(G, {}, str(graph_path), force=True) @@ -275,90 +271,28 @@ def test_build_from_json_preserves_first_direction_on_bidirectional_pair(tmp_pat ) -# Regression tests for #796 — edge_data / edge_datas helpers must tolerate -# MultiGraph and MultiDiGraph, which networkx's node_link_graph() produces -# whenever the loaded JSON has multigraph: true. Plain G.edges[u, v] crashes -# on those with `ValueError: not enough values to unpack (expected 3, got 2)`. +# edge_data / edge_datas helpers operate on the FalkorDB-backed GraphStore. +# (The old MultiGraph variants are gone: FalkorDB stores parallel edges natively, +# so there is no MultiGraph wrapper to tolerate.) -def test_edge_data_simple_graph(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - d = edge_data(G, "a", "b") +def test_edge_data_simple_graph(store): + store.add_nodes_from([("a", {"file_type": "code"}), ("b", {"file_type": "code"})]) + store.add_edge("a", "b", relation="calls", confidence="EXTRACTED") + d = edge_data(store, "a", "b") assert isinstance(d, dict) assert d["relation"] == "calls" assert d["confidence"] == "EXTRACTED" -def test_edge_datas_simple_graph_returns_singleton_list(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - ds = edge_datas(G, "a", "b") +def test_edge_datas_simple_graph_returns_singleton_list(store): + store.add_nodes_from([("a", {"file_type": "code"}), ("b", {"file_type": "code"})]) + store.add_edge("a", "b", relation="calls", confidence="EXTRACTED") + ds = edge_datas(store, "a", "b") assert isinstance(ds, list) assert len(ds) == 1 assert ds[0]["relation"] == "calls" -def test_edge_data_multigraph_with_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - # First parallel edge wins; should be one of the two attribute dicts above. - assert d.get("relation") in ("calls", "references") - - -def test_edge_datas_multigraph_returns_all_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - ds = edge_datas(G, "a", "b") - assert isinstance(ds, list) - assert len(ds) == 2 - relations = {e.get("relation") for e in ds} - assert relations == {"calls", "references"} - - -def test_edge_data_multidigraph(): - G = nx.MultiDiGraph() - G.add_edge("a", "b", relation="calls") - G.add_edge("a", "b", relation="imports") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "imports") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 - - -def test_edge_data_node_link_multigraph_roundtrip(): - """A node_link JSON with multigraph: true must load as MultiGraph and the - helpers must operate on it without raising the 3-tuple unpack ValueError.""" - data = { - "directed": False, - "multigraph": True, - "graph": {}, - "nodes": [ - {"id": "a", "label": "A"}, - {"id": "b", "label": "B"}, - ], - "links": [ - {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "a", "target": "b", "relation": "references", "confidence": "INFERRED"}, - ], - } - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) - assert isinstance(G, nx.MultiGraph) - # Plain G.edges[u, v] would raise here; the helper must not. - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "references") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 - - def test_build_from_json_relativizes_absolute_source_file(tmp_path): """Semantic subagents emit absolute source_file paths; build_from_json must relativize them to root so MCP traversal works correctly (#932).""" @@ -405,16 +339,13 @@ def test_build_from_json_relative_source_file_unchanged(tmp_path): assert G.nodes["foo_bar"]["source_file"] == "src/foo.py" -def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): +def test_build_merge_prune_absolute_paths_match_relative_nodes(store, tmp_path): """#1007: manifest stores absolute paths, graph nodes store relative paths. prune_sources with absolute paths must still remove the right nodes and edges.""" - import networkx as nx - root = tmp_path / "corpus" root.mkdir() - graph_path = tmp_path / "graph.json" - # Simulate a graph with relative source_file paths (as built normally) + # Build a graph with relative source_file paths into the store. chunk = {"nodes": [ {"id": "n1", "label": "login", "file_type": "code", "source_file": "module_a/auth.py"}, {"id": "n2", "label": "format_date", "file_type": "code", "source_file": "module_b/utils.py"}, @@ -422,12 +353,11 @@ def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", "source_file": "module_b/utils.py", "weight": 1.0}, ]} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + build([chunk], dedup=False, store=store) # prune_sources from manifest — absolute paths (what detect_incremental emits) deleted_abs = [str(root / "module_b" / "utils.py")] - G1 = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) + G1 = build_merge([], store.graph_name, prune_sources=deleted_abs, dedup=False, root=root) node_labels = {d["label"] for _, d in G1.nodes(data=True)} assert "format_date" not in node_labels, "stale node from deleted file should be pruned" @@ -436,35 +366,19 @@ def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): assert G1.number_of_edges() == 0, "edge from deleted source_file should be pruned" -def test_build_merge_prune_windows_backslash_paths(tmp_path): +def test_build_merge_prune_windows_backslash_paths(store, tmp_path): """#1007: prune_sources with Windows-style backslash absolute paths must still match.""" - import networkx as nx - root = tmp_path / "corpus" root.mkdir() - graph_path = tmp_path / "graph.json" chunk = {"nodes": [ {"id": "n1", "label": "parse_date", "file_type": "code", "source_file": "module_b/utils.py"}, ], "edges": []} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") + build([chunk], dedup=False, store=store) # Simulate Windows manifest path with backslashes win_path = str(root / "module_b" / "utils.py").replace("/", "\\") - G1 = build_merge([], graph_path, prune_sources=[win_path], dedup=False, root=root) + G1 = build_merge([], store.graph_name, prune_sources=[win_path], dedup=False, root=root) node_labels = {d["label"] for _, d in G1.nodes(data=True)} assert "parse_date" not in node_labels, "node should be pruned even with backslash path" - - -def test_build_merge_rejects_oversized_existing_graph(monkeypatch, tmp_path): - """#F4: build_merge must refuse to read an existing graph.json that - exceeds the size cap, rather than json.loads-ing it into memory.""" - import pytest - - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps({"nodes": [], "links": []}), encoding="utf-8") - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - build_merge([], graph_path, dedup=False) diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index e8a5bc25e..1e2937c39 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -16,13 +16,21 @@ FIXTURES = Path(__file__).parent / "fixtures" +_REPO_ROOT = str(Path(__file__).resolve().parent.parent) + + def _run(args: list[str], cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess: + # Run the CLI as a subprocess from `cwd`. Ensure the repo root is importable + # so `python -m graphify` works even in a non-pip-installed source checkout. + run_env = {**os.environ, **(env or {})} + existing = run_env.get("PYTHONPATH", "") + run_env["PYTHONPATH"] = _REPO_ROOT + (os.pathsep + existing if existing else "") return subprocess.run( [PYTHON, "-m", "graphify"] + args, cwd=cwd, capture_output=True, text=True, - env=env, + env=run_env, ) @@ -36,8 +44,13 @@ def _make_graph(tmp_path: Path) -> Path: from graphify.cluster import cluster, score_all from graphify.analyze import god_nodes, surprising_connections from graphify.export import to_json + from graphify.store import open_store - G = build_from_json(extraction) + # Build into the FalkorDB graph bound to this output dir (writes the pointer + # file) so the CLI subprocess loads the same graph. + G = open_store(out, create=True) + G.clear() + build_from_json(extraction, store=G) communities = cluster(G) cohesion = score_all(G, communities) gods = god_nodes(G) @@ -310,23 +323,23 @@ def test_cluster_only_remaps_labels_to_previous_cids(tmp_path): re-applies labels by raw index and they silently misalign with cluster contents (#1027). Mirror of the watch/update fix from #822. """ + from graphify.store import open_store + out = _make_graph(tmp_path) graph_json = out / "graph.json" labels_json = out / ".graphify_labels.json" - # Tag every node with an out-of-band community id and write a labels file - # keyed on those ids. After cluster-only, at least one of those sentinel - # ids must survive in the labels file (= remap succeeded by node overlap). - # If the cluster-only branch skips remap, Leiden returns small ints - # (0, 1, ...) and the sentinel keys disappear entirely. - g = json.loads(graph_json.read_text(encoding="utf-8")) - nodes = g.get("nodes", []) - assert len(nodes) >= 4, "fixture must have enough nodes to form 2+ communities" + # Tag every node with an out-of-band community id (on the FalkorDB store, + # which is what cluster-only reads as the prior assignment) and write a + # labels file keyed on those ids. After cluster-only, at least one of those + # sentinel ids must survive (= remap succeeded by node overlap). Without the + # remap, Leiden returns small ints (0, 1, ...) and the sentinel keys vanish. + store = open_store(out, create=False) + node_ids = sorted(n for n in store.nodes) + assert len(node_ids) >= 4, "fixture must have enough nodes to form 2+ communities" sentinel_a, sentinel_b = 4242, 9999 - half = len(nodes) // 2 - for i, n in enumerate(nodes): - n["community"] = sentinel_a if i < half else sentinel_b - graph_json.write_text(json.dumps(g), encoding="utf-8") + half = len(node_ids) // 2 + store.set_communities({sentinel_a: node_ids[:half], sentinel_b: node_ids[half:]}) labels_json.write_text( json.dumps({str(sentinel_a): "First Group", str(sentinel_b): "Second Group"}), encoding="utf-8", diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 21fd2ca3a..2ae30ae00 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,6 +1,6 @@ import json import sys -import networkx as nx +from tests import nxcompat as nx from pathlib import Path from graphify.build import build_from_json from graphify.cluster import cluster, cohesion_score, remap_communities_to_previous, score_all diff --git a/tests/test_confidence.py b/tests/test_confidence.py index 299548aca..8ffccb64d 100644 --- a/tests/test_confidence.py +++ b/tests/test_confidence.py @@ -3,7 +3,7 @@ import tempfile from pathlib import Path -import networkx as nx +from tests import nxcompat as nx from graphify.build import build_from_json from graphify.cluster import cluster, score_all diff --git a/tests/test_explain_cli.py b/tests/test_explain_cli.py index 1d00955f0..f47d95482 100644 --- a/tests/test_explain_cli.py +++ b/tests/test_explain_cli.py @@ -4,30 +4,34 @@ import graphify.__main__ as mainmod +_NODES = [ + {"id": "validate", "label": "validateSanitySession()", "source_file": "server/sanity-validate-session.ts", "community": 0, "file_type": "code"}, + {"id": "create_patch", "label": "createPatchHandler()", "source_file": "server/create-patch-handler.ts", "community": 0, "file_type": "code"}, + {"id": "create_edit", "label": "createEditHandler()", "source_file": "server/create-edit-handler.ts", "community": 0, "file_type": "code"}, + {"id": "stable_stringify", "label": "stableStringify()", "source_file": "shared/stringify.ts", "community": 0, "file_type": "code"}, +] +_LINKS = [ + {"source": "create_patch", "target": "validate", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "create_edit", "target": "validate", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "validate", "target": "stable_stringify", "relation": "calls", "confidence": "EXTRACTED"}, +] + + def _write_graph(tmp_path): - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "validate", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", "community": 0}, - {"id": "create_patch", "label": "createPatchHandler()", - "source_file": "server/create-patch-handler.ts", "community": 0}, - {"id": "create_edit", "label": "createEditHandler()", - "source_file": "server/create-edit-handler.ts", "community": 0}, - {"id": "stable_stringify", "label": "stableStringify()", - "source_file": "shared/stringify.ts", "community": 0}, - ], - "links": [ - {"source": "create_patch", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "create_edit", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "validate", "target": "stable_stringify", - "relation": "calls", "confidence": "EXTRACTED"}, - ], - } + """Seed the FalkorDB graph for tmp_path so `explain` (which loads from the + store) finds it.""" + from graphify.store import open_store + + store = open_store(tmp_path, create=True) + store.clear() + store.add_nodes_from([(n["id"], {k: v for k, v in n.items() if k != "id"}) for n in _NODES]) + store.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in _LINKS + ]) p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) + p.write_text(json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": _NODES, "links": _LINKS})) return p diff --git a/tests/test_global_graph.py b/tests/test_global_graph.py index f40d9c6d5..1014b4680 100644 --- a/tests/test_global_graph.py +++ b/tests/test_global_graph.py @@ -1,209 +1,153 @@ """Tests for the global graph infrastructure (graphify/global_graph.py), prefix/prune helpers in graphify/build.py, and the cross-repo guard in -graphify/dedup.py.""" +graphify/dedup.py. FalkorDB-backed.""" from __future__ import annotations import json import pytest -import networkx as nx from unittest.mock import patch -# ── helpers ────────────────────────────────────────────────────────────────── - -def _make_graph(nodes, edges=None): - """Build a simple nx.Graph from node dicts.""" - G = nx.Graph() - for n in nodes: - nid = n["id"] - G.add_node(nid, **{k: v for k, v in n.items() if k != "id"}) - for e in (edges or []): - G.add_edge( - e["source"], - e["target"], - **{k: v for k, v in e.items() if k not in ("source", "target")}, - ) - return G - - -def _graph_to_json(G, path): - from networkx.readwrite import json_graph as jg - try: - data = jg.node_link_data(G, edges="links") - except TypeError: - data = jg.node_link_data(G) - path.write_text(json.dumps(data), encoding="utf-8") - - # ── build.py helpers ────────────────────────────────────────────────────────── -def test_prefix_graph_preserves_label(): +def test_prefix_graph_preserves_label(store, make_store): from graphify.build import prefix_graph_for_global - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - H = prefix_graph_for_global(G, "repoA") + store.add_node("userservice", label="UserService", source_file="src/user.py", file_type="code") + target = make_store() + H = prefix_graph_for_global(store, "repoA", target) assert "repoA::userservice" in H.nodes assert "userservice" not in H.nodes assert H.nodes["repoA::userservice"]["label"] == "UserService" -def test_prefix_graph_sets_repo_and_local_id(): +def test_prefix_graph_sets_repo_and_local_id(store, make_store): from graphify.build import prefix_graph_for_global - G = _make_graph([{"id": "userservice", "label": "UserService"}]) - H = prefix_graph_for_global(G, "repoA") + store.add_node("userservice", label="UserService", file_type="code") + target = make_store() + H = prefix_graph_for_global(store, "repoA", target) data = H.nodes["repoA::userservice"] assert data["repo"] == "repoA" assert data["local_id"] == "userservice" -def test_prefix_graph_rewrites_edges(): +def test_prefix_graph_rewrites_edges(store, make_store): from graphify.build import prefix_graph_for_global - G = _make_graph( - [{"id": "a", "label": "A"}, {"id": "b", "label": "B"}], - [{"source": "a", "target": "b"}], - ) - H = prefix_graph_for_global(G, "repo1") + store.add_nodes_from([("a", {"label": "A", "file_type": "code"}), ("b", {"label": "B", "file_type": "code"})]) + store.add_edge("a", "b", relation="calls") + target = make_store() + H = prefix_graph_for_global(store, "repo1", target) assert H.has_edge("repo1::a", "repo1::b") assert not H.has_edge("a", "b") -def test_prune_repo_removes_correct_nodes(): +def test_prune_repo_removes_correct_nodes(store): from graphify.build import prune_repo_from_graph - G = nx.Graph() - G.add_node("repoA::userservice", repo="repoA", label="UserService") - G.add_node("repoB::userservice", repo="repoB", label="UserService") - G.add_node("repoA::auth", repo="repoA", label="Auth") - removed = prune_repo_from_graph(G, "repoA") + store.add_nodes_from([ + ("repoA::userservice", {"repo": "repoA", "label": "UserService", "file_type": "code"}), + ("repoB::userservice", {"repo": "repoB", "label": "UserService", "file_type": "code"}), + ("repoA::auth", {"repo": "repoA", "label": "Auth", "file_type": "code"}), + ]) + removed = prune_repo_from_graph(store, "repoA") assert removed == 2 - assert "repoB::userservice" in G.nodes - assert "repoA::userservice" not in G.nodes - assert "repoA::auth" not in G.nodes + assert "repoB::userservice" in store.nodes + assert "repoA::userservice" not in store.nodes + assert "repoA::auth" not in store.nodes -def test_prune_repo_returns_zero_if_not_present(): +def test_prune_repo_returns_zero_if_not_present(store): from graphify.build import prune_repo_from_graph - G = nx.Graph() - G.add_node("repoA::x", repo="repoA") - removed = prune_repo_from_graph(G, "repoB") + store.add_node("repoA::x", repo="repoA", file_type="code") + removed = prune_repo_from_graph(store, "repoB") assert removed == 0 - assert G.number_of_nodes() == 1 + assert store.number_of_nodes() == 1 # ── global_graph.py ─────────────────────────────────────────────────────────── -def test_global_add_creates_global_graph(tmp_path): - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G, src_graph) - +@pytest.fixture() +def global_env(tmp_path, falkordb_uri): + """Isolate the global graph to a unique FalkorDB name + temp manifest dir.""" + import os + from graphify.store import GraphStore global_dir = tmp_path / ".graphify" + name = f"pytest_global_{os.getpid()}_{id(tmp_path)}" with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - result = global_add(src_graph, "repoA") + patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"), \ + patch("graphify.global_graph._GLOBAL_NAME", name): + gs = GraphStore(graph_name=name, uri=falkordb_uri) + gs.clear() + try: + yield global_dir + finally: + try: + gs.clear() + except Exception: + pass + + +def test_global_add_creates_global_graph(tmp_path, seed_graph, global_env): + seed_graph(tmp_path, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add + result = global_add(tmp_path / "graph.json", "repoA") assert result["skipped"] is False assert result["nodes_added"] > 0 - manifest_path = global_dir / "global-manifest.json" - assert manifest_path.exists() - manifest = json.loads(manifest_path.read_text()) + manifest = json.loads((global_env / "global-manifest.json").read_text()) assert "repoA" in manifest["repos"] -def test_global_add_skip_on_unchanged_hash(tmp_path): - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G, src_graph) - - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - global_add(src_graph, "repoA") - result2 = global_add(src_graph, "repoA") - +def test_global_add_skip_on_unchanged_hash(tmp_path, seed_graph, global_env): + seed_graph(tmp_path, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add + global_add(tmp_path / "graph.json", "repoA") + result2 = global_add(tmp_path / "graph.json", "repoA") assert result2["skipped"] is True -def test_global_add_two_repos_no_collision(tmp_path): - g1 = tmp_path / "graph1.json" - g2 = tmp_path / "graph2.json" - G1 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - G2 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G1, g1) - _graph_to_json(G2, g2) - - global_dir = tmp_path / ".graphify" - global_graph_path = global_dir / "global-graph.json" - global_manifest_path = global_dir / "global-manifest.json" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_graph_path), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_manifest_path): - from graphify.global_graph import global_add, _load_global_graph - global_add(g1, "repoA") - global_add(g2, "repoB") - G = _load_global_graph() - +def test_global_add_two_repos_no_collision(tmp_path, seed_graph, global_env): + d1 = tmp_path / "r1" + d2 = tmp_path / "r2" + d1.mkdir(); d2.mkdir() + seed_graph(d1, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + seed_graph(d2, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add, _load_global_graph + global_add(d1 / "graph.json", "repoA") + global_add(d2 / "graph.json", "repoB") + G = _load_global_graph() assert "repoA::userservice" in G.nodes assert "repoB::userservice" in G.nodes assert G.number_of_nodes() == 2 # no silent merge -def test_global_remove(tmp_path): - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - _graph_to_json(G, src_graph) - - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add, global_remove - global_add(src_graph, "repoA") - removed = global_remove("repoA") - +def test_global_remove(tmp_path, seed_graph, global_env): + seed_graph(tmp_path, [{"id": "userservice", "label": "UserService", "source_file": "src/user.py", "file_type": "code"}]) + from graphify.global_graph import global_add, global_remove, global_list + global_add(tmp_path / "graph.json", "repoA") + removed = global_remove("repoA") assert removed > 0 - # manifest should no longer list repoA - need to re-patch for list call - global_dir2 = global_dir # same dir - with patch("graphify.global_graph._GLOBAL_DIR", global_dir2), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir2 / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir2 / "global-manifest.json"): - from graphify.global_graph import global_list - repos = global_list() - assert "repoA" not in repos - - -def test_global_remove_unknown_tag_raises(tmp_path): - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_remove - with pytest.raises(KeyError): - global_remove("nonexistent") + assert "repoA" not in global_list() -def test_global_add_collision_warning(tmp_path, capsys): - g1 = tmp_path / "graph1.json" - g2 = tmp_path / "graph2.json" - G = _make_graph([{"id": "x", "label": "X", "source_file": "x.py"}]) - _graph_to_json(G, g1) - _graph_to_json(G, g2) +def test_global_remove_unknown_tag_raises(tmp_path, global_env): + from graphify.global_graph import global_remove + with pytest.raises(KeyError): + global_remove("nonexistent") - global_dir = tmp_path / ".graphify" - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - global_add(g1, "myrepo") - global_add(g2, "myrepo") # different source path, same tag +def test_global_add_collision_warning(tmp_path, seed_graph, global_env, capsys): + d1 = tmp_path / "r1" + d2 = tmp_path / "r2" + d1.mkdir(); d2.mkdir() + seed_graph(d1, [{"id": "x", "label": "X", "source_file": "x.py", "file_type": "code"}]) + seed_graph(d2, [{"id": "x", "label": "X", "source_file": "x.py", "file_type": "code"}]) + from graphify.global_graph import global_add + global_add(d1 / "graph.json", "myrepo") + global_add(d2 / "graph.json", "myrepo") # different source path, same tag captured = capsys.readouterr() assert "warning" in captured.err.lower() or "warning" in captured.out.lower() -# ── dedup guard ─────────────────────────────────────────────────────────────── +# ── dedup guard (needs datasketch) ───────────────────────────────────────────── def test_dedup_raises_on_cross_repo_nodes(): from graphify.dedup import deduplicate_entities @@ -221,8 +165,8 @@ def test_dedup_ok_with_single_repo(): {"id": "repoA::userservice", "label": "UserService", "repo": "repoA"}, {"id": "repoA::auth", "label": "Auth", "repo": "repoA"}, ] - result_nodes, result_edges = deduplicate_entities(nodes, [], communities={}) - assert len(result_nodes) == 2 # no false merge + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 2 def test_dedup_ok_with_no_repo_attr(): @@ -231,68 +175,20 @@ def test_dedup_ok_with_no_repo_attr(): {"id": "userservice", "label": "UserService"}, {"id": "auth", "label": "Auth"}, ] - result_nodes, result_edges = deduplicate_entities(nodes, [], communities={}) + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) assert len(result_nodes) == 2 -# ── merge-graphs prefix ─────────────────────────────────────────────────────── - -def test_merge_graphs_prefixes_ids(tmp_path): - """merge-graphs should prefix node IDs with repo name to avoid silent collision.""" - from graphify.build import prefix_graph_for_global - from networkx.readwrite import json_graph as jg - - # Two graphs with same node ID - G1 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - G2 = _make_graph([{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}]) - - repo1 = tmp_path / "repo1" / "graphify-out" - repo2 = tmp_path / "repo2" / "graphify-out" - repo1.mkdir(parents=True) - repo2.mkdir(parents=True) - - g1_path = repo1 / "graph.json" - g2_path = repo2 / "graph.json" - _graph_to_json(G1, g1_path) - _graph_to_json(G2, g2_path) - - # Simulate what merge-graphs now does (prefix before compose) - graphs = [] - graph_paths = [g1_path, g2_path] - for gp in graph_paths: - data = json.loads(gp.read_text()) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = jg.node_link_graph(data, edges="links") - except TypeError: - G = jg.node_link_graph(data) - repo_tag = gp.parent.parent.name - graphs.append(prefix_graph_for_global(G, repo_tag)) - - merged = nx.Graph() - for G in graphs: - merged = nx.compose(merged, G) - - assert "repo1::userservice" in merged.nodes - assert "repo2::userservice" in merged.nodes - assert merged.number_of_nodes() == 2 # no silent collapse +# ── merge-graphs prefix (plain-dict node-link, no NetworkX) ──────────────────── +def test_merge_graphs_prefixes_ids(): + """merge-graphs prefixes node IDs with repo name to avoid silent collision.""" + from graphify.graphjson import prefix_node_link, merge_node_link, node_count -def test_global_add_rejects_oversized_source_graph(monkeypatch, tmp_path): - """#F4: global_add must refuse to read a source graph.json that - exceeds the size cap, rather than json.loads-ing it into memory.""" - import pytest - - src_graph = tmp_path / "graph.json" - G = _make_graph([{"id": "x", "label": "X", "source_file": "src/x.py"}]) - _graph_to_json(G, src_graph) - - global_dir = tmp_path / ".graphify" - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with patch("graphify.global_graph._GLOBAL_DIR", global_dir), \ - patch("graphify.global_graph._GLOBAL_GRAPH", global_dir / "global-graph.json"), \ - patch("graphify.global_graph._GLOBAL_MANIFEST", global_dir / "global-manifest.json"): - from graphify.global_graph import global_add - with pytest.raises(ValueError, match="exceeds"): - global_add(src_graph, "repoA") + g1 = {"nodes": [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}], "links": []} + g2 = {"nodes": [{"id": "userservice", "label": "UserService", "source_file": "src/user.py"}], "links": []} + merged = merge_node_link([prefix_node_link(g1, "repo1"), prefix_node_link(g2, "repo2")]) + ids = {n["id"] for n in merged["nodes"]} + assert "repo1::userservice" in ids + assert "repo2::userservice" in ids + assert node_count(merged) == 2 # no silent collapse diff --git a/tests/test_hypergraph.py b/tests/test_hypergraph.py index dda8ac793..39f46a092 100644 --- a/tests/test_hypergraph.py +++ b/tests/test_hypergraph.py @@ -4,7 +4,7 @@ import tempfile from pathlib import Path -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.build import build_from_json diff --git a/tests/test_labeling.py b/tests/test_labeling.py index 298d6e62d..246786b97 100644 --- a/tests/test_labeling.py +++ b/tests/test_labeling.py @@ -3,7 +3,7 @@ Backend calls are mocked - no network. Covers the happy path, partial replies, malformed replies, and the no-backend fallback. """ -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.llm import label_communities, generate_community_labels diff --git a/tests/test_multigraph_compat.py b/tests/test_multigraph_compat.py deleted file mode 100644 index 36902e691..000000000 --- a/tests/test_multigraph_compat.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -import networkx as nx - -from graphify.multigraph_compat import ( - CapabilityCheck, - MultigraphCapabilityResult, - probe_multigraph_capabilities, - require_multigraph_capabilities, -) - - -def test_probe_multigraph_capabilities_passes_current_runtime() -> None: - result = probe_multigraph_capabilities() - - assert result.ok, result.error_message() - assert result.python_version - assert result.networkx_version - assert {check.name for check in result.checks} == { - "keyed_parallel_edges", - "node_link_edges_links_round_trip", - "duplicate_key_overwrite_semantics", - "reserved_key_attr_rejected", - "remove_edges_from_two_tuple_semantics", - "to_undirected_preserves_multigraph_type", - } - - -def test_require_multigraph_capabilities_returns_result() -> None: - result = require_multigraph_capabilities() - - assert result.ok - - -def test_failure_message_is_actionable() -> None: - result = MultigraphCapabilityResult( - python_version="3.10.0", - networkx_version="0.0", - checks=(CapabilityCheck("node_link_edges_links_round_trip", False, "boom"),), - ) - - message = result.error_message() - - assert "--multigraph requires NetworkX keyed MultiDiGraph node-link" in message - assert "Default simple graph mode remains available" in message - assert "node_link_edges_links_round_trip: boom" in message - - -def test_networkx_duplicate_key_overwrite_trap_is_real() -> None: - graph = nx.MultiDiGraph() - - graph.add_edge("a", "b", key="same", relation="first") - graph.add_edge("a", "b", key="same", relation="second") - - assert graph.number_of_edges("a", "b") == 1 - assert graph["a"]["b"]["same"]["relation"] == "second" diff --git a/tests/test_multigraph_diagnostics.py b/tests/test_multigraph_diagnostics.py index 8c39b8e23..ab43bdc6c 100644 --- a/tests/test_multigraph_diagnostics.py +++ b/tests/test_multigraph_diagnostics.py @@ -99,8 +99,11 @@ def test_diagnose_extraction_categorizes_same_endpoint_collapse() -> None: assert summary["same_endpoint_group_count"] == 1 assert summary["relation_variant_groups"] == 1 assert summary["source_location_variant_groups"] == 1 - assert summary["post_build_graph_type"] == "DiGraph" - assert summary["post_build_edge_count"] == 2 + assert summary["post_build_graph_type"] == "GraphStore" + # FalkorDB stores parallel edges natively (one per relation type), so + # same-endpoint edges that differ by relation are preserved rather than + # collapsed the way nx.DiGraph used to (which silently dropped one). + assert summary["post_build_edge_count"] == 3 def test_diagnose_extraction_accepts_node_link_links_key() -> None: @@ -193,7 +196,7 @@ def test_diagnose_extraction_defaults_raw_inputs_to_directed(tmp_path: Path) -> summary = diagnose_file(graph_path) assert summary["effective_directed"] is True - assert summary["post_build_graph_type"] == "DiGraph" + assert summary["post_build_graph_type"] == "GraphStore" def test_diagnose_file_reads_json_and_formats_report(tmp_path: Path) -> None: @@ -302,7 +305,7 @@ def test_diagnose_file_defaults_to_json_directed_flag(tmp_path: Path) -> None: summary = diagnose_file(graph_path) assert summary["effective_directed"] is False - assert summary["post_build_graph_type"] == "Graph" + assert summary["post_build_graph_type"] == "GraphStore" def test_diagnose_file_explicit_directed_override(tmp_path: Path) -> None: @@ -314,7 +317,7 @@ def test_diagnose_file_explicit_directed_override(tmp_path: Path) -> None: summary = diagnose_file(graph_path, directed=True) assert summary["effective_directed"] is True - assert summary["post_build_graph_type"] == "DiGraph" + assert summary["post_build_graph_type"] == "GraphStore" def test_scan_producer_suppression_sites_reports_missing_file(tmp_path: Path) -> None: @@ -360,7 +363,7 @@ def test_diagnose_multigraph_cli_undirected_override(monkeypatch, tmp_path: Path out = capsys.readouterr().out assert "effective_directed: False" in out - assert "post_build_graph_type: Graph" in out + assert "post_build_graph_type: GraphStore" in out def test_diagnose_multigraph_cli_max_examples_zero(monkeypatch, tmp_path: Path, capsys) -> None: diff --git a/tests/test_obsidian_filename_cap.py b/tests/test_obsidian_filename_cap.py index 2d7f1c85d..2000aba53 100644 --- a/tests/test_obsidian_filename_cap.py +++ b/tests/test_obsidian_filename_cap.py @@ -1,7 +1,7 @@ """Regression tests for issue #1094: to_obsidian / to_canvas must cap filenames to stay under the 255-byte filesystem limit, instead of crashing with OSError ENAMETOOLONG on long node labels.""" -import networkx as nx +from tests import nxcompat as nx from graphify.export import to_obsidian, to_canvas diff --git a/tests/test_path_cli.py b/tests/test_path_cli.py index de7e8837f..e9e96af7b 100644 --- a/tests/test_path_cli.py +++ b/tests/test_path_cli.py @@ -1,28 +1,16 @@ """Regression tests for `graphify path` arrow direction (#849).""" from __future__ import annotations -import json -import networkx as nx -from networkx.readwrite import json_graph import graphify.__main__ as mainmod - -def _write_graph(tmp_path): - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "create_patch", "label": "createPatchHandler()", - "source_file": "server/create-patch-handler.ts", "community": 0}, - {"id": "validate", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", "community": 0}, - ], - "links": [ - {"source": "create_patch", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - ], - } - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - return p +_NODES = [ + {"id": "create_patch", "label": "createPatchHandler()", + "source_file": "server/create-patch-handler.ts", "community": 0, "file_type": "code"}, + {"id": "validate", "label": "validateSanitySession()", + "source_file": "server/sanity-validate-session.ts", "community": 0, "file_type": "code"}, +] +_LINKS = [ + {"source": "create_patch", "target": "validate", "relation": "calls", "confidence": "EXTRACTED"}, +] def _run(monkeypatch, graph_path, src, tgt, capsys): @@ -33,15 +21,17 @@ def _run(monkeypatch, graph_path, src, tgt, capsys): return capsys.readouterr().out -def test_forward_arrow(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) +def test_forward_arrow(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + p = tmp_path / "graph.json" out = _run(monkeypatch, p, "createPatchHandler", "validateSanitySession", capsys) assert "Shortest path (1 hops):" in out assert "createPatchHandler() --calls [EXTRACTED]--> validateSanitySession()" in out -def test_reverse_arrow(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) +def test_reverse_arrow(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + p = tmp_path / "graph.json" out = _run(monkeypatch, p, "validateSanitySession", "createPatchHandler", capsys) assert "Shortest path (1 hops):" in out assert "validateSanitySession() <--calls [EXTRACTED]-- createPatchHandler()" in out diff --git a/tests/test_prs.py b/tests/test_prs.py index 61ebc140e..3da5fe87b 100644 --- a/tests/test_prs.py +++ b/tests/test_prs.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone from unittest.mock import patch, MagicMock -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.prs import ( diff --git a/tests/test_query_cli.py b/tests/test_query_cli.py index cf8eb6e56..0b9a5cd23 100644 --- a/tests/test_query_cli.py +++ b/tests/test_query_cli.py @@ -1,28 +1,22 @@ """Tests for graphify query CLI context filtering.""" from __future__ import annotations -import json - -import networkx as nx -from networkx.readwrite import json_graph - import graphify.__main__ as mainmod - -def _write_graph(tmp_path): - G = nx.Graph() - G.add_node("n1", label="extract", source_file="extract.py", source_location="L10", community=0) - G.add_node("n2", label="cluster", source_file="cluster.py", source_location="L5", community=0) - G.add_node("n3", label="build", source_file="build.py", source_location="L1", community=1) - G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", context="call") - G.add_edge("n2", "n3", relation="imports", confidence="EXTRACTED", context="import") - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(G, edges="links"))) - return graph_path +_NODES = [ + {"id": "n1", "label": "extract", "source_file": "extract.py", "source_location": "L10", "community": 0, "file_type": "code"}, + {"id": "n2", "label": "cluster", "source_file": "cluster.py", "source_location": "L5", "community": 0, "file_type": "code"}, + {"id": "n3", "label": "build", "source_file": "build.py", "source_location": "L1", "community": 1, "file_type": "code"}, +] +_LINKS = [ + {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", "context": "call"}, + {"source": "n2", "target": "n3", "relation": "imports", "confidence": "EXTRACTED", "context": "import"}, +] -def test_query_cli_explicit_context_filter(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +def test_query_cli_explicit_context_filter(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -36,8 +30,9 @@ def test_query_cli_explicit_context_filter(monkeypatch, tmp_path, capsys): assert "build" not in out -def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys, seed_graph): + seed_graph(tmp_path, _NODES, _LINKS) + graph_path = tmp_path / "graph.json" monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -49,22 +44,3 @@ def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys): assert "Context: call (heuristic)" in out assert "cluster" in out assert "build" not in out - - -def test_query_cli_rejects_oversized_graph(monkeypatch, tmp_path, capsys): - """#F4: query CLI must refuse to parse a graph.json that exceeds the cap.""" - import pytest - - graph_path = _write_graph(tmp_path) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) - monkeypatch.setattr( - mainmod.sys, - "argv", - ["graphify", "query", "extract", "--graph", str(graph_path)], - ) - with pytest.raises(SystemExit): - mainmod.main() - err = capsys.readouterr().err - assert "exceeds" in err - assert "byte cap" in err diff --git a/tests/test_semantic_similarity.py b/tests/test_semantic_similarity.py index 55d9cce34..9f93ca701 100644 --- a/tests/test_semantic_similarity.py +++ b/tests/test_semantic_similarity.py @@ -1,5 +1,5 @@ """Tests for semantically_similar_to edge support.""" -import networkx as nx +from tests import nxcompat as nx import pytest from graphify.build import build_from_json from graphify.analyze import surprising_connections, _surprise_score diff --git a/tests/test_serve.py b/tests/test_serve.py index c98dece2b..d2114acad 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -1,8 +1,7 @@ """Tests for serve.py - MCP graph query helpers (no mcp package required).""" import json import pytest -import networkx as nx -from networkx.readwrite import json_graph +from tests import nxcompat as nx from graphify.serve import ( _communities_from_graph, @@ -259,14 +258,19 @@ def test_query_graph_text_heuristic_context_filter_changes_traversal(): # --- _load_graph --- -def test_load_graph_roundtrip(tmp_path): - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") - p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - G2 = _load_graph(str(p)) - assert G2.number_of_nodes() == G.number_of_nodes() - assert G2.number_of_edges() == G.number_of_edges() +_LOAD_NODES = [ + {"id": "n1", "label": "extract", "source_file": "extract.py", "source_location": "L10", "community": 0, "file_type": "code"}, + {"id": "n2", "label": "cluster", "source_file": "cluster.py", "source_location": "L5", "community": 0, "file_type": "code"}, + {"id": "n3", "label": "build", "source_file": "build.py", "source_location": "L1", "community": 1, "file_type": "code"}, +] +_LOAD_LINKS = [{"source": "n1", "target": "n2", "relation": "calls", "confidence": "INFERRED"}] + + +def test_load_graph_roundtrip(tmp_path, seed_graph): + seed_graph(tmp_path, _LOAD_NODES, _LOAD_LINKS) + G2 = _load_graph(str(tmp_path / "graph.json")) + assert G2.number_of_nodes() == 3 + assert G2.number_of_edges() == 1 def test_load_graph_missing_file(tmp_path): graphify_dir = tmp_path / "graphify-out" @@ -275,41 +279,23 @@ def test_load_graph_missing_file(tmp_path): _load_graph(str(graphify_dir / "nonexistent.json")) -def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path, capsys): - # #F4: oversized graph.json must fail fast (SystemExit) with a clear error. - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") - p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) - with pytest.raises(SystemExit): - _load_graph(str(p)) - err = capsys.readouterr().err - assert "exceeds" in err - assert "byte cap" in err - - -def test_load_graph_accepts_under_cap(monkeypatch, tmp_path): - # Verifies the cap path does not regress the normal load. - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") - p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - # Cap well above the actual file size — load proceeds. - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 10 * 1024 * 1024) - G2 = _load_graph(str(p)) - assert G2.number_of_nodes() == G.number_of_nodes() - - # --- #874: MCP hot-reload --- def _write_graph(path, nodes: list[str]) -> None: - """Write a minimal graph.json with the given node IDs.""" - G = nx.DiGraph() - for n in nodes: - G.add_node(n, label=n, community=0) - data = json_graph.node_link_data(G, edges="links") - path.write_text(json.dumps(data), encoding="utf-8") + """Seed the FalkorDB graph for path's output dir with the given node IDs.""" + from pathlib import Path as _Path + from graphify.store import open_store + + store = open_store(_Path(path).parent, create=True) + store.clear() + store.add_nodes_from([(n, {"label": n, "community": 0, "file_type": "code"}) for n in nodes]) + # Also keep a graph.json artifact so file-stat based tests still work. + import json as _json + _Path(path).write_text( + _json.dumps({"directed": True, "multigraph": False, "graph": {}, + "nodes": [{"id": n, "label": n, "community": 0} for n in nodes], "links": []}), + encoding="utf-8", + ) def test_maybe_reload_detects_graph_change(tmp_path): @@ -470,7 +456,7 @@ def test_query_seeds_from_identifier_not_noise(): def test_query_graph_text_parameter_type_context_filter_changes_traversal(): - import networkx as nx + from tests import nxcompat as nx from graphify.serve import _query_graph_text graph = nx.Graph() @@ -488,7 +474,7 @@ def test_query_graph_text_parameter_type_context_filter_changes_traversal(): def test_query_graph_text_context_filter_aliases_resolve(): - import networkx as nx + from tests import nxcompat as nx from graphify.serve import _normalize_context_filters assert _normalize_context_filters(["param"]) == ["parameter_type"] diff --git a/tests/test_serve_http.py b/tests/test_serve_http.py index e58205339..c665be1ce 100644 --- a/tests/test_serve_http.py +++ b/tests/test_serve_http.py @@ -47,6 +47,19 @@ def _graph_file(tmp_path: Path) -> str: + """Seed the FalkorDB graph for tmp_path so the server's loader finds it.""" + from graphify.store import open_store + + store = open_store(tmp_path, create=True) + store.clear() + store.add_nodes_from([ + (n["id"], dict({k: v for k, v in n.items() if k != "id"}, file_type="code")) + for n in SAMPLE_GRAPH["nodes"] + ]) + store.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in SAMPLE_GRAPH["edges"] + ]) p = tmp_path / "graph.json" p.write_text(json.dumps(SAMPLE_GRAPH), encoding="utf-8") return str(p) diff --git a/tests/test_wiki.py b/tests/test_wiki.py index 063d47ed6..17c254d4a 100644 --- a/tests/test_wiki.py +++ b/tests/test_wiki.py @@ -1,7 +1,7 @@ """Tests for graphify.wiki — Wikipedia-style article generation.""" import pytest from pathlib import Path -import networkx as nx +from tests import nxcompat as nx from graphify.wiki import to_wiki, _index_md, _community_article, _god_node_article diff --git a/tools/skillgen/expected/graphify__skills__amp__references__query.md b/tools/skillgen/expected/graphify__skills__amp__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__query.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__query.md b/tools/skillgen/expected/graphify__skills__claw__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__query.md b/tools/skillgen/expected/graphify__skills__codex__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__query.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__query.md b/tools/skillgen/expected/graphify__skills__copilot__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__query.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__query.md b/tools/skillgen/expected/graphify__skills__droid__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__query.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__query.md b/tools/skillgen/expected/graphify__skills__kilo__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__query.md b/tools/skillgen/expected/graphify__skills__kiro__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__query.md b/tools/skillgen/expected/graphify__skills__opencode__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__query.md b/tools/skillgen/expected/graphify__skills__pi__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__query.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__query.md b/tools/skillgen/expected/graphify__skills__trae__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__query.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__query.md b/tools/skillgen/expected/graphify__skills__vscode__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__query.md b/tools/skillgen/expected/graphify__skills__windows__references__query.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__query.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__query.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` diff --git a/tools/skillgen/fragments/references/query/cli-inline.md b/tools/skillgen/fragments/references/query/cli-inline.md index 25b826f8e..304447582 100644 --- a/tools/skillgen/fragments/references/query/cli-inline.md +++ b/tools/skillgen/fragments/references/query/cli-inline.md @@ -1,6 +1,6 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. +Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline GraphStore traversal otherwise. Two traversal modes - choose based on the question: @@ -12,8 +12,9 @@ Two traversal modes - choose based on the question: First check the graph exists: ```bash $(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): +from graphify.store import open_store +G = open_store('graphify-out', create=False) +if G.number_of_nodes() == 0: print('ERROR: No graph found. Run /graphify first to build the graph.') raise SystemExit(1) " @@ -26,7 +27,7 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: +If the CLI is unavailable, open the FalkorDB-backed graph and run the traversal inline: 1. Find the 1-3 nodes whose label best matches key terms in the question. 2. Run the appropriate traversal from each starting node. @@ -36,13 +37,10 @@ If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal ```bash $(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) question = 'QUESTION' mode = 'MODE' # 'bfs' or 'dfs' @@ -111,7 +109,7 @@ for nid in ranked_nodes: lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') for u, v in subgraph_edges: if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + d = G[u][v] lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') output = '\n'.join(lines) @@ -139,13 +137,10 @@ Find the shortest path between two named concepts in the graph. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) a_term = 'NODE_A' b_term = 'NODE_B' @@ -166,22 +161,20 @@ if not src or not tgt: print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') sys.exit(0) -try: - path = nx.shortest_path(G, src, tgt) +path = G.shortest_path(src, tgt) +if not path: + print(f'No path found between {a_term!r} and {b_term!r}') +else: print(f'Shortest path ({len(path)-1} hops):') for i, nid in enumerate(path): label = G.nodes[nid].get('label', nid) if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][path[i+1]] rel = edge.get('relation', '') conf = edge.get('confidence', '') print(f' {label} --{rel}--> [{conf}]') else: print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') " ``` @@ -201,13 +194,10 @@ Give a plain-language explanation of a single node - everything connected to it. ```bash $(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path +import sys +from graphify.store import open_store -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') +G = open_store('graphify-out', create=False) term = 'NODE_NAME' term_lower = term.lower() @@ -231,7 +221,7 @@ print(f' degree: {G.degree(nid)}') print() print('CONNECTIONS:') for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw + edge = G[nid][neighbor] nlabel = G.nodes[neighbor].get('label', neighbor) rel = edge.get('relation', '') conf = edge.get('confidence', '') diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index f53974a66..66084ca1c 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -94,11 +94,14 @@ new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_tex incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) deleted = list(incremental.get('deleted_files', [])) -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). +# build_merge() merges the new chunk into the existing FalkorDB graph for this +# output dir. Edge direction (calls, implements, imports) is stored natively. +from graphify.store import open_store +_store = open_store('graphify-out', create=True) G = build_merge( [new_extraction], - graph_path='graphify-out/graph.json', + graph_name=_store.graph_name, + uri=_store.uri, prune_sources=deleted or None, ) print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') @@ -107,9 +110,9 @@ print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edge merged_out = { 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} + # Edges are stored in native source->target direction. + {**{k: val for k, val in d.items() if k not in ('source', 'target')}, + 'source': u, 'target': v} for u, v, d in G.edges(data=True) ], # G.graph["hyperedges"] holds hyperedges from both existing graph.json @@ -138,23 +141,25 @@ $(cat graphify-out/.graphify_python) -c " import json from graphify.analyze import graph_diff from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx +from graphify.store import open_store from pathlib import Path -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract) +# Current (post-merge) graph lives in FalkorDB for this output dir. +G_new = open_store('graphify-out', create=False) +# Load old graph (before update) from backup written before merge and rebuild it +# into a scratch FalkorDB graph just for the comparison. +old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') + old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} + G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) if diff['new_edges']: print('New edges:', len(diff['new_edges'])) + G_old.clear() # drop the scratch graph " ``` From 6e689de79f5e814368dc92a71917666a8dadb617 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 09:57:43 +0300 Subject: [PATCH 02/17] refactor: rename graph-connection helpers to reflect they connect, not load Post-migration these open a cache-free FalkorDB store handle (a connection) and never load the graph into memory, so the NetworkX-era "load" naming was misleading: - serve._load_graph -> serve._connect_graph - affected.load_graph -> affected.connect_graph - CLI messages: "Loading existing graph..." -> "Connecting to existing graph..."; "could not load graph" -> "could not connect to graph" - clarified the MCP hot-reload docstring (it reconnects when the graph.json change-sentinel is rewritten; it does not reload the file) Genuine loaders are unchanged: callflow_html.load_graph (parses graph.json into data), the merge-driver graph.json loaders, and _load_graphifyignore/include. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/__main__.py | 30 +++++++++++++++--------------- graphify/affected.py | 6 ++++-- graphify/serve.py | 15 +++++++++------ tests/test_serve.py | 14 +++++++------- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index f4d2de93b..0fccbd92a 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -78,7 +78,7 @@ def _enforce_graph_size_cap_or_exit(gp: Path) -> None: Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. Use this from ``__main__.py`` subcommands that already use the ``print + - sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, + sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._connect_graph``, ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, ``global_graph``, ``watch``, ``export``) call the security helper directly and let the ``ValueError`` propagate. @@ -2511,7 +2511,7 @@ def main() -> None: if len(sys.argv) < 3: print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) sys.exit(1) - from graphify.serve import _query_graph_text, _load_graph + from graphify.serve import _query_graph_text, _connect_graph from graphify.security import sanitize_label from graphify import querylog @@ -2549,7 +2549,7 @@ def main() -> None: else: i += 1 gp = Path(graph_path).resolve() - G = _load_graph(graph_path) + G = _connect_graph(graph_path) import time as _time _t0 = _time.perf_counter() _mode = "dfs" if use_dfs else "bfs" @@ -2576,7 +2576,7 @@ def main() -> None: if len(sys.argv) < 3: print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) sys.exit(1) - from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph + from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, connect_graph query = sys.argv[2] graph_path = "graphify-out/graph.json" depth = 2 @@ -2620,9 +2620,9 @@ def main() -> None: print("error: graph file must be a .json file", file=sys.stderr) sys.exit(1) try: - graph = load_graph(gp) + graph = connect_graph(gp) except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) + print(f"error: could not connect to graph: {exc}", file=sys.stderr) sys.exit(1) print( format_affected( @@ -2660,7 +2660,7 @@ def main() -> None: file=sys.stderr, ) sys.exit(1) - from graphify.serve import _score_nodes, _load_graph + from graphify.serve import _score_nodes, _connect_graph source_label = sys.argv[2] target_label = sys.argv[3] @@ -2670,7 +2670,7 @@ def main() -> None: if a == "--graph" and i + 1 < len(args): graph_path = args[i + 1] gp = Path(graph_path).resolve() - G = _load_graph(graph_path) + G = _connect_graph(graph_path) src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) if not src_scored: @@ -2737,7 +2737,7 @@ def main() -> None: if len(sys.argv) < 3: print('Usage: graphify explain "" [--graph path]', file=sys.stderr) sys.exit(1) - from graphify.serve import _find_node, _load_graph + from graphify.serve import _find_node, _connect_graph label = sys.argv[2] graph_path = _default_graph_path() @@ -2746,7 +2746,7 @@ def main() -> None: if a == "--graph" and i + 1 < len(args): graph_path = args[i + 1] gp = Path(graph_path).resolve() - G = _load_graph(graph_path) + G = _connect_graph(graph_path) matches = _find_node(G, label) if not matches: print(f"No node matching '{label}' found.") @@ -2978,7 +2978,7 @@ def main() -> None: file=sys.stderr, ) sys.exit(1) - from graphify.serve import _load_graph + from graphify.serve import _connect_graph from graphify.cluster import cluster, score_all, remap_communities_to_previous from graphify.analyze import ( god_nodes, @@ -2988,8 +2988,8 @@ def main() -> None: from graphify.report import generate from graphify.export import to_json, to_html - print("Loading existing graph...") - G = _load_graph(str(graph_json)) + print("Connecting to existing graph...") + G = _connect_graph(str(graph_json)) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") print("Re-clustering...") communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) @@ -3429,9 +3429,9 @@ def main() -> None: print(f"callflow HTML written - open in any browser: {out}") sys.exit(0) - from graphify.serve import _load_graph + from graphify.serve import _connect_graph - G = _load_graph(str(graph_path)) + G = _connect_graph(str(graph_path)) # Load optional analysis/labels communities: dict[int, list[str]] = {} diff --git a/graphify/affected.py b/graphify/affected.py index eae6d9dba..6fc19ccab 100644 --- a/graphify/affected.py +++ b/graphify/affected.py @@ -163,8 +163,10 @@ def _label(nid): return "\n".join(lines) -def load_graph(path: Path): - """Open the FalkorDB-backed graph for the output dir containing `path`. +def connect_graph(path: Path): + """Open a connection to the FalkorDB-backed graph for the output dir containing + `path`. Cache-free: returns a store handle (a connection), it does NOT load the + graph into memory. `path` is the legacy graph.json location (e.g. graphify-out/graph.json); we use its parent directory to locate the FalkorDB pointer and open the store. diff --git a/graphify/serve.py b/graphify/serve.py index 3a682bedb..b16c2dc28 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -15,8 +15,10 @@ _jieba = None -def _load_graph(graph_path: str): - """Open the FalkorDB-backed graph for the output dir containing `graph_path`. +def _connect_graph(graph_path: str): + """Open a connection to the FalkorDB-backed graph for the output dir containing + `graph_path`. Cache-free: this returns a store handle (a connection), it does + NOT load the graph into memory. `graph_path` is the legacy graph.json location; its parent directory holds the FalkorDB pointer (falkordb.json). Errors out if no graph has been built yet. @@ -592,8 +594,9 @@ def _build_server(graph_path: str): All graph query tools and resources are registered here over a single ``mcp.server.Server`` instance; the caller picks the transport (stdio or - Streamable HTTP) and runs it. Hot-reload of graph.json works the same way - regardless of transport, since reloads happen inside the tool handlers. + Streamable HTTP) and runs it. Hot-reload (reconnecting to the store when the + graph.json change-sentinel is rewritten on disk) works the same way regardless + of transport, since the reconnect happens inside the tool handlers. """ import threading @@ -604,7 +607,7 @@ def _build_server(graph_path: str): except ImportError as e: raise ImportError('mcp not installed. Run: pip install "graphifyy[mcp]"') from e - G = _load_graph(graph_path) + G = _connect_graph(graph_path) communities = _communities_from_graph(G) # Hot-reload state: mtime+size key lets us detect graph.json changes without @@ -635,7 +638,7 @@ def _maybe_reload() -> None: if key == (_reload_state["mtime_ns"], _reload_state["size"]): return # another thread already reloaded try: - new_G = _load_graph(graph_path) + new_G = _connect_graph(graph_path) except SystemExit: return # keep serving stale graph on transient read error G = new_G diff --git a/tests/test_serve.py b/tests/test_serve.py index d2114acad..87e1911dc 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -17,7 +17,7 @@ _query_graph_text, _resolve_context_filters, _subgraph_to_text, - _load_graph, + _connect_graph, ) @@ -256,7 +256,7 @@ def test_query_graph_text_heuristic_context_filter_changes_traversal(): assert "build" not in text -# --- _load_graph --- +# --- _connect_graph --- _LOAD_NODES = [ {"id": "n1", "label": "extract", "source_file": "extract.py", "source_location": "L10", "community": 0, "file_type": "code"}, @@ -268,7 +268,7 @@ def test_query_graph_text_heuristic_context_filter_changes_traversal(): def test_load_graph_roundtrip(tmp_path, seed_graph): seed_graph(tmp_path, _LOAD_NODES, _LOAD_LINKS) - G2 = _load_graph(str(tmp_path / "graph.json")) + G2 = _connect_graph(str(tmp_path / "graph.json")) assert G2.number_of_nodes() == 3 assert G2.number_of_edges() == 1 @@ -276,7 +276,7 @@ def test_load_graph_missing_file(tmp_path): graphify_dir = tmp_path / "graphify-out" graphify_dir.mkdir() with pytest.raises(SystemExit): - _load_graph(str(graphify_dir / "nonexistent.json")) + _connect_graph(str(graphify_dir / "nonexistent.json")) # --- #874: MCP hot-reload --- @@ -308,15 +308,15 @@ def test_maybe_reload_detects_graph_change(tmp_path): graph_path = out / "graph.json" _write_graph(graph_path, ["alpha", "beta"]) - # Bootstrap _load_graph + _communities_from_graph to verify the reload path - G1 = _load_graph(str(graph_path)) + # Bootstrap _connect_graph + _communities_from_graph to verify the reload path + G1 = _connect_graph(str(graph_path)) assert set(G1.nodes()) == {"alpha", "beta"} # Simulate file changing (bump mtime by touching) time.sleep(0.01) _write_graph(graph_path, ["alpha", "beta", "gamma"]) - G2 = _load_graph(str(graph_path)) + G2 = _connect_graph(str(graph_path)) assert "gamma" in G2.nodes() From b6e54ff410e08a2b1bcb434a9749801e552c1be7 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 10:51:21 +0300 Subject: [PATCH 03/17] review: FalkorDB-only consistency + lazy UDF load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From section-by-section review of the migration: - __main__.py: `affected` and `cluster-only` no longer require the legacy graph.json file to exist on disk. In FalkorDB-only mode the graph lives in the engine (located via the falkordb.json pointer) and the artifact may be absent; these commands now rely on _connect_graph's pointer resolution + empty-graph guard, matching `query`/`path`/`explain`. Previously `query` worked but `affected`/`cluster-only` failed with "graph file not found". - store.py: UDFs (louvain/edgeBetweenness/simpleCycles) are now loaded lazily by the algorithm wrappers that use them instead of in GraphStore.__init__, so the interactive commands (query/path/explain/affected) — which never use UDFs — no longer pay a UDF reload round-trip on every process. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/__main__.py | 19 +++++++------------ graphify/store.py | 7 ++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/graphify/__main__.py b/graphify/__main__.py index 0fccbd92a..c95b2853b 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2612,13 +2612,11 @@ def main() -> None: i += 1 else: i += 1 + # FalkorDB-only: the graph lives in the engine, located via the + # falkordb.json pointer next to graph_path — the legacy graph.json file + # need not exist. connect_graph resolves the pointer and guards the + # empty/missing case (matches the `query` command). gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) try: graph = connect_graph(gp) except Exception as exc: @@ -2971,13 +2969,10 @@ def main() -> None: i_arg += 1 if watch_path is None: watch_path = Path(".") + # FalkorDB-only: graph_json locates the falkordb.json pointer (its parent + # dir); the legacy graph.json artifact need not exist. _connect_graph + # resolves the pointer and errors with "run /graphify" if no graph is built. graph_json = graph_override if graph_override is not None else watch_path / "graphify-out" / "graph.json" - if not graph_json.exists(): - print( - f"error: no graph found at {graph_json} — run /graphify first", - file=sys.stderr, - ) - sys.exit(1) from graphify.serve import _connect_graph from graphify.cluster import cluster, score_all, remap_communities_to_previous from graphify.analyze import ( diff --git a/graphify/store.py b/graphify/store.py index 04bc6ec4a..7c664bb1d 100644 --- a/graphify/store.py +++ b/graphify/store.py @@ -560,7 +560,9 @@ def __init__( self.graph: dict = {} # graph-level metadata only (hyperedges) — NOT the graph self._ensure_resultset_size() self._ensure_schema() - self._ensure_udfs() + # UDFs (louvain/edgeBetweenness/simpleCycles) are loaded lazily by the + # algorithm wrappers that need them — not here — so interactive commands + # (query/path/explain/affected) don't pay a UDF reload on every process. self._load_meta() # ---- views (nx-compatible, cache-free: each access is a live query) ----- @@ -1154,6 +1156,7 @@ def run_louvain(self, edges, resolution: float = 1.0) -> dict: """Run the Louvain UDF over an explicit weighted edge list.""" if not edges: return {} + self._ensure_udfs() rows = self._rows(f"RETURN {_UDF_LIB}.louvain($e, $res)", {"e": edges, "res": resolution}, timeout=_QUERY_TIMEOUT_MS) return {str(k): int(v) for k, v in dict(rows[0][0]).items()} @@ -1168,6 +1171,7 @@ def edge_betweenness(self) -> dict: "MATCH (a:Entity)-[r]->(b:Entity) RETURN a.id, b.id", "id(r)")] if not edges: return {} + self._ensure_udfs() res = self._rows(f"RETURN {_UDF_LIB}.edgeBetweenness($e)", {"e": edges}, timeout=_QUERY_TIMEOUT_MS) out: dict = {} for key, score in dict(res[0][0]).items(): @@ -1179,6 +1183,7 @@ def simple_cycles(self, edges, max_len: int = 5): payload = [[str(u), str(v), 1.0] for u, v in edges] if not payload: return [] + self._ensure_udfs() rows = self._rows(f"RETURN {_UDF_LIB}.simpleCycles($e, $n)", {"e": payload, "n": int(max_len)}, timeout=_QUERY_TIMEOUT_MS) return [[str(x) for x in cyc] for cyc in rows[0][0]] From 905131f4287812e5a526a4b10505308b2e902cdb Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 11:08:06 +0300 Subject: [PATCH 04/17] review: bound two O(n^2)/whole-graph costs on large graphs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From continued section-by-section review: - analyze.suggest_questions: cap exact node-betweenness at 5000 nodes (same guard _cross_community_surprises already uses). Exact betweenness is ~O(V*E); without a cap, report/cluster-only could be very slow on large repos. The bridge-node questions are supplementary — ambiguous-edge + community questions are unaffected. - export.to_svg: render a top-degree overview (>600 nodes) instead of the full graph. The pure-Python force layout (_spring_layout) is O(n^2) per iteration, so a full-graph SVG of a large graph would effectively hang (verified: a 15k-node graph now renders a 600-node overview in ~9s instead of never). An SVG of thousands of nodes is an unreadable hairball anyway. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/analyze.py | 8 ++++++-- graphify/export.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/graphify/analyze.py b/graphify/analyze.py index 2eafc1b9f..597bf3fed 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -431,8 +431,12 @@ def suggest_questions( "why": f"Edge tagged AMBIGUOUS (relation: {relation}) - confidence is low.", }) - # 2. Bridge nodes (high betweenness) → cross-cutting concern questions - if G.number_of_edges() > 0: + # 2. Bridge nodes (high betweenness) → cross-cutting concern questions. + # Exact betweenness is ~O(V·E); cap at 5000 nodes (same guard as + # _cross_community_surprises) so report/cluster-only stays fast on large + # repos. Bridge-node questions are supplementary; the ambiguous-edge and + # community questions below are unaffected. + if G.number_of_edges() > 0 and G.number_of_nodes() <= 5000: # Node betweenness via FalkorDB's built-in algo.betweenness (exact, whole # graph). The old nx path sampled k sources with seed=42 on large graphs; # the built-in computes exactly, so rankings are at least as accurate. diff --git a/graphify/export.py b/graphify/export.py index 2cf6cb3e7..ec65ec979 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1540,6 +1540,12 @@ def _spring_layout(G, seed: int = 42, iterations: int = 60) -> dict: return {nid: (p[0], p[1]) for nid, p in pos.items()} +# Above this node count, to_svg renders a top-degree overview instead of the full +# graph: a full-graph SVG is an unreadable hairball and the pure-Python force +# layout is O(n^2) per iteration, so the full graph would be impractically slow. +_SVG_MAX_NODES = 600 + + def to_svg( G: nx.Graph, communities: dict[int, list[str]], @@ -1562,6 +1568,19 @@ def to_svg( except ImportError as e: raise ImportError("matplotlib not installed. Run: pip install matplotlib") from e + # For large graphs, render a top-degree overview built as an in-memory + # MemGraph (keeps the full node/edge/degree drawing API) so the SVG stays + # readable and the O(n^2) layout stays bounded. + if G.number_of_nodes() > _SVG_MAX_NODES and hasattr(G, "top_degree_nodes"): + from graphify.store import MemGraph + _top = [d["id"] for d in G.top_degree_nodes(limit=_SVG_MAX_NODES)] + _top_set = set(_top) + _attrs = G.node_attrs_batch(_top) + _nodes = [(nid, _attrs.get(nid, {})) for nid in _top] + _edges = [(u, v, d) for u, v, d in G.edges(nbunch=_top, data=True) + if u in _top_set and v in _top_set] + G = MemGraph(_nodes, _edges, directed=G.is_directed()) + node_community = _node_community_map(communities) fig, ax = plt.subplots(figsize=figsize, facecolor="#1a1a2e") From a8e9620b03d1ed68be83862db5e2f5c8da447683 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 12:43:46 +0300 Subject: [PATCH 05/17] review: guard affected.connect_graph + add exporter/error-path tests Addresses the gaps found reviewing PR #1: - affected.connect_graph now guards the empty-graph case (raises so the CLI prints "Re-run /graphify to build" + exits non-zero) instead of handing back an empty store and silently reporting "no unique node match". Parity with serve._connect_graph and the query/path/explain commands. - tests: validate the hand-rolled GraphML writer emits well-formed XML (xml.etree parse), add a to_svg export test (skipped without matplotlib), and add an affected unbuilt-graph test that asserts a non-zero exit (covers the guard above). Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/affected.py | 11 ++++++++++- tests/test_affected_cli.py | 17 +++++++++++++++++ tests/test_cli_export.py | 17 +++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/graphify/affected.py b/graphify/affected.py index 6fc19ccab..0b3c9c89b 100644 --- a/graphify/affected.py +++ b/graphify/affected.py @@ -174,4 +174,13 @@ def connect_graph(path: Path): from .store import open_store out_dir = Path(path).parent if Path(path).suffix else Path(path) - return open_store(out_dir, create=False) + store = open_store(out_dir, create=False) + # open_store(create=False) hands back a store even when no graph was built + # (it derives a name from the pointer/root rather than erroring), so guard the + # empty case here — matching serve._connect_graph — so `graphify affected` + # tells the user to build instead of silently reporting nothing. + if store.number_of_nodes() == 0: + raise FileNotFoundError( + f"No graph found for {out_dir} (FalkorDB graph empty). Re-run /graphify to build." + ) + return store diff --git a/tests/test_affected_cli.py b/tests/test_affected_cli.py index cd03fdac6..47f9c0398 100644 --- a/tests/test_affected_cli.py +++ b/tests/test_affected_cli.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + import graphify.__main__ as mainmod _NODES = [ @@ -53,3 +55,18 @@ def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_ assert "Relations: calls" in out assert "X()" in out assert "__init__.py" not in out + + +def test_affected_cli_unbuilt_graph_exits_with_build_hint(monkeypatch, tmp_path): + """On a never-built graph, `affected` must exit non-zero (telling the user to + build) rather than silently reporting an empty result. Guards the + connect_graph empty-graph check (parity with query/path/explain).""" + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "affected", "Foo", "--graph", str(tmp_path / "graph.json")], + ) + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code != 0 diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 1e2937c39..779262dea 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -152,6 +152,23 @@ def test_export_graphml_creates_file(tmp_path): assert gml.stat().st_size > 0 content = gml.read_text() assert " elements" + + +def test_export_svg_creates_file(tmp_path): + pytest.importorskip("matplotlib") + _make_graph(tmp_path) + r = _run(["export", "svg"], tmp_path) + assert r.returncode == 0, r.stderr + svg = tmp_path / "graphify-out" / "graph.svg" + assert svg.exists() + assert svg.stat().st_size > 0 + assert " Date: Wed, 17 Jun 2026 12:46:48 +0300 Subject: [PATCH 06/17] ci+perf: FalkorDB CI service container; one-pass degree map in cluster() - ci.yml: run falkordb/falkordb as a service container so the conftest session fixture connects (otherwise every graph test silently skips in CI). - cluster.py: materialize dict(G.degree()) once instead of per-node G.degree(n) calls, which against the cache-free store would be O(N) round-trips (twice). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 16 ++++++++++++++++ graphify/cluster.py | 14 ++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e0b1ab08..b8d984a34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,22 @@ jobs: matrix: python-version: ["3.10", "3.12"] + # FalkorDB is the graph backend; without a reachable instance the conftest + # session fixture skips every graph-dependent test, so CI would go green + # without exercising the engine at all. Run it as a service container — the + # fixtures connect to localhost:6379 by default (overridable via FALKORDB_*). + services: + falkordb: + image: falkordb/falkordb:latest + ports: + - 6379:6379 + # falkordb is redis-based, so redis-cli ping gates readiness before tests. + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + steps: - uses: actions/checkout@v6 with: diff --git a/graphify/cluster.py b/graphify/cluster.py index e9d7b452d..9a1fd79b3 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -50,21 +50,27 @@ def cluster( if G.number_of_edges() == 0: return {i: [n] for i, n in enumerate(sorted(G.nodes))} + # Materialize the full degree map in one streamed pass. Per-node G.degree(n) + # against the FalkorDB store is a round-trip each, so the comprehensions below + # would otherwise issue O(N) queries (twice over); one dict() keeps it to a + # single scan, matching the old in-memory nx cost. + degmap = dict(G.degree()) + # Compute hub exclusion set before removing anything so degree is based on full graph hub_nodes: set[str] = set() if exclude_hubs_percentile is not None: - degrees = sorted(d for _, d in G.degree()) + degrees = sorted(degmap.values()) if degrees: idx = max(0, int(len(degrees) * exclude_hubs_percentile / 100) - 1) threshold = degrees[idx] - hub_nodes = {n for n, d in G.degree() if d > threshold} + hub_nodes = {n for n, d in degmap.items() if d > threshold} # Leiden warns and drops isolates - handle them separately # Also exclude hub nodes from partitioning so they don't pull unrelated # subsystems into the same community excluded = hub_nodes - isolates = [n for n in G.nodes() if G.degree(n) == 0 and n not in excluded] - connected_nodes = [n for n in G.nodes() if G.degree(n) > 0 and n not in excluded] + isolates = [n for n, d in degmap.items() if d == 0 and n not in excluded] + connected_nodes = [n for n, d in degmap.items() if d > 0 and n not in excluded] connected = G.subgraph(connected_nodes) raw: dict[int, list[str]] = {} From abd558d628f8a510d2fbb9d133b816e383f49188 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 14:54:37 +0300 Subject: [PATCH 07/17] review: address PR #1 Copilot comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject: add `falkordb` extra so the documented `graphifyy[falkordb]` install resolves (falkordb is a core dep; the extra just re-lists it). - conftest: the FalkorDB reachability skip is no longer session-autouse — the store/seed_graph/make_store fixtures depend on it, so only DB-backed tests skip when no engine is reachable (DB-free tests still run; CI can't silently pass with 0 tests). - update skill doc: the graph-diff step used a global `graphify_diff_scratch` graph name (data-loss if a user has that graph; collisions on concurrent runs). Derive a per-project + pid scratch name from the current graph name. Regenerated skill artifacts + blessed goldens. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/skills/amp/references/update.md | 8 +++++--- graphify/skills/claude/references/update.md | 8 +++++--- graphify/skills/claw/references/update.md | 8 +++++--- graphify/skills/codex/references/update.md | 8 +++++--- graphify/skills/copilot/references/update.md | 8 +++++--- graphify/skills/droid/references/update.md | 8 +++++--- graphify/skills/kilo/references/update.md | 8 +++++--- graphify/skills/kiro/references/update.md | 8 +++++--- graphify/skills/opencode/references/update.md | 8 +++++--- graphify/skills/pi/references/update.md | 8 +++++--- graphify/skills/trae/references/update.md | 8 +++++--- graphify/skills/vscode/references/update.md | 8 +++++--- graphify/skills/windows/references/update.md | 8 +++++--- pyproject.toml | 4 ++++ tests/conftest.py | 13 +++++++++---- .../graphify__skills__amp__references__update.md | 8 +++++--- .../graphify__skills__claude__references__update.md | 8 +++++--- .../graphify__skills__claw__references__update.md | 8 +++++--- .../graphify__skills__codex__references__update.md | 8 +++++--- ...graphify__skills__copilot__references__update.md | 8 +++++--- .../graphify__skills__droid__references__update.md | 8 +++++--- .../graphify__skills__kilo__references__update.md | 8 +++++--- .../graphify__skills__kiro__references__update.md | 8 +++++--- ...raphify__skills__opencode__references__update.md | 8 +++++--- .../graphify__skills__pi__references__update.md | 8 +++++--- .../graphify__skills__trae__references__update.md | 8 +++++--- .../graphify__skills__vscode__references__update.md | 8 +++++--- ...graphify__skills__windows__references__update.md | 8 +++++--- .../skillgen/fragments/references/shared/update.md | 8 +++++--- 29 files changed, 148 insertions(+), 85 deletions(-) diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index 7d06dce32..5d922cc4d 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/pyproject.toml b/pyproject.toml index b7808df3e..5592bb4cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,10 @@ Issues = "https://github.com/safishamsi/graphify/issues" [project.optional-dependencies] mcp = ["mcp"] +# falkordb is a core dependency; this extra exists so the documented +# `graphifyy[falkordb]` install command resolves instead of erroring on an +# unknown extra. +falkordb = ["falkordb>=1.0"] neo4j = ["neo4j"] pdf = ["pypdf", "markdownify"] watch = ["watchdog"] diff --git a/tests/conftest.py b/tests/conftest.py index 0e94d8986..6564bc4a5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,8 +26,13 @@ def falkordb_uri() -> str: return _falkordb_uri() -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="session") def _require_falkordb(falkordb_uri): + """Skip only tests that actually need a live FalkorDB. NOT autouse — the + DB-backed fixtures (store/seed_graph/make_store) depend on it, so DB tests + skip when no engine is reachable while DB-free tests (graphjson, _minhash, + detect, security, ...) still run, instead of the whole suite vanishing. + Session-scoped so the ping happens once.""" pytest.importorskip("falkordb") from graphify.store import _connect @@ -41,7 +46,7 @@ def _require_falkordb(falkordb_uri): @pytest.fixture() -def store(falkordb_uri): +def store(falkordb_uri, _require_falkordb): """A fresh, empty, uniquely-named GraphStore; cleaned up after the test.""" from graphify.store import GraphStore @@ -58,7 +63,7 @@ def store(falkordb_uri): @pytest.fixture() -def seed_graph(falkordb_uri): +def seed_graph(falkordb_uri, _require_falkordb): """Seed a FalkorDB graph for an output dir from node-link data, writing the pointer file so the CLI/serve loaders find it (replaces writing graph.json). @@ -101,7 +106,7 @@ def _seed(out_dir, nodes=(), links=(), *, write_artifact=True): @pytest.fixture() -def make_store(falkordb_uri): +def make_store(falkordb_uri, _require_falkordb): """Factory for tests needing more than one graph (e.g. graph_diff).""" from graphify.store import GraphStore diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index 7d06dce32..5d922cc4d 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -143,7 +143,7 @@ After Step 4, show the graph diff: ```bash $(cat graphify-out/.graphify_python) -c " -import json +import json, os from graphify.analyze import graph_diff from graphify.build import build_from_json from graphify.store import open_store @@ -153,11 +153,13 @@ from pathlib import Path G_new = open_store('graphify-out', create=False) # Load old graph (before update) from backup written before merge and rebuild it -# into a scratch FalkorDB graph just for the comparison. +# into a scratch FalkorDB graph just for the comparison. The scratch name is +# derived from this project's graph name + pid so it never clears or collides +# with an unrelated graph (or a concurrent update on the same project). old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None if old_data: old_extract = {'nodes': old_data.get('nodes', []), 'edges': old_data.get('links', old_data.get('edges', []))} - G_old = build_from_json(old_extract, graph_name='graphify_diff_scratch') + G_old = build_from_json(old_extract, graph_name=f'{G_new.graph_name}__diff_scratch_{os.getpid()}') diff = graph_diff(G_old, G_new) print(diff['summary']) if diff['new_nodes']: From cb702d72a5d6a803f0553c6c73ffd97805fc9dcc Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 14:55:17 +0300 Subject: [PATCH 08/17] fix: _find_node label_tokens used undefined `d` (merge artifact) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v8 merge folded label_tokens matching (#1353) into _find_node, but the cache-free rows are (id, norm_label) tuples with no `d` in scope — a NameError on every _find_node call (explain / get_node / score). Carry the raw label in rows (3-tuple) and tokenize that, mirroring _score_nodes. This fix existed in the working tree but was not staged into the merge commit. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/serve.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graphify/serve.py b/graphify/serve.py index be44a98d0..692d4f9a3 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -540,18 +540,18 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: if c["id"] in seen: continue seen.add(c["id"]) - rows.append((c["id"], c["norm_label"] or _strip_diacritics(c["label"]).lower())) + rows.append((c["id"], c["norm_label"] or _strip_diacritics(c["label"]).lower(), c["label"] or "")) else: rows = [ - (nid, d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower()) + (nid, d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower(), d.get("label") or "") for nid, d in G.nodes(data=True) ] exact: list[str] = [] prefix: list[str] = [] substring: list[str] = [] - for nid, norm_label in rows: + for nid, norm_label, _label in rows: bare_label = norm_label.rstrip("()") - label_tokens = " ".join(_search_tokens(d.get("label") or "")) + label_tokens = " ".join(_search_tokens(_label)) nid_lower = nid.lower() if term == norm_label or term == bare_label or term == label_tokens or term == nid_lower: exact.append(nid) From 9c6016bad2b56b7bc9e2b145c4b8ad8e18906f09 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 15:49:36 +0300 Subject: [PATCH 09/17] fix: preserve _origin so incremental rebuild evicts removed symbols (#1116) A symbol removed from a file that still exists was not pruned on `graphify update` / watch rebuild: the eviction keys off the AST-provenance marker `_origin` in graph.json, but the FalkorDB migration stripped all underscore-prefixed keys at the store boundary (_scalar_props) and in to_json, so `_origin` never reached graph.json and stale AST nodes (e.g. a deleted `foo()`) survived. (Pre-existing since the migration; surfaced once tree-sitter parsers are installed.) - store.add_nodes_from: keep `_origin` (the one underscore-marker that must persist); other `_` keys stay dropped. - export.to_json: keep `_origin` on exported nodes. - watch._canonical_topology_for_compare: exclude `_origin` (provenance, not topology) so an unchanged graph still compares equal across serializers. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/export.py | 4 ++++ graphify/store.py | 6 ++++++ graphify/watch.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/graphify/export.py b/graphify/export.py index b4cb6ab5a..1223653e5 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -515,6 +515,10 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, nodes = [] for nid, attrs in G.nodes(data=True): nd = {k: v for k, v in attrs.items() if not k.startswith("_")} + # Keep _origin (AST provenance): incremental rebuild reads it back to + # evict stale AST symbols removed from a surviving file (#1116). + if isinstance(attrs.get("_origin"), str): + nd["_origin"] = attrs["_origin"] nd["id"] = nid cid = node_community.get(nid) nd["community"] = cid diff --git a/graphify/store.py b/graphify/store.py index 7c664bb1d..5f47e0acf 100644 --- a/graphify/store.py +++ b/graphify/store.py @@ -767,6 +767,12 @@ def add_nodes_from(self, rows, *, fresh: bool = False) -> None: node_id, attrs = item, {} props = _scalar_props(attrs) props["id"] = node_id + # _origin (AST provenance) is the one underscore-marker that must + # survive into the store: incremental rebuild reads it back from + # graph.json to evict stale AST symbols removed from a surviving + # file (#1116). Other _-prefixed keys stay dropped. + if isinstance(attrs.get("_origin"), str): + props["_origin"] = attrs["_origin"] ftype = _safe_label(str(attrs.get("file_type", "Entity")).capitalize()) by_label.setdefault(ftype, []).append(props) materialized.append(1) diff --git a/graphify/watch.py b/graphify/watch.py index 497314dab..b695b1a37 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -264,6 +264,10 @@ def _canonical_topology_for_compare(graph_data: dict) -> dict: n = dict(node) n.pop("community", None) n.pop("norm_label", None) + # _origin is AST provenance (kept in graph.json for #1116 eviction), + # not topology — exclude it so an unchanged graph compares equal + # regardless of which serializer wrote each side. + n.pop("_origin", None) norm_nodes.append(n) canonical["nodes"] = sorted( norm_nodes, From cfc80b33e5a969a4a15e4e1c394aef5db9f73154 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 15:59:19 +0300 Subject: [PATCH 10/17] ci: regenerate uv.lock for the FalkorDB dependency set The lockfile predated the migration's pyproject changes, so it had no falkordb entry (and still pinned networkx). CI installs with `uv sync --all-extras --frozen`, which used that stale lock and left the falkordb client uninstalled -> every DB test ImportError'd ("No module named 'falkordb'"). Re-resolved against the current pyproject: adds falkordb 1.6.1, falkordblite (py>=3.12 marker), redis, psutil; bumps graphifyy to 0.8.40. The FalkorDB service container (added to ci.yml) provides the server the tests connect to. Co-Authored-By: Claude Opus 4.8 (1M context) --- uv.lock | 225 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 167 insertions(+), 58 deletions(-) diff --git a/uv.lock b/uv.lock index 8010f68f2..cee6be794 100644 --- a/uv.lock +++ b/uv.lock @@ -6,11 +6,14 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version < '3.11'", ] @@ -74,6 +77,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/98/f6aa7fe0783e42be3093d8ef1b0ecdc22c34c0d69640dfb37f56925cb141/anytree-2.13.0-py3-none-any.whl", hash = "sha256:4cbcf10df36b1f1cba131b7e487ff3edafc9d6e932a3c70071b5b768bab901ff", size = 45077, upload-time = "2025-04-08T21:06:29.494Z" }, ] +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -586,11 +598,14 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, @@ -987,6 +1002,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] +[[package]] +name = "falkordb" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "redis" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/3a/58da510e5800a5cdbf9591111e4287cbf68b475bf074db12a94e5db8bcea/falkordb-1.6.1.tar.gz", hash = "sha256:bbef448a0b43e00ff3062bd6201368618d7b36e969d16ba71e8b8e3fa90873d4", size = 103185, upload-time = "2026-04-28T13:24:44.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/02/1cf9cef72228ca2b2776b4ed0cb2645298ddcc57c1fe2c545cd46bc11eae/falkordb-1.6.1-py3-none-any.whl", hash = "sha256:cf51caeb433c04db303de5967a0c2590675fcc0354e80997870b1e0497d30c34", size = 37802, upload-time = "2026-04-28T13:24:45.677Z" }, +] + +[[package]] +name = "falkordblite" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "falkordb", marker = "python_full_version >= '3.12'" }, + { name = "psutil", marker = "python_full_version >= '3.12'" }, + { name = "redis", marker = "python_full_version >= '3.12'" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/03/afbb6e0f03c302aa9b64a38e1a2c43664f86921f0dcbf03ec9e31da06ac6/falkordblite-0.10.0.tar.gz", hash = "sha256:65a72abafd30711f699c15571df6959edb8901605053ce940ccdd837832e709b", size = 23675620, upload-time = "2026-05-02T13:12:29.429Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/43/39d8cf13964784447676d24f0cefa3bdc99c10e647c71e6a4172d302dcac/falkordblite-0.10.0-cp312-cp312-macosx_10_13_x86_64.macosx_15_0_arm64.whl", hash = "sha256:741fda166170513db1815d5369870e47d44da2e9b85320fddbc50c88a7338d51", size = 17752268, upload-time = "2026-05-02T13:11:57.906Z" }, + { url = "https://files.pythonhosted.org/packages/61/17/3ec180ca7c2e79a0c3e9912ac04b446e733745cd944845fe4306be016efd/falkordblite-0.10.0-cp312-cp312-manylinux_2_39_aarch64.whl", hash = "sha256:bfe1f47ae03decdc0ad111ec82606bac82ee6cdd7fea04cbcff1283608cc2d2e", size = 34579293, upload-time = "2026-05-02T13:12:01.601Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5e/d343c5249bd24c6614e8c1b718a73bb9f68beb2cf90464bc51c9436d25af/falkordblite-0.10.0-cp312-cp312-manylinux_2_39_x86_64.whl", hash = "sha256:ee9659bd0c7cdf0c2532977f2ec8bf1d1ab01cb3b6776e50c67046481f3162c7", size = 36154066, upload-time = "2026-05-02T13:12:05.435Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1e/18612dc9e75e5c02a281a49d97b56b5504eb5907f971c68e8a5801033f36/falkordblite-0.10.0-cp313-cp313-macosx_10_13_x86_64.macosx_15_0_arm64.whl", hash = "sha256:54a051cbc0bb25b30e65f46ddb1b63149a80bb7004c97615d39c491d492a6d9b", size = 17752240, upload-time = "2026-05-02T13:12:08.837Z" }, + { url = "https://files.pythonhosted.org/packages/29/b3/63f9b1168c4d1abbee4418a0a165496f57cd4b93a261cb481272ce353890/falkordblite-0.10.0-cp313-cp313-manylinux_2_39_aarch64.whl", hash = "sha256:ce1bd408ebaafc3dbdf39e860888382aae69de2b3c949dadfd132779130b99b2", size = 34579294, upload-time = "2026-05-02T13:12:12.069Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cf/d792cf46292f7756f96727916e77ebf9821371e22bff65ed8e90db6b80b9/falkordblite-0.10.0-cp313-cp313-manylinux_2_39_x86_64.whl", hash = "sha256:7d52a3665f6de30cbffc5809a1c6d722492c95bf8dae4a7ee108ca58da939b25", size = 36154069, upload-time = "2026-05-02T13:12:15.742Z" }, + { url = "https://files.pythonhosted.org/packages/ab/57/eb2ac5824f7831fcb550468cc504a944adfee1b0c5a09b4bc684b0217976/falkordblite-0.10.0-cp314-cp314-macosx_10_15_x86_64.macosx_15_0_arm64.whl", hash = "sha256:5eff806e09902b36f34a7c058de0dcf237f0ae047d2b39c0423a5d6a52193341", size = 17752257, upload-time = "2026-05-02T13:12:18.896Z" }, + { url = "https://files.pythonhosted.org/packages/61/13/76932987b43286443088be9d1ebdbc6999ac084d5c40dbfe4a4704aa264b/falkordblite-0.10.0-cp314-cp314-manylinux_2_39_aarch64.whl", hash = "sha256:6cd10bb1b0e35946e0e4c06bd0c16298fe6df70e8e18bc055db0d60359c7271c", size = 34579275, upload-time = "2026-05-02T13:12:22.181Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a6/6eba83964a0633aeee698a1fbd23b954ce39ee4cfd7eced73f52d95ca89a/falkordblite-0.10.0-cp314-cp314-manylinux_2_39_x86_64.whl", hash = "sha256:b36665ebd9e2d5e38b401b34f7ba6f0b22e5b88cbc817971c8c1d71a8ad305f7", size = 36154066, upload-time = "2026-05-02T13:12:26.001Z" }, +] + [[package]] name = "faster-whisper" version = "1.2.1" @@ -1131,11 +1182,10 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.8.37" +version = "0.8.40" source = { editable = "." } dependencies = [ - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "falkordb" }, { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "rapidfuzz" }, @@ -1171,6 +1221,7 @@ dependencies = [ all = [ { name = "anthropic" }, { name = "boto3" }, + { name = "falkordb" }, { name = "faster-whisper", marker = "python_full_version >= '3.11'" }, { name = "graspologic", marker = "python_full_version < '3.13'" }, { name = "jieba" }, @@ -1202,6 +1253,9 @@ chinese = [ dm = [ { name = "tree-sitter-dm" }, ] +falkordb = [ + { name = "falkordb" }, +] gemini = [ { name = "openai" }, { name = "tiktoken" }, @@ -1216,6 +1270,9 @@ kimi = [ leiden = [ { name = "graspologic", marker = "python_full_version < '3.13'" }, ] +lite = [ + { name = "falkordblite", marker = "python_full_version >= '3.12'" }, +] mcp = [ { name = "mcp" }, ] @@ -1284,6 +1341,10 @@ requires-dist = [ { name = "anthropic", marker = "extra == 'anthropic'" }, { name = "boto3", marker = "extra == 'all'" }, { name = "boto3", marker = "extra == 'bedrock'" }, + { name = "falkordb", specifier = ">=1.0" }, + { name = "falkordb", marker = "extra == 'all'" }, + { name = "falkordb", marker = "extra == 'falkordb'", specifier = ">=1.0" }, + { name = "falkordblite", marker = "python_full_version >= '3.12' and extra == 'lite'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'all'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'video'" }, { name = "graspologic", marker = "python_full_version < '3.13' and extra == 'all'" }, @@ -1298,7 +1359,6 @@ requires-dist = [ { name = "mcp", marker = "extra == 'mcp'" }, { name = "neo4j", marker = "extra == 'all'" }, { name = "neo4j", marker = "extra == 'neo4j'" }, - { name = "networkx", specifier = ">=3.4" }, { name = "numpy", specifier = ">=1.21" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'all'", specifier = ">=2.0" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'svg'", specifier = ">=2.0" }, @@ -1320,44 +1380,44 @@ requires-dist = [ { name = "tiktoken", marker = "extra == 'gemini'" }, { name = "tiktoken", marker = "extra == 'kimi'" }, { name = "tiktoken", marker = "extra == 'openai'" }, - { name = "tree-sitter", specifier = ">=0.23.0" }, - { name = "tree-sitter-bash" }, - { name = "tree-sitter-c" }, - { name = "tree-sitter-c-sharp" }, - { name = "tree-sitter-cpp" }, + { name = "tree-sitter", specifier = ">=0.23.0,<0.26" }, + { name = "tree-sitter-bash", specifier = ">=0.23,<0.27" }, + { name = "tree-sitter-c", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-c-sharp", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-cpp", specifier = ">=0.23,<0.25" }, { name = "tree-sitter-dm", marker = "extra == 'all'" }, { name = "tree-sitter-dm", marker = "extra == 'dm'" }, - { name = "tree-sitter-elixir" }, - { name = "tree-sitter-fortran" }, - { name = "tree-sitter-go" }, - { name = "tree-sitter-groovy" }, + { name = "tree-sitter-elixir", specifier = ">=0.3,<0.5" }, + { name = "tree-sitter-fortran", specifier = ">=0.6,<0.8" }, + { name = "tree-sitter-go", specifier = ">=0.23,<0.26" }, + { name = "tree-sitter-groovy", specifier = ">=0.1,<0.3" }, { name = "tree-sitter-hcl", marker = "extra == 'all'" }, { name = "tree-sitter-hcl", marker = "extra == 'terraform'" }, - { name = "tree-sitter-java" }, - { name = "tree-sitter-javascript" }, - { name = "tree-sitter-json" }, - { name = "tree-sitter-julia" }, - { name = "tree-sitter-kotlin" }, - { name = "tree-sitter-lua" }, - { name = "tree-sitter-objc" }, - { name = "tree-sitter-php" }, - { name = "tree-sitter-powershell" }, - { name = "tree-sitter-python" }, - { name = "tree-sitter-ruby" }, - { name = "tree-sitter-rust" }, - { name = "tree-sitter-scala" }, + { name = "tree-sitter-java", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-javascript", specifier = ">=0.23,<0.26" }, + { name = "tree-sitter-json", specifier = ">=0.23,<0.26" }, + { name = "tree-sitter-julia", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-kotlin", specifier = ">=1.0,<2.0" }, + { name = "tree-sitter-lua", specifier = ">=0.2,<0.6" }, + { name = "tree-sitter-objc", specifier = ">=3.0,<4.0" }, + { name = "tree-sitter-php", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-powershell", specifier = ">=0.26,<0.28" }, + { name = "tree-sitter-python", specifier = ">=0.23,<0.26" }, + { name = "tree-sitter-ruby", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-rust", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-scala", specifier = ">=0.23,<0.27" }, { name = "tree-sitter-sql", marker = "extra == 'all'" }, { name = "tree-sitter-sql", marker = "extra == 'sql'" }, - { name = "tree-sitter-swift" }, - { name = "tree-sitter-typescript" }, - { name = "tree-sitter-verilog" }, - { name = "tree-sitter-zig" }, + { name = "tree-sitter-swift", specifier = ">=0.7,<0.9" }, + { name = "tree-sitter-typescript", specifier = ">=0.23,<0.25" }, + { name = "tree-sitter-verilog", specifier = ">=1.0,<2.0" }, + { name = "tree-sitter-zig", specifier = ">=1.0,<2.0" }, { name = "watchdog", marker = "extra == 'all'" }, { name = "watchdog", marker = "extra == 'watch'" }, { name = "yt-dlp", marker = "extra == 'all'" }, { name = "yt-dlp", marker = "extra == 'video'" }, ] -provides-extras = ["mcp", "neo4j", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "dm", "terraform", "all"] +provides-extras = ["mcp", "falkordb", "neo4j", "pdf", "watch", "svg", "leiden", "lite", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "dm", "terraform", "all"] [package.metadata.requires-dev] dev = [ @@ -2381,15 +2441,12 @@ name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ @@ -2467,9 +2524,12 @@ name = "numpy" version = "1.26.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version < '3.11'", ] sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } @@ -2743,9 +2803,12 @@ name = "pandas" version = "3.0.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, @@ -3074,6 +3137,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/95/608f665226bca68b736b79e457fded9a2a38c4f4379a4a7614303d9db3bc/protobuf-7.34.1-py3-none-any.whl", hash = "sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11", size = 170715, upload-time = "2026-03-20T17:34:45.384Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "psycopg" version = "3.3.4" @@ -3680,6 +3771,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/b5/363906b1064fc6fe611783a61764927bbd91919aaaabe8cba82151ca93ef/rapidfuzz-3.14.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:dfef96543ced67d9513a422755db422ae1dc34dade0a1485e0b43e7342ed3ebf", size = 1509889, upload-time = "2026-04-07T11:16:28.487Z" }, ] +[[package]] +name = "redis" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/ae/ed461cca5780b5fc8b9fe8ca0ed98d89508645fb9d880c24cc42c087678f/redis-8.0.0.tar.gz", hash = "sha256:a00c5355432051ac14e593b8b197fc76c887ee12d55a0984f69328a1115fdc49", size = 5101591, upload-time = "2026-05-28T12:45:13.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/e3/b519734372d305bd547534a9f32e4ce9f98552af753dce72cf3483a0ff0b/redis-8.0.0-py3-none-any.whl", hash = "sha256:c938c18338585009f0bc310f4c7e4e4b4d37639356c4ac072cedf3af570c8dc7", size = 499870, upload-time = "2026-05-28T12:45:11.697Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -4108,9 +4211,12 @@ name = "scikit-learn" version = "1.8.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "joblib", marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, @@ -4222,9 +4328,12 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, From fcad53dc5bf85124cbe1245c04bd77adcc474471 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 17 Jun 2026 16:07:31 +0300 Subject: [PATCH 11/17] fix: GraphStore.in_degree/out_degree + build_merge test uses graph_name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI failures (surfaced once the FalkorDB client installs and tree-sitter parsers are present in CI): - test_java_type_resolution used G.in_degree(v); GraphStore only had degree. Add in_degree/out_degree (scoped counts) to GraphStore and MemGraph. - test_build_merge_preserves_call_edge_direction called build_merge([], graph_path) — the migration's build_merge takes a graph_name, not a graph.json path, so the Path reached select_graph() and raised "Expected a string parameter". Reconnect via G1.graph_name instead. (Was masked locally because the JS parser isn't installed.) Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/store.py | 16 ++++++++++++++++ tests/test_build.py | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/graphify/store.py b/graphify/store.py index 5f47e0acf..de0f63e47 100644 --- a/graphify/store.py +++ b/graphify/store.py @@ -403,6 +403,12 @@ def in_edges(self, node_id, data=False): return [(u, v, dict(a)) for u, v, a in rows] return [(u, v) for u, v, _ in rows] + def in_degree(self, node_id) -> int: + return sum(1 for _u, v, _a in self._esorted if v == node_id) + + def out_degree(self, node_id) -> int: + return sum(1 for u, _v, _a in self._esorted if u == node_id) + def subgraph(self, node_ids): return _SubgraphView(self, node_ids) @@ -578,6 +584,16 @@ def edges(self): def degree(self): return _SDegreeView(self) + def in_degree(self, node_id) -> int: + """Number of incoming edges (nx-compatible scoped count).""" + return int(self._rows( + "MATCH (:Entity)-[r]->(:Entity {id:$id}) RETURN count(r)", {"id": node_id})[0][0]) + + def out_degree(self, node_id) -> int: + """Number of outgoing edges (nx-compatible scoped count).""" + return int(self._rows( + "MATCH (:Entity {id:$id})-[r]->(:Entity) RETURN count(r)", {"id": node_id})[0][0]) + # ---- streaming + scoped read helpers (no materialization) --------------- def _stream(self, match_return: str, order_by: str, params: dict | None = None): """Yield rows page-by-page (only one page in memory at a time).""" diff --git a/tests/test_build.py b/tests/test_build.py index 6e8e07581..d3f91983d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -195,8 +195,9 @@ def test_build_merge_preserves_call_edge_direction(tmp_path): assert saved_calls[0]["source"] == truth_src assert saved_calls[0]["target"] == truth_tgt - # Now simulate `--update` with no new chunks — load + re-save. - G2 = build_merge([], graph_path, dedup=False) + # Now simulate `--update` with no new chunks — reconnect to the same + # FalkorDB graph (build_merge takes a graph_name, not a graph.json path). + G2 = build_merge([], G1.graph_name, dedup=False) assert to_json(G2, communities, str(graph_path), force=True) # The calls edge must still go a -> b, not b -> a. From ad18f03f5e2e7005723b40a1ba68a64fa9d9c261 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Thu, 18 Jun 2026 13:40:06 +0300 Subject: [PATCH 12/17] fix: deterministic _find_node tie-ordering across backends Sort each match tier (exact/prefix/substring) by node id so an ambiguous label resolves to the same top match regardless of the backend's row order (FalkorDB scan order vs. in-memory iteration). Found via native-vs-reference parity review. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/serve.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/graphify/serve.py b/graphify/serve.py index 692d4f9a3..2ec309940 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -564,7 +564,11 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: prefix.append(nid) elif term in norm_label or term in label_tokens: substring.append(nid) - return exact + prefix + substring + # Sort within each tier by node id so the result is deterministic and + # independent of the backend's row order (FalkorDB scan order vs. in-memory + # iteration). Without this, an ambiguous label that matches several nodes in + # the same tier could surface a different top match per backend/run (#1175). + return sorted(exact) + sorted(prefix) + sorted(substring) def _filter_blank_stdin() -> None: From f51f720ca5e1257dcda0b6029048c97a4c34ebc1 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Thu, 18 Jun 2026 15:06:38 +0300 Subject: [PATCH 13/17] fix: address PR #1379 review findings (build crash, packaging, edge direction, seed resolution) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four confirmed regressions found in external review of the FalkorDB backend: - build.py: edges missing source_file crashed build_from_json with NameError (referenced an undefined `G`); backfill from node_attrs instead. - pyproject.toml: package the FalkorDB UDFs (udfs/*.js) so wheel/sdist installs don't FileNotFoundError on Louvain/edge-betweenness/cycles. - store.py: directed edge semantics — remove_edges deletes only src->tgt (was nuking reciprocal edges); G[u][v] / _edge_attrs prefer the forward edge while still finding a reverse edge (undirected-model lookup). - affected.py: native resolve_seed gained the bare-callable-name tier (`foo` -> `foo()`) it had only on the in-memory fallback path. Adds regression tests for each. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/affected.py | 10 ++++++- graphify/build.py | 4 +-- graphify/store.py | 42 +++++++++++++++++++++++++----- pyproject.toml | 6 ++++- tests/test_build.py | 13 +++++++++ tests/test_store_edge_semantics.py | 41 +++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 tests/test_store_edge_semantics.py diff --git a/graphify/affected.py b/graphify/affected.py index b05463630..c2109f34e 100644 --- a/graphify/affected.py +++ b/graphify/affected.py @@ -57,7 +57,15 @@ def resolve_seed(graph, query: str) -> str | None: if hasattr(graph, "find_node_ids"): if graph.has_node(query): return query - for kwargs in ({"label": query}, {"source_file": query}, {"label_contains": query}): + # Mirror the in-memory tier order: exact label -> bare callable name -> + # exact source_file -> substring (#1353). Without the bare-name tier a + # query like "foo" failed to resolve to "foo()" on the native path. + for kwargs in ( + {"label": query}, + {"label_bare": _bare_name(query)}, + {"source_file": query}, + {"label_contains": query}, + ): matches = graph.find_node_ids(limit=2, **kwargs) if len(matches) == 1: return str(matches[0]) diff --git a/graphify/build.py b/graphify/build.py index 7d9749a37..28da2c906 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -328,8 +328,8 @@ def build_from_json( # flags and leaves query results with no file reference (#1279). if not attrs.get("source_file"): attrs["source_file"] = ( - G.nodes[src].get("source_file") - or G.nodes[tgt].get("source_file") + node_attrs[src].get("source_file") + or node_attrs[tgt].get("source_file") or "" ) if "source_file" in attrs: diff --git a/graphify/store.py b/graphify/store.py index de0f63e47..c5068d5c6 100644 --- a/graphify/store.py +++ b/graphify/store.py @@ -644,10 +644,18 @@ def _node_attrs(self, nid): return d def _edge_attrs(self, u, v): + # Prefer the forward u->v edge; fall back to a reverse v->u edge so the + # lookup still succeeds in the undirected model (but reports the right + # direction's relation when both exist). rows = self._rows( - "MATCH (a:Entity {id:$u})-[r]-(b:Entity {id:$v}) RETURN properties(r) LIMIT 1", + "MATCH (a:Entity {id:$u})-[r]->(b:Entity {id:$v}) RETURN properties(r) LIMIT 1", {"u": u, "v": v}, timeout=_QUERY_TIMEOUT_MS, ) + if not rows: + rows = self._rows( + "MATCH (a:Entity {id:$u})<-[r]-(b:Entity {id:$v}) RETURN properties(r) LIMIT 1", + {"u": u, "v": v}, timeout=_QUERY_TIMEOUT_MS, + ) return dict(rows[0][0]) if rows else None def _degree_one(self, nid): @@ -736,13 +744,20 @@ def clear(self) -> None: self._ensure_schema() def __getitem__(self, node_id): - """Adjacency access: ``G[u]`` -> {neighbor_id: edge_attrs} (both directions), scoped.""" + """Adjacency access: ``G[u]`` -> {neighbor_id: edge_attrs}. + + Lists neighbors reachable in EITHER direction (the undirected nx.Graph + model graphify uses), but when a pair is connected both ways the FORWARD + (u->m) edge wins, so ``G[u][v]`` reports the u->v relation rather than an + arbitrary direction's attributes. + """ rows = self._rows( - "MATCH (n:Entity {id:$id})-[r]-(m:Entity) RETURN m.id, properties(r)", + "MATCH (n:Entity {id:$id})-[r]-(m:Entity) " + "RETURN m.id, properties(r), startNode(r).id = $id AS fwd ORDER BY fwd DESC", {"id": node_id}, timeout=_QUERY_TIMEOUT_MS, ) adj: dict = {} - for mid, props in rows: + for mid, props, _fwd in rows: adj.setdefault(mid, dict(props)) return adj @@ -843,8 +858,12 @@ def remove_edges(self, pairs) -> None: rows = [{"src": u, "tgt": v} for u, v in pairs] if not rows: return + # DIRECTED delete: pairs come from edges() in native source->target + # order, so only the src->tgt edge is removed. An undirected `-[r]-` + # here would also delete a reciprocal tgt->src edge whose own + # source_file may not be pruned (over-deletion during incremental prune). self._g.query( - "UNWIND $rows AS row MATCH (a:Entity {id: row.src})-[r]-(b:Entity {id: row.tgt}) DELETE r", + "UNWIND $rows AS row MATCH (a:Entity {id: row.src})-[r]->(b:Entity {id: row.tgt}) DELETE r", {"rows": rows}, ) self._invalidate() @@ -1094,12 +1113,21 @@ def confidence_counts(self) -> dict: ) return {str(k): int(v) for k, v in rows} - def find_node_ids(self, *, label=None, source_file=None, label_contains=None, limit=None) -> list: - """Scoped node lookup by exact label / exact source_file / label substring.""" + def find_node_ids(self, *, label=None, source_file=None, label_contains=None, label_bare=None, limit=None) -> list: + """Scoped node lookup by exact label / exact source_file / bare callable + name / label substring.""" if label is not None: q, v = "MATCH (n:Entity) WHERE toLower(n.label) = $v RETURN n.id", label.lower() elif source_file is not None: q, v = "MATCH (n:Entity) WHERE toLower(n.source_file) = $v RETURN n.id", source_file.lower() + elif label_bare is not None: + # Bare callable name: a query like "foo" matches a node labelled + # "foo" OR "foo()" (the "()" decoration callables carry). Mirrors the + # in-memory path's _bare_name tier so `affected foo` resolves natively. + q, v = ( + "MATCH (n:Entity) WHERE toLower(n.label) = $v OR toLower(n.label) = $v + '()' RETURN n.id", + label_bare.lower(), + ) else: q, v = "MATCH (n:Entity) WHERE toLower(n.label) CONTAINS $v RETURN n.id", label_contains.lower() if limit: diff --git a/pyproject.toml b/pyproject.toml index ad84cdb5f..73f39e6e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,11 @@ include-package-data = false # under graphify/skills//references/, and the always-on injection blocks # under graphify/always_on/. There is no graphify/skills//SKILL.md in the # repo, so no SKILL.md glob is needed here. -graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md", "skills/*/references/*.md", "always_on/*.md"] +# "udfs/*.js" ships the FalkorDB server-side algorithms (Louvain, edge +# betweenness, simple cycles) that store.py loads at runtime via +# Path(__file__).parent / "udfs" — without it, wheel/sdist installs hit a +# FileNotFoundError when clustering/analyze run (works only from a checkout). +graphify = ["skill.md", "skill-codex.md", "skill-opencode.md", "skill-kilo.md", "command-kilo.md", "skill-aider.md", "skill-amp.md", "skill-copilot.md", "skill-claw.md", "skill-windows.md", "skill-droid.md", "skill-trae.md", "skill-kiro.md", "skill-vscode.md", "skill-pi.md", "skill-devin.md", "skills/*/references/*.md", "always_on/*.md", "udfs/*.js"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_build.py b/tests/test_build.py index d3f91983d..93e1c41b5 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -50,6 +50,19 @@ def test_legacy_edge_from_to_canonicalized(): assert G.number_of_edges() == 1 +def test_edge_missing_source_file_backfills_from_endpoints(): + """An edge with no source_file must not crash the build; it backfills from + its endpoint nodes (regression: build_from_json referenced an undefined + `G.nodes`, raising NameError on any source_file-less edge).""" + ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"}], + "edges": [{"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED"}], + "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + assert G.number_of_edges() == 1 + assert edge_data(G, "n1", "n2")["source_file"] == "a.py" + + def test_source_file_backslash_normalized(): """Windows backslash paths and POSIX paths for the same file must produce one node.""" extraction = { diff --git a/tests/test_store_edge_semantics.py b/tests/test_store_edge_semantics.py new file mode 100644 index 000000000..7f2c8ddf5 --- /dev/null +++ b/tests/test_store_edge_semantics.py @@ -0,0 +1,41 @@ +"""Regression tests for GraphStore edge-direction and seed-resolution semantics +surfaced by the PR #1379 review.""" +from __future__ import annotations + +from graphify import affected as aff + + +def test_edge_attr_lookup_prefers_forward_direction(store): + """With reciprocal directed edges, ``G[u][v]`` must report the u->v edge's + attributes, not an arbitrary direction (regression: undirected ``-[r]-`` + returned the reverse edge's relation).""" + store.add_nodes_from([("a", {"label": "A"}), ("b", {"label": "B"})]) + store.add_edges_from([("a", "b", {"relation": "ab"}), ("b", "a", {"relation": "ba"})]) + assert store["a"]["b"]["relation"] == "ab" + assert store["b"]["a"]["relation"] == "ba" + + +def test_edge_attr_lookup_falls_back_to_reverse(store): + """A single v->u edge is still found when queried as (u, v): the undirected + graph model must not lose connectivity just because direction is preferred.""" + store.add_nodes_from([("a", {"label": "A"}), ("b", {"label": "B"})]) + store.add_edges_from([("b", "a", {"relation": "ba"})]) + assert store["a"]["b"]["relation"] == "ba" + + +def test_remove_edges_is_directed(store): + """Removing (u, v) must delete only the u->v edge, not a reciprocal v->u + edge (regression: undirected delete nuked both directions, over-pruning + edges whose own source_file was not deleted).""" + store.add_nodes_from([("a", {"label": "A"}), ("b", {"label": "B"})]) + store.add_edges_from([("a", "b", {"relation": "ab"}), ("b", "a", {"relation": "ba"})]) + store.remove_edges([("a", "b")]) + remaining = sorted((u, v, d.get("relation")) for u, v, d in store.edges(data=True)) + assert remaining == [("b", "a", "ba")] + + +def test_resolve_seed_native_bare_name(store): + """`affected foo` must resolve to a uniquely-matching callable node ``foo()`` + on the native store path, matching the in-memory _bare_name tier.""" + store.add_nodes_from([("n1", {"label": "foo()"}), ("n2", {"label": "fooExtra()"})]) + assert aff.resolve_seed(store, "foo") == "n1" From 4a0e9612581408303d8dae8c54117e4f50b4f0a7 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Thu, 18 Jun 2026 15:34:25 +0300 Subject: [PATCH 14/17] fix: query traversal dropped same-level multi-parent edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _traverse marked nodes visited mid-level, so a node reached from several parents in the same BFS level kept only the first parent's edge — thinning ~1/3 of edges in dense subgraphs that `query` (and MCP) return to the user. Defer the visited update to end-of-level (mirroring v8 _bfs) and RETURN DISTINCT to dedup parallel edges. Edge sets now match the in-memory reference exactly (verified on the 15k-node graph: 1964/1964 on the densest seed). Found via PR #1379 comprehensive review. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/store.py | 15 ++++++++++++--- tests/test_store_edge_semantics.py | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/graphify/store.py b/graphify/store.py index c5068d5c6..74bc8de9c 100644 --- a/graphify/store.py +++ b/graphify/store.py @@ -1292,15 +1292,24 @@ def _traverse(self, seeds, depth, hub_threshold, contexts=None): "UNWIND $f AS fid MATCH (n:Entity {id: fid}) " "WITH fid, n, size((n)--()) AS deg " "WHERE fid IN $seeds OR deg < $hub " - f"{edge_match}{edge_where} RETURN fid, m.id", + f"{edge_match}{edge_where} RETURN DISTINCT fid, m.id", params, timeout=_QUERY_TIMEOUT_MS, ) + # Defer the visited update to END-OF-LEVEL (mirrors v8 _bfs). Marking + # a node visited mid-level would drop every parent->child edge except + # the first when a node has multiple parents in the same BFS level + # (e.g. the diamond A->B, A->C, B->D, C->D loses C->D), thinning the + # subgraph the user sees. Record an edge to each newly-discovered node + # from EVERY same-level parent, but enqueue the node only once. nxt = [] + newly: set = set() for fid, mid in sorted(rows, key=lambda r: (str(r[0]), str(r[1]))): if mid not in visited: - visited.add(mid) edges_seen.append((fid, mid)) - nxt.append(mid) + if mid not in newly: + newly.add(mid) + nxt.append(mid) + visited |= newly frontier = nxt return visited, edges_seen diff --git a/tests/test_store_edge_semantics.py b/tests/test_store_edge_semantics.py index 7f2c8ddf5..0f82bbc77 100644 --- a/tests/test_store_edge_semantics.py +++ b/tests/test_store_edge_semantics.py @@ -34,6 +34,31 @@ def test_remove_edges_is_directed(store): assert remaining == [("b", "a", "ba")] +def test_traverse_keeps_all_same_level_parent_edges(store): + """A node reached from multiple parents in the same BFS level must keep an + edge from EVERY parent (regression: marking visited mid-level dropped all but + the first, thinning ~⅓ of edges in the subgraph `query` shows the user). + + Diamond: A->B, A->C, B->D, C->D ; from A at depth 2 both B->D and C->D survive. + """ + store.add_nodes_from([(x, {"label": x}) for x in "ABCD"]) + store.add_edges_from([("A", "B", {}), ("A", "C", {}), ("B", "D", {}), ("C", "D", {})]) + visited, edges = store._traverse(["A"], depth=2, hub_threshold=10 ** 9) + assert set(visited) == set("ABCD") + und = {frozenset((u, v)) for u, v in edges} + assert frozenset(("B", "D")) in und + assert frozenset(("C", "D")) in und + assert len(edges) == 4 + + +def test_traverse_dedups_parallel_edges(store): + """Parallel edges between the same pair must not double-count in the walk.""" + store.add_nodes_from([("A", {"label": "A"}), ("B", {"label": "B"})]) + store.add_edges_from([("A", "B", {"relation": "r1"}), ("A", "B", {"relation": "r2"})]) + _visited, edges = store._traverse(["A"], depth=1, hub_threshold=10 ** 9) + assert edges == [("A", "B")] + + def test_resolve_seed_native_bare_name(store): """`affected foo` must resolve to a uniquely-matching callable node ``foo()`` on the native store path, matching the in-memory _bare_name tier.""" From 779fc4275e601524c37eb9338c4bf2a1603b9081 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Thu, 18 Jun 2026 16:05:32 +0300 Subject: [PATCH 15/17] fix: keep pre-FalkorDB and --no-cluster graphs queryable (auto-import graph.json) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A read command (query/path/explain/export/cluster-only) opening an empty FalkorDB store now imports a node-link graph.json from the output dir on first use, instead of failing with "graph empty". This preserves the prior workflow for projects built before the FalkorDB backend and for `--no-cluster` runs (which write only JSON) — existing graphs stay queryable without a rebuild. One-time, transparent (prints a migration note); no-op when no graph.json exists. Found via PR #1379 comprehensive review (#3). Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/serve.py | 36 ++++++++++++++++++++++++++++++ tests/test_store_edge_semantics.py | 20 +++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/graphify/serve.py b/graphify/serve.py index 2ec309940..ebbe5f7a1 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -27,6 +27,14 @@ def _connect_graph(graph_path: str): resolved = Path(graph_path).resolve() out_dir = resolved.parent if resolved.suffix else resolved store = open_store(out_dir, create=False) + if store.number_of_nodes() == 0: + # Back-compat: the FalkorDB store is empty but a node-link graph.json + # may still hold the graph — an existing pre-FalkorDB project, or a + # `--no-cluster` run that only wrote JSON. Import it into the store on + # first use so query/path/explain/export work without forcing a + # rebuild, preserving the previous workflow. + gj = resolved if resolved.suffix == ".json" else (out_dir / "graph.json") + _import_graph_json_into_store(gj, store) if store.number_of_nodes() == 0: raise FileNotFoundError( f"No graph found for {out_dir} (FalkorDB graph empty). Re-run /graphify to build." @@ -37,6 +45,34 @@ def _connect_graph(graph_path: str): sys.exit(1) +def _import_graph_json_into_store(gj_path: Path, store) -> bool: + """One-time import of a node-link ``graph.json`` into an empty FalkorDB store. + + Back-compat for projects built before the FalkorDB backend (and for + ``--no-cluster`` output, which writes only JSON): rather than erroring with + "graph empty", load the JSON so existing graphs stay queryable without a + rebuild. Returns True if any nodes were imported. + """ + try: + if not gj_path.exists(): + return False + data = json.loads(gj_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + nodes = data.get("nodes") or [] + if not nodes: + return False + edges = data.get("links", data.get("edges", [])) + from graphify.build import build_from_json + build_from_json({"nodes": nodes, "edges": edges}, store=store) + print( + f"[graphify] Imported {len(nodes)} nodes from {gj_path.name} into FalkorDB " + "(one-time migration from a pre-FalkorDB / --no-cluster graph).", + file=sys.stderr, + ) + return True + + def _communities_from_graph(G: nx.Graph) -> dict[int, list[str]]: """Reconstruct community dict from community property stored on nodes.""" communities: dict[int, list[str]] = {} diff --git a/tests/test_store_edge_semantics.py b/tests/test_store_edge_semantics.py index 0f82bbc77..ba9a7a22f 100644 --- a/tests/test_store_edge_semantics.py +++ b/tests/test_store_edge_semantics.py @@ -2,7 +2,10 @@ surfaced by the PR #1379 review.""" from __future__ import annotations +import json + from graphify import affected as aff +from graphify.serve import _import_graph_json_into_store def test_edge_attr_lookup_prefers_forward_direction(store): @@ -64,3 +67,20 @@ def test_resolve_seed_native_bare_name(store): on the native store path, matching the in-memory _bare_name tier.""" store.add_nodes_from([("n1", {"label": "foo()"}), ("n2", {"label": "fooExtra()"})]) assert aff.resolve_seed(store, "foo") == "n1" + + +def test_legacy_graph_json_imports_into_empty_store(store, tmp_path): + """A node-link graph.json (pre-FalkorDB project / --no-cluster output) must + import into an empty store so it stays queryable without a rebuild.""" + gj = tmp_path / "graph.json" + gj.write_text(json.dumps({ + "nodes": [{"id": "a", "label": "A", "source_file": "a.py"}, + {"id": "b", "label": "B", "source_file": "b.py"}], + "links": [{"source": "a", "target": "b", "relation": "calls"}], + })) + assert store.number_of_nodes() == 0 + assert _import_graph_json_into_store(gj, store) is True + assert store.number_of_nodes() == 2 + assert store.number_of_edges() == 1 + # Missing / empty JSON is a no-op, not an error. + assert _import_graph_json_into_store(tmp_path / "missing.json", store) is False From b30e3efe11572d91f236e1c3b1b5e1716a7b2f0c Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Thu, 18 Jun 2026 16:11:19 +0300 Subject: [PATCH 16/17] =?UTF-8?q?feat:=20zero-setup=20default=20=E2=80=94?= =?UTF-8?q?=20auto-fall=20back=20to=20embedded=20FalkorDB=20Lite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When no FalkorDB server is reachable at the local default, _connect now transparently starts the embedded engine (FalkorDB Lite) instead of failing, so a plain install runs with no server to manage — matching the pre-FalkorDB zero-setup experience. falkordblite is now a default dependency on Python >= 3.12; on older interpreters a running server is still required (clear error). A connection failure to an explicitly-configured REMOTE host still surfaces as an error rather than silently using a local empty DB. Shared embedded RDB lives under XDG_DATA_HOME/graphify (override: GRAPHIFY_LITE_DBFILE). Found via PR #1379 comprehensive review (#2). Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/store.py | 42 ++++++++++++++++++++---- pyproject.toml | 5 +++ tests/test_connect_fallback.py | 60 ++++++++++++++++++++++++++++++++++ uv.lock | 4 ++- 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 tests/test_connect_fallback.py diff --git a/graphify/store.py b/graphify/store.py index 74bc8de9c..ef79612d1 100644 --- a/graphify/store.py +++ b/graphify/store.py @@ -148,14 +148,44 @@ def _connect(uri: str, user: str | None = None, password: str | None = None): from falkordb import FalkorDB except ImportError as e: # pragma: no cover raise ImportError("falkordb SDK not installed. Run: pip install falkordb") from e + host = parsed.hostname or "localhost" + port = parsed.port or 6379 connect_user = parsed.username or (user if password else None) connect_password = parsed.password or (password or None) - return FalkorDB( - host=parsed.hostname or "localhost", - port=parsed.port or 6379, - username=connect_user, - password=connect_password, - ) + db = FalkorDB(host=host, port=port, username=connect_user, password=connect_password) + try: + db.connection.ping() + return db + except Exception as exc: + # Server unreachable. For the local default, transparently fall back to + # the embedded engine (FalkorDB Lite) so graphify works with zero setup, + # like the pre-FalkorDB version. An explicitly-configured remote host that + # is down is a real error and must surface, not silently use a local DB. + if host not in ("localhost", "127.0.0.1", "::1"): + raise + try: + return _connect_lite(_default_lite_dbfile()) + except ImportError: + raise ConnectionError( + f"Could not reach a FalkorDB server at {host}:{port}, and the embedded " + "engine (FalkorDB Lite) is not installed. Either start a FalkorDB server " + "or install the embedded engine: pip install falkordblite (Python >= 3.12)." + ) from exc + + +def _default_lite_dbfile() -> str: + """Stable on-disk location for the auto-fallback embedded engine. One shared + RDB holds every project's graph (each project uses a unique graph name, so + they coexist); lives outside any repo, like a server's datadir. Override with + GRAPHIFY_LITE_DBFILE.""" + import os + override = os.environ.get("GRAPHIFY_LITE_DBFILE") + if override: + return override + base = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share") + d = Path(base) / "graphify" + d.mkdir(parents=True, exist_ok=True) + return str(d / "falkordb_lite.rdb") # --------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 73f39e6e1..54ff78e3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,11 @@ keywords = ["claude", "claude-code", "codex", "opencode", "kilo", "cursor", "gem requires-python = ">=3.10" dependencies = [ "falkordb>=1.0", + # Embedded FalkorDB engine — lets graphify run with zero setup (no separate + # server) by auto-falling back to it when no local server is reachable. + # Only available on Python >= 3.12; on older interpreters a running FalkorDB + # server is required. + "falkordblite; python_version >= '3.12'", "numpy>=1.21", "rapidfuzz>=3.0", "tree-sitter>=0.23.0,<0.26", diff --git a/tests/test_connect_fallback.py b/tests/test_connect_fallback.py new file mode 100644 index 000000000..f8660e751 --- /dev/null +++ b/tests/test_connect_fallback.py @@ -0,0 +1,60 @@ +"""Unit tests for the embedded-engine auto-fallback in store._connect. + +These mock the FalkorDB client so they run without a live server (and without +the embedded engine, which needs Python >= 3.12) — they verify the dispatch +logic only.""" +from __future__ import annotations + +import falkordb +import pytest + +import graphify.store as st + + +class _FakeConn: + def __init__(self, ok: bool): + self._ok = ok + + def ping(self): + if not self._ok: + raise ConnectionError("Connection refused") + return True + + +class _FakeDB: + def __init__(self, ok: bool = True, **kwargs): + self.connection = _FakeConn(ok) + + +def test_uses_server_when_reachable(monkeypatch): + fake = _FakeDB(ok=True) + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: fake) + monkeypatch.setattr(st, "_connect_lite", lambda dbfile: pytest.fail("must not fall back")) + assert st._connect("falkordb://localhost:6379") is fake + + +def test_local_server_down_falls_back_to_lite(monkeypatch): + sentinel = object() + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: _FakeDB(ok=False)) + monkeypatch.setattr(st, "_connect_lite", lambda dbfile: sentinel) + assert st._connect("falkordb://localhost:6379") is sentinel + + +def test_remote_server_down_raises(monkeypatch): + """A connection failure to an explicitly-configured remote host must surface, + not silently switch to a local empty embedded DB.""" + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: _FakeDB(ok=False)) + monkeypatch.setattr(st, "_connect_lite", lambda dbfile: pytest.fail("no fallback for remote")) + with pytest.raises(ConnectionError): + st._connect("falkordb://db.internal.example.com:6379") + + +def test_local_down_without_lite_gives_helpful_error(monkeypatch): + def _no_lite(dbfile): + raise ImportError("redislite not installed") + + monkeypatch.setattr(falkordb, "FalkorDB", lambda **kw: _FakeDB(ok=False)) + monkeypatch.setattr(st, "_connect_lite", _no_lite) + with pytest.raises(ConnectionError) as exc: + st._connect("falkordb://localhost:6379") + assert "falkordblite" in str(exc.value) diff --git a/uv.lock b/uv.lock index cee6be794..38aded1ce 100644 --- a/uv.lock +++ b/uv.lock @@ -1182,10 +1182,11 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.8.40" +version = "0.8.41" source = { editable = "." } dependencies = [ { name = "falkordb" }, + { name = "falkordblite", marker = "python_full_version >= '3.12'" }, { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "rapidfuzz" }, @@ -1344,6 +1345,7 @@ requires-dist = [ { name = "falkordb", specifier = ">=1.0" }, { name = "falkordb", marker = "extra == 'all'" }, { name = "falkordb", marker = "extra == 'falkordb'", specifier = ">=1.0" }, + { name = "falkordblite", marker = "python_full_version >= '3.12'" }, { name = "falkordblite", marker = "python_full_version >= '3.12' and extra == 'lite'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'all'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'video'" }, From 1c229debbd9e49ecbbcfbb0d6cc6aafe73fbe025 Mon Sep 17 00:00:00 2001 From: Gal Shubeli Date: Wed, 29 Jul 2026 10:14:10 +0300 Subject: [PATCH 17/17] fix: port the remaining v8 tests onto the FalkorDB backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clears the 20 failures CI reported on the merge commit. Behavioral fixes (not test-only): - build_from_json now records the scan root on the graph (`scan_root` meta) and build_merge reads it back when the caller omits root. This is the FalkorDB replacement for the `graphify-out/.graphify_root` marker; without it the skill's root-less --update never relativized absolute prune_sources and a deleted file's nodes survived as ghosts (#1571). - to_graphml dropped graph-level attributes entirely, losing the hyperedge layer from every export; they are now emitted as for="graph" keys with the same #1831 scalar coercion as nodes/edges. - find_import_cycles required the graph object to expose .simple_cycles(), so it crashed on a plain NetworkX graph passed in by a library caller (networkx exposes simple_cycles as a module function, not a method). The enumeration is a pure function of the edge list, so it moved to store.simple_cycles_from_edges with the FalkorDB UDF kept as the fast path. - merge-graphs emitted directed: true; the combined cross-repo view is undirected, as the old nx.compose path produced (#1606). merge_node_link now takes directed=, defaulting to the first input's flag so the same-graph merge-driver still round-trips. - benchmark could not read a pre-FalkorDB / --no-cluster graph.json; it now falls back to the same back-compat import serve and affected use. - The MCP multi-project loader reported "could not connect to graph"; it is a not-found condition and now says so. Test ports (v8 tests written against file-based loading): - test_build_merge_hyperedges_and_prune: seed the store, pass graph_name; the two #1571 cases now pin the recorded scan_root instead of the marker file. - test_extract: read edge direction from (u, v) — the _src/_tgt markers the undirected NetworkX storage needed no longer exist. - test_export: build the dict/list-attribute graph as a MemGraph, the only shape that can carry non-scalar attrs (the engine stores scalars only). - test_extract_cli: inject the committed semantic layer into the store, which is the incremental baseline now, not graph.json. - test_src_layout_import_resolution: give each build its own store — without one, both builds target the default graph name and the second silently overwrites the first, comparing a graph with itself. Co-Authored-By: Claude Opus 5 (1M context) --- graphify/analyze.py | 9 +- graphify/benchmark.py | 10 ++ graphify/build.py | 23 +++- graphify/cli.py | 7 +- graphify/export.py | 14 ++ graphify/graphjson.py | 16 ++- graphify/serve.py | 5 +- graphify/store.py | 38 ++++++ .../test_build_merge_hyperedges_and_prune.py | 126 ++++++++++-------- tests/test_export.py | 29 ++-- tests/test_extract.py | 15 ++- tests/test_extract_cli.py | 16 +++ tests/test_src_layout_import_resolution.py | 11 +- 13 files changed, 232 insertions(+), 87 deletions(-) diff --git a/graphify/analyze.py b/graphify/analyze.py index fb4c594e0..08dc9e76a 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -712,7 +712,14 @@ def _endpoint_source_file(node_id: str) -> str: multi_edges = [(a, b) for a, b in file_edges if a != b] for f in self_loops: cycles.append([f]) - for cycle in G.simple_cycles(sorted(multi_edges), max_cycle_length): + # Prefer the backend's own enumeration (FalkorDB runs it as a server-side + # UDF); otherwise fall back to the pure-Python one. The fallback keeps this + # working for a plain NetworkX graph passed in by a library caller, which has + # no .simple_cycles() method (networkx exposes it as a module function). + _cycles = getattr(G, "simple_cycles", None) + if _cycles is None: + from graphify.store import simple_cycles_from_edges as _cycles + for cycle in _cycles(sorted(multi_edges), max_cycle_length): if len(cycle) <= max_cycle_length: cycles.append(cycle) if len(cycles) >= top_n * 10: diff --git a/graphify/benchmark.py b/graphify/benchmark.py index e07ba4280..66fa7cec6 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -103,6 +103,16 @@ def run_benchmark( resolved = Path(graph_path or _default_graph_json()) out_dir = resolved.parent if resolved.suffix else resolved G = open_store(out_dir, create=False) + if G.number_of_nodes() == 0: + # Back-compat, matching serve._connect_graph and affected.connect_graph: + # an empty store may still have a node-link graph.json beside it (a + # pre-FalkorDB project, or a `--no-cluster` run that only wrote JSON). + # Import it so `graphify benchmark` works on the same graphs the read + # commands already accept. + from graphify.serve import _import_graph_json_into_store + + gj = resolved if resolved.suffix == ".json" else (out_dir / "graph.json") + _import_graph_json_into_store(gj, G) if corpus_words is None: # Rough estimate: each node label is ~3 words, plus source context diff --git a/graphify/build.py b/graphify/build.py index 46dc46178..9e6efd3dd 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -1070,6 +1070,15 @@ def build_from_json( if kept_hyperedges: store.graph["hyperedges"] = kept_hyperedges store.save_meta() + # Record the scan root on the graph itself. This is the FalkorDB replacement + # for the committed `graphify-out/.graphify_root` marker: a later build_merge + # called WITHOUT root (the skill's --update runbook does exactly that) reads + # it back to relativize absolute prune_sources against the stored relative + # source_file keys. Without it those paths never match and a deleted file's + # nodes survive as ghosts (#1571). + if _root: + store.graph["scan_root"] = _root + store.save_meta() # Warm + persist the hub-threshold (p99 degree) so query traversals don't # recompute it on every invocation. try: @@ -1358,11 +1367,15 @@ def build_merge( had_graph = bool(existing_nodes) # Effective root for relativizing absolute source_file / prune paths back to - # the stored relative source_file keys. The graph lives in FalkorDB, so there - # is no graph.json whose sibling `.graphify_root` marker could supply a - # recorded scan root (#1571); callers that need normalization pass root - # explicitly — cli's extract path and the skill's --update runbook both do. - _eff_root = str(Path(root).resolve()) if root is not None else None + # the stored relative source_file keys. When the caller passes root we use it; + # otherwise fall back to the scan root the graph recorded at build time — the + # FalkorDB replacement for the `.graphify_root` marker. Without that fallback + # a root-less call (the skill's --update runbook) never matches absolute + # prune_sources and the deleted files' nodes survive as ghosts (#1571). + _eff_root = ( + str(Path(root).resolve()) if root is not None + else (store.graph.get("scan_root") or None) + ) # Re-extracted files REPLACE their prior contribution (#1344/#1007). Every # source_file present in new_chunks is dropped from the loaded base before diff --git a/graphify/cli.py b/graphify/cli.py index 816b71350..6922746e5 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1999,9 +1999,10 @@ def dispatch_command(cmd: str) -> None: naive_tags = [gp.parent.parent.name for gp in graph_paths] if len(set(naive_tags)) != len(naive_tags): print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") - merged = merge_node_link([ - prefix_node_link(data, repo_tag) for data, repo_tag in zip(datas, repo_tags) - ]) + merged = merge_node_link( + [prefix_node_link(data, repo_tag) for data, repo_tag in zip(datas, repo_tags)], + directed=False, # the combined cross-repo view is undirected (#1606) + ) out_path.parent.mkdir(parents=True, exist_ok=True) from graphify.paths import write_json_atomic as _wja _wja(out_path, merged, indent=2) diff --git a/graphify/export.py b/graphify/export.py index d0d827c83..5a0d30d69 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1026,11 +1026,25 @@ def esc(x): '', '', ] + # Graph-level attributes (notably the `hyperedges` list attached by + # attach_hyperedges) are part of the export too — dropping them would lose + # the hypergraph layer entirely. Same scalar coercion as nodes/edges (#1831). + graph_rows = { + k: _graphml_safe(v) for k, v in getattr(G, "graph", {}).items() + if not k.startswith("_") + } for name, gtype in node_keys.items(): lines.append(f' ') for name, gtype in edge_keys.items(): lines.append(f' ') + for name, val in graph_rows.items(): + lines.append( + f' ' + ) lines.append(' ') + for name, val in graph_rows.items(): + lines.append(f' {esc(val)}') for nid, attrs in node_rows: lines.append(f' ') for k, v in attrs.items(): diff --git a/graphify/graphjson.py b/graphify/graphjson.py index 56bed0cb2..30ee05800 100644 --- a/graphify/graphjson.py +++ b/graphify/graphjson.py @@ -36,8 +36,16 @@ def _edge_key(e: dict): return (e.get("source"), e.get("target"), e.get("relation")) -def merge_node_link(graphs: list[dict]) -> dict: - """Union several node-link dicts. Later graphs win on node/edge attr conflicts.""" +def merge_node_link(graphs: list[dict], *, directed: bool | None = None) -> dict: + """Union several node-link dicts. Later graphs win on node/edge attr conflicts. + + ``directed`` sets the merged graph's flag. None (the default) inherits the + first input's, so merging two versions of the same graph round-trips. The + cross-repo `merge-graphs` view passes False: per-repo graphs are written by + different extract paths and may disagree on directedness, and the combined + view is undirected — what the old nx.compose path produced by normalizing + every input to a plain Graph (#1606). + """ nodes: dict = {} edges: dict = {} for g in graphs: @@ -48,8 +56,10 @@ def merge_node_link(graphs: list[dict]) -> dict: nodes[nid] = {**nodes.get(nid, {}), **n} for e in g.get("links", []): edges[_edge_key(e)] = e + if directed is None: + directed = bool(graphs[0].get("directed", True)) if graphs else True return { - "directed": True, + "directed": directed, "multigraph": False, "graph": {}, "nodes": list(nodes.values()), diff --git a/graphify/serve.py b/graphify/serve.py index c2b660c5a..bb90b2c59 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -1337,7 +1337,10 @@ def _load_ctx(path: str): try: new_G = _connect_graph(path) except SystemExit as e: # _connect_graph exits on missing/empty graph - raise RuntimeError(f"could not connect to graph at {path}") from e + # Phrased as not-found because that is what it is: no graph has + # been built for that project. Raised (not exited) so a bad + # project_path is one tool error, not a dead server. + raise RuntimeError(f"graph not found for {path} (no graph built there)") from e # Warm the trigram index before exposing the graph so the first query # against it is fast (same rationale as the original startup warm-up). # Skipped for store-backed graphs: they prefilter with an in-engine diff --git a/graphify/store.py b/graphify/store.py index 191ffdcc6..7ca9c1d64 100644 --- a/graphify/store.py +++ b/graphify/store.py @@ -81,6 +81,41 @@ def graph_name_for(path: str | Path) -> str: return f"graphify_{base}_{digest}" +def simple_cycles_from_edges(edges, max_len: int = 5) -> list[list[str]]: + """Directed simple cycles up to ``max_len``, over an explicit edge list. + + Pure-Python counterpart of ``GraphStore.simple_cycles`` (which runs the + FalkorDB UDF). Depends only on the edge list, not on the graph object, so + callers work against a store, a MemGraph, or a plain NetworkX graph handed + in by a library user. Each cycle is returned once, rotated to start at its + smallest node, so the output is canonical and order-independent. + """ + adj: dict = {} + for u, v in edges: + adj.setdefault(str(u), []).append(str(v)) + for succs in adj.values(): + succs.sort() + seen: set[tuple] = set() + out: list[list[str]] = [] + + def _walk(start: str, node: str, path: list[str]) -> None: + for nxt in adj.get(node, ()): + if nxt == start and len(path) > 1: + lo = path.index(min(path)) + canon = tuple(path[lo:] + path[:lo]) + if canon not in seen: + seen.add(canon) + out.append(list(canon)) + elif nxt not in path and len(path) < max_len and nxt > start: + # `nxt > start` keeps each cycle to the single traversal that + # begins at its smallest member — no duplicate rotations. + _walk(start, nxt, path + [nxt]) + + for n in sorted(adj): + _walk(n, n, [n]) + return out + + def pointer_path(out_dir: str | Path = "graphify-out") -> Path: """Path of the FalkorDB pointer file that records the graph name + URI.""" return Path(out_dir) / "falkordb.json" @@ -420,6 +455,9 @@ def has_edge(self, u, v): def has_directed_edge(self, u, v): return any(a == u and b == v for a, b, _ in self._esorted) + def simple_cycles(self, edges, max_len: int = 5): + return simple_cycles_from_edges(edges, max_len) + def edge_attrs_all(self, u, v) -> list[dict]: """Every stored edge between u and v, forward-preferred. diff --git a/tests/test_build_merge_hyperedges_and_prune.py b/tests/test_build_merge_hyperedges_and_prune.py index d3629a73c..5bd8b4015 100644 --- a/tests/test_build_merge_hyperedges_and_prune.py +++ b/tests/test_build_merge_hyperedges_and_prune.py @@ -18,15 +18,27 @@ import pytest -from graphify.build import build_merge, _infer_merge_root - - -def _write_graph(graph_path: Path, nodes, edges, hyperedges) -> None: - """Write a graph.json in the shape to_json emits (top-level hyperedges).""" - graph_path.write_text( - json.dumps({"nodes": nodes, "edges": edges, "hyperedges": hyperedges}), - encoding="utf-8", +from graphify.build import build_merge, build_from_json, _infer_merge_root + + +def _seed(make_store, nodes, edges, hyperedges, *, root=None, scan_root=None): + """Seed a FalkorDB graph the way a prior build would have left it. + + Replaces the old `_write_graph` graph.json fixture: build_merge now reads its + base from the store, so the "existing graph" has to live there. `root` + normalizes source_file the way a rooted build would; `scan_root` records the + root WITHOUT normalizing, which models a graph built with a root whose + semantic nodes still carry absolute source_file paths (#932/#2012). + """ + st = make_store() + build_from_json( + {"nodes": nodes, "edges": edges, "hyperedges": hyperedges}, + store=st, root=root, ) + if scan_root is not None: + st.graph["scan_root"] = str(Path(scan_root).resolve()) + st.save_meta() + return st def _he_ids(G) -> set[str]: @@ -35,10 +47,9 @@ def _he_ids(G) -> set[str]: # ── #1574: hyperedge preservation ───────────────────────────────────────────── -def _seed_two_file_graph(tmp_path): +def _seed_two_file_graph(tmp_path, make_store): root = tmp_path / "corpus" root.mkdir() - graph_path = tmp_path / "graph.json" nodes = [ {"id": "a1", "label": "a1", "file_type": "document", "source_file": "a.md"}, {"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}, @@ -48,19 +59,19 @@ def _seed_two_file_graph(tmp_path): {"id": "he_b", "label": "flow B", "source_file": "b.md", "nodes": ["b1"]}, {"id": "he_global", "label": "cross-file flow", "nodes": ["a1", "b1"]}, # no source_file ] - _write_graph(graph_path, nodes, [], hyperedges) - return root, graph_path + # scan_root recorded so the root-less calls below can still relativize (#1571). + return root, _seed(make_store, nodes, [], hyperedges, root=root, scan_root=root) -def test_update_preserves_hyperedges_of_unchanged_files(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) +def test_update_preserves_hyperedges_of_unchanged_files(tmp_path, make_store): + root, st = _seed_two_file_graph(tmp_path, make_store) # Re-extract only b.md, with a fresh hyperedge for it. new_chunk = { "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], "edges": [], "hyperedges": [{"id": "he_b_v2", "label": "flow B v2", "source_file": "b.md", "nodes": ["b1"]}], } - G = build_merge([new_chunk], graph_path, dedup=False, root=root) + G = build_merge([new_chunk], st.graph_name, dedup=False, root=root) ids = _he_ids(G) assert "he_a" in ids # unchanged file's hyperedge preserved (the bug) assert "he_global" in ids # source_file-less hyperedge preserved @@ -68,24 +79,24 @@ def test_update_preserves_hyperedges_of_unchanged_files(tmp_path): assert "he_b" not in ids # re-extracted file's OLD hyperedge replaced -def test_update_without_root_still_preserves_hyperedges(tmp_path): +def test_update_without_root_still_preserves_hyperedges(tmp_path, make_store): """The runbook omits root; the fallback root must not break preservation.""" - root, graph_path = _seed_two_file_graph(tmp_path) + root, st = _seed_two_file_graph(tmp_path, make_store) new_chunk = { "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], "edges": [], "hyperedges": [{"id": "he_b_v2", "source_file": "b.md", "nodes": ["b1"]}], } - G = build_merge([new_chunk], graph_path, dedup=False) # no root + G = build_merge([new_chunk], st.graph_name, dedup=False) # no root ids = _he_ids(G) assert {"he_a", "he_global", "he_b_v2"} <= ids assert "he_b" not in ids -def test_deleted_file_hyperedges_are_pruned(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) +def test_deleted_file_hyperedges_are_pruned(tmp_path, make_store): + root, st = _seed_two_file_graph(tmp_path, make_store) deleted_abs = [str(root / "a.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) + G = build_merge([], st.graph_name, prune_sources=deleted_abs, dedup=False, root=root) ids = _he_ids(G) assert "he_a" not in ids # deleted file's hyperedge pruned assert "he_b" in ids # untouched file's hyperedge kept @@ -96,41 +107,45 @@ def test_deleted_file_hyperedges_are_pruned(tmp_path): # ── #1571: root-less prune (absolute deleted paths vs relative node keys) ────── -def test_prune_without_root_removes_ghost_nodes_via_grandparent_fallback(tmp_path): +def test_prune_without_root_removes_ghost_nodes_via_recorded_scan_root(tmp_path, make_store): + """#1571, store edition: the graph records its scan root at build time (the + FalkorDB replacement for the `.graphify_root` marker), so a runbook-style + root-less call can still relativize an absolute prune path.""" root = tmp_path / "corpus" (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" nodes = [ {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, ] - _write_graph(graph_path, nodes, [], []) + st = _seed(make_store, nodes, [], [], root=root) # Runbook-style call: absolute prune path, NO root passed. deleted_abs = [str(root / "HANDOFF.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False) + G = build_merge([], st.graph_name, prune_sources=deleted_abs, dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "handoff" not in labels, "deleted file's ghost node must be pruned without root" assert "keep" in labels -def test_prune_without_root_uses_graphify_root_marker(tmp_path): - # graph.json not under a /graphify-out layout, so grandparent wouldn't - # help — the committed .graphify_root marker must be honored instead. +def test_prune_without_root_uses_recorded_root_outside_graphify_out_layout(tmp_path, make_store): + """A scan root that is NOT the output dir's parent must still be honored. + _infer_merge_root still reads the committed `.graphify_root` marker for the + file-based callers; the store carries the same value as `scan_root`.""" out = tmp_path / "out" out.mkdir() graph_path = out / "graph.json" real_root = tmp_path / "elsewhere" / "repo" real_root.mkdir(parents=True) (out / ".graphify_root").write_text(str(real_root), encoding="utf-8") - nodes = [{"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}] - _write_graph(graph_path, nodes, [], []) + graph_path.write_text("{}", encoding="utf-8") assert _infer_merge_root(graph_path) == str(real_root.resolve()) - G = build_merge([], graph_path, prune_sources=[str(real_root / "HANDOFF.md")], dedup=False) + nodes = [{"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}] + st = _seed(make_store, nodes, [], [], root=real_root) + G = build_merge([], st.graph_name, prune_sources=[str(real_root / "HANDOFF.md")], dedup=False) assert "handoff" not in {d["label"] for _, d in G.nodes(data=True)} @pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics") -def test_prune_matches_across_symlinked_root(tmp_path): +def test_prune_matches_across_symlinked_root(tmp_path, make_store): """A symlinked scan root (macOS /var -> /private/var, symlinked home/worktree) makes the absolute prune path and the resolved root differ by prefix. The prune must still match — lexical relative_to fails, so normalization resolves both @@ -139,27 +154,24 @@ def test_prune_matches_across_symlinked_root(tmp_path): (real / "graphify-out").mkdir(parents=True) link = tmp_path / "link" os.symlink(real, link) - graph_path = real / "graphify-out" / "graph.json" - _write_graph(graph_path, [ + st = _seed(make_store, [ {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, - ], [], []) + ], [], [], root=real) # prune path addressed via the SYMLINK, root resolved to the real dir - G = build_merge([], graph_path=graph_path, + G = build_merge([], st.graph_name, prune_sources=[str(link / "HANDOFF.md")], root=str(real), dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "handoff" not in labels and "keep" in labels -def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): +def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path, make_store): """#1796: a file present in BOTH new_chunks (re-extracted) and prune_sources must be REPLACED, not deleted. The old edit-workflow passed the changed file in prune_sources; combined with dedup keeping a same-label node, that used to silently delete the freshly re-extracted concept. Replace wins over delete.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, + st = _seed( + make_store, nodes=[ {"id": "foo_widget_cache", "label": "Widget Cache Design", "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, @@ -168,6 +180,7 @@ def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): ], edges=[], hyperedges=[], + root=tmp_path, ) # foo.md edited: same-label node re-extracted (new content/line) new_chunk = {"nodes": [ @@ -175,19 +188,17 @@ def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} ], "edges": []} - G = build_merge([new_chunk], graph_path=str(graph_path), + G = build_merge([new_chunk], st.graph_name, prune_sources=["docs/foo.md"], root=str(tmp_path)) labels = {G.nodes[n].get("label") for n in G.nodes()} assert "Widget Cache Design" in labels, "re-extracted node was wrongly pruned" -def test_genuine_deletion_still_prunes(tmp_path): +def test_genuine_deletion_still_prunes(tmp_path, make_store): """#1796 guard must not break real deletions: a file in prune_sources but NOT in new_chunks is still removed.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, + st = _seed( + make_store, nodes=[ {"id": "foo_widget_cache", "label": "Widget Cache Design", "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, @@ -196,13 +207,14 @@ def test_genuine_deletion_still_prunes(tmp_path): ], edges=[], hyperedges=[], + root=tmp_path, ) new_chunk = {"nodes": [ {"id": "foo_widget_cache", "label": "Widget Cache Design", "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} ], "edges": []} # bar.md genuinely deleted (not re-extracted) - G = build_merge([new_chunk], graph_path=str(graph_path), + G = build_merge([new_chunk], st.graph_name, prune_sources=["docs/bar.md"], root=str(tmp_path)) labels = {G.nodes[n].get("label") for n in G.nodes()} assert "Other" not in labels, "genuinely deleted file's node should be pruned" @@ -211,7 +223,7 @@ def test_genuine_deletion_still_prunes(tmp_path): # ── #2012: form-insensitive prune (absolute node vs relative prune, and back) ── -def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): +def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path, make_store): """#2012: a node whose source_file survived in ABSOLUTE form must still be pruned when the deletion is expressed relative to root. The runbook calls build_merge WITHOUT root, so build() does not re-normalize the node's stored @@ -221,7 +233,6 @@ def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): now normalizes the node side too + an absolute-identity fallback.""" root = tmp_path / "corpus" (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" nodes = [ # gone.py's node kept an ABSOLUTE source_file (a semantic subagent wrote # it that way, #932); keep.py's is relative. @@ -233,34 +244,35 @@ def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): {"source": "g1", "target": "k1", "type": "calls", "source_file": str(root / "gone.py")}, ] - _write_graph(graph_path, nodes, edges, []) + # scan_root (not root=) so the absolute source_file is NOT re-normalized — + # that mismatched-form survival is exactly what #2012 is about. + st = _seed(make_store, nodes, edges, [], scan_root=root) # Runbook-style: NO root passed (eff_root inferred from the graphify-out # grandparent), so build() leaves the absolute node form intact. Deletion is # expressed RELATIVE — a third form vs the stored absolute node. - G = build_merge([], graph_path, prune_sources=["gone.py"], dedup=False) + G = build_merge([], st.graph_name, prune_sources=["gone.py"], dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "gone" not in labels, "absolute-stored node not pruned by relative delete (#2012)" assert "keep" in labels assert G.number_of_edges() == 0, "edge from the deleted file must be pruned too (#2012)" -def test_prune_reextracted_absolute_node_not_deleted(tmp_path): +def test_prune_reextracted_absolute_node_not_deleted(tmp_path, make_store): """#1796 protection must hold in absolute-identity space too: a file present in BOTH new_chunks and prune_sources (in mismatched forms) is REPLACED, not deleted — the #2012 form-insensitive match must not resurrect the delete for a re-extracted file.""" root = tmp_path / "corpus" (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" - _write_graph(graph_path, [ + st = _seed(make_store, [ {"id": "g1", "label": "gone", "file_type": "code", "source_file": str(root / "mod.py")}, - ], [], []) + ], [], [], scan_root=root) # Re-extracted with a RELATIVE source_file; prune lists it RELATIVE too. # No root passed (runbook), so the stored absolute node is not re-normalized. new_chunk = {"nodes": [ {"id": "g1", "label": "gone", "file_type": "code", "source_file": "mod.py"}, ], "edges": []} - G = build_merge([new_chunk], graph_path, prune_sources=["mod.py"], dedup=False) + G = build_merge([new_chunk], st.graph_name, prune_sources=["mod.py"], dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "gone" in labels, "re-extracted file wrongly pruned across mismatched forms (#2012/#1796)" diff --git a/tests/test_export.py b/tests/test_export.py index 28a4707a7..d5ac61539 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -104,14 +104,27 @@ def test_to_graphml_tolerates_dict_and_list_attribute_values(): metadata, or the graph-level hyperedges list) used to crash the whole export. to_graphml must JSON-serialize them across graph/node/edge scopes (#1831).""" import networkx as nx - G = make_graph() - communities = cluster(G) - a_node = next(iter(G.nodes())) - G.nodes[a_node]["metadata"] = {"kind": "file", "size": 12} - G.nodes[a_node]["tags"] = ["x", "y"] - if G.number_of_edges(): - u, v = next(iter(G.edges())) - G.edges[u, v]["ctx"] = {"k": "v"} + + from graphify.store import MemGraph + + # Built as a MemGraph rather than a FalkorDB-backed store: the engine only + # stores scalar properties, so a dict/list attribute can only ever reach + # to_graphml through the in-memory graph (the aggregated-view and library + # paths). Attributes are set at construction because a store's node view + # hands back a snapshot — assigning into it would not persist. + a_node, b_node = "n_meta", "n_other" + G = MemGraph( + [ + (a_node, {"label": "A", "source_file": "a.py", + "metadata": {"kind": "file", "size": 12}, "tags": ["x", "y"]}), + (b_node, {"label": "B", "source_file": "b.py"}), + ], + [(a_node, b_node, {"relation": "calls", "confidence": "EXTRACTED", + "ctx": {"k": "v"}})], + ) + # A literal mapping rather than cluster(): community detection runs in the + # engine, and this test is about GraphML serialization, not clustering. + communities = {0: [a_node, b_node]} G.graph["hyperedges"] = [{"nodes": [a_node], "label": "h"}] with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) / "graph.graphml" diff --git a/tests/test_extract.py b/tests/test_extract.py index e0a880b89..0314f02ee 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2309,19 +2309,22 @@ def test_extract_json_import_and_extends_targets_are_real_nodes(tmp_path): extracted = extract([package_json, tsconfig], cache_root=tmp_path, parallel=False) graph = build_from_json(extracted, directed=True) + # Edges are stored in their native source->target orientation, so the (u, v) + # yielded by edges() IS the direction — the old _src/_tgt markers that the + # undirected NetworkX storage needed no longer exist. import_targets = { - graph.nodes[data["_tgt"]]["label"] - for _, _, data in graph.edges(data=True) + graph.nodes[v]["label"] + for _, v, data in graph.edges(data=True) if data.get("relation") == "imports" } extends_targets = { - graph.nodes[data["_tgt"]]["label"] - for _, _, data in graph.edges(data=True) + graph.nodes[v]["label"] + for _, v, data in graph.edges(data=True) if data.get("relation") == "extends" } self_loops = [ - data for _, _, data in graph.edges(data=True) - if data.get("relation") in {"imports", "extends"} and data["_src"] == data["_tgt"] + data for u, v, data in graph.edges(data=True) + if data.get("relation") in {"imports", "extends"} and u == v ] assert self_loops == [] assert {"left-pad", "bats"} <= import_targets diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index b1c369a5e..48e3ca92f 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -664,6 +664,22 @@ def _sem_doc_count(g): {"id": "h1", "label": "Shared", "nodes": ["doc_readme_a", "doc_readme_b"], "relation": "participate_in", "source_file": "README.md"}) graph_path.write_text(json.dumps(graph)) + # The FalkorDB store — not graph.json — is the incremental baseline, so the + # committed semantic layer has to be injected there too. graph.json alone is + # a derived artifact and the rebuild would never see these nodes. + from graphify.store import open_store as _open_store + + _st = _open_store(graphify_out, create=False) + _st.add_nodes_from([ + (n["id"], {k: v for k, v in n.items() if k != "id"}) + for n in graph["nodes"] if n["id"].startswith("doc_readme_") + ]) + _st.add_edges_from([ + (e["source"], e["target"], {k: v for k, v in e.items() if k not in ("source", "target")}) + for e in graph["edges"] if str(e.get("source", "")).startswith("doc_readme_") + ]) + _st.graph["hyperedges"] = graph["hyperedges"] + _st.save_meta() (graphify_out / ".graphify_semantic_marker").write_text( json.dumps({"output_tokens": 1})) diff --git a/tests/test_src_layout_import_resolution.py b/tests/test_src_layout_import_resolution.py index beeb6b95b..e25cca8b0 100644 --- a/tests/test_src_layout_import_resolution.py +++ b/tests/test_src_layout_import_resolution.py @@ -63,7 +63,7 @@ def test_resolve_python_module_path_walks_up_to_src_package_root(tmp_path): ) -def test_import_edges_identical_from_root_or_src(tmp_path): +def test_import_edges_identical_from_root_or_src(tmp_path, make_store): """Headline (#2072): the same project yields the same import edges whether scanned from the repo root or from src/ (modulo the `src_` id prefix).""" direct = tmp_path / "direct" @@ -73,8 +73,13 @@ def test_import_edges_identical_from_root_or_src(tmp_path): dpaths = [direct / r for r in _FILES] npaths = [nested / "src" / r for r in _FILES] - dG = build_from_json(extract(dpaths, cache_root=tmp_path / "cd", root=direct, parallel=False), root=str(direct)) - nG = build_from_json(extract(npaths, cache_root=tmp_path / "cn", root=nested, parallel=False), root=str(nested)) + # Distinct stores: build_from_json without an explicit store targets the + # default graph name, so both builds would land in ONE FalkorDB graph and the + # second would silently overwrite the first (comparing a graph with itself). + dG = build_from_json(extract(dpaths, cache_root=tmp_path / "cd", root=direct, parallel=False), + root=str(direct), store=make_store()) + nG = build_from_json(extract(npaths, cache_root=tmp_path / "cn", root=nested, parallel=False), + root=str(nested), store=make_store()) d_edges = _import_edges(dG) # strip the `src_` prefix the nested layout adds to every id.