Add full reference graph mode

search_app.py

@@ -744,6 +744,159 @@ def create_app(config: AppConfig = CONFIG) -> Flask:
             },
         }
 
+    def build_full_graph_payload(
+        max_nodes: int, *, highlight_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Attempt to render the entire reference graph by gathering every video that
+        references another (or is referenced).
+        """
+
+        query = {
+            "bool": {
+                "should": [
+                    {"range": {"internal_references_count": {"gt": 0}}},
+                    {"range": {"referenced_by_count": {"gt": 0}}},
+                    {"exists": {"field": "internal_references"}},
+                    {"exists": {"field": "referenced_by"}},
+                ],
+                "minimum_should_match": 1,
+            }
+        }
+        source_fields = [
+            "video_id",
+            "title",
+            "channel_id",
+            "channel_name",
+            "url",
+            "date",
+            "internal_references",
+            "referenced_by",
+        ]
+        nodes: Dict[str, Dict[str, Any]] = {}
+        links: List[Dict[str, Any]] = []
+        link_seen: Set[Tuple[str, str, str]] = set()
+        batch_size = min(500, max(50, max_nodes * 2))
+        truncated = False
+
+        def ensure_node(node_id: Optional[str], doc: Optional[Dict[str, Any]] = None) -> bool:
+            if not node_id:
+                return False
+            if node_id in nodes:
+                if doc:
+                    existing = nodes[node_id]
+                    if (not existing.get("title") or existing["title"] == node_id) and doc.get("title"):
+                        existing["title"] = doc["title"]
+                    if not existing.get("channel_id") and doc.get("channel_id"):
+                        existing["channel_id"] = doc["channel_id"]
+                    if (
+                        existing.get("channel_name") in {"Unknown", node_id, None}
+                        and (doc.get("channel_name") or doc.get("channel_id"))
+                    ):
+                        existing["channel_name"] = doc.get("channel_name") or doc.get("channel_id")
+                    if not existing.get("url") and doc.get("url"):
+                        existing["url"] = doc.get("url")
+                    if not existing.get("date") and doc.get("date"):
+                        existing["date"] = doc.get("date")
+                return True
+            if len(nodes) >= max_nodes:
+                return False
+            channel_name = None
+            channel_id = None
+            url = None
+            date_val = None
+            title = node_id
+            if doc:
+                title = doc.get("title") or title
+                channel_id = doc.get("channel_id")
+                channel_name = doc.get("channel_name") or channel_id
+                url = doc.get("url")
+                date_val = doc.get("date")
+            nodes[node_id] = {
+                "id": node_id,
+                "title": title,
+                "channel_id": channel_id,
+                "channel_name": channel_name or "Unknown",
+                "url": url,
+                "date": date_val,
+                "is_root": False,
+            }
+            return True
+
+        scroll_id: Optional[str] = None
+        try:
+            body = {"query": query, "_source": source_fields, "sort": ["_doc"]}
+            response = client.search(
+                index=index, body=body, size=batch_size, scroll="1m"
+            )
+            scroll_id = response.get("_scroll_id")
+            stop_fetch = False
+            while not stop_fetch:
+                hits = response.get("hits", {}).get("hits", [])
+                if not hits:
+                    break
+                for hit in hits:
+                    if len(nodes) >= max_nodes:
+                        truncated = True
+                        stop_fetch = True
+                        break
+                    source = hit.get("_source", {}) or {}
+                    video_id = source.get("video_id")
+                    if not video_id:
+                        continue
+                    if not ensure_node(video_id, source):
+                        continue
+                    for target in normalize_reference_list(source.get("internal_references")):
+                        if target == video_id:
+                            continue
+                        if not ensure_node(target):
+                            continue
+                        key = (video_id, target, "references")
+                        if key not in link_seen:
+                            links.append(
+                                {"source": video_id, "target": target, "relation": "references"}
+                            )
+                            link_seen.add(key)
+                    for origin in normalize_reference_list(source.get("referenced_by")):
+                        if origin == video_id:
+                            continue
+                        if not ensure_node(origin):
+                            continue
+                        key = (origin, video_id, "referenced_by")
+                        if key not in link_seen:
+                            links.append(
+                                {"source": origin, "target": video_id, "relation": "referenced_by"}
+                            )
+                            link_seen.add(key)
+                if stop_fetch or not scroll_id:
+                    break
+                response = client.scroll(scroll_id=scroll_id, scroll="1m")
+                scroll_id = response.get("_scroll_id")
+                if not scroll_id:
+                    break
+        finally:
+            if scroll_id:
+                try:
+                    client.clear_scroll(scroll_id=scroll_id)
+                except Exception:
+                    pass
+
+        if highlight_id and highlight_id in nodes:
+            nodes[highlight_id]["is_root"] = True
+
+        return {
+            "root": highlight_id or "",
+            "depth": 0,
+            "nodes": list(nodes.values()),
+            "links": links,
+            "meta": {
+                "node_count": len(nodes),
+                "link_count": len(links),
+                "mode": "full",
+                "truncated": truncated,
+            },
+        }
+
     @app.route("/api/channels")
     def channels():
         include_external = request.args.get("external", default="0", type=str)
@@ -832,7 +985,9 @@ def create_app(config: AppConfig = CONFIG) -> Flask:
     @app.route("/api/graph")
     def graph_api():
         video_id = (request.args.get("video_id") or "").strip()
-        if not video_id:
+        full_graph = request.args.get("full_graph", default="0", type=str)
+        full_graph = full_graph.lower() in {"1", "true", "yes"}
+        if not full_graph and not video_id:
             return jsonify({"error": "video_id is required"}), 400
 
         try:
@@ -845,14 +1000,17 @@ def create_app(config: AppConfig = CONFIG) -> Flask:
             max_nodes = int(request.args.get("max_nodes", "200"))
         except ValueError:
             max_nodes = 200
-        max_nodes = max(10, min(max_nodes, 400))
+        max_nodes = max(10, min(max_nodes, 800 if full_graph else 400))
 
-        payload = build_graph_payload(video_id, depth, max_nodes)
-        if not payload["nodes"]:
-            return (
-                jsonify({"error": f"Video '{video_id}' was not found in the index."}),
-                404,
-            )
+        if full_graph:
+            payload = build_full_graph_payload(max_nodes, highlight_id=video_id or None)
+        else:
+            payload = build_graph_payload(video_id, depth, max_nodes)
+            if not payload["nodes"]:
+                return (
+                    jsonify({"error": f"Video '{video_id}' was not found in the index."}),
+                    404,
+                )
         payload["meta"]["max_nodes"] = max_nodes
         return jsonify(payload)
 

static/graph.html

@@ -54,6 +54,17 @@
             </select>
           </div>
 
+          <div class="field-group">
+            <label class="checkbox">
+              <input type="checkbox" id="graphFullToggle" name="full_graph" />
+              Attempt entire reference graph
+            </label>
+            <p class="field-hint">
+              Includes every video that references another (ignores depth; may be slow). Max nodes still
+              applies.
+            </p>
+          </div>
+
           <div class="field-group">
             <label for="graphLabelSize">Labels</label>
             <select id="graphLabelSize" name="label_size">

static/graph.js

@@ -7,6 +7,7 @@
   const depthInput = document.getElementById("graphDepth");
   const maxNodesInput = document.getElementById("graphMaxNodes");
   const labelSizeInput = document.getElementById("graphLabelSize");
+  const fullGraphToggle = document.getElementById("graphFullToggle");
   const statusEl = document.getElementById("graphStatus");
   const container = document.getElementById("graphContainer");
   const isEmbedded =
@@ -133,6 +134,7 @@
   let currentDepth = sanitizeDepth(depthInput.value);
   let currentMaxNodes = sanitizeMaxNodes(maxNodesInput.value);
   let currentSimulation = null;
+  let currentFullGraph = false;
 
   function setStatus(message, isError = false) {
     if (!statusEl) return;
@@ -148,10 +150,40 @@
     return (value || "").trim();
   }
 
-  async function fetchGraph(videoId, depth, maxNodes) {
+  function isFullGraphMode(forceValue) {
+    if (typeof forceValue === "boolean") {
+      return forceValue;
+    }
+    return fullGraphToggle ? !!fullGraphToggle.checked : false;
+  }
+
+  function applyFullGraphState(forceValue) {
+    const enabled = isFullGraphMode(forceValue);
+    if (typeof forceValue === "boolean" && fullGraphToggle) {
+      fullGraphToggle.checked = forceValue;
+    }
+    if (depthInput) {
+      depthInput.disabled = enabled;
+    }
+    if (videoInput) {
+      if (enabled) {
+        videoInput.removeAttribute("required");
+      } else {
+        videoInput.setAttribute("required", "required");
+      }
+    }
+  }
+
+  async function fetchGraph(videoId, depth, maxNodes, fullGraphMode = false) {
     const params = new URLSearchParams();
-    params.set("video_id", videoId);
-    params.set("depth", String(depth));
+    if (videoId) {
+      params.set("video_id", videoId);
+    }
+    if (fullGraphMode) {
+      params.set("full_graph", "1");
+    } else {
+      params.set("depth", String(depth));
+    }
     params.set("max_nodes", String(maxNodes));
     const response = await fetch(`/api/graph?${params.toString()}`);
     if (!response.ok) {
@@ -399,24 +431,40 @@
     currentSimulation = simulation;
   }
 
-  async function loadGraph(videoId, depth, maxNodes, { updateInputs = false } = {}) {
+  async function loadGraph(
+    videoId,
+    depth,
+    maxNodes,
+    { updateInputs = false, fullGraph } = {}
+  ) {
+    const wantsFull = isFullGraphMode(
+      typeof fullGraph === "boolean" ? fullGraph : undefined
+    );
     const sanitizedId = sanitizeId(videoId);
-    if (!sanitizedId) {
+    if (!wantsFull && !sanitizedId) {
       setStatus("Please enter a video ID.", true);
       return;
     }
-    const safeDepth = sanitizeDepth(depth);
+    const safeDepth = wantsFull ? 0 : sanitizeDepth(depth);
     const safeMaxNodes = sanitizeMaxNodes(maxNodes);
 
     if (updateInputs) {
       videoInput.value = sanitizedId;
-      depthInput.value = String(safeDepth);
+      depthInput.value = String(wantsFull ? currentDepth || 1 : safeDepth);
       maxNodesInput.value = String(safeMaxNodes);
+      applyFullGraphState(wantsFull);
+    } else {
+      applyFullGraphState();
     }
 
-    setStatus("Loading graph…");
+    setStatus(wantsFull ? "Loading full reference graph…" : "Loading graph…");
     try {
-      const data = await fetchGraph(sanitizedId, safeDepth, safeMaxNodes);
+      const data = await fetchGraph(
+        sanitizedId,
+        safeDepth,
+        safeMaxNodes,
+        wantsFull
+      );
       if (!data.nodes || data.nodes.length === 0) {
         setStatus("No nodes returned for this video.", true);
         container.innerHTML = "";
@@ -428,12 +476,21 @@
       currentGraphData = data;
       currentDepth = safeDepth;
       currentMaxNodes = safeMaxNodes;
+      currentFullGraph = wantsFull;
       renderGraph(data, getLabelSize());
       renderLegend(data.nodes);
       setStatus(
-        `Showing ${data.nodes.length} nodes and ${data.links.length} links (depth ${data.depth})`
+        `Showing ${data.nodes.length} nodes and ${data.links.length} links (${
+          data.meta?.mode === "full" ? "full graph" : `depth ${data.depth}`
+        })`
+      );
+      updateUrlState(
+        sanitizedId,
+        safeDepth,
+        safeMaxNodes,
+        getLabelSize(),
+        wantsFull
       );
-      updateUrlState(sanitizedId, safeDepth, safeMaxNodes, getLabelSize());
     } catch (err) {
       console.error(err);
       setStatus(err.message || "Failed to build graph.", true);
@@ -448,6 +505,7 @@
     event.preventDefault();
     await loadGraph(videoInput.value, depthInput.value, maxNodesInput.value, {
       updateInputs: true,
+      fullGraph: isFullGraphMode(),
     });
   }
 
@@ -559,13 +617,23 @@
     }
   }
 
-  function updateUrlState(videoId, depth, maxNodes, labelSize) {
+  function updateUrlState(videoId, depth, maxNodes, labelSize, fullGraphMode) {
     if (isEmbedded) {
       return;
     }
     const next = new URL(window.location.href);
-    next.searchParams.set("video_id", videoId);
-    next.searchParams.set("depth", String(depth));
+    if (videoId) {
+      next.searchParams.set("video_id", videoId);
+    } else {
+      next.searchParams.delete("video_id");
+    }
+    if (fullGraphMode) {
+      next.searchParams.set("full_graph", "1");
+      next.searchParams.delete("depth");
+    } else {
+      next.searchParams.set("depth", String(depth));
+      next.searchParams.delete("full_graph");
+    }
     next.searchParams.set("max_nodes", String(maxNodes));
     if (labelSize && labelSize !== "normal") {
       next.searchParams.set("label_size", labelSize);
@@ -581,25 +649,40 @@
     const depth = sanitizeDepth(params.get("depth") || "");
     const maxNodes = sanitizeMaxNodes(params.get("max_nodes") || "");
     const labelSizeParam = params.get("label_size");
+    const fullGraphParam = params.get("full_graph");
+    const viewFull =
+      fullGraphParam && ["1", "true", "yes"].includes(fullGraphParam.toLowerCase());
     if (videoId) {
       videoInput.value = videoId;
     }
     depthInput.value = String(depth);
     maxNodesInput.value = String(maxNodes);
+    if (fullGraphToggle) {
+      fullGraphToggle.checked = !!viewFull;
+    }
+    applyFullGraphState();
     if (labelSizeParam && isValidLabelSize(labelSizeParam)) {
       setLabelSizeInput(labelSizeParam);
     } else {
       setLabelSizeInput(getLabelSize());
     }
-    if (!videoId || isEmbedded) {
+    if ((isEmbedded && !viewFull) || (!videoId && !viewFull)) {
       return;
     }
-    loadGraph(videoId, depth, maxNodes, { updateInputs: false });
+    loadGraph(videoId, depth, maxNodes, {
+      updateInputs: false,
+      fullGraph: viewFull,
+    });
   }
 
   resizeContainer();
   window.addEventListener("resize", resizeContainer);
   form.addEventListener("submit", handleSubmit);
+  if (fullGraphToggle) {
+    fullGraphToggle.addEventListener("change", () => {
+      applyFullGraphState();
+    });
+  }
   labelSizeInput.addEventListener("change", () => {
     const size = getLabelSize();
     if (currentGraphData) {
@@ -610,7 +693,8 @@
       sanitizeId(videoInput.value),
       currentDepth,
       currentMaxNodes,
-      size
+      size,
+      currentFullGraph
     );
   });
   initFromQuery();
@@ -619,8 +703,23 @@
     load(videoId, depth, maxNodes, options = {}) {
       const targetDepth = depth != null ? depth : currentDepth;
       const targetMax = maxNodes != null ? maxNodes : currentMaxNodes;
+      const explicitFull =
+        typeof options.fullGraph === "boolean"
+          ? options.fullGraph
+          : undefined;
+      if (fullGraphToggle && typeof explicitFull === "boolean") {
+        fullGraphToggle.checked = explicitFull;
+      }
+      applyFullGraphState(
+        typeof explicitFull === "boolean" ? explicitFull : undefined
+      );
+      const fullFlag =
+        typeof explicitFull === "boolean"
+          ? explicitFull
+          : isFullGraphMode();
       return loadGraph(videoId, targetDepth, targetMax, {
         updateInputs: options.updateInputs !== false,
+        fullGraph: fullFlag,
       });
     },
     setLabelSize(size) {
@@ -659,6 +758,7 @@
         labelSize: getLabelSize(),
         nodes: currentGraphData ? currentGraphData.nodes.slice() : [],
         links: currentGraphData ? currentGraphData.links.slice() : [],
+        fullGraph: currentFullGraph,
       };
     },
     isEmbedded,

static/style.css

@@ -196,6 +196,13 @@ body.dimmed {
   font-weight: bold;
 }
 
+.graph-controls .field-hint {
+  font-size: 10px;
+  color: #3c3c3c;
+  margin: 0;
+  max-width: 280px;
+}
+
 .graph-controls input,
 .graph-controls select {
   min-width: 160px;