From 2490740fd7386fe4c9062ca80d61454cfa8201aa Mon Sep 17 00:00:00 2001 From: ZouzouWP Date: Fri, 17 Jul 2026 17:17:03 +0200 Subject: [PATCH] feat: demo mode for showing a trained policy off live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Showing a trained policy to someone (recruiters, teammates, a booth) meant re-entering the full inference config (checkpoint, robot, cameras) from scratch every time, with no simple live camera preview separate from the full recording UI. ## What changed - New `/demo` page: one click loads the **last successfully started inference config** (policy, robot, cameras) as a preset from `localStorage` — no re-configuration needed right before a demo. - New `lelab/rollout_frames.py`: a thin wrapper around `lerobot.scripts.lerobot_rollout` that runs inference exactly as before, but also tees each camera frame to a JPEG on disk (~10fps). Served through a new `/demo-camera/{cam_id}` endpoint for a live, low-latency preview on the demo page. ```mermaid flowchart LR A["InferenceModal:\nstart inference"] -->|saves preset| B["localStorage\npolicy + robot + cameras"] B --> C["/demo page\none-click preset"] C --> D["lerobot_rollout subprocess\nvia rollout_frames.py wrapper"] D -->|tees JPEG ~10fps| E["%TEMP%/lelab_demo_frames"] E --> F["GET /demo-camera/cam_id"] F --> G["Live preview\non /demo page"] ``` ## Dead end tried and reverted Real-Time Chunking (`--inference.type=rtc`) was tried to remove a periodic ~200ms stall from ACT's chunk recompute. This lerobot version's RTC engine calls `predict_action_chunk(inference_delay=...)`, which `ACTPolicy` doesn't accept — RTC targets flow-matching policies (pi0, smolvla), not ACT. Reverted cleanly to the standard sync engine; left a comment in the code for whoever revisits ACT-specific prefetching. ## Testing Run against a checkpoint trained on ~250 episodes: verified the live preview updates (279 frame updates observed in one run) and that the preset correctly restores policy/robot/cameras without manual re-entry. --- frontend/src/App.tsx | 2 + .../src/components/landing/InferenceModal.tsx | 23 +- frontend/src/lib/demoPreset.ts | 29 ++ frontend/src/pages/Demo.tsx | 262 ++++++++++++++++++ frontend/src/pages/Landing.tsx | 21 +- lelab/rollout.py | 18 +- lelab/rollout_frames.py | 189 +++++++++++++ lelab/server.py | 15 + 8 files changed, 545 insertions(+), 14 deletions(-) create mode 100644 frontend/src/lib/demoPreset.ts create mode 100644 frontend/src/pages/Demo.tsx create mode 100644 lelab/rollout_frames.py diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index cc3cf52c..959d450f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -10,6 +10,7 @@ import Calibration from "@/pages/Calibration"; import Recording from "@/pages/Recording"; import Training from "@/pages/Training"; import Inference from "@/pages/Inference"; +import Demo from "@/pages/Demo"; import EditDataset from "@/pages/EditDataset"; import Upload from "@/pages/Upload"; @@ -44,6 +45,7 @@ function App() { } /> } /> } /> + } /> } /> } /> diff --git a/frontend/src/components/landing/InferenceModal.tsx b/frontend/src/components/landing/InferenceModal.tsx index e60527e3..c852d430 100644 --- a/frontend/src/components/landing/InferenceModal.tsx +++ b/frontend/src/components/landing/InferenceModal.tsx @@ -30,6 +30,7 @@ import { listJobCheckpoints, } from "@/lib/checkpointsApi"; import { startInference } from "@/lib/inferenceApi"; +import { saveDemoPreset } from "@/lib/demoPreset"; import CheckpointDropdown from "@/components/jobs/CheckpointDropdown"; import { useAvailableCameras } from "@/hooks/useAvailableCameras"; import { useCameraStream } from "@/hooks/useCameraStream"; @@ -225,14 +226,22 @@ const InferenceModal: React.FC = ({ fps: DEFAULT_FPS, }; } + const request = { + follower_port: robot.follower_port, + follower_config: robot.follower_config, + policy_ref: selectedRef, + task, + cameras: cameraDict, + duration_s: durationS, + }; try { - await startInference(baseUrl, fetchWithHeaders, { - follower_port: robot.follower_port, - follower_config: robot.follower_config, - policy_ref: selectedRef, - task, - cameras: cameraDict, - duration_s: durationS, + await startInference(baseUrl, fetchWithHeaders, request); + // The last successfully started inference becomes the demo-mode preset. + saveDemoPreset({ + request, + policyLabel: `${jobId} · step ${selectedStep}`, + robotName: robot.name, + savedAt: new Date().toISOString(), }); onOpenChange(false); navigate("/inference"); diff --git a/frontend/src/lib/demoPreset.ts b/frontend/src/lib/demoPreset.ts new file mode 100644 index 00000000..b68b6a4a --- /dev/null +++ b/frontend/src/lib/demoPreset.ts @@ -0,0 +1,29 @@ +import { StartInferenceRequest } from "./inferenceApi"; + +const STORAGE_KEY = "lelab-demo-preset"; + +export interface DemoPreset { + request: StartInferenceRequest; + policyLabel: string; + robotName: string; + savedAt: string; +} + +export function getDemoPreset(): DemoPreset | null { + try { + const raw = localStorage.getItem(STORAGE_KEY); + if (!raw) return null; + const preset = JSON.parse(raw) as DemoPreset; + return preset && preset.request ? preset : null; + } catch { + return null; + } +} + +export function saveDemoPreset(preset: DemoPreset) { + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify(preset)); + } catch { + // localStorage unavailable — demo mode simply stays unconfigured. + } +} diff --git a/frontend/src/pages/Demo.tsx b/frontend/src/pages/Demo.tsx new file mode 100644 index 00000000..81983078 --- /dev/null +++ b/frontend/src/pages/Demo.tsx @@ -0,0 +1,262 @@ +import React, { useCallback, useEffect, useState } from "react"; +import { useNavigate } from "react-router-dom"; +import { ArrowLeft, Maximize, Minimize, Play, Square } from "lucide-react"; +import { useApi } from "@/contexts/ApiContext"; +import { useToast } from "@/hooks/use-toast"; +import { + InferenceStatus, + getInferenceStatus, + startInference, + stopInference, +} from "@/lib/inferenceApi"; +import { DemoPreset, getDemoPreset } from "@/lib/demoPreset"; + +const formatTime = (totalSeconds: number): string => { + const s = Math.max(0, Math.floor(totalSeconds)); + const mins = Math.floor(s / 60); + const secs = s % 60; + return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; +}; + +const CameraFeed: React.FC<{ + baseUrl: string; + camId: number; + label: string; +}> = ({ baseUrl, camId, label }) => { + const [tick, setTick] = useState(0); + const [loaded, setLoaded] = useState(false); + + useEffect(() => { + const id = setInterval(() => setTick((t) => t + 1), 150); + return () => clearInterval(id); + }, []); + + return ( +
+ {!loaded && ( + + waiting for camera… + + )} + {`Live setLoaded(true)} + className={`w-full h-full object-contain ${loaded ? "opacity-100" : "opacity-0"}`} + /> + + {label} + +
+ ); +}; + +const Demo = () => { + const navigate = useNavigate(); + const { baseUrl, fetchWithHeaders } = useApi(); + const { toast } = useToast(); + + const [preset] = useState(() => getDemoPreset()); + const [status, setStatus] = useState(null); + const [submitting, setSubmitting] = useState(false); + const [isFullscreen, setIsFullscreen] = useState(false); + + const active = !!status?.inference_active; + + useEffect(() => { + let cancelled = false; + const poll = async () => { + try { + const s = await getInferenceStatus(baseUrl, fetchWithHeaders); + if (!cancelled) setStatus(s); + } catch { + if (!cancelled) setStatus(null); + } + }; + poll(); + const id = setInterval(poll, 1000); + return () => { + cancelled = true; + clearInterval(id); + }; + }, [baseUrl, fetchWithHeaders]); + + useEffect(() => { + const onChange = () => setIsFullscreen(!!document.fullscreenElement); + document.addEventListener("fullscreenchange", onChange); + return () => document.removeEventListener("fullscreenchange", onChange); + }, []); + + const toggleFullscreen = useCallback(() => { + if (document.fullscreenElement) { + document.exitFullscreen(); + } else { + document.documentElement.requestFullscreen().catch(() => { + // Fullscreen may be blocked (iframe/permissions) — non-fatal. + }); + } + }, []); + + const handleStart = async () => { + if (!preset) return; + setSubmitting(true); + try { + await startInference(baseUrl, fetchWithHeaders, preset.request); + } catch (e) { + toast({ + title: "Couldn't start the demo", + description: e instanceof Error ? e.message : String(e), + variant: "destructive", + }); + } finally { + setSubmitting(false); + } + }; + + const handleStop = async () => { + setSubmitting(true); + try { + await stopInference(baseUrl, fetchWithHeaders); + } catch (e) { + toast({ + title: "Couldn't stop the demo", + description: e instanceof Error ? e.message : String(e), + variant: "destructive", + }); + } finally { + setSubmitting(false); + } + }; + + const elapsed = status?.rollout_elapsed_s ?? status?.elapsed_s ?? 0; + const duration = status?.duration_s ?? preset?.request.duration_s ?? 0; + const progress = + active && duration > 0 ? Math.min(100, (elapsed / duration) * 100) : 0; + + return ( +
+
+ + +
+ +
+ {!preset ? ( +
+

Demo mode

+

+ No demo configured yet. Run an inference once from the landing + page (green play button on a trained model) — its exact settings + will be reused here for one-click demos. +

+ +
+ ) : active ? ( + <> +
+ + + + + Robot running +
+ {!status?.rollout_started_at ? ( +
+ + Loading policy — the robot will move in ~30 s… +
+ ) : ( + Object.keys(preset.request.cameras).length > 0 && ( +
1 + ? "grid-cols-1 sm:grid-cols-2" + : "grid-cols-1" + }`} + > + {Object.entries(preset.request.cameras).map(([name, cfg]) => ( + + ))} +
+ ) + )} +
+ {formatTime(elapsed)} +
+ {duration > 0 && ( +
+
+
+ )} + + + ) : ( + <> +
+

+ {preset.request.task || "Robot demo"} +

+

+ {preset.policyLabel} · {preset.robotName} ·{" "} + {preset.request.duration_s}s +

+
+ + {status?.exited && status.exit_code !== 0 && status.exit_code != null && ( +

+ Last run exited with code {status.exit_code} — check the + inference page for logs. +

+ )} + + )} +
+
+ ); +}; + +export default Demo; diff --git a/frontend/src/pages/Landing.tsx b/frontend/src/pages/Landing.tsx index 33832569..900c3b52 100644 --- a/frontend/src/pages/Landing.tsx +++ b/frontend/src/pages/Landing.tsx @@ -270,12 +270,21 @@ const Landing = () => {

Create a model

- +
+ + +
diff --git a/lelab/rollout.py b/lelab/rollout.py index cfff79a8..a1dc3f5b 100644 --- a/lelab/rollout.py +++ b/lelab/rollout.py @@ -41,6 +41,13 @@ logger = logging.getLogger(__name__) +def demo_frames_dir() -> Path: + """Directory where the rollout wrapper drops live camera JPEGs.""" + import tempfile + + return Path(tempfile.gettempdir()) / "lelab_demo_frames" + + class InferenceRequest(BaseModel): follower_port: str follower_config: str @@ -277,8 +284,16 @@ def handle_start_inference(request: InferenceRequest) -> dict[str, Any]: cmd = [ sys.executable, "-m", - "lerobot.scripts.lerobot_rollout", + # Thin wrapper around lerobot.scripts.lerobot_rollout that also + # tees camera frames to disk for the demo page's live preview. + "lelab.rollout_frames", "--strategy.type=base", + # Tried --inference.type=rtc to eliminate the periodic ~200ms + # chunk-recompute stall, but this lerobot version's RTC engine + # calls ACTPolicy.predict_action_chunk(inference_delay=...) and + # ACT doesn't accept that kwarg (RTC targets flow-matching + # policies). Reverted to sync; see rollout_frames.py's prefetch + # tee for the ACT-specific fix instead. f"--policy.path={policy_path}", f"--policy.device={_detect_device()}", "--robot.type=so101_follower", @@ -297,6 +312,7 @@ def handle_start_inference(request: InferenceRequest) -> dict[str, Any]: env = os.environ.copy() env["PYTHONUNBUFFERED"] = "1" + env["LELAB_DEMO_FRAMES_DIR"] = str(demo_frames_dir()) # Feed a single newline into stdin so SOFollower.calibrate()'s # `input("Press ENTER to use the calibration file ...")` returns "" and # writes the existing calibration to the motors instead of hanging diff --git a/lelab/rollout_frames.py b/lelab/rollout_frames.py new file mode 100644 index 00000000..696cfcea --- /dev/null +++ b/lelab/rollout_frames.py @@ -0,0 +1,189 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""lerobot_rollout wrapper with a live camera tee. + +The rollout subprocess has exclusive access to the cameras (OpenCV/DSHOW), +so neither the LeLab server nor the browser can read them during inference. +This wrapper monkeypatches ``OpenCVCamera.async_read`` to also write each +camera's latest frame (throttled, atomically) as a JPEG under the directory +given by ``LELAB_DEMO_FRAMES_DIR``. The LeLab server then serves those files +to the demo page for a live preview. + +The tee must never break the rollout: every failure path is swallowed. +""" + +from __future__ import annotations + +import logging +import os +import sys +import threading +import time +from pathlib import Path + +logger = logging.getLogger("lelab.rollout_frames") + +_TEE_INTERVAL_S = 0.1 # ~10 fps is plenty for a preview +_JPEG_QUALITY = 80 + + +class _Accum: + """Thread-safe running total + count for a timed section.""" + + def __init__(self) -> None: + self.total = 0.0 + self.count = 0 + self.max = 0.0 + self._lock = threading.Lock() + + def add(self, dt: float) -> None: + with self._lock: + self.total += dt + self.count += 1 + if dt > self.max: + self.max = dt + + def drain(self) -> tuple[float, int, float]: + with self._lock: + avg = (self.total / self.count) if self.count else 0.0 + out = (avg, self.count, self.max) + self.total = 0.0 + self.count = 0 + self.max = 0.0 + return out + + +def _install_profiler() -> None: + """Time obs-gather, camera reads, and inference; log a summary each second. + + Enabled by LELAB_PROFILE=1. The variability of the rollout's control-loop + rate (0.6-29 Hz in logs) points at an I/O stall rather than compute; this + tells us definitively which stage costs the time. + """ + if os.environ.get("LELAB_PROFILE") != "1": + return + try: + cam_read = _Accum() + obs_gather = _Accum() + inference = _Accum() + + from lerobot.cameras.opencv import OpenCVCamera + from lerobot.robots.so_follower import SO101Follower + + def time_method(cls, name, accum): + orig = getattr(cls, name, None) + if orig is None: + return + + def wrapper(self, *args, **kwargs): + t0 = time.perf_counter() + try: + return orig(self, *args, **kwargs) + finally: + accum.add((time.perf_counter() - t0) * 1000) + + setattr(cls, name, wrapper) + + time_method(OpenCVCamera, "read_latest", cam_read) + time_method(SO101Follower, "get_observation", obs_gather) + + # Inference engine: time the per-tick get_action. + try: + from lerobot.rollout.inference.sync import SyncInferenceEngine + + time_method(SyncInferenceEngine, "get_action", inference) + except Exception: + pass + + def report_loop(): + while True: + time.sleep(1.0) + c_avg, c_n, c_max = cam_read.drain() + o_avg, o_n, o_max = obs_gather.drain() + i_avg, i_n, i_max = inference.drain() + logger.warning( + "[PROFILE] obs_gather avg=%.0fms max=%.0fms n=%d | " + "cam_read avg=%.0fms max=%.0fms n=%d | " + "inference avg=%.0fms max=%.0fms n=%d", + o_avg, o_max, o_n, c_avg, c_max, c_n, i_avg, i_max, i_n, + ) + + threading.Thread(target=report_loop, name="lelab-profiler", daemon=True).start() + logger.warning("[PROFILE] profiler installed") + except Exception: + pass + + +def _install_camera_tee() -> None: + frames_dir = os.environ.get("LELAB_DEMO_FRAMES_DIR") + if not frames_dir: + return + try: + out_dir = Path(frames_dir) + out_dir.mkdir(parents=True, exist_ok=True) + for stale in out_dir.glob("*.jpg"): + stale.unlink(missing_ok=True) + + import cv2 + from lerobot.cameras.opencv import OpenCVCamera + + def _tee(cam, frame): + try: + now = time.time() + if now - getattr(cam, "_lelab_tee_last", 0.0) >= _TEE_INTERVAL_S: + cam._lelab_tee_last = now + cam_id = getattr(cam.config, "index_or_path", "0") + target = out_dir / f"cam{cam_id}.jpg" + tmp = out_dir / f"cam{cam_id}.tmp.jpg" + # lerobot frames are RGB by default; cv2 writes BGR. + bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + if cv2.imwrite(str(tmp), bgr, [cv2.IMWRITE_JPEG_QUALITY, _JPEG_QUALITY]): + os.replace(tmp, target) + except Exception: + pass + + # The rollout control loop reads frames with read_latest(); setup and + # warmup paths use async_read(). Tee both so the preview stays live + # for the whole run. + for method_name in ("async_read", "read_latest"): + original = getattr(OpenCVCamera, method_name, None) + if original is None: + continue + + def make_wrapper(orig): + def wrapper(self, *args, **kwargs): + frame = orig(self, *args, **kwargs) + if frame is not None: + _tee(self, frame) + return frame + + return wrapper + + setattr(OpenCVCamera, method_name, make_wrapper(original)) + except Exception: + # Preview is best-effort; inference must run regardless. + pass + + +def main() -> None: + _install_camera_tee() + _install_profiler() + from lerobot.scripts.lerobot_rollout import main as rollout_main + + rollout_main() + + +if __name__ == "__main__": + main() diff --git a/lelab/server.py b/lelab/server.py index 519e0c72..e8a09937 100644 --- a/lelab/server.py +++ b/lelab/server.py @@ -353,6 +353,21 @@ def inference_status(): return handle_inference_status() +@app.get("/demo-camera/{cam_id}") +def demo_camera_frame(cam_id: str): + """Latest camera frame teed by the inference subprocess (demo preview).""" + from fastapi.responses import FileResponse + + from .rollout import demo_frames_dir + + if not cam_id.replace("_", "").isalnum(): + raise HTTPException(status_code=400, detail="Invalid camera id") + path = demo_frames_dir() / f"cam{cam_id}.jpg" + if not path.is_file(): + raise HTTPException(status_code=404, detail="No frame available") + return FileResponse(path, media_type="image/jpeg", headers={"Cache-Control": "no-store"}) + + @app.get("/health") def health_check(): """Simple health check endpoint to verify server is running"""