diff --git a/frontend/src/pages/Recording.tsx b/frontend/src/pages/Recording.tsx index 840da851..9701db8f 100644 --- a/frontend/src/pages/Recording.tsx +++ b/frontend/src/pages/Recording.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useCallback, useRef } from "react"; +import React, { useState, useEffect, useCallback, useRef, useMemo } from "react"; import { useNavigate, useLocation } from "react-router-dom"; import { Button } from "@/components/ui/button"; import { @@ -67,6 +67,7 @@ interface BackendStatus { session_elapsed_seconds?: number; session_ended?: boolean; dataset_repo_id?: string; + cameras?: string[]; // Names of the cameras configured for this session available_controls: { stop_recording: boolean; exit_early: boolean; @@ -99,6 +100,60 @@ const Recording = () => { // Guards against React StrictMode double-invocation of the start effect. const startInitiatedRef = useRef(false); + // --- Camera preview layout ------------------------------------------------- + // Aspect ratio (width / height) of the configured cameras, used to size the + // preview windows without letterboxing. Falls back to 4:3 if unknown. + const cameraAspect = useMemo(() => { + const cams = (recordingConfig as unknown as { cameras?: unknown })?.cameras; + const list = Array.isArray(cams) + ? cams + : cams && typeof cams === "object" + ? Object.values(cams as Record) + : []; + const first = list[0] as { width?: number; height?: number } | undefined; + if (first?.width && first?.height) return first.width / first.height; + return 4 / 3; + }, [recordingConfig]); + + // Measure the space left for the camera windows (via ResizeObserver, so it + // re-fits on any viewport change) to size them as large as possible while + // keeping the whole page within one screen (no scroll). + const [cameraArea, setCameraArea] = useState({ w: 0, h: 0 }); + const cameraAreaObserver = useRef(null); + const cameraAreaRef = useCallback((node: HTMLDivElement | null) => { + cameraAreaObserver.current?.disconnect(); + cameraAreaObserver.current = null; + if (node) { + const ro = new ResizeObserver((entries) => { + const r = entries[0].contentRect; + setCameraArea({ w: r.width, h: r.height }); + }); + ro.observe(node); + cameraAreaObserver.current = ro; + } + }, []); + + // Pick the column count and per-window pixel size that maximizes the video + // area within the measured space, given the camera count and aspect ratio. + const cameraCount = backendStatus?.cameras?.length ?? 0; + const cameraWindow = useMemo(() => { + const { w, h } = cameraArea; + if (!cameraCount || w <= 0 || h <= 0) return { width: 0, height: 0 }; + const gap = 12; // matches the grid's gap-3 + let best = { width: 0, height: 0, area: -1 }; + for (let cols = 1; cols <= cameraCount; cols++) { + const rows = Math.ceil(cameraCount / cols); + const cellW = (w - gap * (cols - 1)) / cols; + const cellH = (h - gap * (rows - 1)) / rows; + if (cellW <= 0 || cellH <= 0) continue; + const width = Math.min(cellW, cellH * cameraAspect); + const height = width / cameraAspect; + const area = width * height; + if (area > best.area) best = { width, height, area }; + } + return { width: Math.floor(best.width), height: Math.floor(best.height) }; + }, [cameraArea, cameraCount, cameraAspect]); + const toggleMute = useCallback(() => { setMutedState((prev) => { const next = !prev; @@ -457,9 +512,9 @@ const Recording = () => { const PrimaryIcon = currentPhase === "recording" ? SkipForward : Play; return ( -
-
-
+
+
+
-
-
+
+
Episode {currentEpisode} / {totalEpisodes} @@ -523,7 +578,7 @@ const Recording = () => {
-
+
{
-
+ {/* Camera windows for the cameras configured for this session. Shown + from the preparing phase so the slots are laid out immediately; + each fills with its live feed once the robot is connected and + recording/resetting. This is the flexible region: it absorbs the + space left after the fixed chrome, and cameraWindow sizes each feed + as large as fits so the whole page stays within one screen. */} + {backendStatus.cameras && + backendStatus.cameras.length > 0 && + currentPhase !== "completed" && ( +
+ {backendStatus.cameras.map((name) => ( + + ))} +
+ )} + +
{formatTime(phaseElapsedTime)}
@@ -543,7 +627,7 @@ const Recording = () => {
-
+
{ optimisticPhase !== null || currentPhase === "completed" } - className={`w-full text-white font-semibold py-6 text-lg disabled:opacity-50 ${phaseColor.button}`} + className={`w-full flex-shrink-0 text-white font-semibold py-6 text-lg disabled:opacity-50 ${phaseColor.button}`} > {primaryLabel} @@ -601,4 +685,76 @@ const Recording = () => { ); }; +interface CameraFeedProps { + baseUrl: string; + name: string; + // False during the preparing phase: show an empty slot until the camera is + // connected and streaming. True once recording/resetting, when frames flow. + live: boolean; + // Pixel size computed by the parent to fit the available space. The box is + // sized to the camera's aspect ratio, so the video fills it without letterbox. + width: number; + height: number; +} + +// Renders one recording camera's window at an explicit size. During preparing it +// shows an empty placeholder; once live it plays the backend MJPEG stream. The +// browser renders a `multipart/x-mixed-replace` response natively in an , +// so we just point it at /camera-feed/{name}. If the stream errors before frames +// flow (camera still warming up), retry with a cache-busting key after a delay. +const CameraFeed: React.FC = ({ + baseUrl, + name, + live, + width, + height, +}) => { + const [reloadKey, setReloadKey] = useState(0); + const [hasError, setHasError] = useState(false); + const retryRef = useRef(null); + + const src = `${baseUrl}/camera-feed/${encodeURIComponent(name)}?k=${reloadKey}`; + + useEffect(() => { + return () => { + if (retryRef.current) window.clearTimeout(retryRef.current); + }; + }, []); + + const handleError = useCallback(() => { + setHasError(true); + if (retryRef.current) window.clearTimeout(retryRef.current); + retryRef.current = window.setTimeout(() => { + setHasError(false); + setReloadKey((k) => k + 1); + }, 1500); + }, []); + + // 0 before the first measurement; skip rendering a zero-size box. + if (width <= 0 || height <= 0) return null; + + return ( +
+ {!live ? ( + Getting ready… + ) : hasError ? ( + Connecting feed… + ) : ( + {`${name} + )} + + {name} + +
+ ); +}; + export default Recording; diff --git a/lelab/record.py b/lelab/record.py index 3a086293..f6ecc687 100644 --- a/lelab/record.py +++ b/lelab/record.py @@ -37,6 +37,10 @@ # Global variables for recording state recording_active = False recording_thread: threading.Thread | None = None +# Live SO101Follower for the active session, set once the robot connects so the +# /camera-feed endpoint can peek its cameras' latest frames. Reset to None on +# teardown. Always gated behind `recording_active` by readers. +current_robot = None recording_events = None # Events dict for controlling recording session recording_config = None # Store recording configuration recording_start_time = None # Track when recording started @@ -437,6 +441,10 @@ def handle_recording_status() -> dict[str, Any]: status["current_episode"] = current_episode status["total_episodes"] = recording_config.num_episodes status["saved_episodes"] = saved_episodes # Track completed episodes + # Names of the cameras the user configured for this session. The frontend + # renders a live /camera-feed/{name} preview for exactly these — nothing + # else — so only configured cameras are ever shown. + status["cameras"] = list(recording_config.cameras.keys()) # Add session start time if available if recording_start_time: @@ -459,6 +467,50 @@ def handle_recording_status() -> dict[str, Any]: return status +def camera_feed_frames(cam_key: str, fps: float = 15.0): + """Yield multipart-MJPEG chunks of one recording camera's latest frames. + + Reads via the camera's non-blocking `read_latest()` peek, so it shares the + record loop's lock-protected frame buffer with zero device contention (never + `async_read()`, which would consume the new-frame event and could disturb + record timing). Streams only while a session is live and the named camera + exists; ends cleanly otherwise so the browser stops. Capped well below + the record fps so it stays a passive observer. + """ + import cv2 + + interval = 1.0 / fps + # The session sets `current_robot` only after the robot connects (which is + # preceded by a ~2s camera-release wait), so give it a moment to appear. + deadline = time.time() + 15.0 + while recording_active and current_robot is None and time.time() < deadline: + time.sleep(0.1) + + while recording_active and current_robot is not None: + cam = current_robot.cameras.get(cam_key) + if cam is None: + # Unknown/removed camera — nothing to stream. + break + try: + # read_latest() returns an RGB frame (OpenCVCamera default color + # mode); may raise if the camera hasn't produced a fresh frame yet. + frame = cam.read_latest() + except (TimeoutError, RuntimeError): + time.sleep(interval) + continue + except Exception as e: + logger.warning("camera-feed %s: unexpected read error: %s", cam_key, e) + break + + bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + ok, buf = cv2.imencode(".jpg", bgr) + if not ok: + time.sleep(interval) + continue + yield b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" + buf.tobytes() + b"\r\n" + time.sleep(interval) + + def handle_get_dataset_info(request: DatasetInfoRequest) -> dict[str, Any]: """Return dataset metadata — from the most recent session if it matches, otherwise by loading the local LeRobot cache copy.""" @@ -583,7 +635,7 @@ def record_with_web_events(cfg: RecordConfig, web_events: dict) -> LeRobotDatase from lerobot.utils.feature_utils import hw_to_dataset_features from lerobot.utils.utils import log_say - global current_phase, phase_start_time, current_episode, saved_episodes + global current_phase, phase_start_time, current_episode, saved_episodes, current_robot robot = make_robot_from_config(cfg.robot) teleop = make_teleoperator_from_config(cfg.teleop) if cfg.teleop is not None else None @@ -635,6 +687,9 @@ def record_with_web_events(cfg: RecordConfig, web_events: dict) -> LeRobotDatase logger.info("🔧 ROBOT CONNECTION: Attempting to connect robot...") robot.connect() logger.info("✅ ROBOT CONNECTION: Robot connected successfully") + # Expose the connected robot so /camera-feed can stream its cameras' + # latest frames during the session (cleared in the finally below). + current_robot = robot except Exception as e: logger.error(f"❌ ROBOT CONNECTION: Failed to connect robot: {e}") # If robot connection fails due to camera conflict, provide clear error @@ -862,6 +917,9 @@ def record_with_web_events(cfg: RecordConfig, web_events: dict) -> LeRobotDatase log_say("Stop recording", cfg.play_sounds, blocking=True) finally: + # Stop the camera-feed endpoint from reading before tearing the cameras + # down, so it can't peek a half-disconnected device. + current_robot = None robot.disconnect() if teleop: teleop.disconnect() diff --git a/lelab/server.py b/lelab/server.py index 519e0c72..e7fbc792 100644 --- a/lelab/server.py +++ b/lelab/server.py @@ -28,7 +28,7 @@ from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel from starlette.datastructures import Headers @@ -36,7 +36,8 @@ from starlette.responses import Response from starlette.types import Scope -from . import datasets as dataset_browser +# Import our custom recording functionality +from . import datasets as dataset_browser, record as _record # Import our custom calibration functionality from .calibrate import CalibrationRequest, calibration_manager @@ -47,8 +48,6 @@ JobTarget, job_registry, ) - -# Import our custom recording functionality from .record import ( DatasetInfoRequest, RecordingRequest, @@ -443,6 +442,22 @@ def recording_status(): return handle_recording_status() +@app.get("/camera-feed/{cam_key}") +def camera_feed(cam_key: str): + """Live MJPEG preview of one configured camera during an active recording. + + Browsers render `multipart/x-mixed-replace` directly in an , so the + frontend just points an at this URL. Only valid while a session is + active; the generator ends itself when recording stops. + """ + if not _record.recording_active: + raise HTTPException(status_code=409, detail="No active recording session") + return StreamingResponse( + _record.camera_feed_frames(cam_key), + media_type="multipart/x-mixed-replace; boundary=frame", + ) + + @app.post("/recording-exit-early") def recording_exit_early(): """Skip to next episode (replaces right arrow key)"""