Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions .changeset/milab-6382-gpu-support.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
---
"@platforma-open/milaboratories.gpu-test.workflow": minor
"@platforma-open/milaboratories.gpu-test.gpu-info": minor
"@platforma-open/milaboratories.gpu-test.model": minor
"@platforma-open/milaboratories.gpu-test.ui": minor
---

MILAB-6382: dual-mode operation via `exec.hasGpu` gate

- Workflow gates `.gpuMemory()` on `exec.hasGpu` and passes `--mode gpu` /
`--mode cpu` to the software. Calling `.gpuMemory()` on a backend without
GPU is a permanent runner error; the gate keeps the block runnable on
both GPU and CPU-only clusters.
- Software accepts the new `--mode {gpu,cpu}` flag. In `gpu` mode the
existing detection + benchmark runs; in `cpu` mode the GPU probes are
skipped and the report emits "GPU not available" fast (the block returns
successfully — it never errors when GPU is absent).
- Model: `GpuReport.mode` field added so the UI can render the active
branch.
- UI: summary banner shows "CPU branch (exec.hasGpu = false)" when running
in CPU mode.
5 changes: 5 additions & 0 deletions model/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ export const model = BlockModel.create()

export interface GpuReport {
seed: number;
// 'gpu' = workflow ran on the GPU branch (exec.hasGpu was true) and the
// software ran the full detection + benchmark.
// 'cpu' = workflow ran on the CPU branch; probes were skipped and the
// block returned a fast "no GPU available" report without erroring.
mode: 'gpu' | 'cpu';

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The torch_cuda field is populated by the Python backend (both in gpu and cpu modes) but is currently missing from the TypeScript GpuReport interface. Adding it ensures the TypeScript model fully and accurately represents the backend report schema.

Suggested change
mode: 'gpu' | 'cpu';
mode: 'gpu' | 'cpu';
torch_cuda: {
available: boolean;
device_count: number;
cuda_version: string | null;
cudnn_version: string | null;
devices: Array<{
index: number;
name: string;
total_memory_mb: number;
free_memory_mb?: number;
major: number;
minor: number;
multi_processor_count: number;
}>;
error?: string;
};

gpu_available: boolean;
cupy: {
available: boolean;
Expand Down
88 changes: 69 additions & 19 deletions software/gpu-info/src/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,14 @@ def format_report(report):
lines.append(f"GPU Detection Report (seed: {report.get('seed', 'N/A')})")
lines.append("=" * 60)
lines.append("")
mode = report.get("mode", "gpu")
lines.append(f"Mode: {mode.upper()}")
if mode == "cpu":
lines.append(
" Workflow ran on the CPU branch (exec.hasGpu = false). The block "
"skipped GPU probes and the benchmark."
)
lines.append("")

# CuPy (RAPIDS)
cupy_info = report["cupy"]
Expand Down Expand Up @@ -386,39 +394,81 @@ def format_report(report):
# Summary
gpu_available = cupy_info["available"] or torch_info["available"] or smi_info["available"]
lines.append("=" * 60)
lines.append(f"SUMMARY: GPU {'AVAILABLE' if gpu_available else 'NOT AVAILABLE'}")
if mode == "cpu":
lines.append("SUMMARY: GPU NOT AVAILABLE (CPU branch, no probes attempted)")
else:
lines.append(f"SUMMARY: GPU {'AVAILABLE' if gpu_available else 'NOT AVAILABLE'}")
lines.append("=" * 60)

return "\n".join(lines)


def empty_probe_result(error_msg):
"""Empty probe result used in --mode cpu (no detection attempted)."""
return {
"available": False,
"device_count": 0,
"devices": [],
"cuda_version": None,
"error": error_msg,
}


def main():
import argparse

parser = argparse.ArgumentParser(description="GPU detection and benchmark")
parser.add_argument(
"--mode",
choices=["gpu", "cpu"],
default="gpu",
help=(
"Execution mode (set by the workflow based on exec.hasGpu): "
"'gpu' runs CuPy/PyTorch/nvidia-smi probes and the GPU benchmark; "
"'cpu' skips probes and emits a fast 'no GPU available' report. "
"The block never errors when no GPU is present — the CPU path is "
"a first-class behaviour, not a fallback."
),
)
parser.add_argument("--seed", type=int, default=0, help="Run seed for cache busting")
parser.add_argument("--matrix-size", type=int, default=4000, help="Benchmark matrix size (NxN)")
args = parser.parse_args()

report = {"seed": args.seed}

report["cupy"] = detect_cupy()
report["torch_cuda"] = detect_torch_cuda()
report["nvidia_smi"] = detect_nvidia_smi()
report["environment"] = detect_env_vars()

report["benchmark"] = run_benchmark(
report["cupy"]["available"],
report["torch_cuda"]["available"],
report["nvidia_smi"]["available"],
args.matrix_size,
)
report = {"seed": args.seed, "mode": args.mode}

if args.mode == "cpu":
# Workflow passed --mode cpu (exec.hasGpu = false). Skip every probe
# and emit a minimal but schema-compatible report so the UI renders.
cpu_reason = "workflow ran on the CPU branch (exec.hasGpu = false)"
report["cupy"] = empty_probe_result(cpu_reason)
report["torch_cuda"] = empty_probe_result(cpu_reason)
report["nvidia_smi"] = {
"available": False,
"driver_version": None,
"devices": [],
"error": cpu_reason,
}
report["environment"] = detect_env_vars()
report["benchmark"] = {"ran": False, "skipped": cpu_reason}
report["gpu_available"] = False
else:
report["cupy"] = detect_cupy()
report["torch_cuda"] = detect_torch_cuda()
report["nvidia_smi"] = detect_nvidia_smi()
report["environment"] = detect_env_vars()

report["benchmark"] = run_benchmark(
report["cupy"]["available"],
report["torch_cuda"]["available"],
report["nvidia_smi"]["available"],
args.matrix_size,
)

report["gpu_available"] = (
report["cupy"]["available"]
or report["torch_cuda"]["available"]
or report["nvidia_smi"]["available"]
)
report["gpu_available"] = (
report["cupy"]["available"]
or report["torch_cuda"]["available"]
or report["nvidia_smi"]["available"]
)

# Write JSON output
with open("gpu-info.json", "w") as f:
Expand Down
8 changes: 8 additions & 0 deletions ui/src/pages/GpuInfoPage.vue
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ function rerun() {
<div v-else-if="gpuInfo">
<div class="summary-banner" :class="gpuInfo.gpu_available ? 'gpu-available' : 'gpu-unavailable'">
{{ gpuInfo.gpu_available ? 'GPU AVAILABLE' : 'GPU NOT AVAILABLE' }}
<span v-if="gpuInfo.mode === 'cpu'" class="mode-label">CPU branch (exec.hasGpu = false)</span>
<span class="seed-label">seed: {{ seed }}</span>
</div>

Expand Down Expand Up @@ -273,6 +274,13 @@ function rerun() {
opacity: 0.7;
}

.mode-label {
font-size: 12px;
font-weight: 400;
margin-left: 16px;
opacity: 0.7;
}

.gpu-unavailable {
background: #f8d7da;
color: #721c24;
Expand Down
16 changes: 14 additions & 2 deletions workflow/src/main.tpl.tengo
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,20 @@ wf.body(func(args) {
if args.mem != undefined && args.mem != "" {
b = b.mem(args.mem)
}
if args.gpuMemory != undefined && args.gpuMemory != "" {
b = b.gpuMemory(args.gpuMemory)

// GPU-vs-CPU branch is decided here (tengo), not in the Python runtime.
// Calling .gpuMemory() against a backend without GPU is a permanent
// runner error, so it must be gated on exec.hasGpu. The --mode flag tells
// the software which path to take: 'gpu' runs the full detection +
// benchmark; 'cpu' emits a fast "no GPU available" report and exits
// successfully (gpu-test runs on non-GPU clusters too — it never errors).
if exec.hasGpu {
if args.gpuMemory != undefined && args.gpuMemory != "" {
b = b.gpuMemory(args.gpuMemory)
}
b = b.arg("--mode").arg("gpu")
} else {
b = b.arg("--mode").arg("cpu")
}

b = b.arg("--seed").arg(string(args.seed))
Expand Down
Loading