From 5b2a915edd6a3dc3b315994b0591ffdc5c816289 Mon Sep 17 00:00:00 2001
From: Dmitriy Zhuk <dmitriyzhuk@gmail.com>
Date: Tue, 23 Jun 2026 18:44:01 +0300
Subject: [PATCH] feat(agent): add stop/start to free cluster resources

Restart only redeployed the pod, so there was no way to free a slot on
a full cluster. Add a true Stop (cancel workflow + delete pod, keep the
agent as `stopped`) and a paired Start (deploy a fresh pod).

Backend:
- POST /agents/:id/stop and /agents/:id/start (Owner/Admin)
- AgentDeployService.stopAgent() with deploy-tracker guard so the dying
  pod's Failed event can't flip the row to `failed`
- reconciler ignores a stale Running event for a stopped/restarting agent
- syncStatus skips `stopped`; updateStatus can clear workflowId via null

Admin UI:
- Stop/Start in the agents table row dropdown
- Stop/Start in the agent detail header (top-right) + a "stopped" chat
  overlay with a Start button

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../agent/agent/components/agent/Provider.vue | 101 +++++++++++++++++-
 .../agent/components/agentList/Provider.vue   |  60 ++++++++++-
 admin/slices/agent/agent/stores/agent.ts      |  52 +++++++++
 .../slices/agent/agent/agent.controller.ts    |  29 +++++
 .../slices/agent/agent/data/agent.gateway.ts  |   6 +-
 .../agent/agent/domain/agent.gateway.ts       |   2 +-
 .../agent/agent/domain/agentDeploy.service.ts |  38 +++++++
 .../agent/agent/domain/agentStatus.service.ts |   7 +-
 8 files changed, 286 insertions(+), 9 deletions(-)

diff --git a/admin/slices/agent/agent/components/agent/Provider.vue b/admin/slices/agent/agent/components/agent/Provider.vue
index e964810..cb40f68 100644
--- a/admin/slices/agent/agent/components/agent/Provider.vue
+++ b/admin/slices/agent/agent/components/agent/Provider.vue
@@ -21,7 +21,7 @@ import {
   TableRow,
 } from '#theme/components/ui/table';
 import { Tabs, TabsContent, TabsList, TabsTrigger } from '#theme/components/ui/tabs';
-import { IconAlertTriangle, IconArrowLeft, IconEye, IconEyeOff, IconLoader2, IconRefresh, IconShield, IconX } from '@tabler/icons-vue';
+import { IconAlertTriangle, IconArrowLeft, IconEye, IconEyeOff, IconLoader2, IconPlayerPlay, IconPlayerStop, IconRefresh, IconShield, IconX } from '@tabler/icons-vue';
 import { FileText, X } from 'lucide-vue-next';
 import type { IPaddockScenario } from '#paddock/stores/paddockScenario';
 
@@ -236,6 +236,50 @@ function dismissRestartBanner() {
   agentStore.clearPendingRestart(props.id);
 }
 
+// ── Stop / Start ─────────────────────────────────────────────────────────
+// Stop cancels the workflow and deletes the pod to free cluster CPU/memory
+// (so another agent can start when the cluster is full); Start deploys a fresh
+// pod. Which one we show depends on whether the agent currently holds a pod.
+const RESOURCE_HOLDING: ReadonlySet<AgentStatusTypes> = new Set([
+  'running',
+  'deploying',
+  'pending',
+]);
+const canStop = computed(() =>
+  agent.value ? RESOURCE_HOLDING.has(agent.value.status) : false,
+);
+const toggling = ref(false);
+const toggleError = ref<string | null>(null);
+
+async function onToggleRunning() {
+  if (!agent.value || toggling.value) return;
+  toggling.value = true;
+  toggleError.value = null;
+  const previousStatus = agent.value.status;
+  const stopping = canStop.value;
+  // Optimistic flip so the badge reacts before the API resolves.
+  agent.value = {
+    ...agent.value,
+    status: stopping ? 'stopped' : 'deploying',
+  };
+  try {
+    if (stopping) {
+      await agentStore.stop(agent.value.id);
+    } else {
+      agentStore.markRestartInFlight(agent.value.id);
+      await agentStore.start(agent.value.id);
+    }
+    await refresh();
+  } catch (err) {
+    if (agent.value) agent.value = { ...agent.value, status: previousStatus };
+    if (!stopping) agentStore.clearRestartInFlight(props.id);
+    toggleError.value =
+      (err as Error).message || (stopping ? 'Stop failed' : 'Start failed');
+  } finally {
+    toggling.value = false;
+  }
+}
+
 // ── Live pod state from SSE ─────────────────────────────────────────────
 // Lets the user watch sub-second pod transitions (Pending → ContainerCreating
 // → Running) instead of waiting on the 5s status poll below.
@@ -261,6 +305,7 @@ const podPhaseLabel = computed(() => {
 type ChatOverlay =
   | { kind: 'starting'; title: string; detail: string }
   | { kind: 'failed'; title: string; detail: string }
+  | { kind: 'stopped'; title: string; detail: string }
   | null;
 
 const chatOverlay = computed<ChatOverlay>(() => {
@@ -277,6 +322,15 @@ const chatOverlay = computed<ChatOverlay>(() => {
 
   if (chatLive) return null;
 
+  if (s === 'stopped') {
+    return {
+      kind: 'stopped',
+      title: 'Agent stopped',
+      detail:
+        'The pod was deleted to free cluster resources. Start it to chat again.',
+    };
+  }
+
   if (s === 'failed') {
     return {
       kind: 'failed',
@@ -579,7 +633,29 @@ onBeforeUnmount(stopMetricsPolling);
             </Button>
             <Button
               variant="outline"
-              :disabled="isRestarting"
+              :disabled="toggling || isRestarting"
+              :title="canStop ? 'Cancel the workflow and delete the pod to free cluster resources' : 'Deploy a fresh pod'"
+              @click="onToggleRunning"
+            >
+              <IconLoader2
+                v-if="toggling"
+                class="size-4 animate-spin"
+              />
+              <IconPlayerStop v-else-if="canStop" class="size-4" />
+              <IconPlayerPlay v-else class="size-4" />
+              {{
+                toggling
+                  ? canStop
+                    ? 'Stopping…'
+                    : 'Starting…'
+                  : canStop
+                    ? 'Stop'
+                    : 'Start'
+              }}
+            </Button>
+            <Button
+              variant="outline"
+              :disabled="isRestarting || toggling"
               @click="onRestart"
             >
               <IconLoader2
@@ -591,10 +667,10 @@ onBeforeUnmount(stopMetricsPolling);
             </Button>
           </div>
           <p
-            v-if="restartError"
+            v-if="restartError || toggleError"
             class="text-xs text-destructive"
           >
-            {{ restartError }}
+            {{ restartError || toggleError }}
           </p>
         </div>
       </div>
@@ -659,6 +735,10 @@ onBeforeUnmount(stopMetricsPolling);
                     v-if="chatOverlay.kind === 'starting'"
                     class="size-10 animate-spin text-primary"
                   />
+                  <IconPlayerStop
+                    v-else-if="chatOverlay.kind === 'stopped'"
+                    class="size-10 text-muted-foreground"
+                  />
                   <IconAlertTriangle
                     v-else
                     class="size-10 text-destructive"
@@ -669,6 +749,19 @@ onBeforeUnmount(stopMetricsPolling);
                       {{ chatOverlay.detail }}
                     </p>
                   </div>
+                  <Button
+                    v-if="chatOverlay.kind === 'stopped'"
+                    size="sm"
+                    :disabled="toggling"
+                    @click="onToggleRunning"
+                  >
+                    <IconLoader2
+                      v-if="toggling"
+                      class="size-4 animate-spin"
+                    />
+                    <IconPlayerPlay v-else class="size-4" />
+                    {{ toggling ? 'Starting…' : 'Start agent' }}
+                  </Button>
                   <Button
                     v-if="chatOverlay.kind === 'failed'"
                     size="sm"
diff --git a/admin/slices/agent/agent/components/agentList/Provider.vue b/admin/slices/agent/agent/components/agentList/Provider.vue
index 2d9d776..b39a859 100644
--- a/admin/slices/agent/agent/components/agentList/Provider.vue
+++ b/admin/slices/agent/agent/components/agentList/Provider.vue
@@ -21,6 +21,8 @@ import {
 import {
   IconDotsVertical,
   IconLoader2,
+  IconPlayerPlay,
+  IconPlayerStop,
   IconRefresh,
   IconShield,
   IconTrash,
@@ -66,6 +68,42 @@ async function onRestart(agent: IAgentData) {
   }
 }
 
+// Statuses that consume cluster resources (a pod is or will be running) — the
+// only states where "Stop" makes sense. Everything else gets "Start".
+const RESOURCE_HOLDING: ReadonlySet<AgentStatusTypes> = new Set([
+  'running',
+  'deploying',
+  'pending',
+]);
+
+function canStop(status: AgentStatusTypes): boolean {
+  return RESOURCE_HOLDING.has(status);
+}
+
+// Per-row busy guard for stop/start so one row's spinner doesn't block others.
+const togglingIds = ref<Set<string>>(new Set());
+
+function isToggling(id: string): boolean {
+  return togglingIds.value.has(id);
+}
+
+async function onToggleRunning(agent: IAgentData) {
+  if (togglingIds.value.has(agent.id)) return;
+  togglingIds.value = new Set([...togglingIds.value, agent.id]);
+  try {
+    if (canStop(agent.status)) {
+      await agentStore.stop(agent.id);
+    } else {
+      await agentStore.start(agent.id);
+    }
+    await refresh();
+  } finally {
+    const next = new Set(togglingIds.value);
+    next.delete(agent.id);
+    togglingIds.value = next;
+  }
+}
+
 const pendingRemoval = ref<IAgentData | null>(null);
 const confirmRemoveOpen = computed({
   get: () => pendingRemoval.value !== null,
@@ -159,7 +197,7 @@ async function onRemove() {
                   </DropdownMenuTrigger>
                   <DropdownMenuContent align="end">
                     <DropdownMenuItem
-                      :disabled="isRestarting(agent.id)"
+                      :disabled="isRestarting(agent.id) || isToggling(agent.id)"
                       @select="onRestart(agent)"
                     >
                       <IconLoader2
@@ -169,6 +207,26 @@ async function onRemove() {
                       <IconRefresh v-else class="size-4" />
                       {{ isRestarting(agent.id) ? 'Restarting…' : 'Restart' }}
                     </DropdownMenuItem>
+                    <DropdownMenuItem
+                      :disabled="isToggling(agent.id) || isRestarting(agent.id)"
+                      @select="onToggleRunning(agent)"
+                    >
+                      <IconLoader2
+                        v-if="isToggling(agent.id)"
+                        class="size-4 animate-spin"
+                      />
+                      <IconPlayerStop v-else-if="canStop(agent.status)" class="size-4" />
+                      <IconPlayerPlay v-else class="size-4" />
+                      {{
+                        isToggling(agent.id)
+                          ? canStop(agent.status)
+                            ? 'Stopping…'
+                            : 'Starting…'
+                          : canStop(agent.status)
+                            ? 'Stop'
+                            : 'Start'
+                      }}
+                    </DropdownMenuItem>
                     <DropdownMenuSeparator />
                     <DropdownMenuItem
                       class="text-destructive focus:text-destructive"
diff --git a/admin/slices/agent/agent/stores/agent.ts b/admin/slices/agent/agent/stores/agent.ts
index d201c1f..bcf3d0f 100644
--- a/admin/slices/agent/agent/stores/agent.ts
+++ b/admin/slices/agent/agent/stores/agent.ts
@@ -293,6 +293,56 @@ export const useAgentStore = defineStore('agent', () => {
     }
   }
 
+  // Stop an agent — cancels its workflow and deletes its pod to free cluster
+  // resources, keeping the row so it can be started again. Raw axios call:
+  // the generated SDK hasn't been regenerated for this endpoint yet (same
+  // approach as fetchEnv/fetchMetrics). Optimistic flip to 'stopped'.
+  async function stop(id: string) {
+    const previous = agents.value.find((a) => a.id === id);
+    if (previous && previous.status !== 'stopped') {
+      agents.value = agents.value.map((a) =>
+        a.id === id ? { ...a, status: 'stopped' } : a,
+      );
+    }
+    try {
+      const res = await client.instance.post(`/agents/${id}/stop`);
+      const env = res.data as ApiEnvelope<IAgentData> | undefined;
+      const updated = env?.data;
+      if (!updated) throw new Error('Stop returned no agent data');
+      agents.value = agents.value.map((a) => (a.id === id ? updated : a));
+      return updated;
+    } catch (err) {
+      if (previous) {
+        agents.value = agents.value.map((a) => (a.id === id ? previous : a));
+      }
+      throw err;
+    }
+  }
+
+  // Start a stopped agent — deploys a fresh pod. Optimistic flip to
+  // 'deploying' so the badge reacts before the API resolves.
+  async function start(id: string) {
+    const previous = agents.value.find((a) => a.id === id);
+    if (previous && previous.status !== 'deploying') {
+      agents.value = agents.value.map((a) =>
+        a.id === id ? { ...a, status: 'deploying' } : a,
+      );
+    }
+    try {
+      const res = await client.instance.post(`/agents/${id}/start`);
+      const env = res.data as ApiEnvelope<IAgentData> | undefined;
+      const updated = env?.data;
+      if (!updated) throw new Error('Start returned no agent data');
+      agents.value = agents.value.map((a) => (a.id === id ? updated : a));
+      return updated;
+    } catch (err) {
+      if (previous) {
+        agents.value = agents.value.map((a) => (a.id === id ? previous : a));
+      }
+      throw err;
+    }
+  }
+
   async function remove(id: string, options: { wipeS3?: boolean } = {}) {
     // Use raw fetch because the OpenAPI spec doesn't yet expose `wipeS3`
     // as a typed query param. We re-attach the Bearer token from the
@@ -376,6 +426,8 @@ export const useAgentStore = defineStore('agent', () => {
     create,
     update,
     restart,
+    stop,
+    start,
     remove,
     promoteAdmin,
     demoteAdmin,
diff --git a/api/src/slices/agent/agent/agent.controller.ts b/api/src/slices/agent/agent/agent.controller.ts
index 9e56f88..ece73f8 100644
--- a/api/src/slices/agent/agent/agent.controller.ts
+++ b/api/src/slices/agent/agent/agent.controller.ts
@@ -89,6 +89,9 @@ export class AgentController {
   private async syncStatus(agentId: string) {
     const agent = await this.agentGateway.findById(agentId);
     if (!agent?.workflowId) return agent;
+    // A stopped agent intentionally has no live pod — never resurrect its
+    // status from the (now cancelled) workflow phase.
+    if (agent.status === 'stopped') return agent;
     // Forward-only: only flip to 'failed' (terminal) here. We never demote
     // 'running' back to 'deploying' on a Pending workflow phase — pod-event
     // reconciler owns the running/deploying transitions and a workflow can
@@ -404,6 +407,32 @@ export class AgentController {
     return this.agentGateway.findById(id);
   }
 
+  @Post(':id/stop')
+  @Roles(UserRoleTypes.Owner, UserRoleTypes.Admin)
+  @ApiOperation({
+    summary:
+      'Stop an agent without deleting it: cancels its workflow and deletes its pod to free cluster CPU/memory, then marks it `stopped`. Use this to free a slot so another agent can start. Bring it back with POST :id/start. Admin or Owner.',
+  })
+  async stop(@Param('id') id: string) {
+    const agent = await this.agentGateway.findById(id);
+    if (!agent) throw new NotFoundException('Agent not found');
+    await this.agentDeployService.stopAgent(id);
+    return this.agentGateway.findById(id);
+  }
+
+  @Post(':id/start')
+  @Roles(UserRoleTypes.Owner, UserRoleTypes.Admin)
+  @ApiOperation({
+    summary:
+      'Start a stopped agent: deploys a fresh pod and reattaches the runtime. Inverse of POST :id/stop. Admin or Owner.',
+  })
+  async start(@Param('id') id: string) {
+    const agent = await this.agentGateway.findById(id);
+    if (!agent) throw new NotFoundException('Agent not found');
+    await this.deploy(id);
+    return this.agentGateway.findById(id);
+  }
+
   @Post('restart-by-template/:templateId')
   @Roles(UserRoleTypes.Owner, UserRoleTypes.Admin)
   @ApiOperation({
diff --git a/api/src/slices/agent/agent/data/agent.gateway.ts b/api/src/slices/agent/agent/data/agent.gateway.ts
index 663a664..d4ada3d 100644
--- a/api/src/slices/agent/agent/data/agent.gateway.ts
+++ b/api/src/slices/agent/agent/data/agent.gateway.ts
@@ -90,13 +90,15 @@ export class AgentGateway extends IAgentGateway {
   async updateStatus(
     id: string,
     status: AgentStatusTypes,
-    workflowId?: string,
+    workflowId?: string | null,
   ): Promise<IAgentData> {
     const record = await this.prisma.agent.update({
       where: { id },
       data: {
         status,
-        ...(workflowId && { workflowId }),
+        // `undefined` leaves the column untouched; `null` clears it (used when
+        // stopping an agent so the now-cancelled workflow id isn't kept around).
+        ...(workflowId !== undefined && { workflowId }),
       },
     });
     return this.mapper.toEntity(record);
diff --git a/api/src/slices/agent/agent/domain/agent.gateway.ts b/api/src/slices/agent/agent/domain/agent.gateway.ts
index 57803c0..4ab226a 100644
--- a/api/src/slices/agent/agent/domain/agent.gateway.ts
+++ b/api/src/slices/agent/agent/domain/agent.gateway.ts
@@ -16,7 +16,7 @@ export abstract class IAgentGateway {
   abstract updateStatus(
     id: string,
     status: AgentStatusTypes,
-    workflowId?: string,
+    workflowId?: string | null,
   ): Promise<IAgentData>;
   abstract setWorkflowId(id: string, workflowId: string): Promise<IAgentData>;
   abstract setAdmin(id: string, enabled: boolean): Promise<IAgentData>;
diff --git a/api/src/slices/agent/agent/domain/agentDeploy.service.ts b/api/src/slices/agent/agent/domain/agentDeploy.service.ts
index 1e5253b..090a087 100644
--- a/api/src/slices/agent/agent/domain/agentDeploy.service.ts
+++ b/api/src/slices/agent/agent/domain/agentDeploy.service.ts
@@ -5,6 +5,7 @@ import { ITemplateGateway } from '#/agent/template/domain';
 import { WorkflowService } from '#/workflow/domain/workflow.service';
 import { AuthService } from '#/user/auth/domain';
 import { ISkillGateway } from '#/skill/domain';
+import { IPodGateway } from '#/agent/pod/domain';
 import { DeployTracker } from './deployTracker';
 
 @Injectable()
@@ -18,6 +19,7 @@ export class AgentDeployService {
     private readonly authService: AuthService,
     private readonly fileGateway: IFileGateway,
     private readonly skillGateway: ISkillGateway,
+    private readonly podGateway: IPodGateway,
     private readonly deployTracker: DeployTracker,
   ) {}
 
@@ -62,6 +64,42 @@ export class AgentDeployService {
     await this.deploy(agentId);
   }
 
+  // Stop a running agent: cancel its workflow and delete the pod so the
+  // cluster resources (CPU/memory) are freed, then mark it 'stopped'. The DB
+  // row is kept so the agent can be brought back later via deploy(). Unlike
+  // restartAgent this does NOT redeploy — that's the whole point: free a slot
+  // on the cluster so another agent can start.
+  async stopAgent(agentId: string): Promise<void> {
+    const agent = await this.agentGateway.findById(agentId);
+    if (!agent) return;
+
+    // Mark BEFORE cancel so the dying pod's MODIFIED phase=Failed event
+    // (emitted by Argo's cancellation) is recognised as stale by the
+    // reconciler and doesn't flip the row from 'stopped' to 'failed'.
+    this.deployTracker.mark(agentId);
+
+    // Set 'stopped' (and clear the workflow id) up front so the UI reflects
+    // the intent immediately and the controller's syncStatus stops polling the
+    // about-to-be-cancelled workflow.
+    await this.agentGateway.updateStatus(agentId, 'stopped', null);
+
+    try {
+      await this.workflowService.cancelAgentWorkflow(agent.workflowId);
+    } catch (err) {
+      this.logger.warn(
+        `Cancel workflow failed for agent ${agentId}: ${(err as Error).message}`,
+      );
+    }
+
+    try {
+      await this.podGateway.delete(agentId);
+    } catch (err) {
+      this.logger.warn(
+        `Pod cleanup failed for agent ${agentId}: ${(err as Error).message}`,
+      );
+    }
+  }
+
   async deploy(agentId: string): Promise<void> {
     const agent = await this.agentGateway.findById(agentId);
     if (!agent) return;
diff --git a/api/src/slices/agent/agent/domain/agentStatus.service.ts b/api/src/slices/agent/agent/domain/agentStatus.service.ts
index daa20b1..f088129 100644
--- a/api/src/slices/agent/agent/domain/agentStatus.service.ts
+++ b/api/src/slices/agent/agent/domain/agentStatus.service.ts
@@ -367,7 +367,12 @@ export class AgentStatusService implements OnModuleInit, OnModuleDestroy {
     if (
       podStatus.phase === 'Running' &&
       podStatus.ready &&
-      agent.status !== 'running'
+      agent.status !== 'running' &&
+      // Ignore a Running+Ready event from a pod that predates the current
+      // deploy/stop. Without this, the OLD pod's last Running event (still in
+      // flight while Argo tears it down) could revert a freshly 'stopped' agent
+      // back to 'running', or mask a restart's new pod with the old one.
+      !this.deployTracker.isStale(agent.id, podStatus.startedAt)
     ) {
       this.logger.log(
         `Reconciling agent ${agent.id}: pod ${podStatus.podName} is Running+Ready — marking running`,