Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions openhtf/output/servers/station_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,47 @@ def post(self):
self.finish({'status': 'reported'})


# Station-side hook for escalating a KNOWN fault whose remediation steps did not
# resolve it. Distinct from a routine fault report: the framework sets this to a
# callable (payload_dict -> None) that forwards to an internal Halter service
# raising a higher-priority alert (a PagerDuty incident). As with the report
# handler, the GUI server NEVER calls external services itself — it only invokes
# this injected handler.
_FAULT_ESCALATE_HANDLER = None


def set_fault_escalate_handler(handler):
"""Register the callable invoked with the fault-escalate payload (see above)."""
global _FAULT_ESCALATE_HANDLER
_FAULT_ESCALATE_HANDLER = handler


class FaultEscalateHandler(web_gui_server.CorsRequestHandler):
"""POST endpoint for escalating a known fault from the GUI.

Same delegating contract as FaultReportHandler, but a separate hook so the
framework can route escalations to a higher-priority alert (PagerDuty) rather
than a routine report. Returns 501 if no handler is configured. The handler
runs on a thread so it never blocks the IOLoop.
"""

def post(self):
handler = _FAULT_ESCALATE_HANDLER
if handler is None:
self.set_status(501)
self.finish({'error': 'fault escalation is not configured on this station'})
return
try:
payload = json.loads(self.request.body)
except (ValueError, TypeError):
self.set_status(400)
self.finish({'error': 'invalid JSON body'})
return
threading.Thread(target=handler, args=(payload,), daemon=True).start()
self.set_status(202)
self.finish({'status': 'escalated'})


class AttachmentsHandler(BaseTestHandler):
"""GET endpoint for a file attached to a test."""

Expand Down Expand Up @@ -834,6 +875,7 @@ def __init__(
'attachments/(?P<attachment_name>.+)', AttachmentsHandler),
(r'/commands/(?P<command>.+)', CommandHandler),
(r'/fault-report', FaultReportHandler),
(r'/fault-escalate', FaultEscalateHandler),
))

# Optionally enable history from disk.
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions openhtf/output/web_gui/dist/index.html
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<head><link href="/css/app.e9c4d879332ab5131a29.css" rel="stylesheet"></head><!doctype html>
<head><link href="/css/app.4cad1b230047e7d0f056.css" rel="stylesheet"></head><!doctype html>
<!--
Copyright 2022 Google LLC

Expand All @@ -22,4 +22,4 @@

<base href="/">
<htf-app config="{{ json_encode(config) }}">Loading...</htf-app>
<script type="text/javascript" src="/js/polyfills.e9c4d879332ab5131a29.js"></script><script type="text/javascript" src="/js/vendor.e9c4d879332ab5131a29.js"></script><script type="text/javascript" src="/js/app.e9c4d879332ab5131a29.js"></script>
<script type="text/javascript" src="/js/polyfills.4cad1b230047e7d0f056.js"></script><script type="text/javascript" src="/js/vendor.4cad1b230047e7d0f056.js"></script><script type="text/javascript" src="/js/app.4cad1b230047e7d0f056.js"></script>

Large diffs are not rendered by default.

Large diffs are not rendered by default.

This file was deleted.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
-->
<div
class="htf-layout-widget operator-fault-panel"
*ngIf="test && test.outcomeDetails && test.outcomeDetails.length">
*ngIf="showFaultPanel">
<div class="htf-layout-widget-header operator-fault-header">
<div>TEST FAILED — ACTION REQUIRED</div>
</div>
Expand All @@ -33,18 +33,57 @@
<div *ngIf="detail.whatToDo" class="operator-fault-action">
<div class="operator-fault-action-label">What to do</div>
<pre class="operator-fault-action-body">{{ detail.whatToDo }}</pre>
<!-- Escalation: the steps above didn't resolve it, so let the operator
alert Halter (a higher-priority page, distinct from a routine
report). Same spinner + good/fail feedback as the report button. -->
<div class="operator-fault-escalate">
<span class="operator-fault-escalate-hint">
Steps didn't fix it? This raises an incident with the on-call team.
</span>
<button
type="button"
class="htf-rounded-button-red operator-fault-report-button operator-fault-escalate-button"
[class.operator-fault-report-sent]="escalateStatus(detail) === ReportStatus.sent"
[class.operator-fault-report-error]="escalateStatus(detail) === ReportStatus.error"
[disabled]="!canEscalate(detail)"
(click)="escalateFault(detail)">
<span
*ngIf="escalateStatus(detail) === ReportStatus.sending"
class="operator-fault-spinner"></span>
<span [ngSwitch]="escalateStatus(detail)">
<span *ngSwitchCase="ReportStatus.sending">Reporting…</span>
<span *ngSwitchCase="ReportStatus.sent">✓ Issue reported</span>
<span *ngSwitchCase="ReportStatus.error">✕ Couldn't report — tap to retry</span>
<span *ngSwitchDefault>Report Issue</span>
</span>
</button>
</div>
</div>
<!-- No known remediation: let the operator report it to engineering. -->
<!-- No known remediation: let the operator flag the uncaught error so
engineering can add a fault code. Informational — not a production
incident, so it stays a low-key grey button (contrast the red escalate
button above). -->
<div *ngIf="!detail.whatToDo" class="operator-fault-action">
<div class="operator-fault-action-label">Unexpected error</div>
<div class="operator-fault-action-hint">
This isn't a known fault. Report it so engineering can add a fix.
This isn't a known fault. Let Halter know so engineering can add a fix.
</div>
<button
type="button"
class="htf-rounded-button-grey"
class="htf-rounded-button-grey operator-fault-report-button"
[class.operator-fault-report-sent]="reportStatus(detail) === ReportStatus.sent"
[class.operator-fault-report-error]="reportStatus(detail) === ReportStatus.error"
[disabled]="!canReport(detail)"
(click)="reportFault(detail)">
Report to engineering
<span
*ngIf="reportStatus(detail) === ReportStatus.sending"
class="operator-fault-spinner"></span>
<span [ngSwitch]="reportStatus(detail)">
<span *ngSwitchCase="ReportStatus.sending">Sending…</span>
<span *ngSwitchCase="ReportStatus.sent">✓ Sent to Halter</span>
<span *ngSwitchCase="ReportStatus.error">Couldn't send — tap to retry</span>
<span *ngSwitchDefault>Notify Halter</span>
</span>
</button>
</div>
<!-- Secondary, de-emphasised: engineering context (code + issue). -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,79 @@
margin: 4px 0 8px;
}

// Escalation block under the remediation steps: shown when the steps didn't
// fix it, so the operator can page Halter. Separated from the steps with a
// hairline so it reads as a distinct, last-resort action.
.operator-fault-escalate {
align-items: flex-start;
border-top: 1px dashed $border-light-grey;
display: flex;
flex-direction: column;
gap: 8px;
margin-top: 12px;
padding-top: 10px;
}

.operator-fault-escalate-hint {
font-size: 13px;
}

// Submit buttons: hold the label next to a spinner while in flight, then turn
// green (sent) or red (error) so the operator gets an unambiguous result and
// stops clicking. Two variants:
// - report (grey, htf-rounded-button-grey): informational — flags an uncaught
// error to engineering (Slack). Low-key.
// - escalate (red, htf-rounded-button-red + .operator-fault-escalate-button):
// raises a production incident (PagerDuty + on-call). Made bold + a light
// spinner so it reads as the serious, last-resort action.
.operator-fault-report-button {
align-items: center;
display: inline-flex;
gap: 8px;
}

.operator-fault-escalate-button {
font-weight: bold;

.operator-fault-spinner {
border-color: rgba($theme-white, .45);
border-top-color: $theme-white;
}
}

.operator-fault-report-sent {
background: $theme-green;
border-color: $theme-green;
color: $theme-white;
// Sent is a terminal, disabled state — keep it green (don't grey out).
&:disabled {
background: $theme-green;
border-color: $theme-green;
}
}

.operator-fault-report-error {
background: $theme-red;
border-color: $theme-red;
color: $theme-white;
}

.operator-fault-spinner {
animation: operator-fault-spin 700ms linear infinite;
border: 2px solid rgba($theme-dark-grey, .35);
border-radius: 50%;
border-top-color: $theme-dark-grey;
display: inline-block;
height: 12px;
width: 12px;
}

@keyframes operator-fault-spin {
to {
transform: rotate(360deg);
}
}

// Secondary — engineering context, de-emphasised below the action.
.operator-fault-issue {
border-top: 1px solid $border-light-grey;
Expand Down
Binary file not shown.
Loading