add option to capture cliCommand output

checks/cli.go

@@ -5,6 +5,7 @@ import (
 	"maps"
 	"os"
 	"os/exec"
+	"regexp"
 	"runtime"
 	"strings"
 
@@ -35,16 +36,44 @@ func runCLICommand(command api.CLIStepCLICommand, variables map[string]string) (
 	if command.StdoutFilterTmdl != nil {
 		result.Stdout = ExtractTmdlBlock(result.Stdout, *command.StdoutFilterTmdl)
 	}
+	if err := parseStdoutVariables(result.Stdout, command.StdoutVariables, variables); err != nil {
+		result.Err = err.Error()
+	}
 	result.Variables = maps.Clone(variables)
 	return result
 }
 
+func parseStdoutVariables(stdout string, vardefs []api.CLICommandStdoutVariable, variables map[string]string) error {
+	for _, vardef := range vardefs {
+		if vardef.Name == "" {
+			return fmt.Errorf("invalid stdout variable configuration")
+		}
+		if vardef.Regex == "" {
+			return fmt.Errorf("invalid stdout variable configuration")
+		}
+		re, err := regexp.Compile(vardef.Regex)
+		if err != nil {
+			return fmt.Errorf("invalid stdout variable configuration")
+		}
+		if re.NumSubexp() != 1 {
+			return fmt.Errorf("invalid stdout variable configuration")
+		}
+
+		matches := re.FindStringSubmatch(stdout)
+		if len(matches) == 2 {
+			variables[vardef.Name] = matches[1]
+		}
+	}
+
+	return nil
+}
+
 func prettyPrintCLICommand(test api.CLICommandTest, variables map[string]string) string {
 	if test.ExitCode != nil {
 		return fmt.Sprintf("Expect exit code %d", *test.ExitCode)
 	}
-	if test.StdoutLinesGt != nil {
-		return fmt.Sprintf("Expect > %d lines on stdout", *test.StdoutLinesGt)
+	if test.StdoutLinesGT != nil {
+		return fmt.Sprintf("Expect > %d lines on stdout", *test.StdoutLinesGT)
 	}
 	if test.StdoutContainsAll != nil {
 		var str strings.Builder

checks/cli_test.go

@@ -0,0 +1,103 @@
+package checks
+
+import (
+	"runtime"
+	"testing"
+
+	api "github.com/bootdotdev/bootdev/client"
+)
+
+func TestRunCLICommandCapturesStdoutVariables(t *testing.T) {
+	variables := map[string]string{}
+	result := runCLICommand(api.CLIStepCLICommand{
+		Command: `go env GOOS`,
+		StdoutVariables: []api.CLICommandStdoutVariable{{
+			Name:  "goos",
+			Regex: `([a-z0-9]+)`,
+		}},
+	}, variables)
+
+	if result.Err != "" {
+		t.Fatalf("unexpected command error: %s", result.Err)
+	}
+	if result.Variables["goos"] != runtime.GOOS {
+		t.Fatalf("captured goos = %q, want %q", result.Variables["goos"], runtime.GOOS)
+	}
+	if variables["goos"] != runtime.GOOS {
+		t.Fatalf("shared goos = %q, want %q", variables["goos"], runtime.GOOS)
+	}
+}
+
+func TestRunCLICommandInterpolatesCapturedStdoutVariables(t *testing.T) {
+	variables := map[string]string{}
+
+	first := runCLICommand(api.CLIStepCLICommand{
+		Command: `go env -json GOOS`,
+		StdoutVariables: []api.CLICommandStdoutVariable{{
+			Name:  "goenv",
+			Regex: `"([A-Z]+)"`,
+		}},
+	}, variables)
+	if first.Err != "" {
+		t.Fatalf("unexpected first command error: %s", first.Err)
+	}
+
+	second := runCLICommand(api.CLIStepCLICommand{
+		Command: `go env ${goenv}`,
+	}, variables)
+	if second.Stdout != runtime.GOOS {
+		t.Fatalf("second stdout = %q, want %q", second.Stdout, runtime.GOOS)
+	}
+}
+
+func TestParseStdoutVariablesRequiresOneCaptureGroup(t *testing.T) {
+	variables := map[string]string{}
+	err := parseStdoutVariables("token=abc123", []api.CLICommandStdoutVariable{{
+		Name:  "token",
+		Regex: `token=([a-z]+)([0-9]+)`,
+	}}, variables)
+
+	if err == nil {
+		t.Fatal("expected parse error")
+	}
+	if err.Error() != "invalid stdout variable configuration" {
+		t.Fatalf("error = %q, want invalid stdout variable configuration", err.Error())
+	}
+}
+
+func TestParseStdoutVariablesUsesGenericConfigurationError(t *testing.T) {
+	tests := []struct {
+		name   string
+		vardef api.CLICommandStdoutVariable
+	}{
+		{
+			name:   "missing name",
+			vardef: api.CLICommandStdoutVariable{Regex: `token=([a-z0-9]+)`},
+		},
+		{
+			name:   "missing regex",
+			vardef: api.CLICommandStdoutVariable{Name: "token"},
+		},
+		{
+			name:   "invalid regex",
+			vardef: api.CLICommandStdoutVariable{Name: "token", Regex: `token=([a-z0-9]+`},
+		},
+		{
+			name:   "too many capture groups",
+			vardef: api.CLICommandStdoutVariable{Name: "token", Regex: `token=([a-z]+)([0-9]+)`},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			variables := map[string]string{}
+			err := parseStdoutVariables("token=abc123", []api.CLICommandStdoutVariable{tt.vardef}, variables)
+			if err == nil {
+				t.Fatal("expected parse error")
+			}
+			if err.Error() != "invalid stdout variable configuration" {
+				t.Fatalf("error = %q, want invalid stdout variable configuration", err.Error())
+			}
+		})
+	}
+}

checks/local.go

@@ -61,6 +61,10 @@ func EvaluateCLIResults(cliData api.CLIData, results []api.CLIStepResult) *api.S
 }
 
 func evaluateCLICommandTests(stepIndex int, cmd api.CLIStepCLICommand, result api.CLICommandResult) *api.StructuredErrCLI {
+	if result.Err != "" {
+		return localFailure(stepIndex, 0, result.Err)
+	}
+
 	for testIndex, test := range cmd.Tests {
 		var err error
 
@@ -85,10 +89,10 @@ func evaluateCLICommandTests(stepIndex int, cmd api.CLIStepCLICommand, result ap
 					break
 				}
 			}
-		case test.StdoutLinesGt != nil:
+		case test.StdoutLinesGT != nil:
 			lineCount := stdoutLineCount(result.Stdout)
-			if lineCount <= *test.StdoutLinesGt {
-				err = fmt.Errorf("expected stdout to have more than %d lines, got %d", *test.StdoutLinesGt, lineCount)
+			if lineCount <= *test.StdoutLinesGT {
+				err = fmt.Errorf("expected stdout to have more than %d lines, got %d", *test.StdoutLinesGT, lineCount)
 			}
 		case test.StdoutJq != nil:
 			err = evaluateStdoutJq(result.Stdout, *test.StdoutJq, result.Variables)

checks/local_test.go

@@ -73,6 +73,31 @@ func TestLocalSubmissionEventReportsFirstFailure(t *testing.T) {
 	}
 }
 
+func TestEvaluateCLICommandReportsStdoutVariableParseError(t *testing.T) {
+	cliData := api.CLIData{Steps: []api.CLIStep{
+		{CLICommand: &api.CLIStepCLICommand{Tests: []api.CLICommandTest{
+			{ExitCode: intPtr(0)},
+		}}},
+	}}
+	results := []api.CLIStepResult{
+		{CLICommandResult: &api.CLICommandResult{
+			ExitCode: 0,
+			Err:      "invalid stdout variable configuration",
+		}},
+	}
+
+	event := LocalSubmissionEvent(cliData, results)
+	if event.ResultSlug != api.VerificationResultSlugFailure {
+		t.Fatalf("ResultSlug = %q, want failure", event.ResultSlug)
+	}
+	if event.StructuredErrCLI == nil {
+		t.Fatal("expected structured failure")
+	}
+	if event.StructuredErrCLI.ErrorMessage != "invalid stdout variable configuration" {
+		t.Fatalf("ErrorMessage = %q, want stdout variable error", event.StructuredErrCLI.ErrorMessage)
+	}
+}
+
 func TestEvaluateStdoutJq(t *testing.T) {
 	err := evaluateStdoutJq(
 		"{\"ok\":true}",

client/lessons.go

@@ -34,17 +34,23 @@ type CLIStep struct {
 }
 
 type CLIStepCLICommand struct {
-	Command          string           `yaml:"command"`
-	Tests            []CLICommandTest `yaml:"tests"`
-	SleepAfterMs     *int             `yaml:"sleepAfterMs"`
-	StdoutFilterTmdl *string          `yaml:"stdoutFilterTmdl"`
+	Command          string                     `yaml:"command"`
+	Tests            []CLICommandTest           `yaml:"tests"`
+	StdoutVariables  []CLICommandStdoutVariable `yaml:"stdoutVariables"`
+	SleepAfterMs     *int                       `yaml:"sleepAfterMs"`
+	StdoutFilterTmdl *string                    `yaml:"stdoutFilterTmdl"`
+}
+
+type CLICommandStdoutVariable struct {
+	Name  string `yaml:"name"`
+	Regex string `yaml:"regex"`
 }
 
 type CLICommandTest struct {
 	ExitCode           *int          `yaml:"exitCode"`
 	StdoutContainsAll  []string      `yaml:"stdoutContainsAll"`
 	StdoutContainsNone []string      `yaml:"stdoutContainsNone"`
-	StdoutLinesGt      *int          `yaml:"stdoutLinesGt"`
+	StdoutLinesGT      *int          `yaml:"stdoutLinesGT"`
 	StdoutJq           *StdoutJqTest `yaml:"stdoutJq"`
 }
 
@@ -174,6 +180,7 @@ type CLIStepResult struct {
 
 type CLICommandResult struct {
 	ExitCode     int
+	Err          string            `json:"-"`
 	FinalCommand string            `json:"-"`
 	Command      CLIStepCLICommand `json:"-"`
 	Stdout       string