diff --git a/checks/cli.go b/checks/cli.go index 4410a0d..8e09b24 100644 --- a/checks/cli.go +++ b/checks/cli.go @@ -5,6 +5,7 @@ import ( "maps" "os" "os/exec" + "regexp" "runtime" "strings" @@ -35,16 +36,44 @@ func runCLICommand(command api.CLIStepCLICommand, variables map[string]string) ( if command.StdoutFilterTmdl != nil { result.Stdout = ExtractTmdlBlock(result.Stdout, *command.StdoutFilterTmdl) } + if err := parseStdoutVariables(result.Stdout, command.StdoutVariables, variables); err != nil { + result.Err = err.Error() + } result.Variables = maps.Clone(variables) return result } +func parseStdoutVariables(stdout string, vardefs []api.CLICommandStdoutVariable, variables map[string]string) error { + for _, vardef := range vardefs { + if vardef.Name == "" { + return fmt.Errorf("invalid stdout variable configuration") + } + if vardef.Regex == "" { + return fmt.Errorf("invalid stdout variable configuration") + } + re, err := regexp.Compile(vardef.Regex) + if err != nil { + return fmt.Errorf("invalid stdout variable configuration") + } + if re.NumSubexp() != 1 { + return fmt.Errorf("invalid stdout variable configuration") + } + + matches := re.FindStringSubmatch(stdout) + if len(matches) == 2 { + variables[vardef.Name] = matches[1] + } + } + + return nil +} + func prettyPrintCLICommand(test api.CLICommandTest, variables map[string]string) string { if test.ExitCode != nil { return fmt.Sprintf("Expect exit code %d", *test.ExitCode) } - if test.StdoutLinesGt != nil { - return fmt.Sprintf("Expect > %d lines on stdout", *test.StdoutLinesGt) + if test.StdoutLinesGT != nil { + return fmt.Sprintf("Expect > %d lines on stdout", *test.StdoutLinesGT) } if test.StdoutContainsAll != nil { var str strings.Builder diff --git a/checks/cli_test.go b/checks/cli_test.go new file mode 100644 index 0000000..f094f98 --- /dev/null +++ b/checks/cli_test.go @@ -0,0 +1,103 @@ +package checks + +import ( + "runtime" + "testing" + + api "github.com/bootdotdev/bootdev/client" +) + +func TestRunCLICommandCapturesStdoutVariables(t *testing.T) { + variables := map[string]string{} + result := runCLICommand(api.CLIStepCLICommand{ + Command: `go env GOOS`, + StdoutVariables: []api.CLICommandStdoutVariable{{ + Name: "goos", + Regex: `([a-z0-9]+)`, + }}, + }, variables) + + if result.Err != "" { + t.Fatalf("unexpected command error: %s", result.Err) + } + if result.Variables["goos"] != runtime.GOOS { + t.Fatalf("captured goos = %q, want %q", result.Variables["goos"], runtime.GOOS) + } + if variables["goos"] != runtime.GOOS { + t.Fatalf("shared goos = %q, want %q", variables["goos"], runtime.GOOS) + } +} + +func TestRunCLICommandInterpolatesCapturedStdoutVariables(t *testing.T) { + variables := map[string]string{} + + first := runCLICommand(api.CLIStepCLICommand{ + Command: `go env -json GOOS`, + StdoutVariables: []api.CLICommandStdoutVariable{{ + Name: "goenv", + Regex: `"([A-Z]+)"`, + }}, + }, variables) + if first.Err != "" { + t.Fatalf("unexpected first command error: %s", first.Err) + } + + second := runCLICommand(api.CLIStepCLICommand{ + Command: `go env ${goenv}`, + }, variables) + if second.Stdout != runtime.GOOS { + t.Fatalf("second stdout = %q, want %q", second.Stdout, runtime.GOOS) + } +} + +func TestParseStdoutVariablesRequiresOneCaptureGroup(t *testing.T) { + variables := map[string]string{} + err := parseStdoutVariables("token=abc123", []api.CLICommandStdoutVariable{{ + Name: "token", + Regex: `token=([a-z]+)([0-9]+)`, + }}, variables) + + if err == nil { + t.Fatal("expected parse error") + } + if err.Error() != "invalid stdout variable configuration" { + t.Fatalf("error = %q, want invalid stdout variable configuration", err.Error()) + } +} + +func TestParseStdoutVariablesUsesGenericConfigurationError(t *testing.T) { + tests := []struct { + name string + vardef api.CLICommandStdoutVariable + }{ + { + name: "missing name", + vardef: api.CLICommandStdoutVariable{Regex: `token=([a-z0-9]+)`}, + }, + { + name: "missing regex", + vardef: api.CLICommandStdoutVariable{Name: "token"}, + }, + { + name: "invalid regex", + vardef: api.CLICommandStdoutVariable{Name: "token", Regex: `token=([a-z0-9]+`}, + }, + { + name: "too many capture groups", + vardef: api.CLICommandStdoutVariable{Name: "token", Regex: `token=([a-z]+)([0-9]+)`}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + variables := map[string]string{} + err := parseStdoutVariables("token=abc123", []api.CLICommandStdoutVariable{tt.vardef}, variables) + if err == nil { + t.Fatal("expected parse error") + } + if err.Error() != "invalid stdout variable configuration" { + t.Fatalf("error = %q, want invalid stdout variable configuration", err.Error()) + } + }) + } +} diff --git a/checks/local.go b/checks/local.go index e99103d..b6cc095 100644 --- a/checks/local.go +++ b/checks/local.go @@ -61,6 +61,10 @@ func EvaluateCLIResults(cliData api.CLIData, results []api.CLIStepResult) *api.S } func evaluateCLICommandTests(stepIndex int, cmd api.CLIStepCLICommand, result api.CLICommandResult) *api.StructuredErrCLI { + if result.Err != "" { + return localFailure(stepIndex, 0, result.Err) + } + for testIndex, test := range cmd.Tests { var err error @@ -85,10 +89,10 @@ func evaluateCLICommandTests(stepIndex int, cmd api.CLIStepCLICommand, result ap break } } - case test.StdoutLinesGt != nil: + case test.StdoutLinesGT != nil: lineCount := stdoutLineCount(result.Stdout) - if lineCount <= *test.StdoutLinesGt { - err = fmt.Errorf("expected stdout to have more than %d lines, got %d", *test.StdoutLinesGt, lineCount) + if lineCount <= *test.StdoutLinesGT { + err = fmt.Errorf("expected stdout to have more than %d lines, got %d", *test.StdoutLinesGT, lineCount) } case test.StdoutJq != nil: err = evaluateStdoutJq(result.Stdout, *test.StdoutJq, result.Variables) diff --git a/checks/local_test.go b/checks/local_test.go index f4dd1ae..424af40 100644 --- a/checks/local_test.go +++ b/checks/local_test.go @@ -73,6 +73,31 @@ func TestLocalSubmissionEventReportsFirstFailure(t *testing.T) { } } +func TestEvaluateCLICommandReportsStdoutVariableParseError(t *testing.T) { + cliData := api.CLIData{Steps: []api.CLIStep{ + {CLICommand: &api.CLIStepCLICommand{Tests: []api.CLICommandTest{ + {ExitCode: intPtr(0)}, + }}}, + }} + results := []api.CLIStepResult{ + {CLICommandResult: &api.CLICommandResult{ + ExitCode: 0, + Err: "invalid stdout variable configuration", + }}, + } + + event := LocalSubmissionEvent(cliData, results) + if event.ResultSlug != api.VerificationResultSlugFailure { + t.Fatalf("ResultSlug = %q, want failure", event.ResultSlug) + } + if event.StructuredErrCLI == nil { + t.Fatal("expected structured failure") + } + if event.StructuredErrCLI.ErrorMessage != "invalid stdout variable configuration" { + t.Fatalf("ErrorMessage = %q, want stdout variable error", event.StructuredErrCLI.ErrorMessage) + } +} + func TestEvaluateStdoutJq(t *testing.T) { err := evaluateStdoutJq( "{\"ok\":true}", diff --git a/client/lessons.go b/client/lessons.go index 34a8693..5875d99 100644 --- a/client/lessons.go +++ b/client/lessons.go @@ -34,17 +34,23 @@ type CLIStep struct { } type CLIStepCLICommand struct { - Command string `yaml:"command"` - Tests []CLICommandTest `yaml:"tests"` - SleepAfterMs *int `yaml:"sleepAfterMs"` - StdoutFilterTmdl *string `yaml:"stdoutFilterTmdl"` + Command string `yaml:"command"` + Tests []CLICommandTest `yaml:"tests"` + StdoutVariables []CLICommandStdoutVariable `yaml:"stdoutVariables"` + SleepAfterMs *int `yaml:"sleepAfterMs"` + StdoutFilterTmdl *string `yaml:"stdoutFilterTmdl"` +} + +type CLICommandStdoutVariable struct { + Name string `yaml:"name"` + Regex string `yaml:"regex"` } type CLICommandTest struct { ExitCode *int `yaml:"exitCode"` StdoutContainsAll []string `yaml:"stdoutContainsAll"` StdoutContainsNone []string `yaml:"stdoutContainsNone"` - StdoutLinesGt *int `yaml:"stdoutLinesGt"` + StdoutLinesGT *int `yaml:"stdoutLinesGT"` StdoutJq *StdoutJqTest `yaml:"stdoutJq"` } @@ -174,6 +180,7 @@ type CLIStepResult struct { type CLICommandResult struct { ExitCode int + Err string `json:"-"` FinalCommand string `json:"-"` Command CLIStepCLICommand `json:"-"` Stdout string