From 3337257fe24b67871406217e98862d34647dd725 Mon Sep 17 00:00:00 2001 From: Flu0r1ne Date: Fri, 13 Aug 2021 16:31:04 -0500 Subject: Add colorized output and build traces --- adapters/gtest/adapter.go | 58 ++++++++++++++++++++++-------- adapters/gtest/results.go | 29 ++++++++++----- cmd/planr/main.go | 2 +- cmd/planr/sub/cli.go | 89 +++++++++++++++++++++++++++++++++++++++++++++++ cmd/planr/sub/evaluate.go | 40 +++++++-------------- go.mod | 1 + go.sum | 9 +++++ testcase.go | 15 ++++++-- 8 files changed, 189 insertions(+), 54 deletions(-) create mode 100644 cmd/planr/sub/cli.go diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go index 600c956..3d769e3 100644 --- a/adapters/gtest/adapter.go +++ b/adapters/gtest/adapter.go @@ -2,14 +2,15 @@ package gtest import ( "context" + "errors" "fmt" "io/ioutil" "log" "os" "os/exec" - "time" "path" - "errors" + "time" + "golang.flu0r1ne.net/planr" ) @@ -61,7 +62,7 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) { planr.RunCmd("cmake", "-S", ".", "-B", ".") } -type ResultFromId map[string] planr.TestResult +type ResultFromId map[string] Result func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { buildDir := adapter.Config().Dir() @@ -84,34 +85,35 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { defer cancel() defer os.Remove(f.Name()) - exitFail := false if err := cmd.Run(); err != nil { var exiterr *exec.ExitError - if errors.As(err, &exiterr) && exiterr.ExitCode() == 1{ - exitFail = true - } else { + if !errors.As(err, &exiterr) { log.Printf("%v\n", err) os.Exit(exiterr.ExitCode()) } } - if exitFail { - fmt.Printf("") - } - for _, r := range decodeResults(f) { - lut[exe + "." + r.id] = r.result + lut[exe + "." + r.id] = r } } return lut } +// An executable may contain more than one test +// Gather all executables and deduplicate them func exes(tcs []*planr.TestCase) []string { set := make(map[string] bool, 0) for _, tc := range tcs { + // Tests which have encountered a failure + // may not have an executable + if tc.Result.Status != planr.PASSING { + continue + } + if(!set[tc.Cname]) { set[tc.Cname] = true } @@ -134,13 +136,39 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { buildDir := adapter.Config().Dir() chdir(buildDir) - planr.RunCmd("make", "-k") + for _, tc := range tcs { + cmd := exec.Command("make", tc.Cname) + out, err := cmd.CombinedOutput() + tc.Result = new(planr.TestResult) + + // Don't treat command failure as anything but a build failure + if err != nil{ + var exiterr *exec.ExitError + if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 { + log.Fatal(err) + } + + tc.Result.Status = planr.COMPILATION_FAILURE + } + + tc.Result.DebugOutput = string(out) + } files := exes(tcs) resultById := adapter.execTests(files) for _, tc := range tcs { - result := resultById[id(tc)] - tc.Result = &result + result, ok := resultById[id(tc)] + + // compilation failure + if !ok { + continue + } + + if !result.pass { + tc.Result.Status = planr.RUNTIME_FAILURE + } + + tc.Result.FailureMsg = result.failureMsg } } diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go index 88e4069..c556749 100644 --- a/adapters/gtest/results.go +++ b/adapters/gtest/results.go @@ -6,8 +6,6 @@ import ( "io" "log" "time" - - "golang.flu0r1ne.net/planr" ) type gFailure struct { @@ -48,8 +46,19 @@ type gResults struct { } type Result struct { - id string - result planr.TestResult + id string + pass bool + failureMsg string +} + +func failureMsg(failures []gFailure) string { + failure_msg := "" + + for _, failure := range failures { + failure_msg += failure.Failure + } + + return failure_msg } func decodeResults(r io.Reader) []Result { @@ -69,13 +78,15 @@ func decodeResults(r io.Reader) []Result { for _, test := range suite.Testsuite { n := len(test.Failures) - decoded = append(decoded, Result{ + + res := Result { id: suite.Name + "." + test.Name, - result: planr.TestResult { - Pass: n == 0, - }, - }) + pass: n == 0, + failureMsg: failureMsg(test.Failures), + } + + decoded = append(decoded, res) } } diff --git a/cmd/planr/main.go b/cmd/planr/main.go index c65edb1..3c1c298 100644 --- a/cmd/planr/main.go +++ b/cmd/planr/main.go @@ -38,7 +38,7 @@ func main() { fmt.Printf("%s\n", VERSION) case "build": sub.Build(subargs) - case "evaluate": + case "evaluate","eval": sub.Evaluate(subargs) case "help", "-h", "-help", "--help": printUsage(os.Stdout) diff --git a/cmd/planr/sub/cli.go b/cmd/planr/sub/cli.go new file mode 100644 index 0000000..42d1c81 --- /dev/null +++ b/cmd/planr/sub/cli.go @@ -0,0 +1,89 @@ +package sub + +import ( + "github.com/fatih/color" + "golang.flu0r1ne.net/planr" + "fmt" +) + +var ( + col_pass = color.New(color.FgGreen) + col_fail = color.New(color.FgRed) + col_title = color.New(color.FgHiWhite) + col_label = color.New(color.FgCyan) +); + +func tcTitle(tc planr.TestCase) string { + title := tc.Cname + + if tc.Config.Title != nil { + title = *tc.Config.Title + } + + return title +} + +func tcStatus(tc planr.TestCase) string { + status := "SILENT" + + if tc.Result != nil { + if tc.Result.Status == planr.PASSING { + status = "PASS" + } else { + status = "FAIL" + } + } + + return status +} + +func pprintLabeled(label, value string) { + col_label.Printf(" %s: ", label) + fmt.Println(value) +} + +func tcStatusLine(tc planr.TestCase) { + title := tcTitle(tc) + status := tcStatus(tc) + + if status == "PASS" { + col_pass.Printf("[%s] ", status); + } else { + col_fail.Printf("[%s] ", status); + } + + col_title.Println(title); +} + +func tcPprint(tc planr.TestCase) { + tcStatusLine(tc) + + if tc.Config.Points != nil { + points := fmt.Sprintf("%.1f", *tc.Config.Points) + pprintLabeled("points", points) + } + + if tc.Config.Description != nil { + pprintLabeled("description", *tc.Config.Description) + } + + if tc.Result.DebugOutput != "" { + pprintLabeled("debug output", tc.Result.DebugOutput) + } + + if tc.Result.FailureMsg != "" { + pprintLabeled("failure", tc.Result.FailureMsg); + } + + fmt.Println() +} + +func printResults(passed, tc_total int, earned, points_total float64) { + col_title.Println("Final Results:") + + pprintLabeled("passed", fmt.Sprintf("%d/%d", passed, tc_total)); + + pprintLabeled("score", fmt.Sprintf( + "%.2f/%.2f ~= %.1f%%", earned, points_total, earned / points_total * 100, + )); +} diff --git a/cmd/planr/sub/evaluate.go b/cmd/planr/sub/evaluate.go index 1174b3f..8ce4d81 100644 --- a/cmd/planr/sub/evaluate.go +++ b/cmd/planr/sub/evaluate.go @@ -1,7 +1,6 @@ package sub import ( - "fmt" "golang.flu0r1ne.net/planr" ) @@ -10,43 +9,30 @@ func Evaluate(params []string) { tcs := Runner().Evaluate(rd) - fmt.Printf("\n\nREPORT:\n=======\n\n") - earned := 0.0 total := 0.0 + passed := 0 for _, tc := range tcs { cfg := tc.Config - - name := tc.Cname - if cfg.Title != nil { - name = *cfg.Title - } - var points float64 = 0.0 if cfg.Points != nil { - points = float64(*cfg.Points) - } + points := float64(*cfg.Points) + + total += points - status := "SILENT" - if tc.Result != nil { - if tc.Result.Pass { - status = "PASS" + if tc.Result.Status == planr.PASSING { earned += points - } else { - status = "FAIL" + passed++ } } - total += points - - fmt.Printf("[%s] %s (%f)\n", status, name, points) - - if cfg.Description != nil { - fmt.Printf("> %s\n", *cfg.Description) - } - - fmt.Println() + tcPprint(tc) } - fmt.Printf("Score: %f (%f%%)\n", earned, (earned / total) * 100) + printResults( + passed, + len(tcs), + earned, + total, + ); } diff --git a/go.mod b/go.mod index 5de0411..4f1d784 100644 --- a/go.mod +++ b/go.mod @@ -4,4 +4,5 @@ go 1.16 require ( github.com/BurntSushi/toml v0.3.1 + github.com/fatih/color v1.12.0 // indirect ) diff --git a/go.sum b/go.sum index fca8c3e..5516695 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,13 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w= github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/testcase.go b/testcase.go index 989838c..d3fe8ed 100644 --- a/testcase.go +++ b/testcase.go @@ -1,8 +1,18 @@ package planr +type TestStatus uint + +const ( + PASSING TestStatus = iota + COMPILATION_FAILURE + RUNTIME_FAILURE +) + type TestResult struct { - Id string - Pass bool + Id string + Status TestStatus + FailureMsg string + DebugOutput string } type TestCase struct { @@ -18,6 +28,7 @@ type TestCase struct { Config TestCaseConfig Result *TestResult + } func (tc TestCase) AdapterConfig() InheritableConfig { -- cgit v1.2.3