From 3337257fe24b67871406217e98862d34647dd725 Mon Sep 17 00:00:00 2001 From: Flu0r1ne Date: Fri, 13 Aug 2021 16:31:04 -0500 Subject: Add colorized output and build traces --- adapters/gtest/adapter.go | 58 +++++++++++++++++++++++++++++++++++------------ adapters/gtest/results.go | 29 ++++++++++++++++-------- 2 files changed, 63 insertions(+), 24 deletions(-) (limited to 'adapters') diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go index 600c956..3d769e3 100644 --- a/adapters/gtest/adapter.go +++ b/adapters/gtest/adapter.go @@ -2,14 +2,15 @@ package gtest import ( "context" + "errors" "fmt" "io/ioutil" "log" "os" "os/exec" - "time" "path" - "errors" + "time" + "golang.flu0r1ne.net/planr" ) @@ -61,7 +62,7 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) { planr.RunCmd("cmake", "-S", ".", "-B", ".") } -type ResultFromId map[string] planr.TestResult +type ResultFromId map[string] Result func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { buildDir := adapter.Config().Dir() @@ -84,34 +85,35 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { defer cancel() defer os.Remove(f.Name()) - exitFail := false if err := cmd.Run(); err != nil { var exiterr *exec.ExitError - if errors.As(err, &exiterr) && exiterr.ExitCode() == 1{ - exitFail = true - } else { + if !errors.As(err, &exiterr) { log.Printf("%v\n", err) os.Exit(exiterr.ExitCode()) } } - if exitFail { - fmt.Printf("") - } - for _, r := range decodeResults(f) { - lut[exe + "." + r.id] = r.result + lut[exe + "." + r.id] = r } } return lut } +// An executable may contain more than one test +// Gather all executables and deduplicate them func exes(tcs []*planr.TestCase) []string { set := make(map[string] bool, 0) for _, tc := range tcs { + // Tests which have encountered a failure + // may not have an executable + if tc.Result.Status != planr.PASSING { + continue + } + if(!set[tc.Cname]) { set[tc.Cname] = true } @@ -134,13 +136,39 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { buildDir := adapter.Config().Dir() chdir(buildDir) - planr.RunCmd("make", "-k") + for _, tc := range tcs { + cmd := exec.Command("make", tc.Cname) + out, err := cmd.CombinedOutput() + tc.Result = new(planr.TestResult) + + // Don't treat command failure as anything but a build failure + if err != nil{ + var exiterr *exec.ExitError + if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 { + log.Fatal(err) + } + + tc.Result.Status = planr.COMPILATION_FAILURE + } + + tc.Result.DebugOutput = string(out) + } files := exes(tcs) resultById := adapter.execTests(files) for _, tc := range tcs { - result := resultById[id(tc)] - tc.Result = &result + result, ok := resultById[id(tc)] + + // compilation failure + if !ok { + continue + } + + if !result.pass { + tc.Result.Status = planr.RUNTIME_FAILURE + } + + tc.Result.FailureMsg = result.failureMsg } } diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go index 88e4069..c556749 100644 --- a/adapters/gtest/results.go +++ b/adapters/gtest/results.go @@ -6,8 +6,6 @@ import ( "io" "log" "time" - - "golang.flu0r1ne.net/planr" ) type gFailure struct { @@ -48,8 +46,19 @@ type gResults struct { } type Result struct { - id string - result planr.TestResult + id string + pass bool + failureMsg string +} + +func failureMsg(failures []gFailure) string { + failure_msg := "" + + for _, failure := range failures { + failure_msg += failure.Failure + } + + return failure_msg } func decodeResults(r io.Reader) []Result { @@ -69,13 +78,15 @@ func decodeResults(r io.Reader) []Result { for _, test := range suite.Testsuite { n := len(test.Failures) - decoded = append(decoded, Result{ + + res := Result { id: suite.Name + "." + test.Name, - result: planr.TestResult { - Pass: n == 0, - }, - }) + pass: n == 0, + failureMsg: failureMsg(test.Failures), + } + + decoded = append(decoded, res) } } -- cgit v1.2.3 From 9954b5fa87f74c035739159f4bb831900e5e35be Mon Sep 17 00:00:00 2001 From: Flu0r1ne Date: Fri, 13 Aug 2021 17:03:57 -0500 Subject: Fenced test & compilation output --- adapters/gtest/results.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'adapters') diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go index c556749..0e96edc 100644 --- a/adapters/gtest/results.go +++ b/adapters/gtest/results.go @@ -55,7 +55,7 @@ func failureMsg(failures []gFailure) string { failure_msg := "" for _, failure := range failures { - failure_msg += failure.Failure + failure_msg += failure.Failure + "\n" } return failure_msg -- cgit v1.2.3 From 5d33040ab80b5cce7883b2e5965aa17db2e6515a Mon Sep 17 00:00:00 2001 From: Flu0r1ne Date: Fri, 13 Aug 2021 18:25:17 -0500 Subject: Opt to show CLI output over JSON output --- adapters/gtest/adapter.go | 6 ++++-- adapters/gtest/results.go | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'adapters') diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go index 3d769e3..8dc333d 100644 --- a/adapters/gtest/adapter.go +++ b/adapters/gtest/adapter.go @@ -85,7 +85,8 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { defer cancel() defer os.Remove(f.Name()) - if err := cmd.Run(); err != nil { + out, err := cmd.CombinedOutput() + if err != nil { var exiterr *exec.ExitError if !errors.As(err, &exiterr) { @@ -95,6 +96,7 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { } for _, r := range decodeResults(f) { + r.testOutput = string(out) lut[exe + "." + r.id] = r } } @@ -169,6 +171,6 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { tc.Result.Status = planr.RUNTIME_FAILURE } - tc.Result.FailureMsg = result.failureMsg + tc.Result.TestOutput = result.testOutput } } diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go index 0e96edc..2991823 100644 --- a/adapters/gtest/results.go +++ b/adapters/gtest/results.go @@ -49,6 +49,7 @@ type Result struct { id string pass bool failureMsg string + testOutput string } func failureMsg(failures []gFailure) string { -- cgit v1.2.3 From 24548e87decbdfea38bbf692cecad6d4eefc3ec0 Mon Sep 17 00:00:00 2001 From: Flu0r1ne Date: Sun, 22 Aug 2021 23:27:53 -0500 Subject: Refactoring & Enhanced logging --- adapters/gtest/adapter.go | 75 +++++++++++++++++++++++++++++++---------------- adapters/gtest/config.go | 50 +++++++++++++++---------------- adapters/gtest/results.go | 12 ++++---- 3 files changed, 79 insertions(+), 58 deletions(-) (limited to 'adapters') diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go index 8dc333d..f4fde27 100644 --- a/adapters/gtest/adapter.go +++ b/adapters/gtest/adapter.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path" + "sync" "time" "golang.flu0r1ne.net/planr" @@ -26,11 +27,6 @@ func mkUnit(tc *planr.TestCase) cmakeUnit { }; } -func chdir(dir string) { - if err := os.Chdir(dir); err != nil { - log.Fatal(err) - } -} type GtestAdapter struct {} @@ -49,7 +45,6 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) { units := make([]cmakeUnit, 0) for _, tc := range tcs { - fmt.Printf("[R] Building %s (%s)\n", tc.Cname, tc.Path) cfg := tc.AdapterConfig().(*GtestConfig) cfg.ensureSatisfied(tc.Path) @@ -57,8 +52,7 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) { } genCmake(cmakeFile, units) - - chdir(buildDir) + planr.RunCmd("cmake", "-S", ".", "-B", ".") } @@ -69,7 +63,6 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { lut := make(ResultFromId, 0) for _, exe := range cnames { - fmt.Printf("[R] Evaluating %s\n", exe) exePath := path.Join(buildDir, exe) @@ -95,7 +88,14 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { } } - for _, r := range decodeResults(f) { + results, err := decodeResults(f) + + if err != nil { + log.Printf("Could not collect results from %s: %v", exe, err) + continue + } + + for _, r := range results { r.testOutput = string(out) lut[exe + "." + r.id] = r } @@ -134,27 +134,34 @@ func id(tc *planr.TestCase) string { return tc.Cname + "." + *cfg.Suite + "." + *cfg.Name } -func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { - buildDir := adapter.Config().Dir() - chdir(buildDir) +func compile(wg * sync.WaitGroup, tc *planr.TestCase) { + defer wg.Done() - for _, tc := range tcs { - cmd := exec.Command("make", tc.Cname) - out, err := cmd.CombinedOutput() - tc.Result = new(planr.TestResult) - - // Don't treat command failure as anything but a build failure - if err != nil{ - var exiterr *exec.ExitError - if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 { - log.Fatal(err) - } + cmd := exec.Command("make", tc.Cname) + out, err := cmd.CombinedOutput() + tc.Result = new(planr.TestResult) - tc.Result.Status = planr.COMPILATION_FAILURE + // Don't treat command failure as anything but a build failure + if err != nil{ + var exiterr *exec.ExitError + if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 { + log.Fatal(err) } - tc.Result.DebugOutput = string(out) + tc.Result.Status = planr.COMPILATION_FAILURE + } + + tc.Result.DebugOutput = string(out) +} + +// ./planr eval 0.93s user 0.16s system 100% cpu 1.089 total +func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { + var wg sync.WaitGroup + for _, tc := range tcs { + wg.Add(1) + go compile(&wg, tc) } + wg.Wait() files := exes(tcs) resultById := adapter.execTests(files) @@ -164,6 +171,22 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { // compilation failure if !ok { + fmt.Printf("CAN'T FIND %s: status %d\n", tc.Cname, tc.Result.Status) + + if tc.Result.Status == planr.PASSING { + cfg := tc.AdapterConfig().(*GtestConfig) + + log.Printf( + "Could not find testcase %s with name=\"%s\" and suite=\"%s\". Does such a test exist in the test source?", + tc.Cname, + *cfg.Name, + *cfg.Suite, + ) + + tc.Result.Status = planr.COMPILATION_FAILURE + tc.Result.DebugOutput += fmt.Sprintf("planr: Did not find testcase %s in any test executable\n", id(tc)) + } + continue } diff --git a/adapters/gtest/config.go b/adapters/gtest/config.go index 6a6c8bf..cb5ba75 100644 --- a/adapters/gtest/config.go +++ b/adapters/gtest/config.go @@ -8,23 +8,23 @@ import ( ) type GtestDefaults struct { - Name *string - Suite *string - Testfile *string - Test_root *string - Srcs *[]string - Srcs_root *string + Name *string + Suite *string + Testfile *string + Test_root *string + Srcs *[]string + Srcs_root *string } func (child *GtestDefaults) Inherit(p interface{}) { parent := p.(*GtestDefaults) - if(child.Name == nil) { child.Name = parent.Name } - if(child.Suite == nil) { child.Suite = parent.Suite } - if(child.Testfile == nil) { child.Testfile = parent.Testfile } - if(child.Test_root == nil) { child.Test_root = parent.Test_root } - if(child.Srcs == nil) { child.Srcs = parent.Srcs } - if(child.Srcs_root == nil) { child.Srcs_root = parent.Srcs_root } + if(child.Name == nil) { child.Name = parent.Name } + if(child.Suite == nil) { child.Suite = parent.Suite } + if(child.Testfile == nil) { child.Testfile = parent.Testfile } + if(child.Test_root == nil) { child.Test_root = parent.Test_root } + if(child.Srcs == nil) { child.Srcs = parent.Srcs } + if(child.Srcs_root == nil) { child.Srcs_root = parent.Srcs_root } } @@ -73,24 +73,22 @@ func (cfg GtestConfig) srcList() string { return srcList } -func primitiveDecode(primitive toml.Primitive, config interface{}) { - if err := toml.PrimitiveDecode(primitive, config); err != nil { - log.Fatal(err) - } -} - -func ParseConfig(prim toml.Primitive) planr.InheritableConfig { +func ParseConfig(prim toml.Primitive) (planr.InheritableConfig, error) { config := GtestConfig{} - - primitiveDecode(prim, &config) - - return &config + + if err := toml.PrimitiveDecode(prim, &config); err != nil { + return nil, err + } + + return &config, nil } -func ParseDefaultConfig(prim toml.Primitive) planr.InheritableConfig { +func ParseDefaultConfig(prim toml.Primitive) (planr.InheritableConfig, error) { config := GtestDefaults{} - primitiveDecode(prim, &config) + if err := toml.PrimitiveDecode(prim, &config); err != nil { + return nil, err + } - return &config + return &config, nil } diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go index 2991823..14f5d1d 100644 --- a/adapters/gtest/results.go +++ b/adapters/gtest/results.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "io" - "log" "time" ) @@ -62,19 +61,20 @@ func failureMsg(failures []gFailure) string { return failure_msg } -func decodeResults(r io.Reader) []Result { +func decodeResults(r io.Reader) ([]Result, error) { + decoded := make([]Result, 0) + var results gResults buf := bytes.Buffer{} if _, err := buf.ReadFrom(r); err != nil { - log.Fatal(err) + return decoded, err } if err := json.Unmarshal(buf.Bytes(), &results); err != nil { - log.Fatal(err) + return decoded, err } - decoded := make([]Result, 0) for _, suite := range results.Testsuites { for _, test := range suite.Testsuite { n := len(test.Failures) @@ -91,5 +91,5 @@ func decodeResults(r io.Reader) []Result { } } - return decoded + return decoded, nil } -- cgit v1.2.3