aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--adapters/gtest/adapter.go58
-rw-r--r--adapters/gtest/results.go29
-rw-r--r--cmd/planr/main.go2
-rw-r--r--cmd/planr/sub/cli.go89
-rw-r--r--cmd/planr/sub/evaluate.go40
-rw-r--r--go.mod1
-rw-r--r--go.sum9
-rw-r--r--testcase.go15
8 files changed, 189 insertions, 54 deletions
diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go
index 600c956..3d769e3 100644
--- a/adapters/gtest/adapter.go
+++ b/adapters/gtest/adapter.go
@@ -2,14 +2,15 @@ package gtest
import (
"context"
+ "errors"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
- "time"
"path"
- "errors"
+ "time"
+
"golang.flu0r1ne.net/planr"
)
@@ -61,7 +62,7 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) {
planr.RunCmd("cmake", "-S", ".", "-B", ".")
}
-type ResultFromId map[string] planr.TestResult
+type ResultFromId map[string] Result
func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
buildDir := adapter.Config().Dir()
@@ -84,34 +85,35 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
defer cancel()
defer os.Remove(f.Name())
- exitFail := false
if err := cmd.Run(); err != nil {
var exiterr *exec.ExitError
- if errors.As(err, &exiterr) && exiterr.ExitCode() == 1{
- exitFail = true
- } else {
+ if !errors.As(err, &exiterr) {
log.Printf("%v\n", err)
os.Exit(exiterr.ExitCode())
}
}
- if exitFail {
- fmt.Printf("")
- }
-
for _, r := range decodeResults(f) {
- lut[exe + "." + r.id] = r.result
+ lut[exe + "." + r.id] = r
}
}
return lut
}
+// An executable may contain more than one test
+// Gather all executables and deduplicate them
func exes(tcs []*planr.TestCase) []string {
set := make(map[string] bool, 0)
for _, tc := range tcs {
+ // Tests which have encountered a failure
+ // may not have an executable
+ if tc.Result.Status != planr.PASSING {
+ continue
+ }
+
if(!set[tc.Cname]) {
set[tc.Cname] = true
}
@@ -134,13 +136,39 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
buildDir := adapter.Config().Dir()
chdir(buildDir)
- planr.RunCmd("make", "-k")
+ for _, tc := range tcs {
+ cmd := exec.Command("make", tc.Cname)
+ out, err := cmd.CombinedOutput()
+ tc.Result = new(planr.TestResult)
+
+ // Don't treat command failure as anything but a build failure
+ if err != nil{
+ var exiterr *exec.ExitError
+ if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 {
+ log.Fatal(err)
+ }
+
+ tc.Result.Status = planr.COMPILATION_FAILURE
+ }
+
+ tc.Result.DebugOutput = string(out)
+ }
files := exes(tcs)
resultById := adapter.execTests(files)
for _, tc := range tcs {
- result := resultById[id(tc)]
- tc.Result = &result
+ result, ok := resultById[id(tc)]
+
+ // compilation failure
+ if !ok {
+ continue
+ }
+
+ if !result.pass {
+ tc.Result.Status = planr.RUNTIME_FAILURE
+ }
+
+ tc.Result.FailureMsg = result.failureMsg
}
}
diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go
index 88e4069..c556749 100644
--- a/adapters/gtest/results.go
+++ b/adapters/gtest/results.go
@@ -6,8 +6,6 @@ import (
"io"
"log"
"time"
-
- "golang.flu0r1ne.net/planr"
)
type gFailure struct {
@@ -48,8 +46,19 @@ type gResults struct {
}
type Result struct {
- id string
- result planr.TestResult
+ id string
+ pass bool
+ failureMsg string
+}
+
+func failureMsg(failures []gFailure) string {
+ failure_msg := ""
+
+ for _, failure := range failures {
+ failure_msg += failure.Failure
+ }
+
+ return failure_msg
}
func decodeResults(r io.Reader) []Result {
@@ -69,13 +78,15 @@ func decodeResults(r io.Reader) []Result {
for _, test := range suite.Testsuite {
n := len(test.Failures)
- decoded = append(decoded, Result{
+
+ res := Result {
id: suite.Name + "." + test.Name,
- result: planr.TestResult {
- Pass: n == 0,
- },
- })
+ pass: n == 0,
+ failureMsg: failureMsg(test.Failures),
+ }
+
+ decoded = append(decoded, res)
}
}
diff --git a/cmd/planr/main.go b/cmd/planr/main.go
index c65edb1..3c1c298 100644
--- a/cmd/planr/main.go
+++ b/cmd/planr/main.go
@@ -38,7 +38,7 @@ func main() {
fmt.Printf("%s\n", VERSION)
case "build":
sub.Build(subargs)
- case "evaluate":
+ case "evaluate","eval":
sub.Evaluate(subargs)
case "help", "-h", "-help", "--help":
printUsage(os.Stdout)
diff --git a/cmd/planr/sub/cli.go b/cmd/planr/sub/cli.go
new file mode 100644
index 0000000..42d1c81
--- /dev/null
+++ b/cmd/planr/sub/cli.go
@@ -0,0 +1,89 @@
+package sub
+
+import (
+ "github.com/fatih/color"
+ "golang.flu0r1ne.net/planr"
+ "fmt"
+)
+
+var (
+ col_pass = color.New(color.FgGreen)
+ col_fail = color.New(color.FgRed)
+ col_title = color.New(color.FgHiWhite)
+ col_label = color.New(color.FgCyan)
+);
+
+func tcTitle(tc planr.TestCase) string {
+ title := tc.Cname
+
+ if tc.Config.Title != nil {
+ title = *tc.Config.Title
+ }
+
+ return title
+}
+
+func tcStatus(tc planr.TestCase) string {
+ status := "SILENT"
+
+ if tc.Result != nil {
+ if tc.Result.Status == planr.PASSING {
+ status = "PASS"
+ } else {
+ status = "FAIL"
+ }
+ }
+
+ return status
+}
+
+func pprintLabeled(label, value string) {
+ col_label.Printf(" %s: ", label)
+ fmt.Println(value)
+}
+
+func tcStatusLine(tc planr.TestCase) {
+ title := tcTitle(tc)
+ status := tcStatus(tc)
+
+ if status == "PASS" {
+ col_pass.Printf("[%s] ", status);
+ } else {
+ col_fail.Printf("[%s] ", status);
+ }
+
+ col_title.Println(title);
+}
+
+func tcPprint(tc planr.TestCase) {
+ tcStatusLine(tc)
+
+ if tc.Config.Points != nil {
+ points := fmt.Sprintf("%.1f", *tc.Config.Points)
+ pprintLabeled("points", points)
+ }
+
+ if tc.Config.Description != nil {
+ pprintLabeled("description", *tc.Config.Description)
+ }
+
+ if tc.Result.DebugOutput != "" {
+ pprintLabeled("debug output", tc.Result.DebugOutput)
+ }
+
+ if tc.Result.FailureMsg != "" {
+ pprintLabeled("failure", tc.Result.FailureMsg);
+ }
+
+ fmt.Println()
+}
+
+func printResults(passed, tc_total int, earned, points_total float64) {
+ col_title.Println("Final Results:")
+
+ pprintLabeled("passed", fmt.Sprintf("%d/%d", passed, tc_total));
+
+ pprintLabeled("score", fmt.Sprintf(
+ "%.2f/%.2f ~= %.1f%%", earned, points_total, earned / points_total * 100,
+ ));
+}
diff --git a/cmd/planr/sub/evaluate.go b/cmd/planr/sub/evaluate.go
index 1174b3f..8ce4d81 100644
--- a/cmd/planr/sub/evaluate.go
+++ b/cmd/planr/sub/evaluate.go
@@ -1,7 +1,6 @@
package sub
import (
- "fmt"
"golang.flu0r1ne.net/planr"
)
@@ -10,43 +9,30 @@ func Evaluate(params []string) {
tcs := Runner().Evaluate(rd)
- fmt.Printf("\n\nREPORT:\n=======\n\n")
-
earned := 0.0
total := 0.0
+ passed := 0
for _, tc := range tcs {
cfg := tc.Config
-
- name := tc.Cname
- if cfg.Title != nil {
- name = *cfg.Title
- }
- var points float64 = 0.0
if cfg.Points != nil {
- points = float64(*cfg.Points)
- }
+ points := float64(*cfg.Points)
+
+ total += points
- status := "SILENT"
- if tc.Result != nil {
- if tc.Result.Pass {
- status = "PASS"
+ if tc.Result.Status == planr.PASSING {
earned += points
- } else {
- status = "FAIL"
+ passed++
}
}
- total += points
-
- fmt.Printf("[%s] %s (%f)\n", status, name, points)
-
- if cfg.Description != nil {
- fmt.Printf("> %s\n", *cfg.Description)
- }
-
- fmt.Println()
+ tcPprint(tc)
}
- fmt.Printf("Score: %f (%f%%)\n", earned, (earned / total) * 100)
+ printResults(
+ passed,
+ len(tcs),
+ earned,
+ total,
+ );
}
diff --git a/go.mod b/go.mod
index 5de0411..4f1d784 100644
--- a/go.mod
+++ b/go.mod
@@ -4,4 +4,5 @@ go 1.16
require (
github.com/BurntSushi/toml v0.3.1
+ github.com/fatih/color v1.12.0 // indirect
)
diff --git a/go.sum b/go.sum
index fca8c3e..5516695 100644
--- a/go.sum
+++ b/go.sum
@@ -1,4 +1,13 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=
+github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w=
github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/testcase.go b/testcase.go
index 989838c..d3fe8ed 100644
--- a/testcase.go
+++ b/testcase.go
@@ -1,8 +1,18 @@
package planr
+type TestStatus uint
+
+const (
+ PASSING TestStatus = iota
+ COMPILATION_FAILURE
+ RUNTIME_FAILURE
+)
+
type TestResult struct {
- Id string
- Pass bool
+ Id string
+ Status TestStatus
+ FailureMsg string
+ DebugOutput string
}
type TestCase struct {
@@ -18,6 +28,7 @@ type TestCase struct {
Config TestCaseConfig
Result *TestResult
+
}
func (tc TestCase) AdapterConfig() InheritableConfig {
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-19 23:53:49 +0000