Add colorized output and build traces

8 files changed, 189 insertions, 54 deletions

diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go
index 600c956..3d769e3 100644
--- a/adapters/gtest/adapter.go
+++ b/adapters/gtest/adapter.go
@@ -2,14 +2,15 @@ package gtest
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
-        "time"
 	"path"
-        "errors"
+	"time"
+
 	"golang.flu0r1ne.net/planr"
 )
 
@@ -61,7 +62,7 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) {
   planr.RunCmd("cmake", "-S", ".", "-B", ".")
 }
 
-type ResultFromId map[string] planr.TestResult
+type ResultFromId map[string] Result
 
 func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
   buildDir := adapter.Config().Dir()
@@ -84,34 +85,35 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
       defer cancel()
       defer os.Remove(f.Name())
 
-      exitFail := false
       if err := cmd.Run(); err != nil {
         var exiterr *exec.ExitError
 
-        if errors.As(err, &exiterr) && exiterr.ExitCode() == 1{
-          exitFail = true              
-        } else {
+        if !errors.As(err, &exiterr) {
           log.Printf("%v\n", err)
           os.Exit(exiterr.ExitCode())
         }
       }
 
-      if exitFail {
-        fmt.Printf("")
-      }
-
       for _, r  := range decodeResults(f) {
-        lut[exe + "." + r.id] = r.result
+        lut[exe + "." + r.id] = r
       }
   }
 
   return lut
 }
 
+// An executable may contain more than one test
+// Gather all executables and deduplicate them
 func exes(tcs []*planr.TestCase) []string {
   set := make(map[string] bool, 0)
   
   for _, tc := range tcs {
+    // Tests which have encountered a failure
+    // may not have an executable
+    if tc.Result.Status != planr.PASSING {
+      continue
+    }
+
     if(!set[tc.Cname]) {
       set[tc.Cname] = true
     }
@@ -134,13 +136,39 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
   buildDir := adapter.Config().Dir()
   chdir(buildDir)
 
-  planr.RunCmd("make", "-k")
+  for _, tc := range tcs {
+    cmd := exec.Command("make", tc.Cname)
+    out, err := cmd.CombinedOutput()
+    tc.Result = new(planr.TestResult)
+
+    // Don't treat command failure as anything but a build failure
+    if err != nil{
+      var exiterr *exec.ExitError
+      if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 {
+        log.Fatal(err)
+      }
+
+      tc.Result.Status = planr.COMPILATION_FAILURE
+    }
+
+    tc.Result.DebugOutput = string(out)
+  }
 
   files := exes(tcs)
   resultById := adapter.execTests(files)
 
   for _, tc := range tcs {
-    result := resultById[id(tc)]
-    tc.Result = &result 
+    result, ok := resultById[id(tc)]
+
+    // compilation failure
+    if !ok {
+      continue
+    }
+ 
+    if !result.pass {
+      tc.Result.Status = planr.RUNTIME_FAILURE
+    }
+
+    tc.Result.FailureMsg = result.failureMsg
   }
 }
diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go
index 88e4069..c556749 100644
--- a/adapters/gtest/results.go
+++ b/adapters/gtest/results.go
@@ -6,8 +6,6 @@ import (
 	"io"
 	"log"
 	"time"
-
-	"golang.flu0r1ne.net/planr"
 )
 
 type gFailure struct {
@@ -48,8 +46,19 @@ type gResults struct {
 }
 
 type Result struct {
-  id string
-  result planr.TestResult
+  id         string
+  pass       bool
+  failureMsg string
+}
+
+func failureMsg(failures []gFailure) string {
+  failure_msg := ""
+
+  for _, failure := range failures {
+    failure_msg += failure.Failure
+  }
+
+  return failure_msg
 }
 
 func decodeResults(r io.Reader) []Result {
@@ -69,13 +78,15 @@ func decodeResults(r io.Reader) []Result {
     for _, test := range suite.Testsuite {
       n := len(test.Failures)
 
-      decoded = append(decoded, Result{
+
+      res := Result {
         id: suite.Name + "." + test.Name,
-        result: planr.TestResult {
-          Pass: n == 0,
-        },
-      })
+        pass: n == 0,
+        failureMsg: failureMsg(test.Failures),
+      }
+
 
+      decoded = append(decoded, res)
     }
   }
 
diff --git a/cmd/planr/main.go b/cmd/planr/main.go
index c65edb1..3c1c298 100644
--- a/cmd/planr/main.go
+++ b/cmd/planr/main.go
@@ -38,7 +38,7 @@ func main() {
 		fmt.Printf("%s\n", VERSION)
 	  case "build":
 		sub.Build(subargs)
-	  case "evaluate":
+	  case "evaluate","eval":
 		sub.Evaluate(subargs)
 	  case "help", "-h", "-help", "--help":
 		printUsage(os.Stdout)
diff --git a/cmd/planr/sub/cli.go b/cmd/planr/sub/cli.go
new file mode 100644
index 0000000..42d1c81
--- /dev/null
+++ b/cmd/planr/sub/cli.go
@@ -0,0 +1,89 @@
+package sub
+
+import (
+	"github.com/fatih/color"
+	"golang.flu0r1ne.net/planr"
+        "fmt"
+)
+
+var (
+  col_pass = color.New(color.FgGreen)
+  col_fail = color.New(color.FgRed)
+  col_title = color.New(color.FgHiWhite)
+  col_label = color.New(color.FgCyan)
+);
+
+func tcTitle(tc planr.TestCase) string {
+  title := tc.Cname
+
+  if tc.Config.Title != nil {
+    title = *tc.Config.Title
+  }
+
+  return title
+}
+
+func tcStatus(tc planr.TestCase) string {
+    status := "SILENT"
+
+    if tc.Result != nil {
+      if tc.Result.Status == planr.PASSING {
+        status = "PASS"
+      } else {
+        status = "FAIL"
+      }
+    }
+
+    return status
+}
+
+func pprintLabeled(label, value string) {
+  col_label.Printf("  %s: ", label)
+  fmt.Println(value)
+}
+
+func tcStatusLine(tc planr.TestCase) {
+  title  := tcTitle(tc)
+  status := tcStatus(tc)
+
+  if status == "PASS" {
+    col_pass.Printf("[%s] ", status);
+  } else {
+    col_fail.Printf("[%s] ", status);
+  }
+
+  col_title.Println(title);  
+}
+
+func tcPprint(tc planr.TestCase) {
+  tcStatusLine(tc)
+
+  if tc.Config.Points != nil {
+    points := fmt.Sprintf("%.1f", *tc.Config.Points)
+    pprintLabeled("points", points)
+  }
+
+  if tc.Config.Description != nil {
+    pprintLabeled("description", *tc.Config.Description)
+  }
+
+  if tc.Result.DebugOutput != "" {
+    pprintLabeled("debug output", tc.Result.DebugOutput)
+  }
+
+  if tc.Result.FailureMsg != "" {
+    pprintLabeled("failure", tc.Result.FailureMsg);
+  }
+
+  fmt.Println()
+}
+
+func printResults(passed, tc_total int, earned, points_total float64) {
+  col_title.Println("Final Results:")
+  
+  pprintLabeled("passed", fmt.Sprintf("%d/%d", passed, tc_total));
+
+  pprintLabeled("score", fmt.Sprintf(
+    "%.2f/%.2f ~= %.1f%%", earned, points_total, earned / points_total * 100,
+  ));
+}
diff --git a/cmd/planr/sub/evaluate.go b/cmd/planr/sub/evaluate.go
index 1174b3f..8ce4d81 100644
--- a/cmd/planr/sub/evaluate.go
+++ b/cmd/planr/sub/evaluate.go
@@ -1,7 +1,6 @@
 package sub
 
 import (
-  "fmt"
   "golang.flu0r1ne.net/planr"
 )
 
@@ -10,43 +9,30 @@ func Evaluate(params []string) {
 
   tcs := Runner().Evaluate(rd)
 
-  fmt.Printf("\n\nREPORT:\n=======\n\n")
-
   earned := 0.0
   total  := 0.0
+  passed := 0
   for _, tc := range tcs {
     cfg := tc.Config
-  
-    name := tc.Cname
-    if cfg.Title != nil {
-      name = *cfg.Title
-    }
 
-    var points float64 =  0.0
     if cfg.Points != nil {
-      points = float64(*cfg.Points)
-    }
+      points := float64(*cfg.Points)
+
+      total += points
 
-    status := "SILENT"
-    if tc.Result != nil {
-      if tc.Result.Pass {
-        status = "PASS"
+      if tc.Result.Status == planr.PASSING {
         earned += points
-      } else {
-        status = "FAIL"
+        passed++
       }
     }
 
-    total += points
-
-    fmt.Printf("[%s] %s (%f)\n", status, name, points)
-
-    if cfg.Description != nil {
-      fmt.Printf("> %s\n", *cfg.Description)
-    }
-
-    fmt.Println()
+    tcPprint(tc)
   }
 
-  fmt.Printf("Score: %f (%f%%)\n", earned, (earned / total) * 100)
+  printResults(
+    passed,
+    len(tcs),
+    earned,
+    total,
+  );
 }
diff --git a/go.mod b/go.mod
index 5de0411..4f1d784 100644
--- a/go.mod
+++ b/go.mod
@@ -4,4 +4,5 @@ go 1.16
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/fatih/color v1.12.0 // indirect
 )
diff --git a/go.sum b/go.sum
index fca8c3e..5516695 100644
--- a/go.sum
+++ b/go.sum
@@ -1,4 +1,13 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=
+github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w=
 github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/testcase.go b/testcase.go
index 989838c..d3fe8ed 100644
--- a/testcase.go
+++ b/testcase.go
@@ -1,8 +1,18 @@
 package planr
 
+type TestStatus uint
+
+const (
+  PASSING TestStatus = iota
+  COMPILATION_FAILURE
+  RUNTIME_FAILURE
+)
+
 type TestResult struct {
-  Id   string
-  Pass bool
+  Id          string
+  Status      TestStatus
+  FailureMsg  string
+  DebugOutput string
 }
 
 type TestCase struct {
@@ -18,6 +28,7 @@ type TestCase struct {
   Config TestCaseConfig
 
   Result *TestResult
+
 }
 
 func (tc TestCase) AdapterConfig() InheritableConfig {