Merge branch 'upstream' into ppa

Update to 0.0.2

3 files changed, 131 insertions, 68 deletions

diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go
index 600c956..f4fde27 100644
--- a/adapters/gtest/adapter.go
+++ b/adapters/gtest/adapter.go
@@ -2,14 +2,16 @@ package gtest
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
-        "time"
 	"path"
-        "errors"
+	"sync"
+	"time"
+
 	"golang.flu0r1ne.net/planr"
 )
 
@@ -25,11 +27,6 @@ func mkUnit(tc *planr.TestCase) cmakeUnit {
   };
 }
 
-func chdir(dir string) {
-  if err := os.Chdir(dir); err != nil {
-    log.Fatal(err)
-  }
-}
 
 type GtestAdapter struct {}
 
@@ -48,7 +45,6 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) {
   units := make([]cmakeUnit, 0)
   for _, tc := range tcs {
     
-    fmt.Printf("[R] Building %s (%s)\n", tc.Cname, tc.Path)
     cfg := tc.AdapterConfig().(*GtestConfig)
     cfg.ensureSatisfied(tc.Path)
 
@@ -56,19 +52,17 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) {
   }
 
   genCmake(cmakeFile, units)
-  
-  chdir(buildDir)
+
   planr.RunCmd("cmake", "-S", ".", "-B", ".")
 }
 
-type ResultFromId map[string] planr.TestResult
+type ResultFromId map[string] Result
 
 func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
   buildDir := adapter.Config().Dir()
 
   lut := make(ResultFromId, 0)
   for _, exe := range cnames {
-      fmt.Printf("[R] Evaluating %s\n", exe)
 
       exePath := path.Join(buildDir, exe)
 
@@ -84,34 +78,44 @@ func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
       defer cancel()
       defer os.Remove(f.Name())
 
-      exitFail := false
-      if err := cmd.Run(); err != nil {
+      out, err := cmd.CombinedOutput()
+      if err != nil {
         var exiterr *exec.ExitError
 
-        if errors.As(err, &exiterr) && exiterr.ExitCode() == 1{
-          exitFail = true              
-        } else {
+        if !errors.As(err, &exiterr) {
           log.Printf("%v\n", err)
           os.Exit(exiterr.ExitCode())
         }
       }
 
-      if exitFail {
-        fmt.Printf("")
+      results, err := decodeResults(f)
+
+      if err != nil {
+        log.Printf("Could not collect results from %s: %v", exe, err)
+        continue
       }
 
-      for _, r  := range decodeResults(f) {
-        lut[exe + "." + r.id] = r.result
+      for _, r  := range results {
+        r.testOutput = string(out)
+        lut[exe + "." + r.id] = r
       }
   }
 
   return lut
 }
 
+// An executable may contain more than one test
+// Gather all executables and deduplicate them
 func exes(tcs []*planr.TestCase) []string {
   set := make(map[string] bool, 0)
   
   for _, tc := range tcs {
+    // Tests which have encountered a failure
+    // may not have an executable
+    if tc.Result.Status != planr.PASSING {
+      continue
+    }
+
     if(!set[tc.Cname]) {
       set[tc.Cname] = true
     }
@@ -130,17 +134,66 @@ func id(tc *planr.TestCase) string {
   return tc.Cname + "." + *cfg.Suite + "." + *cfg.Name
 }
 
-func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
-  buildDir := adapter.Config().Dir()
-  chdir(buildDir)
+func compile(wg * sync.WaitGroup, tc *planr.TestCase) {
+  defer wg.Done()
+
+  cmd := exec.Command("make", tc.Cname)
+  out, err := cmd.CombinedOutput()
+  tc.Result = new(planr.TestResult)
+
+  // Don't treat command failure as anything but a build failure
+  if err != nil{
+    var exiterr *exec.ExitError
+    if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 {
+      log.Fatal(err)
+    }
+
+    tc.Result.Status = planr.COMPILATION_FAILURE
+  }
 
-  planr.RunCmd("make", "-k")
+  tc.Result.DebugOutput = string(out)
+}
+
+// ./planr eval  0.93s user 0.16s system 100% cpu 1.089 total
+func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
+  var wg sync.WaitGroup
+  for _, tc := range tcs {
+    wg.Add(1)
+    go compile(&wg, tc)
+  }
+  wg.Wait()
 
   files := exes(tcs)
   resultById := adapter.execTests(files)
 
   for _, tc := range tcs {
-    result := resultById[id(tc)]
-    tc.Result = &result 
+    result, ok := resultById[id(tc)]
+
+    // compilation failure
+    if !ok {
+      fmt.Printf("CAN'T FIND %s: status %d\n", tc.Cname, tc.Result.Status)
+      
+      if tc.Result.Status == planr.PASSING {
+        cfg := tc.AdapterConfig().(*GtestConfig)
+
+        log.Printf(
+          "Could not find testcase %s with name=\"%s\" and suite=\"%s\". Does such a test exist in the test source?",
+          tc.Cname,
+          *cfg.Name,
+          *cfg.Suite,
+        )
+
+        tc.Result.Status = planr.COMPILATION_FAILURE
+        tc.Result.DebugOutput += fmt.Sprintf("planr: Did not find testcase %s in any test executable\n", id(tc))
+      }
+
+      continue
+    }
+ 
+    if !result.pass {
+      tc.Result.Status = planr.RUNTIME_FAILURE
+    }
+
+    tc.Result.TestOutput = result.testOutput
   }
 }
diff --git a/adapters/gtest/config.go b/adapters/gtest/config.go
index 6a6c8bf..cb5ba75 100644
--- a/adapters/gtest/config.go
+++ b/adapters/gtest/config.go
@@ -8,23 +8,23 @@ import (
 )
 
 type GtestDefaults struct {
-  Name      *string
-  Suite     *string
-  Testfile  *string
-  Test_root *string
-  Srcs      *[]string
-  Srcs_root *string
+  Name          *string
+  Suite         *string
+  Testfile      *string
+  Test_root     *string
+  Srcs          *[]string
+  Srcs_root     *string
 }
 
 func (child *GtestDefaults) Inherit(p interface{}) {
   parent := p.(*GtestDefaults)
 
-  if(child.Name == nil)      { child.Name = parent.Name }
-  if(child.Suite == nil)     { child.Suite = parent.Suite }
-  if(child.Testfile == nil)  { child.Testfile = parent.Testfile }
-  if(child.Test_root == nil) { child.Test_root = parent.Test_root }
-  if(child.Srcs == nil)      { child.Srcs = parent.Srcs }
-  if(child.Srcs_root == nil) { child.Srcs_root = parent.Srcs_root }
+  if(child.Name == nil)          { child.Name = parent.Name }
+  if(child.Suite == nil)         { child.Suite = parent.Suite }
+  if(child.Testfile == nil)      { child.Testfile = parent.Testfile }
+  if(child.Test_root == nil)     { child.Test_root = parent.Test_root }
+  if(child.Srcs == nil)          { child.Srcs = parent.Srcs }
+  if(child.Srcs_root == nil)     { child.Srcs_root = parent.Srcs_root }
 }
 
 
@@ -73,24 +73,22 @@ func (cfg GtestConfig) srcList() string {
   return srcList
 }
 
-func primitiveDecode(primitive toml.Primitive, config interface{}) {
-  if err := toml.PrimitiveDecode(primitive, config); err != nil {
-    log.Fatal(err)
-  }
-}
-
-func ParseConfig(prim toml.Primitive) planr.InheritableConfig {
+func ParseConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
     config := GtestConfig{}
-
-    primitiveDecode(prim, &config)
-
-    return &config
+  
+    if err := toml.PrimitiveDecode(prim, &config); err != nil {
+      return nil, err
+    }
+    
+    return &config, nil
 }
 
-func ParseDefaultConfig(prim toml.Primitive) planr.InheritableConfig {
+func ParseDefaultConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
     config := GtestDefaults{}
 
-    primitiveDecode(prim, &config)
+    if err := toml.PrimitiveDecode(prim, &config); err != nil {
+      return nil, err
+    }
 
-    return &config
+    return &config, nil
 }
diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go
index 88e4069..14f5d1d 100644
--- a/adapters/gtest/results.go
+++ b/adapters/gtest/results.go
@@ -4,10 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
-	"log"
 	"time"
-
-	"golang.flu0r1ne.net/planr"
 )
 
 type gFailure struct {
@@ -48,36 +45,51 @@ type gResults struct {
 }
 
 type Result struct {
-  id string
-  result planr.TestResult
+  id         string
+  pass       bool
+  failureMsg string
+  testOutput string
+}
+
+func failureMsg(failures []gFailure) string {
+  failure_msg := ""
+
+  for _, failure := range failures {
+    failure_msg += failure.Failure + "\n"
+  }
+
+  return failure_msg
 }
 
-func decodeResults(r io.Reader) []Result {
+func decodeResults(r io.Reader) ([]Result, error) {
+  decoded := make([]Result, 0)
+  
   var results gResults
   buf := bytes.Buffer{}
 
   if _, err := buf.ReadFrom(r); err != nil {
-    log.Fatal(err) 
+    return decoded, err
   }
 
   if err := json.Unmarshal(buf.Bytes(), &results); err != nil {
-    log.Fatal(err)
+    return decoded, err
   }
 
-  decoded := make([]Result, 0)
   for _, suite := range results.Testsuites {
     for _, test := range suite.Testsuite {
       n := len(test.Failures)
 
-      decoded = append(decoded, Result{
+
+      res := Result {
         id: suite.Name + "." + test.Name,
-        result: planr.TestResult {
-          Pass: n == 0,
-        },
-      })
+        pass: n == 0,
+        failureMsg: failureMsg(test.Failures),
+      }
+
 
+      decoded = append(decoded, res)
     }
   }
 
-  return decoded
+  return decoded, nil
 }