Refactor gtest adapter to fit new pipeline

10 files changed, 289 insertions, 229 deletions

diff --git a/adapters.go b/adapters.go
index b7c4b27..f6c48cb 100644
--- a/adapters.go
+++ b/adapters.go
@@ -17,7 +17,7 @@ type Adapter interface {
   Build(testCase []TestCase)
 
   // Called every time source changes
-  Evaluate(testCase []TestCase) []TestCase
+  Evaluate(testCase []TestCase) []TestResult
 }
 
 // A parser function takes a blob of TOML and decodes it into
diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go
index 7033c7f..ec11748 100644
--- a/adapters/gtest/adapter.go
+++ b/adapters/gtest/adapter.go
@@ -1,34 +1,14 @@
 package gtest
 
 import (
-	"context"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
-	"os/exec"
 	"path"
-	"sync"
-	"time"
 	"golang.flu0r1ne.net/planr"
 )
 
 const GTEST_CMAKE = "CMakeLists.txt"
 
-func makeUnit(tc planr.TestCase, dirs planr.DirConfig) cmakeUnit {
-  cfg := tc.AdapterConfig().(*Config)
-
-  testpath := path.Join(dirs.Tests(), *cfg.Testfile)
-  srclist := cfg.srcList(dirs.Src())
-
-  return cmakeUnit {
-    tc.Cname,
-    testpath,
-    srclist,
-  };
-}
-
 func safeWd() string{
   wd, err := os.Getwd()
 
@@ -39,104 +19,6 @@ func safeWd() string{
   return wd
 }
 
-type ResultFromId map[string] Result
-
-func (adapter *Adapter) execTests(cnames []string) ResultFromId {
-  buildDir := safeWd()
-
-  lut := make(ResultFromId, 0)
-  for _, exe := range cnames {
-
-      exePath := path.Join(buildDir, exe)
-
-      f, err := ioutil.TempFile(buildDir, "gtest_adapter_*.json")
-
-      if err != nil {
-        log.Fatal(err)
-      }
-
-      ctx, cancel := context.WithTimeout(context.Background(), 9999*time.Millisecond)
-      cmd := exec.CommandContext(ctx, exePath, "--gtest_output=json:" + f.Name())
-
-      defer cancel()
-      defer os.Remove(f.Name())
-
-      out, err := cmd.CombinedOutput()
-      if err != nil {
-        var exiterr *exec.ExitError
-
-        if !errors.As(err, &exiterr) {
-          log.Printf("%v\n", err)
-          os.Exit(exiterr.ExitCode())
-        }
-      }
-
-      results, err := decodeResults(f)
-
-      if err != nil {
-        log.Printf("Could not collect results from %s: %v", exe, err)
-        continue
-      }
-
-      for _, r  := range results {
-        r.testOutput = string(out)
-        lut[exe + "." + r.id] = r
-      }
-  }
-
-  return lut
-}
-
-// An executable may contain more than one test
-// Gather all executables and deduplicate them
-func exes(tcs []planr.TestCase) []string {
-  set := make(map[string] bool, 0)
-  
-  for _, tc := range tcs {
-    // Tests which have encountered a failure
-    // may not have an executable
-    if tc.Result.Status != planr.PASSING {
-      continue
-    }
-
-    if(!set[tc.Cname]) {
-      set[tc.Cname] = true
-    }
-  }
- 
-  exes := make([]string, 0)
-  for k := range set {
-    exes = append(exes, k)
-  }
-  
-  return exes
-}
-
-func id(tc planr.TestCase) string {
-  cfg := tc.AdapterConfig().(*Config)
-  return tc.Cname + "." + *cfg.Suite + "." + *cfg.Name
-}
-
-func compile(wg * sync.WaitGroup, tc * planr.TestCase) {
-  defer wg.Done()
-
-  cmd := exec.Command("make", tc.Cname)
-  out, err := cmd.CombinedOutput()
-  tc.Result = new(planr.TestResult)
-
-  // Don't treat command failure as anything but a build failure
-  if err != nil{
-    var exiterr *exec.ExitError
-    if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 {
-      log.Fatal(err)
-    }
-
-    tc.Result.Status = planr.COMPILATION_FAILURE
-  }
-
-  tc.Result.DebugOutput = string(out)
-}
-
 type Adapter struct {
   dirs planr.DirConfig
 }
@@ -155,67 +37,42 @@ func (a *Adapter) Init(dirs planr.DirConfig) {
 
 func (adapter Adapter) Build(tcs []planr.TestCase) {
   buildDir := safeWd()
-  cmakeFile := path.Join(buildDir, GTEST_CMAKE)
 
-  units := make([]cmakeUnit, 0)
-  for _, tc := range tcs {
-    
-    cfg := tc.AdapterConfig().(*Config)
-    cfg.ensureSatisfied(tc.Path)
+  finalizeConfigs(tcs)
 
-    units = append(units, makeUnit(tc, adapter.dirs))
-  }
+  exes := createExecutables(tcs)
 
-  genCmake(cmakeFile, units)
+  cmakeFile := path.Join(buildDir, GTEST_CMAKE)
+  cmakeUnits := cmakeUnits(exes, adapter.dirs)
+
+  generateCmakeScript(cmakeFile, cmakeUnits)
 
   planr.RunCmd("cmake", "-S", ".", "-B", ".")
 }
 
-// ./planr eval  0.93s user 0.16s system 100% cpu 1.089 total
-func (adapter *Adapter) Evaluate(tcs []planr.TestCase) [] planr.TestCase {
-  var wg sync.WaitGroup
-  for i := range tcs {
-    tc := &tcs[i]
-    wg.Add(1)
-    go compile(&wg, tc)
-  }
-  wg.Wait()
-
-  files := exes(tcs)
-  resultById := adapter.execTests(files)
-
-  for i := range tcs {
-    tc := &tcs[i]
-    result, ok := resultById[id(*tc)]
-
-    // compilation failure 
-    if !ok {
-
-      if tc.Result.Status == planr.PASSING {
-        cfg := tc.AdapterConfig().(*Config)
+func (adapter *Adapter) Evaluate(tcs []planr.TestCase) [] planr.TestResult {
+  buildDir := safeWd()
 
-        log.Printf(
-          "Could not find testcase %s with name=\"%s\" and suite=\"%s\". Does such a test exist in the test source?",
-          tc.Cname,
-          *cfg.Name,
-          *cfg.Suite,
-        )
+  finalizeConfigs(tcs)
+  
+  results := make([]planr.TestResult, 0)
+  
+  exes := createExecutables(tcs)
 
-        tc.Result.Status = planr.COMPILATION_FAILURE
-        tc.Result.DebugOutput += fmt.Sprintf("planr: Did not find testcase %s in any test executable\n", id(*tc))
-      }
+  for i := range exes {
+    succeed, buildFailures := exes[i].compile(buildDir)
 
+    if ! succeed {
+      results = append(results, buildFailures...)
       continue
     }
- 
-    if !result.pass {
-      tc.Result.Status = planr.RUNTIME_FAILURE
-    }
 
-    tc.Result.TestOutput = result.testOutput
-  }
+    runtimeResults := exes[i].execute(buildDir)
 
-  return tcs
+    results = append(results, runtimeResults...)
+  }
+ 
+  return results
 }
 
 func NewAdapter() *Adapter {
diff --git a/adapters/gtest/config.go b/adapters/gtest/config.go
index 3305977..04d426c 100644
--- a/adapters/gtest/config.go
+++ b/adapters/gtest/config.go
@@ -2,10 +2,10 @@ package gtest
 
 import (
   "log"
-  "golang.flu0r1ne.net/planr"
   "strings"
-  "github.com/BurntSushi/toml"
   "path"
+  "golang.flu0r1ne.net/planr"
+  "github.com/BurntSushi/toml"
 )
 
 const (
@@ -16,7 +16,7 @@ type Defaults struct {
   Name          *string
   Suite         *string
   Testfile      *string
-  Srcs          *[]string
+  Srcs          []string
   Timeout       *uint
 }
 
@@ -26,7 +26,7 @@ func (child *Defaults) Inherit(p interface{}) {
   if(child.Name == nil)          { child.Name = parent.Name }
   if(child.Suite == nil)         { child.Suite = parent.Suite }
   if(child.Testfile == nil)      { child.Testfile = parent.Testfile }
-  if(child.Srcs == nil)          { child.Srcs = parent.Srcs }
+  if(len(child.Srcs) == 0)       { child.Srcs = parent.Srcs }
   if(child.Timeout == nil)       { child.Timeout = parent.Timeout }
 }
 
@@ -35,34 +35,52 @@ type Config struct {
   Defaults
 }
 
-func (g Config) ensureSatisfied(path string) {
-  if g.Name == nil {
+func (c * Config) finalize(path string) {
+  if c.Name == nil {
     log.Fatalf("\"name\" is not defined for unit: %s\n", path)
-  } else if g.Suite == nil {
+  } else if c.Suite == nil {
     log.Fatalf("\"suite\" is not defined for unit: %s\n", path)
-  } else if g.Testfile == nil {
+  } else if c.Testfile == nil {
     log.Fatalf("\"testfile\" is not defined for unit: %s\n", path)
   }
 
-  if g.Timeout == nil {
-    g.Timeout = new(uint)
-    *g.Timeout = DEFAULT_TIMEOUT;
+  if c.Timeout == nil {
+    c.Timeout = new(uint)
+    *c.Timeout = DEFAULT_TIMEOUT;
   }
 }
 
-func (cfg Config) srcList(srcDir string) string {
-  var srcList string
+func srcList(srcdir string, srcs []string) string {
+  builder := strings.Builder {}
 
-  if cfg.Srcs != nil {
-    srcs := make([]string, len(*cfg.Srcs))
-    for i, src := range *cfg.Srcs {
-      srcs[i] = "\"" + path.Join(srcDir, src) + "\""
-    }
+  for _, src := range srcs {
+    builder.WriteString("\"")
+    builder.WriteString(path.Join(srcdir, src))
+    builder.WriteString("\"\n  ")
+  }
+  
+  return builder.String()
+}
+
+func cmakeUnits(e []executable, dirs planr.DirConfig) []cmakeUnit {
+  
+  units := make([]cmakeUnit, len(e))
+  for i, exe := range e {
+    testpath := path.Join(dirs.Tests(), exe.testpath)
+    srclist  := srcList(dirs.Src(), exe.srcs)
     
-    srcList = strings.Join(srcs, "\n  ")
+    units[i] = cmakeUnit { exe.exeNm, testpath, srclist }
   }
 
-  return srcList
+  return units
+}
+
+func finalizeConfigs(tcs []planr.TestCase) {
+  for i := range tcs {
+    cfg := tcs[i].AdapterConfig().(*Config)
+
+    cfg.finalize(tcs[i].Path)
+  }
 }
 
 func ParseConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
diff --git a/adapters/gtest/executable.go b/adapters/gtest/executable.go
new file mode 100644
index 0000000..78b0b56
--- /dev/null
+++ b/adapters/gtest/executable.go
@@ -0,0 +1,186 @@
+package gtest
+
+import (
+        "os"
+	"errors"
+        "time"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"path"
+	"reflect"
+	"sort"
+        "context"
+
+	"golang.flu0r1ne.net/planr"
+)
+
+type executable struct {
+  exeNm  string
+  testpath string
+  srcs     []string
+  tcs      []planr.TestCase
+}
+
+func createExecutables(tcs []planr.TestCase) []executable {
+  exes := make(map[string] executable, 0)
+
+  for _, tc := range tcs {
+    cfg := tc.AdapterConfig().(*Config)
+    file := *cfg.Testfile
+    exe, contained := exes[file]
+
+    // For set comparison
+    sort.Strings(cfg.Srcs)
+
+    if !contained {
+      exeTcs := make([]planr.TestCase, 1)
+      exeTcs[0] = tc
+
+
+      exe := executable {
+        planr.Cname("", file),
+        file,
+        cfg.Srcs,
+        exeTcs,
+      }
+
+      exes[file] = exe
+      
+      continue
+    }
+
+    // We could create two different executables for each source list
+    // But, that would be confusing so we're going to disallow it
+    if !reflect.DeepEqual(exe.srcs, cfg.Srcs) {
+      log.Fatalf(
+        "Two test case definitions %s and %s have different lists of sources", 
+        exe.testpath, *cfg.Testfile,
+      )
+    }
+
+    exe.tcs = append(exe.tcs, tc)
+    
+    exes[file] = exe
+  }
+
+  exesList := make([]executable, 0)
+
+  for _, exe := range exes {
+    exesList = append(exesList, exe)
+  }
+
+  return exesList
+}
+
+func (exe executable) compile(builddir string) (succeeded bool, buildFailures []planr.TestResult) {
+  cmd := exec.Command("make", "-C", builddir, exe.exeNm)
+  out, err := cmd.CombinedOutput()
+  buildFailures = make([]planr.TestResult, 0)
+
+  outputLog := string(out)
+
+  if err != nil{
+    var exiterr *exec.ExitError
+    if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 {
+      log.Fatalf("Unrecoverable build failure: %v", err)
+    }
+
+    for i := range exe.tcs {
+      res := planr.TestResult {}
+      res.Tc = exe.tcs[i]
+      res.DebugOutput = outputLog
+      res.Status = planr.COMPILATION_FAILURE
+
+      buildFailures = append(buildFailures, res)
+    }
+
+    succeeded = false
+
+    return
+  }
+
+  succeeded = true
+  return
+}
+
+const TMPFILENAME = "gtest_adapter_*.json"
+
+func runGtest(exe string, tc planr.TestCase, builddir string) planr.TestResult {
+  result := planr.TestResult {}
+  result.Tc = tc
+
+  exePath := path.Join(builddir, exe)
+  cfg := tc.AdapterConfig().(*Config)
+
+  f, err := ioutil.TempFile(builddir, TMPFILENAME)
+      
+  if err != nil {
+    log.Fatal(err)
+  }
+
+  timeout := time.Duration(*cfg.Timeout) * time.Millisecond
+
+  ctx, cancel := context.WithTimeout(context.Background(), timeout)
+
+  jsonFlag := "--gtest_output=json:" + f.Name()
+  testFlag := "--gtest_filter=" + *cfg.Suite + "." + *cfg.Name
+
+  cmd := exec.CommandContext(ctx, exePath, jsonFlag, testFlag)
+
+  defer cancel()
+  defer os.Remove(f.Name())
+
+  out, err := cmd.CombinedOutput()
+  if err != nil {
+    var exiterr *exec.ExitError
+
+    if !errors.As(err, &exiterr) {
+      log.Printf("%v\n", err)
+      os.Exit(exiterr.ExitCode())
+    }
+  }
+
+  results, err := decodeResults(f)
+
+  if err != nil {
+    log.Fatalf("Could not collect results from %s: %v", exe, err)
+  }
+
+  if len(results) < 1 {
+    log.Fatalf(
+      "Could not find testcase %s with name=\"%s\" and suite=\"%s\". Does such a test exist in the test source?",
+      tc.Cname,
+      *cfg.Name,
+      *cfg.Suite,
+    )
+  }
+
+  // TODO: Cleanup -- ZERO TESTS?
+  if len(results) > 1 {
+    log.Fatalf("Unexpected number of results")
+  }
+
+  decodeResult := results[0]
+  
+  result.TestOutput = string(out)
+
+  if decodeResult.pass {
+    result.Status = planr.PASSING
+  } else {
+    result.Status = planr.RUNTIME_FAILURE
+  }
+
+  return result
+}
+
+func (exe executable) execute(builddir string) []planr.TestResult {
+  results := make([]planr.TestResult, len(exe.tcs))
+
+  for i := range exe.tcs {
+    results[i] = runGtest(exe.exeNm, exe.tcs[i], builddir)
+  }
+
+  return results
+}
+
diff --git a/adapters/gtest/templating.go b/adapters/gtest/templating.go
index a9a3b07..57532fa 100644
--- a/adapters/gtest/templating.go
+++ b/adapters/gtest/templating.go
@@ -8,12 +8,12 @@ import (
 )
 
 type cmakeUnit struct {
-  Cname string
+  ExeNm string
   File  string
   Srcs  string
 };
 
-func genCmake(out string, units []cmakeUnit) {
+func generateCmakeScript(out string, units []cmakeUnit) {
   file, err := os.OpenFile(out, os.O_RDWR | os.O_CREATE, 0644)
   defer func () {
     err := file.Close()
@@ -33,27 +33,29 @@ func genCmake(out string, units []cmakeUnit) {
 
   for _, unit := range units {
     if err := tmpl.Execute(file, unit); err != nil {
-      log.Fatalf("Failed to generate unit %s: %v", unit.Cname, err);
+      log.Fatalf("Failed to generate unit %s: %v", unit.ExeNm, err);
     }
   }
 }
 
 
+// TODO: Add comments
 func unitTemplate() *template.Template {
   tmpl, err := template.New("gtest_unit").Parse(`
+
 add_executable(
-  "{{.Cname}}"
+  "{{.ExeNm}}"
   "{{.File}}"
   {{.Srcs}}
 )
 
 target_link_libraries(
-  "{{.Cname}}"
+  "{{.ExeNm}}"
   gtest_main
 )
 
 gtest_discover_tests(
-  "{{.Cname}}"
+  "{{.ExeNm}}"
 )
 `)
 
diff --git a/cmd/planr/sub/cli.go b/cmd/planr/sub/cli.go
index d667b88..3c58f4e 100644
--- a/cmd/planr/sub/cli.go
+++ b/cmd/planr/sub/cli.go
@@ -14,25 +14,23 @@ var (
   col_label = color.New(color.FgCyan)
 );
 
-func tcTitle(tc planr.TestCase) string {
-  title := tc.Cname
+func tcTitle(tr planr.TestResult) string {
+  title := tr.Tc.Cname
 
-  if tc.Config.Title != nil {
-    title = *tc.Config.Title
+  if tr.Tc.Config.Title != nil {
+    title = *tr.Tc.Config.Title
   }
 
   return title
 }
 
-func tcStatus(tc planr.TestCase) string {
+func tcStatus(tc planr.TestResult) string {
     status := "SILENT"
 
-    if tc.Result != nil {
-      if tc.Result.Status == planr.PASSING {
-        status = "PASS"
-      } else {
-        status = "FAIL"
-      }
+    if tc.Status == planr.PASSING {
+      status = "PASS"
+    } else {
+      status = "FAIL"
     }
 
     return status
@@ -59,9 +57,9 @@ func pprintFenced(title, value string) {
   fmt.Println(fence)
 }
 
-func tcStatusLine(tc planr.TestCase) {
-  title  := tcTitle(tc)
-  status := tcStatus(tc)
+func tcStatusLine(tr planr.TestResult) {
+  title  := tcTitle(tr)
+  status := tcStatus(tr)
 
   if status == "PASS" {
     col_pass.Printf("[%s] ", status);
@@ -72,8 +70,10 @@ func tcStatusLine(tc planr.TestCase) {
   col_title.Println(title);  
 }
 
-func tcPprint(tc planr.TestCase) {
-  tcStatusLine(tc)
+func tcPprint(tr planr.TestResult) {
+  tcStatusLine(tr)
+
+  tc := tr.Tc
 
   pprintLabeled("id", tc.Cname)
 
@@ -86,22 +86,20 @@ func tcPprint(tc planr.TestCase) {
     pprintLabeled("description", *tc.Config.Description)
   }
 
-  res := tc.Result
-
-  if res.Status == planr.COMPILATION_FAILURE {
+  if tr.Status == planr.COMPILATION_FAILURE {
     
-    if res.DebugOutput != "" {
+    if tr.DebugOutput != "" {
       fmt.Println()
-      pprintFenced("compilation output", tc.Result.DebugOutput);
+      pprintFenced("compilation output", tr.DebugOutput);
     } else {
       fmt.Println("WARN: No debug output provided")
     }
 
-  } else if res.Status == planr.RUNTIME_FAILURE {
+  } else if tr.Status == planr.RUNTIME_FAILURE {
 
-    if tc.Result.TestOutput != "" {
+    if tr.TestOutput != "" {
       fmt.Println()
-      pprintFenced("test output", tc.Result.TestOutput);
+      pprintFenced("test output", tr.TestOutput);
     }
     
   }
diff --git a/cmd/planr/sub/evaluate.go b/cmd/planr/sub/evaluate.go
index 0366d44..18cccb1 100644
--- a/cmd/planr/sub/evaluate.go
+++ b/cmd/planr/sub/evaluate.go
@@ -6,26 +6,26 @@ import (
 
 func Evaluate(runner planr.Runner, params []string) {
   tcs := runner.CollectCases()
-  tcs = runner.Evaluate(tcs)
+  trs := runner.Evaluate(tcs)
 
   earned := 0.0
   total  := 0.0
   passed := 0
-  for _, tc := range tcs {
-    cfg := tc.Config
+  for _, tr := range trs {
+    cfg := tr.Tc.Config
 
     if cfg.Points != nil {
       points := float64(*cfg.Points)
 
       total += points
 
-      if tc.Result.Status == planr.PASSING {
+      if tr.Status == planr.PASSING {
         earned += points
         passed++
       }
     }
 
-    tcPprint(tc)
+    tcPprint(tr)
   }
 
   printResults(
diff --git a/fs.go b/fs.go
index 575517c..a42ff2c 100644
--- a/fs.go
+++ b/fs.go
@@ -50,7 +50,7 @@ func basename(path string) string {
   return path[0:len(path) - len(ext)]
 }
 
-func cname(root string, path string) string {
+func Cname(root string, path string) string {
   rel, err := filepath.Rel(root, path) 
   
   if err != nil {
@@ -76,7 +76,7 @@ func collectUnits(root string, cfgs []AdapterConfig) []TestCase {
   collectFromDir(root, nil, cfgs, &tcs)
 
   for i := range tcs {
-    tcs[i].Cname = cname(root, tcs[i].Path)
+    tcs[i].Cname = Cname(root, tcs[i].Path)
   }
 
   return tcs
diff --git a/runner.go b/runner.go
index f613d44..2d66dc6 100644
--- a/runner.go
+++ b/runner.go
@@ -108,9 +108,9 @@ func (r Runner) Build(tcs []TestCase) {
   safeCd(r.dirs.Config())
 }
 
-func (r Runner) Evaluate(tcs []TestCase) []TestCase {
+func (r Runner) Evaluate(tcs []TestCase) []TestResult {
   testSets := r.groupByAdapter(tcs)
-  results := make([]TestCase, 0)
+  results := make([]TestResult, 0)
 
   for _, pair := range testSets {
     adapter := pair.adapter
diff --git a/testcase.go b/testcase.go
index 7e0bf17..19f1e58 100644
--- a/testcase.go
+++ b/testcase.go
@@ -7,7 +7,8 @@ import (
 type TestStatus uint
 
 const (
-  PASSING  TestStatus = iota
+  NOT_RUN TestStatus = iota
+  PASSING  
   COMPILATION_FAILURE
   RUNTIME_FAILURE
 )
@@ -17,6 +18,7 @@ type TestResult struct {
   Status      TestStatus
   DebugOutput string
   TestOutput  string
+  Tc          TestCase
 }
 
 // Program-wide testcase config
@@ -43,9 +45,6 @@ type TestCase struct {
   Cname  string
   
   Config TestCaseConfig
-
-  Result *TestResult
-
 }
 
 func (tc TestCase) AdapterConfig() InheritableConfig {