Improvised a basic text report, finished evaluation mappings

7 files changed, 150 insertions, 46 deletions

diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go
index 09e6ac1..600c956 100644
--- a/adapters/gtest/adapter.go
+++ b/adapters/gtest/adapter.go
@@ -16,7 +16,7 @@ import (
 const GTEST_CMAKE = "CMakeLists.txt"
 
 func mkUnit(tc *planr.TestCase) cmakeUnit {
-  cfg := tc.AdapterConfig("gtest").(*GtestConfig)
+  cfg := tc.AdapterConfig().(*GtestConfig)
 
   return cmakeUnit {
     tc.Cname,
@@ -49,7 +49,7 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) {
   for _, tc := range tcs {
     
     fmt.Printf("[R] Building %s (%s)\n", tc.Cname, tc.Path)
-    cfg := tc.AdapterConfig("gtest").(*GtestConfig)
+    cfg := tc.AdapterConfig().(*GtestConfig)
     cfg.ensureSatisfied(tc.Path)
 
     units = append(units, mkUnit(tc))
@@ -61,15 +61,16 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) {
   planr.RunCmd("cmake", "-S", ".", "-B", ".")
 }
 
-func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
-  planr.RunCmd("make", "-k")
+type ResultFromId map[string] planr.TestResult
+
+func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
   buildDir := adapter.Config().Dir()
 
-  results := make([]planr.TestResult, 0)
-  for _, tc := range tcs {
-      fmt.Printf("[R] Evaluating %s (%s)\n", tc.Cname, tc.Path)
+  lut := make(ResultFromId, 0)
+  for _, exe := range cnames {
+      fmt.Printf("[R] Evaluating %s\n", exe)
 
-      exe := path.Join(buildDir, tc.Cname)
+      exePath := path.Join(buildDir, exe)
 
       f, err := ioutil.TempFile(buildDir, "gtest_adapter_*.json")
 
@@ -78,7 +79,7 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
       }
 
       ctx, cancel := context.WithTimeout(context.Background(), 9999*time.Millisecond)
-      cmd := exec.CommandContext(ctx, exe, "--gtest_output=json:" + f.Name())
+      cmd := exec.CommandContext(ctx, exePath, "--gtest_output=json:" + f.Name())
 
       defer cancel()
       defer os.Remove(f.Name())
@@ -96,11 +97,50 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
       }
 
       if exitFail {
-        fmt.Printf("Failure detected")
+        fmt.Printf("")
       }
 
-      results = append(results, decodeResults(f)...)
+      for _, r  := range decodeResults(f) {
+        lut[exe + "." + r.id] = r.result
+      }
   }
 
-  fmt.Println(results)
+  return lut
+}
+
+func exes(tcs []*planr.TestCase) []string {
+  set := make(map[string] bool, 0)
+  
+  for _, tc := range tcs {
+    if(!set[tc.Cname]) {
+      set[tc.Cname] = true
+    }
+  }
+ 
+  exes := make([]string, 0)
+  for k := range set {
+    exes = append(exes, k)
+  }
+  
+  return exes
+}
+
+func id(tc *planr.TestCase) string {
+  cfg := tc.AdapterConfig().(*GtestConfig)
+  return tc.Cname + "." + *cfg.Suite + "." + *cfg.Name
+}
+
+func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
+  buildDir := adapter.Config().Dir()
+  chdir(buildDir)
+
+  planr.RunCmd("make", "-k")
+
+  files := exes(tcs)
+  resultById := adapter.execTests(files)
+
+  for _, tc := range tcs {
+    result := resultById[id(tc)]
+    tc.Result = &result 
+  }
 }
diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go
index e5280e3..88e4069 100644
--- a/adapters/gtest/results.go
+++ b/adapters/gtest/results.go
@@ -47,7 +47,12 @@ type gResults struct {
   Testsuites []gTestsuites `json:"testsuites"`
 }
 
-func decodeResults(r io.Reader) []planr.TestResult {
+type Result struct {
+  id string
+  result planr.TestResult
+}
+
+func decodeResults(r io.Reader) []Result {
   var results gResults
   buf := bytes.Buffer{}
 
@@ -59,14 +64,18 @@ func decodeResults(r io.Reader) []planr.TestResult {
     log.Fatal(err)
   }
 
-  decoded := make([]planr.TestResult, 0)
+  decoded := make([]Result, 0)
   for _, suite := range results.Testsuites {
     for _, test := range suite.Testsuite {
       n := len(test.Failures)
 
-      decoded = append(decoded, planr.TestResult {
-        Pass: n == 0,
+      decoded = append(decoded, Result{
+        id: suite.Name + "." + test.Name,
+        result: planr.TestResult {
+          Pass: n == 0,
+        },
       })
+
     }
   }
 
diff --git a/cmd/sub/build.go b/cmd/sub/build.go
index 8108768..58c3a38 100644
--- a/cmd/sub/build.go
+++ b/cmd/sub/build.go
@@ -5,13 +5,15 @@ import (
 	"golang.flu0r1ne.net/planr/adapters/gtest"
 )
 
-func Build(params []string) {
-  gtestAdapter := gtest.GtestAdapter {}
+func Runner() planr.Runner {
+  r := planr.Runner {}
+  r.RegisterAdapter(&gtest.GtestAdapter{})
+  return r
+}
 
-  r := planr.Runner{}
-  r.RegisterAdapter(&gtestAdapter)
+func Build(params []string) {
 
   rd := planr.RubricDir()
 
-  r.Run(rd)
+  Runner().Build(rd)
 }
diff --git a/cmd/sub/evaluate.go b/cmd/sub/evaluate.go
index 35dd48d..a0f858b 100644
--- a/cmd/sub/evaluate.go
+++ b/cmd/sub/evaluate.go
@@ -2,8 +2,44 @@ package sub
 
 import (
   "fmt"
+  "golang.flu0r1ne.net/planr"
 )
 
 func Evaluate(params []string) {
-  fmt.Print(params) 
+  rd := planr.RubricDir()
+
+  tcs := Runner().Evaluate(rd)
+
+  fmt.Printf("\n\nREPORT:\n=======\n\n")
+
+  for _, tc := range tcs {
+    cfg := tc.Config
+  
+    name := tc.Cname
+    if cfg.Title != nil {
+      name = *cfg.Title
+    }
+
+    status := "NOT RUN"
+    if tc.Result != nil {
+      if tc.Result.Pass {
+        status = "PASS"
+      } else {
+        status = "FAIL"
+      }
+    }
+
+    var points float32 =  0.0
+    if cfg.Points != nil {
+      points = *cfg.Points
+    }
+
+    fmt.Printf("[%s] %s (%f)\n", status, name, points)
+
+    if cfg.Description != nil {
+      fmt.Printf("> %s\n", *cfg.Description)
+    }
+
+    fmt.Println()
+  }
 }
diff --git a/config.go b/config.go
index 6dd264b..6efa90d 100644
--- a/config.go
+++ b/config.go
@@ -60,6 +60,7 @@ type AdapterConfig struct {
 // in defaults.toml
 type Defaults struct {
   Points      *float32
+  Adapter     *string
 
   /*
     The TOML library only parses exported fields.
@@ -89,6 +90,12 @@ type TestCaseConfig struct {
   Description *string
 }
 
+func (c TestCaseConfig) ensureSatisfied(name string) {
+  if (c.Adapter == nil) {
+    log.Fatalf("Adapter must be provided for testcase %s", name)
+  }
+}
+
 // The default configuration must be able in inherit from
 // other defaults further up the tree
 //
@@ -101,6 +108,10 @@ func (child *Defaults) Inherit(p interface{}) {
     child.Points = parent.Points;
   }
 
+  if child.Adapter == nil {
+    child.Adapter = parent.Adapter;
+  } 
+
   // Call the inherit method as defined by the adapters
   // If an adapter is undefined, inherit the parent configuration
   //
diff --git a/runner.go b/runner.go
index 5b76cf1..96bcd19 100644
--- a/runner.go
+++ b/runner.go
@@ -27,15 +27,22 @@ func (r Runner) buildTcLUT(tcs []TestCase) TcTab {
 
   for i := range tcs {
     tc := &tcs[i]
-    for nm := range tc.Config.adapters_ {
-      m[nm] = append(m[nm], tc)
-    }
+    nm := *tc.Config.Adapter
+    m[nm] = append(m[nm], tc)
   }
 
   return m 
 }
 
-func (r Runner) Build(tcs []TestCase) {
+func (r Runner) checkConfig(tcs []TestCase) {
+  for _, tc := range tcs {
+    tc.Config.ensureSatisfied(tc.Path)
+  }
+}
+
+func (r Runner) build(tcs []TestCase) {
+  r.checkConfig(tcs)
+
   tcTab := r.buildTcLUT(tcs)
 
   for _, adapter := range r.adapters {
@@ -46,9 +53,17 @@ func (r Runner) Build(tcs []TestCase) {
   }
 }
 
-func (r Runner) Evaluate(tcs []TestCase) []TestResult {
+func (r Runner) units(root string) []TestCase {
+  return collectUnits(root, r.adapterCfgs())
+}
+
+func (r Runner) Build(root string) {
+  units := r.units(root) 
+  r.build(units)
+}
+
+func (r Runner) evaluate(tcs []TestCase) {
   tcTab := r.buildTcLUT(tcs)
-  results := make([]TestResult, 0)
   
   for _, adapter := range r.adapters {
     nm := adapter.Config().Name
@@ -57,15 +72,12 @@ func (r Runner) Evaluate(tcs []TestCase) []TestResult {
     adapter.Evaluate(tcTab[nm])
   }
 
-  return results
 }
 
+func (r Runner) Evaluate(root string) []TestCase {
+  units := r.units(root)
 
-func (r Runner) Run(root string) [] TestResult {
-  tcs := collectUnits(root, r.adapterCfgs())
-
-  r.Build(tcs)
-  trs := r.Evaluate(tcs)
+  r.evaluate(units)
 
-  return trs
+  return units
 }
diff --git a/testcase.go b/testcase.go
index 70c188e..989838c 100644
--- a/testcase.go
+++ b/testcase.go
@@ -1,6 +1,7 @@
 package planr
 
 type TestResult struct {
+  Id   string
   Pass bool
 }
 
@@ -13,19 +14,12 @@ type TestCase struct {
   // configuration with dots. The `toml` extension is also stripped.
   // rubric/alpha/beta/tc1.toml -> alpha.beta.tc1
   Cname  string
+  
   Config TestCaseConfig
-}
-
-func (tc TestCase) ContainsAdapter(name string) bool {
-  for adapter := range tc.Config.adapters_ {
-    if adapter == name {
-      return true;
-    }
-  }
 
-  return false;
+  Result *TestResult
 }
 
-func (tc TestCase) AdapterConfig(name string) InheritableConfig {
-  return tc.Config.adapters_[name]
+func (tc TestCase) AdapterConfig() InheritableConfig {
+  return tc.Config.adapters_[*tc.Config.Adapter]
 }