From cbc5fbdf941cd40dc6a9546fb924689f5d6978c7 Mon Sep 17 00:00:00 2001 From: Flu0r1ne Date: Thu, 5 Aug 2021 18:41:53 -0500 Subject: Improvised a basic text report, finished evaluation mappings --- adapters/gtest/adapter.go | 64 ++++++++++++++++++++++++++++++++++++++--------- adapters/gtest/results.go | 17 ++++++++++--- cmd/sub/build.go | 12 +++++---- cmd/sub/evaluate.go | 38 +++++++++++++++++++++++++++- config.go | 11 ++++++++ runner.go | 38 ++++++++++++++++++---------- testcase.go | 16 ++++-------- 7 files changed, 150 insertions(+), 46 deletions(-) diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go index 09e6ac1..600c956 100644 --- a/adapters/gtest/adapter.go +++ b/adapters/gtest/adapter.go @@ -16,7 +16,7 @@ import ( const GTEST_CMAKE = "CMakeLists.txt" func mkUnit(tc *planr.TestCase) cmakeUnit { - cfg := tc.AdapterConfig("gtest").(*GtestConfig) + cfg := tc.AdapterConfig().(*GtestConfig) return cmakeUnit { tc.Cname, @@ -49,7 +49,7 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) { for _, tc := range tcs { fmt.Printf("[R] Building %s (%s)\n", tc.Cname, tc.Path) - cfg := tc.AdapterConfig("gtest").(*GtestConfig) + cfg := tc.AdapterConfig().(*GtestConfig) cfg.ensureSatisfied(tc.Path) units = append(units, mkUnit(tc)) @@ -61,15 +61,16 @@ func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) { planr.RunCmd("cmake", "-S", ".", "-B", ".") } -func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { - planr.RunCmd("make", "-k") +type ResultFromId map[string] planr.TestResult + +func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId { buildDir := adapter.Config().Dir() - results := make([]planr.TestResult, 0) - for _, tc := range tcs { - fmt.Printf("[R] Evaluating %s (%s)\n", tc.Cname, tc.Path) + lut := make(ResultFromId, 0) + for _, exe := range cnames { + fmt.Printf("[R] Evaluating %s\n", exe) - exe := path.Join(buildDir, tc.Cname) + exePath := path.Join(buildDir, exe) f, err := ioutil.TempFile(buildDir, "gtest_adapter_*.json") @@ -78,7 +79,7 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { } ctx, cancel := context.WithTimeout(context.Background(), 9999*time.Millisecond) - cmd := exec.CommandContext(ctx, exe, "--gtest_output=json:" + f.Name()) + cmd := exec.CommandContext(ctx, exePath, "--gtest_output=json:" + f.Name()) defer cancel() defer os.Remove(f.Name()) @@ -96,11 +97,50 @@ func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { } if exitFail { - fmt.Printf("Failure detected") + fmt.Printf("") } - results = append(results, decodeResults(f)...) + for _, r := range decodeResults(f) { + lut[exe + "." + r.id] = r.result + } } - fmt.Println(results) + return lut +} + +func exes(tcs []*planr.TestCase) []string { + set := make(map[string] bool, 0) + + for _, tc := range tcs { + if(!set[tc.Cname]) { + set[tc.Cname] = true + } + } + + exes := make([]string, 0) + for k := range set { + exes = append(exes, k) + } + + return exes +} + +func id(tc *planr.TestCase) string { + cfg := tc.AdapterConfig().(*GtestConfig) + return tc.Cname + "." + *cfg.Suite + "." + *cfg.Name +} + +func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) { + buildDir := adapter.Config().Dir() + chdir(buildDir) + + planr.RunCmd("make", "-k") + + files := exes(tcs) + resultById := adapter.execTests(files) + + for _, tc := range tcs { + result := resultById[id(tc)] + tc.Result = &result + } } diff --git a/adapters/gtest/results.go b/adapters/gtest/results.go index e5280e3..88e4069 100644 --- a/adapters/gtest/results.go +++ b/adapters/gtest/results.go @@ -47,7 +47,12 @@ type gResults struct { Testsuites []gTestsuites `json:"testsuites"` } -func decodeResults(r io.Reader) []planr.TestResult { +type Result struct { + id string + result planr.TestResult +} + +func decodeResults(r io.Reader) []Result { var results gResults buf := bytes.Buffer{} @@ -59,14 +64,18 @@ func decodeResults(r io.Reader) []planr.TestResult { log.Fatal(err) } - decoded := make([]planr.TestResult, 0) + decoded := make([]Result, 0) for _, suite := range results.Testsuites { for _, test := range suite.Testsuite { n := len(test.Failures) - decoded = append(decoded, planr.TestResult { - Pass: n == 0, + decoded = append(decoded, Result{ + id: suite.Name + "." + test.Name, + result: planr.TestResult { + Pass: n == 0, + }, }) + } } diff --git a/cmd/sub/build.go b/cmd/sub/build.go index 8108768..58c3a38 100644 --- a/cmd/sub/build.go +++ b/cmd/sub/build.go @@ -5,13 +5,15 @@ import ( "golang.flu0r1ne.net/planr/adapters/gtest" ) -func Build(params []string) { - gtestAdapter := gtest.GtestAdapter {} +func Runner() planr.Runner { + r := planr.Runner {} + r.RegisterAdapter(>est.GtestAdapter{}) + return r +} - r := planr.Runner{} - r.RegisterAdapter(>estAdapter) +func Build(params []string) { rd := planr.RubricDir() - r.Run(rd) + Runner().Build(rd) } diff --git a/cmd/sub/evaluate.go b/cmd/sub/evaluate.go index 35dd48d..a0f858b 100644 --- a/cmd/sub/evaluate.go +++ b/cmd/sub/evaluate.go @@ -2,8 +2,44 @@ package sub import ( "fmt" + "golang.flu0r1ne.net/planr" ) func Evaluate(params []string) { - fmt.Print(params) + rd := planr.RubricDir() + + tcs := Runner().Evaluate(rd) + + fmt.Printf("\n\nREPORT:\n=======\n\n") + + for _, tc := range tcs { + cfg := tc.Config + + name := tc.Cname + if cfg.Title != nil { + name = *cfg.Title + } + + status := "NOT RUN" + if tc.Result != nil { + if tc.Result.Pass { + status = "PASS" + } else { + status = "FAIL" + } + } + + var points float32 = 0.0 + if cfg.Points != nil { + points = *cfg.Points + } + + fmt.Printf("[%s] %s (%f)\n", status, name, points) + + if cfg.Description != nil { + fmt.Printf("> %s\n", *cfg.Description) + } + + fmt.Println() + } } diff --git a/config.go b/config.go index 6dd264b..6efa90d 100644 --- a/config.go +++ b/config.go @@ -60,6 +60,7 @@ type AdapterConfig struct { // in defaults.toml type Defaults struct { Points *float32 + Adapter *string /* The TOML library only parses exported fields. @@ -89,6 +90,12 @@ type TestCaseConfig struct { Description *string } +func (c TestCaseConfig) ensureSatisfied(name string) { + if (c.Adapter == nil) { + log.Fatalf("Adapter must be provided for testcase %s", name) + } +} + // The default configuration must be able in inherit from // other defaults further up the tree // @@ -101,6 +108,10 @@ func (child *Defaults) Inherit(p interface{}) { child.Points = parent.Points; } + if child.Adapter == nil { + child.Adapter = parent.Adapter; + } + // Call the inherit method as defined by the adapters // If an adapter is undefined, inherit the parent configuration // diff --git a/runner.go b/runner.go index 5b76cf1..96bcd19 100644 --- a/runner.go +++ b/runner.go @@ -27,15 +27,22 @@ func (r Runner) buildTcLUT(tcs []TestCase) TcTab { for i := range tcs { tc := &tcs[i] - for nm := range tc.Config.adapters_ { - m[nm] = append(m[nm], tc) - } + nm := *tc.Config.Adapter + m[nm] = append(m[nm], tc) } return m } -func (r Runner) Build(tcs []TestCase) { +func (r Runner) checkConfig(tcs []TestCase) { + for _, tc := range tcs { + tc.Config.ensureSatisfied(tc.Path) + } +} + +func (r Runner) build(tcs []TestCase) { + r.checkConfig(tcs) + tcTab := r.buildTcLUT(tcs) for _, adapter := range r.adapters { @@ -46,9 +53,17 @@ func (r Runner) Build(tcs []TestCase) { } } -func (r Runner) Evaluate(tcs []TestCase) []TestResult { +func (r Runner) units(root string) []TestCase { + return collectUnits(root, r.adapterCfgs()) +} + +func (r Runner) Build(root string) { + units := r.units(root) + r.build(units) +} + +func (r Runner) evaluate(tcs []TestCase) { tcTab := r.buildTcLUT(tcs) - results := make([]TestResult, 0) for _, adapter := range r.adapters { nm := adapter.Config().Name @@ -57,15 +72,12 @@ func (r Runner) Evaluate(tcs []TestCase) []TestResult { adapter.Evaluate(tcTab[nm]) } - return results } +func (r Runner) Evaluate(root string) []TestCase { + units := r.units(root) -func (r Runner) Run(root string) [] TestResult { - tcs := collectUnits(root, r.adapterCfgs()) - - r.Build(tcs) - trs := r.Evaluate(tcs) + r.evaluate(units) - return trs + return units } diff --git a/testcase.go b/testcase.go index 70c188e..989838c 100644 --- a/testcase.go +++ b/testcase.go @@ -1,6 +1,7 @@ package planr type TestResult struct { + Id string Pass bool } @@ -13,19 +14,12 @@ type TestCase struct { // configuration with dots. The `toml` extension is also stripped. // rubric/alpha/beta/tc1.toml -> alpha.beta.tc1 Cname string + Config TestCaseConfig -} - -func (tc TestCase) ContainsAdapter(name string) bool { - for adapter := range tc.Config.adapters_ { - if adapter == name { - return true; - } - } - return false; + Result *TestResult } -func (tc TestCase) AdapterConfig(name string) InheritableConfig { - return tc.Config.adapters_[name] +func (tc TestCase) AdapterConfig() InheritableConfig { + return tc.Config.adapters_[*tc.Config.Adapter] } -- cgit v1.2.3