summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--CHANGELOG.md52
-rw-r--r--Makefile8
-rw-r--r--adapters.go28
-rw-r--r--adapters/bash/adapter.go103
-rw-r--r--adapters/bash/config.go64
-rw-r--r--adapters/gtest/adapter.go202
-rw-r--r--adapters/gtest/config.go97
-rw-r--r--adapters/gtest/executable.go185
-rw-r--r--adapters/gtest/templating.go57
-rw-r--r--cmd/planr/main.go77
-rw-r--r--cmd/planr/sub/build.go16
-rw-r--r--cmd/planr/sub/clean.go7
-rw-r--r--cmd/planr/sub/cli.go60
-rw-r--r--cmd/planr/sub/common.go15
-rw-r--r--cmd/planr/sub/config.go28
-rw-r--r--cmd/planr/sub/evaluate.go89
-rw-r--r--config.go202
-rw-r--r--dirtyscripting.go (renamed from softscript.go)0
-rw-r--r--fs.go128
-rw-r--r--rubric_config.go166
-rw-r--r--runner.go161
-rw-r--r--runner_builder.go42
-rw-r--r--scoring.go31
-rw-r--r--stddirs.go190
-rw-r--r--testcase.go33
-rw-r--r--version.go3
26 files changed, 1409 insertions, 635 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..f48554b
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,52 @@
+# Changelog
+
+## [ v0.1.0 ]
+
+### Added
+- Changelog
+- Clean subcommand to clean build folder
+- Test selection via `Evaluate` command
+- Env overrides for standard directory structure
+ + `PLANR_CONFIG_DIR` - E.g. `planr` directory
+ + `PLANR_SRC_DIR`
+ + `PLANR_BUILD_DIR`
+- Flags overrides for standard directory structure
+ + `-configdir`
+ + `-builddir`
+ + `-srcdir`
+- `bash` adapter
+- `config` command to get directory configuration settings
+- "Primary" configuration file in "planr/config.toml" for rubric version information
+- `json` output for the `evaluate` command
+
+### Changed
+- Better executable handling in the `gtest` adapter
+ + Generates commends in `CMakeLists.txt` for debugging
+- Better concurrency in test pipeline
+- Much refactoring
+
+### Fixed
+- BUG: Fix decoding when no defaults file are present
+
+## [ v0.0.3 ]
+
+### Fixed
+- BUG: Paths to source or test files containing spaces would fail to compile
+
+## [ v0.0.2 ]
+
+### Added
+- Improve CLI output by fencing traces
+- Add colorized compilation traces
+- Internal refactoring
+
+## [ v0.0.1 ]
+
+### Added
+- Basic build pipeline
+- Gtest Test Adapter
+- Configuration decoder
+- Basic CLI
+ + `build` command
+ + `clean` command
+ + `help` command
diff --git a/Makefile b/Makefile
index 424c67d..dd6031f 100644
--- a/Makefile
+++ b/Makefile
@@ -8,8 +8,14 @@ endif
CMD := planr
+ifdef DEBUG
+ FLAGS=-ldflags=-w
+endif
+
+MAIN_PKG := ./cmd/planr/main.go
+
$(CMD):
- go build -o $(CMD) ./cmd/planr/main.go
+ go build $(FLAGS) -o $(CMD) $(MAIN_PKG)
install:
mkdir -p $(DESTDIR)$(PREFIX)$(BINDIR)/
diff --git a/adapters.go b/adapters.go
index 8419c8b..f6c48cb 100644
--- a/adapters.go
+++ b/adapters.go
@@ -1,16 +1,38 @@
package planr
+import (
+ "github.com/BurntSushi/toml"
+)
+
// Test adapters must implement all life cycle hooks
// This allows common config, code generation, etc
// Test cases matching adapter configurations will be
// fed into the adapter interface
type Adapter interface {
- //
Config() AdapterConfig
+ Init(dirs DirConfig)
+
// Called once to preform expensive code generation
- Build(testCase []*TestCase)
+ Build(testCase []TestCase)
// Called every time source changes
- Evaluate(testCase []*TestCase)
+ Evaluate(testCase []TestCase) []TestResult
+}
+
+// A parser function takes a blob of TOML and decodes it into
+// configuration relevant to an adapter
+type TomlParser func (toml.Primitive) (InheritableConfig, error)
+
+// The name under which an adapter registers corresponds
+// to a table under the super-table adapters. All corresponding
+// TOML will be passed to the ParseConfig method or ParseDefaultConfig
+// for parsing. The ParseConfig file parses options in test case files.
+// The ParseDefaultConfig is parsed by `defaults.toml` files and can
+// be used to establish default configuration that will be inherited
+// by all units in a common directory (collection)
+type AdapterConfig struct {
+ Name string
+ ParseConfig TomlParser
+ ParseDefaultConfig TomlParser
}
diff --git a/adapters/bash/adapter.go b/adapters/bash/adapter.go
new file mode 100644
index 0000000..65a6c80
--- /dev/null
+++ b/adapters/bash/adapter.go
@@ -0,0 +1,103 @@
+package bash
+
+import (
+ "context"
+ "errors"
+ "log"
+ "os"
+ "os/exec"
+ "path"
+ "strings"
+ "time"
+ "fmt"
+
+ "golang.flu0r1ne.net/planr"
+)
+
+type Adapter struct {
+ dirs planr.DirConfig
+}
+
+func (a *Adapter) Config() planr.AdapterConfig {
+ return planr.AdapterConfig {
+ Name: "bash",
+ ParseConfig: ParseConfig,
+ ParseDefaultConfig: ParseDefaultConfig,
+ }
+}
+
+func safeWd() string{
+ wd, err := os.Getwd()
+
+ if err != nil {
+ log.Fatalf("Could not get GtestBuildDir %s %v\n", wd, err)
+ }
+
+ return wd
+}
+
+func (a *Adapter) Init(dirs planr.DirConfig) {
+ a.dirs = dirs
+}
+
+func (adapter Adapter) Build(tcs []planr.TestCase) { }
+
+func executeScriptedTest(testdir string, tc planr.TestCase) planr.TestResult {
+ cfg := tc.AdapterConfig().(*Config)
+
+ timeout := time.Duration(cfg.Timeout) * time.Millisecond
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+
+ defer cancel()
+
+ path := path.Join(testdir, cfg.Testfile)
+
+ result := planr.TestResult {}
+ result.Tc = tc
+
+ cmd := exec.CommandContext(ctx, "bash", path)
+
+ if out, err := cmd.CombinedOutput(); err != nil {
+ result.Status = planr.RUNTIME_FAILURE
+ result.TestOutput = string(out)
+
+ var exiterr *exec.ExitError
+ if !errors.As(err, &exiterr) {
+ log.Fatalf("Test script %s failed with unknown error %v\n", path, err)
+ } else {
+ if strings.Contains(exiterr.String(), "killed") {
+ result.TestOutput += fmt.Sprintf("TEST TERMINATED (Timeout=%d)\n", cfg.Timeout)
+ }
+ }
+
+ return result
+ }
+
+ result.Status = planr.PASSING
+
+
+ return result
+}
+
+func (adapter Adapter) Evaluate(tcs []planr.TestCase) [] planr.TestResult {
+ finalizeConfigs(tcs)
+
+ trs := make([]planr.TestResult, 0)
+ c := make(chan planr.TestResult, 0)
+ for i := range tcs {
+ go func(i int) {
+ c <- executeScriptedTest(adapter.dirs.Tests(), tcs[i])
+ }(i)
+ }
+
+ for range tcs {
+ trs = append(trs, <-c)
+ }
+
+ return trs
+}
+
+func NewAdapter() *Adapter {
+ return new(Adapter)
+}
diff --git a/adapters/bash/config.go b/adapters/bash/config.go
new file mode 100644
index 0000000..aaa405a
--- /dev/null
+++ b/adapters/bash/config.go
@@ -0,0 +1,64 @@
+package bash
+
+import (
+ "log"
+ "golang.flu0r1ne.net/planr"
+ "github.com/BurntSushi/toml"
+)
+
+const (
+ DEFAULT_TIMEOUT=1000
+)
+
+type Defaults struct {
+ Testfile string
+ Timeout uint
+}
+
+func (child *Defaults) Inherit(p interface{}) {
+ parent := p.(*Defaults)
+
+ if(child.Timeout == 0) { child.Timeout = parent.Timeout }
+}
+
+type Config struct {
+ Defaults
+}
+
+func (c *Config) finalize(path string) {
+ if c.Testfile == "" {
+ log.Fatalf("\"Testfile\" is not defined for unit %s\n", path)
+ }
+
+ if c.Timeout == 0 {
+ c.Timeout = DEFAULT_TIMEOUT;
+ }
+}
+
+func finalizeConfigs(tcs []planr.TestCase) {
+ for i := range tcs {
+ cfg := tcs[i].AdapterConfig().(*Config)
+
+ cfg.finalize(tcs[i].Path)
+ }
+}
+
+func ParseConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
+ config := Config {}
+
+ if err := toml.PrimitiveDecode(prim, &config); err != nil {
+ return nil, err
+ }
+
+ return &config, nil
+}
+
+func ParseDefaultConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
+ config := Defaults{}
+
+ if err := toml.PrimitiveDecode(prim, &config); err != nil {
+ return nil, err
+ }
+
+ return &config, nil
+}
diff --git a/adapters/gtest/adapter.go b/adapters/gtest/adapter.go
index f4fde27..2961f29 100644
--- a/adapters/gtest/adapter.go
+++ b/adapters/gtest/adapter.go
@@ -1,199 +1,87 @@
package gtest
import (
- "context"
- "errors"
- "fmt"
- "io/ioutil"
"log"
"os"
- "os/exec"
"path"
- "sync"
- "time"
-
"golang.flu0r1ne.net/planr"
)
const GTEST_CMAKE = "CMakeLists.txt"
-func mkUnit(tc *planr.TestCase) cmakeUnit {
- cfg := tc.AdapterConfig().(*GtestConfig)
+func safeWd() string{
+ wd, err := os.Getwd()
- return cmakeUnit {
- tc.Cname,
- cfg.joinTests(*cfg.Testfile),
- cfg.srcList(),
- };
-}
+ if err != nil {
+ log.Fatalf("Could not get GtestBuildDir %s %v\n", wd, err)
+ }
+ return wd
+}
-type GtestAdapter struct {}
+type Adapter struct {
+ dirs planr.DirConfig
+}
-func (a *GtestAdapter) Config() planr.AdapterConfig {
+func (a *Adapter) Config() planr.AdapterConfig {
return planr.AdapterConfig {
Name: "gtest",
ParseConfig: ParseConfig,
ParseDefaultConfig: ParseDefaultConfig,
- }
-}
-
-func (adapter *GtestAdapter) Build(tcs []*planr.TestCase) {
- buildDir := adapter.Config().Dir()
- cmakeFile := path.Join(buildDir, GTEST_CMAKE)
-
- units := make([]cmakeUnit, 0)
- for _, tc := range tcs {
-
- cfg := tc.AdapterConfig().(*GtestConfig)
- cfg.ensureSatisfied(tc.Path)
-
- units = append(units, mkUnit(tc))
}
-
- genCmake(cmakeFile, units)
-
- planr.RunCmd("cmake", "-S", ".", "-B", ".")
}
-type ResultFromId map[string] Result
-
-func (adapter *GtestAdapter) execTests(cnames []string) ResultFromId {
- buildDir := adapter.Config().Dir()
-
- lut := make(ResultFromId, 0)
- for _, exe := range cnames {
-
- exePath := path.Join(buildDir, exe)
-
- f, err := ioutil.TempFile(buildDir, "gtest_adapter_*.json")
-
- if err != nil {
- log.Fatal(err)
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 9999*time.Millisecond)
- cmd := exec.CommandContext(ctx, exePath, "--gtest_output=json:" + f.Name())
-
- defer cancel()
- defer os.Remove(f.Name())
+func (a *Adapter) Init(dirs planr.DirConfig) {
+ a.dirs = dirs
+}
- out, err := cmd.CombinedOutput()
- if err != nil {
- var exiterr *exec.ExitError
+func (adapter Adapter) Build(tcs []planr.TestCase) {
+ buildDir := safeWd()
- if !errors.As(err, &exiterr) {
- log.Printf("%v\n", err)
- os.Exit(exiterr.ExitCode())
- }
- }
+ finalizeConfigs(tcs)
- results, err := decodeResults(f)
+ exes := createExecutables(tcs)
- if err != nil {
- log.Printf("Could not collect results from %s: %v", exe, err)
- continue
- }
+ cmakeFile := path.Join(buildDir, GTEST_CMAKE)
+ cmakeUnits := cmakeUnits(exes, adapter.dirs)
- for _, r := range results {
- r.testOutput = string(out)
- lut[exe + "." + r.id] = r
- }
- }
+ generateCmakeScript(cmakeFile, cmakeUnits)
- return lut
+ planr.RunCmd("cmake", "-S", ".", "-B", ".")
}
-// An executable may contain more than one test
-// Gather all executables and deduplicate them
-func exes(tcs []*planr.TestCase) []string {
- set := make(map[string] bool, 0)
+func (adapter *Adapter) Evaluate(tcs []planr.TestCase) [] planr.TestResult {
+ buildDir := safeWd()
+
+ finalizeConfigs(tcs)
- for _, tc := range tcs {
- // Tests which have encountered a failure
- // may not have an executable
- if tc.Result.Status != planr.PASSING {
- continue
- }
-
- if(!set[tc.Cname]) {
- set[tc.Cname] = true
- }
- }
-
- exes := make([]string, 0)
- for k := range set {
- exes = append(exes, k)
- }
+ results := make([]planr.TestResult, 0)
- return exes
-}
-
-func id(tc *planr.TestCase) string {
- cfg := tc.AdapterConfig().(*GtestConfig)
- return tc.Cname + "." + *cfg.Suite + "." + *cfg.Name
-}
+ exes := createExecutables(tcs)
-func compile(wg * sync.WaitGroup, tc *planr.TestCase) {
- defer wg.Done()
+ c := make(chan []planr.TestResult, len(exes))
+ for i := range exes {
+ go func(exe *executable) {
+ succeed, buildFailures := exe.compile(buildDir)
- cmd := exec.Command("make", tc.Cname)
- out, err := cmd.CombinedOutput()
- tc.Result = new(planr.TestResult)
+ if ! succeed {
+ c <- buildFailures
+ return
+ }
- // Don't treat command failure as anything but a build failure
- if err != nil{
- var exiterr *exec.ExitError
- if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 {
- log.Fatal(err)
- }
+ runtimeResults := exe.execute(buildDir)
- tc.Result.Status = planr.COMPILATION_FAILURE
+ c <- runtimeResults
+ }(&exes[i])
}
- tc.Result.DebugOutput = string(out)
-}
-
-// ./planr eval 0.93s user 0.16s system 100% cpu 1.089 total
-func (adapter *GtestAdapter) Evaluate(tcs []*planr.TestCase) {
- var wg sync.WaitGroup
- for _, tc := range tcs {
- wg.Add(1)
- go compile(&wg, tc)
+ for range exes {
+ results = append(results, (<-c)...)
}
- wg.Wait()
-
- files := exes(tcs)
- resultById := adapter.execTests(files)
-
- for _, tc := range tcs {
- result, ok := resultById[id(tc)]
-
- // compilation failure
- if !ok {
- fmt.Printf("CAN'T FIND %s: status %d\n", tc.Cname, tc.Result.Status)
-
- if tc.Result.Status == planr.PASSING {
- cfg := tc.AdapterConfig().(*GtestConfig)
-
- log.Printf(
- "Could not find testcase %s with name=\"%s\" and suite=\"%s\". Does such a test exist in the test source?",
- tc.Cname,
- *cfg.Name,
- *cfg.Suite,
- )
-
- tc.Result.Status = planr.COMPILATION_FAILURE
- tc.Result.DebugOutput += fmt.Sprintf("planr: Did not find testcase %s in any test executable\n", id(tc))
- }
-
- continue
- }
- if !result.pass {
- tc.Result.Status = planr.RUNTIME_FAILURE
- }
+ return results
+}
- tc.Result.TestOutput = result.testOutput
- }
+func NewAdapter() *Adapter {
+ return new(Adapter)
}
diff --git a/adapters/gtest/config.go b/adapters/gtest/config.go
index cff45fa..533f266 100644
--- a/adapters/gtest/config.go
+++ b/adapters/gtest/config.go
@@ -2,79 +2,88 @@ package gtest
import (
"log"
- "golang.flu0r1ne.net/planr"
"strings"
+ "path"
+ "golang.flu0r1ne.net/planr"
"github.com/BurntSushi/toml"
)
-type GtestDefaults struct {
- Name *string
- Suite *string
- Testfile *string
- Test_root *string
- Srcs *[]string
- Srcs_root *string
+const (
+ DEFAULT_TIMEOUT = 1000
+)
+
+type Defaults struct {
+ Name string
+ Suite string
+ Testfile string
+ Srcs []string
+ Timeout uint
}
-func (child *GtestDefaults) Inherit(p interface{}) {
- parent := p.(*GtestDefaults)
+func (child *Defaults) Inherit(p interface{}) {
+ parent := p.(*Defaults)
- if(child.Name == nil) { child.Name = parent.Name }
- if(child.Suite == nil) { child.Suite = parent.Suite }
- if(child.Testfile == nil) { child.Testfile = parent.Testfile }
- if(child.Test_root == nil) { child.Test_root = parent.Test_root }
- if(child.Srcs == nil) { child.Srcs = parent.Srcs }
- if(child.Srcs_root == nil) { child.Srcs_root = parent.Srcs_root }
+ if(child.Name == "") { child.Name = parent.Name }
+ if(child.Suite == "") { child.Suite = parent.Suite }
+ if(child.Testfile == "") { child.Testfile = parent.Testfile }
+ if(len(child.Srcs) == 0) { child.Srcs = parent.Srcs }
+ if(child.Timeout == 0) { child.Timeout = parent.Timeout }
}
-type GtestConfig struct {
- GtestDefaults
+type Config struct {
+ Defaults
}
-func (g GtestConfig) ensureSatisfied(path string) {
- if g.Name == nil {
+func (c * Config) finalize(path string) {
+ if c.Name == "" {
log.Fatalf("\"name\" is not defined for unit: %s\n", path)
- } else if g.Suite == nil {
+ } else if c.Suite == "" {
log.Fatalf("\"suite\" is not defined for unit: %s\n", path)
- } else if g.Testfile == nil {
+ } else if c.Testfile == "" {
log.Fatalf("\"testfile\" is not defined for unit: %s\n", path)
}
+
+ if c.Timeout == 0 {
+ c.Timeout = DEFAULT_TIMEOUT;
+ }
}
-func (cfg GtestConfig) joinTests(path_ string) string {
- if cfg.Test_root == nil {
- return planr.JoinConfigDir("tests", path_)
+func srcList(srcdir string, srcs []string) string {
+ builder := strings.Builder {}
+
+ for _, src := range srcs {
+ builder.WriteString("\"")
+ builder.WriteString(path.Join(srcdir, src))
+ builder.WriteString("\"\n ")
}
-
- return planr.JoinConfigDir(*cfg.Test_root, path_)
+
+ return builder.String()
}
-func (cfg GtestConfig) joinSrcs(path_ string) string {
- if cfg.Srcs_root == nil {
- return planr.JoinConfigDir("../src", path_)
+func cmakeUnits(e []executable, dirs planr.DirConfig) []cmakeUnit {
+
+ units := make([]cmakeUnit, len(e))
+ for i, exe := range e {
+ testpath := path.Join(dirs.Tests(), exe.testpath)
+ srclist := srcList(dirs.Src(), exe.srcs)
+
+ units[i] = cmakeUnit { exe.exeNm, testpath, srclist }
}
- return planr.JoinConfigDir(*cfg.Srcs_root, path_)
+ return units
}
-func (cfg GtestConfig) srcList() string {
- var srcList string
+func finalizeConfigs(tcs []planr.TestCase) {
+ for i := range tcs {
+ cfg := tcs[i].AdapterConfig().(*Config)
- if cfg.Srcs != nil {
- srcs := make([]string, len(*cfg.Srcs))
- for i, src := range *cfg.Srcs {
- srcs[i] = "\"" + cfg.joinSrcs(src) + "\""
- }
-
- srcList = strings.Join(srcs, "\n ")
+ cfg.finalize(tcs[i].Path)
}
-
- return srcList
}
func ParseConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
- config := GtestConfig{}
+ config := Config{}
if err := toml.PrimitiveDecode(prim, &config); err != nil {
return nil, err
@@ -84,7 +93,7 @@ func ParseConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
}
func ParseDefaultConfig(prim toml.Primitive) (planr.InheritableConfig, error) {
- config := GtestDefaults{}
+ config := Defaults{}
if err := toml.PrimitiveDecode(prim, &config); err != nil {
return nil, err
diff --git a/adapters/gtest/executable.go b/adapters/gtest/executable.go
new file mode 100644
index 0000000..25c83c1
--- /dev/null
+++ b/adapters/gtest/executable.go
@@ -0,0 +1,185 @@
+package gtest
+
+import (
+ "os"
+ "errors"
+ "time"
+ "io/ioutil"
+ "log"
+ "os/exec"
+ "path"
+ "reflect"
+ "sort"
+ "context"
+
+ "golang.flu0r1ne.net/planr"
+)
+
+type executable struct {
+ exeNm string
+ testpath string
+ srcs []string
+ tcs []planr.TestCase
+}
+
+func createExecutables(tcs []planr.TestCase) []executable {
+ exes := make(map[string] executable, 0)
+
+ for _, tc := range tcs {
+ cfg := tc.AdapterConfig().(*Config)
+ file := cfg.Testfile
+ exe, contained := exes[file]
+
+ // For set comparison
+ sort.Strings(cfg.Srcs)
+
+ if !contained {
+ exeTcs := make([]planr.TestCase, 1)
+ exeTcs[0] = tc
+
+
+ exe := executable {
+ planr.Cname("", file),
+ file,
+ cfg.Srcs,
+ exeTcs,
+ }
+
+ exes[file] = exe
+
+ continue
+ }
+
+ // We could create two different executables for each source list
+ // But, that would be confusing so we're going to disallow it
+ if !reflect.DeepEqual(exe.srcs, cfg.Srcs) {
+ log.Fatalf(
+ "Two test case definitions %s and %s have different lists of sources",
+ exe.testpath, cfg.Testfile,
+ )
+ }
+
+ exe.tcs = append(exe.tcs, tc)
+
+ exes[file] = exe
+ }
+
+ exesList := make([]executable, 0)
+
+ for _, exe := range exes {
+ exesList = append(exesList, exe)
+ }
+
+ return exesList
+}
+
+func (exe executable) compile(builddir string) (succeeded bool, buildFailures []planr.TestResult) {
+ cmd := exec.Command("make", "-C", builddir, exe.exeNm)
+ out, err := cmd.CombinedOutput()
+ buildFailures = make([]planr.TestResult, 0)
+
+ outputLog := string(out)
+
+ if err != nil{
+ var exiterr *exec.ExitError
+ if errors.As(err, &exiterr) && exiterr.ExitCode() == 0 {
+ log.Fatalf("Unrecoverable build failure: %v", err)
+ }
+
+ for i := range exe.tcs {
+ res := planr.TestResult {}
+ res.Tc = exe.tcs[i]
+ res.DebugOutput = outputLog
+ res.Status = planr.COMPILATION_FAILURE
+
+ buildFailures = append(buildFailures, res)
+ }
+
+ succeeded = false
+
+ return
+ }
+
+ succeeded = true
+ return
+}
+
+const TMPFILENAME = "gtest_adapter_*.json"
+
+func runGtest(exe string, tc planr.TestCase, builddir string) planr.TestResult {
+ result := planr.TestResult {}
+ result.Tc = tc
+
+ exePath := path.Join(builddir, exe)
+ cfg := tc.AdapterConfig().(*Config)
+
+ f, err := ioutil.TempFile(builddir, TMPFILENAME)
+
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ timeout := time.Duration(cfg.Timeout) * time.Millisecond
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+
+ jsonFlag := "--gtest_output=json:" + f.Name()
+ testFlag := "--gtest_filter=" + cfg.Suite + "." + cfg.Name
+
+ cmd := exec.CommandContext(ctx, exePath, jsonFlag, testFlag)
+
+ defer cancel()
+ defer os.Remove(f.Name())
+
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ var exiterr *exec.ExitError
+
+ if !errors.As(err, &exiterr) {
+ log.Printf("%v\n", err)
+ os.Exit(exiterr.ExitCode())
+ }
+ }
+
+ results, err := decodeResults(f)
+
+ if err != nil {
+ log.Fatalf("Could not collect results from %s: %v", exe, err)
+ }
+
+ if len(results) < 1 {
+ log.Fatalf(
+ "Could not find testcase %s with name=\"%s\" and suite=\"%s\". Does such a test exist in the test source?",
+ tc.Cname,
+ cfg.Name,
+ cfg.Suite,
+ )
+ }
+
+ if len(results) > 1 {
+ log.Fatalf("Unexpected number of results, filter should have produced one result")
+ }
+
+ decodeResult := results[0]
+
+ result.TestOutput = string(out)
+
+ if decodeResult.pass {
+ result.Status = planr.PASSING
+ } else {
+ result.Status = planr.RUNTIME_FAILURE
+ }
+
+ return result
+}
+
+func (exe executable) execute(builddir string) []planr.TestResult {
+ results := make([]planr.TestResult, len(exe.tcs))
+
+ for i := range exe.tcs {
+ results[i] = runGtest(exe.exeNm, exe.tcs[i], builddir)
+ }
+
+ return results
+}
+
diff --git a/adapters/gtest/templating.go b/adapters/gtest/templating.go
index c49f170..41c54c1 100644
--- a/adapters/gtest/templating.go
+++ b/adapters/gtest/templating.go
@@ -3,17 +3,18 @@ package gtest
import (
"io"
"log"
- "text/template"
"os"
+ "golang.flu0r1ne.net/planr"
+ "text/template"
)
type cmakeUnit struct {
- Cname string
+ ExeNm string
File string
Srcs string
};
-func genCmake(out string, units []cmakeUnit) {
+func generateCmakeScript(out string, units []cmakeUnit) {
file, err := os.OpenFile(out, os.O_RDWR | os.O_CREATE, 0644)
defer func () {
err := file.Close()
@@ -27,45 +28,67 @@ func genCmake(out string, units []cmakeUnit) {
log.Fatalf("Could not open CMakeFile (%s)\n%v", out, err)
}
- writeBoiler(file)
+ writeCmakeBoilerplate(file)
tmpl := unitTemplate()
for _, unit := range units {
if err := tmpl.Execute(file, unit); err != nil {
- log.Fatalf("Failed to generate unit %s: %v", unit.Cname, err);
+ log.Fatalf("Failed to generate unit %s: %v", unit.ExeNm, err);
}
}
}
+// TODO: Add comments
func unitTemplate() *template.Template {
tmpl, err := template.New("gtest_unit").Parse(`
+
+################################################
+
+## {{.ExeNm}}
+
add_executable(
- "{{.Cname}}"
+ "{{.ExeNm}}"
"{{.File}}"
{{.Srcs}}
)
target_link_libraries(
- "{{.Cname}}"
+ "{{.ExeNm}}"
gtest_main
)
gtest_discover_tests(
- "{{.Cname}}"
+ "{{.ExeNm}}"
)
`)
if err != nil {
- log.Fatalf("Cannot load Gtest Unit Template %v", err)
+ log.Fatalf("Cannot load Gtest unit template %v", err)
}
return tmpl
}
-func writeBoiler(w io.Writer) {
- w.Write([]byte(`
+const GOOGLE_TEST_URL = "https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip"
+
+func writeCmakeBoilerplate(w io.Writer) {
+ tmpl := boilderTemplate()
+
+ tmpl.Execute(w, struct {
+ Url string
+ Version string
+ }{
+ Url: GOOGLE_TEST_URL,
+ Version: planr.VERSION,
+ })
+}
+
+func boilderTemplate() *template.Template {
+ tmpl, err := template.New("gtest_boilerplate").Parse(`
+# AUTOMATICALLY GENERATED BY PLANR VERSION {{.Version}}
+
cmake_minimum_required (VERSION 3.1.0)
project(PlanRGtestAdapter)
@@ -73,12 +96,16 @@ project(PlanRGtestAdapter)
include(FetchContent)
FetchContent_Declare(
googletest
- URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
+ URL {{.Url}}
)
include(GoogleTest)
FetchContent_MakeAvailable(googletest)
-`))
-}
-
+`)
+
+ if err != nil {
+ log.Fatalf("Cannot load Gtest Cmake boilerplate")
+ }
+ return tmpl
+}
diff --git a/cmd/planr/main.go b/cmd/planr/main.go
index e2a85ed..2243203 100644
--- a/cmd/planr/main.go
+++ b/cmd/planr/main.go
@@ -1,24 +1,28 @@
package main
-import (
+import ( "flag"
"fmt"
"io"
"log"
"os"
+ "runtime/pprof"
+ "golang.flu0r1ne.net/planr"
+ "golang.flu0r1ne.net/planr/adapters/gtest"
+ "golang.flu0r1ne.net/planr/adapters/bash"
"golang.flu0r1ne.net/planr/cmd/planr/sub"
)
-const (
- VERSION = "0.0.3"
-)
-
func printUsage(w io.Writer) {
fmt.Fprintf (w, "usage: %s command args ... \n", os.Args[0])
fmt.Fprintln(w, " help ")
fmt.Fprintln(w, " version ")
fmt.Fprintln(w, " build ")
fmt.Fprintln(w, " evaluate ")
+ fmt.Fprintln(w, " evaluate <test0> <test1> ... ")
+ fmt.Fprintln(w, " evaluate -json ")
+ fmt.Fprintln(w, " clean ")
+ fmt.Fprintln(w, " config <key> ")
}
func dieUsage() {
@@ -26,25 +30,74 @@ func dieUsage() {
os.Exit(1)
}
+var src = flag.String("srcdir", "", "source directory")
+var config = flag.String("configdir", "", "config directory")
+var build = flag.String("builddir", "", "build directory")
+var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
+
+func getConfiguredRunner() planr.Runner {
+ r := planr.ConfigureRunner()
+ r = planr.RegisterAdapter(r, gtest.NewAdapter())
+ r = planr.RegisterAdapter(r, bash.NewAdapter())
+
+ if wd, err := os.Getwd(); err == nil {
+ r = planr.SetConfigDirFromTree(r, wd)
+ }
+
+ if *src != "" {
+ r = planr.SetSrcDir(r, *src)
+ }
+
+ if *config != "" {
+ r = planr.SetConfigDir(r, *config)
+ }
+
+ if *build != "" {
+ r = planr.SetBuildDir(r, *build)
+ }
+
+ return r.New()
+}
+
func main() {
+ flag.Parse()
+
+ if *cpuprofile != "" {
+ f, err := os.Create(*cpuprofile)
+
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ pprof.StartCPUProfile(f)
+ defer pprof.StopCPUProfile()
+ }
log.SetFlags(log.Llongfile | log.Lmsgprefix)
log.SetPrefix("planr: ")
- if len(os.Args) < 2 {
+ if flag.NArg() < 1 {
dieUsage()
}
- subcommand := os.Args[1]
- subargs := os.Args[2:]
+ runner := getConfiguredRunner()
+
+ cfg := planr.DecodeConfig(runner.ConfigDir())
+
+ subcommand := flag.Arg(0)
+ subargs := flag.Args()[1:]
switch subcommand {
case "version":
- fmt.Printf("%s\n", VERSION)
+ fmt.Printf("%s\n", planr.VERSION)
case "build":
- sub.Build(subargs)
- case "evaluate","eval":
- sub.Evaluate(subargs)
+ sub.Build(runner, subargs, cfg)
+ case "evaluate", "eval":
+ sub.Evaluate(runner, subargs, cfg)
+ case "clean":
+ sub.Clean(runner, subargs)
+ case "config":
+ sub.Config(runner, subargs)
case "help", "-h", "-help", "--help":
printUsage(os.Stdout)
default:
diff --git a/cmd/planr/sub/build.go b/cmd/planr/sub/build.go
index 58c3a38..e93e0cb 100644
--- a/cmd/planr/sub/build.go
+++ b/cmd/planr/sub/build.go
@@ -2,18 +2,10 @@ package sub
import (
"golang.flu0r1ne.net/planr"
- "golang.flu0r1ne.net/planr/adapters/gtest"
)
-func Runner() planr.Runner {
- r := planr.Runner {}
- r.RegisterAdapter(&gtest.GtestAdapter{})
- return r
-}
-
-func Build(params []string) {
-
- rd := planr.RubricDir()
-
- Runner().Build(rd)
+func Build(runner planr.Runner, params []string, cfg * planr.Config) {
+ dieIncompatibleVersion(cfg)
+ tcs := runner.CollectCases()
+ runner.Build(tcs)
}
diff --git a/cmd/planr/sub/clean.go b/cmd/planr/sub/clean.go
new file mode 100644
index 0000000..d658c10
--- /dev/null
+++ b/cmd/planr/sub/clean.go
@@ -0,0 +1,7 @@
+package sub
+
+import "golang.flu0r1ne.net/planr"
+
+func Clean(runner planr.Runner, params []string) {
+ runner.Clean()
+}
diff --git a/cmd/planr/sub/cli.go b/cmd/planr/sub/cli.go
index 0e6a942..e6f2256 100644
--- a/cmd/planr/sub/cli.go
+++ b/cmd/planr/sub/cli.go
@@ -14,25 +14,23 @@ var (
col_label = color.New(color.FgCyan)
);
-func tcTitle(tc planr.TestCase) string {
- title := tc.Cname
+func tcTitle(tr planr.TestResult) string {
+ title := tr.Tc.Cname
- if tc.Config.Title != nil {
- title = *tc.Config.Title
+ if tr.Tc.Config.Title != nil {
+ title = *tr.Tc.Config.Title
}
return title
}
-func tcStatus(tc planr.TestCase) string {
+func tcStatus(tc planr.TestResult) string {
status := "SILENT"
- if tc.Result != nil {
- if tc.Result.Status == planr.PASSING {
- status = "PASS"
- } else {
- status = "FAIL"
- }
+ if tc.Status == planr.PASSING {
+ status = "PASS"
+ } else {
+ status = "FAIL"
}
return status
@@ -59,9 +57,9 @@ func pprintFenced(title, value string) {
fmt.Println(fence)
}
-func tcStatusLine(tc planr.TestCase) {
- title := tcTitle(tc)
- status := tcStatus(tc)
+func tcStatusLine(tr planr.TestResult) {
+ title := tcTitle(tr)
+ status := tcStatus(tr)
if status == "PASS" {
col_pass.Printf("[%s] ", status);
@@ -72,8 +70,10 @@ func tcStatusLine(tc planr.TestCase) {
col_title.Println(title);
}
-func tcPprint(tc planr.TestCase) {
- tcStatusLine(tc)
+func tcPprint(tr planr.TestResult) {
+ tcStatusLine(tr)
+
+ tc := tr.Tc
pprintLabeled("id", tc.Cname)
@@ -86,22 +86,20 @@ func tcPprint(tc planr.TestCase) {
pprintLabeled("description", *tc.Config.Description)
}
- res := tc.Result
-
- if res.Status == planr.COMPILATION_FAILURE {
+ if tr.Status == planr.COMPILATION_FAILURE {
- if res.DebugOutput != "" {
+ if tr.DebugOutput != "" {
fmt.Println()
- pprintFenced("compilation output", tc.Result.DebugOutput);
+ pprintFenced("compilation output", tr.DebugOutput);
} else {
fmt.Println("WARN: No debug output provided")
}
- } else if res.Status == planr.RUNTIME_FAILURE {
+ } else if tr.Status == planr.RUNTIME_FAILURE {
- if tc.Result.TestOutput != "" {
+ if tr.TestOutput != "" {
fmt.Println()
- pprintFenced("test output", tc.Result.TestOutput);
+ pprintFenced("test output", tr.TestOutput);
}
}
@@ -109,14 +107,16 @@ func tcPprint(tc planr.TestCase) {
fmt.Println()
}
-func printResults(passed, tc_total int, earned, points_total float64) {
+func printScoring(score planr.Scoring) {
col_title.Println("Final Results:")
- pprintLabeled("passed", fmt.Sprintf("%d/%d", passed, tc_total));
+ pprintLabeled("passed", fmt.Sprintf("%d/%d", score.Passed, score.Total));
- percent := earned / points_total * 100
+ percent := score.EarnedPoints / score.TotalPoints * 100
- pprintLabeled("score", fmt.Sprintf(
- "%.2f/%.2f ~= %.1f%%", earned, points_total, percent,
- ));
+ if score.TotalPoints != 0 {
+ pprintLabeled("score", fmt.Sprintf(
+ "%.2f/%.2f ~= %.1f%%", score.EarnedPoints, score.TotalPoints, percent,
+ ));
+ }
}
diff --git a/cmd/planr/sub/common.go b/cmd/planr/sub/common.go
new file mode 100644
index 0000000..9126f3c
--- /dev/null
+++ b/cmd/planr/sub/common.go
@@ -0,0 +1,15 @@
+package sub
+
+import (
+ "golang.flu0r1ne.net/planr"
+ "os"
+ "fmt"
+)
+
+func dieIncompatibleVersion(cfg *planr.Config) {
+ if cfg != nil && cfg.IncompatibleWithVersion() {
+ fmt.Fprintf(os.Stderr, "This version of PlanR (%v) is incompatible with config version %s\n", planr.VERSION, cfg.Version)
+ fmt.Fprintf(os.Stderr, "Please upgrade to version %s or greater\n", cfg.Version)
+ os.Exit(1)
+ }
+}
diff --git a/cmd/planr/sub/config.go b/cmd/planr/sub/config.go
new file mode 100644
index 0000000..ee372c8
--- /dev/null
+++ b/cmd/planr/sub/config.go
@@ -0,0 +1,28 @@
+package sub
+
+import (
+ "golang.flu0r1ne.net/planr"
+ "fmt"
+ "os"
+)
+
+
+func Config(runner planr.Runner, params []string) {
+ if len(params) != 1 {
+ fmt.Fprintf(os.Stderr, "Usage: planr config <parameter>\n")
+ os.Exit(1)
+ }
+
+ key := params[0]
+
+ switch key {
+ case "builddir":
+ fmt.Printf("%s\n", runner.BuildDir())
+ case "configdir":
+ fmt.Printf("%s\n", runner.ConfigDir())
+ case "srcdir":
+ fmt.Printf("%s\n", runner.SrcDir())
+ default:
+ fmt.Fprintf(os.Stderr, "\"%s\" not found in configuration\n", key)
+ }
+}
diff --git a/cmd/planr/sub/evaluate.go b/cmd/planr/sub/evaluate.go
index 8ce4d81..30d30d2 100644
--- a/cmd/planr/sub/evaluate.go
+++ b/cmd/planr/sub/evaluate.go
@@ -1,38 +1,81 @@
package sub
import (
- "golang.flu0r1ne.net/planr"
+ "encoding/json"
+ "fmt"
+ "log"
+ "flag"
+
+ "golang.flu0r1ne.net/planr"
)
-func Evaluate(params []string) {
- rd := planr.RubricDir()
+type gradingResults struct {
+ TestResults []planr.TestResult
+ Score planr.Scoring
+}
+
+func prettyPrint(results gradingResults, summarize bool) {
+ for _, tr := range results.TestResults {
+ tcPprint(tr)
+ }
+
+ if summarize {
+ printScoring(results.Score)
+ }
+}
- tcs := Runner().Evaluate(rd)
+func jsonPrint(results gradingResults) {
+ res, err := json.Marshal(results)
- earned := 0.0
- total := 0.0
- passed := 0
- for _, tc := range tcs {
- cfg := tc.Config
+ if err != nil {
+ log.Fatalf("Error printing JSON: %v\n", err)
+ }
- if cfg.Points != nil {
- points := float64(*cfg.Points)
+ fmt.Println(string(res))
+}
- total += points
+func Evaluate(runner planr.Runner, params []string, cfg *planr.Config) {
+ f := flag.NewFlagSet("evaluate", flag.ExitOnError)
- if tc.Result.Status == planr.PASSING {
- earned += points
- passed++
- }
+ jsonOutput := f.Bool("json", false, "print json output")
+
+ dieIncompatibleVersion(cfg)
+
+ f.Parse(params)
+
+ tcs := runner.CollectCases()
+
+ // Filter those tests which patch IDs in params
+ filteredTcs := make([]planr.TestCase, 0)
+ summarizeScore := false
+ if f.NArg() > 0 {
+ ids := f.Args()
+
+ membershipFun := make(map[string] bool, 0)
+ for _, id := range ids {
+ membershipFun[id] = true
}
- tcPprint(tc)
+ for i := range tcs {
+ if membershipFun[tcs[i].Cname] {
+ filteredTcs = append(filteredTcs, tcs[i])
+ }
+ }
+ } else {
+ summarizeScore = true
+ filteredTcs = tcs
}
- printResults(
- passed,
- len(tcs),
- earned,
- total,
- );
+ trs := runner.Evaluate(filteredTcs)
+
+ results := gradingResults {
+ TestResults: trs,
+ Score: planr.Score(trs),
+ }
+
+ if *jsonOutput {
+ jsonPrint(results)
+ } else {
+ prettyPrint(results, summarizeScore)
+ }
}
diff --git a/config.go b/config.go
index bc0fa6a..d7cd3e4 100644
--- a/config.go
+++ b/config.go
@@ -1,199 +1,51 @@
package planr
import (
- "log"
"github.com/BurntSushi/toml"
+ "log"
+ "path"
+ "strings"
)
-
-/*
- TODO: Every property defined within the defaults currently
- has to implement the "inherit" method to conditionally inherit a
- property in relation to a parent. (Ostensibly so that test cases
- can override default configuration.) This is pedantic because
- most properties will end up writing boilerplate amounting to:
-
- parent.inherit()
- ...
- if config.property == nil {
- config.property = config.parent.property
- }
-
- This library provides copying behavior between structs
- with common properties using reflection. It seems like
- a slight abuse of reflection... But, it could be
- retro-fitted to implement this behavior if a "onlyCopyZeroFields"
- option was provided.
-
- > "github.com/jinzhu/copier"
-*/
-
-// Inheritable configuration can inherit properties defined in a
-// defaults file. This happens on a per-directory basis so multiple
-// tests can share common configuration.
-//
-// The parent will always be of the same type as the child and an
-// assertion is required to define the proper behavior.
-type InheritableConfig interface {
- Inherit(parent interface{})
-}
-
-// A parser function takes a blob of TOML and decodes it into
-// configuration relevant to an adapter
-type TomlParser func (toml.Primitive) (InheritableConfig, error)
-
-// The name under which an adapter registers corresponds
-// to a table under the super-table adapters. All corresponding
-// TOML will be passed to the ParseConfig method or ParseDefaultConfig
-// for parsing. The ParseConfig file parses options in test case files.
-// The ParseDefaultConfig is parsed by `defaults.toml` files and can
-// be used to establish default configuration that will be inherited
-// by all units in a common directory (collection)
-type AdapterConfig struct {
- Name string
- ParseConfig TomlParser
- ParseDefaultConfig TomlParser
-}
-
-// Program-wide configuration which is recognized
-// in defaults.toml
-type Defaults struct {
- Points *float32
- Adapter *string
-
- /*
- The TOML library only parses exported fields.
- The Adapters field is an intermediate mapping
- individual adapters to their locally defined
- configuration. After they individually process
- the configuration, it is mapped to the adapters_
- field.
-
- See: decodeAdapters()
- */
- Adapters *map[string] toml.Primitive
- adapters_ map[string] InheritableConfig
-
- /*
- The configs_ field is necessary to property
- implement the Inherit method using a common
- interface.
- */
- configs_ *[]AdapterConfig
-}
-
-// Program-wide testcase config
-type TestCaseConfig struct {
- Defaults
- Title *string
- Description *string
-}
-
-func (c TestCaseConfig) ensureSatisfied(name string) {
- if (c.Adapter == nil) {
- log.Fatalf("Adapter must be provided for testcase %s", name)
- }
+type Config struct {
+ Version string
}
-// The default configuration must be able in inherit from
-// other defaults further up the tree
-//
-// This provides multiple levels of configurability
-func (child *Defaults) Inherit(p interface{}) {
- parent := p.(Defaults)
-
- // Inherit properties which haven't been configured
- if child.Points == nil {
- child.Points = parent.Points;
- }
-
- if child.Adapter == nil {
- child.Adapter = parent.Adapter;
- }
+const PLANR_CONFIG_FILE = "config.toml"
- // Call the inherit method as defined by the adapters
- // If an adapter is undefined, inherit the parent configuration
- //
- // _configs represents all adapters (registered to a runner)
- for _, adapter := range *child.configs_ {
- parent_adapter, parent_exists := parent.adapters_[adapter.Name]
- child_adapter, child_exists := child.adapters_[adapter.Name]
+// TODO: REMOVE
+const STRICTLY_REQUIRE_CONFIG = false
- if parent_exists {
- if child_exists {
- child_adapter.Inherit(parent_adapter)
- } else {
- child.adapters_[adapter.Name] = parent_adapter
- }
- }
- }
-}
+func DecodeConfig(configDir string) *Config {
+ cfg := new(Config)
-// Parses the intermediate adapters Adapters containing TOML primitives
-// according to methods registered with the runner
-// Once parsed, they are stored alongside the registered name to determine
-// which adapter will receive the configuration
-func (defaults *Defaults) decodeAdapters(
- adapters []AdapterConfig,
- asDefault bool,
-) error {
- defaults.configs_ = &adapters
- defaults.adapters_ = make(map[string]InheritableConfig)
+ configFile := path.Join(configDir, PLANR_CONFIG_FILE)
- if defaults.Adapters != nil {
- for _, config := range adapters {
- primitive, exists := (*defaults.Adapters)[config.Name]
-
- if exists {
- var parsed InheritableConfig
- var err error
- if asDefault {
- parsed, err = config.ParseDefaultConfig(primitive)
- } else {
- parsed, err = config.ParseConfig(primitive)
- }
+ if _, err := toml.DecodeFile(configFile, cfg); err != nil {
+ cfg = nil
- if err != nil {
- return err
- }
-
- defaults.adapters_[config.Name] = parsed
- }
+ // TODO: handle missing config
+ if STRICTLY_REQUIRE_CONFIG {
+ log.Fatalf("Could not decode global configuration %s: %v", configFile, err)
}
}
- return nil
+ return cfg
}
-// Decode defaults.toml
-func DecodeDefaults(path string, adapterCfg []AdapterConfig) (Defaults, error) {
- defaults := Defaults { }
-
-
-
- if _, err := toml.DecodeFile(path, &defaults); err != nil {
- return defaults, err
+func (cfg Config) IncompatibleWithVersion() bool {
+ if strings.Count(cfg.Version, ".") != 2 {
+ log.Fatalf("Version %s is not semantic", cfg.Version)
}
- if err := defaults.decodeAdapters(adapterCfg, true); err != nil {
- return defaults, err
- }
-
-
- return defaults, nil
-}
-
-// Decode an individual unit
-func DecodeConfig(path string, adapterCfg []AdapterConfig) (TestCaseConfig, error) {
- config := TestCaseConfig { }
-
- if _, err := toml.DecodeFile(path, &config); err != nil {
- return config, nil
- }
+ cfgbits := strings.SplitN(cfg.Version, ".", 2)
+ bits := strings.SplitN(VERSION, ".", 2)
- if err := config.decodeAdapters(adapterCfg, false); err != nil {
- return config, err
+ // major version change
+ if cfgbits[0] != bits[0] {
+ return true
}
- return config, nil
+ // Config newer, possible feature additions
+ return cfgbits[1] > bits[1]
}
diff --git a/softscript.go b/dirtyscripting.go
index 9924d54..9924d54 100644
--- a/softscript.go
+++ b/dirtyscripting.go
diff --git a/fs.go b/fs.go
index 86de16b..b76f182 100644
--- a/fs.go
+++ b/fs.go
@@ -45,121 +45,12 @@ func directoryExists(path string) bool {
return info.IsDir()
}
-// Find the configuration directory
-// Uses:
-// 1. PlANR_DIRECTORY env if set
-// 2. planr
-// 3. .planr
-func ConfigDir() string {
-
- // Return environmental override if set
- if dir, isSet := os.LookupEnv("PLANR_DIRECTORY"); isSet {
-
- if !directoryExists(dir) {
- log.Fatalf("Cannot find planr directory %s", dir);
- }
-
- return dir;
- }
-
- cwd, err := os.Getwd()
-
- if err != nil {
- log.Fatal(err)
- }
-
- var rubricDir string
-
- rubric_search_dirs := [2]string{
- "planr",
- ".planr",
- }
-
- found := traverseUp(cwd, func (path string) bool {
-
- for _, dir := range rubric_search_dirs {
- rubricDir = filepath.Join(path, dir)
-
- if directoryExists(rubricDir) {
- return true
- }
- }
-
- return false
- });
-
- if !found {
- log.Fatal("Could not find planr directory");
- }
-
- return rubricDir
-}
-
-func JoinConfigDir(path_ string, file string) string {
- if path.IsAbs(path_) {
- return path.Join(path_, file)
- }
-
- return path.Join(ConfigDir(), path_, file)
-}
-
-func RootDir() string {
- return path.Join(ConfigDir(), "..")
-}
-
-// Find rubric directory at PLANR_DIR/rubric
-func RubricDir() string {
- rubricDir := path.Join(ConfigDir(), "rubric");
-
- if !directoryExists(rubricDir) {
- log.Fatal("Could not find the rubric directory inside of planr")
- }
-
- return rubricDir
-}
-
-func BuildDir() string {
- buildDir := path.Join(ConfigDir(), "build")
-
- if !directoryExists(buildDir) {
- err := os.Mkdir(buildDir, 0755)
-
- if err != nil {
- log.Fatalf("Cannot create build directory %v\n", err)
- }
- }
-
- return buildDir
-}
-
-func CleanBuildDir() {
- buildDir := path.Join(ConfigDir(), "build")
- if err := os.RemoveAll(buildDir); err != nil {
- log.Fatalf("Cannot clean (removeAll) in build directory %v\n", err)
- }
-}
-
-func (ac AdapterConfig) Dir() string {
- dir := BuildDir()
- dir = path.Join(dir, ac.Name)
-
- if !directoryExists(dir) {
- err := os.Mkdir(dir, 0755)
-
- if err != nil {
- log.Fatalf("Cannot create build/%s directory %v\n", ac.Name, err)
- }
- }
-
- return dir
-}
-
func basename(path string) string {
ext := filepath.Ext(path)
return path[0:len(path) - len(ext)]
}
-func cname(root string, path string) string {
+func Cname(root string, path string) string {
rel, err := filepath.Rel(root, path)
if err != nil {
@@ -185,7 +76,8 @@ func collectUnits(root string, cfgs []AdapterConfig) []TestCase {
collectFromDir(root, nil, cfgs, &tcs)
for i := range tcs {
- tcs[i].Cname = cname(root, tcs[i].Path)
+ tcs[i].Cname = Cname(root, tcs[i].Path)
+ tcs[i].readIdx = i
}
return tcs
@@ -208,7 +100,7 @@ func collectFromDir(
// Process defaults for this directory if a defaults.toml is found
defaultsPath := path.Join(dir, DEFAULTS)
if info, err := os.Stat(defaultsPath); err == nil && !info.IsDir() {
- d, err := DecodeDefaults(defaultsPath, cfgs)
+ d, err := DecodeRubricDefaults(defaultsPath, cfgs)
if err != nil {
log.Fatalf("Error encounter in %s: %v\n", defaultsPath, err);
@@ -244,13 +136,15 @@ func collectFromDir(
}
// Decode a unit
- config, err := DecodeConfig(child, cfgs)
+ config, err := DecodeRubricConfig(child, cfgs)
if err != nil {
log.Fatalf("Error encountered in %s: %v", child, config)
}
- config.Inherit(*defaults)
+ if defaults != nil {
+ config.Inherit(*defaults)
+ }
tc := TestCase {
Path: child,
@@ -262,3 +156,9 @@ func collectFromDir(
}
}
}
+
+func safeCd(newWd string) {
+ if err := os.Chdir(newWd); err != nil {
+ log.Fatalf("Could not change into directory %v\n", err)
+ }
+}
diff --git a/rubric_config.go b/rubric_config.go
new file mode 100644
index 0000000..322e58a
--- /dev/null
+++ b/rubric_config.go
@@ -0,0 +1,166 @@
+package planr
+
+import (
+ "github.com/BurntSushi/toml"
+)
+/*
+ TODO: Every property defined within the defaults currently
+ has to implement the "inherit" method to conditionally inherit a
+ property in relation to a parent. (Ostensibly so that test cases
+ can override default configuration.) This is pedantic because
+ most properties will end up writing boilerplate amounting to:
+
+ parent.inherit()
+ ...
+ if config.property == nil {
+ config.property = config.parent.property
+ }
+
+ This library provides copying behavior between structs
+ with common properties using reflection. It seems like
+ a slight abuse of reflection... But, it could be
+ retro-fitted to implement this behavior if a "onlyCopyZeroFields"
+ option was provided.
+
+ > "github.com/jinzhu/copier"
+*/
+
+// Inheritable configuration can inherit properties defined in a
+// defaults file. This happens on a per-directory basis so multiple
+// tests can share common configuration.
+//
+// The parent will always be of the same type as the child and an
+// assertion is required to define the proper behavior.
+type InheritableConfig interface {
+ Inherit(parent interface{})
+}
+
+
+// Program-wide configuration which is recognized
+// in defaults.toml
+type Defaults struct {
+ Points *float32
+ Adapter *string
+
+ /*
+ The TOML library only parses exported fields.
+ The Adapters field is an intermediate mapping
+ individual adapters to their locally defined
+ configuration. After they individually process
+ the configuration, it is mapped to the adapters_
+ field.
+
+ See: decodeAdapters()
+ */
+ Adapters *map[string] toml.Primitive
+ adapters_ map[string] InheritableConfig
+
+ /*
+ The configs_ field is necessary to property
+ implement the Inherit method using a common
+ interface.
+ */
+ configs_ *[]AdapterConfig
+}
+
+
+// The default configuration must be able in inherit from
+// other defaults further up the tree
+//
+// This provides multiple levels of configurability
+func (child *Defaults) Inherit(p interface{}) {
+ parent := p.(Defaults)
+
+ // Inherit properties which haven't been configured
+ if child.Points == nil {
+ child.Points = parent.Points;
+ }
+
+ if child.Adapter == nil {
+ child.Adapter = parent.Adapter;
+ }
+
+ // Call the inherit method as defined by the adapters
+ // If an adapter is undefined, inherit the parent configuration
+ //
+ // _configs represents all adapters (registered to a runner)
+ for _, adapter := range *child.configs_ {
+ parent_adapter, parent_exists := parent.adapters_[adapter.Name]
+ child_adapter, child_exists := child.adapters_[adapter.Name]
+
+ if parent_exists {
+ if child_exists {
+ child_adapter.Inherit(parent_adapter)
+ } else {
+ child.adapters_[adapter.Name] = parent_adapter
+ }
+ }
+ }
+}
+
+// Parses the intermediate adapters Adapters containing TOML primitives
+// according to methods registered with the runner
+// Once parsed, they are stored alongside the registered name to determine
+// which adapter will receive the configuration
+func (defaults *Defaults) decodeAdapters(
+ adapters []AdapterConfig,
+ asDefault bool,
+) error {
+ defaults.configs_ = &adapters
+ defaults.adapters_ = make(map[string]InheritableConfig)
+
+ if defaults.Adapters != nil {
+ for _, config := range adapters {
+ primitive, exists := (*defaults.Adapters)[config.Name]
+
+ if exists {
+ var parsed InheritableConfig
+ var err error
+ if asDefault {
+ parsed, err = config.ParseDefaultConfig(primitive)
+ } else {
+ parsed, err = config.ParseConfig(primitive)
+ }
+
+ if err != nil {
+ return err
+ }
+
+ defaults.adapters_[config.Name] = parsed
+ }
+ }
+ }
+
+ return nil
+}
+
+// Decode defaults.toml
+func DecodeRubricDefaults(path string, adapterCfg []AdapterConfig) (Defaults, error) {
+ defaults := Defaults { }
+
+ if _, err := toml.DecodeFile(path, &defaults); err != nil {
+ return defaults, err
+ }
+
+ if err := defaults.decodeAdapters(adapterCfg, true); err != nil {
+ return defaults, err
+ }
+
+
+ return defaults, nil
+}
+
+// Decode an individual unit
+func DecodeRubricConfig(path string, adapterCfg []AdapterConfig) (TestCaseConfig, error) {
+ config := TestCaseConfig { }
+
+ if _, err := toml.DecodeFile(path, &config); err != nil {
+ return config, nil
+ }
+
+ if err := config.decodeAdapters(adapterCfg, false); err != nil {
+ return config, err
+ }
+
+ return config, nil
+}
diff --git a/runner.go b/runner.go
index 3bee17a..15f08ae 100644
--- a/runner.go
+++ b/runner.go
@@ -3,14 +3,13 @@ package planr
import (
"log"
"os"
+ "path"
+ "sort"
)
type Runner struct {
- adapters []Adapter
-}
-
-func (r *Runner) RegisterAdapter(a Adapter) {
- r.adapters = append(r.adapters, a)
+ adapters map[string] Adapter
+ dirs DirConfig
}
func (r Runner) adapterCfgs() []AdapterConfig {
@@ -23,71 +22,143 @@ func (r Runner) adapterCfgs() []AdapterConfig {
return cgs
}
-type TcTab map[string] []*TestCase
-
-func (r Runner) buildTcLUT(tcs []TestCase) TcTab {
- m := make(TcTab, 0)
-
- for i := range tcs {
- tc := &tcs[i]
- nm := *tc.Config.Adapter
- m[nm] = append(m[nm], tc)
- }
-
- return m
-}
-
+// TODO: Move into configuration parsing
func (r Runner) checkConfig(tcs []TestCase) {
for _, tc := range tcs {
tc.Config.ensureSatisfied(tc.Path)
}
}
-func cdBuild(adapter Adapter) {
- dir := adapter.Config().Dir()
+func (r Runner) setupEnv(adapter Adapter) {
+ nm := adapter.Config().Name
+ wd := path.Join(r.dirs.Build(), nm)
- if err := os.Chdir(dir); err != nil {
- log.Fatal(err)
- }
+ if !directoryExists(wd) {
+ if err := os.Mkdir(wd, 0755); err != nil {
+ log.Fatalf("Could not create adapter config %s %v\n", wd, err)
+ }
+ }
+
+ safeCd(wd)
+}
+
+type adapterTestSet struct {
+ adapter Adapter
+ tcs []TestCase
}
-func (r Runner) build(tcs []TestCase) {
+func (r Runner) groupByAdapter(tcs []TestCase) []adapterTestSet {
r.checkConfig(tcs)
+
+ pairs := make(map[string] adapterTestSet, 0)
- tcTab := r.buildTcLUT(tcs)
+ for _, tc := range tcs {
+ // TODO: Make non-pointer
+ adptNm := *tc.Config.Adapter
+
+ // See if adapter if contained in map
+ adapter, contained := r.adapters[adptNm]
- for _, adapter := range r.adapters {
- nm := adapter.Config().Name
- cdBuild(adapter)
+ if !contained {
+ log.Fatalf("Cannot find adapter \"%s\" for testcase \"%s\"", adptNm, tc.Cname)
+ }
- adapter.Build(tcTab[nm])
+ pair, exists := pairs[adptNm]
+
+ if !exists {
+ pair.adapter = adapter
+ }
+
+ pair.tcs = append(pair.tcs, tc)
+
+ pairs[adptNm] = pair
}
+
+
+ // Convert to slice
+ set := make([]adapterTestSet, 0)
+
+ for _, pair := range pairs {
+ set = append(set, pair)
+ }
+
+ return set
}
-func (r Runner) units(root string) []TestCase {
- return collectUnits(root, r.adapterCfgs())
+func (r Runner) CollectCases() []TestCase {
+ return collectUnits(r.dirs.Rubric(), r.adapterCfgs())
}
-func (r Runner) Build(root string) {
- units := r.units(root)
- r.build(units)
+func (r Runner) Build(tcs []TestCase) {
+
+ if !directoryExists(r.dirs.Build()) {
+ r.dirs.MkBuild()
+ }
+
+ testSets := r.groupByAdapter(tcs)
+
+ for _, pair := range testSets {
+ adapter := pair.adapter
+ cases := pair.tcs
+
+ r.setupEnv(adapter)
+
+ adapter.Build(cases)
+ }
+
+ safeCd(r.dirs.Config())
}
-func (r Runner) evaluate(tcs []TestCase) {
- tcTab := r.buildTcLUT(tcs)
+func (r Runner) Evaluate(tcs []TestCase) []TestResult {
+ testSets := r.groupByAdapter(tcs)
+ results := make([]TestResult, 0)
- for _, adapter := range r.adapters {
- nm := adapter.Config().Name
- cdBuild(adapter)
+ c := make(chan []TestResult)
+ for _, pair := range testSets {
+ go func (pair adapterTestSet) {
+ adapter := pair.adapter
+ cases := pair.tcs
- adapter.Evaluate(tcTab[nm])
+ r.setupEnv(adapter)
+ resultSet := adapter.Evaluate(cases)
+
+ c <- resultSet
+ }(pair)
+ }
+
+ for range testSets {
+ results = append(results, (<-c)...)
}
+
+ sort.Sort(ByReadIdx(results))
+
+ safeCd(r.dirs.Config())
+
+ return results
+}
+
+func (r Runner) Clean() {
+ r.dirs.CleanBuild()
}
-func (r Runner) Evaluate(root string) []TestCase {
- units := r.units(root)
+func (r Runner) BuildDir() string {
+ return r.dirs.Build();
+}
+
+func (r Runner) ConfigDir() string {
+ return r.dirs.Config()
+}
- r.evaluate(units)
+func (r Runner) SrcDir() string {
+ return r.dirs.Src()
+}
+
+func NewRunner(adapters map[string]Adapter, dirs DirConfig) Runner {
+ r := Runner{adapters, dirs}
+
+ for _, adapter := range r.adapters {
+ adapter.Init(dirs)
+ }
- return units
+ return r
}
diff --git a/runner_builder.go b/runner_builder.go
new file mode 100644
index 0000000..b3c07d9
--- /dev/null
+++ b/runner_builder.go
@@ -0,0 +1,42 @@
+package planr
+
+type RunnerBuilder struct {
+ adapters map[string] Adapter
+ dirs DirConfig
+}
+
+func ConfigureRunner() RunnerBuilder {
+ builder := RunnerBuilder{}
+ builder.adapters = make(map[string] Adapter, 0)
+ return builder
+}
+
+func RegisterAdapter(b RunnerBuilder, a Adapter) RunnerBuilder {
+ nm := a.Config().Name
+ b.adapters[nm] = a
+ return b
+}
+
+func SetConfigDirFromTree(b RunnerBuilder, childPath string) RunnerBuilder {
+ b.dirs.SetConfigFromTree(childPath)
+ return b
+}
+
+func SetConfigDir(b RunnerBuilder, dir string) RunnerBuilder {
+ b.dirs.SetConfig(dir)
+ return b
+}
+
+func SetBuildDir(b RunnerBuilder, dir string) RunnerBuilder {
+ b.dirs.SetBuild(dir)
+ return b
+}
+
+func SetSrcDir(b RunnerBuilder, dir string) RunnerBuilder {
+ b.dirs.SetSrc(dir)
+ return b
+}
+
+func (b RunnerBuilder) New() Runner {
+ return NewRunner(b.adapters, b.dirs)
+}
diff --git a/scoring.go b/scoring.go
new file mode 100644
index 0000000..675058a
--- /dev/null
+++ b/scoring.go
@@ -0,0 +1,31 @@
+package planr
+
+type Scoring struct {
+ EarnedPoints float64
+ TotalPoints float64
+ Passed int
+ Total int
+}
+
+func Score(trs []TestResult) Scoring {
+ score := Scoring {}
+
+ for _, tr := range trs {
+ cfg := tr.Tc.Config
+ points := 0.0
+
+ if cfg.Points != nil {
+ points = float64(*cfg.Points)
+ }
+
+ score.TotalPoints += points
+ if tr.Status == PASSING {
+ score.EarnedPoints += points
+ score.Passed++
+ }
+
+ score.Total += 1
+ }
+
+ return score
+}
diff --git a/stddirs.go b/stddirs.go
new file mode 100644
index 0000000..581af87
--- /dev/null
+++ b/stddirs.go
@@ -0,0 +1,190 @@
+package planr
+
+import (
+ "os"
+ "log"
+ "path"
+ "path/filepath"
+)
+
+// Standard Directories:
+//
+// Planr relies on a standard directory structure to resolve paths to source code
+// These directories can be overridden via environmental variables //
+// Directories:
+// - `.` root directory
+// - `src` contains source code
+// - `planr` contains test metadata (also called configuration directory)
+// - `planr/rubric` contains test case configuration
+// - `planr/tests` contains source code for test cases
+// - `planr/build` contains all files written during the build process (ephemeral)
+
+// Env overrides
+const (
+ ENV_CONFIG_DIR="PLANR_CONFIG_DIR"
+ ENV_SRC_DIR="PLANR_SRC_DIR"
+ ENV_BUILD_DIR="PLANR_BUILD_DIR"
+)
+
+// Try these search directories
+var CONFIG_SEARCH_DIRS = [] string {
+ "planr",
+ ".planr",
+}
+
+// Path are relative to the "planr" project config directory
+const (
+ DEFAULT_PATH_SRC="../src"
+ DEFAULT_PATH_BUILD="build"
+ DEFAULT_PATH_RUBRIC="rubric"
+ DEFAULT_PATH_TESTS="tests"
+)
+
+type DirConfig struct {
+ src string
+ config string
+ build string
+
+ // Config falls back to the config found in the parent directory if the env variable hasn't been overridden
+ pdFallback string
+}
+
+func dieDirAbsent(name, path string) {
+ if !directoryExists(path) {
+ log.Fatalf("Could not find %s directory tried %s", name, path)
+ }
+}
+
+func dirFromEnv(name, env string) *string {
+ if dir, isSet := os.LookupEnv(env); isSet {
+
+ dieDirAbsent(name, dir)
+
+ return &dir
+ }
+
+ return nil
+}
+
+func (c *DirConfig) SetSrc(srcDir string) {
+ dieDirAbsent("src", srcDir)
+ c.src = abs(srcDir)
+}
+
+func (c *DirConfig) SetConfig(configDir string) {
+ dieDirAbsent("planr (config)", configDir)
+ c.config = abs(configDir)
+}
+
+func (c *DirConfig) SetBuild(buildDir string) {
+ dieDirAbsent("build", buildDir)
+ c.build = abs(buildDir)
+}
+
+func (c *DirConfig) SetConfigFromTree(cdir string) {
+ var configDir string
+
+ found := traverseUp(cdir, func (path string) bool {
+
+ for _, dir := range CONFIG_SEARCH_DIRS {
+ configDir = filepath.Join(path, dir)
+
+ if directoryExists(configDir) {
+ return true
+ }
+ }
+
+ return false
+ });
+
+
+ if found {
+ c.pdFallback = configDir
+ }
+}
+
+func (c DirConfig) Config() string {
+ if c.config != "" {
+ return c.config
+ }
+
+ if dir := dirFromEnv("config", ENV_CONFIG_DIR); dir != nil {
+ c.config = abs(*dir)
+ return c.config
+ }
+
+ if c.pdFallback == "" {
+ log.Fatal("Could not find planr directory");
+ }
+
+ c.config = abs(c.pdFallback);
+ return c.config
+}
+
+func (c DirConfig) Src() string {
+ if c.src != "" {
+ return c.src
+ }
+
+ if dir := dirFromEnv("src", ENV_SRC_DIR); dir != nil {
+ c.src = *dir
+ return c.src
+ }
+
+ // set path relative to config
+ dir := c.Config()
+ return path.Join(dir, DEFAULT_PATH_SRC)
+}
+
+func abs(path string) string {
+ apath, err := filepath.Abs(path)
+
+ if err != nil {
+ log.Fatalf("Could not find path %s", path)
+ }
+
+ return apath
+}
+
+func (c DirConfig) Build() string {
+ if c.build != "" {
+ return c.build
+ }
+
+ if dir := dirFromEnv("build", ENV_BUILD_DIR); dir != nil {
+ c.build = *dir
+ return c.build
+ }
+
+ dir := c.Config()
+ return path.Join(dir, DEFAULT_PATH_BUILD)
+}
+
+func (c DirConfig) CleanBuild() {
+ build := c.Build()
+
+ if err := os.RemoveAll(build); err != nil {
+ log.Fatalf("Cannot build directory %v\n", err)
+ }
+
+}
+
+func (c DirConfig) MkBuild() {
+ build := c.Build()
+
+ if err := os.Mkdir(build, 0755); err != nil {
+ log.Fatalf("Could not create build directory %v\n", err)
+ }
+}
+
+func (c DirConfig) Rubric() string {
+ rubric := path.Join(c.Config(), DEFAULT_PATH_RUBRIC)
+ dieDirAbsent("rubric", rubric)
+ return rubric
+}
+
+func (c DirConfig) Tests() string {
+ tests := path.Join(c.Config(), DEFAULT_PATH_TESTS)
+ dieDirAbsent("tests", tests)
+ return tests
+}
diff --git a/testcase.go b/testcase.go
index 8506b84..d1db292 100644
--- a/testcase.go
+++ b/testcase.go
@@ -1,9 +1,14 @@
package planr
+import (
+ "log"
+)
+
type TestStatus uint
const (
- PASSING TestStatus = iota
+ NOT_RUN TestStatus = iota
+ PASSING
COMPILATION_FAILURE
RUNTIME_FAILURE
)
@@ -13,6 +18,20 @@ type TestResult struct {
Status TestStatus
DebugOutput string
TestOutput string
+ Tc TestCase
+}
+
+// Program-wide testcase config
+type TestCaseConfig struct {
+ Defaults
+ Title *string
+ Description *string
+}
+
+func (c TestCaseConfig) ensureSatisfied(name string) {
+ if (c.Adapter == nil) {
+ log.Fatalf("Adapter must be provided for testcase %s", name)
+ }
}
type TestCase struct {
@@ -26,11 +45,17 @@ type TestCase struct {
Cname string
Config TestCaseConfig
-
- Result *TestResult
-
+
+ // Reorder according to original read order after concurrent operation
+ readIdx int
}
func (tc TestCase) AdapterConfig() InheritableConfig {
return tc.Config.adapters_[*tc.Config.Adapter]
}
+
+type ByReadIdx []TestResult
+
+func (a ByReadIdx) Len() int { return len(a) }
+func (a ByReadIdx) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByReadIdx) Less(i, j int) bool { return a[i].Tc.readIdx < a[j].Tc.readIdx }
diff --git a/version.go b/version.go
new file mode 100644
index 0000000..52c7c5d
--- /dev/null
+++ b/version.go
@@ -0,0 +1,3 @@
+package planr
+
+const VERSION = "0.1.0"
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 13:26:17 +0000