123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327 |
- // Copyright 2018 The Cockroach Authors.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- // implied. See the License for the specific language governing
- // permissions and limitations under the License.
- package datadriven
- import (
- "bufio"
- "flag"
- "fmt"
- "io"
- "io/ioutil"
- "os"
- "path/filepath"
- "strconv"
- "strings"
- "testing"
- )
- var (
- rewriteTestFiles = flag.Bool(
- "rewrite", false,
- "ignore the expected results and rewrite the test files with the actual results from this "+
- "run. Used to update tests when a change affects many cases; please verify the testfile "+
- "diffs carefully!",
- )
- )
- // RunTest invokes a data-driven test. The test cases are contained in a
- // separate test file and are dynamically loaded, parsed, and executed by this
- // testing framework. By convention, test files are typically located in a
- // sub-directory called "testdata". Each test file has the following format:
- //
- // <command>[,<command>...] [arg | arg=val | arg=(val1, val2, ...)]...
- // <input to the command>
- // ----
- // <expected results>
- //
- // The command input can contain blank lines. However, by default, the expected
- // results cannot contain blank lines. This alternate syntax allows the use of
- // blank lines:
- //
- // <command>[,<command>...] [arg | arg=val | arg=(val1, val2, ...)]...
- // <input to the command>
- // ----
- // ----
- // <expected results>
- //
- // <more expected results>
- // ----
- // ----
- //
- // To execute data-driven tests, pass the path of the test file as well as a
- // function which can interpret and execute whatever commands are present in
- // the test file. The framework invokes the function, passing it information
- // about the test case in a TestData struct. The function then returns the
- // actual results of the case, which this function compares with the expected
- // results, and either succeeds or fails the test.
- func RunTest(t *testing.T, path string, f func(d *TestData) string) {
- t.Helper()
- file, err := os.OpenFile(path, os.O_RDWR, 0644 /* irrelevant */)
- if err != nil {
- t.Fatal(err)
- }
- defer func() {
- _ = file.Close()
- }()
- runTestInternal(t, path, file, f, *rewriteTestFiles)
- }
- // RunTestFromString is a version of RunTest which takes the contents of a test
- // directly.
- func RunTestFromString(t *testing.T, input string, f func(d *TestData) string) {
- t.Helper()
- runTestInternal(t, "<string>" /* optionalPath */, strings.NewReader(input), f, *rewriteTestFiles)
- }
- func runTestInternal(
- t *testing.T, sourceName string, reader io.Reader, f func(d *TestData) string, rewrite bool,
- ) {
- t.Helper()
- r := newTestDataReader(t, sourceName, reader, rewrite)
- for r.Next(t) {
- t.Run("", func(t *testing.T) {
- d := &r.data
- actual := func() string {
- defer func() {
- if r := recover(); r != nil {
- fmt.Printf("\npanic during %s:\n%s\n", d.Pos, d.Input)
- panic(r)
- }
- }()
- actual := f(d)
- if !strings.HasSuffix(actual, "\n") {
- actual += "\n"
- }
- return actual
- }()
- if r.rewrite != nil {
- r.emit("----")
- if hasBlankLine(actual) {
- r.emit("----")
- r.rewrite.WriteString(actual)
- r.emit("----")
- r.emit("----")
- } else {
- r.emit(actual)
- }
- } else if d.Expected != actual {
- t.Fatalf("\n%s: %s\nexpected:\n%s\nfound:\n%s", d.Pos, d.Input, d.Expected, actual)
- } else if testing.Verbose() {
- input := d.Input
- if input == "" {
- input = "<no input to command>"
- }
- // TODO(tbg): it's awkward to reproduce the args, but it would be helpful.
- fmt.Printf("\n%s:\n%s [%d args]\n%s\n----\n%s", d.Pos, d.Cmd, len(d.CmdArgs), input, actual)
- }
- })
- if t.Failed() {
- t.FailNow()
- }
- }
- if r.rewrite != nil {
- data := r.rewrite.Bytes()
- if l := len(data); l > 2 && data[l-1] == '\n' && data[l-2] == '\n' {
- data = data[:l-1]
- }
- if dest, ok := reader.(*os.File); ok {
- if _, err := dest.WriteAt(data, 0); err != nil {
- t.Fatal(err)
- }
- if err := dest.Truncate(int64(len(data))); err != nil {
- t.Fatal(err)
- }
- if err := dest.Sync(); err != nil {
- t.Fatal(err)
- }
- } else {
- t.Logf("input is not a file; rewritten output is:\n%s", data)
- }
- }
- }
- // Walk goes through all the files in a subdirectory, creating subtests to match
- // the file hierarchy; for each "leaf" file, the given function is called.
- //
- // This can be used in conjunction with RunTest. For example:
- //
- // datadriven.Walk(t, path, func (t *testing.T, path string) {
- // // initialize per-test state
- // datadriven.RunTest(t, path, func (d *datadriven.TestData) {
- // // ...
- // }
- // }
- //
- // Files:
- // testdata/typing
- // testdata/logprops/scan
- // testdata/logprops/select
- //
- // If path is "testdata/typing", the function is called once and no subtests
- // care created.
- //
- // If path is "testdata/logprops", the function is called two times, in
- // separate subtests /scan, /select.
- //
- // If path is "testdata", the function is called three times, in subtest
- // hierarchy /typing, /logprops/scan, /logprops/select.
- //
- func Walk(t *testing.T, path string, f func(t *testing.T, path string)) {
- finfo, err := os.Stat(path)
- if err != nil {
- t.Fatal(err)
- }
- if !finfo.IsDir() {
- f(t, path)
- return
- }
- files, err := ioutil.ReadDir(path)
- if err != nil {
- t.Fatal(err)
- }
- for _, file := range files {
- t.Run(file.Name(), func(t *testing.T) {
- Walk(t, filepath.Join(path, file.Name()), f)
- })
- }
- }
- // TestData contains information about one data-driven test case that was
- // parsed from the test file.
- type TestData struct {
- Pos string // reader and line number
- // Cmd is the first string on the directive line (up to the first whitespace).
- Cmd string
- CmdArgs []CmdArg
- Input string
- Expected string
- }
- // ScanArgs looks up the first CmdArg matching the given key and scans it into
- // the given destinations in order. If the arg does not exist, the number of
- // destinations does not match that of the arguments, or a destination can not
- // be populated from its matching value, a fatal error results.
- //
- // For example, for a TestData originating from
- //
- // cmd arg1=50 arg2=yoruba arg3=(50, 50, 50)
- //
- // the following would be valid:
- //
- // var i1, i2, i3, i4 int
- // var s string
- // td.ScanArgs(t, "arg1", &i1)
- // td.ScanArgs(t, "arg2", &s)
- // td.ScanArgs(t, "arg3", &i2, &i3, &i4)
- func (td *TestData) ScanArgs(t *testing.T, key string, dests ...interface{}) {
- t.Helper()
- var arg CmdArg
- for i := range td.CmdArgs {
- if td.CmdArgs[i].Key == key {
- arg = td.CmdArgs[i]
- break
- }
- }
- if arg.Key == "" {
- t.Fatalf("missing argument: %s", key)
- }
- if len(dests) != len(arg.Vals) {
- t.Fatalf("%s: got %d destinations, but %d values", arg.Key, len(dests), len(arg.Vals))
- }
- for i := range dests {
- arg.Scan(t, i, dests[i])
- }
- }
- // CmdArg contains information about an argument on the directive line. An
- // argument is specified in one of the following forms:
- // - argument
- // - argument=value
- // - argument=(values, ...)
- type CmdArg struct {
- Key string
- Vals []string
- }
- func (arg CmdArg) String() string {
- switch len(arg.Vals) {
- case 0:
- return arg.Key
- case 1:
- return fmt.Sprintf("%s=%s", arg.Key, arg.Vals[0])
- default:
- return fmt.Sprintf("%s=(%s)", arg.Key, strings.Join(arg.Vals, ", "))
- }
- }
- // Scan attempts to parse the value at index i into the dest.
- func (arg CmdArg) Scan(t *testing.T, i int, dest interface{}) {
- if i < 0 || i >= len(arg.Vals) {
- t.Fatalf("cannot scan index %d of key %s", i, arg.Key)
- }
- val := arg.Vals[i]
- switch dest := dest.(type) {
- case *string:
- *dest = val
- case *int:
- n, err := strconv.ParseInt(val, 10, 64)
- if err != nil {
- t.Fatal(err)
- }
- *dest = int(n) // assume 64bit ints
- case *uint64:
- n, err := strconv.ParseUint(val, 10, 64)
- if err != nil {
- t.Fatal(err)
- }
- *dest = n
- case *bool:
- b, err := strconv.ParseBool(val)
- if err != nil {
- t.Fatal(err)
- }
- *dest = b
- default:
- t.Fatalf("unsupported type %T for destination #%d (might be easy to add it)", dest, i+1)
- }
- }
- // Fatalf wraps a fatal testing error with test file position information, so
- // that it's easy to locate the source of the error.
- func (td TestData) Fatalf(tb testing.TB, format string, args ...interface{}) {
- tb.Helper()
- tb.Fatalf("%s: %s", td.Pos, fmt.Sprintf(format, args...))
- }
- func hasBlankLine(s string) bool {
- scanner := bufio.NewScanner(strings.NewReader(s))
- for scanner.Scan() {
- if strings.TrimSpace(scanner.Text()) == "" {
- return true
- }
- }
- return false
- }
|