Add support for test timeouts

4 files changed, 661 insertions, 19 deletions

diff --git a/tests/test/script/runner/driver.cxx b/tests/test/script/runner/driver.cxx
index d5a74a4..935541d 100644
--- a/tests/test/script/runner/driver.cxx
+++ b/tests/test/script/runner/driver.cxx
@@ -1,6 +1,13 @@
 // file      : tests/test/script/runner/driver.cxx -*- C++ -*-
 // license   : MIT; see accompanying LICENSE file
 
+#ifndef _WIN32
+#  include <thread> // this_thread::sleep_for()
+#  include <chrono>
+#else
+#  include <libbutl/win32-utility.hxx>
+#endif
+
 #include <limits>    // numeric_limits
 #include <string>
 #include <cstdlib>   // abort()
@@ -36,10 +43,10 @@ main (int argc, char* argv[])
   // Usage: driver [-i <int>] (-o <string>)* (-e <string>)* (-f <file>)*
   //        (-d <dir>)* (-v <name>)* [(-t (a|m|s|z)) | (-s <int>)]
   //
-  // Execute actions specified by -i, -o, -e, -f, -d, and -v options in the
-  // order as they appear on the command line. After that terminate abnormally
-  // if -t option is provided, otherwise exit normally with the status
-  // specified by -s option (0 by default).
+  // Execute actions specified by -i, -o, -e, -f, -d, -v, and -l options in
+  // the order as they appear on the command line. After that terminate
+  // abnormally if -t option is provided, otherwise exit normally with the
+  // status specified by -s option (0 by default).
   //
   // -i <fd>
   //    Forward stdin data to the standard stream denoted by the file
@@ -62,6 +69,9 @@ main (int argc, char* argv[])
   //    If the specified variable is set the print its value to stdout and the
   //    string '<none>' otherwise.
   //
+  // -l <sec>
+  //    Sleep the specified number of seconds.
+  //
   // -t <method>
   //    Abnormally terminate itself using one of the following methods:
   //
@@ -144,6 +154,18 @@ main (int argc, char* argv[])
       optional<string> var (getenv (v));
       cout << (var ? *var : "<none>") << endl;
     }
+    else if (o == "-l")
+    {
+      size_t t (toi (v));
+
+       // MinGW GCC 4.9 doesn't implement this_thread so use Win32 Sleep().
+       //
+#ifndef _WIN32
+      this_thread::sleep_for (chrono::seconds (t));
+#else
+      Sleep (static_cast<DWORD> (t * 1000));
+#endif
+    }
     else if (o == "-t")
     {
       assert (aterm == '\0' && !status); // Make sure exit method is not set.
diff --git a/tests/test/script/runner/env.testscript b/tests/test/script/runner/env.testscript
index 6fcedfa..ef90c3b 100644
--- a/tests/test/script/runner/env.testscript
+++ b/tests/test/script/runner/env.testscript
@@ -3,5 +3,30 @@
 
 .include ../common.testscript
 
-$c <'env abc=xyz -- $* -v abc >xyz' && $b                         : set
-$c <'env --unset=abc -- $* -v abc >"<none>"' && env abc=xyz -- $b : unset
+: variables
+:
+{
+  $c <'env abc=xyz -- $* -v abc >xyz' && $b                         : set
+  $c <'env --unset=abc -- $* -v abc >"<none>"' && env abc=xyz -- $b : unset
+}
+
+: timeout
+:
+{
+  : expired
+  :
+  $c <'env --timeout 1 -- $* -l 5' && $b 2>>~%EOE% != 0
+  %testscript:1:1: error: .+ terminated: execution timeout expired%
+    info: test id: 1
+  EOE
+
+  : not-expired
+  :
+  $c <'env --timeout 5 -- $* -l 1' && $b
+
+  : invalid
+  :
+  $c <'env --timeout a -- $*' && $b 2>>EOE != 0
+  testscript:1:15: error: env: invalid value 'a' for option '--timeout'
+  EOE
+}
diff --git a/tests/test/script/runner/set.testscript b/tests/test/script/runner/set.testscript
index 9219cbb..41709e4 100644
--- a/tests/test/script/runner/set.testscript
+++ b/tests/test/script/runner/set.testscript
@@ -94,10 +94,28 @@
 {
   : non-exact
   :
-  $c <<EOI && $b
-  set -w baz <' foo   bar ';
-  echo $baz >'foo bar'
-  EOI
+  {
+    : non-empty
+    :
+    $c <<EOI && $b
+    set -w baz <' foo   bar ';
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar"'
+    EOI
+
+    : empty
+    :
+    $c <<EOI && $b
+    set -w baz <:'';
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >''
+    EOI
+
+    : spaces
+    :
+    $c <<EOI && $b
+    set -w baz <'  ';
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >''
+    EOI
+  }
 
   : exact
   :
@@ -106,7 +124,7 @@
     :
     $c <<EOI && $b
     set --exact --whitespace baz <' foo   bar ';
-    echo $baz >'foo bar '
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar" ""'
     EOI
 
     : no-trailing-ws
@@ -115,8 +133,22 @@
     : ':' modifier.
     :
     $c <<EOI && $b
-    set --exact --whitespace baz <:' foo bar';
-    echo $baz >'foo bar'
+    set -e -w baz <:' foo bar';
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >'"foo" "bar"'
+    EOI
+
+    : empty
+    :
+    $c <<EOI && $b
+    set -e -w baz <:'';
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >''
+    EOI
+
+    : spaces
+    :
+    $c <<EOI && $b
+    set -e -w baz <'  ';
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >'""'
     EOI
   }
 }
@@ -134,7 +166,7 @@
   bar
 
   EOF
-  echo $baz >' foo  bar '
+  echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" ""'
   EOI
 
   : exact
@@ -150,7 +182,7 @@
     bar
 
     EOF
-    echo $baz >' foo  bar  '
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar" "" ""'
     EOI
 
     : no-trailing-newline
@@ -162,7 +194,7 @@
 
     bar
     EOF
-    echo $baz >' foo  bar'
+    echo $regex.apply($baz, '^(.*)$', '"\1"') >'"" "foo" "" "bar"'
     EOI
   }
 }
@@ -180,7 +212,7 @@
   bar
 
   EOF
-  echo $baz >>EOO
+  echo ($baz[0]) >>EOO
 
   foo
 
@@ -209,7 +241,7 @@
 
     bar
     EOF
-    echo "$baz" >>EOO
+    echo ($baz[0]) >>EOO
 
     foo
 
@@ -227,7 +259,7 @@
 
     bar
     EOF
-    echo "$baz" >>EOO
+    echo ($baz[0]) >>EOO
 
     foo
 
@@ -237,6 +269,66 @@
   }
 }
 
+: deadline
+:
+{
+  : not-reached
+  :
+  $c <<EOI && $b
+  env -t 10 -- $* -o 'foo' | set bar;
+  echo $bar >foo 2>|
+  EOI
+
+  : set-reached
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  $* -o 'foo' -l 10 | env -t 1 -- set bar
+  EOI
+  %testscript:.*: error: set terminated: execution timeout expired%
+  %.
+  EOE
+
+  : driver-reached
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  env -t 1 -- $* -o 'foo' -l 10 | set bar
+  EOI
+  %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+  %.
+  EOE
+
+  : read-some-data
+  :
+  {
+    s="----------------------------------------------------------------------"
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s"
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s"
+
+    : failure
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+    echo "$s" >=f;
+    $* -o 'foo' -l 10 | cat f - | env -t 2 -- set bar
+    EOI
+    %testscript:.*: error: set terminated: execution timeout expired%
+    %.
+    EOE
+
+    : success
+    :
+    : Note that the cat builtin ends up with the 'broken pipe' diagnostics or
+    : similar.
+    :
+    $c <<EOI && $b
+    echo "$s" >=f;
+    timeout --success 2;
+    $* -o 'foo' -l 10 | cat f - 2>>~%EOE% | set bar
+      %cat: .+%
+      EOE
+    EOI
+  }
+}
+
 : attributes
 :
 {
diff --git a/tests/test/script/runner/timeout.testscript b/tests/test/script/runner/timeout.testscript
new file mode 100644
index 0000000..ae8f535
--- /dev/null
+++ b/tests/test/script/runner/timeout.testscript
@@ -0,0 +1,503 @@
+# file      : tests/test/script/runner/timeout.testscript
+# license   : MIT; see accompanying LICENSE file
+
+.include ../common.testscript
+
+: test
+:
+{
+  : fragment-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      timeout 1;
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      timeout 1;
+      timeout 0;
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout /10
+
+      {
+        +timeout /10
+
+        timeout 1;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : successful
+    :
+    $c <<EOI && $b
+      timeout --success 1;
+      $* -l 3
+      EOI
+  }
+
+  : missing
+  :
+  $c <'timeout' && $b 2>>~%EOE% != 0
+    testscript:1:1: error: missing timeout
+    %.
+    EOE
+
+  : invalid
+  :
+  $c <'timeout foo' && $b 2>>~%EOE% != 0
+    testscript:1:1: error: invalid test fragment timeout 'foo'
+    %.
+    EOE
+}
+
+: group
+:
+{
+  : group-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout 1
+
+        $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      {
+        +timeout 1
+        +timeout 0
+
+        $* -l 3
+      }
+      EOI
+
+    : override
+    :
+    : Also test slash usage inside the timeout value.
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 10/10
+
+      {
+        +timeout 1/
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout foo/
+      }
+      EOI
+      testscript:2:4: error: invalid test group timeout 'foo'
+      %.
+      EOE
+  }
+
+  : test-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout /1
+
+        $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      {
+        +timeout /1
+        +timeout /0
+
+        $* -l 3
+      }
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 10/10
+
+      {
+        +timeout /1
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout /foo
+      }
+      EOI
+      testscript:2:4: error: invalid test timeout 'foo'
+      %.
+      EOE
+  }
+}
+
+: script
+:
+{
+  : group-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 1
+
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      +timeout 1
+      +timeout 0
+
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout 1
+
+      {
+        +timeout 10
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout foo/
+      EOI
+      testscript:1:2: error: invalid testscript timeout 'foo'
+      %.
+      EOE
+
+    : successful
+    :
+    $c <<EOI && $b
+      +timeout --success 1
+
+      $* -l 3
+      EOI
+  }
+
+  : test-timeout
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout /1
+
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b
+      +timeout /1
+      +timeout /0
+
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      +timeout /1
+
+      {
+        +timeout --success /1
+
+        {
+          +timeout 10/10
+
+          timeout 10;
+          env -t 10 -- $* -l 3
+        }
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : successful
+    :
+    $c <<EOI && $b
+      +timeout --success /1
+
+      $* -l 3
+      EOI
+
+    : invalid
+    :
+    $c <<EOI && $b 2>>~%EOE% != 0
+      {
+        +timeout /foo
+      }
+      EOI
+      testscript:2:4: error: invalid test timeout 'foo'
+      %.
+      EOE
+  }
+}
+
+: config
+:
+{
+  : operation
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b config.test.timeout=1/10 2>>~%EOE% != 0
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b config.test.timeout=1/10 config.test.timeout=0/10
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b config.test.timeout=1/10 2>>~%EOE% != 0
+      +timeout 10
+
+      {
+        +timeout 10/10
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c && $b config.test.timeout=foo 2>>EOE != 0
+      error: invalid config.test.timeout test operation timeout value 'foo'
+      EOE
+  }
+
+  : test
+  :
+  {
+    : set
+    :
+    $c <<EOI && $b config.test.timeout=10/1 2>>~%EOE% != 0
+      $* -l 3
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : reset
+    :
+    $c <<EOI && $b config.test.timeout=10/1 config.test.timeout=10/0
+      $* -l 3
+      EOI
+
+    : override
+    :
+    $c <<EOI && $b config.test.timeout=10/1 2>>~%EOE% != 0
+      +timeout 10
+
+      {
+        +timeout 10/10
+
+        timeout 10;
+        env -t 10 -- $* -l 3
+      }
+      EOI
+      %testscript:.*: error: .+ terminated: execution timeout expired%
+      %.
+      EOE
+
+    : invalid
+    :
+    $c && $b config.test.timeout=/foo 2>>EOE != 0
+      error: invalid config.test.timeout test timeout value 'foo'
+      EOE
+  }
+}
+
+: failures
+:
+: Here we test that the left-hand side processes are terminated on failure.
+:
+{
+  : set
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  env -t 1 -- $* -l 86400 -o 'foo' | set --foo bar
+  EOI
+  %testscript:.*: error: set: unknown option '--foo'%
+  %.
+  EOE
+
+  : exit
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  env -t 1 -- $* -l 86400 -o 'foo' | exit 0
+  EOI
+  %testscript:.*: error: exit builtin must be the only pipe command%
+  %.
+  EOE
+
+  : redirect
+  :
+  $c <<EOI && $b 2>>~%EOE% != 0
+  env -t 1 -- $* -l 86400 -o 'foo' | touch $~/foo/bar
+  EOI
+  %testscript:.*: error: touch exited with code 1%
+  %.+
+  EOE
+}
+
+: pipeline
+:
+{
+  : prog-tm-prog
+  :
+  $c <'$* -l 10 | env -t 1 -- $* -i 0' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-prog-prog
+  :
+  $c <'env -t 1 -- $* -l 10 | $* -i 0' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-cat-prog
+  :
+  $c <'env -t 1 -- cat <"test" | $* -l 10' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: cat terminated: execution timeout expired%
+    %.
+    EOE
+
+  : cat-tm-prog
+  :
+  $c <'cat <"test" | env -t 1 -- $* -l 10' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-prog-cat
+  :
+  $c <'env -t 1 -- $* -l 10 | cat >-' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: .+driver.* terminated: execution timeout expired%
+    %.
+    EOE
+
+  : tm-echo-prog
+  :
+  $c <'env -t 1 -- echo "test" | $* -l 10' && $b 2>>~%EOE% != 0
+    %testscript:.*: error: echo terminated: execution timeout expired%
+    %.
+    EOE
+
+  : successful
+  :
+  {
+    : prog-prog
+    :
+    $c <<EOI && $b
+    timeout --success 1;
+    $* -l 10 | $* -i 0
+    EOI
+
+    : prog-cat
+    :
+    $c <<EOI && $b
+    timeout --success 1;
+    $* -l 10 | cat
+    EOI
+
+    : cat-prog
+    :
+    $c <<EOI && $b
+    s="----------------------------------------------------------------------";
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s";
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s";
+    s="$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s$s";
+    timeout --success 1;
+    cat <"$s" 2>>~%EOE% | $* -l 10 -i 0
+    %cat: unable to print stdin: .+%
+    EOE
+    EOI
+  }
+}