Implement description support in testscript

11 files changed, 447 insertions, 26 deletions

diff --git a/unit-tests/test/script/lexer/buildfile b/unit-tests/test/script/lexer/buildfile
index 30f6b9f..d9b17e8 100644
--- a/unit-tests/test/script/lexer/buildfile
+++ b/unit-tests/test/script/lexer/buildfile
@@ -8,7 +8,7 @@ import libs = libbutl%lib{butl}
 src = token lexer diagnostics utility variable name test/script/{token lexer}
 
 exe{driver}: cxx{driver} ../../../../build2/cxx{$src} $libs \
-test{script-line command-line first-token second-token variable-line variable \
-     comment}
+test{script-line command-line first-token second-token variable-line \
+     description-line variable comment}
 
 include ../../../../build2/
diff --git a/unit-tests/test/script/lexer/description-line.test b/unit-tests/test/script/lexer/description-line.test
new file mode 100644
index 0000000..f0f0c9f
--- /dev/null
+++ b/unit-tests/test/script/lexer/description-line.test
@@ -0,0 +1,17 @@
+test.arguments += description-line
+
+$* <" foo  bar " >>EOO # full
+' foo  bar '
+EOO
+
+$* <"  " >>EOO # space
+'  '
+EOO
+
+$* <"" >>EOO # empty
+''
+EOO
+
+$* <:"foo" 2>>EOE != 0 # eof
+stdin:1:4: error: expected newline at the end of description line
+EOE
diff --git a/unit-tests/test/script/lexer/driver.cxx b/unit-tests/test/script/lexer/driver.cxx
index c5beebb..abd32ba 100644
--- a/unit-tests/test/script/lexer/driver.cxx
+++ b/unit-tests/test/script/lexer/driver.cxx
@@ -29,13 +29,14 @@ namespace build2
           assert (argc == 2);
           string s (argv[1]);
 
-          if      (s == "script-line")   m = lexer_mode::script_line;
-          else if (s == "first-token")   m = lexer_mode::first_token;
-          else if (s == "second-token")  m = lexer_mode::second_token;
-          else if (s == "variable-line") m = lexer_mode::variable_line;
-          else if (s == "command-line")  m = lexer_mode::command_line;
-          else if (s == "here-line")     m = lexer_mode::here_line;
-          else if (s == "variable")      m = lexer_mode::variable;
+          if      (s == "script-line")      m = lexer_mode::script_line;
+          else if (s == "first-token")      m = lexer_mode::first_token;
+          else if (s == "second-token")     m = lexer_mode::second_token;
+          else if (s == "variable-line")    m = lexer_mode::variable_line;
+          else if (s == "command-line")     m = lexer_mode::command_line;
+          else if (s == "here-line")        m = lexer_mode::here_line;
+          else if (s == "description-line") m = lexer_mode::description_line;
+          else if (s == "variable")         m = lexer_mode::variable;
           else                           assert (false);
         }
 
@@ -45,9 +46,10 @@ namespace build2
 
           // Some modes auto-expire so we need something underneath.
           //
-          bool u (m == lexer_mode::first_token   ||
-                  m == lexer_mode::second_token  ||
-                  m == lexer_mode::variable_line ||
+          bool u (m == lexer_mode::first_token      ||
+                  m == lexer_mode::second_token     ||
+                  m == lexer_mode::variable_line    ||
+                  m == lexer_mode::description_line ||
                   m == lexer_mode::variable);
 
           lexer l (cin, path ("stdin"), u ? lexer_mode::script_line : m);
diff --git a/unit-tests/test/script/lexer/first-token.test b/unit-tests/test/script/lexer/first-token.test
index a433362..456ef6d 100644
--- a/unit-tests/test/script/lexer/first-token.test
+++ b/unit-tests/test/script/lexer/first-token.test
@@ -7,6 +7,11 @@ $* <";" >>EOO # semi
 <newline>
 EOO
 
+$* <":" >>EOO # colon
+:
+<newline>
+EOO
+
 $* <"{" >>EOO # lcbrace
 {
 <newline>
diff --git a/unit-tests/test/script/lexer/script-line.test b/unit-tests/test/script/lexer/script-line.test
index 6c5038a..96eb19c 100644
--- a/unit-tests/test/script/lexer/script-line.test
+++ b/unit-tests/test/script/lexer/script-line.test
@@ -17,6 +17,25 @@ $* <";" >>EOO # semi-only
 <newline>
 EOO
 
+$* <"cmd: dsc" >>EOO # colon
+'cmd'
+:
+'dsc'
+<newline>
+EOO
+
+$* <"cmd :dsc" >>EOO # colon-separated
+'cmd'
+:
+'dsc'
+<newline>
+EOO
+
+$* <":" >>EOO # colon-only
+:
+<newline>
+EOO
+
 $* <"cmd <+ 1>+" >>EOO # pass-redirect
 'cmd'
 <+
diff --git a/unit-tests/test/script/lexer/second-token.test b/unit-tests/test/script/lexer/second-token.test
index 058dc65..5238a82 100644
--- a/unit-tests/test/script/lexer/second-token.test
+++ b/unit-tests/test/script/lexer/second-token.test
@@ -7,6 +7,11 @@ $* <";" >>EOO # semi
 <newline>
 EOO
 
+$* <":" >>EOO # colon
+:
+<newline>
+EOO
+
 $* <"=foo" >>EOO # assign
 =
 'foo'
diff --git a/unit-tests/test/script/parser/buildfile b/unit-tests/test/script/parser/buildfile
index e64159e..c65dd73 100644
--- a/unit-tests/test/script/parser/buildfile
+++ b/unit-tests/test/script/parser/buildfile
@@ -11,7 +11,7 @@ filesystem config/{utility init operation} dump types-parsers \
 test/{target script/{token lexer parser script}}
 
 exe{driver}: cxx{driver} ../../../../build2/cxx{$src} $libs \
-test{cleanup command-re-parse expansion here-document pre-parse redirect \
-     scope setup-teardown}
+test{cleanup command-re-parse description expansion here-document here-string \
+     pre-parse redirect scope setup-teardown}
 
 include ../../../../build2/
diff --git a/unit-tests/test/script/parser/description.test b/unit-tests/test/script/parser/description.test
new file mode 100644
index 0000000..881cb5b
--- /dev/null
+++ b/unit-tests/test/script/parser/description.test
@@ -0,0 +1,332 @@
+$* <<EOI >>EOO                                         # id
+: foo
+cmd
+EOI
+: id:foo
+cmd
+EOO
+
+$* <<EOI >>EOO                                         # summary
+: foo bar
+cmd
+EOI
+: sm:foo bar
+cmd
+EOO
+
+$* <<EOI >>EOO                                         # id-summary
+: foo-bar
+: foo bar
+cmd
+EOI
+: id:foo-bar
+: sm:foo bar
+cmd
+EOO
+
+# Initially assumed summary.
+#
+$* <<EOI >>EOO                                         # details-summary
+: foo bar
+: bar baz
+cmd
+EOI
+: foo bar
+: bar baz
+cmd
+EOO
+
+# Initially assumed id and summary.
+#
+$* <<EOI >>EOO                                         # details-id-summary
+: foo-bar
+: bar baz
+: baz fox
+cmd
+EOI
+: foo-bar
+: bar baz
+: baz fox
+cmd
+EOO
+
+$* <<EOI >>EOO                                         # id-details
+: foo-bar
+:
+: foo bar
+: bar baz
+cmd
+EOI
+: id:foo-bar
+:
+: foo bar
+: bar baz
+cmd
+EOO
+
+$* <<EOI >>EOO                                         # summary-details
+: foo bar
+:
+: foo bar
+: bar baz
+cmd
+EOI
+: sm:foo bar
+:
+: foo bar
+: bar baz
+cmd
+EOO
+
+$* <<EOI >>EOO                                         # id-summary-details
+: foo-bar
+: foo bar
+:
+: foo bar
+: bar baz
+cmd
+EOI
+: id:foo-bar
+: sm:foo bar
+:
+: foo bar
+: bar baz
+cmd
+EOO
+
+$* <<EOI >>EOO                                         # blanks
+:
+:
+: foo bar
+: bar baz
+:
+: baz fox
+:
+:
+cmd
+EOI
+: foo bar
+: bar baz
+:
+: baz fox
+cmd
+EOO
+
+$* <<EOI >>EOO                                         # strip
+:  	  foo-bar
+:  	  bar baz
+:
+:  	    baz fox
+: fox biz
+:biz buz
+:
+cmd
+EOI
+: id:foo-bar
+: sm:bar baz
+:
+:   baz fox
+:  fox biz
+: biz buz
+cmd
+EOO
+
+# Legal places for a description.
+#
+$* <<EOI >>EOO                                         # legal-var
+: foo bar
+x = y;
+cmd \$x
+EOI
+: sm:foo bar
+cmd y
+EOO
+
+# Illegal places for a description.
+#
+$* <": foo" 2>>EOE != 0                                 # illegal-eof
+testscript:2:1: error: description before <end of file>
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # illegal-rcbrace
+{
+  cmd
+  : foo
+}
+EOI
+testscript:4:1: error: description before '}'
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # illegal-setup
+: foo
++cmd
+EOI
+testscript:2:1: error: description before setup command
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # illegal-tdown
+: foo
+-cmd
+EOI
+testscript:2:1: error: description before teardown command
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # illegal-var
+: foo
+x = y
+EOI
+testscript:2:1: error: description before setup/teardown variable
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # illegal-test
+cmd1;
+: foo
+cmd2
+EOI
+testscript:2:1: error: description inside test
+EOE
+
+# Id uniqueness.
+#
+$* <<EOI 2>>EOE != 0                                    # id-dup-test-test
+: foo
+cmd
+: foo
+cmd
+EOI
+testscript:3:1: error: duplicate id foo
+  testscript:1:1: info: previously used here
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # id-dup-test-group
+: foo
+cmd
+: foo
+{
+  cmd
+  cmd
+}
+EOI
+testscript:3:1: error: duplicate id foo
+  testscript:1:1: info: previously used here
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # id-dup-group-test
+: foo
+{
+  cmd
+  cmd
+}
+: foo
+cmd
+EOI
+testscript:6:1: error: duplicate id foo
+  testscript:1:1: info: previously used here
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # id-dup-group-group
+: foo
+{
+  cmd
+  cmd
+}
+: foo
+{
+  cmd
+  cmd
+}
+EOI
+testscript:6:1: error: duplicate id foo
+  testscript:1:1: info: previously used here
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # id-dup-group-derived
+: 3
+cmd
+{
+  cmd
+  cmd
+}
+EOI
+testscript:3:1: error: duplicate id 3
+  testscript:1:1: info: previously used here
+EOE
+
+$* <<EOI 2>>EOE != 0                                    # id-dup-test-derived
+: 3
+cmd
+cmd
+EOI
+testscript:3:1: error: duplicate id 3
+  testscript:1:1: info: previously used here
+EOE
+
+# Interaction with test scope merging.
+#
+
+# No merge since both have description.
+#
+$* -s -i <<EOI >>EOO                    # test-scope-both
+: foo
+{
+  : bar
+  cmd
+}
+EOI
+{
+  : id:foo
+  { # foo
+    : id:bar
+    { # foo/bar
+      cmd
+    }
+  }
+}
+EOO
+
+$* -s -i <<EOI >>EOO                    # test-scope-group
+: foo-bar
+: foo bar
+{
+  cmd
+}
+EOI
+{
+  : id:foo-bar
+  : sm:foo bar
+  { # foo-bar
+    cmd
+  }
+}
+EOO
+
+$* -s -i <<EOI >>EOO                    # test-scope-test
+{
+  : foo-bar
+  : foo bar
+  cmd
+}
+EOI
+{
+  : id:foo-bar
+  : sm:foo bar
+  { # foo-bar
+    cmd
+  }
+}
+EOO
+
+# Id conflict once moved to outer scope.
+#
+$* <<EOI 2>>EOE != 0                    # test-scope-id-dup
+: foo
+cmd
+{
+  : foo
+  cmd
+}
+cmd
+EOI
+testscript:4:3: error: duplicate id foo
+  testscript:1:1: info: previously used here
+EOE
diff --git a/unit-tests/test/script/parser/driver.cxx b/unit-tests/test/script/parser/driver.cxx
index aad94f9..18fcdce 100644
--- a/unit-tests/test/script/parser/driver.cxx
+++ b/unit-tests/test/script/parser/driver.cxx
@@ -35,12 +35,47 @@ namespace build2
         virtual void
         enter (scope& s, const location&) override
         {
+          if (s.desc)
+          {
+            const auto& d (*s.desc);
+
+            if (!d.id.empty ())
+              cout << ind_ << ": id:" << d.id << endl;
+
+            if (!d.summary.empty ())
+              cout << ind_ << ": sm:" << d.summary << endl;
+
+            if (!d.details.empty ())
+            {
+              if (!d.id.empty () || !d.summary.empty ())
+                cout << ind_ << ":" << endl; // Blank.
+
+              const auto& s (d.details);
+              for (size_t b (0), e (0), n; e != string::npos; b = e + 1)
+              {
+                e = s.find ('\n', b);
+                n = ((e != string::npos ? e : s.size ()) - b);
+
+                cout << ind_ << ':';
+                if (n != 0)
+                {
+                  cout << ' ';
+                  cout.write (s.c_str () + b, static_cast<streamsize> (n));
+                }
+                cout << endl;
+              }
+            }
+          }
+
           if (scope_)
           {
+            cout << ind_ << "{";
+
             if (id_ && !s.id_path.empty ()) // Skip empty root scope id.
-              cout << ind_ << ": " << s.id_path.string () << endl;
+              cout << " # " << s.id_path.string ();
+
+            cout << endl;
 
-            cout << ind_ << "{" << endl;
             ind_ += "  ";
           }
         }
diff --git a/unit-tests/test/script/parser/here-string.test b/unit-tests/test/script/parser/here-string.test
new file mode 100644
index 0000000..9f44bb2
--- /dev/null
+++ b/unit-tests/test/script/parser/here-string.test
@@ -0,0 +1,11 @@
+$* <<EOI >>EOO # empty
+cmd <""
+EOI
+cmd <""
+EOO
+
+$* <<EOI >>EOO # empty-nn
+cmd <:""
+EOI
+cmd <:""
+EOO
diff --git a/unit-tests/test/script/parser/scope.test b/unit-tests/test/script/parser/scope.test
index a903959..38b6a76 100644
--- a/unit-tests/test/script/parser/scope.test
+++ b/unit-tests/test/script/parser/scope.test
@@ -57,8 +57,7 @@ $* -s -i <<EOI >>EOO                 # test-scope
 }
 EOI
 {
-  : 1
-  {
+  { # 1
     cmd
   }
 }
@@ -72,8 +71,7 @@ $* -s -i <<EOI >>EOO                    # test-scope-nested
 }
 EOI
 {
-  : 1
-  {
+  { # 1
     cmd
   }
 }
@@ -86,8 +84,7 @@ $* -s -i <<EOI >>EOO                    # test-scope-var
 }
 EOI
 {
-  : 1
-  {
+  { # 1
     cmd abc
   }
 }
@@ -101,11 +98,9 @@ $* -s -i <<EOI >>EOO                    # test-scope-setup
 }
 EOI
 {
-  : 1
-  {
+  { # 1
     setup
-    : 1/4
-    {
+    { # 1/4
       cmd abc
     }
   }