Add dynamic prerequisites to $< unless --adhoc is specified

Also add a few tests for depdb-dyndep.

10 files changed, 260 insertions, 87 deletions

diff --git a/NEWS b/NEWS
index d2c0090..ee9c009 100644
--- a/NEWS
+++ b/NEWS
@@ -93,6 +93,11 @@ Version 0.15.0
       option is currently only valid in the --byproduct mode (in the normal
       mode relative paths indicate non-existent files).
 
+    --adhoc
+
+      Treat dynamically discovered prerequisites as ad hoc (so they don't end
+      up in $<; only in the normal mode).
+
     --drop-cycles
 
       Drop prerequisites that are also targets. Only use this option if you
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
index f4f3af9..c64dbfb 100644
--- a/libbuild2/adhoc-rule-buildscript.cxx
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -1229,6 +1229,8 @@ namespace build2
   // dependency counts straight. But we don't consider them for the "renders
   // us out-of-date" check assuming this has already been done.
   //
+  // See also environment::set_special_variables().
+  //
   optional<target_state> adhoc_buildscript_rule::
   execute_update_prerequisites (action a, const target& t, timestamp mt) const
   {
diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx
index dc54194..04cd1c2 100644
--- a/libbuild2/build/script/builtin-options.cxx
+++ b/libbuild2/build/script/builtin-options.cxx
@@ -412,6 +412,7 @@ namespace build2
         include_path_specified_ (false),
         default_type_ (),
         default_type_specified_ (false),
+        adhoc_ (),
         cwd_ (),
         cwd_specified_ (false),
         drop_cycles_ ()
@@ -509,6 +510,8 @@ namespace build2
           _cli_depdb_dyndep_options_map_["--default-type"] =
           &::build2::build::script::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::default_type_,
             &depdb_dyndep_options::default_type_specified_ >;
+          _cli_depdb_dyndep_options_map_["--adhoc"] =
+          &::build2::build::script::cli::thunk< depdb_dyndep_options, bool, &depdb_dyndep_options::adhoc_ >;
           _cli_depdb_dyndep_options_map_["--cwd"] =
           &::build2::build::script::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::cwd_,
             &depdb_dyndep_options::cwd_specified_ >;
diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx
index 39b8667..60020c9 100644
--- a/libbuild2/build/script/builtin-options.hxx
+++ b/libbuild2/build/script/builtin-options.hxx
@@ -418,6 +418,15 @@ namespace build2
         void
         default_type_specified (bool);
 
+        const bool&
+        adhoc () const;
+
+        bool&
+        adhoc ();
+
+        void
+        adhoc (const bool&);
+
         const dir_path&
         cwd () const;
 
@@ -465,6 +474,7 @@ namespace build2
         bool include_path_specified_;
         string default_type_;
         bool default_type_specified_;
+        bool adhoc_;
         dir_path cwd_;
         bool cwd_specified_;
         bool drop_cycles_;
diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx
index 075bad8..6f91b2c 100644
--- a/libbuild2/build/script/builtin-options.ixx
+++ b/libbuild2/build/script/builtin-options.ixx
@@ -329,6 +329,24 @@ namespace build2
         this->default_type_specified_ = x;
       }
 
+      inline const bool& depdb_dyndep_options::
+      adhoc () const
+      {
+        return this->adhoc_;
+      }
+
+      inline bool& depdb_dyndep_options::
+      adhoc ()
+      {
+        return this->adhoc_;
+      }
+
+      inline void depdb_dyndep_options::
+      adhoc (const bool& x)
+      {
+        this->adhoc_ = x;
+      }
+
       inline const dir_path& depdb_dyndep_options::
       cwd () const
       {
diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli
index 9f3f2ba..6292f48 100644
--- a/libbuild2/build/script/builtin.cli
+++ b/libbuild2/build/script/builtin.cli
@@ -58,6 +58,11 @@ namespace build2
         string --default-type;        // Default prerequisite type to use
                                       // if none could be derived from ext.
 
+        bool --adhoc;                 // Treat dynamically discovered
+                                      // prerequisites as ad hoc (so they
+                                      // don't end up in $<; only in the
+                                      // normal mode).
+
         dir_path --cwd;               // Builtin's working directory used
                                       // to complete relative paths (only
                                       // in --byproduct mode).
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index 6f3c300..dd6fa2d 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -1604,6 +1604,14 @@ namespace build2
         else
           def_pt = &file::static_type;
 
+        // --adhoc
+        //
+        if (ops.adhoc ())
+        {
+          if (byprod)
+            fail (ll) << "depdb dyndep: --adhoc specified with --byproduct";
+        }
+
         // Update prerequisite targets.
         //
         using dyndep = dyndep_rule;
@@ -1858,10 +1866,6 @@ namespace build2
         // Enter as a target, update, and add to the list of prerequisite
         // targets a file.
         //
-        // Note that these targets don't end up in $< (which is the right
-        // thing) because that variable has already been initialized (in the
-        // environment ctor).
-        //
         size_t skip_count (0);
 
         auto add = [this, &trace, what,
@@ -1959,10 +1963,19 @@ namespace build2
                   trace, what,
                   a, t,
                   *ft, mt,
-                  false /* fail */,
-                  false /* adhoc */,
-                  1     /* data */))
+                  false        /* fail */,
+                  ops.adhoc () /* adhoc */))
             {
+              prerequisite_target& pt (pts.back ());
+
+              if (pt.adhoc)
+              {
+                pt.data = reinterpret_cast<uintptr_t> (pt.target);
+                pt.target = nullptr;
+              }
+              else
+                pt.data = 1; // Already updated.
+
               if (!cache)
                 dd.expect (ft->path ()); // @@ Use fp (or verify match)?
 
@@ -2221,6 +2234,12 @@ namespace build2
         // Add the terminating blank line (we are updating depdb).
         //
         dd.expect ("");
+
+        // Reload $< and $> to make sure they contain the newly discovered
+        // prerequisites and targets.
+        //
+        if (update)
+          environment_->set_special_variables (a);
       }
 
       // When add a special variable don't forget to update lexer::word().
diff --git a/libbuild2/build/script/script.cxx b/libbuild2/build/script/script.cxx
index f4f8da8..480903e 100644
--- a/libbuild2/build/script/script.cxx
+++ b/libbuild2/build/script/script.cxx
@@ -40,42 +40,50 @@ namespace build2
               redirect (redirect_type::pass)),
             target (t),
             vars (context, false /* global */),
+            var_ts (var_pool.insert (">")),
+            var_ps (var_pool.insert ("<")),
             script_deadline (to_deadline (dl, false /* success */))
       {
-        // Set special variables.
-        //
+        set_special_variables (a);
+
+        if (temp)
+          set_temp_dir_variable ();
+      }
+
+      void environment::
+      set_special_variables (action a)
+      {
         {
           // $>
           //
           names ns;
-          for (const target_type* m (&t); m != nullptr; m = m->adhoc_member)
+          for (const target_type* m (&target);
+               m != nullptr;
+               m = m->adhoc_member)
             m->as_name (ns);
 
-          assign (var_pool.insert (">")) = move (ns);
+          assign (var_ts) = move (ns);
         }
 
         {
           // $<
           //
-          // Note that at this stage (after execute_prerequisites()) ad hoc
-          // prerequisites are no longer in prerequisite_targets which means
-          // they won't end up in $< either. While at first thought ad hoc
-          // prerequisites in ad hoc recipes don't seem to make much sense,
-          // they could be handy to exclude certain preresquisites from $<
-          // while still treating them as such.
+          // Note that ad hoc prerequisites don't end up in $<. While at first
+          // thought ad hoc prerequisites in ad hoc recipes don't seem to make
+          // much sense, they could be handy to exclude certain prerequisites
+          // from $< while still treating them as such, especially in rule.
           //
           names ns;
-          for (const target_type* pt: t.prerequisite_targets[a])
+          for (const prerequisite_target& pt: target.prerequisite_targets[a])
           {
-            if (pt != nullptr)
-              pt->as_name (ns);
+            // See adhoc_buildscript_rule::execute_update_prerequisites().
+            //
+            if (pt.target != nullptr && !pt.adhoc)
+              pt.target->as_name (ns);
           }
 
-          assign (var_pool.insert ("<")) = move (ns);
+          assign (var_ps) = move (ns);
         }
-
-        if (temp)
-          set_temp_dir_variable ();
       }
 
       void environment::
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index 4e88785..0619253 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -94,6 +94,17 @@ namespace build2
                      bool temp_dir,
                      const optional<timestamp>& deadline = nullopt);
 
+        // (Re)set special $< and $> variables.
+        //
+        void
+        set_special_variables (action);
+
+        // Create the temporary directory (if it doesn't exist yet) and set
+        // the $~ special variable to its path.
+        //
+        void
+        set_temp_dir_variable ();
+
         environment (environment&&) = delete;
         environment (const environment&) = delete;
         environment& operator= (environment&&) = delete;
@@ -117,6 +128,9 @@ namespace build2
         variable_pool var_pool;
         variable_map vars;
 
+        const variable& var_ts; // $>
+        const variable& var_ps; // $<
+
         // Temporary directory for the script run.
         //
         // Currently this directory is removed regardless of the script
@@ -146,12 +160,6 @@ namespace build2
         //
         size_t exec_line = 1;
 
-        // Create the temporary directory (if it doesn't exist yet) and set
-        // the $~ special variable to its path.
-        //
-        void
-        set_temp_dir_variable ();
-
         virtual void
         set_variable (string&& name,
                       names&&,
diff --git a/tests/recipe/buildscript/testscript b/tests/recipe/buildscript/testscript
index 12c5717..910ee67 100644
--- a/tests/recipe/buildscript/testscript
+++ b/tests/recipe/buildscript/testscript
@@ -250,6 +250,97 @@ posix = ($cxx.target.class != 'windows')
       $* clean 2>-
     }
 
+    : preamble
+    :
+    {
+      : valid
+      :
+      {
+        echo 'bar' >=bar;
+
+        cat <<EOI >=buildfile;
+          s = $process.run(cat bar)
+          foo:
+          {{
+            depdb clear
+
+            s1 = 'abc'
+            s2 = 'xyz'
+
+            if echo "$s" >>>? 'bar'
+              v = "$s1"
+            else
+              echo "$s2" | set v
+            end
+
+            depdb string "$v"
+
+            echo "$v" >$path($>)
+          }}
+          EOI
+
+        $* 2>'echo file{foo}';
+        cat <<<foo >'abc';
+
+        $* 2>/'info: dir{./} is up to date';
+
+        echo 'baz' >=bar;
+        $* 2>'echo file{foo}';
+        cat <<<foo >'xyz';
+
+        $* clean 2>-
+      }
+
+      : invalid
+      :
+      {
+        cat <<EOI >=buildfile;
+          foo:
+          {{
+            v = 'abc'
+            echo "$v" >$path($>)
+            depdb string "$v"
+          }}
+          EOI
+
+        $* 2>>~%EOE% != 0;
+          buildfile:4:3: error: disallowed command in depdb preamble
+            info: only variable assignments are allowed in depdb preamble
+            buildfile:5:3: info: depdb preamble ends here
+          %.+
+          EOE
+
+        $* clean 2>-
+      }
+
+      : temp-dir
+      :
+      {
+        cat <<EOI >=buildfile;
+          foo:
+          {{
+            touch $~/f | set dummy
+
+            if test -f $~/f
+              v = "yes"
+            else
+              v = "no"
+            end
+
+            depdb string "$v"
+            diag echo $>
+
+            test -f $~/f
+            echo "$v" >$path($>)
+          }}
+          EOI
+
+        $* 2>'echo file{foo.}';
+
+        $* clean 2>-
+      }
+    }
+
     : string
     :
     {
@@ -368,94 +459,98 @@ posix = ($cxx.target.class != 'windows')
       }
     }
 
-    : preamble
+    : dyndep
     :
     {
-      : valid
+      : normal
       :
       {
-        echo 'bar' >=bar;
+	cat <<EOI >=bar.h;
+          bar
+          EOI
 
         cat <<EOI >=buildfile;
-          s = $process.run(cat bar)
-          foo:
-          {{
-            depdb clear
+          define h: file
+          h{*}: extension = h
 
-            s1 = 'abc'
-            s2 = 'xyz'
+          ./: h{foo baz}
 
-            if echo "$s" >>>? 'bar'
-              v = "$s1"
-            else
-              echo "$s2" | set v
-            end
+          h{foo}:
+          {{
+            # Note that strictly speaking we should return $out_base/baz.h
+            # on the second invocation (since it now exists). But our dyndep
+            # machinery skips the entry which it has already seen, so this
+            # works for now.
+            #
+            depdb dyndep "-I$out_base" --what=header --default-type=h -- \
+              echo "$out_base/foo.h: $src_base/bar.h baz.h"
 
-            depdb string "$v"
+            diag gen $>
 
-            echo "$v" >$path($>)
+            cat $path($<) >$path($>)
           }}
-          EOI
-
-        $* 2>'echo file{foo}';
-        cat <<<foo >'abc';
-
-        $* 2>/'info: dir{./} is up to date';
-
-        echo 'baz' >=bar;
-        $* 2>'echo file{foo}';
-        cat <<<foo >'xyz';
 
-        $* clean 2>-
-      }
-
-      : invalid
-      :
-      {
-        cat <<EOI >=buildfile;
-          foo:
+          h{baz}:
           {{
-            v = 'abc'
-            echo "$v" >$path($>)
-            depdb string "$v"
+            diag gen $>
+            echo baz >$path($>)
           }}
           EOI
 
-        $* 2>>~%EOE% != 0;
-          buildfile:4:3: error: disallowed command in depdb preamble
-            info: only variable assignments are allowed in depdb preamble
-            buildfile:5:3: info: depdb preamble ends here
-          %.+
+	$* 2>>EOE;
+          gen h{baz.h}
+          gen h{foo.h}
           EOE
 
-        $* clean 2>-
+	cat foo.h >>EOO;
+	  bar
+	  baz
+	  EOO
+
+	$* clean 2>-
       }
 
-      : temp-dir
+      : byproduct
       :
       {
+	cat <<EOI >=bar.h;
+          bar
+          EOI
+
         cat <<EOI >=buildfile;
-          foo:
+          define h: file
+          h{*}: extension = h
+
+          h{foo}: h{baz}
           {{
-            touch $~/f | set dummy
+            o = $path($>)
+            t = $path($>).t
 
-            if test -f $~/f
-              v = "yes"
-            else
-              v = "no"
-            end
+            depdb dyndep --byproduct --what=header --default-type=h --file $t
 
-            depdb string "$v"
-            diag echo $>
+            diag gen $>
+            cat $src_base/bar.h $out_base/baz.h >$o
+            echo "$out_base/foo.h: $src_base/bar.h $out_base/baz.h" >$t
+          }}
 
-            test -f $~/f
-            echo "$v" >$path($>)
+          h{baz}:
+          {{
+            diag gen $>
+            echo baz >$path($>)
           }}
           EOI
 
-        $* 2>'echo file{foo.}';
+	$* 2>>EOE;
+          gen h{baz.h}
+          gen h{foo.h}
+          EOE
 
-        $* clean 2>-
+	cat foo.h >>EOO;
+	  bar
+	  baz
+	  EOO
+
+	$* clean 2>-
       }
     }
   }