Add support for dynamic target extraction in addition to prerequisites

This functionality is enabled with the depdb-dyndep --dyn-target option. Only
the make format is supported, where the listed targets are added as ad hoc
group members (unless already specified as static members). This functionality
is not available in the --byproduct mode.

12 files changed, 864 insertions, 128 deletions

diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx
index ce4ab8b..f3e3560 100644
--- a/libbuild2/adhoc-rule-buildscript.cxx
+++ b/libbuild2/adhoc-rule-buildscript.cxx
@@ -5,6 +5,8 @@
 
 #include <sstream>
 
+#include <libbutl/filesystem.hxx> // try_rm_file()
+
 #include <libbuild2/depdb.hxx>
 #include <libbuild2/scope.hxx>
 #include <libbuild2/target.hxx>
@@ -225,6 +227,8 @@ namespace build2
     build::script::default_runner run;
 
     path dd;
+    paths dyn_targets;
+    paths old_dyn_targets;
 
     const scope* bs;
     timestamp mt;
@@ -457,10 +461,120 @@ namespace build2
       }
     }
 
+    // Read the list of dynamic targets from depdb, if exists (used in a few
+    // depdb-dyndep --dyn-target handling places below).
+    //
+    auto read_dyn_targets = [] (path ddp) -> paths
+    {
+      depdb dd (move (ddp), true /* read_only */);
+
+      paths r;
+      while (dd.reading ()) // Breakout loop.
+      {
+        string* l;
+        auto read = [&dd, &l] () -> bool
+        {
+          return (l = dd.read ()) != nullptr;
+        };
+
+        if (!read ()) // Rule id.
+          break;
+
+        // We can omit this for as long as we don't break our blank line
+        // anchors semantics (e.g., by adding another blank somewhere, say
+        // after the custom depdb builtins).
+        //
+#if 0
+        if (*l != rule_id_)
+          fail << "unable to clean dynamic target group " << t
+               << " with old depdb";
+#endif
+
+        // Note that we cannot read out expected lines since there can be
+        // custom depdb builtins. We also need to skip the prerequisites
+        // list. So we read until the blank line that always terminates the
+        // prerequisites list.
+        //
+        {
+          bool r;
+          while ((r = read ()) && !l->empty ()) ;
+          if (!r)
+            break;
+        }
+
+        // Read the dynamic target files. We should always end with a blank
+        // line.
+        //
+        for (;;)
+        {
+          if (!read () || l->empty ())
+            break;
+
+          r.push_back (path (*l));
+        }
+
+        break;
+      }
+
+      return r;
+    };
+
     // See if we are providing the standard clean as a fallback.
     //
     if (me.fallback)
-      return &perform_clean_file;
+    {
+      // For depdb-dyndep --dyn-target use depdb to clean dynamic targets.
+      //
+      if (script.depdb_dyndep && script.depdb_dyndep_dyn_target)
+      {
+        file& t (xt.as<file> ());
+
+        // Note that only removing the relevant filesystem entries is not
+        // enough: we actually have to populate the group with members since
+        // this information could be used to clean derived targets (for
+        // example, object files). So we just do that and let the standard
+        // clean logic take care of them the same as static members.
+        //
+        using dyndep = dyndep_rule;
+
+        function<dyndep::map_extension_func> map_ext (
+          [] (const scope& bs, const string& n, const string& e)
+          {
+            return dyndep::map_extension (bs, n, e, nullptr);
+          });
+
+        for (path& f: read_dyn_targets (t.path () + ".d"))
+        {
+          // Note that this logic should be consistent with what we have in
+          // exec_depdb_dyndep().
+          //
+          // Note that here we don't bother cleaning any old dynamic targets
+          // -- the more we can clean, the merrier.
+          //
+          // @@ We don't have --target-what, --target-default-type here.
+          //    Could we do the same thing as byproduct to get them? That
+          //    would require us running the first half of the depdb preamble
+          //    but ignoring all the depdb builtins (we still want all the
+          //    variable assignments -- maybe we could automatically skip them
+          //    if we see depdb is not open). Wonder if there would be any
+          //    other complications...
+          //
+          //    BTW, this sort of works for --target-default-type since we
+          //    just clean them as file{} targets (but diagnostics is off). It
+          //    does break, however, if there is s batch, since then we end up
+          //    detecting different targets sharing a path. This will also not
+          //    work at all if/when we support specifying custom extension to
+          //    type mapping in order to resolve ambiguities.
+          //
+          dyndep::inject_adhoc_group_member ("file",
+                                             a, bs, t,
+                                             move (f),
+                                             map_ext, file::static_type);
+        }
+      }
+
+      return perform_clean_file;
+    }
 
     // If we have any update during match prerequisites, now is the time to
     // update them.
@@ -575,10 +689,49 @@ namespace build2
       }
     }
 
+    // Note that while it's tempting to turn match_data* into recipes, some of
+    // their members are not movable. And in the end we will have the same
+    // result: one dynamic memory allocation.
+    //
+    unique_ptr<match_data> md;
+    unique_ptr<match_data_byproduct> mdb;
+
+    if (script.depdb_dyndep_byproduct)
+    {
+      mdb.reset (new match_data_byproduct (
+                   a, t, bs, script.depdb_preamble_temp_dir));
+    }
+    else
+    {
+      md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir));
+
+      // If the set of dynamic targets can change based on changes to the
+      // inputs (say, each entity, such as a type, in the input file gets its
+      // own output file), then we can end up with a large number of old
+      // output files laying around because they are not part of the new
+      // dynamic target set. So we try to clean them up based on the old depdb
+      // information, similar to how we do it for perform_clean above (except
+      // here we will just keep the list of old files).
+      //
+      // Note: do before opening depdb, which can start over-writing it.
+      //
+      // We also have to do this speculatively, without knowing whether we
+      // will need to update. Oh, well, being dynamic ain't free.
+      //
+      if (script.depdb_dyndep_dyn_target)
+        md->old_dyn_targets = read_dyn_targets (tp + ".d");
+    }
+
     depdb dd (tp + ".d");
 
     // NOTE: see the "static dependencies" version (with comments) below.
     //
+    // NOTE: We use blank lines as anchors to skip directly to certain entries
+    //       (e.g., dynamic targets). So make sure none of the other entries
+    //       can be blank (for example, see `depdb string` builtin).
+    //
+    // NOTE: KEEP IN SYNC WITH read_dyn_targets ABOVE!
+    //
     if (dd.expect ("<ad hoc buildscript recipe> 1") != nullptr)
       l4 ([&]{trace << "rule mismatch forcing update of " << t;});
 
@@ -613,10 +766,21 @@ namespace build2
           l4 ([&]{trace << "recipe variable change forcing update of " << t;});
       }
 
+      // Static targets and prerequisites (there can also be dynamic targets;
+      // see dyndep --dyn-target).
+      //
+      // There is a nuance: in an operation batch (e.g., `b update update`) we
+      // will already have the dynamic targets as members on the subsequent
+      // operations and we need to make sure we don't treat them as static.
+      // Using target_decl to distinguish the two seems like a natural way.
+      //
       {
         sha256 tcs;
         for (const target* m (&t); m != nullptr; m = m->adhoc_member)
-          hash_target (tcs, *m, storage);
+        {
+          if (m->decl == target_decl::real)
+            hash_target (tcs, *m, storage);
+        }
 
         if (dd.expect (tcs.string ()) != nullptr)
           l4 ([&]{trace << "target set change forcing update of " << t;});
@@ -634,22 +798,8 @@ namespace build2
       }
     }
 
-    // Note that while it's tempting to turn match_data* into recipes, some of
-    // their members are not movable. And in the end we will have the same
-    // result: one dynamic memory allocation.
+    // Get ready to run the depdb preamble.
     //
-    unique_ptr<match_data> md;
-    unique_ptr<match_data_byproduct> mdb;
-
-    if (script.depdb_dyndep_byproduct)
-    {
-      mdb.reset (new match_data_byproduct (
-                   a, t, bs, script.depdb_preamble_temp_dir));
-    }
-    else
-      md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir));
-
-
     build::script::environment& env (mdb != nullptr ? mdb->env : md->env);
     build::script::default_runner& run (mdb != nullptr ? mdb->run : md->run);
 
@@ -828,20 +978,22 @@ namespace build2
     else
     {
       // Run the second half of the preamble (depdb-dyndep commands) to update
-      // our prerequisite targets and extract dynamic dependencies.
+      // our prerequisite targets and extract dynamic dependencies (targets and
+      // prerequisites).
       //
       // Note that this should be the last update to depdb (the invalidation
       // order semantics).
       //
-      bool deferred_failure (false);
+      md->deferred_failure = false;
       {
         build::script::parser p (ctx);
         p.execute_depdb_preamble_dyndep (a, bs, t,
                                          env, script, run,
                                          dd,
+                                         md->dyn_targets,
                                          update,
                                          mt,
-                                         deferred_failure);
+                                         md->deferred_failure);
       }
 
       if (update && dd.reading () && !ctx.dry_run)
@@ -854,7 +1006,6 @@ namespace build2
       //
       md->bs = &bs;
       md->mt = update ? timestamp_nonexistent : mt;
-      md->deferred_failure = deferred_failure;
 
       return [this, md = move (md)] (action a, const target& t)
       {
@@ -1066,7 +1217,7 @@ namespace build2
               if (r.second.empty ())
                 continue;
 
-              // @@ TODO: what should we do about targets?
+              // Note: no support for dynamic targets in byproduct mode.
               //
               if (r.first == make_type::target)
                 continue;
@@ -1076,10 +1227,10 @@ namespace build2
               if (f.relative ())
               {
                 if (!byp.cwd)
-                  fail (il) << "relative path '" << f << "' in make dependency"
-                            << " declaration" <<
+                  fail (il) << "relative prerequisite path '" << f
+                            << "' in make dependency declaration" <<
                     info << "consider using --cwd to specify relative path "
-                            << "base";
+                         << "base";
 
                 f = *byp.cwd / f;
               }
@@ -1143,6 +1294,30 @@ namespace build2
       return *ps;
     }
 
+    // Remove previous dynamic targets since their set may change with changes
+    // to the inputs (see apply() for details).
+    //
+    // The dry-run mode complicates things: if we don't remove the old files,
+    // then that information will be gone (since we update depdb even in the
+    // dry-run mode). But if we remove everything in the dry-run mode, then we
+    // may also remove some of the current files, which would be incorrect. So
+    // let's always remove but only files that are not in the current set.
+    //
+    for (const path& f: md.old_dyn_targets)
+    {
+      if (find (md.dyn_targets.begin (), md.dyn_targets.end (), f) ==
+          md.dyn_targets.end ())
+      {
+        // This is an optimization so best effort.
+        //
+        if (optional<rmfile_status> s = butl::try_rmfile_ignore_error (f))
+        {
+          if (s == rmfile_status::success && verb >= 2)
+            text << "rm " << f;
+        }
+      }
+    }
+
     // Sequence start time for mtime checks below.
     //
     timestamp start (!ctx.dry_run && depdb::mtime_check ()
@@ -1627,6 +1802,8 @@ namespace build2
     // Finally, we print the entire ad hoc group at verbosity level 1, similar
     // to the default update diagnostics.
     //
+    // @@ TODO: .t may also be a temporary directory.
+    //
     return perform_clean_extra (a,
                                 t.as<file> (),
                                 {".d", ".t"},
diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx
index 9e4213d..3b64de1 100644
--- a/libbuild2/build/script/builtin-options.cxx
+++ b/libbuild2/build/script/builtin-options.cxx
@@ -289,7 +289,13 @@ namespace build2
         adhoc_ (),
         cwd_ (),
         cwd_specified_ (false),
-        drop_cycles_ ()
+        drop_cycles_ (),
+        target_what_ (),
+        target_what_specified_ (false),
+        target_default_type_ (),
+        target_default_type_specified_ (false),
+        target_cwd_ (),
+        target_cwd_specified_ (false)
       {
       }
 
@@ -391,6 +397,15 @@ namespace build2
             &depdb_dyndep_options::cwd_specified_ >;
           _cli_depdb_dyndep_options_map_["--drop-cycles"] =
           &::build2::build::cli::thunk< depdb_dyndep_options, &depdb_dyndep_options::drop_cycles_ >;
+          _cli_depdb_dyndep_options_map_["--target-what"] =
+          &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_what_,
+            &depdb_dyndep_options::target_what_specified_ >;
+          _cli_depdb_dyndep_options_map_["--target-default-type"] =
+          &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_default_type_,
+            &depdb_dyndep_options::target_default_type_specified_ >;
+          _cli_depdb_dyndep_options_map_["--target-cwd"] =
+          &::build2::build::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::target_cwd_,
+            &depdb_dyndep_options::target_cwd_specified_ >;
         }
       };
 
diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx
index 590d3b2..2a00072 100644
--- a/libbuild2/build/script/builtin-options.hxx
+++ b/libbuild2/build/script/builtin-options.hxx
@@ -174,6 +174,51 @@ namespace build2
         void
         drop_cycles (const bool&);
 
+        const string&
+        target_what () const;
+
+        string&
+        target_what ();
+
+        void
+        target_what (const string&);
+
+        bool
+        target_what_specified () const;
+
+        void
+        target_what_specified (bool);
+
+        const string&
+        target_default_type () const;
+
+        string&
+        target_default_type ();
+
+        void
+        target_default_type (const string&);
+
+        bool
+        target_default_type_specified () const;
+
+        void
+        target_default_type_specified (bool);
+
+        const dir_path&
+        target_cwd () const;
+
+        dir_path&
+        target_cwd ();
+
+        void
+        target_cwd (const dir_path&);
+
+        bool
+        target_cwd_specified () const;
+
+        void
+        target_cwd_specified (bool);
+
         // Implementation details.
         //
         protected:
@@ -201,6 +246,12 @@ namespace build2
         dir_path cwd_;
         bool cwd_specified_;
         bool drop_cycles_;
+        string target_what_;
+        bool target_what_specified_;
+        string target_default_type_;
+        bool target_default_type_specified_;
+        dir_path target_cwd_;
+        bool target_cwd_specified_;
       };
     }
   }
diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx
index ea06a0f..3e3787f 100644
--- a/libbuild2/build/script/builtin-options.ixx
+++ b/libbuild2/build/script/builtin-options.ixx
@@ -233,6 +233,96 @@ namespace build2
       {
         this->drop_cycles_ = x;
       }
+
+      inline const string& depdb_dyndep_options::
+      target_what () const
+      {
+        return this->target_what_;
+      }
+
+      inline string& depdb_dyndep_options::
+      target_what ()
+      {
+        return this->target_what_;
+      }
+
+      inline void depdb_dyndep_options::
+      target_what (const string& x)
+      {
+        this->target_what_ = x;
+      }
+
+      inline bool depdb_dyndep_options::
+      target_what_specified () const
+      {
+        return this->target_what_specified_;
+      }
+
+      inline void depdb_dyndep_options::
+      target_what_specified (bool x)
+      {
+        this->target_what_specified_ = x;
+      }
+
+      inline const string& depdb_dyndep_options::
+      target_default_type () const
+      {
+        return this->target_default_type_;
+      }
+
+      inline string& depdb_dyndep_options::
+      target_default_type ()
+      {
+        return this->target_default_type_;
+      }
+
+      inline void depdb_dyndep_options::
+      target_default_type (const string& x)
+      {
+        this->target_default_type_ = x;
+      }
+
+      inline bool depdb_dyndep_options::
+      target_default_type_specified () const
+      {
+        return this->target_default_type_specified_;
+      }
+
+      inline void depdb_dyndep_options::
+      target_default_type_specified (bool x)
+      {
+        this->target_default_type_specified_ = x;
+      }
+
+      inline const dir_path& depdb_dyndep_options::
+      target_cwd () const
+      {
+        return this->target_cwd_;
+      }
+
+      inline dir_path& depdb_dyndep_options::
+      target_cwd ()
+      {
+        return this->target_cwd_;
+      }
+
+      inline void depdb_dyndep_options::
+      target_cwd (const dir_path& x)
+      {
+        this->target_cwd_ = x;
+      }
+
+      inline bool depdb_dyndep_options::
+      target_cwd_specified () const
+      {
+        return this->target_cwd_specified_;
+      }
+
+      inline void depdb_dyndep_options::
+      target_cwd_specified (bool x)
+      {
+        this->target_cwd_specified_ = x;
+      }
     }
   }
 }
diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli
index 7d0936f..2fba0b0 100644
--- a/libbuild2/build/script/builtin.cli
+++ b/libbuild2/build/script/builtin.cli
@@ -17,8 +17,8 @@ namespace build2
       //
       class depdb_dyndep_options
       {
-        // Note that --byproduct, if any, must be the first option and is
-        // handled ad hoc, kind of as a sub-command.
+        // Note that --byproduct or --dyn-target, if any, must be the first
+        // option and is handled ad hoc.
         //
         // Similarly, --update-{include,exclude} are handled ad hoc and must
         // be literals, similar to the -- separator. They specify prerequisite
@@ -44,16 +44,19 @@ namespace build2
         // and the other for prerequisite, we omit "prerequisite" as that's
         // what we extract by default and most commonly. For example:
         //
-        // --what          --what-target
-        // --default-type  --default-target-type
+        // --what          --target-what
+        // --default-type  --target-default-type
         //
         path --file;                  // Read from file rather than stdin.
 
         string --format;              // Dependency format: make (default).
 
-        string --what;                // Dependency kind, e.g., "header".
+        // Dynamic dependency extraction options.
+        //
+        string --what;                // Prerequisite kind, e.g., "header".
 
-        dir_paths --include-path|-I;  // Search paths for generated files.
+        dir_paths --include-path|-I;  // Search paths for generated
+                                      // prerequisites.
 
         string --default-type;        // Default prerequisite type to use
                                       // if none could be derived from ext.
@@ -64,14 +67,36 @@ namespace build2
                                       // normal mode).
 
         dir_path --cwd;               // Builtin's working directory used
-                                      // to complete relative paths (only
-                                      // in --byproduct mode).
+                                      // to complete relative paths of
+                                      // prerequisites (only in --byproduct
+                                      // mode).
 
         bool --drop-cycles;           // Drop prerequisites that are also
                                       // targets. Only use if you are sure
                                       // such cycles are harmless, that is,
                                       // the output is not affected by such
                                       // prerequisites' content.
+
+        // Dynamic target extraction options.
+        //
+        // This functionality is enabled with the --dyn-target option. Only
+        // the make format is supported, where the listed targets are added as
+        // ad hoc group members (unless already specified as static members).
+        // This functionality is not available in the byproduct mode.
+        //
+        // @@ BTW, here what would likely be more useful than default target
+        //    is the ability to specify custom extension-to-type mapping in
+        //    order to resolve ambiguities. See also the issue with getting
+        //    these options during clean.
+        //
+        string --target-what;         // Target kind, e.g., "source".
+
+        string --target-default-type; // Default target type to use if none
+                                      // could be derived from ext.
+
+        dir_path --target-cwd;        // Builtin's working directory used to
+                                      // complete relative paths of targets.
+
       };
     }
   }
diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx
index df0a419..e7268f9 100644
--- a/libbuild2/build/script/parser.cxx
+++ b/libbuild2/build/script/parser.cxx
@@ -158,6 +158,7 @@ namespace build2
         {
           s.depdb_dyndep = depdb_dyndep_->second;
           s.depdb_dyndep_byproduct = depdb_dyndep_byproduct_;
+          s.depdb_dyndep_dyn_target = depdb_dyndep_dyn_target_;
         }
         s.depdb_preamble = move (depdb_preamble_);
 
@@ -830,8 +831,18 @@ namespace build2
                   fail (l) << "multiple 'depdb dyndep' calls" <<
                     info (depdb_dyndep_->first) << "previous call is here";
 
-                if (peek () == type::word && peeked ().value == "--byproduct")
-                  depdb_dyndep_byproduct_ = true;
+                if (peek () == type::word)
+                {
+                  const string& v (peeked ().value);
+
+                  // Note: --byproduct and --dyn-target are mutually
+                  // exclusive.
+                  //
+                  if (v == "--byproduct")
+                    depdb_dyndep_byproduct_ = true;
+                  else if (v == "--dyn-target")
+                    depdb_dyndep_dyn_target_ = true;
+                }
               }
               else
               {
@@ -1280,6 +1291,7 @@ namespace build2
                            environment& e, const script& s, runner& r,
                            lines_iterator begin, lines_iterator end,
                            depdb& dd,
+                           paths* dyn_targets,
                            bool* update,
                            optional<timestamp> mt,
                            bool* deferred_failure,
@@ -1308,12 +1320,17 @@ namespace build2
           const script& scr;
 
           depdb& dd;
+          paths* dyn_targets;
           bool* update;
           bool* deferred_failure;
           optional<timestamp> mt;
           dyndep_byproduct* byp;
 
-        } data {trace, a, bs, t, e, s, dd, update, deferred_failure, mt, byp};
+        } data {
+          trace,
+          a, bs, t,
+          e, s,
+          dd, dyn_targets, update, deferred_failure, mt, byp};
 
         auto exec_cmd = [this, &data] (token& t,
                                        build2::script::token_type& tt,
@@ -1345,6 +1362,7 @@ namespace build2
                                  li, ll,
                                  data.a, data.bs, const_cast<file&> (data.t),
                                  data.dd,
+                                 *data.dyn_targets,
                                  *data.update,
                                  *data.mt,
                                  *data.deferred_failure,
@@ -1354,35 +1372,29 @@ namespace build2
             {
               names ns (exec_special (t, tt, true /* skip <cmd> */));
 
+              string v;
+              const char* w (nullptr);
               if (cmd == "hash")
               {
                 sha256 cs;
                 for (const name& n: ns)
                   to_checksum (cs, n);
 
-                if (data.dd.expect (cs.string ()) != nullptr)
-                  l4 ([&] {
-                      data.trace (ll)
-                        << "'depdb hash' argument change forcing update of "
-                        << data.t;});
+                v = cs.string ();
+                w = "argument";
               }
               else if (cmd == "string")
               {
-                string s;
                 try
                 {
-                  s = convert<string> (move (ns));
+                  v = convert<string> (move (ns));
                 }
                 catch (const invalid_argument& e)
                 {
                   fail (ll) << "invalid 'depdb string' argument: " << e;
                 }
 
-                if (data.dd.expect (s) != nullptr)
-                  l4 ([&] {
-                      data.trace (ll)
-                        << "'depdb string' argument change forcing update of "
-                        << data.t;});
+                w = "argument";
               }
               else if (cmd == "env")
               {
@@ -1403,14 +1415,32 @@ namespace build2
                   fail (ll) << pf << e;
                 }
 
-                if (data.dd.expect (cs.string ()) != nullptr)
-                  l4 ([&] {
-                      data.trace (ll)
-                        << "'depdb env' environment change forcing update of "
-                        << data.t;});
+                v = cs.string ();
+                w = "environment";
               }
               else
                 assert (false);
+
+              // Prefix the value with the type letter. This serves two
+              // purposes:
+              //
+              // 1. It makes sure the result is never a blank line. We use
+              //    blank lines as anchors to skip directly to certain entries
+              //    (e.g., dynamic targets).
+              //
+              // 2. It allows us to detect the beginning of prerequisites
+              //    since an absolute path will be distinguishable from these
+              //    entries (in the future we may want to add an explicit
+              //    blank after such custom entries to make this easier).
+              //
+              v.insert (0, 1, ' ');
+              v.insert (0, 1, cmd[0]); // `h`, `s`, or `e`
+
+              if (data.dd.expect (v) != nullptr)
+                l4 ([&] {
+                    data.trace (ll)
+                      << "'depdb " << cmd << "' " << w << " change forcing "
+                      << "update of " << data.t;});
             }
           }
           else
@@ -1617,6 +1647,7 @@ namespace build2
                          size_t li, const location& ll,
                          action a, const scope& bs, file& t,
                          depdb& dd,
+                         paths& dyn_targets,
                          bool& update,
                          timestamp mt,
                          bool& deferred_failure,
@@ -1629,6 +1660,7 @@ namespace build2
         depdb_dyndep_options ops;
         bool prog (false);
         bool byprod (false);
+        bool dyn_tgt (false);
 
         // Prerequisite update filter (--update-*).
         //
@@ -1671,11 +1703,9 @@ namespace build2
 
           next (t, tt); // Skip the 'dyndep' command.
 
-          if (tt == type::word && t.value == "--byproduct")
-          {
-            byprod = true;
+          if (tt == type::word && ((byprod = (t.value == "--byproduct")) ||
+                                   (dyn_tgt = (t.value == "--dyn-target"))))
             next (t, tt);
-          }
 
           assert (byprod == (byprod_result != nullptr));
 
@@ -1894,10 +1924,23 @@ namespace build2
                 continue;
               }
 
-              // Handle --byproduct in the wrong place.
+              // Handle --byproduct and --dyn-target in the wrong place.
               //
               if (strcmp (a, "--byproduct") == 0)
-                fail (ll) << "depdb dyndep: --byproduct must be first option";
+              {
+                fail (ll) << "depdb dyndep: "
+                          << (dyn_tgt
+                              ? "--byproduct specified with --dyn-target"
+                              : "--byproduct must be first option");
+              }
+
+              if (strcmp (a, "--dyn-target") == 0)
+              {
+                fail (ll) << "depdb dyndep: "
+                          << (byprod
+                              ? "--dyn-target specified with --byproduct"
+                              : "--dyn-target must be first option");
+              }
 
               // Handle non-literal --update-*.
               //
@@ -1922,16 +1965,9 @@ namespace build2
           }
         }
 
-        // --what
-        //
-        const char* what (ops.what_specified ()
-                          ? ops.what ().c_str ()
-                          : "file");
-
         // --format
         //
         dyndep_format format (dyndep_format::make);
-
         if (ops.format_specified ())
         {
           const string& f (ops.format ());
@@ -1941,10 +1977,18 @@ namespace build2
                       << f << "'";
         }
 
+        // Prerequisite-specific options.
+        //
+
+        // --what
+        //
+        const char* what (ops.what_specified ()
+                          ? ops.what ().c_str ()
+                          : "file");
+
         // --cwd
         //
         optional<dir_path> cwd;
-
         if (ops.cwd_specified ())
         {
           if (!byprod)
@@ -1964,28 +2008,6 @@ namespace build2
             fail (ll) << "depdb dyndep: -I specified with --byproduct";
         }
 
-        // --file
-        //
-        // Note that if --file is specified without a program, then we assume
-        // it is one of the static prerequisites.
-        //
-        optional<path> file;
-
-        if (ops.file_specified ())
-        {
-          file = move (ops.file ());
-
-          if (file->relative ())
-          {
-            if (!cwd)
-              fail (ll) << "depdb dyndep: relative path specified with --file";
-
-            *file = *cwd / *file;
-          }
-        }
-        else if (!prog)
-          fail (ll) << "depdb dyndep: program or --file expected";
-
         // --default-type
         //
         // Get the default prerequisite type falling back to file{} if not
@@ -1997,7 +2019,7 @@ namespace build2
         // translation unit would want to make sure it resolves extracted
         // system headers to h{} targets analogous to the c module's rule.
         //
-        const target_type* def_pt;
+        const target_type* def_pt (&file::static_type);
         if (ops.default_type_specified ())
         {
           const string& t (ops.default_type ());
@@ -2005,10 +2027,8 @@ namespace build2
           def_pt = bs.find_target_type (t);
           if (def_pt == nullptr)
             fail (ll) << "depdb dyndep: unknown target type '" << t
-                      << "' specific with --default-type";
+                      << "' specified with --default-type";
         }
-        else
-          def_pt = &file::static_type;
 
         // --adhoc
         //
@@ -2018,6 +2038,76 @@ namespace build2
             fail (ll) << "depdb dyndep: --adhoc specified with --byproduct";
         }
 
+        // Target-specific options.
+        //
+
+        // --target-what
+        //
+        const char* what_tgt ("file");
+        if (ops.target_what_specified ())
+        {
+          if (!dyn_tgt)
+            fail (ll) << "depdb dyndep: --target-what specified without "
+                      << "--dyn-target";
+
+          what_tgt = ops.target_what ().c_str ();
+        }
+
+        // --target-cwd
+        //
+        optional<dir_path> cwd_tgt;
+        if (ops.target_cwd_specified ())
+        {
+          if (!dyn_tgt)
+            fail (ll) << "depdb dyndep: --target-cwd specified without "
+                      << "--dyn-target";
+
+          cwd_tgt = move (ops.target_cwd ());
+
+          if (cwd_tgt->relative ())
+            fail (ll) << "depdb dyndep: relative path specified with "
+                      << "--target-cwd";
+        }
+
+        // --target-default-type
+        //
+        const target_type* def_tt (&file::static_type);
+        if (ops.target_default_type_specified ())
+        {
+          if (!dyn_tgt)
+            fail (ll) << "depdb dyndep: --target-default-type specified "
+                      << "without --dyn-target";
+
+          const string& t (ops.target_default_type ());
+
+          def_tt = bs.find_target_type (t);
+          if (def_tt == nullptr)
+            fail (ll) << "depdb dyndep: unknown target type '" << t
+                      << "' specified with --target-default-type";
+        }
+
+
+        // --file (last since need --*cwd)
+        //
+        // Note that if --file is specified without a program, then we assume
+        // it is one of the static prerequisites.
+        //
+        optional<path> file;
+        if (ops.file_specified ())
+        {
+          file = move (ops.file ());
+
+          if (file->relative ())
+          {
+            if (!cwd && !cwd_tgt)
+              fail (ll) << "depdb dyndep: relative path specified with --file";
+
+            *file = (cwd ? *cwd : *cwd_tgt) / *file;
+          }
+        }
+        else if (!prog)
+          fail (ll) << "depdb dyndep: program or --file expected";
+
         // Update prerequisite targets.
         //
         using dyndep = dyndep_rule;
@@ -2148,6 +2238,8 @@ namespace build2
           return;
         }
 
+        const scope& rs (*bs.root_scope ());
+
         // This code is based on the prior work in the cc module (specifically
         // extract_headers()) where you can often find more detailed rationale
         // for some of the steps performed.
@@ -2161,6 +2253,7 @@ namespace build2
           [] (const scope& bs, const string& n, const string& e)
           {
             // NOTE: another version in adhoc_buildscript_rule::apply().
+            // NOTE: now also used for dynamic targets below!
 
             // @@ TODO: allow specifying base target types.
             //
@@ -2275,14 +2368,17 @@ namespace build2
             // they include the line index in their names to avoid clashes
             // between lines).
             //
-            // Cleanups are not an issue, they will simply replaced. And
+            // Cleanups are not an issue, they will simply be replaced. And
             // overriding the contents of the special files seems harmless and
             // consistent with what would happen if the command redirects its
             // output to a non-special file.
             //
+            // Note: make it a maybe-cleanup in case the command cleans it
+            // up itself.
+            //
             if (file)
               environment_->clean (
-                {build2::script::cleanup_type::always, *file},
+                {build2::script::cleanup_type::maybe, *file},
                 true /* implicit */);
           }
         };
@@ -2429,13 +2525,27 @@ namespace build2
                  << t;
           });
 
+        // While in the make format targets come before prerequisites, in
+        // depdb we store them after since any change to prerequisites can
+        // invalidate the set of targets. So we save them first and process
+        // later.
+        //
+        // Note also that we need to return them to the caller in case we are
+        // updating.
+
         // If nothing so far has invalidated the dependency database, then try
         // the cached data before running the program.
         //
         bool cache (!update);
+        bool skip_blank (false);
 
         for (bool restart (true), first_run (true); restart; cache = false)
         {
+          // Clear the state in case we are restarting.
+          //
+          if (dyn_tgt)
+            dyn_targets.clear ();
+
           restart = false;
 
           if (cache)
@@ -2444,7 +2554,8 @@ namespace build2
             //
             assert (skip_count == 0);
 
-            // We should always end with a blank line.
+            // We should always end with a blank line after the list of
+            // dynamic prerequisites.
             //
             for (;;)
             {
@@ -2458,8 +2569,11 @@ namespace build2
                 break;
               }
 
-              if (l->empty ()) // Done, nothing changed.
-                return;
+              if (l->empty ()) // Done with prerequisites, nothing changed.
+              {
+                skip_blank = true;
+                break;
+              }
 
               if (optional<bool> r = add (path (move (*l)), nullptr, mt))
               {
@@ -2481,6 +2595,36 @@ namespace build2
                 return;
               }
             }
+
+            if (!restart) // Nothing changed.
+            {
+              if (dyn_tgt)
+              {
+                // We should always end with a blank line after the list of
+                // dynamic targets.
+                //
+                for (;;)
+                {
+                  string* l (dd.read ());
+
+                  // If the line is invalid, run the compiler.
+                  //
+                  if (l == nullptr)
+                  {
+                    restart = true;
+                    break;
+                  }
+
+                  if (l->empty ()) // Done with target.
+                    break;
+
+                  dyn_targets.push_back (path (move (*l)));
+                }
+              }
+
+              if (!restart) // Done, nothing changed.
+                break; // Break earliy to keep cache=true.
+            }
           }
           else
           {
@@ -2614,31 +2758,63 @@ namespace build2
                     if (r.second.empty ())
                       continue;
 
-                    // @@ TODO: what should we do about targets?
+                    // Skip targets unless requested to extract.
                     //
-                    // Note that if we take GCC as an example, things are
+                    // BTW, if you are wondering why don't we extract targets
+                    // by default, take GCC as an example, where things are
                     // quite messed up: by default it ignores -o and just
                     // takes the source file name and replaces the extension
                     // with a platform-appropriate object file extension. One
                     // can specify a custom target (or even multiple targets)
-                    // with -MT or with -MQ (quoting). Though MinGW GCC still
-                    // does not quote `:` with -MQ. So in this case it's
+                    // with -MT or with -MQ (quoting). So in this case it's
                     // definitely easier for the user to ignore the targets
                     // and just specify everything in the buildfile.
                     //
-                    // On the other hand, other tools are likely to produce
-                    // more sensible output (except perhaps for quoting).
-                    //
-                    // @@ Maybe in the lax mode we should only recognize `:`
-                    //    if it's separated on at least one side?
-                    //
-                    //    Alternatively, we could detect Windows drives in
-                    //    paths and "handle" them (I believe this is what GNU
-                    //    make does). Maybe we should have three formats:
-                    //    make-lax, make, make-strict?
-                    //
                     if (r.first == make_type::target)
+                    {
+                      if (dyn_tgt)
+                      {
+                        path& f (r.second);
+
+                        if (f.relative ())
+                        {
+                          if (!cwd_tgt)
+                            fail (il) << "relative target path '" << f
+                                      << "' in make dependency declaration" <<
+                              info << "consider using --target-cwd to specify "
+                                   << "relative path base";
+
+                          f = *cwd_tgt / f;
+                        }
+
+                        // Note that unlike prerequisites, here we don't need
+                        // normalize_external() since we expect the targets to
+                        // be within this project.
+                        //
+                        try
+                        {
+                          f.normalize ();
+                        }
+                        catch (const invalid_path&)
+                        {
+                          fail << "invalid " << what_tgt << " path '"
+                               << f.string () << "'";
+                        }
+
+                        // The target must be within this project.
+                        //
+                        if (!f.sub (rs.out_path ()))
+                        {
+                          fail << what_tgt << " target path " << f
+                               << " must be inside project output directory "
+                               << rs.out_path ();
+                        }
+
+                        dyn_targets.push_back (move (f));
+                      }
+
                       continue;
+                    }
 
                     if (optional<bool> u = add (move (r.second), &skip, rmt))
                     {
@@ -2667,7 +2843,7 @@ namespace build2
                     break;
                 }
 
-                break;
+                break; // case
               }
             }
 
@@ -2678,9 +2854,88 @@ namespace build2
           }
         }
 
-        // Add the terminating blank line (we are updating depdb).
+        // Add the dynamic prerequisites terminating blank line if we are
+        // updating depdb and unless it's already there.
+        //
+        if (!cache && !skip_blank)
+          dd.expect ("");
+
+        // Handle dynamic targets.
         //
-        dd.expect ("");
+        if (dyn_tgt)
+        {
+          // There is one more level (at least that we know of) to this rabbit
+          // hole: if the set of dynamic targets changes between clean and
+          // update and we do a `clean update` batch, then we will end up with
+          // old targets (as entered by clean from old depdb information)
+          // being present during update. So we need to clean them out.
+          //
+          // Optimize this for a first/single batch (common case) by noticing
+          // that there are only real targets to start with.
+          //
+          optional<vector<const target*>> dts;
+          for (const target* m (&t); m != nullptr; m = m->adhoc_member)
+          {
+            if (m->decl != target_decl::real)
+              dts = vector<const target*> ();
+          }
+
+          for (const path& f: dyn_targets)
+          {
+            // Note that this logic should be consistent with what we have in
+            // adhoc_buildscript_rule::apply() for perform_clean.
+            //
+            pair<const build2::file&, bool> r (
+              dyndep::inject_adhoc_group_member (
+                what_tgt,
+                a, bs, t,
+                f, // Can't move since need to return dyn_targets.
+                map_ext, *def_tt));
+
+            // Note that we have to track the dynamic target even if it was
+            // already a member (think `b update && b clean update`).
+            //
+            if (r.second || r.first.decl != target_decl::real)
+            {
+              if (!cache)
+                dd.expect (f);
+
+              if (dts)
+                dts->push_back (&r.first);
+            }
+          }
+
+          // Add the dynamic targets terminating blank line.
+          //
+          if (!cache)
+            dd.expect ("");
+
+          // Clean out old dynamic targets (skip the primary member).
+          //
+          if (dts)
+          {
+            for (target* p (&t); p->adhoc_member != nullptr; )
+            {
+              target* m (p->adhoc_member);
+
+              if (m->decl != target_decl::real)
+              {
+                // While there could be quite a few dynamic targets (think
+                // something like Doxygen), this will hopefully be optimized
+                // down to a contiguous memory region scan for an integer and
+                // so should be fast.
+                //
+                if (find (dts->begin (), dts->end (), m) == dts->end ())
+                {
+                  p->adhoc_member = m->adhoc_member; // Drop m.
+                  continue;
+                }
+              }
+
+              p = m;
+            }
+          }
+        }
 
         // Reload $< and $> to make sure they contain the newly discovered
         // prerequisites and targets.
diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx
index 70e24aa..b615a90 100644
--- a/libbuild2/build/script/parser.hxx
+++ b/libbuild2/build/script/parser.hxx
@@ -123,14 +123,16 @@ namespace build2
         execute_depdb_preamble_dyndep (
           action a, const scope& base, file& t,
           environment& e, const script& s, runner& r,
-          depdb& dd, bool& update, timestamp mt, bool& deferred_failure)
+          depdb& dd,
+          paths& dyn_targets,
+          bool& update, timestamp mt, bool& deferred_failure)
         {
           exec_depdb_preamble (
             a, base, t,
             e, s, r,
             s.depdb_preamble.begin () + *s.depdb_dyndep,
             s.depdb_preamble.end (),
-            dd, &update, mt, &deferred_failure);
+            dd, &dyn_targets, &update, mt, &deferred_failure);
         }
 
         // This version doesn't actually execute the depdb-dyndep builtin (but
@@ -161,6 +163,7 @@ namespace build2
           // This is getting a bit ugly (we also don't really need to pass
           // depdb here). One day we will find a better way...
           //
+          paths dyn_targets;
           bool deferred_failure; // Dymmy.
 
           dyndep_byproduct v;
@@ -169,7 +172,7 @@ namespace build2
             e, s, r,
             s.depdb_preamble.begin () + *s.depdb_dyndep,
             s.depdb_preamble.end (),
-            dd, &update, mt, &deferred_failure, &v);
+            dd, &dyn_targets, &update, mt, &deferred_failure, &v);
           return v;
         }
 
@@ -221,6 +224,7 @@ namespace build2
                              environment&, const script&, runner&,
                              lines_iterator begin, lines_iterator end,
                              depdb&,
+                             paths* dyn_targets = nullptr,
                              bool* update = nullptr,
                              optional<timestamp> mt = nullopt,
                              bool* deferred_failure = nullptr,
@@ -231,6 +235,7 @@ namespace build2
                            size_t line_index, const location&,
                            action, const scope& base, file&,
                            depdb&,
+                           paths& dyn_targets,
                            bool& update,
                            timestamp,
                            bool& deferred_failure,
@@ -355,6 +360,7 @@ namespace build2
         optional<pair<location, size_t>>
                            depdb_dyndep_;   // depdb-dyndep location/position.
         bool               depdb_dyndep_byproduct_ = false; // --byproduct
+        bool               depdb_dyndep_dyn_target_ = false; // --dyn-target
         lines              depdb_preamble_; // Note: excluding depdb-clear.
 
         // If present, the first impure function called in the body of the
diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx
index 57a893e..08f1bf4 100644
--- a/libbuild2/build/script/script.hxx
+++ b/libbuild2/build/script/script.hxx
@@ -82,8 +82,9 @@ namespace build2
         bool             depdb_value;                    // String or hash.
         optional<size_t> depdb_dyndep;                   // Pos of first dyndep.
         bool             depdb_dyndep_byproduct = false; // dyndep --byproduct
+        bool             depdb_dyndep_dyn_target = false;// dyndep --dyn-target
         lines            depdb_preamble;                 // Note include vars.
-        bool             depdb_preamble_temp_dir = false; // True if refs $~.
+        bool             depdb_preamble_temp_dir = false;// True if refs $~.
 
         location start_loc;
         location end_loc;
diff --git a/libbuild2/dyndep.cxx b/libbuild2/dyndep.cxx
index ace901b..b793de8 100644
--- a/libbuild2/dyndep.cxx
+++ b/libbuild2/dyndep.cxx
@@ -162,12 +162,6 @@ namespace build2
       dr << info << "consider listing it as static prerequisite of " << t;
   }
 
-  // Reverse-lookup target type(s) from file name/extension.
-  //
-  // If the list of base target types is specified, then only these types and
-  // those derived from them are considered. Otherwise, any file-based type is
-  // considered but not the file type itself.
-  //
   small_vector<const target_type*, 2> dyndep_rule::
   map_extension (const scope& bs,
                  const string& n, const string& e,
@@ -877,4 +871,99 @@ namespace build2
 
     return t;
   }
+
+  pair<const file&, bool> dyndep_rule::
+  inject_adhoc_group_member (const char* what,
+                             action, const scope& bs, target& t,
+                             path f,
+                             const function<map_extension_func>& map_ext,
+                             const target_type& fallback)
+  {
+    path n (f.leaf ());
+    string e (n.extension ());
+    n.make_base ();
+
+    // Map extension to the target type, falling back to def_tt.
+    //
+    small_vector<const target_type*, 2> tts;
+    if (map_ext != nullptr)
+      tts = map_ext (bs, n.string (), e);
+
+    // Not sure what else we can do in this case.
+    //
+    if (tts.size () > 1)
+    {
+      diag_record dr (fail);
+
+      dr << "mapping of " << what << " target file " << f
+         << " to target type is ambiguous";
+
+      for (const target_type* tt: tts)
+        dr << info << "can be " << tt->name << "{}";
+    }
+
+    const target_type& tt (tts.empty () ? fallback : *tts.front ());
+
+    if (!tt.is_a<file> ())
+    {
+      fail << what << " target file " << f << " mapped to non-file-based "
+           << "target type " << tt.name << "{}";
+    }
+
+    // Assume nobody else can insert these members (seems reasonable seeing
+    // that their names are dynamically discovered).
+    //
+    auto l (search_new_locked (
+              bs.ctx,
+              tt,
+              f.directory (),
+              dir_path (), // Always in out.
+              move (n).string (),
+              &e,
+              &bs));
+
+    file* ft (&l.first.as<file> ()); // Note: non-const only if locked.
+
+    // Skip if this is one of the static targets (or a duplicate of the
+    // dynamic target).
+    //
+    // In particular, we expect to skip all the targets that we could not lock
+    // (e.g., in case all of this has already been done for the previous
+    // operation in a batch; make sure to test `update update update` and
+    // `update clean update ...` batches if changing anything here).
+    //
+    // While at it also find the ad hoc members list tail.
+    //
+    const_ptr<target>* tail (&t.adhoc_member);
+    for (target* m (&t); m != nullptr; m = m->adhoc_member)
+    {
+      if (ft == m)
+      {
+        tail = nullptr;
+        break;
+      }
+
+      tail = &m->adhoc_member;
+    }
+
+    if (tail == nullptr)
+      return pair<const file&, bool> (*ft, false);
+
+    if (!l.second)
+      fail << "dynamic " << what << " target " << *ft << " already exists "
+           << "and cannot be made ad hoc member of group " << t;
+
+    ft->group = &t;
+    l.second.unlock ();
+
+    // We need to be able to distinguish static targets from dynamic (see the
+    // static set hashing in adhoc_buildscript_rule::apply() for details).
+    //
+    assert (ft->decl != target_decl::real);
+
+    *tail = ft;
+    ft->path (move (f));
+
+    return pair<const file&, bool> (*ft, true);
+  }
 }
diff --git a/libbuild2/dyndep.hxx b/libbuild2/dyndep.hxx
index a6f800e..6ae585e 100644
--- a/libbuild2/dyndep.hxx
+++ b/libbuild2/dyndep.hxx
@@ -94,6 +94,10 @@ namespace build2
     // and those derived from them are considered. Otherwise, any file-based
     // type is considered but not the file type itself.
     //
+    // It's possible the extension-to-target type mapping is ambiguous (for
+    // example, because both C and C++-language headers use the same .h
+    // extension). So this function can return multiple target types.
+    //
     static small_vector<const target_type*, 2>
     map_extension (const scope& base,
                    const string& name, const string& ext,
@@ -244,6 +248,23 @@ namespace build2
       return inject_group_member (
         a, bs, g, move (p), T::static_type).template as<T> ();
     }
+
+    // Find or insert a target file path as a target, make it a member of the
+    // specified ad hoc group unless it already is, and set its path. Return
+    // the target and an indication of whether it was added as a member.
+    //
+    // The target type is determined using the map_extension function if
+    // specified, falling back to the fallback type if unable to.
+    //
+    // The file path must be absolute and normalized. Note that this function
+    // assumes that this target can only be known as a member of this group.
+    //
+    static pair<const file&, bool>
+    inject_adhoc_group_member (const char* what,
+                               action, const scope& base, target& g,
+                               path,
+                               const function<map_extension_func>&,
+                               const target_type& fallback);
   };
 }
 
diff --git a/libbuild2/script/script.cxx b/libbuild2/script/script.cxx
index b8dfc68..4a6ca33 100644
--- a/libbuild2/script/script.cxx
+++ b/libbuild2/script/script.cxx
@@ -768,7 +768,9 @@ namespace build2
     {
       using script::cleanup;
 
-      assert (!implicit || c.type == cleanup_type::always);
+      // Implicit never-cleanup doesn't make sense.
+      //
+      assert (!implicit || c.type != cleanup_type::never);
 
       const path& p (c.path);
 
diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx
index 037b18c..e935477 100644
--- a/libbuild2/target.hxx
+++ b/libbuild2/target.hxx
@@ -272,6 +272,10 @@ namespace build2
   //    fuzzy: they feel more `real` than `implied`. Maybe introduce
   //    `synthesized` in-between?
   //
+  // @@ There are also now dynamically-discovered targets (ad hoc group
+  //    members; see depdb-dyndep --dyn-target) which currently end up
+  //    with prereq_new.
+  //
   enum class target_decl: uint8_t
   {
     prereq_new = 1, // Created from prerequisite (create_new_target()).