diff options
| -rw-r--r-- | libbuild2/adhoc-rule-buildscript.cxx | 227 | ||||
| -rw-r--r-- | libbuild2/build/script/builtin-options.cxx | 17 | ||||
| -rw-r--r-- | libbuild2/build/script/builtin-options.hxx | 51 | ||||
| -rw-r--r-- | libbuild2/build/script/builtin-options.ixx | 90 | ||||
| -rw-r--r-- | libbuild2/build/script/builtin.cli | 41 | ||||
| -rw-r--r-- | libbuild2/build/script/parser.cxx | 421 | ||||
| -rw-r--r-- | libbuild2/build/script/parser.hxx | 12 | ||||
| -rw-r--r-- | libbuild2/build/script/script.hxx | 3 | ||||
| -rw-r--r-- | libbuild2/dyndep.cxx | 101 | ||||
| -rw-r--r-- | libbuild2/dyndep.hxx | 21 | ||||
| -rw-r--r-- | libbuild2/script/script.cxx | 4 | ||||
| -rw-r--r-- | libbuild2/target.hxx | 4 |
12 files changed, 864 insertions, 128 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index ce4ab8b..f3e3560 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -5,6 +5,8 @@ #include <sstream> +#include <libbutl/filesystem.hxx> // try_rm_file() + #include <libbuild2/depdb.hxx> #include <libbuild2/scope.hxx> #include <libbuild2/target.hxx> @@ -225,6 +227,8 @@ namespace build2 build::script::default_runner run; path dd; + paths dyn_targets; + paths old_dyn_targets; const scope* bs; timestamp mt; @@ -457,10 +461,120 @@ namespace build2 } } + // Read the list of dynamic targets from depdb, if exists (used in a few + // depdb-dyndep --dyn-target handling places below). + // + auto read_dyn_targets = [] (path ddp) -> paths + { + depdb dd (move (ddp), true /* read_only */); + + paths r; + while (dd.reading ()) // Breakout loop. + { + string* l; + auto read = [&dd, &l] () -> bool + { + return (l = dd.read ()) != nullptr; + }; + + if (!read ()) // Rule id. + break; + + // We can omit this for as long as we don't break our blank line + // anchors semantics (e.g., by adding another blank somewhere, say + // after the custom depdb builtins). + // +#if 0 + if (*l != rule_id_) + fail << "unable to clean dynamic target group " << t + << " with old depdb"; +#endif + + // Note that we cannot read out expected lines since there can be + // custom depdb builtins. We also need to skip the prerequisites + // list. So we read until the blank line that always terminates the + // prerequisites list. + // + { + bool r; + while ((r = read ()) && !l->empty ()) ; + if (!r) + break; + } + + // Read the dynamic target files. We should always end with a blank + // line. + // + for (;;) + { + if (!read () || l->empty ()) + break; + + r.push_back (path (*l)); + } + + break; + } + + return r; + }; + // See if we are providing the standard clean as a fallback. // if (me.fallback) - return &perform_clean_file; + { + // For depdb-dyndep --dyn-target use depdb to clean dynamic targets. + // + if (script.depdb_dyndep && script.depdb_dyndep_dyn_target) + { + file& t (xt.as<file> ()); + + // Note that only removing the relevant filesystem entries is not + // enough: we actually have to populate the group with members since + // this information could be used to clean derived targets (for + // example, object files). So we just do that and let the standard + // clean logic take care of them the same as static members. + // + using dyndep = dyndep_rule; + + function<dyndep::map_extension_func> map_ext ( + [] (const scope& bs, const string& n, const string& e) + { + return dyndep::map_extension (bs, n, e, nullptr); + }); + + for (path& f: read_dyn_targets (t.path () + ".d")) + { + // Note that this logic should be consistent with what we have in + // exec_depdb_dyndep(). + // + // Note that here we don't bother cleaning any old dynamic targets + // -- the more we can clean, the merrier. + // + // @@ We don't have --target-what, --target-default-type here. + // Could we do the same thing as byproduct to get them? That + // would require us running the first half of the depdb preamble + // but ignoring all the depdb builtins (we still want all the + // variable assignments -- maybe we could automatically skip them + // if we see depdb is not open). Wonder if there would be any + // other complications... + // + // BTW, this sort of works for --target-default-type since we + // just clean them as file{} targets (but diagnostics is off). It + // does break, however, if there is s batch, since then we end up + // detecting different targets sharing a path. This will also not + // work at all if/when we support specifying custom extension to + // type mapping in order to resolve ambiguities. + // + dyndep::inject_adhoc_group_member ("file", + a, bs, t, + move (f), + map_ext, file::static_type); + } + } + + return perform_clean_file; + } // If we have any update during match prerequisites, now is the time to // update them. @@ -575,10 +689,49 @@ namespace build2 } } + // Note that while it's tempting to turn match_data* into recipes, some of + // their members are not movable. And in the end we will have the same + // result: one dynamic memory allocation. + // + unique_ptr<match_data> md; + unique_ptr<match_data_byproduct> mdb; + + if (script.depdb_dyndep_byproduct) + { + mdb.reset (new match_data_byproduct ( + a, t, bs, script.depdb_preamble_temp_dir)); + } + else + { + md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir)); + + // If the set of dynamic targets can change based on changes to the + // inputs (say, each entity, such as a type, in the input file gets its + // own output file), then we can end up with a large number of old + // output files laying around because they are not part of the new + // dynamic target set. So we try to clean them up based on the old depdb + // information, similar to how we do it for perform_clean above (except + // here we will just keep the list of old files). + // + // Note: do before opening depdb, which can start over-writing it. + // + // We also have to do this speculatively, without knowing whether we + // will need to update. Oh, well, being dynamic ain't free. + // + if (script.depdb_dyndep_dyn_target) + md->old_dyn_targets = read_dyn_targets (tp + ".d"); + } + depdb dd (tp + ".d"); // NOTE: see the "static dependencies" version (with comments) below. // + // NOTE: We use blank lines as anchors to skip directly to certain entries + // (e.g., dynamic targets). So make sure none of the other entries + // can be blank (for example, see `depdb string` builtin). + // + // NOTE: KEEP IN SYNC WITH read_dyn_targets ABOVE! + // if (dd.expect ("<ad hoc buildscript recipe> 1") != nullptr) l4 ([&]{trace << "rule mismatch forcing update of " << t;}); @@ -613,10 +766,21 @@ namespace build2 l4 ([&]{trace << "recipe variable change forcing update of " << t;}); } + // Static targets and prerequisites (there can also be dynamic targets; + // see dyndep --dyn-target). + // + // There is a nuance: in an operation batch (e.g., `b update update`) we + // will already have the dynamic targets as members on the subsequent + // operations and we need to make sure we don't treat them as static. + // Using target_decl to distinguish the two seems like a natural way. + // { sha256 tcs; for (const target* m (&t); m != nullptr; m = m->adhoc_member) - hash_target (tcs, *m, storage); + { + if (m->decl == target_decl::real) + hash_target (tcs, *m, storage); + } if (dd.expect (tcs.string ()) != nullptr) l4 ([&]{trace << "target set change forcing update of " << t;}); @@ -634,22 +798,8 @@ namespace build2 } } - // Note that while it's tempting to turn match_data* into recipes, some of - // their members are not movable. And in the end we will have the same - // result: one dynamic memory allocation. + // Get ready to run the depdb preamble. // - unique_ptr<match_data> md; - unique_ptr<match_data_byproduct> mdb; - - if (script.depdb_dyndep_byproduct) - { - mdb.reset (new match_data_byproduct ( - a, t, bs, script.depdb_preamble_temp_dir)); - } - else - md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir)); - - build::script::environment& env (mdb != nullptr ? mdb->env : md->env); build::script::default_runner& run (mdb != nullptr ? mdb->run : md->run); @@ -828,20 +978,22 @@ namespace build2 else { // Run the second half of the preamble (depdb-dyndep commands) to update - // our prerequisite targets and extract dynamic dependencies. + // our prerequisite targets and extract dynamic dependencies (targets and + // prerequisites). // // Note that this should be the last update to depdb (the invalidation // order semantics). // - bool deferred_failure (false); + md->deferred_failure = false; { build::script::parser p (ctx); p.execute_depdb_preamble_dyndep (a, bs, t, env, script, run, dd, + md->dyn_targets, update, mt, - deferred_failure); + md->deferred_failure); } if (update && dd.reading () && !ctx.dry_run) @@ -854,7 +1006,6 @@ namespace build2 // md->bs = &bs; md->mt = update ? timestamp_nonexistent : mt; - md->deferred_failure = deferred_failure; return [this, md = move (md)] (action a, const target& t) { @@ -1066,7 +1217,7 @@ namespace build2 if (r.second.empty ()) continue; - // @@ TODO: what should we do about targets? + // Note: no support for dynamic targets in byproduct mode. // if (r.first == make_type::target) continue; @@ -1076,10 +1227,10 @@ namespace build2 if (f.relative ()) { if (!byp.cwd) - fail (il) << "relative path '" << f << "' in make dependency" - << " declaration" << + fail (il) << "relative prerequisite path '" << f + << "' in make dependency declaration" << info << "consider using --cwd to specify relative path " - << "base"; + << "base"; f = *byp.cwd / f; } @@ -1143,6 +1294,30 @@ namespace build2 return *ps; } + // Remove previous dynamic targets since their set may change with changes + // to the inputs (see apply() for details). + // + // The dry-run mode complicates things: if we don't remove the old files, + // then that information will be gone (since we update depdb even in the + // dry-run mode). But if we remove everything in the dry-run mode, then we + // may also remove some of the current files, which would be incorrect. So + // let's always remove but only files that are not in the current set. + // + for (const path& f: md.old_dyn_targets) + { + if (find (md.dyn_targets.begin (), md.dyn_targets.end (), f) == + md.dyn_targets.end ()) + { + // This is an optimization so best effort. + // + if (optional<rmfile_status> s = butl::try_rmfile_ignore_error (f)) + { + if (s == rmfile_status::success && verb >= 2) + text << "rm " << f; + } + } + } + // Sequence start time for mtime checks below. // timestamp start (!ctx.dry_run && depdb::mtime_check () @@ -1627,6 +1802,8 @@ namespace build2 // Finally, we print the entire ad hoc group at verbosity level 1, similar // to the default update diagnostics. // + // @@ TODO: .t may also be a temporary directory. + // return perform_clean_extra (a, t.as<file> (), {".d", ".t"}, diff --git a/libbuild2/build/script/builtin-options.cxx b/libbuild2/build/script/builtin-options.cxx index 9e4213d..3b64de1 100644 --- a/libbuild2/build/script/builtin-options.cxx +++ b/libbuild2/build/script/builtin-options.cxx @@ -289,7 +289,13 @@ namespace build2 adhoc_ (), cwd_ (), cwd_specified_ (false), - drop_cycles_ () + drop_cycles_ (), + target_what_ (), + target_what_specified_ (false), + target_default_type_ (), + target_default_type_specified_ (false), + target_cwd_ (), + target_cwd_specified_ (false) { } @@ -391,6 +397,15 @@ namespace build2 &depdb_dyndep_options::cwd_specified_ >; _cli_depdb_dyndep_options_map_["--drop-cycles"] = &::build2::build::cli::thunk< depdb_dyndep_options, &depdb_dyndep_options::drop_cycles_ >; + _cli_depdb_dyndep_options_map_["--target-what"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_what_, + &depdb_dyndep_options::target_what_specified_ >; + _cli_depdb_dyndep_options_map_["--target-default-type"] = + &::build2::build::cli::thunk< depdb_dyndep_options, string, &depdb_dyndep_options::target_default_type_, + &depdb_dyndep_options::target_default_type_specified_ >; + _cli_depdb_dyndep_options_map_["--target-cwd"] = + &::build2::build::cli::thunk< depdb_dyndep_options, dir_path, &depdb_dyndep_options::target_cwd_, + &depdb_dyndep_options::target_cwd_specified_ >; } }; diff --git a/libbuild2/build/script/builtin-options.hxx b/libbuild2/build/script/builtin-options.hxx index 590d3b2..2a00072 100644 --- a/libbuild2/build/script/builtin-options.hxx +++ b/libbuild2/build/script/builtin-options.hxx @@ -174,6 +174,51 @@ namespace build2 void drop_cycles (const bool&); + const string& + target_what () const; + + string& + target_what (); + + void + target_what (const string&); + + bool + target_what_specified () const; + + void + target_what_specified (bool); + + const string& + target_default_type () const; + + string& + target_default_type (); + + void + target_default_type (const string&); + + bool + target_default_type_specified () const; + + void + target_default_type_specified (bool); + + const dir_path& + target_cwd () const; + + dir_path& + target_cwd (); + + void + target_cwd (const dir_path&); + + bool + target_cwd_specified () const; + + void + target_cwd_specified (bool); + // Implementation details. // protected: @@ -201,6 +246,12 @@ namespace build2 dir_path cwd_; bool cwd_specified_; bool drop_cycles_; + string target_what_; + bool target_what_specified_; + string target_default_type_; + bool target_default_type_specified_; + dir_path target_cwd_; + bool target_cwd_specified_; }; } } diff --git a/libbuild2/build/script/builtin-options.ixx b/libbuild2/build/script/builtin-options.ixx index ea06a0f..3e3787f 100644 --- a/libbuild2/build/script/builtin-options.ixx +++ b/libbuild2/build/script/builtin-options.ixx @@ -233,6 +233,96 @@ namespace build2 { this->drop_cycles_ = x; } + + inline const string& depdb_dyndep_options:: + target_what () const + { + return this->target_what_; + } + + inline string& depdb_dyndep_options:: + target_what () + { + return this->target_what_; + } + + inline void depdb_dyndep_options:: + target_what (const string& x) + { + this->target_what_ = x; + } + + inline bool depdb_dyndep_options:: + target_what_specified () const + { + return this->target_what_specified_; + } + + inline void depdb_dyndep_options:: + target_what_specified (bool x) + { + this->target_what_specified_ = x; + } + + inline const string& depdb_dyndep_options:: + target_default_type () const + { + return this->target_default_type_; + } + + inline string& depdb_dyndep_options:: + target_default_type () + { + return this->target_default_type_; + } + + inline void depdb_dyndep_options:: + target_default_type (const string& x) + { + this->target_default_type_ = x; + } + + inline bool depdb_dyndep_options:: + target_default_type_specified () const + { + return this->target_default_type_specified_; + } + + inline void depdb_dyndep_options:: + target_default_type_specified (bool x) + { + this->target_default_type_specified_ = x; + } + + inline const dir_path& depdb_dyndep_options:: + target_cwd () const + { + return this->target_cwd_; + } + + inline dir_path& depdb_dyndep_options:: + target_cwd () + { + return this->target_cwd_; + } + + inline void depdb_dyndep_options:: + target_cwd (const dir_path& x) + { + this->target_cwd_ = x; + } + + inline bool depdb_dyndep_options:: + target_cwd_specified () const + { + return this->target_cwd_specified_; + } + + inline void depdb_dyndep_options:: + target_cwd_specified (bool x) + { + this->target_cwd_specified_ = x; + } } } } diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli index 7d0936f..2fba0b0 100644 --- a/libbuild2/build/script/builtin.cli +++ b/libbuild2/build/script/builtin.cli @@ -17,8 +17,8 @@ namespace build2 // class depdb_dyndep_options { - // Note that --byproduct, if any, must be the first option and is - // handled ad hoc, kind of as a sub-command. + // Note that --byproduct or --dyn-target, if any, must be the first + // option and is handled ad hoc. // // Similarly, --update-{include,exclude} are handled ad hoc and must // be literals, similar to the -- separator. They specify prerequisite @@ -44,16 +44,19 @@ namespace build2 // and the other for prerequisite, we omit "prerequisite" as that's // what we extract by default and most commonly. For example: // - // --what --what-target - // --default-type --default-target-type + // --what --target-what + // --default-type --target-default-type // path --file; // Read from file rather than stdin. string --format; // Dependency format: make (default). - string --what; // Dependency kind, e.g., "header". + // Dynamic dependency extraction options. + // + string --what; // Prerequisite kind, e.g., "header". - dir_paths --include-path|-I; // Search paths for generated files. + dir_paths --include-path|-I; // Search paths for generated + // prerequisites. string --default-type; // Default prerequisite type to use // if none could be derived from ext. @@ -64,14 +67,36 @@ namespace build2 // normal mode). dir_path --cwd; // Builtin's working directory used - // to complete relative paths (only - // in --byproduct mode). + // to complete relative paths of + // prerequisites (only in --byproduct + // mode). bool --drop-cycles; // Drop prerequisites that are also // targets. Only use if you are sure // such cycles are harmless, that is, // the output is not affected by such // prerequisites' content. + + // Dynamic target extraction options. + // + // This functionality is enabled with the --dyn-target option. Only + // the make format is supported, where the listed targets are added as + // ad hoc group members (unless already specified as static members). + // This functionality is not available in the byproduct mode. + // + // @@ BTW, here what would likely be more useful than default target + // is the ability to specify custom extension-to-type mapping in + // order to resolve ambiguities. See also the issue with getting + // these options during clean. + // + string --target-what; // Target kind, e.g., "source". + + string --target-default-type; // Default target type to use if none + // could be derived from ext. + + dir_path --target-cwd; // Builtin's working directory used to + // complete relative paths of targets. + }; } } diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index df0a419..e7268f9 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -158,6 +158,7 @@ namespace build2 { s.depdb_dyndep = depdb_dyndep_->second; s.depdb_dyndep_byproduct = depdb_dyndep_byproduct_; + s.depdb_dyndep_dyn_target = depdb_dyndep_dyn_target_; } s.depdb_preamble = move (depdb_preamble_); @@ -830,8 +831,18 @@ namespace build2 fail (l) << "multiple 'depdb dyndep' calls" << info (depdb_dyndep_->first) << "previous call is here"; - if (peek () == type::word && peeked ().value == "--byproduct") - depdb_dyndep_byproduct_ = true; + if (peek () == type::word) + { + const string& v (peeked ().value); + + // Note: --byproduct and --dyn-target are mutually + // exclusive. + // + if (v == "--byproduct") + depdb_dyndep_byproduct_ = true; + else if (v == "--dyn-target") + depdb_dyndep_dyn_target_ = true; + } } else { @@ -1280,6 +1291,7 @@ namespace build2 environment& e, const script& s, runner& r, lines_iterator begin, lines_iterator end, depdb& dd, + paths* dyn_targets, bool* update, optional<timestamp> mt, bool* deferred_failure, @@ -1308,12 +1320,17 @@ namespace build2 const script& scr; depdb& dd; + paths* dyn_targets; bool* update; bool* deferred_failure; optional<timestamp> mt; dyndep_byproduct* byp; - } data {trace, a, bs, t, e, s, dd, update, deferred_failure, mt, byp}; + } data { + trace, + a, bs, t, + e, s, + dd, dyn_targets, update, deferred_failure, mt, byp}; auto exec_cmd = [this, &data] (token& t, build2::script::token_type& tt, @@ -1345,6 +1362,7 @@ namespace build2 li, ll, data.a, data.bs, const_cast<file&> (data.t), data.dd, + *data.dyn_targets, *data.update, *data.mt, *data.deferred_failure, @@ -1354,35 +1372,29 @@ namespace build2 { names ns (exec_special (t, tt, true /* skip <cmd> */)); + string v; + const char* w (nullptr); if (cmd == "hash") { sha256 cs; for (const name& n: ns) to_checksum (cs, n); - if (data.dd.expect (cs.string ()) != nullptr) - l4 ([&] { - data.trace (ll) - << "'depdb hash' argument change forcing update of " - << data.t;}); + v = cs.string (); + w = "argument"; } else if (cmd == "string") { - string s; try { - s = convert<string> (move (ns)); + v = convert<string> (move (ns)); } catch (const invalid_argument& e) { fail (ll) << "invalid 'depdb string' argument: " << e; } - if (data.dd.expect (s) != nullptr) - l4 ([&] { - data.trace (ll) - << "'depdb string' argument change forcing update of " - << data.t;}); + w = "argument"; } else if (cmd == "env") { @@ -1403,14 +1415,32 @@ namespace build2 fail (ll) << pf << e; } - if (data.dd.expect (cs.string ()) != nullptr) - l4 ([&] { - data.trace (ll) - << "'depdb env' environment change forcing update of " - << data.t;}); + v = cs.string (); + w = "environment"; } else assert (false); + + // Prefix the value with the type letter. This serves two + // purposes: + // + // 1. It makes sure the result is never a blank line. We use + // blank lines as anchors to skip directly to certain entries + // (e.g., dynamic targets). + // + // 2. It allows us to detect the beginning of prerequisites + // since an absolute path will be distinguishable from these + // entries (in the future we may want to add an explicit + // blank after such custom entries to make this easier). + // + v.insert (0, 1, ' '); + v.insert (0, 1, cmd[0]); // `h`, `s`, or `e` + + if (data.dd.expect (v) != nullptr) + l4 ([&] { + data.trace (ll) + << "'depdb " << cmd << "' " << w << " change forcing " + << "update of " << data.t;}); } } else @@ -1617,6 +1647,7 @@ namespace build2 size_t li, const location& ll, action a, const scope& bs, file& t, depdb& dd, + paths& dyn_targets, bool& update, timestamp mt, bool& deferred_failure, @@ -1629,6 +1660,7 @@ namespace build2 depdb_dyndep_options ops; bool prog (false); bool byprod (false); + bool dyn_tgt (false); // Prerequisite update filter (--update-*). // @@ -1671,11 +1703,9 @@ namespace build2 next (t, tt); // Skip the 'dyndep' command. - if (tt == type::word && t.value == "--byproduct") - { - byprod = true; + if (tt == type::word && ((byprod = (t.value == "--byproduct")) || + (dyn_tgt = (t.value == "--dyn-target")))) next (t, tt); - } assert (byprod == (byprod_result != nullptr)); @@ -1894,10 +1924,23 @@ namespace build2 continue; } - // Handle --byproduct in the wrong place. + // Handle --byproduct and --dyn-target in the wrong place. // if (strcmp (a, "--byproduct") == 0) - fail (ll) << "depdb dyndep: --byproduct must be first option"; + { + fail (ll) << "depdb dyndep: " + << (dyn_tgt + ? "--byproduct specified with --dyn-target" + : "--byproduct must be first option"); + } + + if (strcmp (a, "--dyn-target") == 0) + { + fail (ll) << "depdb dyndep: " + << (byprod + ? "--dyn-target specified with --byproduct" + : "--dyn-target must be first option"); + } // Handle non-literal --update-*. // @@ -1922,16 +1965,9 @@ namespace build2 } } - // --what - // - const char* what (ops.what_specified () - ? ops.what ().c_str () - : "file"); - // --format // dyndep_format format (dyndep_format::make); - if (ops.format_specified ()) { const string& f (ops.format ()); @@ -1941,10 +1977,18 @@ namespace build2 << f << "'"; } + // Prerequisite-specific options. + // + + // --what + // + const char* what (ops.what_specified () + ? ops.what ().c_str () + : "file"); + // --cwd // optional<dir_path> cwd; - if (ops.cwd_specified ()) { if (!byprod) @@ -1964,28 +2008,6 @@ namespace build2 fail (ll) << "depdb dyndep: -I specified with --byproduct"; } - // --file - // - // Note that if --file is specified without a program, then we assume - // it is one of the static prerequisites. - // - optional<path> file; - - if (ops.file_specified ()) - { - file = move (ops.file ()); - - if (file->relative ()) - { - if (!cwd) - fail (ll) << "depdb dyndep: relative path specified with --file"; - - *file = *cwd / *file; - } - } - else if (!prog) - fail (ll) << "depdb dyndep: program or --file expected"; - // --default-type // // Get the default prerequisite type falling back to file{} if not @@ -1997,7 +2019,7 @@ namespace build2 // translation unit would want to make sure it resolves extracted // system headers to h{} targets analogous to the c module's rule. // - const target_type* def_pt; + const target_type* def_pt (&file::static_type); if (ops.default_type_specified ()) { const string& t (ops.default_type ()); @@ -2005,10 +2027,8 @@ namespace build2 def_pt = bs.find_target_type (t); if (def_pt == nullptr) fail (ll) << "depdb dyndep: unknown target type '" << t - << "' specific with --default-type"; + << "' specified with --default-type"; } - else - def_pt = &file::static_type; // --adhoc // @@ -2018,6 +2038,76 @@ namespace build2 fail (ll) << "depdb dyndep: --adhoc specified with --byproduct"; } + // Target-specific options. + // + + // --target-what + // + const char* what_tgt ("file"); + if (ops.target_what_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-what specified without " + << "--dyn-target"; + + what_tgt = ops.target_what ().c_str (); + } + + // --target-cwd + // + optional<dir_path> cwd_tgt; + if (ops.target_cwd_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-cwd specified without " + << "--dyn-target"; + + cwd_tgt = move (ops.target_cwd ()); + + if (cwd_tgt->relative ()) + fail (ll) << "depdb dyndep: relative path specified with " + << "--target-cwd"; + } + + // --target-default-type + // + const target_type* def_tt (&file::static_type); + if (ops.target_default_type_specified ()) + { + if (!dyn_tgt) + fail (ll) << "depdb dyndep: --target-default-type specified " + << "without --dyn-target"; + + const string& t (ops.target_default_type ()); + + def_tt = bs.find_target_type (t); + if (def_tt == nullptr) + fail (ll) << "depdb dyndep: unknown target type '" << t + << "' specified with --target-default-type"; + } + + + // --file (last since need --*cwd) + // + // Note that if --file is specified without a program, then we assume + // it is one of the static prerequisites. + // + optional<path> file; + if (ops.file_specified ()) + { + file = move (ops.file ()); + + if (file->relative ()) + { + if (!cwd && !cwd_tgt) + fail (ll) << "depdb dyndep: relative path specified with --file"; + + *file = (cwd ? *cwd : *cwd_tgt) / *file; + } + } + else if (!prog) + fail (ll) << "depdb dyndep: program or --file expected"; + // Update prerequisite targets. // using dyndep = dyndep_rule; @@ -2148,6 +2238,8 @@ namespace build2 return; } + const scope& rs (*bs.root_scope ()); + // This code is based on the prior work in the cc module (specifically // extract_headers()) where you can often find more detailed rationale // for some of the steps performed. @@ -2161,6 +2253,7 @@ namespace build2 [] (const scope& bs, const string& n, const string& e) { // NOTE: another version in adhoc_buildscript_rule::apply(). + // NOTE: now also used for dynamic targets below! // @@ TODO: allow specifying base target types. // @@ -2275,14 +2368,17 @@ namespace build2 // they include the line index in their names to avoid clashes // between lines). // - // Cleanups are not an issue, they will simply replaced. And + // Cleanups are not an issue, they will simply be replaced. And // overriding the contents of the special files seems harmless and // consistent with what would happen if the command redirects its // output to a non-special file. // + // Note: make it a maybe-cleanup in case the command cleans it + // up itself. + // if (file) environment_->clean ( - {build2::script::cleanup_type::always, *file}, + {build2::script::cleanup_type::maybe, *file}, true /* implicit */); } }; @@ -2429,13 +2525,27 @@ namespace build2 << t; }); + // While in the make format targets come before prerequisites, in + // depdb we store them after since any change to prerequisites can + // invalidate the set of targets. So we save them first and process + // later. + // + // Note also that we need to return them to the caller in case we are + // updating. + // If nothing so far has invalidated the dependency database, then try // the cached data before running the program. // bool cache (!update); + bool skip_blank (false); for (bool restart (true), first_run (true); restart; cache = false) { + // Clear the state in case we are restarting. + // + if (dyn_tgt) + dyn_targets.clear (); + restart = false; if (cache) @@ -2444,7 +2554,8 @@ namespace build2 // assert (skip_count == 0); - // We should always end with a blank line. + // We should always end with a blank line after the list of + // dynamic prerequisites. // for (;;) { @@ -2458,8 +2569,11 @@ namespace build2 break; } - if (l->empty ()) // Done, nothing changed. - return; + if (l->empty ()) // Done with prerequisites, nothing changed. + { + skip_blank = true; + break; + } if (optional<bool> r = add (path (move (*l)), nullptr, mt)) { @@ -2481,6 +2595,36 @@ namespace build2 return; } } + + if (!restart) // Nothing changed. + { + if (dyn_tgt) + { + // We should always end with a blank line after the list of + // dynamic targets. + // + for (;;) + { + string* l (dd.read ()); + + // If the line is invalid, run the compiler. + // + if (l == nullptr) + { + restart = true; + break; + } + + if (l->empty ()) // Done with target. + break; + + dyn_targets.push_back (path (move (*l))); + } + } + + if (!restart) // Done, nothing changed. + break; // Break earliy to keep cache=true. + } } else { @@ -2614,31 +2758,63 @@ namespace build2 if (r.second.empty ()) continue; - // @@ TODO: what should we do about targets? + // Skip targets unless requested to extract. // - // Note that if we take GCC as an example, things are + // BTW, if you are wondering why don't we extract targets + // by default, take GCC as an example, where things are // quite messed up: by default it ignores -o and just // takes the source file name and replaces the extension // with a platform-appropriate object file extension. One // can specify a custom target (or even multiple targets) - // with -MT or with -MQ (quoting). Though MinGW GCC still - // does not quote `:` with -MQ. So in this case it's + // with -MT or with -MQ (quoting). So in this case it's // definitely easier for the user to ignore the targets // and just specify everything in the buildfile. // - // On the other hand, other tools are likely to produce - // more sensible output (except perhaps for quoting). - // - // @@ Maybe in the lax mode we should only recognize `:` - // if it's separated on at least one side? - // - // Alternatively, we could detect Windows drives in - // paths and "handle" them (I believe this is what GNU - // make does). Maybe we should have three formats: - // make-lax, make, make-strict? - // if (r.first == make_type::target) + { + if (dyn_tgt) + { + path& f (r.second); + + if (f.relative ()) + { + if (!cwd_tgt) + fail (il) << "relative target path '" << f + << "' in make dependency declaration" << + info << "consider using --target-cwd to specify " + << "relative path base"; + + f = *cwd_tgt / f; + } + + // Note that unlike prerequisites, here we don't need + // normalize_external() since we expect the targets to + // be within this project. + // + try + { + f.normalize (); + } + catch (const invalid_path&) + { + fail << "invalid " << what_tgt << " path '" + << f.string () << "'"; + } + + // The target must be within this project. + // + if (!f.sub (rs.out_path ())) + { + fail << what_tgt << " target path " << f + << " must be inside project output directory " + << rs.out_path (); + } + + dyn_targets.push_back (move (f)); + } + continue; + } if (optional<bool> u = add (move (r.second), &skip, rmt)) { @@ -2667,7 +2843,7 @@ namespace build2 break; } - break; + break; // case } } @@ -2678,9 +2854,88 @@ namespace build2 } } - // Add the terminating blank line (we are updating depdb). + // Add the dynamic prerequisites terminating blank line if we are + // updating depdb and unless it's already there. + // + if (!cache && !skip_blank) + dd.expect (""); + + // Handle dynamic targets. // - dd.expect (""); + if (dyn_tgt) + { + // There is one more level (at least that we know of) to this rabbit + // hole: if the set of dynamic targets changes between clean and + // update and we do a `clean update` batch, then we will end up with + // old targets (as entered by clean from old depdb information) + // being present during update. So we need to clean them out. + // + // Optimize this for a first/single batch (common case) by noticing + // that there are only real targets to start with. + // + optional<vector<const target*>> dts; + for (const target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (m->decl != target_decl::real) + dts = vector<const target*> (); + } + + for (const path& f: dyn_targets) + { + // Note that this logic should be consistent with what we have in + // adhoc_buildscript_rule::apply() for perform_clean. + // + pair<const build2::file&, bool> r ( + dyndep::inject_adhoc_group_member ( + what_tgt, + a, bs, t, + f, // Can't move since need to return dyn_targets. + map_ext, *def_tt)); + + // Note that we have to track the dynamic target even if it was + // already a member (think `b update && b clean update`). + // + if (r.second || r.first.decl != target_decl::real) + { + if (!cache) + dd.expect (f); + + if (dts) + dts->push_back (&r.first); + } + } + + // Add the dynamic targets terminating blank line. + // + if (!cache) + dd.expect (""); + + // Clean out old dynamic targets (skip the primary member). + // + if (dts) + { + for (target* p (&t); p->adhoc_member != nullptr; ) + { + target* m (p->adhoc_member); + + if (m->decl != target_decl::real) + { + // While there could be quite a few dynamic targets (think + // something like Doxygen), this will hopefully be optimized + // down to a contiguous memory region scan for an integer and + // so should be fast. + // + if (find (dts->begin (), dts->end (), m) == dts->end ()) + { + p->adhoc_member = m->adhoc_member; // Drop m. + continue; + } + } + + p = m; + } + } + } // Reload $< and $> to make sure they contain the newly discovered // prerequisites and targets. diff --git a/libbuild2/build/script/parser.hxx b/libbuild2/build/script/parser.hxx index 70e24aa..b615a90 100644 --- a/libbuild2/build/script/parser.hxx +++ b/libbuild2/build/script/parser.hxx @@ -123,14 +123,16 @@ namespace build2 execute_depdb_preamble_dyndep ( action a, const scope& base, file& t, environment& e, const script& s, runner& r, - depdb& dd, bool& update, timestamp mt, bool& deferred_failure) + depdb& dd, + paths& dyn_targets, + bool& update, timestamp mt, bool& deferred_failure) { exec_depdb_preamble ( a, base, t, e, s, r, s.depdb_preamble.begin () + *s.depdb_dyndep, s.depdb_preamble.end (), - dd, &update, mt, &deferred_failure); + dd, &dyn_targets, &update, mt, &deferred_failure); } // This version doesn't actually execute the depdb-dyndep builtin (but @@ -161,6 +163,7 @@ namespace build2 // This is getting a bit ugly (we also don't really need to pass // depdb here). One day we will find a better way... // + paths dyn_targets; bool deferred_failure; // Dymmy. dyndep_byproduct v; @@ -169,7 +172,7 @@ namespace build2 e, s, r, s.depdb_preamble.begin () + *s.depdb_dyndep, s.depdb_preamble.end (), - dd, &update, mt, &deferred_failure, &v); + dd, &dyn_targets, &update, mt, &deferred_failure, &v); return v; } @@ -221,6 +224,7 @@ namespace build2 environment&, const script&, runner&, lines_iterator begin, lines_iterator end, depdb&, + paths* dyn_targets = nullptr, bool* update = nullptr, optional<timestamp> mt = nullopt, bool* deferred_failure = nullptr, @@ -231,6 +235,7 @@ namespace build2 size_t line_index, const location&, action, const scope& base, file&, depdb&, + paths& dyn_targets, bool& update, timestamp, bool& deferred_failure, @@ -355,6 +360,7 @@ namespace build2 optional<pair<location, size_t>> depdb_dyndep_; // depdb-dyndep location/position. bool depdb_dyndep_byproduct_ = false; // --byproduct + bool depdb_dyndep_dyn_target_ = false; // --dyn-target lines depdb_preamble_; // Note: excluding depdb-clear. // If present, the first impure function called in the body of the diff --git a/libbuild2/build/script/script.hxx b/libbuild2/build/script/script.hxx index 57a893e..08f1bf4 100644 --- a/libbuild2/build/script/script.hxx +++ b/libbuild2/build/script/script.hxx @@ -82,8 +82,9 @@ namespace build2 bool depdb_value; // String or hash. optional<size_t> depdb_dyndep; // Pos of first dyndep. bool depdb_dyndep_byproduct = false; // dyndep --byproduct + bool depdb_dyndep_dyn_target = false;// dyndep --dyn-target lines depdb_preamble; // Note include vars. - bool depdb_preamble_temp_dir = false; // True if refs $~. + bool depdb_preamble_temp_dir = false;// True if refs $~. location start_loc; location end_loc; diff --git a/libbuild2/dyndep.cxx b/libbuild2/dyndep.cxx index ace901b..b793de8 100644 --- a/libbuild2/dyndep.cxx +++ b/libbuild2/dyndep.cxx @@ -162,12 +162,6 @@ namespace build2 dr << info << "consider listing it as static prerequisite of " << t; } - // Reverse-lookup target type(s) from file name/extension. - // - // If the list of base target types is specified, then only these types and - // those derived from them are considered. Otherwise, any file-based type is - // considered but not the file type itself. - // small_vector<const target_type*, 2> dyndep_rule:: map_extension (const scope& bs, const string& n, const string& e, @@ -877,4 +871,99 @@ namespace build2 return t; } + + pair<const file&, bool> dyndep_rule:: + inject_adhoc_group_member (const char* what, + action, const scope& bs, target& t, + path f, + const function<map_extension_func>& map_ext, + const target_type& fallback) + { + path n (f.leaf ()); + string e (n.extension ()); + n.make_base (); + + // Map extension to the target type, falling back to def_tt. + // + small_vector<const target_type*, 2> tts; + if (map_ext != nullptr) + tts = map_ext (bs, n.string (), e); + + // Not sure what else we can do in this case. + // + if (tts.size () > 1) + { + diag_record dr (fail); + + dr << "mapping of " << what << " target file " << f + << " to target type is ambiguous"; + + for (const target_type* tt: tts) + dr << info << "can be " << tt->name << "{}"; + } + + const target_type& tt (tts.empty () ? fallback : *tts.front ()); + + if (!tt.is_a<file> ()) + { + fail << what << " target file " << f << " mapped to non-file-based " + << "target type " << tt.name << "{}"; + } + + // Assume nobody else can insert these members (seems reasonable seeing + // that their names are dynamically discovered). + // + auto l (search_new_locked ( + bs.ctx, + tt, + f.directory (), + dir_path (), // Always in out. + move (n).string (), + &e, + &bs)); + + file* ft (&l.first.as<file> ()); // Note: non-const only if locked. + + // Skip if this is one of the static targets (or a duplicate of the + // dynamic target). + // + // In particular, we expect to skip all the targets that we could not lock + // (e.g., in case all of this has already been done for the previous + // operation in a batch; make sure to test `update update update` and + // `update clean update ...` batches if changing anything here). + // + // While at it also find the ad hoc members list tail. + // + const_ptr<target>* tail (&t.adhoc_member); + for (target* m (&t); m != nullptr; m = m->adhoc_member) + { + if (ft == m) + { + tail = nullptr; + break; + } + + tail = &m->adhoc_member; + } + + if (tail == nullptr) + return pair<const file&, bool> (*ft, false); + + if (!l.second) + fail << "dynamic " << what << " target " << *ft << " already exists " + << "and cannot be made ad hoc member of group " << t; + + ft->group = &t; + l.second.unlock (); + + // We need to be able to distinguish static targets from dynamic (see the + // static set hashing in adhoc_buildscript_rule::apply() for details). + // + assert (ft->decl != target_decl::real); + + *tail = ft; + ft->path (move (f)); + + return pair<const file&, bool> (*ft, true); + } } diff --git a/libbuild2/dyndep.hxx b/libbuild2/dyndep.hxx index a6f800e..6ae585e 100644 --- a/libbuild2/dyndep.hxx +++ b/libbuild2/dyndep.hxx @@ -94,6 +94,10 @@ namespace build2 // and those derived from them are considered. Otherwise, any file-based // type is considered but not the file type itself. // + // It's possible the extension-to-target type mapping is ambiguous (for + // example, because both C and C++-language headers use the same .h + // extension). So this function can return multiple target types. + // static small_vector<const target_type*, 2> map_extension (const scope& base, const string& name, const string& ext, @@ -244,6 +248,23 @@ namespace build2 return inject_group_member ( a, bs, g, move (p), T::static_type).template as<T> (); } + + // Find or insert a target file path as a target, make it a member of the + // specified ad hoc group unless it already is, and set its path. Return + // the target and an indication of whether it was added as a member. + // + // The target type is determined using the map_extension function if + // specified, falling back to the fallback type if unable to. + // + // The file path must be absolute and normalized. Note that this function + // assumes that this target can only be known as a member of this group. + // + static pair<const file&, bool> + inject_adhoc_group_member (const char* what, + action, const scope& base, target& g, + path, + const function<map_extension_func>&, + const target_type& fallback); }; } diff --git a/libbuild2/script/script.cxx b/libbuild2/script/script.cxx index b8dfc68..4a6ca33 100644 --- a/libbuild2/script/script.cxx +++ b/libbuild2/script/script.cxx @@ -768,7 +768,9 @@ namespace build2 { using script::cleanup; - assert (!implicit || c.type == cleanup_type::always); + // Implicit never-cleanup doesn't make sense. + // + assert (!implicit || c.type != cleanup_type::never); const path& p (c.path); diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx index 037b18c..e935477 100644 --- a/libbuild2/target.hxx +++ b/libbuild2/target.hxx @@ -272,6 +272,10 @@ namespace build2 // fuzzy: they feel more `real` than `implied`. Maybe introduce // `synthesized` in-between? // + // @@ There are also now dynamically-discovered targets (ad hoc group + // members; see depdb-dyndep --dyn-target) which currently end up + // with prereq_new. + // enum class target_decl: uint8_t { prereq_new = 1, // Created from prerequisite (create_new_target()). |