diff options
Diffstat (limited to 'libbuild2')
78 files changed, 4077 insertions, 971 deletions
diff --git a/libbuild2/adhoc-rule-buildscript.cxx b/libbuild2/adhoc-rule-buildscript.cxx index 3e868a6..e3ed0a4 100644 --- a/libbuild2/adhoc-rule-buildscript.cxx +++ b/libbuild2/adhoc-rule-buildscript.cxx @@ -223,6 +223,19 @@ namespace build2 using dynamic_target = build::script::parser::dynamic_target; using dynamic_targets = build::script::parser::dynamic_targets; + // Return true if the path exist and is a symlink. + // + static inline bool + path_symlink (const path& p) + { + pair<bool, butl::entry_stat> r ( + butl::path_entry (p, + false /* follow_symlinks */, + true /* ignore_errors */)); + + return r.first && r.second.type == butl::entry_type::symlink; + }; + struct adhoc_buildscript_rule::match_data { match_data (action a, const target& t, const scope& bs, bool temp_dir) @@ -236,6 +249,7 @@ namespace build2 const scope* bs; timestamp mt; + bool symlink; bool deferred_failure; }; @@ -257,6 +271,7 @@ namespace build2 const scope* bs; timestamp mt; + bool symlink; }; bool adhoc_buildscript_rule:: @@ -1015,8 +1030,12 @@ namespace build2 bool update (false); timestamp mt; + // Support creating file symlinks using ad hoc recipes. + // + bool symlink (false); + if (dd.writing ()) - update = true; + update = true; // Will re-query symlink. else { if (g == nullptr) @@ -1025,6 +1044,8 @@ namespace build2 if ((mt = ft.mtime ()) == timestamp_unknown) ft.mtime (mt = mtime (tp)); // Cache. + + symlink = mt != timestamp_nonexistent && path_symlink (tp); } else { @@ -1038,13 +1059,21 @@ namespace build2 : nullptr)); if (p != nullptr) + { mt = g->load_mtime (*p); + symlink = mt != timestamp_nonexistent && path_symlink (*p); + } else - update = true; + update = true; // Will re-query symlink. } if (!update) - update = dd.mtime > mt; + { + // If this is a symlink, depdb mtime could be greater than the symlink + // target. + // + update = dd.mtime > mt && !symlink; + } } if (update) @@ -1178,9 +1207,11 @@ namespace build2 } // Note that in case of dry run we will have an incomplete (but valid) - // database which will be updated on the next non-dry run. + // database which will be updated on the next non-dry run. Except that + // we may still end up performing a non-dry-run update due to update + // during match or load. // - if (!update || ctx.dry_run_option) + if (!update /*|| ctx.dry_run_option*/) dd.close (false /* mtime_check */); else mdb->dd = dd.close_to_reopen (); @@ -1189,6 +1220,7 @@ namespace build2 // mdb->bs = &bs; mdb->mt = update ? timestamp_nonexistent : mt; + mdb->symlink = symlink; return [this, md = move (mdb)] (action a, const target& t) { @@ -1216,8 +1248,24 @@ namespace build2 md->deferred_failure); } - if (update && dd.reading () && !ctx.dry_run_option) - dd.touch = timestamp_unknown; + // Update depdb timestamp if nothing changed. Failed that, we will keep + // re-validating the information store in depdb (see similar logic in + // cc::compile_rule). + // + if (update && dd.reading ()) + { + // What will happen if dry_run_option is true but we still end up + // performing a non-dry-run update due to update during match or + // load? In this case the target will become up-to-date and we will + // keep re-validating the cache until the depdb will get touched due + // to other reasons, which would be bad. So it feels like the least + // bad option is to keep re-touching the database on dry-run. + // +#if 0 + if (!ctx.dry_run_option) +#endif + dd.touch = timestamp_unknown; + } dd.close (false /* mtime_check */); @@ -1265,6 +1313,7 @@ namespace build2 md->bs = &bs; md->dd = move (dd.path); md->mt = update ? timestamp_nonexistent : mt; + md->symlink = symlink; return [this, md = move (md)] (action a, const target& t) { @@ -1592,18 +1641,32 @@ namespace build2 timestamp now (system_clock::now ()); + const path* tp (nullptr); if (!ctx.dry_run) { // Only now we know for sure there must be a member in the group. // const file& ft ((g == nullptr ? t : *g->members.front ()).as<file> ()); + tp = &ft.path (); + + md.symlink = path_symlink (*tp); // Re-query. - depdb::check_mtime (start, md.dd.path, ft.path (), now); + // Again, if this is a symlink, depdb mtime will be greater than + // the symlink target. + // + if (!md.symlink) + depdb::check_mtime (start, md.dd.path, *tp, now); } + // Symlinks don't play well with dry-run (see full description in + // perform_update_file_or_group()). + // (g == nullptr ? static_cast<const mtime_target&> (t.as<file> ()) - : static_cast<const mtime_target&> (*g)).mtime (now); + : static_cast<const mtime_target&> (*g)).mtime ( + md.symlink && tp != nullptr + ? build2::mtime (*tp) + : now); return target_state::changed; } @@ -1662,17 +1725,32 @@ namespace build2 timestamp now (system_clock::now ()); + const path* tp (nullptr); if (!ctx.dry_run) { // Note: in case of deferred failure we may not have any members. // const file& ft ((g == nullptr ? t : *g->members.front ()).as<file> ()); - depdb::check_mtime (start, md.dd, ft.path (), now); + tp = &ft.path (); + + md.symlink = path_symlink (*tp); // Re-query. + + // Again, if this is a symlink, depdb mtime will be greater than the + // symlink target. + // + if (!md.symlink) + depdb::check_mtime (start, md.dd, *tp, now); } + // Symlinks don't play well with dry-run (see full description in + // perform_update_file_or_group()). + // (g == nullptr ? static_cast<const mtime_target&> (t) - : static_cast<const mtime_target&> (*g)).mtime (now); + : static_cast<const mtime_target&> (*g)).mtime ( + md.symlink && tp != nullptr + ? build2::mtime (*tp) + : now); return target_state::changed; } @@ -1695,35 +1773,14 @@ namespace build2 const file& ft ((g == nullptr ? t : *g->members.front ()).as<file> ()); const path& tp (ft.path ()); - // Support creating file symlinks using ad hoc recipes. - // - auto path_symlink = [&tp] () - { - pair<bool, butl::entry_stat> r ( - butl::path_entry (tp, - false /* follow_symlinks */, - true /* ignore_errors */)); - - return r.first && r.second.type == butl::entry_type::symlink; - }; - // Update prerequisites and determine if any of them render this target // out-of-date. // - // If the file entry exists, check if its a symlink. - // - bool symlink (false); - timestamp mt; + timestamp mt (g == nullptr ? ft.load_mtime () : g->load_mtime (tp)); - if (g == nullptr) - { - mt = ft.load_mtime (); - - if (mt != timestamp_nonexistent) - symlink = path_symlink (); - } - else - mt = g->load_mtime (tp); + // Support creating file symlinks using ad hoc recipes. + // + bool symlink (mt != timestamp_nonexistent && path_symlink (tp)); // This is essentially ps=execute_prerequisites(a, t, mt) which we // cannot use because we need to see ad hoc prerequisites. @@ -1923,8 +1980,7 @@ namespace build2 { if (!ctx.dry_run) { - if (g == nullptr) - symlink = path_symlink (); + symlink = path_symlink (tp); // Re-query. // Again, if this is a symlink, depdb mtime will be greater than // the symlink target. diff --git a/libbuild2/algorithm.cxx b/libbuild2/algorithm.cxx index 16f1503..d1cd474 100644 --- a/libbuild2/algorithm.cxx +++ b/libbuild2/algorithm.cxx @@ -861,7 +861,7 @@ namespace build2 // bool ambig (false); - diag_record dr; + maybe_diag_record dr; for (++i; i != rs.second; ++i) { const rule_match* r1 (&*i); @@ -2077,7 +2077,11 @@ namespace build2 case target_state::unchanged: break; case target_state::postponed: - ts = t[a].state = target_state::unchanged; + // Keep the target state postponed (see group_action() for details) + // but translate the result from postponed to unchanged, similar to + // executed_state_impl(). + // + ts = target_state::unchanged; break; case target_state::group: ts = (*t.group)[a].state; diff --git a/libbuild2/bash/rule.cxx b/libbuild2/bash/rule.cxx index 6e96b34..be7b2ff 100644 --- a/libbuild2/bash/rule.cxx +++ b/libbuild2/bash/rule.cxx @@ -473,7 +473,7 @@ namespace build2 { if (!*md.for_install) fail << "incompatible " << t << " build" << - info << "target already built not for install"; + info << "target already updated but not for install"; } else md.for_install = true; diff --git a/libbuild2/bin/init.cxx b/libbuild2/bin/init.cxx index 610082e..f01adfb 100644 --- a/libbuild2/bin/init.cxx +++ b/libbuild2/bin/init.cxx @@ -12,7 +12,6 @@ #include <libbuild2/test/module.hxx> -#include <libbuild2/install/rule.hxx> #include <libbuild2/install/utility.hxx> #include <libbuild2/bin/rule.hxx> @@ -31,6 +30,7 @@ namespace build2 static const obj_rule obj_; static const libul_rule libul_; static const lib_rule lib_; + static const install_lib_rule install_lib_; static const def_rule def_; // Default config.bin.*.lib values. @@ -631,10 +631,8 @@ namespace build2 // if (install_loaded) { - auto& gr (install::group_rule::instance); - - r.insert<lib> (perform_install_id, "bin.lib", gr); - r.insert<lib> (perform_uninstall_id, "bin.lib", gr); + r.insert<lib> (perform_install_id, "bin.lib", install_lib_); + r.insert<lib> (perform_uninstall_id, "bin.lib", install_lib_); } if (const test::module* m = rs.find_module<test::module> ("test")) diff --git a/libbuild2/bin/rule.cxx b/libbuild2/bin/rule.cxx index c7147bf..1fea558 100644 --- a/libbuild2/bin/rule.cxx +++ b/libbuild2/bin/rule.cxx @@ -218,5 +218,18 @@ namespace build2 const target* m[] = {t.a, t.s}; return execute_members (a, t, m); } + + // install_lib_rule + // + pair<const target*, uint64_t> install_lib_rule:: + filter (const scope* is, + action a, const target& t, const prerequisite& p, + match_extra& me) const + { + if (p.is_a<lib> ()) + return pair<const target*, uint64_t> (nullptr, 0); + + return install::group_rule::filter (is, a, t, p, me); + } } } diff --git a/libbuild2/bin/rule.hxx b/libbuild2/bin/rule.hxx index 9dd1d14..74f4301 100644 --- a/libbuild2/bin/rule.hxx +++ b/libbuild2/bin/rule.hxx @@ -10,6 +10,7 @@ #include <libbuild2/rule.hxx> #include <libbuild2/dist/rule.hxx> +#include <libbuild2/install/rule.hxx> #include <libbuild2/bin/export.hxx> @@ -78,6 +79,26 @@ namespace build2 static target_state perform (action, const target&); }; + + // Install rule for lib{} group. + // + // The only difference compared to the standard install::group_rule is + // that it ignores the lib{} prerequisites, instead expecting the correct + // things to be installed via the liba{}/libs{} members. This is important + // due to the presence of match options (see lib{} target for details). + // + class LIBBUILD2_BIN_SYMEXPORT install_lib_rule: public install::group_rule + { + public: + install_lib_rule () {} + + virtual pair<const target*, uint64_t> + filter (const scope*, + action, const target&, const prerequisite&, + match_extra&) const override; + + using install::group_rule::filter; // "Unhide" to make Clang happy. + }; } } diff --git a/libbuild2/build/script/builtin.cli b/libbuild2/build/script/builtin.cli index 5aea034..9639477 100644 --- a/libbuild2/build/script/builtin.cli +++ b/libbuild2/build/script/builtin.cli @@ -103,9 +103,10 @@ namespace build2 // Dynamic target extraction options. // // This functionality is enabled with the --dyn-target option. Only - // the make format is supported, where the listed targets are added as - // ad hoc group members (unless already specified as static members). - // This functionality is not available in the byproduct mode. + // the `make` and `lines` formats are supported (see above), where the + // listed targets are added as ad hoc group members (unless already + // specified as static members). This functionality is not available + // in the byproduct mode. // string --target-what; // Target kind, e.g., "source". diff --git a/libbuild2/build/script/parser.cxx b/libbuild2/build/script/parser.cxx index 3ecf23d..c362776 100644 --- a/libbuild2/build/script/parser.cxx +++ b/libbuild2/build/script/parser.cxx @@ -106,7 +106,7 @@ namespace build2 // name from the script operation first. // { - diag_record dr; + maybe_diag_record dr; if (!diag_name_ && diag_preamble_.empty ()) { @@ -131,7 +131,7 @@ namespace build2 << "'"; } - if (!dr.empty ()) + if (dr) { dr << info << "consider specifying it explicitly with the 'diag' " << "recipe attribute"; @@ -1529,7 +1529,7 @@ namespace build2 { // Copy the tokens and start playing. // - replay_data (replay_tokens (dl.tokens)); + replay_data (dl.tokens); token t; build2::script::token_type tt; @@ -2501,10 +2501,14 @@ namespace build2 // auto fail = [this, what, &ctx] (const auto& f) -> optional<bool> { + // Note that this test will give a false negative if this target + // ends up being updated during load or match. At least it's + // conservative. + // bool df (!ctx.match_only && !ctx.dry_run_option); - diag_record dr; - dr << error << what << ' ' << f << " not found and no rule to " + diag_record dr (error); + dr << what << ' ' << f << " not found and no rule to " << "generate it"; if (df) diff --git a/libbuild2/cc/common.cxx b/libbuild2/cc/common.cxx index fcc8961..dfc78e7 100644 --- a/libbuild2/cc/common.cxx +++ b/libbuild2/cc/common.cxx @@ -10,6 +10,8 @@ #include <libbuild2/filesystem.hxx> #include <libbuild2/diagnostics.hxx> +#include <libbuild2/config/utility.hxx> // config::lookup_config() + #include <libbuild2/cc/utility.hxx> using namespace std; @@ -948,13 +950,36 @@ namespace build2 const prerequisite_key& p, bool exist) const { - tracer trace (x, "search_library"); - assert (p.scope != nullptr && (!exist || act)); + tracer trace (x, "search_library"); + context& ctx (p.scope->ctx); const scope& rs (*p.scope->root_scope ()); + // Note: since we are searching for a (presumably) installed library, + // utility libraries do not apply. + // + bool l (p.is_a<lib> ()); + const string& name (*p.tk.name); + const optional<string>& ext (l ? nullopt : p.tk.ext); // Only liba/libs. + + // Import phase 1 may pass us a path specified by the user with + // config.import.<proj>.<name>.<type>. The possible cases are: + // + // 1. Empty or relative directory for liba{} and libs{} (absolute would + // be taken care of by phase 1 since these tragets are path-based). + // + // 2. Empty, relative, or absolute directory for lib{} (since it's not a + // path-based target). + // + const dir_path& dir (*p.tk.dir); + + // Same semantics as in lookup_import() below. + // + if (!dir.empty () && dir.relative ()) + fail << "relative path in imported " << p; + // Here is the problem: we may be building for two different toolchains // simultaneously that use the same installed library. But our search is // toolchain-specific. To make sure we end up with different targets for @@ -965,18 +990,99 @@ namespace build2 ? cpath : cast<process_path> (rs["bin.ld.path"])); - // @@ This is hairy enough to warrant a separate implementation for - // Windows. - - // Note: since we are searching for a (presumably) installed library, - // utility libraries do not apply. + // If this prerequisite is project-qualified do an ad hoc check for + // config.import.<proj>.<name>.{liba,libs} which can be used to specify + // different path (see import_search() for background). Note that for + // importing liba{}/libs{} directly this is handled by the standard + // import machinery. // - bool l (p.is_a<lib> ()); - const optional<string>& ext (l ? nullopt : p.tk.ext); // Only liba/libs. + // Note that we also support simple names, which are then searched in + // the standard directories. The standard import machinery also does the + // right thing by delegating the resolution of relative paths to phase + // 2. + // + // Note that we can only do this if in the load phase since we need to + // enter variables and mark them as saved (via lookup_config() call), + // which means this will only work for immediate import with the rule + // hint. And doing this, strictly speaking, is racy (there could be both + // delayed and immediate imports and the delayed could get handled + // first, for example if the immediate import is handled in the + // interrupting load phase). + // + // @@ Perhaps in the future we can try to carefully switch the phase? + // Note, however, that in the existing mode I believe we may end up + // being calling from the execute phase but in this mode we can probably + // assume import has already been done and can just lookup the variable + // and value in the read-only mode. See GH issue #449. + // + auto lookup_import = [&rs, + &act, + namev = + (p.proj + ? sanitize_identifier (name) + : string ()), + projv = + (p.proj + ? p.proj->variable () + : string ())] (const char* tt) -> optional<path> + { + if (!projv.empty ()) + { + // Note: we know tt is liba or libs and need not to be sanitized. + // + string varn ("config.import." + projv + '.' + namev + '.' + tt); + + if (config::specified_config (rs, varn, true /* exact */)) + { + if (act) + fail << varn << " can only be specified for immediate import"; + + scope& s (rs.rw ()); // Safe because in the load phase. + + // Note: qualified so go straight for the public variable pool. + // + auto& vp (s.var_pool (true /* public */)); + + const variable& var (vp.insert (move (varn))); + + bool nv (false); + auto l (config::lookup_config (nv, s, var)); + + if (l.defined ()) + { + const path* p (cast_null<path> (l)); + + if (const char* w = ( + p == nullptr ? "null" : + p->empty () ? "empty path" : + p->relative () && !p->simple () ? "relative path" : + nullptr)) + fail << w << " in " << var; + + path r (*p); + + if (r.absolute ()) + { + try + { + r.normalize (); + } + catch (const invalid_path&) + { + fail << "invalid path in " << var; + } + } + + return r; + } + } + } + + return nullopt; + }; // First figure out what we need to search for. // - const string& name (*p.tk.name); // liba // @@ -985,37 +1091,47 @@ namespace build2 if (l || p.is_a<liba> ()) { - // We are trying to find a library in the search paths extracted from - // the compiler. It would only be natural if we used the library - // prefix/extension that correspond to this compiler and/or its - // target. - // - // Unlike MinGW, VC's .lib/.dll.lib naming is by no means standard and - // we might need to search for other names. In fact, there is no - // reliable way to guess from the file name what kind of library it - // is, static or import and we will have to do deep inspection of such - // alternative names. However, if we did find .dll.lib, then we can - // assume that .lib is the static library without any deep inspection - // overhead. - // - const char* e (""); - - if (tsys == "win32-msvc") + if (optional<path> p = lookup_import ("liba")) { - an = path (name); - e = "lib"; + an = move (*p); + ae = an.extension (); } else { - an = path ("lib" + name); - e = "a"; - } + // We are trying to find a library in the search paths extracted + // from the compiler. It would only be natural if we used the + // library prefix/extension that correspond to this compiler and/or + // its target. + // + // Unlike MinGW, VC's .lib/.dll.lib naming is by no means standard + // and we might need to search for other names. In fact, there is no + // reliable way to guess from the file name what kind of library it + // is, static or import and we will have to do deep inspection of + // such alternative names. However, if we did find .dll.lib, then we + // can assume that .lib is the static library without any deep + // inspection overhead. + // + const char* e (""); - ae = ext ? ext : string (e); - if (!ae->empty ()) - { - an += '.'; - an += *ae; + an = dir; // Empty or absolute. + + if (tsys == "win32-msvc") + { + an /= path (name); + e = "lib"; + } + else + { + an /= path ("lib" + name); + e = "a"; + } + + ae = ext ? ext : string (e); + if (!ae->empty ()) + { + an += '.'; + an += *ae; + } } } @@ -1026,27 +1142,37 @@ namespace build2 if (l || p.is_a<libs> ()) { - const char* e (""); - - if (tsys == "win32-msvc") + if (optional<path> p = lookup_import ("libs")) { - sn = path (name); - e = "dll.lib"; + sn = move (*p); + se = sn.extension (); } else { - sn = path ("lib" + name); + const char* e (""); - if (tsys == "darwin") e = "dylib"; - else if (tsys == "mingw32") e = "dll.a"; // See search code below. - else e = "so"; - } + sn = dir; - se = ext ? ext : string (e); - if (!se->empty ()) - { - sn += '.'; - sn += *se; + if (tsys == "win32-msvc") + { + sn /= path (name); + e = "dll.lib"; + } + else + { + sn /= path ("lib" + name); + + if (tsys == "darwin") e = "dylib"; + else if (tsys == "mingw32") e = "dll.a"; // See search code below. + else e = "so"; + } + + se = ext ? ext : string (e); + if (!se->empty ()) + { + sn += '.'; + sn += *se; + } } } @@ -1268,10 +1394,6 @@ namespace build2 return a != nullptr || s != nullptr; }; - // First try user directories (i.e., -L or /LIBPATH). - // - bool sys (false); - if (!usrd) { usrd = extract_library_search_dirs (*p.scope); @@ -1304,13 +1426,57 @@ namespace build2 } } + bool sys (false); const dir_path* pd (nullptr); - for (const dir_path& d: *usrd) + + // First see if an absolute path was specified with import. + // + // Note: an, sn are either simple of absolute. + // + dir_path id; + if (an.absolute () || sn.absolute ()) { - if (search (d)) + if (an.absolute ()) { - pd = &d; - break; + id = an.directory (); + an.make_leaf (); + } + + if (sn.absolute ()) + { + dir_path d (sn.directory ()); + sn.make_leaf (); + + if (id.empty ()) + id = move (d); + else if (id != d) + fail << "inconsistent imported " << an << " and " << sn + << " directories" << + info << an << ": " << id << + info << sn << ": " << d; + } + + if (!search (id)) + { + fail << "imported " << (an.empty () ? sn : an) + << " does not exist in " << id; + } + + sys = find (sysd.begin (), sysd.end (), id) != sysd.end (); + pd = &id; + } + + if (pd == nullptr) + { + // Next try user directories (i.e., -L or /LIBPATH). + // + for (const dir_path& d: *usrd) + { + if (search (d)) + { + pd = &d; + break; + } } } @@ -1401,8 +1567,7 @@ namespace build2 // string d ("-DLIB"); - d += sanitize_identifier ( - ucase (const_cast<const string&> (t.name))); + d += sanitize_identifier (ucase (t.name)); d += '_'; d += suffix; diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx index cebd244..53d38ac 100644 --- a/libbuild2/cc/compile-rule.cxx +++ b/libbuild2/cc/compile-rule.cxx @@ -25,6 +25,7 @@ #include <libbuild2/cc/target.hxx> // h #include <libbuild2/cc/module.hxx> #include <libbuild2/cc/utility.hxx> +#include <libbuild2/cc/compiledb.hxx> using std::exit; using std::strlen; @@ -1181,6 +1182,11 @@ namespace build2 fsdir_rule::perform_update_direct (a, *dir); } + // Use the subset of the depdb checks to detect changes to the + // compilation database entry. + // + bool compiledb_changed (false); + // Note: the leading '@' is reserved for the module map prefix (see // extract_modules()) and no other line must start with it. // @@ -1198,8 +1204,14 @@ namespace build2 // but only in what it targets, then the checksum will still change. // if (dd.expect (cast<string> (rs[x_checksum])) != nullptr) + { l4 ([&]{trace << "compiler mismatch forcing update of " << t;}); + // The checksum includes the absolute compiler path. + // + compiledb_changed = true; + } + // Then the compiler environment checksum. // if (dd.expect (env_checksum) != nullptr) @@ -1263,7 +1275,17 @@ namespace build2 append_sys_hdr_options (cs); // Extra system header dirs (last). if (dd.expect (cs.string ()) != nullptr) + { l4 ([&]{trace << "options mismatch forcing update of " << t;}); + + // Note that this doesn't include any of the "plumbing" options + // like -x, -c, -o, etc. In the unlikely event that there are + // changes in this area that also affect the semantics of the + // compilation database (options reordering doesn't, for example), + // then we can resort to incrementing the rule version. + // + compiledb_changed = true; + } } // Finally the source file. @@ -1273,7 +1295,10 @@ namespace build2 assert (!p.empty ()); // Sanity check. if (dd.expect (p) != nullptr) + { l4 ([&]{trace << "source file mismatch forcing update of " << t;}); + compiledb_changed = true; + } } // If any of the above checks resulted in a mismatch (different @@ -1296,6 +1321,14 @@ namespace build2 u = dd.mtime > mt; } + // Confirm the entry in the compilation database, if any. + // + if (compiledb::match (bs, t, tp, src, compiledb_changed) && !u) + { + l4 ([&]{trace << "compilation database forcing update of " << t;}); + u = true; + } + // If updating for any of the above reasons, treat it as if doesn't // exist. // @@ -1511,8 +1544,20 @@ namespace build2 // to keep re-validating the file on every subsequent dry-run as well // on the real run). // - if (u && dd.reading () && !ctx.dry_run_option) - dd.touch = timestamp_unknown; + if (u && dd.reading ()) + { + // What will happen if dry_run_option is true but we still end up + // performing a non-dry-run update due to update during match or + // load? In this case the target will become up-to-date and we will + // keep re-validating the cache until the depdb will get touched due + // to other reasons, which would be bad. So it feels like the least + // bad option is to keep re-touching the database on dry-run. + // +#if 0 + if (!ctx.dry_run_option) +#endif + dd.touch = timestamp_unknown; + } dd.close (false /* mtime_check */); md.dd = move (dd.path); @@ -2305,9 +2350,9 @@ namespace build2 if (verb > 2) { - diag_record dr; - dr << error << "header " << f << " not found and no " - << "rule to generate it"; + diag_record dr (error); + dr << "header " << f << " not found and no rule to " + << "generate it"; if (verb < 4) dr << info << "re-run with --verbose=4 for more information"; @@ -2880,8 +2925,8 @@ namespace build2 if (verb > 2) { - diag_record dr; - dr << error << "header '" << f << "' not found"; + diag_record dr (error); + dr << "header '" << f << "' not found"; if (verb < 4) dr << info << "re-run with --verbose=4 for more information"; @@ -3148,7 +3193,7 @@ namespace build2 } hk.file = move (fp); - hk.hash = hash<path> () (hk.file); + hk.hash = hash<string> () (hk.file.string ()); slock l (hc.header_map_mutex); auto i (hc.header_map.find (hk)); @@ -3201,7 +3246,7 @@ namespace build2 // path has changed (header has been remapped). // if (!e || r.second) - hk.hash = hash<path> () (hk.file); + hk.hash = hash<string> () (hk.file.string ()); const file* f; { @@ -3214,6 +3259,8 @@ namespace build2 { //cache_cls.fetch_add (1, memory_order_relaxed); + // @@ TMP cleanup. + // #if 0 assert (r.first == f); #else @@ -4007,11 +4054,14 @@ namespace build2 // auto fail = [&ctx] (const auto& h) -> optional<bool> { + // Note that this test will give a false negative if this target + // ends up being updated during load or match. At least it's + // conservative. + // bool df (!ctx.match_only && !ctx.dry_run_option); - diag_record dr; - dr << error << "header " << h << " not found and no rule to " - << "generate it"; + diag_record dr (error); + dr << "header " << h << " not found and no rule to generate it"; if (df) dr << info << "failure deferred to compiler diagnostics"; @@ -4069,7 +4119,6 @@ namespace build2 this] (path hp, path bp, timestamp mt) -> optional<bool> { context& ctx (t.ctx); - bool df (!ctx.match_only && !ctx.dry_run_option); const file* ht ( enter_header (a, bs, t, li, @@ -4078,9 +4127,14 @@ namespace build2 if (ht == nullptr) // hp is still valid. { - diag_record dr; - dr << error << "header " << hp << " not found and no rule to " - << "generate it"; + // Note that this test will give a false negative if this target + // ends up being updated during load or match. At least it's + // conservative. + // + bool df (!ctx.match_only && !ctx.dry_run_option); + + diag_record dr (error); + dr << "header " << hp << " not found and no rule to generate it"; if (df) dr << info << "failure deferred to compiler diagnostics"; @@ -4929,7 +4983,7 @@ namespace build2 if (pr.wait ()) { { - diag_record dr; + maybe_diag_record dr; if (bad_error) dr << fail << "expected error exit status from " @@ -5037,7 +5091,7 @@ namespace build2 // preprocessed source files). // { - diag_record dr; + maybe_diag_record dr; if (force_gen_skip && *force_gen_skip == skip_count) { dr << @@ -5045,11 +5099,11 @@ namespace build2 info << "run the following two commands to investigate"; dr << info; - print_process (dr, args.data ()); // No pipes. + print_process (*dr, args.data ()); // No pipes. init_args ((gen = true)); dr << info << ""; - print_process (dr, args.data ()); // No pipes. + print_process (*dr, args.data ()); // No pipes. } if (dbuf.is_open ()) @@ -7352,8 +7406,11 @@ namespace build2 // apply()). For named modules there may be no obj*{} if this is a // sidebuild (obj*{} is already in the library binary). // - path relm; + const path* abso (nullptr); + const path* absm (nullptr); path relo; + path relm; + switch (ut) { case unit_type::module_header: @@ -7363,12 +7420,18 @@ namespace build2 case unit_type::module_impl_part: { if (const file* o = find_adhoc_member<file> (t, tts.obj)) - relo = relative (o->path ()); + { + abso = &o->path (); + relo = relative (*abso); + } break; } default: - relo = relative (tp); + { + abso = &tp; + relo = relative (tp); + } } // Build the command line. @@ -7398,6 +7461,9 @@ namespace build2 small_vector<string, 2> header_args; // Header unit options storage. small_vector<string, 2> module_args; // Module options storage. + // NOTE: see a note in apply() on the compilation database implications + // if changing anything below. + // switch (cclass) { case compiler_class::msvc: @@ -7532,6 +7598,7 @@ namespace build2 { assert (ut != unit_type::module_header); // @@ MODHDR + absm = &tp; relm = relative (tp); args.push_back ("/ifcOutput"); @@ -7745,6 +7812,9 @@ namespace build2 // Output module file is specified in the mapping file, the // same as input. // + // We set neither relm nor absm since they are not on the + // command line. + // if (ut == unit_type::module_header) // No obj, -c implied. break; @@ -7773,6 +7843,7 @@ namespace build2 { assert (ut != unit_type::module_header); // @@ MODHDR + absm = &tp; relm = relative (tp); // Without this option Clang's .pcm will reference source @@ -7884,6 +7955,15 @@ namespace build2 else if (verb == 2) print_process (args); + // Insert or update the entry in the compilation database, if any. + // + compiledb::execute ( + bs, + t, tp, s, *sp, + cpath, args, + relo, abso != nullptr ? *abso : empty_path, + relm, absm != nullptr ? *absm : empty_path); + // If we have the (partially) preprocessed output, switch to that. // // But we remember the original source/position to restore later. diff --git a/libbuild2/cc/compiledb.cxx b/libbuild2/cc/compiledb.cxx new file mode 100644 index 0000000..ccf08a9 --- /dev/null +++ b/libbuild2/cc/compiledb.cxx @@ -0,0 +1,1100 @@ +// file : libbuild2/cc/compiledb.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <libbuild2/cc/compiledb.hxx> + +#include <cstring> // strlen() +#include <iostream> // cout + +#ifndef BUILD2_BOOTSTRAP +# include <libbutl/json/parser.hxx> +#endif + +#include <libbuild2/filesystem.hxx> +#include <libbuild2/diagnostics.hxx> + +#include <libbuild2/cc/module.hxx> + +#include <libbuild2/cc/target.hxx> +#include <libbuild2/bin/target.hxx> + +using namespace std; + +namespace build2 +{ + namespace cc + { + compiledb_set compiledbs; + + // compiledb + // + compiledb:: + ~compiledb () + { + } + + // Return true if this entry should be written to the database with the + // specified name. + // + static bool + filter (const scope& rs, + const core_module& m, + const string& name, + const file& ot, const file& it) + { + tracer trace ("cc::compiledb_filter"); + + bool r (true); + const char* w (nullptr); // Why r is false. + + // First check if writing to this database is enabled. + // + // No filter means not enabled. + // + if (m.cdb_filter_ == nullptr) + { + r = false; + w = "no database name filter"; + } + else + { + // Iterate in reverse (so that later values override earlier) and take + // the first name match. + // + r = false; + for (const pair<optional<string>, bool>& p: + reverse_iterate (*m.cdb_filter_)) + { + if (!p.first || *p.first == name) + { + r = p.second; + break; + } + } + + if (!r) + w = "no match in database name filter"; + } + + // Verify the name is known in this amalgamation. Note that without + // this check we may end up writing to unrelated databases in other + // amalgamations (think linked configurations). + // + if (r) + { + r = false; + for (const core_module* pm (&m); + pm != nullptr; + pm = pm->outer_module_) + { + const strings& ns (pm->cdb_names_); + + if (find (ns.begin (), ns.end (), name) != ns.end ()) + { + r = true; + break; + } + } + + if (!r) + w = "database name unknown in amalgamation"; + } + + // Filter based on the output target. + // + // If there is no filter specified, then accept all targets. + // + if (r && m.cdb_filter_output_ != nullptr) + { + // If the filter is empty, then there is no match. + // + if (m.cdb_filter_output_->empty ()) + { + r = false; + w = "empty output target type filter"; + } + else + { + const target_type& ott (ot.type ()); + + // Iterate in reverse (so that later values override earlier) and + // take the first name match. + // + r = false; + for (const pair<optional<string>, string>& p: + reverse_iterate (*m.cdb_filter_output_)) + { + if (p.first && *p.first != name) + continue; + + using namespace bin; + + const string& n (p.second); + + if (ott.name == n || n == "target") + { + r = true; + } + // + // Handle obj/bmi/hbmi{} groups ad hoc. + // + else if (n == "obj") + { + r = ott.is_a<obje> () || ott.is_a<objs> () || ott.is_a<obja> (); + } + else if (n == "bmi") + { + r = ott.is_a<bmie> () || ott.is_a<bmis> () || ott.is_a<bmia> (); + } + else if (n == "hbmi") + { + r = ott.is_a<hbmie> () || ott.is_a<hbmis> () || ott.is_a<hbmia> (); + } + else + { + // Handle the commonly-used, well-known targets directly (see + // note in core_config_init() for why we cannot pre-lookup + // them). + // + const target_type* tt ( + n == "obje" ? &obje::static_type : + n == "objs" ? &objs::static_type : + n == "obja" ? &obja::static_type : + n == "bmie" ? &bmie::static_type : + n == "bmis" ? &bmis::static_type : + n == "bmia" ? &bmia::static_type : + n == "hbmie" ? &hbmie::static_type : + n == "hbmis" ? &hbmis::static_type : + n == "hbmia" ? &hbmia::static_type : + rs.find_target_type (n)); + + if (tt == nullptr) + fail << "unknown target type '" << n << "' in " + << "config.cc.compiledb.filter.output value"; + + r = ott.is_a (*tt); + } + + if (r) + break; + } + + if (!r) + w = "no match in output target type filter"; + } + } + + // Filter based on the input target. + // + // If there is no filter specified, then accept all targets. + // + if (r && m.cdb_filter_input_ != nullptr) + { + // If the filter is empty, then there is no match. + // + if (m.cdb_filter_input_->empty ()) + { + r = false; + w = "empty input target type filter"; + } + else + { + const target_type& itt (it.type ()); + + // Iterate in reverse (so that later values override earlier) and + // take the first name match. + // + r = false; + for (const pair<optional<string>, string>& p: + reverse_iterate (*m.cdb_filter_input_)) + { + if (p.first && *p.first != name) + continue; + + const string& n (p.second); + + if (itt.name == n || n == "target") + r = true; + else + { + // The same optimization as above. Note: cxx{}, etc., are in the + // cxx module so we have to look them up. + // + const target_type* tt ( + n == "c" ? &c::static_type : + n == "m" ? &m::static_type : + n == "S" ? &m::static_type : + rs.find_target_type (n)); + + if (tt == nullptr) + fail << "unknown target type '" << n << "' in " + << "config.cc.compiledb.filter.input value"; + + r = itt.is_a (*tt); + } + + if (r) + break; + } + + if (!r) + w = "no match in input target type filter"; + } + } + + l6 ([&] + { + if (r) + trace << "keep " << ot << " in " << name; + else + trace << "omit " << ot << " from " << name << ": " << w; + }); + + return r; + } + + bool compiledb:: + match (const scope& bs, + const file& ot, const path_type& op, + const file& it, + bool changed) + { + if (compiledbs.empty ()) + return false; + + const scope& rs (*bs.root_scope ()); + const auto* m (rs.find_module<core_module> (core_module::name)); + + assert (m != nullptr); + + bool u (false); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + if (filter (rs, *m, db->name, ot, it)) + u = db->match (ot, op, changed) || u; + } + + return u; + } + + void compiledb:: + execute (const scope& bs, + const file& ot, const path_type& op, + const file& it, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + if (compiledbs.empty ()) + return; + + const scope& rs (*bs.root_scope ()); + const auto* m (rs.find_module<core_module> (core_module::name)); + + assert (m != nullptr); + + assert (relo.empty () == abso.empty () && + relm.empty () == absm.empty ()); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + if (filter (rs, *m, db->name, ot, it)) + db->execute (ot, op, it, ip, cpath, args, relo, abso, relm, absm); + } + } + + void + compiledb_pre (context& ctx, action a, const action_targets&) + { + // Note: won't be registered if compiledbs is empty. + + // Note: may be called directly with empty action_targets. + + assert (a.inner_action () == perform_update_id); + + tracer trace ("cc::compiledb_pre"); + + bool mctx (ctx.module_context == &ctx); + + l6 ([&]{trace << (mctx ? "module" : "normal") << " context " << &ctx;}); + + for (const unique_ptr<compiledb>& db: compiledbs) + db->pre (ctx); + } + + void + compiledb_post (context& ctx, + action a, + const action_targets& ts, + bool failed) + { + // Note: won't be registered if compiledbs is empty. + + assert (a.inner_action () == perform_update_id); + + tracer trace ("cc::compiledb_post"); + + bool mctx (ctx.module_context == &ctx); + + l6 ([&]{trace << (mctx ? "module" : "normal") << " context " << &ctx + << ", failed: " << failed;}); + + for (const unique_ptr<compiledb>& db: compiledbs) + db->post (ctx, ts, failed); + } + +#ifndef BUILD2_BOOTSTRAP + + namespace json = butl::json; + + // compiledb_stdout + // + compiledb_stdout:: + compiledb_stdout (string n) + : compiledb (move (n), path_type ()), + state_ (state::init), + nesting_ (0), + js_ (cout, 0 /* indentation */, "" /* multi_value_separator */) + { + } + + void compiledb_stdout:: + pre (context&) + { + // If the previous operation batch failed, then we shouldn't be here. + // + assert (state_ != state::failed); + + // The module context (used to build build system modules) poses a + // problem: we can receive its callbacks before the main context's or + // nested in the pre/post calls of the main context (or both, in + // fact). Plus there may be multiple pre/post sequences corresponding to + // the module context of both kinds. The three distinct cases are: + // + // 1. Module is loaded as part of the initial buildfile load (e.g., from + // root.build) -- in this case we will observe module pre/post before + // the main context's pre/post. + // + // In fact, to be precise, we will only observe them if cc is loaded + // before such a module. + // + // 2. Module is loaded via the interrupting load (e.g., from a directory + // buildfile that is loaded implicitly during match) -- in this case + // we will observe pre/post calls nested into the main context's + // pre/post. + // + // 3. The module context is used to build an ad hoc C++ recipe -- in + // this case we also get nested calls like in (2) since this happens + // during the recipe's match(). + // + // One thing to keep in mind (and which we rely upon quite a bit below) + // is that the main context's post will always be last (within any given + // operation; there could be another for the subsequent operation in a + // batch). + // + // Handling the nested case is relatively straightforward: we can keep + // track and ignore all the nested calls. + // + // The before case is where things get complicated. We could "take" the + // first module pre call and then wait until the main post, unless we + // see a module post call with failed=true, in which case there will be + // no further pre/post calls. There is, however, a nuance: the module is + // loaded and build for any operation, not just update, which means that + // if the main operation is not update (say, it's clean), we won't see + // any of the main context's pre/post calls. + // + // The way we are going to resolve this problem is different for the + // stdout and file implementations: + // + // For stdout we will just say that it should only be used with the + // update operation. There is really no good reason to use it with + // anything else anyway. See compiledb_stdout::post() for additional + // details. + // + // For file we will rely on its persistence and simply close and reopen + // the database for each pre/post sequence, the same way as if they were + // separate operations in a batch. + // + if (nesting_++ != 0) // Nested pre() call. + return; + + if (state_ == state::init) // First pre() call. + { + state_ = state::empty; + cout << "[\n"; + } + } + + bool compiledb_stdout:: + match (const file&, const path_type&, bool) + { + return true; + } + + static inline const char* + rel_to_abs (const char* a, + const string& rs, const string& as, + string& buf) + { + if (size_t rn = rs.size ()) + { + size_t an (strlen (a)); + + if (an >= rn && rs.compare (0, rn, a, rn) == 0) + { + if (an == rn) + return as.c_str (); + + buf = as; + buf.append (a + rn, an - rn); + + return buf.c_str (); + } + } + + return nullptr; + } + + void compiledb_stdout:: + execute (const file&, const path_type& op, + const file&, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + const string& ro (relo.string ()); + const string& ao (abso.string ()); + + const string& rm (relm.string ()); + const string& am (absm.string ()); + + mlock l (mutex_); + + switch (state_) + { + case state::full: + { + cout << ",\n"; + break; + } + case state::empty: + { + state_ = state::full; + break; + } + case state::failed: + return; + case state::init: + assert (false); + return; + } + + try + { + // Duplicate what we have in the file implementation (instead of + // factoring it out to something common) in case here we need to + // adjust things (change order, omit some values; for example to + // accommodate broken consumers). We have this freedom here but not + // there. + // + js_.begin_object (); + { + js_.member ("output", op.string ()); + js_.member ("file", ip.string ()); + + js_.member_begin_array ("arguments"); + { + string buf; // Reuse. + for (auto b (args.begin ()), i (b), e (args.end ()); + i != e && *i != nullptr; + ++i) + { + const char* r; + + if (i == b) + r = cpath.effect_string (); + else + { + // Untranslate relative paths back to absolute. + // + const char* a (*i); + + if ((r = rel_to_abs (a, ro, ao, buf)) == nullptr && + (r = rel_to_abs (a, rm, am, buf)) == nullptr) + r = a; + } + + js_.value (r); + } + } + js_.end_array (); + + js_.member ("directory", work.string ()); + } + js_.end_object (); + } + catch (const json::invalid_json_output& e) + { + // There is no way (nor reason; the output will most likely be invalid + // anyway) to reuse the failed json serializer so make sure we ignore + // all the subsequent callbacks. + // + state_ = state::failed; + + l.unlock (); + + fail << "invalid compilation database json output: " << e; + } + } + + void compiledb_stdout:: + post (context& ctx, const action_targets&, bool failed) + { + assert (nesting_ != 0); + if (--nesting_ != 0) // Nested post() call. + return; + + bool mctx (ctx.module_context == &ctx); + + switch (state_) + { + case state::empty: + case state::full: + { + // If this is a module context's post, wait for the main context's + // post (last) unless the module load failed (in which case there + // will be no main pre/post). + // + // Note that there is no easy way to diagnose the case where we + // won't get the main pre/post calls. Instead, we will just produce + // invalid JSON (array won't be closed). In a somewhat hackish way, + // this actually makes the `b [-n] clean update` sequence work: we + // will take the pre() call from clean and the main post() from + // update. + // + if (mctx && !failed) + return; + + if (state_ == state::full) + cout << '\n'; + + cout << "]\n"; + break; + } + case state::failed: + return; + case state::init: + assert (false); + } + + state_ = state::init; + } + + // compiledb_file + // + compiledb_file:: + compiledb_file (string n, path_type p) + : compiledb (move (n), move (p)), + state_ (state::closed), + nesting_ (0) + { + } + + void compiledb_file:: + pre (context&) + { + // If the previous operation batch failed, then we shouldn't be here. + // + assert (state_ != state::failed); + + // See compiledb_stdout::pre() for background on dealing with the module + // context. Here are some file-specific nuances: + // + // We are going to load the database on the first pre call and flush + // (but not close) it on the matching post. Flushing means that we will + // update the file but still keep the in-memory state, in case there is + // another pre/post session coming. This is both a performance + // optimization but also the way we handle prunning no longer present + // entries, which gets tricky across multiple pre/post sessions (see + // post() for details). + // + if (nesting_++ != 0) // Nested pre() call. + return; + + if (state_ == state::closed) // First pre() call. + { + // Load the contents of the file if it exists, marking all the entries + // as (presumed) absent. + // + if (exists (path)) + { + uint64_t line (1); + try + { + ifdstream ifs (path, ifdstream::badbit); + + // Parse the top-level array manually (see post() for the expected + // format). + // + auto throw_invalid_input = [] (const string& d) + { + throw json::invalid_json_input ("", 0, 1, 0, d); + }; + + enum {first, second, next, last, end} s (first); + + for (string l; !eof (getline (ifs, l)); line++) + { + switch (s) + { + case first: + { + if (l != "[") + throw_invalid_input ("beginning of array expected"); + + s = second; + continue; + } + case second: + { + if (l == "]") + { + s = end; + continue; + } + + s = next; + } + // Fall through. + case next: + { + if (!l.empty () && l.back () == ',') + l.pop_back (); + else + s = last; + + break; + } + case last: + { + if (l != "]") + throw_invalid_input ("end of array expected"); + + s = end; + continue; + } + case end: + { + throw_invalid_input ("junk after end of array"); + } + } + + // Parse just the output target path, which must come first. + // + json::parser jp (l, "" /* name */); + + jp.next_expect (json::event::begin_object); + string op (move (jp.next_expect_member_string ("output"))); + + auto r (db_.emplace (move (op), entry {entry_status::absent, l})); + if (!r.second) + throw_invalid_input ( + "duplicate output value '" + r.first->first + '\''); + } + + if (s != end) + throw_invalid_input ("corrupt input text"); + } + catch (const json::invalid_json_input& e) + { + state_ = state::failed; + + location l (path, line, e.column); + fail (l) << "invalid compilation database json input: " << e << + info << "remove this file if it was produced by a different tool"; + } + catch (const io_error& e) + { + state_ = state::failed; + fail << "unable to read " << path << ": " << e; + } + } + + absent_ = db_.size (); + changed_ = false; + + state_ = state::open; + } + } + + bool compiledb_file:: + match (const file&, const path_type& op, bool changed) + { + mlock l (mutex_); + + switch (state_) + { + case state::open: + break; + case state::failed: + return false; + case state::closed: + assert (false); + return false; + } + + // Mark an existing entry as present or changed. And if one does not + // exist, then (for now) as missing. + // + auto i (db_.find (op.string ())); + + if (i != db_.end ()) + { + entry& e (i->second); + + // Note: we can end up with present entries via the module context + // (see post() below). And we can see changed entries in a subsequent + // nested module context. + // + switch (e.status) + { + case entry_status::present: + case entry_status::changed: + assert (!changed); + break; + case entry_status::absent: + { + e.status = changed ? entry_status::changed : entry_status::present; + + absent_--; + changed_ = changed_ || (e.status == entry_status::changed); + break; + } + case entry_status::missing: + assert (false); + } + + return false; + } + else + { + db_.emplace (op.string (), entry {entry_status::missing, string ()}); + + changed_ = true; + + return true; + } + } + + void compiledb_file:: + execute (const file&, const path_type& op, + const file&, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + const string& ro (relo.string ()); + const string& ao (abso.string ()); + + const string& rm (relm.string ()); + const string& am (absm.string ()); + + mlock l (mutex_); + + switch (state_) + { + case state::open: + break; + case state::failed: + return; + case state::closed: + assert (false); + return; + } + + auto i (db_.find (op.string ())); + + // We should have had the match() call before execute(). + // + assert (i != db_.end () && i->second.status != entry_status::absent); + + entry& e (i->second); + + if (e.status == entry_status::present) // Present and unchanged. + return; + + // The entry is either missing or changed. + // + try + { + e.json.clear (); + json::buffer_serializer js (e.json, 0 /* indentation */); + + js.begin_object (); + { + js.member ("output", op.string ()); // Note: must come first. + js.member ("file", ip.string ()); + + js.member_begin_array ("arguments"); + { + string buf; // Reuse. + for (auto b (args.begin ()), i (b), e (args.end ()); + i != e && *i != nullptr; + ++i) + { + const char* r; + + if (i == b) + r = cpath.effect_string (); + else + { + // Untranslate relative paths back to absolute. + // + const char* a (*i); + + if ((r = rel_to_abs (a, ro, ao, buf)) == nullptr && + (r = rel_to_abs (a, rm, am, buf)) == nullptr) + r = a; + } + + js.value (r); + } + } + js.end_array (); + + js.member ("directory", work.string ()); + } + js.end_object (); + } + catch (const json::invalid_json_output& e) + { + // There is no way (nor reason; the output will most likely be invalid + // anyway) to reuse the failed json serializer so make sure we ignore + // all the subsequent callbacks. + // + state_ = state::failed; + + l.unlock (); + + fail << "invalid compilation database json output: " << e; + } + + e.status = entry_status::changed; + } + + void compiledb_file:: + post (context& ctx, const action_targets& ts, bool failed) + { + assert (nesting_ != 0); + if (--nesting_ != 0) // Nested post() call. + return; + + switch (state_) + { + case state::open: + break; + case state::failed: + return; + case state::closed: + assert (false); + return; + } + + bool mctx (ctx.module_context == &ctx); + + tracer trace ("cc::compiledb_file::post"); + + // See if we need to update the file. + // + if (changed_) + l6 ([&]{trace << "updating due to missing/changed entries: " << path;}); + + // Don't prune the stale entries if the operation failed since we may + // not have gotten to execute some of them. + // + // And if this is a module context's post, then also don't prune the + // stale entries, instead waiting for the main context's post (if there + // will be one; this means we will only prune on update). + // + // Actually, this pruning business is even trickier than that: if we + // are not updating the entire project (say, rather only a subdirectory + // or even a specific target), then we will naturally not get any + // match/execute calls for targets of this project that don't get pulled + // into this build. Which means that we cannot just prune entries that + // we did not match/execute. It feels the correct semantics is to only + // prune the entries if they are in a subdirectory of the dir{} targets + // which we are building. + // + // What do we do about the module context, where we always update a + // specific libs{}? We could use its directory instead but that may lead + // to undesirable results. For example, if there are unit tests in the + // same directory, we will end up dropping their entries. It feels like + // the correct approach is to just ignore module context's entries + // entirely. If someone wants to prune the compilation database of a + // module, they will just need to update it directly (i.e., via the main + // context). Note that we cannot apply the same "simplification" to the + // changed entries since we will only observe the change once. + // + bool absent (false); + + if (!failed && !mctx && absent_ != 0) + { + // Pre-scan the entries and drop the appropriate absent ones. + // + for (auto i (db_.begin ()); i != db_.end (); ) + { + const entry& e (i->second); + + if (e.status == entry_status::absent) + { + // Absent entries should be rare enough during the normal + // development that we don't need to bother with caching the + // directories. + // + bool a (false); + for (const action_target& at: ts) + { + const target& t (at.as<target> ()); + if (t.is_a<dir> ()) + { + const string& p (i->first); + const string& d (t.out_dir ().string ()); + + if (path_traits::sub (p.c_str (), p.size (), + d.c_str (), d.size ())) + { + // Remove this entry from the in-memory state so that it + // matches the file state. + // + i = db_.erase (i); + --absent_; + a = absent = true; + break; + } + } + } + + if (a) + continue; + } + + ++i; + } + } + + if (absent) + l6 ([&]{trace << "updating due to absent entries: " << path;}); + + try + { + auto_rmfile rm; + ofdstream ofs; + + bool u (changed_ || absent); // Update the file. + + if (u) + { + rm = auto_rmfile (path); + ofs.open (path); + + // We parse the top-level array manually (see pre() above) and the + // expected format is as follows: + // + // [ + // {"output":...}, + // ... + // {"output":...} + // ] + // + ofs.write ("[\n", 2); + } + + // Iterate over the entries resetting their status and writing them to + // the file if necessary. + // + bool first (true); + for (auto& p: db_) + { + entry& e (p.second); + + // First sort out the status also skipping appropriate entries. + // + switch (e.status) + { + case entry_status::absent: + { + // This is an absent entry that we should keep (see pre-scan + // above). + // + break; + } + case entry_status::missing: + { + // This should only happen if this operation has failed (see + // also below) or we are in the match-only mode. + // + assert (failed || ctx.match_only); + continue; + } + case entry_status::present: + case entry_status::changed: + { + // This is tricky: if this is a module context, then we don't + // want to mark the entries as absent since they will then get + // dropped by the main operation context. + // + if (mctx) + e.status = entry_status::present; + else + { + // Note: this is necessary for things to work across multiple + // operations in a batch. + // + e.status = entry_status::absent; + absent_++; + } + } + } + + if (u) + { + if (first) + first = false; + else + ofs.write (",\n", 2); + + ofs.write (e.json.c_str (), e.json.size ()); + } + } + + if (u) + { + ofs.write (first ? "]\n" : "\n]\n", first ? 2 : 3); + + ofs.close (); + rm.cancel (); + } + } + catch (const io_error& e) + { + state_ = state::failed; + fail << "unable to write to " << path << ": " << e; + } + + // If this operation has failed, then our state may not be accurate + // (e.g., entries with missing status) but we also don't expect any + // further pre calls. Let's change out state to failed as a sanity + // check. + // + if (failed) + state_ = state::failed; + else + changed_ = false; + + // Note: keep in the open state (see pre() for details). + } + +#endif // BUILD2_BOOTSTRAP + } +} diff --git a/libbuild2/cc/compiledb.hxx b/libbuild2/cc/compiledb.hxx new file mode 100644 index 0000000..8288cf5 --- /dev/null +++ b/libbuild2/cc/compiledb.hxx @@ -0,0 +1,236 @@ +// file : libbuild2/cc/compiledb.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_CC_COMPILEDB_HXX +#define LIBBUILD2_CC_COMPILEDB_HXX + +#include <unordered_map> + +#ifndef BUILD2_BOOTSTRAP +# include <libbutl/json/serializer.hxx> +#endif + +#include <libbuild2/types.hxx> +#include <libbuild2/utility.hxx> + +#include <libbuild2/target.hxx> +#include <libbuild2/action.hxx> +#include <libbuild2/context.hxx> + +namespace build2 +{ + namespace cc + { + using compiledb_name_filter = vector<pair<optional<string>, bool>>; + using compiledb_type_filter = vector<pair<optional<string>, string>>; + + class compiledb + { + public: + // Match callback where we confirm an entry in the database and also + // signal whether it has changes (based on change tracking in depdb). + // Return true to force compilation of this target and thus make sure + // the below execute() is called (unless something before that failed). + // + // Besides noticing changes, this callback is also necessary to notice + // and delete entries that should no longer be in the database (e.g., a + // source file was removed from the project). + // + // Note that output is either obj*{}, bmi*{}, of hbmi*{}. + // + static bool + match (const scope& bs, + const file& output, const path& output_path, + const file& input, + bool changed); + + // Execute callback where we insert or update an entry in the database. + // + // The {relo, abso}, and {relm, absm} pairs are used to "untranslate" + // relative paths to absolute. Specifically, any argument that has rel? + // as a prefix has this prefix replaced with the corresponding abs?. + // Note that this means we won't be able to handle old MSVC and + // clang-cl, which don't support the `/F?: <path>` form, only + // `/F?<path>`. Oh, well. Note also that either relo or relm (but not + // both) could be empty if unused. + // + // Note that we assume the source file is always absolute and is the + // last argument. + // + // Why do we want absolute paths? That's a good question. Our initial + // plan was to compare command lines in order to detect when we need to + // update the database. And if those changed with every change of CWD, + // that would be of little use. But then we realized we could do better + // by using depdb to detect changes. So now we actually don't have a + // need to get rid of the relative paths in the command line. But seeing + // that we already have it, let's keep it for now in case it makes a + // different to some broken/legacy consumers. Note also that C++ module + // name-to-BMI mapping is not untranslated (see append_module_options()). + // + static void + execute (const scope& bs, + const file& output, const path& output_path, + const file& input, const path& input_path, + const process_path& cpath, const cstrings& args, + const path& relo, const path& abso, + const path& relm, const path& absm); + + public: + using path_type = build2::path; + + string name; + path_type path; + + // The path is expected to be absolute and normalized or empty if the + // name is `-` (stdout). + // + compiledb (string n, path_type p) + : name (move (n)), path (move (p)) + { + } + + virtual void + pre (context&) = 0; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) = 0; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) = 0; + + virtual void + post (context&, const action_targets&, bool failed) = 0; + + virtual + ~compiledb (); + }; + + using compiledb_set = vector<unique_ptr<compiledb>>; + + // Populated by core_config_init() during serial load. + // + extern compiledb_set compiledbs; + + // Context operation callbacks. + // + void + compiledb_pre (context&, action, const action_targets&); + + void + compiledb_post (context&, action, const action_targets&, bool failed); + +#ifndef BUILD2_BOOTSTRAP + + // Implementation that writes to stdout. + // + // Note that this implementation forces compilation of all the targets for + // which it is called to make sure their entries are in the database. So + // typically used in the dry run mode. + // + class compiledb_stdout: public compiledb + { + public: + // The path is expected to be empty. + // + explicit + compiledb_stdout (string name); + + virtual void + pre (context&) override; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) override; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) override; + + virtual void + post (context&, const action_targets&, bool failed) override; + + private: + mutex mutex_; + enum class state {init, empty, full, failed} state_; + size_t nesting_; + butl::json::stream_serializer js_; + }; + + // Implementation that maintains a file. + // + class compiledb_file: public compiledb + { + public: + compiledb_file (string name, path_type path); + + virtual void + pre (context&) override; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) override; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) override; + + virtual void + post (context&, const action_targets&, bool failed) override; + + private: + mutex mutex_; + enum class state {closed, open, failed} state_; + size_t nesting_; + + // We want to optimize the performance for the incremental update case + // where only a few files will be recompiled and most of the time there + // will be no change in the command line, which means we won't need to + // rewrite the file. + // + // As a result, our in-memory representation is a hashmap (we could have + // thousands of entries) of absolute and normalized output file paths + // (stored as strings for lookup efficiency) to their serialized JSON + // text lines plus the status: absent, present, changed, or missing + // (entry should be there but is not). This way we don't waste + // (completely) parsing (and re-serializing) each line knowing that we + // won't need to touch most of them. + // + // In fact, we could have gone even further and used a sorted vector + // since insertions will be rare in this case. But we will need to + // lookup every entry on each update, so it's unclear this is a win. + // + enum class entry_status {absent, present, changed, missing}; + + struct entry + { + entry_status status; + string json; + }; + + using map_type = std::unordered_map<string, entry>; + map_type db_; + + // Number/presence of various entries in the database (used to determine + // whether we need to update the file without iterating over all the + // entries). + // + size_t absent_; // Number of absent entries. + bool changed_; // Presence of changed or missing entries. + }; + +#endif // BUILD2_BOOTSTRAP + } +} + +#endif // LIBBUILD2_CC_COMPILEDB_HXX diff --git a/libbuild2/cc/functions.cxx b/libbuild2/cc/functions.cxx index 9d408af..0adcf5f 100644 --- a/libbuild2/cc/functions.cxx +++ b/libbuild2/cc/functions.cxx @@ -76,7 +76,9 @@ namespace build2 for (auto i (ts_ns.begin ()); i != ts_ns.end (); ++i) { name& n (*i), o; - const target& t (to_target (*bs, move (n), move (n.pair ? *++i : o))); + const target& t (to_target (*bs, + move (n), move (n.pair ? *++i : o), + true /* in_recipe */)); if (!t.matched (a)) fail << t << " is not matched" << @@ -193,7 +195,9 @@ namespace build2 for (auto i (ts_ns.begin ()); i != ts_ns.end (); ++i) { name& n (*i), o; - const target& t (to_target (*bs, move (n), move (n.pair ? *++i : o))); + const target& t (to_target (*bs, + move (n), move (n.pair ? *++i : o), + true /* in_recipe */)); bool la (false); if (li diff --git a/libbuild2/cc/guess.cxx b/libbuild2/cc/guess.cxx index f092933..2d079f0 100644 --- a/libbuild2/cc/guess.cxx +++ b/libbuild2/cc/guess.cxx @@ -949,9 +949,15 @@ namespace build2 // there is nothing like -m32/-m64 or /MACHINE). Targeting // 64-bit seems like as good of a default as any. // - fb = ((dir_path (mi->msvc_dir) /= "bin") /= "Hostx64") /= - "x64"; - + fb = dir_path (mi->msvc_dir) /= "bin"; + +#if defined(_M_ARM64) || defined(__aarch64__) + fb /= "HostARM64"; + fb /= "ARM64"; +#else + fb /= "Hostx64"; + fb /= "x64"; +#endif search_info = info_ptr ( new msvc_info (move (*mi)), msvc_info_deleter); } @@ -1629,15 +1635,25 @@ namespace build2 // Seeing that we only do 64-bit on Windows, let's always use 64-bit // MSVC tools (link.exe, etc). In case of the Platform SDK, it's unclear - // what the CPU signifies (host, target, both). + // what the CPU signifies (host, target, both). It appears to be host. // - r = (((dir_path (mi.msvc_dir) /= "bin") /= "Hostx64") /= cpu). - representation (); + r = (((dir_path (mi.msvc_dir) /= "bin") /= +#if defined(_M_ARM64) || defined(__aarch64__) + "HostARM64" +#else + "Hostx64" +#endif + ) /= cpu).representation (); r += path::traits_type::path_separator; - r += (((dir_path (mi.psdk_dir) /= "bin") /= mi.psdk_ver) /= cpu). - representation (); + r += (((dir_path (mi.psdk_dir) /= "bin") /= mi.psdk_ver) /= +#if defined(_M_ARM64) || defined(__aarch64__) + "arm64" +#else + "x64" +#endif + ).representation (); return r; } diff --git a/libbuild2/cc/init.cxx b/libbuild2/cc/init.cxx index e124450..64c70f8 100644 --- a/libbuild2/cc/init.cxx +++ b/libbuild2/cc/init.cxx @@ -10,8 +10,10 @@ #include <libbuild2/config/utility.hxx> +#include <libbuild2/cc/module.hxx> #include <libbuild2/cc/target.hxx> #include <libbuild2/cc/utility.hxx> +#include <libbuild2/cc/compiledb.hxx> using namespace std; using namespace butl; @@ -23,7 +25,7 @@ namespace build2 // Scope operation callback that cleans up module sidebuilds. // static target_state - clean_module_sidebuilds (action, const scope& rs, const dir&) + clean_module_sidebuilds (const scope& rs) { context& ctx (rs.ctx); @@ -67,6 +69,131 @@ namespace build2 return target_state::unchanged; } + // Scope operation callback that cleans up compilation databases. + // + static target_state + clean_compiledb (const scope& rs) + { + context& ctx (rs.ctx); + + target_state r (target_state::unchanged); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + const path& p (db->path); + + if (p.empty () || + ctx.scopes.find_out (p.directory ()).root_scope () != &rs) + continue; + + if (rmfile (ctx, p)) + r = target_state::changed; + } + + return r; + } + + // Scope operation callback for cleaning module sidebuilds and compilation + // databases. + // + static target_state + clean_callback (action, const scope& rs, const dir&) + { + target_state r (clean_module_sidebuilds (rs)); + + if (!compiledbs.empty ()) + r |= clean_compiledb (rs); + + return r; + } + + // Detect if just <name> in the <name>[@<path>] form is actually <path>. + // We assume it is <path> and not <name> if it contains a directory + // component or is the special directory name (`.`/`..`) . If that's the + // case, return canonicalized name representing <path>. See the call site + // in core_config_init() below for background. + // + static optional<name> + compiledb_name_to_path (const name& n) + { + if (n.directory ()) + return n; + + if (n.file ()) + { + if (!n.dir.empty () || + path_traits::find_separator (n.value) != string::npos) + { + name r (n); + r.canonicalize (); + return r; + } + else if (n.value == "." || n.value == "..") + { + return name (dir_path (n.value)); + } + } + + return nullopt; + } + + // Custom save function that completes relative paths in the + // config.cc.compiledb and config.cc.compiledb.name values. + // + static pair<names_view, const char*> + save_compiledb_name (const scope&, + const value& v, + const value*, + names& storage) + { + const names& ns (v.as<names> ()); // Value is untyped. + + // Detect and handle the case where just <name> is actually <path>. + // + if (ns.size () == 1) + { + const name& n (ns.back ()); + + if (optional<name> otn = compiledb_name_to_path (n)) + { + name& tn (*otn); + + if (tn.dir.relative ()) + tn.dir.complete (); + + tn.dir.normalize (); + + storage.push_back (move (tn)); + return make_pair (names_view (storage), "="); + } + } + + if (find_if (ns.begin (), ns.end (), + [] (const name& n) {return n.pair;}) == ns.end ()) + { + return make_pair (names_view (ns), "="); + } + + storage = ns; + for (auto i (storage.begin ()); i != storage.end (); ++i) + { + if (i->pair) + { + name& n (*++i); + + if (!n.directory ()) + n.canonicalize (); + + if (n.dir.relative ()) + n.dir.complete (); + + n.dir.normalize (); + } + } + + return make_pair (names_view (storage), "="); + } + bool core_vars_init (scope& rs, scope&, @@ -107,6 +234,22 @@ namespace build2 vp.insert<abs_dir_path> ("config.cc.pkgconfig.sysroot"); + // Compilation database. + // + // See the manual for the semantics. + // + // config.cc.compiledb -- <name>[@<path>]|<path> (untyped) + // config.cc.compiledb.name -- <name>[@<path>]... (untyped) + // config.cc.compiledb.filter -- [<name>@]<bool>... + // config.cc.compiledb.filter.input -- [<name>@]<target-type>... + // config.cc.compiledb.filter.output -- [<name>@]<target-type>... + // + vp.insert ("config.cc.compiledb"); + vp.insert ("config.cc.compiledb.name"); + vp.insert<compiledb_name_filter> ("config.cc.compiledb.filter"); + vp.insert<compiledb_type_filter> ("config.cc.compiledb.filter.input"); + vp.insert<compiledb_type_filter> ("config.cc.compiledb.filter.output"); + vp.insert<strings> ("cc.poptions"); vp.insert<strings> ("cc.coptions"); vp.insert<strings> ("cc.loptions"); @@ -192,16 +335,6 @@ namespace build2 // vp.insert<bool> ("cc.serialize"); - // Register scope operation callback. - // - // It feels natural to clean up sidebuilds as a post operation but that - // prevents the (otherwise-empty) out root directory to be cleaned up - // (via the standard fsdir{} chain). - // - rs.operation_callbacks.emplace ( - perform_clean_id, - scope::operation_callback {&clean_module_sidebuilds, nullptr /*post*/}); - return true; } @@ -292,6 +425,8 @@ namespace build2 assert (first); + context& ctx (rs.ctx); + // Load cc.core.guess. // load_module (rs, rs, "cc.core.guess", loc); @@ -312,7 +447,6 @@ namespace build2 // // @@ Same nonsense as in module. // - // rs.assign ("cc.poptions") += cast_null<strings> ( lookup_config (rs, "config.cc.poptions", nullptr)); @@ -363,21 +497,16 @@ namespace build2 if (!cast_false<bool> (rs["bin.config.loaded"])) { // Prepare configuration hints (pretend it belongs to root scope). - // They are only used on the first load of bin.config so we only - // populate them on our first load. // variable_map h (rs); - if (first) - { - // Note that all these variables have already been registered. - // - h.assign ("config.bin.target") = - cast<target_triplet> (rs["cc.target"]).representation (); + // Note that all these variables have already been registered. + // + h.assign ("config.bin.target") = + cast<target_triplet> (rs["cc.target"]).representation (); - if (auto l = extra.hints["config.bin.pattern"]) - h.assign ("config.bin.pattern") = cast<string> (l); - } + if (auto l = extra.hints["config.bin.pattern"]) + h.assign ("config.bin.pattern") = cast<string> (l); init_module (rs, rs, "bin.config", loc, false /* optional */, h); } @@ -386,7 +515,6 @@ namespace build2 // ourselves since the target can come from the configuration and not // our hint). // - if (first) { const auto& ct (cast<target_triplet> (rs["cc.target"])); const auto& bt (cast<target_triplet> (rs["bin.target"])); @@ -416,6 +544,447 @@ namespace build2 if (tsys == "mingw32") load_module (rs, rs, "bin.rc.config", loc); + // Find the innermost outer core_module, if any. + // + const core_module* om (nullptr); + for (const scope* s (&rs); + (s = s->parent_scope ()->root_scope ()) != nullptr; ) + { + if ((om = s->find_module<core_module> (core_module::name)) != nullptr) + break; + } + + auto& m (extra.set_module (new core_module (om))); + + // config.cc.compiledb.* + // + { + // For config.cc.compiledb and config.cc.compiledb.name we only + // consider a value in this root scope (if it's inherited from the + // outer scope, then that's where it will be handled). One special + // case is when it's specified on a scope that doesn't load the cc + // module (including, ultimately, the global scope for a global + // override). We handle it by assuming the value belongs to the + // outermost amalgamation that loads the cc module. + // + // Note: cache the result. + // + auto find_outermost = + [&rs, o = optional<pair<scope*, core_module*>> ()] () mutable + { + if (!o) + { + o = pair<scope*, core_module*> (&rs, nullptr); + for (scope* s (&rs); + (s = s->parent_scope ()->root_scope ()) != nullptr; ) + { + if (auto* m = s->find_module<core_module> (core_module::name)) + { + o->first = s; + o->second = m; + } + } + } + + return *o; + }; + + auto belongs = [&rs, &find_outermost] (const lookup& l) + { + return l.belongs (rs) || find_outermost ().first == &rs; + }; + + // Add compilation databases specified in ns as <name>[@<path>] pairs, + // appending their names to cdb_names. If <path> is absent, then place + // the database into the base directory. Return the last added name. + // + auto add_cdbs = [&ctx, + &loc, + &trace] (strings& cdb_names, + const names& ns, + const dir_path& base) -> const string& + { + // Check that names and paths match. Return false if this entry + // already exist. + // + // Note that before we also checked that the same paths are not used + // across contexts. But, actually, there doesn't seem to be anything + // wrong with that and this can actually be useful, for example, + // when developing build system modules. + // + auto check = [&loc] (const string& n, const path& p) + { + for (const unique_ptr<compiledb>& db: compiledbs) + { + bool nm (db->name == n); + bool pm (db->path == p); + + if (nm != pm) + fail (loc) << "inconsistent compilation database names/paths" << + info << p << " is called " << n << + info << db->path << " is called " << db->name; + + if (nm) + return false; + } + + return true; + }; + + const string* r (&empty_string); + + bool reg (false); + size_t j (compiledbs.size ()); // First newly added database. + for (auto i (ns.begin ()); i != ns.end (); ++i) + { + // Each element has the <name>[@<path>] form. + // + // The special `-` <name> signifies stdout. + // + // If <path> is absent, then the file is called <name>.json and + // placed into the output directory of the amalgamation or project + // root scope (passed as the base argument). + // + // If <path> is (syntactically) a directory, then the file path is + // <path>/<name>.json. + // + if (!i->simple () || i->empty ()) + fail (loc) << "invalid compilation database name '" << *i << "'"; + + // Don't allow names that have (or are) directory components. + // + if (compiledb_name_to_path (*i)) + fail (loc) << "directory component in compilation database name '" + << *i << "'"; + + string n (i->value); + + path p; + if (i->pair) + { + ++i; + + if (n == "-") + fail (loc) << "compilation database path specified for stdout " + << "name"; + try + { + if (i->directory ()) + p = i->dir / n + ".json"; + else if (i->file ()) + { + if (i->dir.empty ()) + p = path (i->value); + else + p = i->dir / i->value; + } + else + throw invalid_path (""); + + if (p.relative ()) + p.complete (); + + p.normalize (); + } + catch (const invalid_path&) + { + fail (loc) << "invalid compilation database path '" << *i + << "'"; + } + } + else if (n != "-") + { + p = base / n + ".json"; + } + + if (check (n, p)) + { + reg = compiledbs.empty (); // First time. + +#ifdef BUILD2_BOOTSTRAP + fail (loc) << "compilation database requested during bootstrap"; +#else + if (n == "-") + compiledbs.push_back ( + unique_ptr<compiledb> ( + new compiledb_stdout (n))); + else + compiledbs.push_back ( + unique_ptr<compiledb> ( + new compiledb_file (n, move (p)))); +#endif + } + + // We may end up with duplicates via the config.cc.compiledb + // logic. + // + auto k (find (cdb_names.begin (), cdb_names.end (), n)); + + if (k == cdb_names.end ()) + { + cdb_names.push_back (move (n)); + r = &cdb_names.back (); + } + else + r = &*k; + } + + // Register context operation callback for compiledb generation. + // + // We have two complications here: + // + // 1. We could be performing all this from the load phase that + // interrupted the match phase, which means the point where the + // pre callback would have been called is already gone (but the + // post callback will still be called). This will happen if we, + // say, import a project that has a compilation database from a + // project that doesn't. + // + // (Note that if you think that this can be solved by simply + // always registering the callbacks, regardless of whether we + // have any databases or not, consider a slightly different + // scenario where we import a project that loads the cc module + // from a project that does not). + // + // What we are going to do in this case is simply call the pre + // callback manually. + // + // 2. We could again be performing all this from the load phase that + // interrupted the match phase, but this time the pre callback + // has already been called, which means there will be no pre() + // call for the newly added database(s). This will happen if we, + // say, import a project that has a compilation database from a + // project that also has one. + // + // Again, what we are going to do in this case is simply call the + // pre callback for the new database(s) manually. + // + if (reg) + ctx.operation_callbacks.emplace ( + perform_update_id, + context::operation_callback {&compiledb_pre, &compiledb_post}); + + if (!ctx.phase_mutex.unlocked ()) // Interrupting load. + { + action a (ctx.current_action ()); + + if (a.inner_action () == perform_update_id) + { + if (reg) // Case #1. + { + l6 ([&]{trace << "direct compiledb_pre for context " << &ctx;}); + compiledb_pre (ctx, a, action_targets {}); + } + else // Case #2. + { + size_t n (compiledbs.size ()); + + if (j != n) + { + l6 ([&]{trace << "additional compiledb for context " << &ctx;}); + + for (; j != n; ++j) + compiledbs[j]->pre (ctx); + } + } + } + } + + return *r; + }; + + lookup l; + + // config.cc.compiledb + // + // The semantics of this value is as follows: + // + // Location: outermost amalgamation that loads the cc module. + // Name filter: enable from this scope unless specified explicitly. + // Type filter: enable from this scope unless specified explicitly. + // + // Note: save omitted. + // + optional<string> enable_filter; + + l = lookup_config (rs, "config.cc.compiledb", 0, &save_compiledb_name); + if (l && belongs (l)) + { + l6 ([&]{trace << "config.cc.compiledb specified on " << rs;}); + + const names& ns (cast<names> (l)); + + // Make sure it's one name/path. + // + size_t n (ns.size ()); + if (n == 0 || n != (ns.front ().pair ? 2 : 1)) + fail (loc) << "invalid compilation database name '" << ns << "'"; + + // Detect and translate just <name> which is actually <path> to the + // <name>@<path> form: + // + // - The <name> part is the name of the directory where the database + // file will reside (typically project/repository or package + // name). + // + // - If <path> is a directory, then the database name is + // compile_commands.json. + // + names tns; + if (n == 1) + { + const name& n (ns.front ()); + + if (optional<name> otn = compiledb_name_to_path (n)) + { + name& tn (*otn); + + // Note: the add_cdbs() call below completes and normalizes the + // path but we need to do it earlier in order to be able to + // derive the name (the last component can be `.`/`..`). + // + if (tn.dir.relative ()) + tn.dir.complete (); + + tn.dir.normalize (); + + if (!exists (tn.dir)) + fail (loc) << "compilation database directory " << tn.dir + << " does not exist"; + + if (tn.value.empty ()) + tn.value = "compile_commands.json"; + + tns.push_back (name (tn.dir.leaf ().string ())); + tns.back ().pair = '@'; + tns.push_back (move (tn)); + } + } + + // We inject the database directly into the outer amalgamation's + // module, as-if config.cc.compiledb.name was specified in its + // scope. Unless there isn't one, in which case it's us. + // + pair<scope*, core_module*> p (find_outermost ()); + + // Save the name for the name filter below. + // + enable_filter = add_cdbs ( + (p.second != nullptr ? *p.second : m).cdb_names_, + tns.empty () ? ns : tns, + p.first->out_path ()); + } + + // config.cc.compiledb.name + // + // Note: save omitted. + // + l = lookup_config (rs, + "config.cc.compiledb.name", + 0, + &save_compiledb_name); + if (l && belongs (l)) + { + l6 ([&]{trace << "config.cc.compiledb.name specified on " << rs;}); + + add_cdbs (m.cdb_names_, cast<names> (l), rs.out_path ()); + } + + // config.cc.compiledb.filter + // + // Note: save omitted. + // + l = lookup_config (rs, "config.cc.compiledb.filter"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_ = &cast<compiledb_name_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_ != nullptr) + m.cdb_filter_storage_ = *om->cdb_filter_; + + m.cdb_filter_storage_.emplace_back (*enable_filter, true); + m.cdb_filter_ = &m.cdb_filter_storage_; + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_ = om->cdb_filter_; + } + + // config.cc.compiledb.filter.input + // config.cc.compiledb.filter.output + // + // Note that filtering happens before we take into account the change + // status, which means for larger projects there would be a lot of + // targets to filter even during the incremental update. So it feels + // it would have been better to pre-lookup the target types. However, + // the targets that would normally be used are registered by other + // modules (bin, c/cxx) and which haven't been loaded yet. So instead + // we try to optimize the lookup for the commonly used targets. + // + // Note: save omitted. + // + l = lookup_config (rs, "config.cc.compiledb.filter.input"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_input_ = &cast<compiledb_type_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_input_ != nullptr) + { + m.cdb_filter_input_storage_ = *om->cdb_filter_input_; + m.cdb_filter_input_storage_.emplace_back (*enable_filter, "target"); + m.cdb_filter_input_ = &m.cdb_filter_input_storage_; + } + else + m.cdb_filter_input_ = nullptr; // Enable all. + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_input_ = om->cdb_filter_input_; + } + + l = lookup_config (rs, "config.cc.compiledb.filter.output"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_output_ = &cast<compiledb_type_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_output_ != nullptr) + { + m.cdb_filter_output_storage_ = *om->cdb_filter_output_; + m.cdb_filter_output_storage_.emplace_back (*enable_filter, "target"); + m.cdb_filter_output_ = &m.cdb_filter_output_storage_; + } + else + m.cdb_filter_output_ = nullptr; // Enable all. + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_output_ = om->cdb_filter_output_; + } + } + + // Register scope operation callback for cleaning module sidebuilds and + // compilation databases. + // + // It feels natural to clean this stuff up as a post operation but that + // prevents the (otherwise-empty) out root directory to be cleaned up + // (via the standard fsdir{} chain). + // + rs.operation_callbacks.emplace ( + perform_clean_id, + scope::operation_callback {&clean_callback, nullptr /*post*/}); + return true; } diff --git a/libbuild2/cc/install-rule.cxx b/libbuild2/cc/install-rule.cxx index 6758e03..c4f924a 100644 --- a/libbuild2/cc/install-rule.cxx +++ b/libbuild2/cc/install-rule.cxx @@ -76,86 +76,6 @@ namespace build2 otype ot (link_type (t).type); - // @@ TMP: drop eventually. - // -#if 0 - // If this is a shared library prerequisite, install it as long as it is - // in the installation scope. - // - // Less obvious: we also want to install a static library prerequisite - // of a library (since it could be referenced from its .pc file, etc). - // - // Note: for now we assume these prerequisites never come from see- - // through groups. - // - // Note: we install ad hoc prerequisites by default. - // - - // Note: at least one must be true since we only register this rule for - // exe{}, and lib[as]{} (this makes sure the following if-condition will - // always be true for libx{}). - // - bool st (t.is_a<exe> () || t.is_a<libs> ()); // Target needs shared. - bool at (t.is_a<liba> () || t.is_a<libs> ()); // Target needs static. - assert (st || at); - - if ((st && (p.is_a<libx> () || p.is_a<libs> ())) || - (at && (p.is_a<libx> () || p.is_a<liba> ()))) - { - const target* pt (&search (t, p)); - - // If this is the lib{}/libu*{} group, pick a member which we would - // link. For libu*{} we want the "see through" logic. - // - if (const libx* l = pt->is_a<libx> ()) - pt = link_member (*l, a, link_info (t.base_scope (), ot)); - - // Note: not redundant since we could be returning a member. - // - if ((st && pt->is_a<libs> ()) || (at && pt->is_a<liba> ())) - { - // Adjust match options. - // - if (a.operation () != update_id) - { - if (t.is_a<exe> ()) - options = lib::option_install_runtime; - else - { - // This is a library prerequisite of a library target and - // runtime-only begets runtime-only. - // - if (me.cur_options == lib::option_install_runtime) - options = lib::option_install_runtime; - } - } - - return make_pair (is == nullptr || pt->in (*is) ? pt : nullptr, - options); - } - - // See through to libu*{} members. Note that we are always in the same - // project (and thus amalgamation). - // - if (pt->is_a<libux> ()) - { - // Adjust match options (similar to above). - // - if (a.operation () != update_id && !pt->is_a<libue> ()) - { - if (t.is_a<exe> ()) - options = lib::option_install_runtime; - else - { - if (me.cur_options == lib::option_install_runtime) - options = lib::option_install_runtime; - } - } - - return make_pair (pt, options); - } - } -#else // Note that at first it may seem like we don't need to install static // library prerequisites of executables. But such libraries may still // have prerequisites that are needed at runtime (say, some data files). @@ -189,7 +109,21 @@ namespace build2 // This is a library prerequisite of a library target and // runtime-only begets runtime-only. // - if (me.cur_options == lib::option_install_runtime) + // @@ But it goes further: while an interface prerequisite should + // match the target's options, it feels an implementation can be + // runtime-only, at least for shared library targets (for static + // the consumer would still need to link the prerequisite + // explicitly, which means it is more like buildtime). We could + // probably distinguish between interface/implementation by + // examining the *.export.libs variable and looking for the + // prerequisite (or its group). Feels hairy, though. So for now we + // only do this for target-shared/prerequisite-static case where + // we can assume the prerequisite is always implementation. See GH + // issue #448. See also apply_posthoc() as well as + // libux_install_rule below. + // + if (me.cur_options == lib::option_install_runtime || + (t.is_a<libs> () && pt->is_a<liba> ())) options = lib::option_install_runtime; } } @@ -209,7 +143,6 @@ namespace build2 return make_pair (pt, options); } } -#endif // The rest of the tests only succeed if the base filter() succeeds. // @@ -348,7 +281,7 @@ namespace build2 // if (!*md.for_install) fail << "incompatible " << t << " build" << - info << "target already built not for install"; + info << "target already updated but not for install"; } else md.for_install = true; @@ -396,6 +329,9 @@ namespace build2 p.match_options = lib::option_install_runtime; else { + // @@ Hm, maybe runtime should be unconditional here since a + // plugin is always an implementation dependency? + // if (me.cur_options == lib::option_install_runtime) p.match_options = lib::option_install_runtime; } @@ -561,56 +497,6 @@ namespace build2 // above. In particular, here we use libue/libua/libus{} as proxies for // exe/liba/libs{} there. // - - // @@ TMP: drop eventually. - // -#if 0 - bool st (t.is_a<libue> () || t.is_a<libus> ()); // Target needs shared. - bool at (t.is_a<libua> () || t.is_a<libus> ()); // Target needs static. - assert (st || at); - - if ((st && (p.is_a<libx> () || p.is_a<libs> ())) || - (at && (p.is_a<libx> () || p.is_a<liba> ()))) - { - const target* pt (&search (t, p)); - - if (const libx* l = pt->is_a<libx> ()) - pt = link_member (*l, a, link_info (t.base_scope (), ot)); - - if ((st && pt->is_a<libs> ()) || (at && pt->is_a<liba> ())) - { - if (a.operation () != update_id) - { - if (t.is_a<libue> ()) - options = lib::option_install_runtime; - else - { - if (me.cur_options == lib::option_install_runtime) - options = lib::option_install_runtime; - } - } - - return make_pair (is == nullptr || pt->in (*is) ? pt : nullptr, - options); - } - - if (pt->is_a<libux> ()) - { - if (a.operation () != update_id && !pt->is_a<libue> ()) - { - if (t.is_a<libue> ()) - options = lib::option_install_runtime; - else - { - if (me.cur_options == lib::option_install_runtime) - options = lib::option_install_runtime; - } - } - - return make_pair (pt, options); - } - } -#else if (p.is_a<libx> () || p.is_a<libs> () || p.is_a<liba> ()) { const target* pt (&search (t, p)); @@ -624,7 +510,8 @@ namespace build2 options = lib::option_install_runtime; else { - if (me.cur_options == lib::option_install_runtime) + if (me.cur_options == lib::option_install_runtime || + (t.is_a<libus> () && pt->is_a<liba> ())) options = lib::option_install_runtime; } } @@ -637,7 +524,6 @@ namespace build2 else return make_pair (pt, options); } -#endif const target* pt (file_rule::instance.filter (is, a, t, p, me).first); if (pt == nullptr) diff --git a/libbuild2/cc/link-rule.cxx b/libbuild2/cc/link-rule.cxx index db82507..d43e7e8 100644 --- a/libbuild2/cc/link-rule.cxx +++ b/libbuild2/cc/link-rule.cxx @@ -2509,12 +2509,19 @@ namespace build2 // appear after the preceding static library of which this binless // library is a dependency. // + // Note that we omit the duplicate suppression if we are linking the + // whole archive since the previous instance may not necessarily do + // the same (see GH issue #411; we could have complicated things and + // stored the flag in appended_libraries but it doesn't feel + // worthwhile in this case). + // // From the process_libraries() semantics we know that this callback // is always called and always after the options callbacks. // - appended_library* al (l != nullptr - ? &d.ls.append (*l, d.args.size ()) - : d.ls.append (ns, d.args.size ())); + appended_library* al ( + f & lflag_whole ? nullptr : + l != nullptr ? &d.ls.append (*l, d.args.size ()) : + d.ls.append (ns, d.args.size ())); if (al != nullptr && al->end != appended_library::npos) // Closed. { @@ -2591,8 +2598,8 @@ namespace build2 // if (*md->for_install != *d.for_install) fail << "incompatible " << *l << " build" << - info << "library is built " << (*md->for_install ? "" : "not ") - << "for install"; + info << "target is already updated but " + << (*md->for_install ? "" : "not ") << "for install"; } auto newer = [&d, l] () @@ -2713,13 +2720,13 @@ namespace build2 else { d.args.push_back ("-Wl,--whole-archive"); - d.args.push_back (move (p)); + d.args.push_back (move (p)); p.clear (); d.args.push_back ("-Wl,--no-whole-archive"); - goto done; } } - d.args.push_back (move (p)); + if (!p.empty ()) + d.args.push_back (move (p)); } if (d.cs != nullptr) @@ -3515,10 +3522,6 @@ namespace build2 // args.push_back ("/NOLOGO"); - // Add /MACHINE. - // - args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); - // For utility libraries use thin archives if possible. // // LLVM's lib replacement had the /LLVMLIBTHIN option at least from @@ -3585,7 +3588,16 @@ namespace build2 { // Are we using the compiler or the linker (e.g., link.exe) directly? // - bool ldc (tsys != "win32-msvc"); + bool ldc; + + if (tsys == "win32-msvc") + { + args.push_back ("/NOLOGO"); + ldc = false; + } + else + ldc = true; + if (ldc) { @@ -3995,6 +4007,20 @@ namespace build2 if (lt.shared_library () && (tsys == "win32-msvc" || tsys == "mingw32")) reli = relative (find_adhoc_member<libi> (t)->path ()); + if (tsys == "win32-msvc") + { + // Add /MACHINE unless there is a custom value (/MACHINE:ARM64EC). + // + // Note that we don't bother hashing it since to change its value one + // would have to use a different MSVC toolchain (which means things + // would be rebuilt from scratch anyway). + // + if (!find_option_prefix ("/MACHINE:", args, true)) + { + args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); + } + } + const process_path* ld (nullptr); if (lt.static_library ()) { @@ -4018,15 +4044,10 @@ namespace build2 // Using link.exe directly. // ld = &cast<process_path> (rs["bin.ld.path"]); - args.push_back ("/NOLOGO"); if (ot == otype::s) args.push_back ("/DLL"); - // Add /MACHINE. - // - args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); - // Unless explicitly enabled with /INCREMENTAL, disable incremental // linking (it is implicitly enabled if /DEBUG is specified). The // reason is the .ilk file: its name cannot be changed and if we diff --git a/libbuild2/cc/module.cxx b/libbuild2/cc/module.cxx index cf6c6e4..a3c64d9 100644 --- a/libbuild2/cc/module.cxx +++ b/libbuild2/cc/module.cxx @@ -22,6 +22,12 @@ namespace build2 { namespace cc { + // cc.core_module + // + const string core_module::name ("cc.core.config"); + + // x.config_module + // void config_module:: guess (scope& rs, const location& loc, const variable_map&) { @@ -891,6 +897,9 @@ namespace build2 config::save_environment (rs, xi.platform_environment); } + // x module + // + // Global cache of ad hoc importable headers. // // The key is a hash of the system header search directories diff --git a/libbuild2/cc/module.hxx b/libbuild2/cc/module.hxx index 4213516..d4e9a67 100644 --- a/libbuild2/cc/module.hxx +++ b/libbuild2/cc/module.hxx @@ -14,6 +14,8 @@ #include <libbuild2/cc/common.hxx> +#include <libbuild2/cc/compiledb.hxx> + #include <libbuild2/cc/compile-rule.hxx> #include <libbuild2/cc/link-rule.hxx> #include <libbuild2/cc/install-rule.hxx> @@ -27,6 +29,35 @@ namespace build2 { struct compiler_info; + // cc.core module + // + class core_module: public build2::module + { + public: + static const string name; + + explicit + core_module (const core_module* om) + : outer_module_ (om) + { + } + + public: + const core_module* outer_module_; + + strings cdb_names_; + + const compiledb_name_filter* cdb_filter_ = nullptr; + const compiledb_type_filter* cdb_filter_input_ = nullptr; + const compiledb_type_filter* cdb_filter_output_ = nullptr; + + compiledb_name_filter cdb_filter_storage_; + compiledb_type_filter cdb_filter_input_storage_; + compiledb_type_filter cdb_filter_output_storage_; + }; + + // x.config module + // class LIBBUILD2_CC_SYMEXPORT config_module: public build2::module, public config_data { @@ -92,13 +123,36 @@ namespace build2 // struct header_key { - path file; + // We used to use path comparison/hash which are case-insensitive on + // Windows. While this sounds right on the surface, the catch is that + // our target names are always case-sensitive, even on Windows. So + // what can happen is that the same header spelled in different case + // ends up with distinct targets but trying to occupy the same cache + // entry. See GH issue #390 for details. + // + // The conceptually correct way to fix this would be to actualize the + // header name. But that would be prohibitively expensive, especially + // on Windows. Plus the header may be generated and thus not yet + // exist. + // + // On the other hand, we can already end up with different targets + // mapped to the same filesystem entry in other situations (h{} vs + // hxx{} is the canonical example). And we are ok with that provided + // they have noop recipes. So it feels like the simplest solution here + // is to go along by having case-sensitive cache entries. This means + // that auto-generated headers will have to be included using the same + // spelling, but that seems like a sensible restriction (doing + // otherwise won't be portable). + // + path file; size_t hash; friend bool operator== (const header_key& x, const header_key& y) { - return x.file == y.file; // Note: hash was already compared. + // Note: hash was already compared. + // + return x.file.string () == y.file.string (); } }; @@ -130,6 +184,8 @@ namespace build2 msvc_library_search_dirs (const compiler_info&, scope&) const; }; + // x module + // class LIBBUILD2_CC_SYMEXPORT module: public build2::module, public virtual common, public link_rule, @@ -139,7 +195,6 @@ namespace build2 public predefs_rule { public: - explicit module (data&& d, const scope& rs) : common (move (d)), link_rule (move (d)), diff --git a/libbuild2/cc/msvc.cxx b/libbuild2/cc/msvc.cxx index d21969c..66223b5 100644 --- a/libbuild2/cc/msvc.cxx +++ b/libbuild2/cc/msvc.cxx @@ -33,10 +33,10 @@ namespace build2 const char* msvc_cpu (const string& cpu) { - const char* m (cpu == "i386" || cpu == "i686" ? "x86" : - cpu == "x86_64" ? "x64" : + const char* m (cpu == "x86_64" ? "x64" : + cpu == "i386" || cpu == "i686" ? "x86" : + cpu == "aarch64" ? "arm64" : cpu == "arm" ? "arm" : - cpu == "arm64" ? "arm64" : nullptr); if (m == nullptr) @@ -51,10 +51,10 @@ namespace build2 const char* msvc_machine (const string& cpu) { - const char* m (cpu == "i386" || cpu == "i686" ? "/MACHINE:x86" : - cpu == "x86_64" ? "/MACHINE:x64" : + const char* m (cpu == "x86_64" ? "/MACHINE:x64" : + cpu == "i386" || cpu == "i686" ? "/MACHINE:x86" : + cpu == "aarch64" ? "/MACHINE:ARM64" : cpu == "arm" ? "/MACHINE:ARM" : - cpu == "arm64" ? "/MACHINE:ARM64" : nullptr); if (m == nullptr) @@ -511,8 +511,8 @@ namespace build2 if (!run_finish_code (args, pr, s, 2 /* verbosity */) || io) { - diag_record dr; - dr << warn << "unable to detect " << l << " library type, ignoring" << + diag_record dr (warn); + dr << "unable to detect " << l << " library type, ignoring" << info << "run the following command to investigate" << info; print_process (dr, args); return otype::e; diff --git a/libbuild2/cc/pkgconfig.cxx b/libbuild2/cc/pkgconfig.cxx index 7e47534..d4be03b 100644 --- a/libbuild2/cc/pkgconfig.cxx +++ b/libbuild2/cc/pkgconfig.cxx @@ -1,6 +1,8 @@ // file : libbuild2/cc/pkgconfig.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file +#include <libbuild2/cc/pkgconfig.hxx> + #include <libbuild2/scope.hxx> #include <libbuild2/target.hxx> #include <libbuild2/context.hxx> @@ -18,7 +20,6 @@ #include <libbuild2/cc/utility.hxx> #include <libbuild2/cc/common.hxx> -#include <libbuild2/cc/pkgconfig.hxx> #include <libbuild2/cc/compile-rule.hxx> #include <libbuild2/cc/link-rule.hxx> @@ -692,6 +693,7 @@ namespace build2 cmp ("msxml", 5) || // msxml* cmp ("netapi32") || cmp ("normaliz") || + cmp ("ntdll") || cmp ("odbc32") || cmp ("ole32") || cmp ("oleaut32") || @@ -707,6 +709,7 @@ namespace build2 cmp ("user32") || cmp ("userenv") || cmp ("uuid") || + cmp ("uxtheme") || cmp ("version") || cmp ("windowscodecs") || cmp ("winhttp") || diff --git a/libbuild2/cc/windows-manifest.cxx b/libbuild2/cc/windows-manifest.cxx index 14f4a53..ff77327 100644 --- a/libbuild2/cc/windows-manifest.cxx +++ b/libbuild2/cc/windows-manifest.cxx @@ -23,8 +23,9 @@ namespace build2 const char* windows_manifest_arch (const string& tcpu) { - const char* pa (tcpu == "i386" || tcpu == "i686" ? "x86" : - tcpu == "x86_64" ? "amd64" : + const char* pa (tcpu == "x86_64" ? "amd64" : + tcpu == "i386" || tcpu == "i686" ? "x86" : + tcpu == "aarch64" ? "arm64" : nullptr); if (pa == nullptr) diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx index 2f134c4..776299c 100644 --- a/libbuild2/config/init.cxx +++ b/libbuild2/config/init.cxx @@ -27,6 +27,7 @@ namespace build2 namespace config { static const file_rule file_rule_ (true /* check_type */); + static const noop_rule noop_rule_ (true /* exclude_group */); void functions (function_map&); // functions.cxx @@ -38,7 +39,10 @@ namespace build2 // the entire values. // static pair<names_view, const char*> - save_environment (const value& d, const value* b, names& storage) + save_environment (const scope&, + const value& d, + const value* b, + names& storage) { if (b == nullptr) return make_pair (reverse (d, storage, true /* reduce */), "="); @@ -730,7 +734,7 @@ namespace build2 // This allows a custom configure rule while doing nothing by default. // - rs.insert_rule<target> (configure_id, 0, "config.noop", noop_rule::instance); + rs.insert_rule<target> (configure_id, 0, "config.noop", noop_rule_); // We need this rule for out-of-any-project dependencies (for example, // libraries imported from /usr/lib). We are registering it on the diff --git a/libbuild2/config/module.cxx b/libbuild2/config/module.cxx index 713d30c..faae865 100644 --- a/libbuild2/config/module.cxx +++ b/libbuild2/config/module.cxx @@ -14,7 +14,7 @@ namespace build2 bool module:: save_variable (const variable& var, optional<uint64_t> flags, - save_variable_function* save) + save_variable_function* func) { const string& n (var.name); @@ -45,15 +45,18 @@ namespace build2 return false; } - sv.push_back (saved_variable {var, flags, save}); + sv.push_back (saved_variable {var, flags, func}); return true; } void module:: - save_variable (scope& rs, const variable& var, optional<uint64_t> flags) + save_variable (scope& rs, + const variable& var, + optional<uint64_t> flags, + save_variable_function* func) { if (module* m = rs.find_module<module> (module::name)) - m->save_variable (var, flags); + m->save_variable (var, flags, func); } void module:: diff --git a/libbuild2/config/module.hxx b/libbuild2/config/module.hxx index 8d3ff67..77109ce 100644 --- a/libbuild2/config/module.hxx +++ b/libbuild2/config/module.hxx @@ -22,20 +22,12 @@ namespace build2 namespace config { // An ordered list of build system modules each with an ordered list of - // config.* variables and their "save flags" (see save_variable()) that - // are used (as opposed to just being specified) in this configuration. - // Populated by the config utility functions (required(), optional()) and - // saved in the order populated. If flags are absent, then this variable - // was marked as "unsaved" (always transient). + // config.* variables and their save flags/function (see save_variable()) + // that are used (as opposed to just being specified) in this + // configuration. Populated by the config utility functions (required(), + // optional()) and saved in the order populated. If flags are absent, then + // this variable was marked as "unsaved" (always transient). // - // The optional save function can be used to implement custom variable - // saving, for example, as a difference appended to the base value. The - // second half of the result is the assignment operator to use. - // - using save_variable_function = - pair<names_view, const char*> (const value&, - const value* base, - names& storage); struct saved_variable { reference_wrapper<const variable> var; @@ -151,7 +143,10 @@ namespace build2 save_variable_function* = nullptr); static void - save_variable (scope&, const variable&, optional<uint64_t>); + save_variable (scope&, + const variable&, + optional<uint64_t>, + save_variable_function*); bool save_module (const char* name, int prio = 0); diff --git a/libbuild2/config/operation.cxx b/libbuild2/config/operation.cxx index b1716cf..5c06f3c 100644 --- a/libbuild2/config/operation.cxx +++ b/libbuild2/config/operation.cxx @@ -302,8 +302,8 @@ namespace build2 continue; } - diag_record dr; - dr << warn (on) << "saving no longer used variable " << *var; + diag_record dr (warn (on)); + dr << "saving no longer used variable " << *var; info_import (dr, var->name); if (verb >= 2) info_value (dr, v); @@ -313,8 +313,8 @@ namespace build2 { if (r.second) // warn { - diag_record dr; - dr << warn (on) << "dropping no longer used variable " << *var; + diag_record dr (warn (on)); + dr << "dropping no longer used variable " << *var; info_import (dr, var->name); info_value (dr, v); } @@ -479,9 +479,8 @@ namespace build2 // if (checked && org == ovr) { - diag_record dr; - dr << warn (on) << "saving previously inherited variable " - << var; + diag_record dr (warn (on)); + dr << "saving previously inherited variable " << var; dr << info << "because project " << *r << " no longer uses " << "it in its configuration"; @@ -558,7 +557,7 @@ namespace build2 storage.clear (); pair<names_view, const char*> p ( sv.save != nullptr - ? sv.save (v, base, storage) + ? sv.save (rs, v, base, storage) : make_pair (reverse (v, storage, true /* reduce */), "=")); // Might becomes empty after a custom save function had at it. @@ -1018,8 +1017,20 @@ namespace build2 if (oif->operation_pre != nullptr) oif->operation_pre (ctx, {}, true /* inner */, location ()); + // Use perform_match() instead of direct match_sync() to handle + // posthoc targets (similar to dist). + // +#if 0 phase_lock pl (ctx, run_phase::match); match_sync (action (configure_id, id), t); +#else + action a (configure_id, id); + action_targets ts {&t}; + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + perform_post_operation_callbacks (ctx, a, ts, false /*failed*/); +#endif if (oif->operation_post != nullptr) oif->operation_post (ctx, {}, true /* inner */); diff --git a/libbuild2/config/utility.cxx b/libbuild2/config/utility.cxx index 6574367..35ff3ff 100644 --- a/libbuild2/config/utility.cxx +++ b/libbuild2/config/utility.cxx @@ -7,7 +7,14 @@ using namespace std; namespace build2 { - void (*config_save_variable) (scope&, const variable&, optional<uint64_t>); + void + (*config_save_variable) (scope&, + const variable&, + optional<uint64_t>, + pair<names_view, const char*> (*)(const scope&, + const value&, + const value*, + names&)); void (*config_save_environment) (scope&, const char*); void (*config_save_module) (scope&, const char*, int); const string& (*config_preprocess_create) (context&, @@ -21,7 +28,10 @@ namespace build2 namespace config { pair<lookup, bool> - lookup_config_impl (scope& rs, const variable& var, uint64_t sflags) + lookup_config_impl (scope& rs, + const variable& var, + uint64_t sflags, + save_variable_function* sfunc) { // This is a stripped-down version of the default value case. @@ -71,16 +81,19 @@ namespace build2 } if (l.defined ()) - save_variable (rs, var, sflags); + save_variable (rs, var, sflags, sfunc); return pair<lookup, bool> (l, n); } bool - specified_config (scope& rs, - const string& n, - initializer_list<const char*> ig) + specified_config (const scope& rs, + const string& ns, + initializer_list<const char*> ig, + bool exact) { + assert (!exact || ig.size () == 0); + // Note: go straight for the public variable pool. // auto& vp (rs.ctx.var_pool); @@ -93,8 +106,7 @@ namespace build2 // any original values, they will be "visible"; see find_override() for // details. // - const string ns ("config." + n); - for (scope* s (&rs); s != nullptr; s = s->parent_scope ()) + for (const scope* s (&rs); s != nullptr; s = s->parent_scope ()) { for (auto p (s->vars.lookup_namespace (ns)); p.first != p.second; @@ -107,17 +119,25 @@ namespace build2 if (size_t n = v->override ()) v = vp.find (string (v->name, 0, n)); - auto match_tail = [&ns, v] (const char* t) + if (exact) { - return v->name.compare (ns.size () + 1, string::npos, t) == 0; - }; - - // Ignore config.*.configured and user-supplied names. - // - if (v->name.size () <= ns.size () || - (!match_tail ("configured") && - find_if (ig.begin (), ig.end (), match_tail) == ig.end ())) - return true; + if (v->name.size () == ns.size ()) + return true; + } + else + { + auto match_tail = [&ns, v] (const char* t) + { + return v->name.compare (ns.size () + 1, string::npos, t) == 0; + }; + + // Ignore config.*.configured and user-supplied names. + // + if (v->name.size () <= ns.size () || // @@ Hm, when can it be < ? + (!match_tail ("configured") && + find_if (ig.begin (), ig.end (), match_tail) == ig.end ())) + return true; + } } } diff --git a/libbuild2/config/utility.hxx b/libbuild2/config/utility.hxx index 1e2ff53..57f253f 100644 --- a/libbuild2/config/utility.hxx +++ b/libbuild2/config/utility.hxx @@ -29,13 +29,22 @@ namespace build2 // disfigure hooks (for example, for second-level configuration). These are // accessed through the config module entry points (which are NULL for // transient configurations). Note also that the exact interpretation of the - // save flags and module order depends on the config module implementation - // (which may ignore them as not applicable). An implementation may also - // define custom save flags (for example, accessible through the config.save - // attribute). Such flags should start from 0x100000000. + // save flags/function and module order depends on the config module + // implementation (which may ignore them as not applicable). An + // implementation may also define custom save flags (for example, accessible + // through the config.save attribute). Such flags should start from + // 0x100000000. + // + // See below for the save function (last argument) semantics. // LIBBUILD2_SYMEXPORT extern void - (*config_save_variable) (scope&, const variable&, optional<uint64_t>); + (*config_save_variable) (scope&, + const variable&, + optional<uint64_t>, + pair<names_view, const char*> (*)(const scope&, + const value&, + const value*, + names&)); LIBBUILD2_SYMEXPORT extern void (*config_save_environment) (scope&, const char*); @@ -75,11 +84,27 @@ namespace build2 const uint64_t save_false_omitted = 0x08; // Treat false as undefined. const uint64_t save_base = 0x10; // Custom save with base. + // The optional save function can be used to implement custom variable + // saving, for example, as a difference appended to the base value or to + // complete relative paths (if abs_dir_path does not fit). The base + // argument is the value of this variable from the outer scope (if any) + // and is only calculated if the save_base flag is specified. The second + // half of the result is the assignment operator to use. + // + using save_variable_function = + pair<names_view, const char*> (const scope& rs, + const value&, + const value* base, + names& storage); + inline void - save_variable (scope& rs, const variable& var, uint64_t flags = 0) + save_variable (scope& rs, + const variable& var, + uint64_t flags = 0, + save_variable_function* func = nullptr) { if (config_save_variable != nullptr) - config_save_variable (rs, var, flags); + config_save_variable (rs, var, flags, func); } // Mark a variable as "unsaved" (always transient). @@ -91,7 +116,7 @@ namespace build2 unsave_variable (scope& rs, const variable& var) { if (config_save_variable != nullptr) - config_save_variable (rs, var, nullopt); + config_save_variable (rs, var, nullopt, nullptr); } // Mark an environment variable to be saved during hermetic configuration. @@ -256,35 +281,40 @@ namespace build2 lookup lookup_config (scope& rs, const variable&, - uint64_t save_flags = 0); + uint64_t save_flags = 0, + save_variable_function* = nullptr); lookup lookup_config (bool& new_value, scope& rs, const variable&, - uint64_t save_flags = 0); + uint64_t save_flags = 0, + save_variable_function* = nullptr); // Note that the variable is expected to have already been entered. // inline lookup lookup_config (scope& rs, const string& var, - uint64_t save_flags = 0) + uint64_t save_flags = 0, + save_variable_function* func = nullptr) { // Note: go straight for the public variable pool. // - return lookup_config (rs, rs.ctx.var_pool[var], save_flags); + return lookup_config (rs, rs.ctx.var_pool[var], save_flags, func); } inline lookup lookup_config (bool& new_value, scope& rs, const string& var, - uint64_t save_flags = 0) + uint64_t save_flags = 0, + save_variable_function* func = nullptr) { // Note: go straight for the public variable pool. // - return lookup_config (new_value, rs, rs.ctx.var_pool[var], save_flags); + return lookup_config ( + new_value, rs, rs.ctx.var_pool[var], save_flags, func); } // Lookup a config.* variable value and, if the value is undefined, set it @@ -476,17 +506,23 @@ namespace build2 // that it is unconfigured (e.g., in order to avoid re-running the tests, // etc; see below). Additional variables (e.g., unsaved) can be ignored // with the third argument. If specified, it should contain the part(s) - // after config.<name>. + // after the namespace (config.<name>). + // + // Note that <name> may include several components (separated with `.`). + // And you can request the exact match rather than the prefix. + // + // Note: unlike the above functions, can be called from any phase. // LIBBUILD2_SYMEXPORT bool - specified_config (scope& rs, - const string& var, - initializer_list<const char*> ignore); + specified_config (const scope& rs, + const string& ns, + initializer_list<const char*> ignore, + bool exact = false); inline bool - specified_config (scope& rs, const string& var) + specified_config (const scope& rs, const string& ns, bool exact = false) { - return specified_config (rs, var, {}); + return specified_config (rs, ns, {}, exact); } // Check if there is a false config.*.configured value. This mechanism can diff --git a/libbuild2/config/utility.ixx b/libbuild2/config/utility.ixx index d8348bd..87f628c 100644 --- a/libbuild2/config/utility.ixx +++ b/libbuild2/config/utility.ixx @@ -6,25 +6,32 @@ namespace build2 namespace config { LIBBUILD2_SYMEXPORT pair<lookup, bool> - lookup_config_impl (scope&, const variable&, uint64_t); + lookup_config_impl (scope&, + const variable&, + uint64_t, + save_variable_function*); template <typename T> pair<lookup, bool> lookup_config_impl (scope&, const variable&, T&&, uint64_t, bool); inline lookup - lookup_config (scope& rs, const variable& var, uint64_t sflags) + lookup_config (scope& rs, + const variable& var, + uint64_t sflags, + save_variable_function* func) { - return lookup_config_impl (rs, var, sflags).first; + return lookup_config_impl (rs, var, sflags, func).first; } inline lookup lookup_config (bool& new_value, scope& rs, const variable& var, - uint64_t sflags) + uint64_t sflags, + save_variable_function* func) { - auto r (lookup_config_impl (rs, var, sflags)); + auto r (lookup_config_impl (rs, var, sflags, func)); new_value = new_value || r.second; return r.first; } diff --git a/libbuild2/context.cxx b/libbuild2/context.cxx index 6e4fd6f..31d218b 100644 --- a/libbuild2/context.cxx +++ b/libbuild2/context.cxx @@ -1061,6 +1061,13 @@ namespace build2 return r ? optional<bool> (s) : nullopt; } + bool run_phase_mutex:: + unlocked () const + { + mlock l (m_); + return lc_ == 0 && mc_ == 0 && ec_ == 0; + } + // C++17 deprecated uncaught_exception() so use uncaught_exceptions() if // available. // diff --git a/libbuild2/context.hxx b/libbuild2/context.hxx index 828c41e..12b11d5 100644 --- a/libbuild2/context.hxx +++ b/libbuild2/context.hxx @@ -49,6 +49,12 @@ namespace build2 optional<bool> relock (run_phase unlock, run_phase lock); + // Return true if the mutex is unlocked, meaning we are in the initial + // load phase. + // + bool + unlocked () const; + // Statistics. // public: @@ -78,7 +84,7 @@ namespace build2 // context& ctx_; - mutex m_; + mutable mutex m_; bool fail_; @@ -178,38 +184,43 @@ namespace build2 // match - search prerequisites and match rules // execute - execute the matched rule // - // The build system starts with a "serial load" phase and then continues - // with parallel match and execute. Match, however, can be interrupted - // both with load and execute. + // The build system starts with a serial "initial load" phase and then + // continues with parallel match and execute. Match, however, can be + // interrupted both with load and execute. // - // Match can be interrupted with "exclusive load" in order to load - // additional buildfiles. Similarly, it can be interrupted with (parallel) - // execute in order to build targetd required to complete the match (for - // example, generated source code or source code generators themselves). + // Match can be interrupted with a (serial) "interrupting load" in order + // to load additional buildfiles. Similarly, it can be interrupted with + // (parallel) execute in order to build targetd required to complete the + // match (for example, generated source code or source code generators + // themselves). // // Such interruptions are performed by phase change that is protected by // phase_mutex (which is also used to synchronize the state changes // between phases). // - // Serial load can perform arbitrary changes to the build state. Exclusive - // load, however, can only perform "island appends". That is, it can - // create new "nodes" (variables, scopes, etc) but not (semantically) - // change already existing nodes or invalidate any references to such (the - // idea here is that one should be able to load additional buildfiles as - // long as they don't interfere with the existing build state). The - // "islands" are identified by the load_generation number (1 for the - // initial/serial load). It is incremented in case of a phase switch and - // can be stored in various "nodes" to verify modifications are only done - // "within the islands". Another example of invalidation would be - // insertion of a new scope "under" an existing target thus changing its - // scope hierarchy (and potentially even its base scope). This would be - // bad because we may have made decisions based on the original hierarchy, - // for example, we may have queried a variable which in the new hierarchy - // would "see" a new value from the newly inserted scope. + // Initial load can perform arbitrary changes to the build state. + // Interrupting load, however, can only perform what we call "island + // appends". That is, it can create new "nodes" (variables, scopes, etc) + // but not (semantically) change already existing nodes or invalidate any + // references to such (the idea here is that one should be able to load + // additional buildfiles as long as they don't interfere with the existing + // build state). The "islands" are identified by the load_generation + // number (1 for the initial load). It is incremented in case of a phase + // switch and can be stored in various "nodes" to verify modifications are + // only done "within the islands". Another example of invalidation would + // be insertion of a new scope "under" an existing target thus changing + // its scope hierarchy (and potentially even its base scope). This would + // be bad because we may have made decisions based on the original + // hierarchy, for example, we may have queried a variable which in the new + // hierarchy would "see" a new value from the newly inserted scope. // // The special load_generation value 0 indicates initialization before - // anything has been loaded. Currently, it is changed to 1 at the end - // of the context constructor. + // anything has been loaded. Currently, it is changed to 1 at the end of + // the context constructor. Note also that subsequent operations in a + // batch may trigger loading of additional buildfiles, in fact, entire new + // projects. As a result, load_generation is also incremented after each + // operation in a batch. If you need to detect the initial load in each + // operation, check that phase_mutex is unlocked. // // Note must come (and thus initialized) before the data_ member. // @@ -229,6 +240,10 @@ namespace build2 // Match only flag/level (see --{load,match}-only but also dist). // + // See also dry_run, which is in some sense a weaker version of match- + // only: the target is executed but nothing is actually being done (unless + // executed during match or load, that is). + // optional<match_only_level> match_only; // Skip booting external modules flag (see --no-external-modules). @@ -262,11 +277,22 @@ namespace build2 // // Note also that sometimes it makes sense to do a bit more than // absolutely necessary or to discard information in order to keep the - // rule logic sane. And some rules may choose to ignore this flag + // rule logic sane. And some rules may choose to ignore this flag // altogether. In this case, however, the rule should be careful not to // rely on functions (notably from filesystem) that respect this flag in // order not to end up with a job half done. // + // Finally, sometimes you may need to know during match whether there will + // be a non-dry-run execute and use the dry_run_option for that. This can + // be problematic because even when dry_run_option is true, the target may + // end up being executed in the non-dry-run mode during load or match. As + // a result, any logic that is based on dry_run_option should be capable + // of functioning correctly in the non-dry-run execute. + // + // See also match_only, which is in some sense a stronger version of + // dry-run: the target is not executed at all, again, unless during match + // or load. + // bool dry_run = false; bool dry_run_option; @@ -351,10 +377,52 @@ namespace build2 (current_mname.empty () && current_oname == mo)); }; + // Operation callbacks. + // + // An entity (module, core) can register a function that will be called + // when an action is executed on a set of targets. The pre callback is + // called before any recipes for the action are matched and the post -- + // after all have been executed. The post callback is called even if + // execution has failed. + // + // The callback should only be registered during the load phase. Note + // that it's registered for the inner action, meaning that it will be + // called for any outer action (which is discernible from the first + // argument of the callback). Note also that meta-operations other than + // perform never actually execute any recipes and it probably only makes + // sense to register these callbacks for the perform_* actions. + // + // Note that the callbacks will also be called when building a build + // system module or an ad hoc C++ recipe. See create_module_context() for + // details. + // + // Note also that if the callbacks are registered from a module load + // function, then there are nuances with interrupted load phases. See the + // compilation database handling in the cc module for details. + // + // See also scope::operation_callback. + // + struct operation_callback + { + using pre_callback = + void (context&, action, const action_targets&); + + using post_callback = + void (context&, action, const action_targets&, bool failed); + + function<pre_callback> pre; + function<post_callback> post; + }; + + using operation_callback_map = multimap<action_id, operation_callback>; + + operation_callback_map operation_callbacks; + // Meta/operation-specific context-global auxiliary data storage. // - // Note: cleared by current_[meta_]operation() below. Normally set by - // meta/operation-specific callbacks from [mate_]operation_info. + // Normally set by meta/operation-specific callbacks from + // [mata_]operation_info. The operation data is cleared by + // current_operation() below. // // Note also: watch out for MT-safety in the data itself. // @@ -759,6 +827,12 @@ namespace build2 // Set current meta-operation and operation. // + // Remember to also increment load_generation for subsequent operations in + // a batch if additional buildfiles are loaded between them. + // + // Note that the context instance is not to be re-used between different + // meta-operations. + // void current_meta_operation (const meta_operation_info&); diff --git a/libbuild2/diagnostics.cxx b/libbuild2/diagnostics.cxx index c795e44..091d6e5 100644 --- a/libbuild2/diagnostics.cxx +++ b/libbuild2/diagnostics.cxx @@ -559,7 +559,10 @@ namespace build2 const process_env& pe, const char* const* args, size_t n) { if (pe.env ()) - dr << pe << ' '; + { + to_stream (dr.os, pe, process_env_format::cwd | process_env_format::vars); + dr << ' '; + } dr << butl::process_args {args, n}; } @@ -836,7 +839,7 @@ namespace build2 // is inseparable from any buffered diagnostics. So we prepare the record // first and then write both while holding the diagnostics stream lock. // - diag_record dr; + maybe_diag_record dr; if (!pe) { // Note: see similar code in run_finish_impl(). @@ -852,7 +855,7 @@ namespace build2 if (verb >= 1 && verb <= v) { dr << info << "command line: "; - print_process (dr, args); + print_process (*dr, args); } } } @@ -861,7 +864,7 @@ namespace build2 } void diag_buffer:: - close (diag_record&& dr) + close (maybe_diag_record&& dr) { assert (state_ != state::closed); @@ -901,7 +904,7 @@ namespace build2 args0 = nullptr; state_ = state::closed; - if (!buf.empty () || !dr.empty ()) + if (!buf.empty () || dr) { diag_stream_lock l; @@ -911,14 +914,14 @@ namespace build2 buf.clear (); } - if (!dr.empty ()) - dr.flush ([] (const butl::diag_record& r) - { - // Similar to default_writer(). - // - *diag_stream << r.os.str () << '\n'; - diag_stream->flush (); - }); + if (dr) + (*dr).flush ([] (const butl::diag_record& r) + { + // Similar to default_writer(). + // + *diag_stream << r.os.str () << '\n'; + diag_stream->flush (); + }); else diag_stream->flush (); } @@ -927,11 +930,15 @@ namespace build2 // diag_do(), etc. // string - diag_do (context& ctx, const action&) + diag_do (context& ctx, const action& a) { const meta_operation_info& m (*ctx.current_mif); const operation_info& io (*ctx.current_inner_oif); - const operation_info* oo (ctx.current_outer_oif); + const operation_info* oo (a.outer () ? ctx.current_outer_oif : nullptr); + + assert (m.id == a.meta_operation () && + io.id == a.operation () && + (a.inner () || (oo != nullptr && oo->id == a.outer_operation ()))); string r; @@ -968,11 +975,15 @@ namespace build2 } string - diag_doing (context& ctx, const action&) + diag_doing (context& ctx, const action& a) { const meta_operation_info& m (*ctx.current_mif); const operation_info& io (*ctx.current_inner_oif); - const operation_info* oo (ctx.current_outer_oif); + const operation_info* oo (a.outer () ? ctx.current_outer_oif : nullptr); + + assert (m.id == a.meta_operation () && + io.id == a.operation () && + (a.inner () || (oo != nullptr && oo->id == a.outer_operation ()))); string r; @@ -1005,11 +1016,15 @@ namespace build2 } string - diag_did (context& ctx, const action&) + diag_did (context& ctx, const action& a) { const meta_operation_info& m (*ctx.current_mif); const operation_info& io (*ctx.current_inner_oif); - const operation_info* oo (ctx.current_outer_oif); + const operation_info* oo (a.outer () ? ctx.current_outer_oif : nullptr); + + assert (m.id == a.meta_operation () && + io.id == a.operation () && + (a.inner () || (oo != nullptr && oo->id == a.outer_operation ()))); string r; @@ -1046,11 +1061,17 @@ namespace build2 } void - diag_done (ostream& os, const action&, const target& t) + diag_done (ostream& os, const action& a, const target& t) { - const meta_operation_info& m (*t.ctx.current_mif); - const operation_info& io (*t.ctx.current_inner_oif); - const operation_info* oo (t.ctx.current_outer_oif); + context& ctx (t.ctx); + + const meta_operation_info& m (*ctx.current_mif); + const operation_info& io (*ctx.current_inner_oif); + const operation_info* oo (a.outer () ? ctx.current_outer_oif : nullptr); + + assert (m.id == a.meta_operation () && + io.id == a.operation () && + (a.inner () || (oo != nullptr && oo->id == a.outer_operation ()))); // perform(update(x)) -> "x is up to date" // configure(update(x)) -> "updating x is configured" diff --git a/libbuild2/diagnostics.hxx b/libbuild2/diagnostics.hxx index ef41f22..3452ad8 100644 --- a/libbuild2/diagnostics.hxx +++ b/libbuild2/diagnostics.hxx @@ -4,6 +4,8 @@ #ifndef LIBBUILD2_DIAGNOSTICS_HXX #define LIBBUILD2_DIAGNOSTICS_HXX +#include <cstddef> // max_align_t + #include <libbutl/diagnostics.hxx> #include <libbuild2/types.hxx> @@ -447,15 +449,110 @@ namespace build2 return *this; } - diag_record () = default; + // Use maybe_diag_record below for maybe-used records. + // + diag_record () = delete; + + // Create empty record that will be used. + // + explicit + diag_record (nullptr_t): butl::diag_record () {} template <typename B> explicit - diag_record (const diag_prologue<B>& p): diag_record () { *this << p;} + diag_record (const diag_prologue<B>& p) + : butl::diag_record () + { + *this << p; + } template <typename B> explicit - diag_record (const diag_mark<B>& m): diag_record () { *this << m;} + diag_record (const diag_mark<B>& m) + : butl::diag_record () + { + *this << m; + } + }; + + // A diag_record wrapper similar to std::optional that allows creating an + // uninitialized record that may or may not be used. The initializers are + // the diagnostics marks, for example: + // + // maybe_diag_record dr; // Uninitialized. + // dr << fail << "bad"; // Initialized. + // if (dr) + // *dr << "extra info"; + // + // The reason we need this is because the std::ostringstream member of + // butl::diag_record is quite expensive to construct but to never use. + // + struct maybe_diag_record + { + maybe_diag_record (): empty_ (true) {} + + explicit operator bool () const + { + return !empty_; + } + + diag_record& + operator* () + { + return *reinterpret_cast<diag_record*> (data_); + } + + template <typename B> + diag_record& + operator<< (const diag_mark<B>& m) + { + diag_record* r; + + if (empty_) + { + r = new (&data_) diag_record (m); + empty_ = false; + } + else + *(r = reinterpret_cast<diag_record*> (data_)) << m; + + return *r; + } + + template <typename B> + diag_record& + operator<< (const diag_prologue<B>& p) + { + diag_record* r; + + if (empty_) + { + r = new (&data_) diag_record (p); + empty_ = false; + } + else + *(r = reinterpret_cast<diag_record*> (data_)) << p; + + return *r; + } + + maybe_diag_record (maybe_diag_record&&) = delete; + maybe_diag_record& operator= (maybe_diag_record&&) = delete; + + maybe_diag_record (const maybe_diag_record&) = delete; + maybe_diag_record& operator= (const maybe_diag_record&) = delete; + + ~maybe_diag_record () noexcept (false) // butl::diag_record may throw + { + if (!empty_) + reinterpret_cast<diag_record*> (data_)->~diag_record (); + } + + private: + bool empty_; + + static constexpr size_t size_ = sizeof (diag_record); + alignas (std::max_align_t) unsigned char data_[size_]; }; template <typename B> @@ -467,7 +564,7 @@ namespace build2 diag_record operator<< (const T& x) const { - diag_record r; + diag_record r (nullptr); r.append (this->indent, this->epilogue); B::operator() (r); r << x; @@ -981,7 +1078,7 @@ namespace build2 // function will throw. // void - close (diag_record&& = {}); + close (maybe_diag_record&& = {}); // Direct access to the underlying stream and buffer for custom processing // (see read() above for details). diff --git a/libbuild2/dist/init.cxx b/libbuild2/dist/init.cxx index 48a3e15..32cbff2 100644 --- a/libbuild2/dist/init.cxx +++ b/libbuild2/dist/init.cxx @@ -132,7 +132,7 @@ namespace build2 // Note: ignore config.dist.bootstrap. // - bool s (specified_config (rs, "dist", {"bootstrap"})); + bool s (specified_config (rs, "config.dist", {"bootstrap"})); // config.dist.root // diff --git a/libbuild2/dist/operation.cxx b/libbuild2/dist/operation.cxx index dd1c87b..0ed02c6 100644 --- a/libbuild2/dist/operation.cxx +++ b/libbuild2/dist/operation.cxx @@ -60,8 +60,13 @@ namespace build2 const path& arc, const dir_path& dir, const string& ext); static operation_id - dist_operation_pre (context&, const values&, operation_id o) + dist_operation_pre (context& ctx, const values&, operation_id o) { + // Note: cannot be --load-only which requires perform(update). + // + if (ctx.match_only) + fail << "--match-only specified for dist meta-operation"; + if (o != default_id) fail << "explicit operation specified for dist meta-operation"; @@ -155,8 +160,12 @@ namespace build2 for (const dir_entry& e: dir_iterator (d, dir_iterator::no_follow)) { const path& n (e.path ()); + const string& s (n.string ()); - if (!n.empty () && n.string ().front () != '.') + if (s.compare (0, 4, ".git") != 0 && + s != ".bdep" && + s != ".bpkg" && + s != ".build2") try { if (e.type () == entry_type::directory) // Can throw. @@ -278,120 +287,136 @@ namespace build2 const location loc (pn); // Dummy location. action_targets ts {tgt}; - auto process_postponed = [&ctx, &mod] () { - if (!mod.postponed.list.empty ()) + // Signal to the rules we will not be executing. + // + auto mog = make_guard ([&ctx] () {ctx.match_only = nullopt;}); + ctx.match_only = match_only_level::all; + + auto process_postponed = [&ctx, &mod, &ts] (action a) { - // Re-grab the phase lock similar to perform_match(). - // - phase_lock l (ctx, run_phase::match); + if (!mod.postponed.list.empty ()) + { + auto eg ( + make_exception_guard ( + [&ctx, a, &ts] () + { + perform_post_operation_callbacks ( + ctx, a, ts, true /* failed */); + })); - // Note that we don't need to bother with the mutex since we do - // all of this serially. But we can end up with new elements at - // the end. - // - // Strictly speaking, to handle this correctly we would need to do - // multiple passes over this list and only give up when we cannot - // make any progress since earlier entries that we cannot resolve - // could be "fixed" by later entries. But this feels far-fetched - // and so let's wait for a real example before complicating this. - // - for (auto i (mod.postponed.list.begin ()); - i != mod.postponed.list.end (); - ++i) - rule::match_postponed (*i); - } - }; + // Re-grab the phase lock similar to perform_match(). + // + phase_lock l (ctx, run_phase::match); - auto mog = make_guard ([&ctx] () {ctx.match_only = nullopt;}); - ctx.match_only = match_only_level::all; + // Note that we don't need to bother with the mutex since we do + // all of this serially. But we can end up with new elements at + // the end. + // + // Strictly speaking, to handle this correctly we would need to + // do multiple passes over this list and only give up when we + // cannot make any progress since earlier entries that we cannot + // resolve could be "fixed" by later entries. But this feels + // far-fetched and so let's wait for a real example before + // complicating this. + // + for (auto i (mod.postponed.list.begin ()); + i != mod.postponed.list.end (); + ++i) + rule::match_postponed (*i); + } + }; - const operations& ops (rs.root_extra->operations); - for (operations::size_type id (default_id + 1); // Skip default_id. - id < ops.size (); - ++id) - { - if (const operation_info* oif = ops[id]) + const operations& ops (rs.root_extra->operations); + for (operations::size_type id (default_id + 1); // Skip default_id. + id < ops.size (); + ++id) { - // Skip aliases (e.g., update-for-install). In fact, one can argue - // the default update should be sufficient since it is assumed to - // update all prerequisites and we no longer support ad hoc stuff - // like test.input. Though here we are using the dist - // meta-operation, not perform. - // - if (oif->id != id) - continue; - - // Use standard (perform) match. - // - if (auto pp = oif->pre_operation) + if (const operation_info* oif = ops[id]) { - if (operation_id pid = pp (ctx, {}, dist_id, loc)) + // Skip aliases (e.g., update-for-install). In fact, one can + // argue the default update should be sufficient since it is + // assumed to update all prerequisites and we no longer support + // ad hoc stuff like test.input. Though here we are using the + // dist meta-operation, not perform. + // + if (oif->id != id) + continue; + + // Use standard (perform) match. + // + if (auto pp = oif->pre_operation) { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); + if (operation_id pid = pp (ctx, {}, dist_id, loc)) + { + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); - if (oif->operation_pre != nullptr) - oif->operation_pre (ctx, {}, false /* inner */, loc); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); - if (poif->operation_pre != nullptr) - poif->operation_pre (ctx, {}, true /* inner */, loc); + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); - action a (dist_id, poif->id, oif->id); - mod.postponed.list.clear (); - perform_match ({}, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - process_postponed (); + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (a); + perform_post_operation_callbacks (ctx, a, ts, false /*failed*/); - if (poif->operation_post != nullptr) - poif->operation_post (ctx, {}, true /* inner */); + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); - if (oif->operation_post != nullptr) - oif->operation_post (ctx, {}, false /* inner */); + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); + } } - } - ctx.current_operation (*oif, nullptr, false /* diag_noise */); + ctx.current_operation (*oif, nullptr, false /* diag_noise */); - if (oif->operation_pre != nullptr) - oif->operation_pre (ctx, {}, true /* inner */, loc); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, true /* inner */, loc); - action a (dist_id, oif->id); - mod.postponed.list.clear (); - perform_match ({}, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - process_postponed (); + action a (dist_id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (a); + perform_post_operation_callbacks (ctx, a, ts, false /*failed*/); - if (oif->operation_post != nullptr) - oif->operation_post (ctx, {}, true /* inner */); + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, true /* inner */); - if (auto po = oif->post_operation) - { - if (operation_id pid = po (ctx, {}, dist_id)) + if (auto po = oif->post_operation) { - const operation_info* poif (ops[pid]); - ctx.current_operation (*poif, oif, false /* diag_noise */); + if (operation_id pid = po (ctx, {}, dist_id)) + { + const operation_info* poif (ops[pid]); + ctx.current_operation (*poif, oif, false /* diag_noise */); - if (oif->operation_pre != nullptr) - oif->operation_pre (ctx, {}, false /* inner */, loc); + if (oif->operation_pre != nullptr) + oif->operation_pre (ctx, {}, false /* inner */, loc); - if (poif->operation_pre != nullptr) - poif->operation_pre (ctx, {}, true /* inner */, loc); + if (poif->operation_pre != nullptr) + poif->operation_pre (ctx, {}, true /* inner */, loc); - action a (dist_id, poif->id, oif->id); - mod.postponed.list.clear (); - perform_match ({}, a, ts, - 1 /* diag (failures only) */, - false /* progress */); - process_postponed (); + action a (dist_id, poif->id, oif->id); + mod.postponed.list.clear (); + perform_match ({}, a, ts, + 1 /* diag (failures only) */, + false /* progress */); + process_postponed (a); + perform_post_operation_callbacks (ctx, a, ts, false /*failed*/); - if (poif->operation_post != nullptr) - poif->operation_post (ctx, {}, true /* inner */); + if (poif->operation_post != nullptr) + poif->operation_post (ctx, {}, true /* inner */); - if (oif->operation_post != nullptr) - oif->operation_post (ctx, {}, false /* inner */); + if (oif->operation_post != nullptr) + oif->operation_post (ctx, {}, false /* inner */); + } } } } @@ -643,8 +668,8 @@ namespace build2 { l5 ([&]{trace << "bootstrap dist " << rs;}); - // Recursively enter/collect file targets in src_root ignoring those - // that start with a dot. + // Recursively enter/collect file targets in src_root ignoring the + // following ones: .git*, .bdep, .bpkg, and .build2. // // Note that, in particular, we also collect the symlinks which point // outside src_root (think of third-party project packaging with the diff --git a/libbuild2/dump.cxx b/libbuild2/dump.cxx index 9b7f5b1..9fcfca8 100644 --- a/libbuild2/dump.cxx +++ b/libbuild2/dump.cxx @@ -242,7 +242,8 @@ namespace build2 h_pair = true; } else if (t.is_a<map<optional<string>, string>> () || - t.is_a<vector<pair<optional<string>, string>>> ()) + t.is_a<vector<pair<optional<string>, string>>> () || + t.is_a<vector<pair<optional<string>, bool>>> ()) { h_array = true; h_pair = false; diff --git a/libbuild2/dyndep.cxx b/libbuild2/dyndep.cxx index dbeb47e..9d9dfa4 100644 --- a/libbuild2/dyndep.cxx +++ b/libbuild2/dyndep.cxx @@ -40,9 +40,8 @@ namespace build2 if (!f) return nullopt; - diag_record dr; - dr << fail << what << ' ' << pt << " not found and no rule to " - << "generate it"; + diag_record dr (fail); + dr << what << ' ' << pt << " not found and no rule to generate it"; if (verb < 4) dr << info << "re-run with --verbose=4 for more information"; @@ -107,9 +106,8 @@ namespace build2 if (!f) return nullopt; - diag_record dr; - dr << fail << what << ' ' << pt << " not found and no rule to " - << "generate it"; + diag_record dr (fail); + dr << what << ' ' << pt << " not found and no rule to generate it"; if (verb < 4) dr << info << "re-run with --verbose=4 for more information"; @@ -139,7 +137,7 @@ namespace build2 action a, const target& t, size_t pts_n, const file& pt) { - diag_record dr; + const char* err (nullptr); if (pt.matched (a, memory_order_acquire)) { @@ -148,7 +146,7 @@ namespace build2 { if (pts_n == 0 || !updated_during_match (a, t, pts_n, pt)) { - dr << fail << what << ' ' << pt << " has non-noop recipe"; + err = "has non-noop recipe"; } } } @@ -157,12 +155,14 @@ namespace build2 // Note that this target could not possibly be updated during match // since it's not matched. // - dr << fail << what << ' ' << pt << " is explicitly declared as " - << "target and may have non-noop recipe"; + err = "is explicitly declared as target and may have non-noop recipe"; } - if (!dr.empty ()) - dr << info << "consider listing it as static prerequisite of " << t; + if (err != nullptr) + { + fail << what << ' ' << pt << ' ' << err << + info << "consider listing it as static prerequisite of " << t; + } } small_vector<const target_type*, 2> dyndep_rule:: diff --git a/libbuild2/file.cxx b/libbuild2/file.cxx index f834d8c..14b60bd 100644 --- a/libbuild2/file.cxx +++ b/libbuild2/file.cxx @@ -2004,9 +2004,8 @@ namespace build2 if (!opt) { - diag_record dr; - dr << error (loc) << "invalid metadata signature in " << args[0] - << " output" << + diag_record dr (error (loc)); + dr << "invalid metadata signature in " << args[0] << " output" << info << "expected '" << s << "'"; if (verb >= 1 && verb <= 2) @@ -2042,8 +2041,8 @@ namespace build2 // if (!opt) { - diag_record dr; - dr << error (loc) << "unable to extract metadata from " << args[0] << + diag_record dr (error (loc)); + dr << "unable to extract metadata from " << args[0] << info << "process " << args[0] << " " << *pr.exit; if (verb >= 1 && verb <= 2) @@ -2111,14 +2110,17 @@ namespace build2 dr << info << "use config.import." << pv << " configuration variable to " << "specify its " << (qual != nullptr ? qual : "") << "project out_root"; - // Suggest ad hoc import but only if it's a path-based target (doing it - // for lib{} is very confusing). + // Suggest ad hoc import. Note that now we do it even for non-path-based + // targets since we have ad hoc import phase 2 logic for lib{}. // - if (tt != nullptr && tt->is_a<path_target> ()) + if (tt != nullptr) { - string v (tt->is_a<exe> () && (pv == tn || pn == tn) + string nv (sanitize_identifier (tn)); + + string v (tt->is_a<exe> () && pv == nv ? "config." + pv - : "config.import." + pv + '.' + tn + '.' + tt->name); + : ("config.import." + pv + '.' + nv + '.' + + sanitize_identifier (tt->name))); dr << info << "or use " << v << " configuration variable to specify " << "its " << (qual != nullptr ? qual : "") << "path"; @@ -2266,12 +2268,14 @@ namespace build2 // recognize the special config.<proj> (tool importation; we could // also handle the case where <proj> is not the same as <name> via // the config.<proj>.<name> variable). For backwards-compatibility - // reasons, it takes precedence over config.import. + // reasons, it takes precedence over config.import. Note that both + // <name> and <type> must be sanitized to a valid identifier (think + // libs{build2-foo} and cli.cxx{}). // - // Note: see import phase 2 diagnostics if changing anything here. + // Note also that phase 2 import may handle these imports in an ad hoc + // manner (see cc::search_library() for an example). // - // @@ How will this work for snake-case targets, say libs{build2-foo}? - // As well as for dot-separated target types, say, cli.cxx{}? + // Note: see import phase 2 diagnostics if changing anything here. // // @@ This duality has a nasty side-effect: if we have config.<proj> // configured, then specifying config.<proj>.import has no effect @@ -2330,6 +2334,8 @@ namespace build2 // const path* p (nullptr); + string valv (sanitize_identifier (tgt.value)); + if (tgt.typed ()) { bool e (tgt.type == "exe"); @@ -2338,15 +2344,18 @@ namespace build2 // overridable variable of type path. The config.<proj> we have to // type manually. // - if (e && (projv == tgt.value || proj == tgt.value)) + if (e && projv == valv) p = lookup (vp.insert<path> ("config." + projv), e); if (p == nullptr) - p = lookup (vp.insert (n + '.' + tgt.value + '.' + tgt.type), e); + { + string ttv (sanitize_identifier (tgt.type)); + p = lookup (vp.insert (n + '.' + valv + '.' + ttv), e); + } } if (p == nullptr) - p = lookup (vp.insert (n + '.' + tgt.value), false); + p = lookup (vp.insert (n + '.' + valv), false); if (p != nullptr) { @@ -2356,18 +2365,29 @@ namespace build2 { string on (move (tgt.value)); // Original name as imported. - tgt.dir = p->directory (); - tgt.value = p->leaf ().string (); + // Keep the original name if the path is (syntactically) to + // directory. + // + if (p->to_directory ()) + { + tgt.dir = path_cast<dir_path> (*p); + tgt.value = on; + } + else + { + tgt.dir = p->directory (); + tgt.value = p->leaf ().string (); + } // If the path is relative, then keep it project-qualified // assuming import phase 2 knows what to do with it. Think: // // config.import.build2.b=b-boot // - // @@ Maybe we should still complete it if it's not simple? After - // all, this is a path, do we want interpretations other than - // relative to CWD? Maybe we do, who knows. Doesn't seem to - // harm anything at the moment. + // Maybe we should still complete it if it's not simple? After + // all, this is a path, do we want interpretations other than + // relative to CWD? Maybe we do, who knows. Doesn't seem to harm + // anything at the moment. Yes, we do, see cc::search_library(). // // Why not call import phase 2 directly here? Well, one good // reason would be to allow for rule-specific import resolution. @@ -2376,48 +2396,59 @@ namespace build2 tgt.proj = move (proj); else { - // Enter the target and assign its path (this will most commonly - // be some out of project file). - // - // @@ Should we check that the file actually exists (and cache - // the extracted timestamp)? Or just let things take their - // natural course? - // name n (tgt); const target_type* tt (ibase.find_target_type (n, loc).first); if (tt == nullptr) fail (loc) << "unknown target type " << n.type << " in " << n; - // Note: not using the extension extracted by find_target_type() - // to be consistent with import phase 2. - // - target& t (insert_target (trace, ctx, *tt, *p).first); - - // Load the metadata, similar to import phase 2. + // If this is not a path-based target, then delegate to import + // phase 2 as above (see cc::search_library() for an example). // - if (meta) + if (!tt->is_a<path_target> ()) + { + tgt.proj = move (proj); + } + else { - if (exe* e = t.is_a<exe> ()) + // Enter the target and assign its path (this will most + // commonly be some out of project file). + // + // @@ Should we check that the file actually exists (and cache + // the extracted timestamp)? Or just let things take their + // natural course? + // + + // Note: not using the extension extracted by + // find_target_type() to be consistent with import phase 2. + // + target& t (insert_target (trace, ctx, *tt, *p).first); + + // Load the metadata, similar to import phase 2. + // + if (meta) { - if (!e->vars[ctx.var_export_metadata].defined ()) + if (exe* e = t.is_a<exe> ()) { - optional<string> md; + if (!e->vars[ctx.var_export_metadata].defined ()) { - auto df = make_diag_frame ( - [&proj, tt, &on] (const diag_record& dr) - { - import_suggest ( - dr, proj, tt, on, false, "alternative "); - }); - - md = extract_metadata (e->process_path (), - *meta, - false /* optional */, - loc); + optional<string> md; + { + auto df = make_diag_frame ( + [&proj, tt, &on] (const diag_record& dr) + { + import_suggest ( + dr, proj, tt, on, false, "alternative "); + }); + + md = extract_metadata (e->process_path (), + *meta, + false /* optional */, + loc); + } + + parse_metadata (*e, move (*md), loc); } - - parse_metadata (*e, move (*md), loc); } } } @@ -2664,12 +2695,10 @@ namespace build2 } else { - diag_record dr; - dr << fail (loc) << "unable to determine src_root for imported "; - if (proj) - dr << *proj; - else - dr << out_root; + diag_record dr (fail (loc)); + dr << "unable to determine src_root for imported "; + if (proj) dr << *proj; + else dr << out_root; dr << info << "consider configuring " << out_root; } @@ -3247,8 +3276,8 @@ namespace build2 if (opt || exist) return nullptr; - diag_record dr; - dr << fail (loc) << "unable to import target " << pk; + diag_record dr (fail (loc)); + dr << "unable to import target " << pk; if (proj.empty ()) dr << info << "consider adding its installation location" << @@ -3295,8 +3324,8 @@ namespace build2 if (opt || exist) return nullptr; - diag_record dr; - dr << fail (loc) << "unable to import target " << ns; + diag_record dr (fail (loc)); + dr << "unable to import target " << ns; import_suggest (dr, *n.proj, nullptr, string (), meta.has_value ()); } } @@ -3374,8 +3403,8 @@ namespace build2 if (opt) return names {}; - diag_record dr; - dr << fail (loc) << "unable to import target " << n; + diag_record dr (fail (loc)); + dr << "unable to import target " << n; import_suggest (dr, *n.proj, nullptr /* tt */, n.value, false); diff --git a/libbuild2/file.hxx b/libbuild2/file.hxx index 36e4c00..ff8a821 100644 --- a/libbuild2/file.hxx +++ b/libbuild2/file.hxx @@ -30,8 +30,9 @@ namespace build2 // export.build -- export stub // export/ -- exported buildfiles // - // The build/, bootstrap/, root/, and config.build entries are in .gitignore - // as generated by bdep-new. + // The build/, /bootstrap/, /root/, and config.build entries are in + // .gitignore as generated by bdep-new (note that build/ is ignored + // recursively; see below). // // The rest of the filesystem entries are shared between the project and the // modules that it loads. In particular, if a project loads module named diff --git a/libbuild2/filesystem.hxx b/libbuild2/filesystem.hxx index 7b45a08..44f5d92 100644 --- a/libbuild2/filesystem.hxx +++ b/libbuild2/filesystem.hxx @@ -24,6 +24,9 @@ namespace build2 { using butl::entry_type; + using butl::dir_entry; + using butl::dir_iterator; + using butl::auto_rmfile; using butl::auto_rmdir; diff --git a/libbuild2/function.cxx b/libbuild2/function.cxx index 3110547..3515c8e 100644 --- a/libbuild2/function.cxx +++ b/libbuild2/function.cxx @@ -239,9 +239,9 @@ namespace build2 // No match. // - diag_record dr; + diag_record dr (fail (loc)); - dr << fail (loc) << "unmatched call to "; print_call (dr.os); + dr << "unmatched call to "; print_call (dr.os); if (all_ovls != nullptr) { @@ -284,8 +284,8 @@ namespace build2 { // Ambigous match. // - diag_record dr; - dr << fail (loc) << "ambiguous call to "; print_call (dr.os); + diag_record dr (fail (loc)); + dr << "ambiguous call to "; print_call (dr.os); for (auto f: ovls) dr << info << "candidate: " << *f; diff --git a/libbuild2/functions-json.cxx b/libbuild2/functions-json.cxx index e06d9a5..9c59c4b 100644 --- a/libbuild2/functions-json.cxx +++ b/libbuild2/functions-json.cxx @@ -295,8 +295,8 @@ namespace build2 } catch (const invalid_json_output& e) { - diag_record dr; - dr << fail << "invalid json value: " << e; + diag_record dr (fail); + dr << "invalid json value: " << e; if (e.event) dr << info << "while serializing " << to_string (*e.event); diff --git a/libbuild2/functions-name.cxx b/libbuild2/functions-name.cxx index 456f85b..cb32d09 100644 --- a/libbuild2/functions-name.cxx +++ b/libbuild2/functions-name.cxx @@ -49,7 +49,10 @@ namespace build2 } const target& - to_target (const scope& s, name&& n, name&& o) + to_target (const scope& s, + name&& n, name&& o, + bool in_recipe, + const location& loc) { // Note: help the user out and search in both out and src like a // prerequisite. @@ -62,13 +65,13 @@ namespace build2 // bool typed (n.typed ()); - diag_record dr (fail); + diag_record dr (fail (loc)); dr << "target " << (n.pair ? names {move (n), move (o)} : names {move (n)}) << " not found"; - if (!typed) + if (in_recipe && !typed) dr << info << "wrap it in ([names] ...) if this is literal target name " << "specified inside recipe"; @@ -76,12 +79,15 @@ namespace build2 } const target& - to_target (const scope& s, names&& ns) + to_target (const scope& s, names&& ns, bool in_recipe, const location& loc) { assert (ns.size () == (ns[0].pair ? 2 : 1)); name o; - return to_target (s, move (ns[0]), move (ns[0].pair ? ns[1] : o)); + return to_target (s, + move (ns[0]), move (ns[0].pair ? ns[1] : o), + in_recipe, + loc); } static bool diff --git a/libbuild2/functions-name.hxx b/libbuild2/functions-name.hxx index 34fa4b8..30fc8ad 100644 --- a/libbuild2/functions-name.hxx +++ b/libbuild2/functions-name.hxx @@ -18,13 +18,16 @@ namespace build2 // Resolve the name to target issuing diagnostics and failing if not found. // LIBBUILD2_SYMEXPORT const target& - to_target (const scope&, name&&, name&& out); + to_target (const scope&, + name&&, name&& out, + bool in_recipe, + const location& = {}); // As above but from the names vector which should contain a single name // or an out-qualified name pair (asserted). // LIBBUILD2_SYMEXPORT const target& - to_target (const scope&, names&&); + to_target (const scope&, names&&, bool in_recipe, const location& = {}); } #endif // LIBBUILD2_FUNCTIONS_NAME_HXX diff --git a/libbuild2/functions-process.cxx b/libbuild2/functions-process.cxx index 6faa798..bcd91a8 100644 --- a/libbuild2/functions-process.cxx +++ b/libbuild2/functions-process.cxx @@ -177,8 +177,8 @@ namespace build2 // While assuming that the builtin has issued the diagnostics on failure // we still print the error message (see process_finish() for details). // - diag_record dr; - dr << fail << "builtin " << bn << " " << process_exit (rs); + diag_record dr (fail); + dr << "builtin " << bn << " " << process_exit (rs); if (verb >= 1 && verb <= 2) { diff --git a/libbuild2/functions-regex.cxx b/libbuild2/functions-regex.cxx index c46f6f5..de34d63 100644 --- a/libbuild2/functions-regex.cxx +++ b/libbuild2/functions-regex.cxx @@ -138,12 +138,24 @@ namespace build2 // string s (to_string (move (v))); + // Match flags. + // + // Note that by default std::regex_search() matches the empty substrings + // in non-empty strings for all the major implementations. We suppress + // such a counter-intuitive behavior with the match_not_null flag (see the + // butl::regex_replace_search() function implementation for details). + // + regex_constants::match_flag_type mf (regex_constants::match_default); + + if (!s.empty ()) + mf |= regex_constants::match_not_null; + if (!match && !subs) - return value (regex_search (s, rge)); // Return boolean value. + return value (regex_search (s, rge, mf)); // Return boolean value. match_results<string::const_iterator> m; - if (regex_search (s, m, rge)) + if (regex_search (s, m, rge, mf)) { assert (!m.empty ()); @@ -483,7 +495,19 @@ namespace build2 for (auto& n: ns) { - if (regex_search (convert<string> (move (n)), rge)) + string s (convert<string> (move (n))); + + // Match flags. + // + // Suppress matching of empty substrings in non-empty strings (see above + // for details). + // + regex_constants::match_flag_type mf (regex_constants::match_default); + + if (!s.empty ()) + mf |= regex_constants::match_not_null; + + if (regex_search (s, rge, mf)) return true; } @@ -516,7 +540,17 @@ namespace build2 bool s (n.simple ()); string v (convert<string> (s ? move (n) : name (n))); - if (regex_search (v, rge) == matching) + // Match flags. + // + // Suppress matching of empty substrings in non-empty strings (see above + // for details). + // + regex_constants::match_flag_type mf (regex_constants::match_default); + + if (!v.empty ()) + mf |= regex_constants::match_not_null; + + if (regex_search (v, rge, mf) == matching) r.emplace_back (s ? name (move (v)) : move (n)); } diff --git a/libbuild2/functions-target.cxx b/libbuild2/functions-target.cxx index d564aa2..55baeda 100644 --- a/libbuild2/functions-target.cxx +++ b/libbuild2/functions-target.cxx @@ -23,10 +23,10 @@ namespace build2 // // Return the path of a target (or a list of paths for a list of // targets). The path must be assigned, which normally happens during - // match. As a result, this function is normally called form a recipe. + // match. As a result, this function is normally called from a recipe. // // Note that while this function is technically not pure, we don't mark it - // as such since it can only be called (normally form a recipe) after the + // as such since it can only be called (normally from a recipe) after the // target has been matched, meaning that this target is a prerequisite and // therefore this impurity has been accounted for. // @@ -35,6 +35,8 @@ namespace build2 if (s == nullptr) fail << "target.path() called out of scope"; + context& ctx (s->ctx); + // Most of the time we will have a single target so optimize for that. // small_vector<path, 1> r; @@ -42,7 +44,10 @@ namespace build2 for (auto i (ns.begin ()); i != ns.end (); ++i) { name& n (*i), o; - const target& t (to_target (*s, move (n), move (n.pair ? *++i : o))); + const target& t ( + to_target (*s, + move (n), move (n.pair ? *++i : o), + ctx.phase != run_phase::load /* in_recipe */)); if (const auto* pt = t.is_a<path_target> ()) { @@ -53,6 +58,10 @@ namespace build2 else fail << "target " << t << " path is not assigned"; } + else if (t.is_a<dir> () || t.is_a<fsdir> ()) + { + r.push_back (t.out_dir ()); + } else fail << "target " << t << " is not path-based"; } @@ -90,7 +99,9 @@ namespace build2 name o; const target& t ( - to_target (*s, move (ns[0]), move (ns[0].pair ? ns[1] : o))); + to_target (*s, + move (ns[0]), move (ns[0].pair ? ns[1] : o), + s->ctx.phase != run_phase::load /* in_recipe */)); if (const auto* et = t.is_a<exe> ()) { diff --git a/libbuild2/install/init.cxx b/libbuild2/install/init.cxx index 3df912f..19c57d4 100644 --- a/libbuild2/install/init.cxx +++ b/libbuild2/install/init.cxx @@ -427,7 +427,7 @@ namespace build2 // Note: ignore config.install.{scope,manifest} (see below). // - bool s (specified_config (rs, "install", {"scope", "manifest"})); + bool s (specified_config (rs, "config.install", {"scope", "manifest"})); // Adjust module priority so that the (numerous) config.install.* // values are saved at the end of config.build. diff --git a/libbuild2/install/operation.cxx b/libbuild2/install/operation.cxx index ce5d24a..029a5f6 100644 --- a/libbuild2/install/operation.cxx +++ b/libbuild2/install/operation.cxx @@ -369,7 +369,10 @@ namespace build2 // files, there is not going to be much speedup from doing it in parallel. // There is also now the installation manifest, which relies on us // installing all the filesystem entries of a target serially. - + // + // Additionally, we stop on first error since there is no sense in + // continuing. + // const operation_info op_install { install_id, 0, @@ -379,7 +382,8 @@ namespace build2 "installed", "has nothing to install", // We cannot "be installed". execution_mode::first, - 0 /* concurrency */, // Run serially. + 0 /* concurrency */, // Run serially. + false /* keep_going */, // Stop on first error. &pre_install, nullptr, &install_pre, @@ -406,7 +410,8 @@ namespace build2 "uninstalled", "is not installed", execution_mode::last, - 0 /* concurrency */, // Run serially + 0 /* concurrency */, // Run serially. + false /* keep_going */, // Stop on first error. &pre_uninstall, nullptr, nullptr, @@ -427,6 +432,7 @@ namespace build2 op_update.name_done, op_update.mode, op_update.concurrency, + op_update.keep_going, op_update.pre_operation, op_update.post_operation, op_update.operation_pre, diff --git a/libbuild2/install/rule.hxx b/libbuild2/install/rule.hxx index b023af5..3dbb68d 100644 --- a/libbuild2/install/rule.hxx +++ b/libbuild2/install/rule.hxx @@ -119,7 +119,7 @@ namespace build2 virtual recipe apply (action, target&, match_extra&) const override; - group_rule (bool sto): see_through_only (sto) {} + group_rule (bool sto = false): see_through_only (sto) {} static const group_rule instance; bool see_through_only; diff --git a/libbuild2/module.cxx b/libbuild2/module.cxx index 1aaa38d..9a12d01 100644 --- a/libbuild2/module.cxx +++ b/libbuild2/module.cxx @@ -96,18 +96,28 @@ namespace build2 nullopt)); /* module_context */ // We use the same context for building any nested modules that might be - // required while building modules. + // required while building modules. Note: this is also used to detect + // module building context. @@ Maybe we should invent special build.mode? // context& mctx (*(ctx.module_context = ctx.module_context_storage->get ())); mctx.module_context = &mctx; + // Copy over any operation callbacks. If a callback implementation does + // not wish to see module context's calls, it can filter them out based on + // the passed context. + // + // Note also that only the callbacks registered before we need to build + // the first module will be in effect. Probably good enough for now. + // + mctx.operation_callbacks = ctx.operation_callbacks; + // Setup the context to perform update. In a sense we have a long-running // perform meta-operation batch (indefinite, in fact, since we never call // the meta-operation's *_post() callbacks) in which we periodically // execute update operations. // // Note that we perform each build in a separate update operation. Failed - // that, if the same target is update twice (which may happen with ad hoc + // that, if the same target is updated twice (which may happen with ad hoc // recipes) we will see the old state. // if (mo_perform.meta_operation_pre != nullptr) @@ -178,18 +188,24 @@ namespace build2 loc, tgs); + // Note that we suppress all outcome diagnostics, even for failure since + // the extra `info: failed to update <target>` is not very useful (we + // expect an appropriate diagnostics frame explains what's going on). + // mo_perform.match ({}, /* parameters */ a, tgs, - 1, /* diag (failures only) */ + 0, /* diag (none) */ false /* progress */); mo_perform.execute ({}, /* parameters */ a, tgs, - 1, /* diag (failures only) */ + 0, /* diag (none) */ false /* progress */); + ctx.module_context->load_generation++; + assert (tgs.size () == 1); return tgs[0].as<target> (); } @@ -470,7 +486,7 @@ namespace build2 // if (nested) { - // This could be initial or exclusive load. + // This could be initial or interrupting load. // // @@ TODO: see the ad hoc recipe case as a reference. // diff --git a/libbuild2/name.hxx b/libbuild2/name.hxx index f5cb2c5..c6aac45 100644 --- a/libbuild2/name.hxx +++ b/libbuild2/name.hxx @@ -136,7 +136,7 @@ namespace build2 // value to dir. Throw invalid_argument if value would become empty. May // also throw invalid_path. // - void + LIBBUILD2_SYMEXPORT void canonicalize (); }; diff --git a/libbuild2/operation.cxx b/libbuild2/operation.cxx index 6f88e38..8b28dd8 100644 --- a/libbuild2/operation.cxx +++ b/libbuild2/operation.cxx @@ -271,9 +271,12 @@ namespace build2 struct monitor_data { size_t incr; - string what; - atomic<timestamp::rep> time {timestamp_nonexistent_rep}; + string what1; + string what2; + size_t exec = 0; // Number of targets executed during match. + timestamp time = timestamp_nonexistent; } md; // Note: must outlive monitor_guard. + scheduler::monitor_guard mg; if (prog && show_progress (2 /* max_verb */)) @@ -282,39 +285,69 @@ namespace build2 // the up-to-date check on some projects (e.g., Boost). So we jump // through a few hoops to make sure we don't overindulge. // + // Note also that the higher the increment, the less accurate our + // executed during match number will be. + // + // Note that we strip the outer operation from "(... during match)" + // not to repeat the same "(for <operation>)" twice. + // md.incr = stderr_term // Scale depending on output type. - ? (ctx.sched->serial () ? 1 : 5) + ? (ctx.sched->serial () ? 1 : 2) : 100; - md.what = " targets to " + diag_do (ctx, a); + md.what1 = " targets to " + diag_do (ctx, a); + md.what2 = ' ' + diag_did (ctx, a.inner_action ()) + " during match)"; mg = ctx.sched->monitor ( ctx.target_count, md.incr, - [&md] (size_t c) -> size_t + [&md, &ctx] (size_t p, size_t c) -> size_t { - size_t r (c + md.incr); + if (p > c) + md.exec += p - c; if (stderr_term) { - timestamp o (duration (md.time.load (memory_order_consume))); timestamp n (system_clock::now ()); - if (n - o < chrono::milliseconds (80)) - return r; + if (n - md.time < chrono::milliseconds (80)) + return md.incr; - md.time.store (n.time_since_epoch ().count (), - memory_order_release); + md.time = n; } diag_progress_lock pl; diag_progress = ' '; diag_progress += to_string (c); - diag_progress += md.what; + diag_progress += md.what1; + + if (md.exec != 0) + { + // Offset by the number of targets skipped. + // + size_t s (ctx.skip_count.load (memory_order_relaxed)); - return r; + if (md.exec > s) + { + diag_progress += " ("; + diag_progress += to_string (md.exec - s); + diag_progress += md.what2; + } + } + + return md.incr; }); } + // Call the pre operation callbacks. + // + // See a comment in perform_execute() for why we are doing it here + // (short answer: phase switches). + // + auto cs (ctx.operation_callbacks.equal_range (a)); + for (auto i (cs.first); i != cs.second; ++i) + if (const auto& f = i->second.pre) + f (ctx, a, ts); + // Start asynchronous matching of prerequisites keeping track of how // many we have started. Wait with unlocked phase to allow phase // switching. @@ -417,7 +450,11 @@ namespace build2 diag_progress.clear (); } - // We are now running serially. Re-examine targets that we have matched. + // We are now running serially. + // + + // Re-examine targets that we have matched and determine whether we have + // failed. // for (size_t j (0); j != n; ++j) { @@ -443,11 +480,8 @@ namespace build2 case target_state::postponed: { // We bailed before matching it (leave state in action_target as - // unknown). + // unknown for the structured result printing). // - if (verb != 0 && diag >= 1) - info << "not " << diag_did (a, t); - break; } case target_state::unknown: @@ -460,9 +494,6 @@ namespace build2 { // Things didn't go well for this target. // - if (verb != 0 && diag >= 1) - info << "failed to " << diag_do (a, t); - at.state = s; fail = true; break; @@ -472,6 +503,36 @@ namespace build2 } } + // Call the post operation callbacks if perform_execute() won't be + // called. + // + if (fail) + perform_post_operation_callbacks (ctx, a, ts, fail); + + // Re-examine targets that we have matched and print diagnostics. + // + if (verb != 0 && diag >= 1) + { + for (size_t j (0); j != n; ++j) + { + action_target& at (ts[j]); + const target& t (at.as<target> ()); + + if (at.state == target_state::failed) + { + // Things didn't go well for this target. + // + info << "failed to " << diag_do (a, t); + } + else if (j >= i || t.matched_state (a) == target_state::postponed) + { + // We bailed before matching it. + // + info << "not " << diag_did (a, t); + } + } + } + if (fail) throw failed (); @@ -602,40 +663,55 @@ namespace build2 switch (ctx.current_inner_oif->concurrency) { case 0: sched_tune = tune_guard (*ctx.sched, 1); break; // Run serially. - case 1: break; // Run as is. - default: assert (false); // Not supported. + case 1: break; // Run as is. + default: assert (false); // Not supported. } + // Override the keep_going flag if requested by the operation. + // + auto kgg = make_guard ([&ctx, o = ctx.keep_going] () + { + ctx.keep_going = o; + }); + if (!ctx.current_inner_oif->keep_going) + ctx.keep_going = false; + // Set the dry-run flag. // ctx.dry_run = ctx.dry_run_option; // Setup progress reporting if requested. // - string what; // Note: must outlive monitor_guard. + struct monitor_data + { + size_t init; + size_t incr; + string what; + } md; // Note: must outlive monitor_guard. + scheduler::monitor_guard mg; if (prog && show_progress (1 /* max_verb */)) { - size_t init (ctx.target_count.load (memory_order_relaxed)); - size_t incr (init > 100 ? init / 100 : 1); // 1%. + md.init = ctx.target_count.load (memory_order_relaxed); + md.incr = md.init > 100 ? md.init / 100 : 1; // 1%. - if (init != incr) + if (md.init != md.incr) { - what = "% of targets " + diag_did (ctx, a); + md.what = "% of targets " + diag_did (ctx, a); mg = ctx.sched->monitor ( ctx.target_count, - init - incr, - [init, incr, &what, &ctx] (size_t c) -> size_t + md.incr, + [&md, &ctx] (size_t, size_t c) -> size_t { - size_t p ((init - c) * 100 / init); + size_t p ((md.init - c) * 100 / md.init); size_t s (ctx.skip_count.load (memory_order_relaxed)); diag_progress_lock pl; diag_progress = ' '; diag_progress += to_string (p); - diag_progress += what; + diag_progress += md.what; if (s != 0) { @@ -644,11 +720,18 @@ namespace build2 diag_progress += " skipped)"; } - return c - incr; + return md.incr; }); } } + // Note that while this would seem like the natural place to call the + // pre operation callbacks, it is actually too late since during match + // we may switch to the execute phase and execute some recipes (think + // building a tool to generate some code). So we have to do this in + // perform_match() and then carefully make sure the post callbacks are + // called for all the exit paths (match failed, match_only, etc). + // In the 'last' execution mode run post hoc first. // if (ctx.current_mode == execution_mode::last) @@ -697,9 +780,44 @@ namespace build2 // We are now running serially. // - // Clear the dry-run flag. + // Re-examine all the targets and determine whether we have failed. // - ctx.dry_run = false; + for (action_target& at: ts) + { + const target& t (at.as<target> ()); + + // Similar to match we cannot attribute post hoc failures to specific + // targets so it seems the best we can do is just fail them all. + // + if (!posthoc_fail) + { + // Note that here we call executed_state() directly instead of + // execute_complete() since we know there is no need to wait. + // + at.state = t.executed_state (a, false /* fail */); + } + else + at.state = /*t.state[a].state =*/ target_state::failed; + + switch (at.state) + { + case target_state::unknown: + case target_state::unchanged: + case target_state::changed: + break; + case target_state::failed: + { + fail = true; + break; + } + default: + assert (false); + } + } + + // Call the post operation callbacks. + // + perform_post_operation_callbacks (ctx, a, ts, fail); // Clear the progress if present. // @@ -709,7 +827,11 @@ namespace build2 diag_progress.clear (); } - // Restore original scheduler settings. + // Clear the dry-run flag. + // + ctx.dry_run = false; + + // Restore original scheduler and keep_going settings. } // Print skip count if not zero. Note that we print it regardless of the @@ -733,19 +855,6 @@ namespace build2 { const target& t (at.as<target> ()); - // Similar to match we cannot attribute post hoc failures to specific - // targets so it seems the best we can do is just fail them all. - // - if (!posthoc_fail) - { - // Note that here we call executed_state() directly instead of - // execute_complete() since we know there is no need to wait. - // - at.state = t.executed_state (a, false /* fail */); - } - else - at.state = /*t.state[a].state =*/ target_state::failed; - switch (at.state) { case target_state::unknown: @@ -780,7 +889,6 @@ namespace build2 if (verb != 0 && diag >= 1) info << "failed to " << diag_do (a, t); - fail = true; break; } default: @@ -956,8 +1064,8 @@ namespace build2 : 0); }; - diag_record dr; - dr << info << "detected unexecuted matched targets:"; + diag_record dr (info); + dr << "detected unexecuted matched targets:"; for (const auto& pt: ctx.targets) { @@ -978,6 +1086,19 @@ namespace build2 #endif } + void + perform_post_operation_callbacks (context& ctx, + action a, + const action_targets& ts, + bool failed) + { + auto cs (ctx.operation_callbacks.equal_range (a)); + + for (auto i (cs.first); i != cs.second; ++i) + if (const auto& f = i->second.post) + f (ctx, a, ts, failed); + } + const meta_operation_info mo_perform { perform_id, "perform", @@ -1391,7 +1512,8 @@ namespace build2 "", "", execution_mode::first, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, nullptr, nullptr, nullptr, @@ -1419,7 +1541,8 @@ namespace build2 "updated", "is up to date", execution_mode::first, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, nullptr, nullptr, nullptr, @@ -1437,7 +1560,8 @@ namespace build2 "cleaned", "is clean", execution_mode::last, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, nullptr, nullptr, nullptr, diff --git a/libbuild2/operation.hxx b/libbuild2/operation.hxx index e8ff38a..4ca1305 100644 --- a/libbuild2/operation.hxx +++ b/libbuild2/operation.hxx @@ -177,9 +177,19 @@ namespace build2 // diagnostics (unless quiet). // LIBBUILD2_SYMEXPORT void - perform_execute (const values&, action, const action_targets&, + perform_execute (const values&, action, action_targets&, uint16_t diag, bool prog); + // Call the context-wide post operation callbacks. Should be called after + // perfrom_match() if perform_execute() will not be called. Note that + // perform_match() handles its own failures but not the match_only case. + // + LIBBUILD2_SYMEXPORT void + perform_post_operation_callbacks (context&, + action, + const action_targets&, + bool failed); + LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_noop; LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_perform; LIBBUILD2_SYMEXPORT extern const meta_operation_info mo_info; @@ -225,6 +235,12 @@ namespace build2 // const size_t concurrency; + // Whether to keep going in case of errors for this operation. If the + // value is false, then the context's keep_going flag is overridden for + // the duration of the operation. + // + const bool keep_going; + // The values argument in the callbacks is the operation parameters. If // the operation expects parameters, then it should have a non-NULL // operation_pre() callback. Failed that, any parameters will be diagnosed diff --git a/libbuild2/parser.cxx b/libbuild2/parser.cxx index e417d39..babe4c9 100644 --- a/libbuild2/parser.cxx +++ b/libbuild2/parser.cxx @@ -2259,9 +2259,9 @@ namespace build2 if (tt != type::word) { - diag_record dr; + diag_record dr (fail (t)); - dr << fail (t) << "unterminated recipe "; + dr << "unterminated recipe "; if (kind.empty ()) dr << "block"; else dr << kind << "-block"; dr << info (st) << "recipe "; @@ -2385,9 +2385,16 @@ namespace build2 // size_t b (0), e (0); for (size_t m (0), n (text.size ()); - next_word (text, n, b, e, m, '\n', '\r'), b != n; - sloc.line++) + next_word (text, n, b, e, m, '\n', '\r'), b != n; ) { + // Treat consecutive \r\n (CRLF) as if they were a single + // delimiter. + // + if (b != 0 && text[b - 1] == '\r' && + e != n && text[e] == '\n' && + m != 2) + continue; + s.assign (text, b, e - b); if (!trim (s).empty ()) @@ -2401,6 +2408,8 @@ namespace build2 break; } } + + sloc.line++; } if (b == e) @@ -2998,8 +3007,8 @@ namespace build2 // rule-specific search) can possibly succeed so we can fail now and // with a more accurate reason. See import2(names) for background. // - diag_record dr; - dr << fail (ploc) << "unable to import target " << n; + diag_record dr (fail (ploc)); + dr << "unable to import target " << n; import_suggest (dr, *n.proj, nullptr, string (), false); } else @@ -3954,7 +3963,7 @@ namespace build2 proj = n.variable (); } - diag_record dr; + maybe_diag_record dr; do // Breakout loop. { if (!config) @@ -4058,7 +4067,7 @@ namespace build2 } while (false); - if (!dr.empty ()) + if (dr) dr << info << "expected variable name in the 'config[.**]." << (proj.empty () ? "<project>" : proj.c_str ()) << ".**' form"; } @@ -5607,7 +5616,7 @@ namespace build2 } d {var, val_attrs, line, block, lhs, is}; - function<void (value&&, bool first)> iteration = + function<bool (value&&, bool first)> iteration = [this, &d] (value&& v, bool first) { // Rewind the stream. @@ -5645,12 +5654,15 @@ namespace build2 << "instead of " << t; lexer_ = ol; + return true; }; if (!iterate) { for (auto b (ns->begin ()), i (b), e (ns->end ()); i != e; ++i) { + bool first (i == b); + // Set the variable value. // bool pair (i->pair); @@ -5662,7 +5674,7 @@ namespace build2 if (etype != nullptr) typify (v, *etype, &var); - iteration (move (v), i == b); + iteration (move (v), first); } } else @@ -5751,9 +5763,9 @@ namespace build2 void parser:: parse_diag (token& t, type& tt) { - diag_record dr; const location l (get_location (t)); + diag_record dr (nullptr); switch (t.value[0]) { case 'f': dr << fail (l); break; @@ -6995,7 +7007,13 @@ namespace build2 // The directory/value must not be empty if we have a type. // if (d.empty () && v.empty () && !t.empty ()) - fail (loc) << "typed empty name"; + { + // We sneak +/- in type if in pattern_mode::detect (think {+{}}). + // + fail (loc) << (t == "+" ? "empty name pattern inclusion group" : + t == "-" ? "empty name pattern exclusion group" : + "typed empty name"); + } ns.emplace_back (move (p), move (d), move (t), move (v), pat); return ns.back (); @@ -7255,7 +7273,9 @@ namespace build2 (root_ != nullptr && root_->root_extra != nullptr && m.to_directory () && - exists (d.sp / m / root_->root_extra->buildignore_file))) + exists (m.relative () + ? d.sp / m / root_->root_extra->buildignore_file + : m / root_->root_extra->buildignore_file))) return !interm; // Note that we have to make copies of the extension since there will @@ -7311,9 +7331,11 @@ namespace build2 return true; }); + path pat (move (p)); + try { - path_search (path (move (p)), + path_search (pat, process, *sp, path_match_flags::follow_symlinks, @@ -7321,7 +7343,8 @@ namespace build2 } catch (const system_error& e) { - fail (l) << "unable to scan " << *sp << ": " << e; + fail (l) << "unable to scan for '" + << (pat.relative () ? *sp / pat : pat) << "': " << e; } }; @@ -7566,7 +7589,10 @@ namespace build2 // and look for some wildcards since the pattern can be the result of an // expansion (or, worse, concatenation). Thus pattern_mode::detect: we // are going to ask parse_names() to detect for us if the first name is - // a pattern. And if it is, to refrain from adding pair/dir/type. + // a pattern. And if it is, to refrain from adding pair/dir/type (note: + // for the pattern inclusions and exclusions the name's type member will + // be set to "+" and "-", respectively, and will later be cleared by + // expand_name_pattern()). // optional<const target_type*> pat_tt ( parse_names ( @@ -8310,16 +8336,42 @@ namespace build2 fail (loc) << "invalid path '" << e.path << "'"; } - count = parse_names_trailer ( - t, tt, ns, pmode, what, separators, pairn, *pp1, dp1, tp1, cross); + // Note that for a pattern inclusion group (see above) we make sure + // that the resulting patterns are simple names, passing NULL as the + // directory path (the names' type members will still be set to "+" + // thought; see the parse_names_trailer::parse() lambda + // implementation for details). + // + assert (!pinc || (tp1 != nullptr && *tp1 == "+")); - // If empty group or empty name, then this is not a pattern inclusion - // group (see above). + count = parse_names_trailer ( + t, tt, + ns, + pmode, + what, + separators, pairn, + *pp1, (!pinc ? dp1 : nullptr), tp1, + cross); + + // If empty group, then this is not a pattern inclusion group. // if (pinc) { - if (count != 0 && (count > 1 || !ns.back ().empty ())) + if (count != 0) + { + // Note that we can never end up with the empty name here. For + // example, for the below constructs the above + // parse_names_trailer() call would fail with appropriate + // diagnostics since the empty name's type will be set to "+" + // (see above for details): + // + // foo/{hxx cxx}{+{}} + // foo/{+{}} + // + assert (count > 1 || !ns.back ().empty ()); + pattern_detected (ttp); + } ppat = pinc = false; } diff --git a/libbuild2/parser.hxx b/libbuild2/parser.hxx index 3e1d0a0..b68bf7e 100644 --- a/libbuild2/parser.hxx +++ b/libbuild2/parser.hxx @@ -893,6 +893,9 @@ namespace build2 // Set the data and start playing. // + // Note that using the const reference version is better than making a + // copy since it may reuse the existing capacity. + // void replay_data (replay_tokens&& d) { @@ -905,6 +908,18 @@ namespace build2 replay_ = replay::play; } + void + replay_data (const replay_tokens& d) + { + assert (replay_ == replay::stop); + + replay_path_ = path_; // Save old path. + + replay_data_ = d; + replay_i_ = 0; + replay_ = replay::play; + } + // Implementation details, don't call directly. // replay_token diff --git a/libbuild2/prerequisite.cxx b/libbuild2/prerequisite.cxx index bb77c9e..ec18665 100644 --- a/libbuild2/prerequisite.cxx +++ b/libbuild2/prerequisite.cxx @@ -91,4 +91,8 @@ namespace build2 return r; } + + // prerequisites + // + const prerequisites empty_prerequisites; } diff --git a/libbuild2/prerequisite.hxx b/libbuild2/prerequisite.hxx index 9b9cccf..008fc11 100644 --- a/libbuild2/prerequisite.hxx +++ b/libbuild2/prerequisite.hxx @@ -190,6 +190,8 @@ namespace build2 } using prerequisites = vector<prerequisite>; + + LIBBUILD2_SYMEXPORT extern const prerequisites empty_prerequisites; } #endif // LIBBUILD2_PREREQUISITE_HXX diff --git a/libbuild2/rule.cxx b/libbuild2/rule.cxx index dc1c96c..b504dc7 100644 --- a/libbuild2/rule.cxx +++ b/libbuild2/rule.cxx @@ -430,9 +430,9 @@ namespace build2 // noop_rule // bool noop_rule:: - match (action, target&) const + match (action, target& t) const { - return true; + return !exclude_group_ || !t.is_a<group> (); } recipe noop_rule:: diff --git a/libbuild2/rule.hxx b/libbuild2/rule.hxx index eceb6ad..71782c0 100644 --- a/libbuild2/rule.hxx +++ b/libbuild2/rule.hxx @@ -209,8 +209,17 @@ namespace build2 virtual recipe apply (action, target&) const override; - noop_rule () {} - static const noop_rule instance; + // If exclude_group is true then exclude the group-based targets (since + // their membership can only be accurately determined by the ad hoc + // recipe). + // + explicit + noop_rule (bool exclude_group = false): exclude_group_ (exclude_group) {} + + static const noop_rule instance; // Note: does not exclude group. + + private: + bool exclude_group_; }; // Ad hoc rule. diff --git a/libbuild2/scheduler.cxx b/libbuild2/scheduler.cxx index 69673e6..ebb38e4 100644 --- a/libbuild2/scheduler.cxx +++ b/libbuild2/scheduler.cxx @@ -701,7 +701,7 @@ namespace build2 } scheduler::monitor_guard scheduler:: - monitor (atomic_count& c, size_t t, function<size_t (size_t)> f) + monitor (atomic_count& c, size_t t, function<size_t (size_t, size_t)> f) { assert (monitor_count_ == nullptr && t != 0); @@ -713,7 +713,7 @@ namespace build2 monitor_count_ = &c; monitor_tshold_.store (t, memory_order_relaxed); - monitor_init_ = c.load (memory_order_relaxed); + monitor_prev_ = c.load (memory_order_relaxed); monitor_func_ = move (f); return monitor_guard (this); diff --git a/libbuild2/scheduler.hxx b/libbuild2/scheduler.hxx index 3cc206e..2d4d189 100644 --- a/libbuild2/scheduler.hxx +++ b/libbuild2/scheduler.hxx @@ -452,9 +452,11 @@ namespace build2 // should be set before any tasks are queued and cleared after all of // them have completed. // - // The counter must go in one direction, either increasing or decreasing, - // and should contain the initial value during the call. Zero threshold - // value is reserved. + // The counter can go in either direction and should contain the initial + // value during the call. The callback function is called when the current + // value differs form the previous/initial by at least the specified + // threshold. The callback returns the new threshold. Zero threshold value + // is reserved. The callback calls are serialized and synchronized. // struct monitor_guard { @@ -488,7 +490,9 @@ namespace build2 }; monitor_guard - monitor (atomic_count&, size_t threshold, function<size_t (size_t)>); + monitor (atomic_count&, + size_t threshold, + function<size_t (size_t previous, size_t current)>); // If initially active thread(s) (besides the one that calls startup()) // exist before the call to startup(), then they must call join() before @@ -598,10 +602,10 @@ namespace build2 // Monitor. // - atomic_count* monitor_count_ = nullptr; // NULL if not used. - atomic_count monitor_tshold_; // 0 means locked. - size_t monitor_init_; // Initial count. - function<size_t (size_t)> monitor_func_; + atomic_count* monitor_count_ = nullptr; // NULL if not used. + atomic_count monitor_tshold_; // 0 means locked. + size_t monitor_prev_; // Previous values. + function<size_t (size_t, size_t)> monitor_func_; build2::mutex mutex_; bool shutdown_ = true; // Shutdown flag. @@ -914,19 +918,19 @@ namespace build2 if (monitor_tshold_.compare_exchange_strong ( t, 0, - memory_order_release, + memory_order_acq_rel, // Synchronize on success. memory_order_relaxed)) { - // Now we are the only ones messing with this. + // Now we are the only ones messing with this and everything + // is synchronized. // + size_t p (monitor_prev_); size_t v (monitor_count_->load (memory_order_relaxed)); - if (v != monitor_init_) + if ((p > v ? p - v : p < v ? v - p : 0) >= t) { - // See which direction we are going. - // - if (v > monitor_init_ ? (v >= t) : (v <= t)) - t = monitor_func_ (v); + t = monitor_func_ (p, v); + monitor_prev_= v; } monitor_tshold_.store (t, memory_order_release); diff --git a/libbuild2/scheduler.txx b/libbuild2/scheduler.txx index 87c9384..15fefcc 100644 --- a/libbuild2/scheduler.txx +++ b/libbuild2/scheduler.txx @@ -29,12 +29,13 @@ namespace build2 // if (monitor_count_ != nullptr) { + size_t t (monitor_tshold_.load (memory_order_relaxed)); + size_t p (monitor_prev_); size_t v (monitor_count_->load (memory_order_relaxed)); - if (v != monitor_init_) + if ((p > v ? p - v : p < v ? v - p : 0) >= t) { - size_t t (monitor_tshold_.load (memory_order_relaxed)); - if (v > monitor_init_ ? (v >= t) : (v <= t)) - monitor_tshold_.store (monitor_func_ (v), memory_order_relaxed); + monitor_tshold_.store (monitor_func_ (p, v), memory_order_relaxed); + monitor_prev_ = v; } } diff --git a/libbuild2/scope.hxx b/libbuild2/scope.hxx index ece78b7..d792ed2 100644 --- a/libbuild2/scope.hxx +++ b/libbuild2/scope.hxx @@ -459,14 +459,20 @@ namespace build2 // when an action is executed on the dir{} target that corresponds to this // scope. The pre callback is called just before the recipe and the post // -- immediately after. The callbacks are only called if the recipe - // (including noop recipe) is executed for the corresponding target. The - // callbacks should only be registered during the load phase. + // (including noop recipe) is executed for the corresponding target. + // + // The callback should only be registered during the load phase. Note that + // it's registered for the inner action, meaning that it will be called + // for any outer action (which is discernible from the first argument of + // the callback). // // It only makes sense for callbacks to return target_state changed or // unchanged and to throw failed in case of an error. These pre/post // states will be merged with the recipe state and become the target // state. See execute_recipe() for details. // + // See also context::operation_callback. + // public: struct operation_callback { diff --git a/libbuild2/script/parser.cxx b/libbuild2/script/parser.cxx index 84d2afc..30c7ff6 100644 --- a/libbuild2/script/parser.cxx +++ b/libbuild2/script/parser.cxx @@ -1668,9 +1668,9 @@ namespace build2 if (es > 255) { - diag_record dr; + diag_record dr (fail (l)); - dr << fail (l) << "expected exit status instead of "; + dr << "expected exit status instead of "; to_stream (dr.os, ns, quote_mode::normal); dr << info << "exit status is an unsigned integer less than 256"; @@ -2218,7 +2218,7 @@ namespace build2 // Copy the tokens and start playing. // - replay_data (replay_tokens (ln.tokens)); + replay_data (ln.tokens); // We don't really need to change the mode since we already know // the line type. @@ -2433,7 +2433,7 @@ namespace build2 // Prepare for the condition reevaluation. // - replay_data (replay_tokens (ln.tokens)); + replay_data (ln.tokens); next (t, tt); li = wli; } @@ -2460,19 +2460,19 @@ namespace build2 // struct loop_data { - lines::const_iterator i; - lines::const_iterator e; const function<exec_set_function>& exec_set; const function<exec_cmd_function>& exec_cmd; const function<exec_cond_function>& exec_cond; const function<exec_for_function>& exec_for; + lines::const_iterator i; + lines::const_iterator e; const iteration_index* ii; size_t& li; variable_pool* var_pool; decltype (fcend)& fce; lines::const_iterator& fe; - } ld {i, e, - exec_set, exec_cmd, exec_cond, exec_for, + } ld {exec_set, exec_cmd, exec_cond, exec_for, + i, e, ii, li, var_pool, fcend, @@ -2558,7 +2558,6 @@ namespace build2 const location& ll; size_t fli; iteration_index& fi; - } d {ld, env, vname, attrs, ll, fli, fi}; function<void (string&&)> f ( @@ -2676,12 +2675,19 @@ namespace build2 if (val) { - // If this value is a vector, then save its element type so + // If the value type provides custom iterate function, then + // use that (see value_type::iterate for details). + // + auto iterate (val.type != nullptr + ? val.type->iterate + : nullptr); + + // If this value is a container, then save its element type so // that we can typify each element below. // const value_type* etype (nullptr); - if (val.type != nullptr) + if (!iterate && val.type != nullptr) { etype = val.type->element_type; @@ -2693,37 +2699,84 @@ namespace build2 size_t fli (li); iteration_index fi {1, ii}; - names& ns (val.as<names> ()); - for (auto ni (ns.begin ()), ne (ns.end ()); ni != ne; ++ni) + names* ns (!iterate ? &val.as<names> () : nullptr); + + // Similar to above. + // + struct loop_data + { + const function<exec_set_function>& exec_set; + const function<exec_cmd_function>& exec_cmd; + const function<exec_cond_function>& exec_cond; + const function<exec_for_function>& exec_for; + lines::const_iterator i; + lines::const_iterator e; + const location& ll; + size_t& li; + variable_pool* var_pool; + const variable& var; + const attributes& val_attrs; + decltype (fcend)& fce; + lines::const_iterator& fe; + iteration_index& fi; + + } ld {exec_set, exec_cmd, exec_cond, exec_for, + i, e, + ll, li, + var_pool, *var, val_attrs, + fcend, fe, fi}; + + function<bool (value&&, bool first)> iteration = + [this, &ld] (value&& v, bool) { - li = fli; + ld.exec_for (ld.var, move (v), ld.val_attrs, ld.ll); - // Set the variable value. + // Find the construct end, if it is not found yet. // - bool pair (ni->pair); - names n; - n.push_back (move (*ni)); - if (pair) n.push_back (move (*++ni)); - value v (move (n)); // Untyped. + if (ld.fe == ld.e) + ld.fe = ld.fce (ld.i, true, false); + + if (!exec_lines ( + ld.i + 1, ld.fe, + ld.exec_set, ld.exec_cmd, ld.exec_cond, ld.exec_for, + &ld.fi, ld.li, + ld.var_pool)) + return false; + + ld.fi.index++; + return true; + }; - if (etype != nullptr) - typify (v, *etype, var); + if (!iterate) + { + for (auto nb (ns->begin ()), ni (nb), ne (ns->end ()); + ni != ne; + ++ni) + { + bool first (ni == nb); - exec_for (*var, move (v), val_attrs, ll); + li = fli; - // Find the construct end, if it is not found yet. - // - if (fe == e) - fe = fcend (i, true, false); + // Set the variable value. + // + bool pair (ni->pair); + names n; + n.push_back (move (*ni)); + if (pair) n.push_back (move (*++ni)); + value v (move (n)); // Untyped. - if (!exec_lines (i + 1, fe, - exec_set, exec_cmd, exec_cond, exec_for, - &fi, li, - var_pool)) - return false; + if (etype != nullptr) + typify (v, *etype, var); - fi.index++; + if (!iteration (move (v), first)) + return false; + } + } + else + { + if (!iterate (val, iteration)) + return false; } } diff --git a/libbuild2/script/run.cxx b/libbuild2/script/run.cxx index f8f98c1..31e4537 100644 --- a/libbuild2/script/run.cxx +++ b/libbuild2/script/run.cxx @@ -1301,7 +1301,6 @@ namespace build2 // Terminate processes gracefully and set the terminate flag for the // pipe commands. // - diag_record dr; for (pipe_command* c (pc); c != nullptr; c = c->prev) { if (process* p = c->proc) @@ -1324,6 +1323,10 @@ namespace build2 c->terminated = true; } + // Delay any failure until we process the entire pipeline. + // + maybe_diag_record dr; + // Wait a bit for the processes to terminate and kill the remaining // ones. // @@ -1810,6 +1813,24 @@ namespace build2 // const string& program (c.program.recall.string ()); + // Optimize standalone true/false builtins (used in conditions, etc). We + // know they don't read/write stdin/stdout/stderr so none of the below + // complications are necessary. + // + if (resolve && (program == "true" || program == "false") && + first && last && cf == nullptr && + !c.in && !c.out && !c.err && !c.exit) + { + // Note: we can safely ignore agruments, env vars, and timeout. + // + // Don't print the true and false builtins, since they are normally + // used for the commands execution flow control (also below). + // + return program == "true"; + } + + // Standard streams. + // const redirect& in ((c.in ? *c.in : env.in).effective ()); const redirect* out (!last || (cf != nullptr && !last_cmd) @@ -1818,6 +1839,36 @@ namespace build2 const redirect& err ((c.err ? *c.err : env.err).effective ()); + // If the output redirect is `none` (default for testscript) or `null` + // and this is a builtin which is known not to write to stdout, then + // redirect its stdout to stderr, unless stderr itself is redirected to + // stdout. This way we will give up an expensive fdopen() in favor of a + // cheap fddup(). + // + // Note that we ignore the pseudo builtins since they are handled prior + // to opening any file descriptors for stdout. + // + const redirect out_merge (redirect_type::merge, 2); + if (out != nullptr && + (out->type == redirect_type::none || + out->type == redirect_type::null) && + err.type != redirect_type::merge && + resolve && + (program == "cp" || + program == "false" || + program == "ln" || + program == "mkdir" || + program == "mv" || + program == "rm" || + program == "rmdir" || + program == "sleep" || + program == "test" || + program == "touch" || + program == "true")) + { + out = &out_merge; + } + auto process_args = [&c] () -> cstrings { return build2::process_args (c.program.recall_string (), c.arguments); @@ -1964,11 +2015,14 @@ namespace build2 // If this is the first pipeline command, then open stdin descriptor // according to the redirect specified. // + // Specifically, save into ifd the descriptor of a newly created file, + // existing file, stdin descriptor duplicate, or null-device descriptor + // for the here_*_literal, file, pass, or null/none redirects + // respectively (not creating any pipe). + // path isp; - if (!first) - assert (!c.in); // No redirect expected. - else + if (first) { // Open a file for passing to the command stdin. // @@ -2002,25 +2056,58 @@ namespace build2 break; } case redirect_type::none: - // Somehow need to make sure that the child process doesn't read - // from stdin. That is tricky to do in a portable way. Here we - // suppose that the program which (erroneously) tries to read some - // data from stdin being redirected to /dev/null fails not being - // able to read the expected data, and so the command doesn't pass - // through. - // - // @@ Obviously doesn't cover the case when the process reads - // whatever available. - // @@ Another approach could be not to redirect stdin and let the - // process to hang which can be interpreted as a command failure. - // @@ Both ways are quite ugly. Is there some better way to do - // this? - // @@ Maybe we can create a pipe, write a byte into it, close the - // writing end, and after the process terminates make sure we can - // still read this byte out? - // + { + // Somehow need to make sure that the child process doesn't read + // from stdin. That is tricky to do in a portable way. Here we + // suppose that the program which (erroneously) tries to read some + // data from stdin being redirected to /dev/null fails not being + // able to read the expected data, and so the command doesn't pass + // through. + // + // @@ Obviously doesn't cover the case when the process reads + // whatever available. + // @@ Another approach could be not to redirect stdin and let the + // process to hang which can be interpreted as a command + // failure. + // @@ Both ways are quite ugly. Is there some better way to do + // this? + // @@ Maybe we can create a pipe, write a byte into it, close the + // writing end, and after the process terminates make sure we + // can still read this byte out? + // + // Let's optimize for the builtins, which are known not to read + // any input, by not opening /dev/null but duplicating the stdin + // file descriptor instead. + // + if (resolve && + (program == "cp" || + program == "date" || + program == "echo" || + program == "false" || + program == "find" || + program == "ln" || + program == "mkdir" || + program == "mv" || + program == "rm" || + program == "rmdir" || + program == "sleep" || + program == "test" || + program == "touch" || + program == "true")) + { + try + { + ifd = fddup (0); + } + catch (const io_error& e) + { + fail (ll) << "unable to duplicate stdin: " << e; + } + + break; + } + } // Fall through. - // case redirect_type::null: { ifd = open_null (); @@ -2057,8 +2144,10 @@ namespace build2 case redirect_type::here_doc_ref: assert (false); break; } } + else + assert (!c.in); // No redirect expected. - assert (ifd.get () != -1); + assert (ifd != nullfd); // Calculate the process/builtin execution deadline. Note that we should // also consider the left-hand side processes deadlines, not to keep @@ -2152,14 +2241,16 @@ namespace build2 else pc.next = &pc; // Points to itself. - // Open a file for command output redirect if requested explicitly - // (file overwrite/append redirects) or for the purpose of the output + // Open a file for command output redirect if requested explicitly (file + // overwrite/append redirects) or for the purpose of the output // validation (none, here_*, file comparison redirects), register the // file for cleanup, return the file descriptor. Interpret trace // redirect according to the verbosity level (as null if below 2, as - // pass otherwise). Return nullfd, standard stream descriptor duplicate - // or null-device descriptor for merge, pass or null redirects - // respectively (not opening any file). + // pass otherwise). Return nullfd, standard stream descriptor duplicate, + // or null-device descriptor for merge, pass (except for the buffered + // stderr), or null redirects respectively (not opening/creating any + // file/pipe). Create the pipe and return its write end for the pass + // redirect of the buffered stderr. // auto open = [&env, &wdir, &ll, &std_path, &c, &pc] (const redirect& r, int dfd, @@ -2276,10 +2367,12 @@ namespace build2 path osp; fdpipe ofd; - // If this is the last command in the pipeline than redirect the - // command process stdout to a file. Otherwise create a pipe and - // redirect the stdout to the write-end of the pipe. The read-end will - // be passed as stdin for the next command in the pipeline. + // If this is either not the last command in the pipeline or the + // command's output needs to be read by the specified function, then + // create a pipe and redirect the stdout to the write-end of the + // pipe. The read-end will be passed as stdin for the next command in + // the pipeline or the function. Otherwise, proceed according to the + // specified redirect (see open() lambda for details). // // @@ Shouldn't we allow the here-* and file output redirects for a // command with pipelined output? Say if such redirect is present @@ -2290,16 +2383,21 @@ namespace build2 // script failures investigation and, for example, for validation // "tightening". // - if (last && out != nullptr) - ofd.out = open (*out, 1, osp); - else + if (!last || out == nullptr) { assert (!c.out); // No redirect expected. + + // Otherwise we wouldn't be here. + // + assert (!last || (cf != nullptr && !last_cmd)); + ofd = open_pipe (); } + else + ofd.out = open (*out, 1, osp); // Note: may or may not open file. path esp; - auto_fd efd (open (err, 2, esp)); + auto_fd efd (open (err, 2, esp)); // Note: may or may not open file/pipe. // Merge standard streams. // @@ -2547,7 +2645,7 @@ namespace build2 // Verify the exit status and issue the diagnostics on failure. // - diag_record dr; + maybe_diag_record dr; path pr (cmd_path (cmd)); @@ -2563,16 +2661,16 @@ namespace build2 if (c->unread_stdout) { - dr << "stdout "; + *dr << "stdout "; if (c->unread_stderr) - dr << "and "; + *dr << "and "; } if (c->unread_stderr) - dr << "stderr "; + *dr << "stderr "; - dr << "not closed after exit"; + *dr << "not closed after exit"; }; // Fail if the process is terminated due to reaching the deadline. @@ -2588,7 +2686,7 @@ namespace build2 if (verb == 1) { dr << info << "command line: "; - print_process (dr, *c->args); + print_process (*dr, *c->args); } fail = true; @@ -2643,23 +2741,23 @@ namespace build2 dr << error (ll) << w << ' ' << pr << ' '; if (!exit->normal ()) - dr << *exit; + *dr << *exit; else { uint16_t ec (exit->code ()); // Make sure printed as integer. if (!valid) { - dr << "exit code " << ec << " out of 0-255 range"; + *dr << "exit code " << ec << " out of 0-255 range"; } else { if (cmd.exit) - dr << "exit code " << ec - << (cmp == exit_comparison::eq ? " != " : " == ") - << exc; + *dr << "exit code " << ec + << (cmp == exit_comparison::eq ? " != " : " == ") + << exc; else - dr << "exited with code " << ec; + *dr << "exited with code " << ec; } } @@ -2669,7 +2767,7 @@ namespace build2 if (verb == 1) { dr << info << "command line: "; - print_process (dr, *c->args); + print_process (*dr, *c->args); } if (non_empty (*c->esp, ll) && avail_on_failure (*c->esp, env)) @@ -2683,7 +2781,7 @@ namespace build2 // Print cached stderr. // - print_file (dr, *c->esp, ll); + print_file (*dr, *c->esp, ll); } else if (c->unread_stdout || c->unread_stderr) unread_output_diag (true /* main_error */); @@ -2754,7 +2852,7 @@ namespace build2 // Execute the builtin. // // Don't print the true and false builtins, since they are normally - // used for the commands execution flow control. + // used for the commands execution flow control (also above). // if (verb >= 2 && program != "true" && program != "false") print_process (args); diff --git a/libbuild2/script/run.hxx b/libbuild2/script/run.hxx index c4c2aa2..955c37e 100644 --- a/libbuild2/script/run.hxx +++ b/libbuild2/script/run.hxx @@ -39,7 +39,9 @@ namespace build2 // can be used in diagnostics. // // Optionally, execute the specified function at the end of the pipe, - // either after the last command or instead of it. + // either after the last command (last_cmd=false) or instead of it + // (last_cmd=true). Note that the last_cmd argument is only meaningful if + // the function is specified. // void run (environment&, diff --git a/libbuild2/target-state.hxx b/libbuild2/target-state.hxx index a6106f7..df54876 100644 --- a/libbuild2/target-state.hxx +++ b/libbuild2/target-state.hxx @@ -25,7 +25,8 @@ namespace build2 // enum class target_state: uint8_t { - unknown = 1, + uninitialized = 0, + unknown, unchanged, postponed, busy, diff --git a/libbuild2/target.cxx b/libbuild2/target.cxx index 65e18d3..b915b25 100644 --- a/libbuild2/target.cxx +++ b/libbuild2/target.cxx @@ -76,8 +76,6 @@ namespace build2 // target // - const target::prerequisites_type target::empty_prerequisites_; - target:: ~target () { diff --git a/libbuild2/target.hxx b/libbuild2/target.hxx index b008347..1b7b755 100644 --- a/libbuild2/target.hxx +++ b/libbuild2/target.hxx @@ -733,8 +733,6 @@ namespace build2 atomic<uint8_t> prerequisites_state_ {0}; prerequisites_type prerequisites_; - static const prerequisites_type empty_prerequisites_; - // Target-specific variables. // // See also rule-specific variables below. @@ -953,10 +951,13 @@ namespace build2 mutable bool recipe_keep; // Keep after execution. bool recipe_group_action; // Recipe is group_action. - // Target state for this operation. Note that it is undetermined until - // a rule is matched and recipe applied (see set_recipe()). + // Target state for this operation. + // + // Note that it is undetermined until a rule is matched and recipe + // applied (see set_recipe()). However, we need it to be not postponed + // for ad hoc members that are not matched (see group_state()). // - target_state state; + target_state state = target_state::uninitialized; // Set to true (only for the inner action) if this target has been // matched but not executed as a result of the resolve_members() call. @@ -1485,9 +1486,9 @@ namespace build2 { public: explicit - group_prerequisites (const target& t); + group_prerequisites (const target&); - group_prerequisites (const target& t, const target* g); + group_prerequisites (const target&, const target* group); using prerequisites_type = target::prerequisites_type; using base_iterator = prerequisites_type::const_iterator; @@ -1501,8 +1502,8 @@ namespace build2 using iterator_category = std::bidirectional_iterator_tag; iterator () {} - iterator (const target* t, - const target* g, + iterator (const prerequisites_type* t, + const prerequisites_type* g, const prerequisites_type* c, base_iterator i): t_ (t), g_ (g), c_ (c), i_ (i) {} @@ -1531,8 +1532,8 @@ namespace build2 operator!= (const iterator& x, const iterator& y) {return !(x == y);} private: - const target* t_ = nullptr; - const target* g_ = nullptr; + const prerequisites_type* t_ = nullptr; + const prerequisites_type* g_ = nullptr; const prerequisites_type* c_ = nullptr; base_iterator i_; }; @@ -1555,8 +1556,8 @@ namespace build2 size () const; private: - const target& t_; - const target* g_; + const prerequisites_type* t_; // NULL if empty. + const prerequisites_type* g_; // NULL if no group or empty. }; // A member of a prerequisite. If 'member' is NULL, then this is the diff --git a/libbuild2/target.ixx b/libbuild2/target.ixx index 39b81e7..1dc667d 100644 --- a/libbuild2/target.ixx +++ b/libbuild2/target.ixx @@ -207,7 +207,7 @@ namespace build2 { return prerequisites_state_.load (memory_order_acquire) == 2 ? prerequisites_ - : empty_prerequisites_; + : empty_prerequisites; } inline bool target:: @@ -286,6 +286,7 @@ namespace build2 // @@ Hm, I wonder why not just return s.recipe_group_action now that we // cache it. // + const opstate& s (state[a]); // This special hack allows us to do things like query an ad hoc member's // state or mtime without matching/executing the member, only the group. @@ -294,12 +295,15 @@ namespace build2 // execute phase). // // Note: this test must come first since the member may not be matched and - // thus its state uninitialized. + // thus its state set (but it won't be postponed; see opstate::state). // if (ctx.phase == run_phase::execute && adhoc_group_member ()) - return true; - - const opstate& s (state[a]); + { + // Note: if the member state is postponed, then the group state may not + // be yet known (see group_action() for details). + // + return s.state != target_state::postponed; + } if (s.state == target_state::group) return true; @@ -342,7 +346,14 @@ namespace build2 inline target_state target:: executed_state_impl (action a) const { - return (group_state (a) ? group->state : state)[a].state; + target_state ts ((group_state (a) ? group->state : state)[a].state); + + // Translate postponed to unchanged, similar to execute_recipe(). + // + if (ts == target_state::postponed) + ts = target_state::unchanged; + + return ts; } inline target_state target:: @@ -480,42 +491,64 @@ namespace build2 // inline group_prerequisites:: group_prerequisites (const target& t) - : t_ (t), - g_ (t_.group == nullptr || - t_.group->adhoc_member != nullptr || // Ad hoc group member. - t_.group->prerequisites ().empty () - ? nullptr : t_.group) + : t_ (nullptr), g_ (nullptr) { + // Take "snapshot" of prerequisites, both for target and group. + // + const auto& ps (t.prerequisites ()); + if (!ps.empty ()) + t_ = &ps; + + if (const target* g = t.group) + { + if (g->adhoc_member == nullptr) // Not ad hoc group member. + { + const auto& ps (g->prerequisites ()); + if (!ps.empty ()) + g_ = &ps; + } + } } inline group_prerequisites:: group_prerequisites (const target& t, const target* g) - : t_ (t), - g_ (g == nullptr || - g->prerequisites ().empty () - ? nullptr : g) + : t_ (nullptr), g_ (nullptr) { + const auto& ps (t.prerequisites ()); + if (!ps.empty ()) + t_ = &ps; + + if (g != nullptr) + { + const auto& ps (g->prerequisites ()); + if (!ps.empty ()) + g_ = &ps; + } } inline auto group_prerequisites:: begin () const -> iterator { - auto& c ((g_ != nullptr ? *g_ : t_).prerequisites ()); - return iterator (&t_, g_, &c, c.begin ()); + auto* c (g_ != nullptr ? g_ : + t_ != nullptr ? t_ : + &empty_prerequisites); + return iterator (t_, g_, c, c->begin ()); } inline auto group_prerequisites:: end () const -> iterator { - auto& c (t_.prerequisites ()); - return iterator (&t_, g_, &c, c.end ()); + auto* c (t_ != nullptr ? t_ : + g_ != nullptr ? g_ : + &empty_prerequisites); + return iterator (t_, g_, c, c->end ()); } inline size_t group_prerequisites:: size () const { - return t_.prerequisites ().size () + - (g_ != nullptr ? g_->prerequisites ().size () : 0); + return ((t_ != nullptr ? t_->size () : 0) + + (g_ != nullptr ? g_->size () : 0)); } // group_prerequisites::iterator @@ -523,9 +556,9 @@ namespace build2 inline auto group_prerequisites::iterator:: operator++ () -> iterator& { - if (++i_ == c_->end () && c_ != &t_->prerequisites ()) + if (++i_ == c_->end () && c_ == g_ && t_ != nullptr) { - c_ = &t_->prerequisites (); + c_ = t_; i_ = c_->begin (); } return *this; @@ -535,9 +568,9 @@ namespace build2 inline auto group_prerequisites::iterator:: operator-- () -> iterator& { - if (i_ == c_->begin () && c_ == &t_->prerequisites ()) + if (i_ == c_->begin () && c_ == t_) { - c_ = &g_->prerequisites (); + c_ = g_; i_ = c_->end (); } diff --git a/libbuild2/test/operation.cxx b/libbuild2/test/operation.cxx index 2535adb..640d2fe 100644 --- a/libbuild2/test/operation.cxx +++ b/libbuild2/test/operation.cxx @@ -63,7 +63,8 @@ namespace build2 "tested", "has nothing to test", // We cannot "be tested". execution_mode::first, - 1 /* concurrency */, + 1 /* concurrency */, + true /* keep_going */, &pre_test, nullptr, nullptr, @@ -84,6 +85,7 @@ namespace build2 op_update.name_done, op_update.mode, op_update.concurrency, + op_update.keep_going, op_update.pre_operation, op_update.post_operation, op_update.operation_pre, diff --git a/libbuild2/test/rule.cxx b/libbuild2/test/rule.cxx index 28eb35b..b3a73ce 100644 --- a/libbuild2/test/rule.cxx +++ b/libbuild2/test/rule.cxx @@ -745,7 +745,9 @@ namespace build2 // auto term_pipe = [&timed_wait] (pipe_process* pp) { - diag_record dr; + // Delay the failure until we process the entire pipeline. + // + maybe_diag_record dr; // Terminate processes gracefully and set the terminate flag for // them. @@ -962,7 +964,7 @@ namespace build2 // if (!fail) { - diag_record dr; + maybe_diag_record dr; // Note that there can be a race, so that the process we have // terminated due to reaching the deadline has in fact exited @@ -983,14 +985,14 @@ namespace build2 if (!pe) { - dr << "terminated: execution timeout expired"; + *dr << "terminated: execution timeout expired"; if (p->unread_stderr) dr << error << "stderr not closed after exit"; } else if (!pe->normal () || pe->code () != 0) { - dr << *pe; + *dr << *pe; if (p->unread_stderr) dr << error << "stderr not closed after exit"; @@ -999,7 +1001,7 @@ namespace build2 { assert (p->unread_stderr); - dr << "stderr not closed after exit"; + *dr << "stderr not closed after exit"; } if (verb == 1) @@ -1009,9 +1011,9 @@ namespace build2 for (pipe_process* p (b); p != nullptr; p = p->next) { if (p != b) - dr << " | "; + *dr << " | "; - print_process (dr, p->args); + print_process (*dr, p->args); } } } diff --git a/libbuild2/utility.cxx b/libbuild2/utility.cxx index ae7c9b0..83947b2 100644 --- a/libbuild2/utility.cxx +++ b/libbuild2/utility.cxx @@ -364,8 +364,8 @@ namespace build2 // // Note: make sure keep the above trace if decide not to print. // - diag_record dr; - dr << error (loc) << "process " << args[0] << " " << pe; + diag_record dr (error (loc)); + dr << "process " << args[0] << " " << pe; if (verb >= 1 && verb <= v) { diff --git a/libbuild2/utility.hxx b/libbuild2/utility.hxx index f4fd7bc..99d6806 100644 --- a/libbuild2/utility.hxx +++ b/libbuild2/utility.hxx @@ -73,7 +73,9 @@ namespace build2 using butl::icase_compare_string; using butl::icase_compare_c_string; using butl::lcase; + using butl::make_lcase; using butl::ucase; + using butl::make_ucase; using butl::alpha; using butl::alnum; using butl::digit; @@ -82,6 +84,7 @@ namespace build2 using butl::trim; using butl::next_word; using butl::sanitize_identifier; + using butl::make_sanitized_identifier; using butl::sanitize_strlit; using butl::make_guard; diff --git a/libbuild2/variable.cxx b/libbuild2/variable.cxx index fb9e840..086fa7a 100644 --- a/libbuild2/variable.cxx +++ b/libbuild2/variable.cxx @@ -1872,8 +1872,8 @@ namespace build2 // // Note: the same diagnostics as in $json.serialize(). // - diag_record dr; - dr << fail << "invalid json value: " << e; + diag_record dr (fail); + dr << "invalid json value: " << e; if (e.event) dr << info << "while serializing " << to_string (*e.event); @@ -2099,8 +2099,8 @@ namespace build2 // We will likely be trying to interpret a member name as an integer // due to the incorrect value type so issue appropriate diagnostics. // - diag_record dr; - dr << fail (sloc) << "invalid json value subscript: " << e; + diag_record dr (fail (sloc)); + dr << "invalid json value subscript: " << e; if (jv != nullptr && jv->type != json_type::object) dr << info << "json value type is " << jv->type; @@ -2122,9 +2122,9 @@ namespace build2 return r; } - static void + static bool json_iterate (const value& val, - const function<void (value&&, bool first)>& f) + const function<bool (value&&, bool first)>& f) { // Implement in terms of subscript for consistency (in particular, // iterating over simple values like number, string). @@ -2136,8 +2136,11 @@ namespace build2 if (!e.second) break; - f (move (e.first), i == 0); + if (!f (move (e.first), i == 0)) + return false; } + + return true; } const json_value value_traits<json_value>::empty_instance; @@ -3317,10 +3320,13 @@ namespace build2 value_traits<vector<pair<string, optional<string>>>>; template struct LIBBUILD2_DEFEXPORT + value_traits<vector<pair<string, optional<bool>>>>; + + template struct LIBBUILD2_DEFEXPORT value_traits<vector<pair<optional<string>, string>>>; template struct LIBBUILD2_DEFEXPORT - value_traits<vector<pair<string, optional<bool>>>>; + value_traits<vector<pair<optional<string>, bool>>>; template struct LIBBUILD2_DEFEXPORT value_traits<set<string>>; template struct LIBBUILD2_DEFEXPORT value_traits<set<json_value>>; diff --git a/libbuild2/variable.hxx b/libbuild2/variable.hxx index eebb767..e55a121 100644 --- a/libbuild2/variable.hxx +++ b/libbuild2/variable.hxx @@ -124,12 +124,13 @@ namespace build2 const location& sloc, const location& bloc); - // Custom iteration function. It should invoked the specified function for + // Custom iteration function. It should invoke the specified function for // each element in order. If NULL, then the generic implementation is - // used. The passed value is never NULL. + // used. The passed value is never NULL. If the specified function returns + // false, then stop the iteration and return false. Otherwise return true. // - void (*const iterate) (const value&, - const function<void (value&&, bool first)>&); + bool (*const iterate) (const value&, + const function<bool (value&&, bool first)>&); }; // The order of the enumerators is arranged so that their integral values @@ -1379,6 +1380,9 @@ namespace build2 extern template struct LIBBUILD2_DECEXPORT value_traits<vector<pair<string, optional<bool>>>>; + extern template struct LIBBUILD2_DECEXPORT + value_traits<vector<pair<optional<string>, bool>>>; + extern template struct LIBBUILD2_DECEXPORT value_traits<set<string>>; extern template struct LIBBUILD2_DECEXPORT value_traits<set<json_value>>; diff --git a/libbuild2/variable.txx b/libbuild2/variable.txx index a1ee340..1d27ec7 100644 --- a/libbuild2/variable.txx +++ b/libbuild2/variable.txx @@ -135,30 +135,34 @@ namespace build2 { size_t n (ns.size ()); - diag_record dr; - if (value_traits<T>::empty_value ? n <= 1 : n == 1) + try { - try + if (value_traits<T>::empty_value ? n <= 1 : n == 1) { + // Throws invalid argument with complete diagnostics, in which case ns + // is not moved from. + // value_traits<T>::assign ( v, (n == 0 ? T () : value_traits<T>::convert (move (ns.front ()), nullptr))); } - catch (const invalid_argument& e) - { - dr << fail << e; - } + else + throw invalid_argument (""); } - else - dr << fail << "invalid " << value_traits<T>::value_type.name - << " value: " << (n == 0 ? "empty" : "multiple names"); - - if (!dr.empty ()) + catch (const invalid_argument& e) { + diag_record dr (fail); + + if (*e.what () != '\0') + dr << e; + else + dr << "invalid " << value_traits<T>::value_type.name << " value: " + << (n == 0 ? "empty value" : "multiple names"); + if (var != nullptr) - dr << " in variable " << var->name; + dr << " in variable " << var->name; // Note: appended to above line. dr << info << "while converting '" << ns << "'"; } @@ -170,30 +174,34 @@ namespace build2 { size_t n (ns.size ()); - diag_record dr; - if (value_traits<T>::empty_value ? n <= 1 : n == 1) + try { - try + if (value_traits<T>::empty_value ? n <= 1 : n == 1) { + // Throws invalid argument with complete diagnostics, in which case ns + // is not moved from. + // value_traits<T>::append ( v, (n == 0 ? T () : value_traits<T>::convert (move (ns.front ()), nullptr))); } - catch (const invalid_argument& e) - { - dr << fail << e; - } + else + throw invalid_argument (""); } - else - dr << fail << "invalid " << value_traits<T>::value_type.name - << " value: " << (n == 0 ? "empty" : "multiple names"); - - if (!dr.empty ()) + catch (const invalid_argument& e) { + diag_record dr (fail); + + if (*e.what () != '\0') + dr << e; + else + dr << "invalid " << value_traits<T>::value_type.name << " value: " + << (n == 0 ? "empty value" : "multiple names"); + if (var != nullptr) - dr << " in variable " << var->name; + dr << " in variable " << var->name; // Note: appended to above line. dr << info << "while converting '" << ns << "'"; } @@ -205,30 +213,34 @@ namespace build2 { size_t n (ns.size ()); - diag_record dr; - if (value_traits<T>::empty_value ? n <= 1 : n == 1) + try { - try + if (value_traits<T>::empty_value ? n <= 1 : n == 1) { + // Throws invalid argument with complete diagnostics, in which case ns + // is not moved from. + // value_traits<T>::prepend ( v, (n == 0 ? T () : value_traits<T>::convert (move (ns.front ()), nullptr))); } - catch (const invalid_argument& e) - { - dr << fail << e; - } + else + throw invalid_argument (""); } - else - dr << fail << "invalid " << value_traits<T>::value_type.name - << " value: " << (n == 0 ? "empty" : "multiple names"); - - if (!dr.empty ()) + catch (const invalid_argument& e) { + diag_record dr (fail); + + if (*e.what () != '\0') + dr << e; + else + dr << "invalid " << value_traits<T>::value_type.name << " value: " + << (n == 0 ? "empty value" : "multiple names"); + if (var != nullptr) - dr << " in variable " << var->name; + dr << " in variable " << var->name; // Note: appended to above line. dr << info << "while converting '" << ns << "'"; } @@ -686,16 +698,19 @@ namespace build2 // Provide iterate for vector<T> for efficiency. // template <typename T> - void + bool vector_iterate (const value& val, - const function<void (value&&, bool first)>& f) + const function<bool (value&&, bool first)>& f) { const auto& v (val.as<vector<T>> ()); // Never NULL. for (auto b (v.begin ()), i (b), e (v.end ()); i != e; ++i) { - f (value (*i), i == b); + if (!f (value (*i), i == b)) + return false; } + + return true; } // Make sure these are static-initialized together. Failed that VC will make @@ -1071,16 +1086,19 @@ namespace build2 // Provide iterate for set<T> for efficiency. // template <typename T> - void + bool set_iterate (const value& val, - const function<void (value&&, bool first)>& f) + const function<bool (value&&, bool first)>& f) { const auto& v (val.as<set<T>> ()); // Never NULL. for (auto b (v.begin ()), i (b), e (v.end ()); i != e; ++i) { - f (value (*i), i == b); + if (!f (value (*i), i == b)) + return false; } + + return true; } // Make sure these are static-initialized together. Failed that VC will make |