diff options
Diffstat (limited to 'libbuild2/cc')
| -rw-r--r-- | libbuild2/cc/common.cxx | 12 | ||||
| -rw-r--r-- | libbuild2/cc/compile-rule.cxx | 84 | ||||
| -rw-r--r-- | libbuild2/cc/compiledb.cxx | 1100 | ||||
| -rw-r--r-- | libbuild2/cc/compiledb.hxx | 236 | ||||
| -rw-r--r-- | libbuild2/cc/guess.cxx | 42 | ||||
| -rw-r--r-- | libbuild2/cc/guess.hxx | 3 | ||||
| -rw-r--r-- | libbuild2/cc/init.cxx | 617 | ||||
| -rw-r--r-- | libbuild2/cc/link-rule.cxx | 447 | ||||
| -rw-r--r-- | libbuild2/cc/module.cxx | 9 | ||||
| -rw-r--r-- | libbuild2/cc/module.hxx | 61 | ||||
| -rw-r--r-- | libbuild2/cc/msvc.cxx | 12 | ||||
| -rw-r--r-- | libbuild2/cc/pkgconfig.cxx | 8 | ||||
| -rw-r--r-- | libbuild2/cc/windows-manifest.cxx | 5 |
13 files changed, 2514 insertions, 122 deletions
diff --git a/libbuild2/cc/common.cxx b/libbuild2/cc/common.cxx index 9a4a07c..fcc8961 100644 --- a/libbuild2/cc/common.cxx +++ b/libbuild2/cc/common.cxx @@ -185,7 +185,7 @@ namespace build2 // const string* pt ( cast_null<string> ( - l.state[a].lookup_original (c_type, true /* target_only */).first)); + l.state[a].lookup_original (c_type, lookup_limit::target).first)); // cc.type value format is <lang>[,...]. // @@ -242,7 +242,7 @@ namespace build2 // { const variable& v (impl ? c_export_impl_libs : c_export_libs); - c_e_libs = l.lookup_original (v, false, &bs).first; + c_e_libs = l.lookup_original (v, &bs).first; } if (!cc) @@ -251,7 +251,7 @@ namespace build2 same ? (impl ? x_export_impl_libs : x_export_libs) : vp[t + (impl ? ".export.impl_libs" : ".export.libs")]); - x_e_libs = l.lookup_original (v, false, &bs).first; + x_e_libs = l.lookup_original (v, &bs).first; } // Process options first. @@ -669,7 +669,7 @@ namespace build2 // See the link rule for the lookup semantics. // lookup l ( - t->lookup_original (var, true /* target_only */).first); + t->lookup_original (var, lookup_limit::target).first); if (l ? cast<bool> (*l) : u) lf |= lflag_whole; @@ -778,8 +778,8 @@ namespace build2 if (proc_lib) { const variable& v (same ? x_libs : vp[t + ".libs"]); - proc_impl (l.lookup_original (c_libs, false, &bs).first); - proc_impl (l.lookup_original (v, false, &bs).first); + proc_impl (l.lookup_original (c_libs, &bs).first); + proc_impl (l.lookup_original (v, &bs).first); } } } diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx index 95ba89f..29a26b5 100644 --- a/libbuild2/cc/compile-rule.cxx +++ b/libbuild2/cc/compile-rule.cxx @@ -25,6 +25,7 @@ #include <libbuild2/cc/target.hxx> // h #include <libbuild2/cc/module.hxx> #include <libbuild2/cc/utility.hxx> +#include <libbuild2/cc/compiledb.hxx> using std::exit; using std::strlen; @@ -1181,6 +1182,11 @@ namespace build2 fsdir_rule::perform_update_direct (a, *dir); } + // Use the subset of the depdb checks to detect changes to the + // compilation database entry. + // + bool compiledb_changed (false); + // Note: the leading '@' is reserved for the module map prefix (see // extract_modules()) and no other line must start with it. // @@ -1198,8 +1204,14 @@ namespace build2 // but only in what it targets, then the checksum will still change. // if (dd.expect (cast<string> (rs[x_checksum])) != nullptr) + { l4 ([&]{trace << "compiler mismatch forcing update of " << t;}); + // The checksum includes the absolute compiler path. + // + compiledb_changed = true; + } + // Then the compiler environment checksum. // if (dd.expect (env_checksum) != nullptr) @@ -1263,7 +1275,17 @@ namespace build2 append_sys_hdr_options (cs); // Extra system header dirs (last). if (dd.expect (cs.string ()) != nullptr) + { l4 ([&]{trace << "options mismatch forcing update of " << t;}); + + // Note that this doesn't include any of the "plumbing" options + // like -x, -c, -o, etc. In the unlikely event that there are + // changes in this area that also affect the semantics of the + // compilation database (options reordering doesn't, for example), + // then we can resort to incrementing the rule version. + // + compiledb_changed = true; + } } // Finally the source file. @@ -1273,7 +1295,10 @@ namespace build2 assert (!p.empty ()); // Sanity check. if (dd.expect (p) != nullptr) + { l4 ([&]{trace << "source file mismatch forcing update of " << t;}); + compiledb_changed = true; + } } // If any of the above checks resulted in a mismatch (different @@ -1296,6 +1321,14 @@ namespace build2 u = dd.mtime > mt; } + // Confirm the entry in the compilation database, if any. + // + if (compiledb::match (bs, t, tp, src, compiledb_changed) && !u) + { + l4 ([&]{trace << "compilation database forcing update of " << t;}); + u = true; + } + // If updating for any of the above reasons, treat it as if doesn't // exist. // @@ -1950,8 +1983,8 @@ namespace build2 // // @@ Should we print the pid we are talking to? It gets hard to // follow once things get nested. But if all our diag will include - // some kind of id (chain, thread?), then this will not be strictly - // necessary. + // some kind of id (dependency chain, thread?), then this will not + // be strictly necessary. // diag_record dr (text); for (size_t i (0); i != batch_n; ++i) @@ -2628,8 +2661,8 @@ namespace build2 // @@ MODHDR: Should we print the pid we are talking to? It gets hard to // follow once things get nested. But if all our diag will - // include some kind of id (chain, thread?), then this will - // not be strictly necessary. + // include some kind of id (dependency chain, thread?), then + // this will not be strictly necessary. // if (verb >= 3) text << " > " << rq; @@ -3148,7 +3181,7 @@ namespace build2 } hk.file = move (fp); - hk.hash = hash<path> () (hk.file); + hk.hash = hash<string> () (hk.file.string ()); slock l (hc.header_map_mutex); auto i (hc.header_map.find (hk)); @@ -3201,7 +3234,7 @@ namespace build2 // path has changed (header has been remapped). // if (!e || r.second) - hk.hash = hash<path> () (hk.file); + hk.hash = hash<string> () (hk.file.string ()); const file* f; { @@ -3214,6 +3247,8 @@ namespace build2 { //cache_cls.fetch_add (1, memory_order_relaxed); + // @@ TMP cleanup. + // #if 0 assert (r.first == f); #else @@ -4167,9 +4202,10 @@ namespace build2 if (l->empty ()) // Done, nothing changed. { // If modules are enabled, then we keep the preprocessed output - // around (see apply() for details). + // around (see apply() for details). Unless reprocessing was + // requested. // - if (modules) + if (modules && !reprocess) { result.first = ctx.fcache->create_existing (t.path () + pext); result.second = true; @@ -7351,8 +7387,11 @@ namespace build2 // apply()). For named modules there may be no obj*{} if this is a // sidebuild (obj*{} is already in the library binary). // - path relm; + const path* abso (nullptr); + const path* absm (nullptr); path relo; + path relm; + switch (ut) { case unit_type::module_header: @@ -7362,12 +7401,18 @@ namespace build2 case unit_type::module_impl_part: { if (const file* o = find_adhoc_member<file> (t, tts.obj)) - relo = relative (o->path ()); + { + abso = &o->path (); + relo = relative (*abso); + } break; } default: - relo = relative (tp); + { + abso = &tp; + relo = relative (tp); + } } // Build the command line. @@ -7397,6 +7442,9 @@ namespace build2 small_vector<string, 2> header_args; // Header unit options storage. small_vector<string, 2> module_args; // Module options storage. + // NOTE: see a note in apply() on the compilation database implications + // if changing anything below. + // switch (cclass) { case compiler_class::msvc: @@ -7531,6 +7579,7 @@ namespace build2 { assert (ut != unit_type::module_header); // @@ MODHDR + absm = &tp; relm = relative (tp); args.push_back ("/ifcOutput"); @@ -7744,6 +7793,9 @@ namespace build2 // Output module file is specified in the mapping file, the // same as input. // + // We set neither relm nor absm since they are not on the + // command line. + // if (ut == unit_type::module_header) // No obj, -c implied. break; @@ -7772,6 +7824,7 @@ namespace build2 { assert (ut != unit_type::module_header); // @@ MODHDR + absm = &tp; relm = relative (tp); // Without this option Clang's .pcm will reference source @@ -7883,6 +7936,15 @@ namespace build2 else if (verb == 2) print_process (args); + // Insert or update the entry in the compilation database, if any. + // + compiledb::execute ( + bs, + t, tp, s, *sp, + cpath, args, + relo, abso != nullptr ? *abso : empty_path, + relm, absm != nullptr ? *absm : empty_path); + // If we have the (partially) preprocessed output, switch to that. // // But we remember the original source/position to restore later. diff --git a/libbuild2/cc/compiledb.cxx b/libbuild2/cc/compiledb.cxx new file mode 100644 index 0000000..ccf08a9 --- /dev/null +++ b/libbuild2/cc/compiledb.cxx @@ -0,0 +1,1100 @@ +// file : libbuild2/cc/compiledb.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <libbuild2/cc/compiledb.hxx> + +#include <cstring> // strlen() +#include <iostream> // cout + +#ifndef BUILD2_BOOTSTRAP +# include <libbutl/json/parser.hxx> +#endif + +#include <libbuild2/filesystem.hxx> +#include <libbuild2/diagnostics.hxx> + +#include <libbuild2/cc/module.hxx> + +#include <libbuild2/cc/target.hxx> +#include <libbuild2/bin/target.hxx> + +using namespace std; + +namespace build2 +{ + namespace cc + { + compiledb_set compiledbs; + + // compiledb + // + compiledb:: + ~compiledb () + { + } + + // Return true if this entry should be written to the database with the + // specified name. + // + static bool + filter (const scope& rs, + const core_module& m, + const string& name, + const file& ot, const file& it) + { + tracer trace ("cc::compiledb_filter"); + + bool r (true); + const char* w (nullptr); // Why r is false. + + // First check if writing to this database is enabled. + // + // No filter means not enabled. + // + if (m.cdb_filter_ == nullptr) + { + r = false; + w = "no database name filter"; + } + else + { + // Iterate in reverse (so that later values override earlier) and take + // the first name match. + // + r = false; + for (const pair<optional<string>, bool>& p: + reverse_iterate (*m.cdb_filter_)) + { + if (!p.first || *p.first == name) + { + r = p.second; + break; + } + } + + if (!r) + w = "no match in database name filter"; + } + + // Verify the name is known in this amalgamation. Note that without + // this check we may end up writing to unrelated databases in other + // amalgamations (think linked configurations). + // + if (r) + { + r = false; + for (const core_module* pm (&m); + pm != nullptr; + pm = pm->outer_module_) + { + const strings& ns (pm->cdb_names_); + + if (find (ns.begin (), ns.end (), name) != ns.end ()) + { + r = true; + break; + } + } + + if (!r) + w = "database name unknown in amalgamation"; + } + + // Filter based on the output target. + // + // If there is no filter specified, then accept all targets. + // + if (r && m.cdb_filter_output_ != nullptr) + { + // If the filter is empty, then there is no match. + // + if (m.cdb_filter_output_->empty ()) + { + r = false; + w = "empty output target type filter"; + } + else + { + const target_type& ott (ot.type ()); + + // Iterate in reverse (so that later values override earlier) and + // take the first name match. + // + r = false; + for (const pair<optional<string>, string>& p: + reverse_iterate (*m.cdb_filter_output_)) + { + if (p.first && *p.first != name) + continue; + + using namespace bin; + + const string& n (p.second); + + if (ott.name == n || n == "target") + { + r = true; + } + // + // Handle obj/bmi/hbmi{} groups ad hoc. + // + else if (n == "obj") + { + r = ott.is_a<obje> () || ott.is_a<objs> () || ott.is_a<obja> (); + } + else if (n == "bmi") + { + r = ott.is_a<bmie> () || ott.is_a<bmis> () || ott.is_a<bmia> (); + } + else if (n == "hbmi") + { + r = ott.is_a<hbmie> () || ott.is_a<hbmis> () || ott.is_a<hbmia> (); + } + else + { + // Handle the commonly-used, well-known targets directly (see + // note in core_config_init() for why we cannot pre-lookup + // them). + // + const target_type* tt ( + n == "obje" ? &obje::static_type : + n == "objs" ? &objs::static_type : + n == "obja" ? &obja::static_type : + n == "bmie" ? &bmie::static_type : + n == "bmis" ? &bmis::static_type : + n == "bmia" ? &bmia::static_type : + n == "hbmie" ? &hbmie::static_type : + n == "hbmis" ? &hbmis::static_type : + n == "hbmia" ? &hbmia::static_type : + rs.find_target_type (n)); + + if (tt == nullptr) + fail << "unknown target type '" << n << "' in " + << "config.cc.compiledb.filter.output value"; + + r = ott.is_a (*tt); + } + + if (r) + break; + } + + if (!r) + w = "no match in output target type filter"; + } + } + + // Filter based on the input target. + // + // If there is no filter specified, then accept all targets. + // + if (r && m.cdb_filter_input_ != nullptr) + { + // If the filter is empty, then there is no match. + // + if (m.cdb_filter_input_->empty ()) + { + r = false; + w = "empty input target type filter"; + } + else + { + const target_type& itt (it.type ()); + + // Iterate in reverse (so that later values override earlier) and + // take the first name match. + // + r = false; + for (const pair<optional<string>, string>& p: + reverse_iterate (*m.cdb_filter_input_)) + { + if (p.first && *p.first != name) + continue; + + const string& n (p.second); + + if (itt.name == n || n == "target") + r = true; + else + { + // The same optimization as above. Note: cxx{}, etc., are in the + // cxx module so we have to look them up. + // + const target_type* tt ( + n == "c" ? &c::static_type : + n == "m" ? &m::static_type : + n == "S" ? &m::static_type : + rs.find_target_type (n)); + + if (tt == nullptr) + fail << "unknown target type '" << n << "' in " + << "config.cc.compiledb.filter.input value"; + + r = itt.is_a (*tt); + } + + if (r) + break; + } + + if (!r) + w = "no match in input target type filter"; + } + } + + l6 ([&] + { + if (r) + trace << "keep " << ot << " in " << name; + else + trace << "omit " << ot << " from " << name << ": " << w; + }); + + return r; + } + + bool compiledb:: + match (const scope& bs, + const file& ot, const path_type& op, + const file& it, + bool changed) + { + if (compiledbs.empty ()) + return false; + + const scope& rs (*bs.root_scope ()); + const auto* m (rs.find_module<core_module> (core_module::name)); + + assert (m != nullptr); + + bool u (false); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + if (filter (rs, *m, db->name, ot, it)) + u = db->match (ot, op, changed) || u; + } + + return u; + } + + void compiledb:: + execute (const scope& bs, + const file& ot, const path_type& op, + const file& it, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + if (compiledbs.empty ()) + return; + + const scope& rs (*bs.root_scope ()); + const auto* m (rs.find_module<core_module> (core_module::name)); + + assert (m != nullptr); + + assert (relo.empty () == abso.empty () && + relm.empty () == absm.empty ()); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + if (filter (rs, *m, db->name, ot, it)) + db->execute (ot, op, it, ip, cpath, args, relo, abso, relm, absm); + } + } + + void + compiledb_pre (context& ctx, action a, const action_targets&) + { + // Note: won't be registered if compiledbs is empty. + + // Note: may be called directly with empty action_targets. + + assert (a.inner_action () == perform_update_id); + + tracer trace ("cc::compiledb_pre"); + + bool mctx (ctx.module_context == &ctx); + + l6 ([&]{trace << (mctx ? "module" : "normal") << " context " << &ctx;}); + + for (const unique_ptr<compiledb>& db: compiledbs) + db->pre (ctx); + } + + void + compiledb_post (context& ctx, + action a, + const action_targets& ts, + bool failed) + { + // Note: won't be registered if compiledbs is empty. + + assert (a.inner_action () == perform_update_id); + + tracer trace ("cc::compiledb_post"); + + bool mctx (ctx.module_context == &ctx); + + l6 ([&]{trace << (mctx ? "module" : "normal") << " context " << &ctx + << ", failed: " << failed;}); + + for (const unique_ptr<compiledb>& db: compiledbs) + db->post (ctx, ts, failed); + } + +#ifndef BUILD2_BOOTSTRAP + + namespace json = butl::json; + + // compiledb_stdout + // + compiledb_stdout:: + compiledb_stdout (string n) + : compiledb (move (n), path_type ()), + state_ (state::init), + nesting_ (0), + js_ (cout, 0 /* indentation */, "" /* multi_value_separator */) + { + } + + void compiledb_stdout:: + pre (context&) + { + // If the previous operation batch failed, then we shouldn't be here. + // + assert (state_ != state::failed); + + // The module context (used to build build system modules) poses a + // problem: we can receive its callbacks before the main context's or + // nested in the pre/post calls of the main context (or both, in + // fact). Plus there may be multiple pre/post sequences corresponding to + // the module context of both kinds. The three distinct cases are: + // + // 1. Module is loaded as part of the initial buildfile load (e.g., from + // root.build) -- in this case we will observe module pre/post before + // the main context's pre/post. + // + // In fact, to be precise, we will only observe them if cc is loaded + // before such a module. + // + // 2. Module is loaded via the interrupting load (e.g., from a directory + // buildfile that is loaded implicitly during match) -- in this case + // we will observe pre/post calls nested into the main context's + // pre/post. + // + // 3. The module context is used to build an ad hoc C++ recipe -- in + // this case we also get nested calls like in (2) since this happens + // during the recipe's match(). + // + // One thing to keep in mind (and which we rely upon quite a bit below) + // is that the main context's post will always be last (within any given + // operation; there could be another for the subsequent operation in a + // batch). + // + // Handling the nested case is relatively straightforward: we can keep + // track and ignore all the nested calls. + // + // The before case is where things get complicated. We could "take" the + // first module pre call and then wait until the main post, unless we + // see a module post call with failed=true, in which case there will be + // no further pre/post calls. There is, however, a nuance: the module is + // loaded and build for any operation, not just update, which means that + // if the main operation is not update (say, it's clean), we won't see + // any of the main context's pre/post calls. + // + // The way we are going to resolve this problem is different for the + // stdout and file implementations: + // + // For stdout we will just say that it should only be used with the + // update operation. There is really no good reason to use it with + // anything else anyway. See compiledb_stdout::post() for additional + // details. + // + // For file we will rely on its persistence and simply close and reopen + // the database for each pre/post sequence, the same way as if they were + // separate operations in a batch. + // + if (nesting_++ != 0) // Nested pre() call. + return; + + if (state_ == state::init) // First pre() call. + { + state_ = state::empty; + cout << "[\n"; + } + } + + bool compiledb_stdout:: + match (const file&, const path_type&, bool) + { + return true; + } + + static inline const char* + rel_to_abs (const char* a, + const string& rs, const string& as, + string& buf) + { + if (size_t rn = rs.size ()) + { + size_t an (strlen (a)); + + if (an >= rn && rs.compare (0, rn, a, rn) == 0) + { + if (an == rn) + return as.c_str (); + + buf = as; + buf.append (a + rn, an - rn); + + return buf.c_str (); + } + } + + return nullptr; + } + + void compiledb_stdout:: + execute (const file&, const path_type& op, + const file&, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + const string& ro (relo.string ()); + const string& ao (abso.string ()); + + const string& rm (relm.string ()); + const string& am (absm.string ()); + + mlock l (mutex_); + + switch (state_) + { + case state::full: + { + cout << ",\n"; + break; + } + case state::empty: + { + state_ = state::full; + break; + } + case state::failed: + return; + case state::init: + assert (false); + return; + } + + try + { + // Duplicate what we have in the file implementation (instead of + // factoring it out to something common) in case here we need to + // adjust things (change order, omit some values; for example to + // accommodate broken consumers). We have this freedom here but not + // there. + // + js_.begin_object (); + { + js_.member ("output", op.string ()); + js_.member ("file", ip.string ()); + + js_.member_begin_array ("arguments"); + { + string buf; // Reuse. + for (auto b (args.begin ()), i (b), e (args.end ()); + i != e && *i != nullptr; + ++i) + { + const char* r; + + if (i == b) + r = cpath.effect_string (); + else + { + // Untranslate relative paths back to absolute. + // + const char* a (*i); + + if ((r = rel_to_abs (a, ro, ao, buf)) == nullptr && + (r = rel_to_abs (a, rm, am, buf)) == nullptr) + r = a; + } + + js_.value (r); + } + } + js_.end_array (); + + js_.member ("directory", work.string ()); + } + js_.end_object (); + } + catch (const json::invalid_json_output& e) + { + // There is no way (nor reason; the output will most likely be invalid + // anyway) to reuse the failed json serializer so make sure we ignore + // all the subsequent callbacks. + // + state_ = state::failed; + + l.unlock (); + + fail << "invalid compilation database json output: " << e; + } + } + + void compiledb_stdout:: + post (context& ctx, const action_targets&, bool failed) + { + assert (nesting_ != 0); + if (--nesting_ != 0) // Nested post() call. + return; + + bool mctx (ctx.module_context == &ctx); + + switch (state_) + { + case state::empty: + case state::full: + { + // If this is a module context's post, wait for the main context's + // post (last) unless the module load failed (in which case there + // will be no main pre/post). + // + // Note that there is no easy way to diagnose the case where we + // won't get the main pre/post calls. Instead, we will just produce + // invalid JSON (array won't be closed). In a somewhat hackish way, + // this actually makes the `b [-n] clean update` sequence work: we + // will take the pre() call from clean and the main post() from + // update. + // + if (mctx && !failed) + return; + + if (state_ == state::full) + cout << '\n'; + + cout << "]\n"; + break; + } + case state::failed: + return; + case state::init: + assert (false); + } + + state_ = state::init; + } + + // compiledb_file + // + compiledb_file:: + compiledb_file (string n, path_type p) + : compiledb (move (n), move (p)), + state_ (state::closed), + nesting_ (0) + { + } + + void compiledb_file:: + pre (context&) + { + // If the previous operation batch failed, then we shouldn't be here. + // + assert (state_ != state::failed); + + // See compiledb_stdout::pre() for background on dealing with the module + // context. Here are some file-specific nuances: + // + // We are going to load the database on the first pre call and flush + // (but not close) it on the matching post. Flushing means that we will + // update the file but still keep the in-memory state, in case there is + // another pre/post session coming. This is both a performance + // optimization but also the way we handle prunning no longer present + // entries, which gets tricky across multiple pre/post sessions (see + // post() for details). + // + if (nesting_++ != 0) // Nested pre() call. + return; + + if (state_ == state::closed) // First pre() call. + { + // Load the contents of the file if it exists, marking all the entries + // as (presumed) absent. + // + if (exists (path)) + { + uint64_t line (1); + try + { + ifdstream ifs (path, ifdstream::badbit); + + // Parse the top-level array manually (see post() for the expected + // format). + // + auto throw_invalid_input = [] (const string& d) + { + throw json::invalid_json_input ("", 0, 1, 0, d); + }; + + enum {first, second, next, last, end} s (first); + + for (string l; !eof (getline (ifs, l)); line++) + { + switch (s) + { + case first: + { + if (l != "[") + throw_invalid_input ("beginning of array expected"); + + s = second; + continue; + } + case second: + { + if (l == "]") + { + s = end; + continue; + } + + s = next; + } + // Fall through. + case next: + { + if (!l.empty () && l.back () == ',') + l.pop_back (); + else + s = last; + + break; + } + case last: + { + if (l != "]") + throw_invalid_input ("end of array expected"); + + s = end; + continue; + } + case end: + { + throw_invalid_input ("junk after end of array"); + } + } + + // Parse just the output target path, which must come first. + // + json::parser jp (l, "" /* name */); + + jp.next_expect (json::event::begin_object); + string op (move (jp.next_expect_member_string ("output"))); + + auto r (db_.emplace (move (op), entry {entry_status::absent, l})); + if (!r.second) + throw_invalid_input ( + "duplicate output value '" + r.first->first + '\''); + } + + if (s != end) + throw_invalid_input ("corrupt input text"); + } + catch (const json::invalid_json_input& e) + { + state_ = state::failed; + + location l (path, line, e.column); + fail (l) << "invalid compilation database json input: " << e << + info << "remove this file if it was produced by a different tool"; + } + catch (const io_error& e) + { + state_ = state::failed; + fail << "unable to read " << path << ": " << e; + } + } + + absent_ = db_.size (); + changed_ = false; + + state_ = state::open; + } + } + + bool compiledb_file:: + match (const file&, const path_type& op, bool changed) + { + mlock l (mutex_); + + switch (state_) + { + case state::open: + break; + case state::failed: + return false; + case state::closed: + assert (false); + return false; + } + + // Mark an existing entry as present or changed. And if one does not + // exist, then (for now) as missing. + // + auto i (db_.find (op.string ())); + + if (i != db_.end ()) + { + entry& e (i->second); + + // Note: we can end up with present entries via the module context + // (see post() below). And we can see changed entries in a subsequent + // nested module context. + // + switch (e.status) + { + case entry_status::present: + case entry_status::changed: + assert (!changed); + break; + case entry_status::absent: + { + e.status = changed ? entry_status::changed : entry_status::present; + + absent_--; + changed_ = changed_ || (e.status == entry_status::changed); + break; + } + case entry_status::missing: + assert (false); + } + + return false; + } + else + { + db_.emplace (op.string (), entry {entry_status::missing, string ()}); + + changed_ = true; + + return true; + } + } + + void compiledb_file:: + execute (const file&, const path_type& op, + const file&, const path_type& ip, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) + { + const string& ro (relo.string ()); + const string& ao (abso.string ()); + + const string& rm (relm.string ()); + const string& am (absm.string ()); + + mlock l (mutex_); + + switch (state_) + { + case state::open: + break; + case state::failed: + return; + case state::closed: + assert (false); + return; + } + + auto i (db_.find (op.string ())); + + // We should have had the match() call before execute(). + // + assert (i != db_.end () && i->second.status != entry_status::absent); + + entry& e (i->second); + + if (e.status == entry_status::present) // Present and unchanged. + return; + + // The entry is either missing or changed. + // + try + { + e.json.clear (); + json::buffer_serializer js (e.json, 0 /* indentation */); + + js.begin_object (); + { + js.member ("output", op.string ()); // Note: must come first. + js.member ("file", ip.string ()); + + js.member_begin_array ("arguments"); + { + string buf; // Reuse. + for (auto b (args.begin ()), i (b), e (args.end ()); + i != e && *i != nullptr; + ++i) + { + const char* r; + + if (i == b) + r = cpath.effect_string (); + else + { + // Untranslate relative paths back to absolute. + // + const char* a (*i); + + if ((r = rel_to_abs (a, ro, ao, buf)) == nullptr && + (r = rel_to_abs (a, rm, am, buf)) == nullptr) + r = a; + } + + js.value (r); + } + } + js.end_array (); + + js.member ("directory", work.string ()); + } + js.end_object (); + } + catch (const json::invalid_json_output& e) + { + // There is no way (nor reason; the output will most likely be invalid + // anyway) to reuse the failed json serializer so make sure we ignore + // all the subsequent callbacks. + // + state_ = state::failed; + + l.unlock (); + + fail << "invalid compilation database json output: " << e; + } + + e.status = entry_status::changed; + } + + void compiledb_file:: + post (context& ctx, const action_targets& ts, bool failed) + { + assert (nesting_ != 0); + if (--nesting_ != 0) // Nested post() call. + return; + + switch (state_) + { + case state::open: + break; + case state::failed: + return; + case state::closed: + assert (false); + return; + } + + bool mctx (ctx.module_context == &ctx); + + tracer trace ("cc::compiledb_file::post"); + + // See if we need to update the file. + // + if (changed_) + l6 ([&]{trace << "updating due to missing/changed entries: " << path;}); + + // Don't prune the stale entries if the operation failed since we may + // not have gotten to execute some of them. + // + // And if this is a module context's post, then also don't prune the + // stale entries, instead waiting for the main context's post (if there + // will be one; this means we will only prune on update). + // + // Actually, this pruning business is even trickier than that: if we + // are not updating the entire project (say, rather only a subdirectory + // or even a specific target), then we will naturally not get any + // match/execute calls for targets of this project that don't get pulled + // into this build. Which means that we cannot just prune entries that + // we did not match/execute. It feels the correct semantics is to only + // prune the entries if they are in a subdirectory of the dir{} targets + // which we are building. + // + // What do we do about the module context, where we always update a + // specific libs{}? We could use its directory instead but that may lead + // to undesirable results. For example, if there are unit tests in the + // same directory, we will end up dropping their entries. It feels like + // the correct approach is to just ignore module context's entries + // entirely. If someone wants to prune the compilation database of a + // module, they will just need to update it directly (i.e., via the main + // context). Note that we cannot apply the same "simplification" to the + // changed entries since we will only observe the change once. + // + bool absent (false); + + if (!failed && !mctx && absent_ != 0) + { + // Pre-scan the entries and drop the appropriate absent ones. + // + for (auto i (db_.begin ()); i != db_.end (); ) + { + const entry& e (i->second); + + if (e.status == entry_status::absent) + { + // Absent entries should be rare enough during the normal + // development that we don't need to bother with caching the + // directories. + // + bool a (false); + for (const action_target& at: ts) + { + const target& t (at.as<target> ()); + if (t.is_a<dir> ()) + { + const string& p (i->first); + const string& d (t.out_dir ().string ()); + + if (path_traits::sub (p.c_str (), p.size (), + d.c_str (), d.size ())) + { + // Remove this entry from the in-memory state so that it + // matches the file state. + // + i = db_.erase (i); + --absent_; + a = absent = true; + break; + } + } + } + + if (a) + continue; + } + + ++i; + } + } + + if (absent) + l6 ([&]{trace << "updating due to absent entries: " << path;}); + + try + { + auto_rmfile rm; + ofdstream ofs; + + bool u (changed_ || absent); // Update the file. + + if (u) + { + rm = auto_rmfile (path); + ofs.open (path); + + // We parse the top-level array manually (see pre() above) and the + // expected format is as follows: + // + // [ + // {"output":...}, + // ... + // {"output":...} + // ] + // + ofs.write ("[\n", 2); + } + + // Iterate over the entries resetting their status and writing them to + // the file if necessary. + // + bool first (true); + for (auto& p: db_) + { + entry& e (p.second); + + // First sort out the status also skipping appropriate entries. + // + switch (e.status) + { + case entry_status::absent: + { + // This is an absent entry that we should keep (see pre-scan + // above). + // + break; + } + case entry_status::missing: + { + // This should only happen if this operation has failed (see + // also below) or we are in the match-only mode. + // + assert (failed || ctx.match_only); + continue; + } + case entry_status::present: + case entry_status::changed: + { + // This is tricky: if this is a module context, then we don't + // want to mark the entries as absent since they will then get + // dropped by the main operation context. + // + if (mctx) + e.status = entry_status::present; + else + { + // Note: this is necessary for things to work across multiple + // operations in a batch. + // + e.status = entry_status::absent; + absent_++; + } + } + } + + if (u) + { + if (first) + first = false; + else + ofs.write (",\n", 2); + + ofs.write (e.json.c_str (), e.json.size ()); + } + } + + if (u) + { + ofs.write (first ? "]\n" : "\n]\n", first ? 2 : 3); + + ofs.close (); + rm.cancel (); + } + } + catch (const io_error& e) + { + state_ = state::failed; + fail << "unable to write to " << path << ": " << e; + } + + // If this operation has failed, then our state may not be accurate + // (e.g., entries with missing status) but we also don't expect any + // further pre calls. Let's change out state to failed as a sanity + // check. + // + if (failed) + state_ = state::failed; + else + changed_ = false; + + // Note: keep in the open state (see pre() for details). + } + +#endif // BUILD2_BOOTSTRAP + } +} diff --git a/libbuild2/cc/compiledb.hxx b/libbuild2/cc/compiledb.hxx new file mode 100644 index 0000000..8288cf5 --- /dev/null +++ b/libbuild2/cc/compiledb.hxx @@ -0,0 +1,236 @@ +// file : libbuild2/cc/compiledb.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LIBBUILD2_CC_COMPILEDB_HXX +#define LIBBUILD2_CC_COMPILEDB_HXX + +#include <unordered_map> + +#ifndef BUILD2_BOOTSTRAP +# include <libbutl/json/serializer.hxx> +#endif + +#include <libbuild2/types.hxx> +#include <libbuild2/utility.hxx> + +#include <libbuild2/target.hxx> +#include <libbuild2/action.hxx> +#include <libbuild2/context.hxx> + +namespace build2 +{ + namespace cc + { + using compiledb_name_filter = vector<pair<optional<string>, bool>>; + using compiledb_type_filter = vector<pair<optional<string>, string>>; + + class compiledb + { + public: + // Match callback where we confirm an entry in the database and also + // signal whether it has changes (based on change tracking in depdb). + // Return true to force compilation of this target and thus make sure + // the below execute() is called (unless something before that failed). + // + // Besides noticing changes, this callback is also necessary to notice + // and delete entries that should no longer be in the database (e.g., a + // source file was removed from the project). + // + // Note that output is either obj*{}, bmi*{}, of hbmi*{}. + // + static bool + match (const scope& bs, + const file& output, const path& output_path, + const file& input, + bool changed); + + // Execute callback where we insert or update an entry in the database. + // + // The {relo, abso}, and {relm, absm} pairs are used to "untranslate" + // relative paths to absolute. Specifically, any argument that has rel? + // as a prefix has this prefix replaced with the corresponding abs?. + // Note that this means we won't be able to handle old MSVC and + // clang-cl, which don't support the `/F?: <path>` form, only + // `/F?<path>`. Oh, well. Note also that either relo or relm (but not + // both) could be empty if unused. + // + // Note that we assume the source file is always absolute and is the + // last argument. + // + // Why do we want absolute paths? That's a good question. Our initial + // plan was to compare command lines in order to detect when we need to + // update the database. And if those changed with every change of CWD, + // that would be of little use. But then we realized we could do better + // by using depdb to detect changes. So now we actually don't have a + // need to get rid of the relative paths in the command line. But seeing + // that we already have it, let's keep it for now in case it makes a + // different to some broken/legacy consumers. Note also that C++ module + // name-to-BMI mapping is not untranslated (see append_module_options()). + // + static void + execute (const scope& bs, + const file& output, const path& output_path, + const file& input, const path& input_path, + const process_path& cpath, const cstrings& args, + const path& relo, const path& abso, + const path& relm, const path& absm); + + public: + using path_type = build2::path; + + string name; + path_type path; + + // The path is expected to be absolute and normalized or empty if the + // name is `-` (stdout). + // + compiledb (string n, path_type p) + : name (move (n)), path (move (p)) + { + } + + virtual void + pre (context&) = 0; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) = 0; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) = 0; + + virtual void + post (context&, const action_targets&, bool failed) = 0; + + virtual + ~compiledb (); + }; + + using compiledb_set = vector<unique_ptr<compiledb>>; + + // Populated by core_config_init() during serial load. + // + extern compiledb_set compiledbs; + + // Context operation callbacks. + // + void + compiledb_pre (context&, action, const action_targets&); + + void + compiledb_post (context&, action, const action_targets&, bool failed); + +#ifndef BUILD2_BOOTSTRAP + + // Implementation that writes to stdout. + // + // Note that this implementation forces compilation of all the targets for + // which it is called to make sure their entries are in the database. So + // typically used in the dry run mode. + // + class compiledb_stdout: public compiledb + { + public: + // The path is expected to be empty. + // + explicit + compiledb_stdout (string name); + + virtual void + pre (context&) override; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) override; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) override; + + virtual void + post (context&, const action_targets&, bool failed) override; + + private: + mutex mutex_; + enum class state {init, empty, full, failed} state_; + size_t nesting_; + butl::json::stream_serializer js_; + }; + + // Implementation that maintains a file. + // + class compiledb_file: public compiledb + { + public: + compiledb_file (string name, path_type path); + + virtual void + pre (context&) override; + + virtual bool + match (const file& output, const path_type& output_path, + bool changed) override; + + virtual void + execute (const file& output, const path_type& output_path, + const file& input, const path_type& input_path, + const process_path& cpath, const cstrings& args, + const path_type& relo, const path_type& abso, + const path_type& relm, const path_type& absm) override; + + virtual void + post (context&, const action_targets&, bool failed) override; + + private: + mutex mutex_; + enum class state {closed, open, failed} state_; + size_t nesting_; + + // We want to optimize the performance for the incremental update case + // where only a few files will be recompiled and most of the time there + // will be no change in the command line, which means we won't need to + // rewrite the file. + // + // As a result, our in-memory representation is a hashmap (we could have + // thousands of entries) of absolute and normalized output file paths + // (stored as strings for lookup efficiency) to their serialized JSON + // text lines plus the status: absent, present, changed, or missing + // (entry should be there but is not). This way we don't waste + // (completely) parsing (and re-serializing) each line knowing that we + // won't need to touch most of them. + // + // In fact, we could have gone even further and used a sorted vector + // since insertions will be rare in this case. But we will need to + // lookup every entry on each update, so it's unclear this is a win. + // + enum class entry_status {absent, present, changed, missing}; + + struct entry + { + entry_status status; + string json; + }; + + using map_type = std::unordered_map<string, entry>; + map_type db_; + + // Number/presence of various entries in the database (used to determine + // whether we need to update the file without iterating over all the + // entries). + // + size_t absent_; // Number of absent entries. + bool changed_; // Presence of changed or missing entries. + }; + +#endif // BUILD2_BOOTSTRAP + } +} + +#endif // LIBBUILD2_CC_COMPILEDB_HXX diff --git a/libbuild2/cc/guess.cxx b/libbuild2/cc/guess.cxx index 5ae6fb2..2d079f0 100644 --- a/libbuild2/cc/guess.cxx +++ b/libbuild2/cc/guess.cxx @@ -949,9 +949,15 @@ namespace build2 // there is nothing like -m32/-m64 or /MACHINE). Targeting // 64-bit seems like as good of a default as any. // - fb = ((dir_path (mi->msvc_dir) /= "bin") /= "Hostx64") /= - "x64"; - + fb = dir_path (mi->msvc_dir) /= "bin"; + +#if defined(_M_ARM64) || defined(__aarch64__) + fb /= "HostARM64"; + fb /= "ARM64"; +#else + fb /= "Hostx64"; + fb /= "x64"; +#endif search_info = info_ptr ( new msvc_info (move (*mi)), msvc_info_deleter); } @@ -1629,15 +1635,25 @@ namespace build2 // Seeing that we only do 64-bit on Windows, let's always use 64-bit // MSVC tools (link.exe, etc). In case of the Platform SDK, it's unclear - // what the CPU signifies (host, target, both). + // what the CPU signifies (host, target, both). It appears to be host. // - r = (((dir_path (mi.msvc_dir) /= "bin") /= "Hostx64") /= cpu). - representation (); + r = (((dir_path (mi.msvc_dir) /= "bin") /= +#if defined(_M_ARM64) || defined(__aarch64__) + "HostARM64" +#else + "Hostx64" +#endif + ) /= cpu).representation (); r += path::traits_type::path_separator; - r += (((dir_path (mi.psdk_dir) /= "bin") /= mi.psdk_ver) /= cpu). - representation (); + r += (((dir_path (mi.psdk_dir) /= "bin") /= mi.psdk_ver) /= +#if defined(_M_ARM64) || defined(__aarch64__) + "arm64" +#else + "x64" +#endif + ).representation (); return r; } @@ -1878,7 +1894,7 @@ namespace build2 move (ver), nullopt, move (gr.signature), - "", + "", // Checksum to be calculated from signature. move (t), move (ot), move (cpat), @@ -2128,7 +2144,7 @@ namespace build2 move (ver), nullopt, move (gr.signature), - move (gr.checksum), // Calculated on whole -v output. + "", // Checksum calculated on whole -v output. move (t), move (ot), move (pat), @@ -2898,7 +2914,7 @@ namespace build2 move (ver), move (var_ver), move (gr.signature), - move (gr.checksum), // Calculated on whole -v output. + "", // Checksum calculated on whole -v output. move (t), move (ot), move (cpat), @@ -3216,7 +3232,7 @@ namespace build2 move (ver), nullopt, move (gr.signature), - "", + "", // Checksum to be calculated from signature. move (t), move (ot), move (pat), @@ -3370,6 +3386,8 @@ namespace build2 cs.append (gr.type_signature); } + cs.append (r.path.effect_string ()); + r.checksum = cs.string (); // Derive binutils pattern unless this has already been done by the diff --git a/libbuild2/cc/guess.hxx b/libbuild2/cc/guess.hxx index 7cbbd87..dfa8aa2 100644 --- a/libbuild2/cc/guess.hxx +++ b/libbuild2/cc/guess.hxx @@ -160,6 +160,9 @@ namespace build2 // checksum will still change. This is currently the case for all the // compilers that we support. // + // And we assume that the checksum incorporates the absolute compiler + // path. This is used to detect compilation database changes. + // // The target is the compiler's traget architecture triplet. Note that // unlike all the preceding fields, this one takes into account the // compile options (e.g., -m32). diff --git a/libbuild2/cc/init.cxx b/libbuild2/cc/init.cxx index e124450..d691bc5 100644 --- a/libbuild2/cc/init.cxx +++ b/libbuild2/cc/init.cxx @@ -10,8 +10,10 @@ #include <libbuild2/config/utility.hxx> +#include <libbuild2/cc/module.hxx> #include <libbuild2/cc/target.hxx> #include <libbuild2/cc/utility.hxx> +#include <libbuild2/cc/compiledb.hxx> using namespace std; using namespace butl; @@ -23,7 +25,7 @@ namespace build2 // Scope operation callback that cleans up module sidebuilds. // static target_state - clean_module_sidebuilds (action, const scope& rs, const dir&) + clean_module_sidebuilds (const scope& rs) { context& ctx (rs.ctx); @@ -67,6 +69,131 @@ namespace build2 return target_state::unchanged; } + // Scope operation callback that cleans up compilation databases. + // + static target_state + clean_compiledb (const scope& rs) + { + context& ctx (rs.ctx); + + target_state r (target_state::unchanged); + + for (const unique_ptr<compiledb>& db: compiledbs) + { + const path& p (db->path); + + if (p.empty () || + ctx.scopes.find_out (p.directory ()).root_scope () != &rs) + continue; + + if (rmfile (ctx, p)) + r = target_state::changed; + } + + return r; + } + + // Scope operation callback for cleaning module sidebuilds and compilation + // databases. + // + static target_state + clean_callback (action, const scope& rs, const dir&) + { + target_state r (clean_module_sidebuilds (rs)); + + if (!compiledbs.empty ()) + r |= clean_compiledb (rs); + + return r; + } + + // Detect if just <name> in the <name>[@<path>] form is actually <path>. + // We assume it is <path> and not <name> if it contains a directory + // component or is the special directory name (`.`/`..`) . If that's the + // case, return canonicalized name representing <path>. See the call site + // in core_config_init() below for background. + // + static optional<name> + compiledb_name_to_path (const name& n) + { + if (n.directory ()) + return n; + + if (n.file ()) + { + if (!n.dir.empty () || + path_traits::find_separator (n.value) != string::npos) + { + name r (n); + r.canonicalize (); + return r; + } + else if (n.value == "." || n.value == "..") + { + return name (dir_path (n.value)); + } + } + + return nullopt; + } + + // Custom save function that completes relative paths in the + // config.cc.compiledb and config.cc.compiledb.name values. + // + static pair<names_view, const char*> + save_compiledb_name (const scope&, + const value& v, + const value*, + names& storage) + { + const names& ns (v.as<names> ()); // Value is untyped. + + // Detect and handle the case where just <name> is actually <path>. + // + if (ns.size () == 1) + { + const name& n (ns.back ()); + + if (optional<name> otn = compiledb_name_to_path (n)) + { + name& tn (*otn); + + if (tn.dir.relative ()) + tn.dir.complete (); + + tn.dir.normalize (); + + storage.push_back (move (tn)); + return make_pair (names_view (storage), "="); + } + } + + if (find_if (ns.begin (), ns.end (), + [] (const name& n) {return n.pair;}) == ns.end ()) + { + return make_pair (names_view (ns), "="); + } + + storage = ns; + for (auto i (storage.begin ()); i != storage.end (); ++i) + { + if (i->pair) + { + name& n (*++i); + + if (!n.directory ()) + n.canonicalize (); + + if (n.dir.relative ()) + n.dir.complete (); + + n.dir.normalize (); + } + } + + return make_pair (names_view (storage), "="); + } + bool core_vars_init (scope& rs, scope&, @@ -107,6 +234,22 @@ namespace build2 vp.insert<abs_dir_path> ("config.cc.pkgconfig.sysroot"); + // Compilation database. + // + // See the manual for the semantics. + // + // config.cc.compiledb -- <name>[@<path>]|<path> (untyped) + // config.cc.compiledb.name -- <name>[@<path>]... (untyped) + // config.cc.compiledb.filter -- [<name>@]<bool>... + // config.cc.compiledb.filter.input -- [<name>@]<target-type>... + // config.cc.compiledb.filter.output -- [<name>@]<target-type>... + // + vp.insert ("config.cc.compiledb"); + vp.insert ("config.cc.compiledb.name"); + vp.insert<compiledb_name_filter> ("config.cc.compiledb.filter"); + vp.insert<compiledb_type_filter> ("config.cc.compiledb.filter.input"); + vp.insert<compiledb_type_filter> ("config.cc.compiledb.filter.output"); + vp.insert<strings> ("cc.poptions"); vp.insert<strings> ("cc.coptions"); vp.insert<strings> ("cc.loptions"); @@ -192,16 +335,6 @@ namespace build2 // vp.insert<bool> ("cc.serialize"); - // Register scope operation callback. - // - // It feels natural to clean up sidebuilds as a post operation but that - // prevents the (otherwise-empty) out root directory to be cleaned up - // (via the standard fsdir{} chain). - // - rs.operation_callbacks.emplace ( - perform_clean_id, - scope::operation_callback {&clean_module_sidebuilds, nullptr /*post*/}); - return true; } @@ -292,6 +425,8 @@ namespace build2 assert (first); + context& ctx (rs.ctx); + // Load cc.core.guess. // load_module (rs, rs, "cc.core.guess", loc); @@ -312,7 +447,6 @@ namespace build2 // // @@ Same nonsense as in module. // - // rs.assign ("cc.poptions") += cast_null<strings> ( lookup_config (rs, "config.cc.poptions", nullptr)); @@ -363,21 +497,16 @@ namespace build2 if (!cast_false<bool> (rs["bin.config.loaded"])) { // Prepare configuration hints (pretend it belongs to root scope). - // They are only used on the first load of bin.config so we only - // populate them on our first load. // variable_map h (rs); - if (first) - { - // Note that all these variables have already been registered. - // - h.assign ("config.bin.target") = - cast<target_triplet> (rs["cc.target"]).representation (); + // Note that all these variables have already been registered. + // + h.assign ("config.bin.target") = + cast<target_triplet> (rs["cc.target"]).representation (); - if (auto l = extra.hints["config.bin.pattern"]) - h.assign ("config.bin.pattern") = cast<string> (l); - } + if (auto l = extra.hints["config.bin.pattern"]) + h.assign ("config.bin.pattern") = cast<string> (l); init_module (rs, rs, "bin.config", loc, false /* optional */, h); } @@ -386,7 +515,6 @@ namespace build2 // ourselves since the target can come from the configuration and not // our hint). // - if (first) { const auto& ct (cast<target_triplet> (rs["cc.target"])); const auto& bt (cast<target_triplet> (rs["bin.target"])); @@ -416,6 +544,447 @@ namespace build2 if (tsys == "mingw32") load_module (rs, rs, "bin.rc.config", loc); + // Find the innermost outer core_module, if any. + // + const core_module* om (nullptr); + for (const scope* s (&rs); + (s = s->parent_scope ()->root_scope ()) != nullptr; ) + { + if ((om = s->find_module<core_module> (core_module::name)) != nullptr) + break; + } + + auto& m (extra.set_module (new core_module (om))); + + // config.cc.compiledb.* + // + { + // For config.cc.compiledb and config.cc.compiledb.name we only + // consider a value in this root scope (if it's inherited from the + // outer scope, then that's where it will be handled). One special + // case is when it's specified on a scope that doesn't load the cc + // module (including, ultimately, the global scope for a global + // override). We handle it by assuming the value belongs to the + // outermost amalgamation that loads the cc module. + // + // Note: cache the result. + // + auto find_outermost = + [&rs, o = optional<pair<scope*, core_module*>> ()] () mutable + { + if (!o) + { + o = pair<scope*, core_module*> (&rs, nullptr); + for (scope* s (&rs); + (s = s->parent_scope ()->root_scope ()) != nullptr; ) + { + if (auto* m = s->find_module<core_module> (core_module::name)) + { + o->first = s; + o->second = m; + } + } + } + + return *o; + }; + + auto belongs = [&rs, &find_outermost] (const lookup& l) + { + return l.belongs (rs) || find_outermost ().first == &rs; + }; + + // Add compilation databases specified in ns as <name>[@<path>] pairs, + // appending their names to cdb_names. If <path> is absent, then place + // the database into the base directory. Return the last added name. + // + auto add_cdbs = [&ctx, + &loc, + &trace] (strings& cdb_names, + const names& ns, + const dir_path& base) -> const string& + { + // Check that names and paths match. Return false if this entry + // already exist. + // + // Note that before we also checked that the same paths are not used + // across contexts. But, actually, there doesn't seem to be anything + // wrong with that and this can actually be useful, for example, + // when developing build system modules. + // + auto check = [&loc] (const string& n, const path& p) + { + for (const unique_ptr<compiledb>& db: compiledbs) + { + bool nm (db->name == n); + bool pm (db->path == p); + + if (nm != pm) + fail (loc) << "inconsistent compilation database names/paths" << + info << p << " is called " << n << + info << db->path << " is called " << db->name; + + if (nm) + return false; + } + + return true; + }; + + const string* r (&empty_string); + + bool reg (false); + size_t j (compiledbs.size ()); // First newly added database. + for (auto i (ns.begin ()); i != ns.end (); ++i) + { + // Each element has the <name>[@<path>] form. + // + // The special `-` <name> signifies stdout. + // + // If <path> is absent, then the file is called <name>.json and + // placed into the output directory of the amalgamation or project + // root scope (passed as the base argument). + // + // If <path> is (syntactically) a directory, then the file path is + // <path>/<name>.json. + // + if (!i->simple () || i->empty ()) + fail (loc) << "invalid compilation database name '" << *i << "'"; + + // Don't allow names that have (or are) directory components. + // + if (compiledb_name_to_path (*i)) + fail (loc) << "directory component in compilation database name '" + << *i << "'"; + + string n (i->value); + + path p; + if (i->pair) + { + ++i; + + if (n == "-") + fail (loc) << "compilation database path specified for stdout " + << "name"; + try + { + if (i->directory ()) + p = i->dir / n + ".json"; + else if (i->file ()) + { + if (i->dir.empty ()) + p = path (i->value); + else + p = i->dir / i->value; + } + else + throw invalid_path (""); + + if (p.relative ()) + p.complete (); + + p.normalize (); + } + catch (const invalid_path&) + { + fail (loc) << "invalid compilation database path '" << *i + << "'"; + } + } + else if (n != "-") + { + p = base / n + ".json"; + } + + if (check (n, p)) + { + reg = compiledbs.empty (); // First time. + +#ifdef BUILD2_BOOTSTRAP + fail (loc) << "compilation database requested during bootstrap"; +#else + if (n == "-") + compiledbs.push_back ( + unique_ptr<compiledb> ( + new compiledb_stdout (n))); + else + compiledbs.push_back ( + unique_ptr<compiledb> ( + new compiledb_file (n, move (p)))); +#endif + } + + // We may end up with duplicates via the config.cc.compiledb + // logic. + // + auto k (find (cdb_names.begin (), cdb_names.end (), n)); + + if (k == cdb_names.end ()) + { + cdb_names.push_back (move (n)); + r = &cdb_names.back (); + } + else + r = &*k; + } + + // Register context operation callback for compiledb generation. + // + // We have two complications here: + // + // 1. We could be performing all this from the load phase that + // interrupted the match phase, which means the point where the + // pre callback would have been called is already gone (but the + // post callback will still be called). This will happen if we, + // say, import a project that has a compilation database from a + // project that doesn't. + // + // (Note that if you think that this can be solved by simply + // always registering the callbacks, regardless of whether we + // have any databases or not, consider a slightly different + // scenario where we import a project that loads the cc module + // from a project that does not). + // + // What we are going to do in this case is simply call the pre + // callback manually. + // + // 2. We could again be performing all this from the load phase that + // interrupted the match phase, but this time the pre callback + // has already been called, which means there will be no pre() + // call for the newly added database(s). This will happen if we, + // say, import a project that has a compilation database from a + // project that also has one. + // + // Again, what we are going to do in this case is simply call the + // pre callback for the new database(s) manually. + // + if (reg) + ctx.operation_callbacks.emplace ( + perform_update_id, + context::operation_callback {&compiledb_pre, &compiledb_post}); + + if (ctx.load_generation > 1) + { + action a (ctx.current_action ()); + + if (a.inner_action () == perform_update_id) + { + if (reg) // Case #1. + { + l6 ([&]{trace << "direct compiledb_pre for context " << &ctx;}); + compiledb_pre (ctx, a, action_targets {}); + } + else // Case #2. + { + size_t n (compiledbs.size ()); + + if (j != n) + { + l6 ([&]{trace << "additional compiledb for context " << &ctx;}); + + for (; j != n; ++j) + compiledbs[j]->pre (ctx); + } + } + } + } + + return *r; + }; + + lookup l; + + // config.cc.compiledb + // + // The semantics of this value is as follows: + // + // Location: outermost amalgamation that loads the cc module. + // Name filter: enable from this scope unless specified explicitly. + // Type filter: enable from this scope unless specified explicitly. + // + // Note: save omitted. + // + optional<string> enable_filter; + + l = lookup_config (rs, "config.cc.compiledb", 0, &save_compiledb_name); + if (l && belongs (l)) + { + l6 ([&]{trace << "config.cc.compiledb specified on " << rs;}); + + const names& ns (cast<names> (l)); + + // Make sure it's one name/path. + // + size_t n (ns.size ()); + if (n == 0 || n != (ns.front ().pair ? 2 : 1)) + fail (loc) << "invalid compilation database name '" << ns << "'"; + + // Detect and translate just <name> which is actually <path> to the + // <name>@<path> form: + // + // - The <name> part is the name of the directory where the database + // file will reside (typically project/repository or package + // name). + // + // - If <path> is a directory, then the database name is + // compile_commands.json. + // + names tns; + if (n == 1) + { + const name& n (ns.front ()); + + if (optional<name> otn = compiledb_name_to_path (n)) + { + name& tn (*otn); + + // Note: the add_cdbs() call below completes and normalizes the + // path but we need to do it earlier in order to be able to + // derive the name (the last component can be `.`/`..`). + // + if (tn.dir.relative ()) + tn.dir.complete (); + + tn.dir.normalize (); + + if (!exists (tn.dir)) + fail (loc) << "compilation database directory " << tn.dir + << " does not exist"; + + if (tn.value.empty ()) + tn.value = "compile_commands.json"; + + tns.push_back (name (tn.dir.leaf ().string ())); + tns.back ().pair = '@'; + tns.push_back (move (tn)); + } + } + + // We inject the database directly into the outer amalgamation's + // module, as-if config.cc.compiledb.name was specified in its + // scope. Unless there isn't one, in which case it's us. + // + pair<scope*, core_module*> p (find_outermost ()); + + // Save the name for the name filter below. + // + enable_filter = add_cdbs ( + (p.second != nullptr ? *p.second : m).cdb_names_, + tns.empty () ? ns : tns, + p.first->out_path ()); + } + + // config.cc.compiledb.name + // + // Note: save omitted. + // + l = lookup_config (rs, + "config.cc.compiledb.name", + 0, + &save_compiledb_name); + if (l && belongs (l)) + { + l6 ([&]{trace << "config.cc.compiledb.name specified on " << rs;}); + + add_cdbs (m.cdb_names_, cast<names> (l), rs.out_path ()); + } + + // config.cc.compiledb.filter + // + // Note: save omitted. + // + l = lookup_config (rs, "config.cc.compiledb.filter"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_ = &cast<compiledb_name_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_ != nullptr) + m.cdb_filter_storage_ = *om->cdb_filter_; + + m.cdb_filter_storage_.emplace_back (*enable_filter, true); + m.cdb_filter_ = &m.cdb_filter_storage_; + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_ = om->cdb_filter_; + } + + // config.cc.compiledb.filter.input + // config.cc.compiledb.filter.output + // + // Note that filtering happens before we take into account the change + // status, which means for larger projects there would be a lot of + // targets to filter even during the incremental update. So it feels + // it would have been better to pre-lookup the target types. However, + // the targets that would normally be used are registered by other + // modules (bin, c/cxx) and which haven't been loaded yet. So instead + // we try to optimize the lookup for the commonly used targets. + // + // Note: save omitted. + // + l = lookup_config (rs, "config.cc.compiledb.filter.input"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_input_ = &cast<compiledb_type_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_input_ != nullptr) + { + m.cdb_filter_input_storage_ = *om->cdb_filter_input_; + m.cdb_filter_input_storage_.emplace_back (*enable_filter, "target"); + m.cdb_filter_input_ = &m.cdb_filter_input_storage_; + } + else + m.cdb_filter_input_ = nullptr; // Enable all. + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_input_ = om->cdb_filter_input_; + } + + l = lookup_config (rs, "config.cc.compiledb.filter.output"); + if (l && belongs (l)) // Custom. + { + m.cdb_filter_output_ = &cast<compiledb_type_filter> (l); + } + else if (enable_filter) // Override. + { + // Inherit outer filter. + // + if (om != nullptr && om->cdb_filter_output_ != nullptr) + { + m.cdb_filter_output_storage_ = *om->cdb_filter_output_; + m.cdb_filter_output_storage_.emplace_back (*enable_filter, "target"); + m.cdb_filter_output_ = &m.cdb_filter_output_storage_; + } + else + m.cdb_filter_output_ = nullptr; // Enable all. + } + else if (om != nullptr) // Inherit. + { + m.cdb_filter_output_ = om->cdb_filter_output_; + } + } + + // Register scope operation callback for cleaning module sidebuilds and + // compilation databases. + // + // It feels natural to clean this stuff up as a post operation but that + // prevents the (otherwise-empty) out root directory to be cleaned up + // (via the standard fsdir{} chain). + // + rs.operation_callbacks.emplace ( + perform_clean_id, + scope::operation_callback {&clean_callback, nullptr /*post*/}); + return true; } diff --git a/libbuild2/cc/link-rule.cxx b/libbuild2/cc/link-rule.cxx index 417cba5..a669f37 100644 --- a/libbuild2/cc/link-rule.cxx +++ b/libbuild2/cc/link-rule.cxx @@ -102,13 +102,13 @@ namespace build2 // const scope& bs (t->base_scope ()); - if (lookup l = t->lookup_original (c_export_libs, false, &bs).first) + if (lookup l = t->lookup_original (c_export_libs, &bs).first) { if (!deduplicate_export_libs (bs, cast<vector<name>> (l), r, seen)) return false; } - if (lookup l = t->lookup_original (x_export_libs, false, &bs).first) + if (lookup l = t->lookup_original (x_export_libs, &bs).first) { if (!deduplicate_export_libs (bs, cast<vector<name>> (l), r, seen)) return false; @@ -503,8 +503,8 @@ namespace build2 return false; } - // We will only chain a C source if there is also an X source or we were - // explicitly told to. + // We will only synthesize a dependency for a C source if there is also + // an X source or we were explicitly told to. // if (r.seen_c && !r.seen_x && hint.empty ()) { @@ -915,8 +915,8 @@ namespace build2 const fsdir* dir (inject_fsdir (a, t)); // Process prerequisites, pass 1: search and match prerequisite - // libraries, search obj/bmi{} targets, and search targets we do rule - // chaining for. + // libraries, search obj/bmi{} targets, and search targets we do + // dependency synthesis for. // // Also clear the binless flag if we see any source or object files. // Note that if we don't see any this still doesn't mean the library is @@ -928,7 +928,7 @@ namespace build2 // compile::apply() to unmatch them and therefore not to hinder // parallelism (or mess up for-install'ness). // - // We also create obj/bmi{} chain targets because we need to add + // We also synthesize obj/bmi{} dependencies because we need to add // (similar to lib{}) all the bmi{} as prerequisites to all the other // obj/bmi{} that we are creating. Note that this doesn't mean that the // compile rule will actually treat them all as prerequisite targets. @@ -955,6 +955,20 @@ namespace build2 auto& pts (t.prerequisite_targets[a]); size_t start (pts.size ()); + // Compile rule options specified on lib/exe{} to propagate to obj/bmi{} + // during dependency synthesis (see below). + // + struct cr_options + { + const strings* c_p; // cc.poptions + const strings* x_p; // x.poptions + const strings* c_c; // cc.coptions + const strings* x_c; // x.coptions + + bool member; // Any value came from member as opposed to group. + }; + optional<cr_options> cr_ops; // Lookup lazily. + for (prerequisite_member p: group_prerequisite_members (a, t)) { // Note that we have to recognize update=match for *(update), not just @@ -1025,7 +1039,14 @@ namespace build2 continue; } - const target*& pt (pto); + const target*& pt (pto.target); + + // Auxiliary data in prerequisite_target: + // + // - for libraries it stores link flags (lflag_whole) + // - for synthesized obj/bmi{} it strores the group flag + // + uintptr_t& pd (pto.data); // Mark (2 bits): // @@ -1045,7 +1066,7 @@ namespace build2 { binless = binless && (mod ? user_binless : false); - // Rule chaining, part 1. + // Dependency synthesis, part 1. // // Which scope shall we use to resolve the root? Unlikely, but // possible, the prerequisite is from a different project @@ -1058,6 +1079,62 @@ namespace build2 // bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + // Lookup all the relevant binary-specific compile option values if + // this hasn't already been done. + // + if (!cr_ops) + { + bool m (false); + + auto lookup = [&bs, &t, &m] (const variable& var) -> const strings* + { + // We don't want to pick anything beyond target-specific + // values (but including target type/pattern-specific) since + // they will also be in effect for obj/bmi{} and setting them + // to the same values is a waste (and, in fact, it could be + // that when this feature is not used, different obj/bmi{} are + // compiled with different options). + // + // Note that we need to take into account target type/pattern- + // specific append/prepend since it could modify the scope + // value but only be applicable to lib/exe{}. Something like + // this: + // + // cc.poptions += -DFOO + // lib{*}: cc.poptions += -DBAR + // + // Then there is this nuance: if any value came from the member + // and not from the group, then we need to override the above + // group semantics. In particular, this allows us to do: + // + // liba{hello}: cxx.poptions += -DLIBHELLO_STATIC_BUILD + // libs{hello}: cxx.poptions += -DLIBHELLO_SHARED_BUILD + // + // Note also that the variables we are dealing with are not + // overridable. + // + pair<build2::lookup, size_t> p ( + t.lookup_original (var, + lookup_limit::target_type, + &bs)); + + const strings* r (cast_null<strings> (p.first)); + + if (r != nullptr) + m = m || p.second == 1; // Found in the target itself. + + return r; + }; + + cr_ops = cr_options {lookup (c_poptions), lookup (x_poptions), + lookup (c_coptions), lookup (x_coptions), + false}; + cr_ops->member = m; + } + + if (group && cr_ops->member) + group = false; + const target_type& rtt (mod ? (group ? bmi::static_type : tts.bmi) : (group ? obj::static_type : tts.obj)); @@ -1091,44 +1168,278 @@ namespace build2 // obj/bmi{} is always in the out tree. Note that currently it could // be the group -- we will pick a member in part 2 below. // + // Note: d is used below. + // pair<target&, ulock> r ( search_new_locked ( ctx, rtt, d, dir_path (), *cp.tk.name, nullptr, cp.scope)); - // If we shouldn't clean obj{}, then it is fair to assume we - // shouldn't clean the source either (generated source will be in - // the same directory as obj{} and if not, well, go find yourself - // another build system ;-)). - // - if (skip (r.first)) - { - pt = nullptr; - continue; - } + const target& cpt (r.first); + bool locked (r.second.owns_lock ()); - // Either set of verify the bin.binless value on this bmi*{} target + // Either set or verify the bin.binless value on this bmi*{} target // (see config_data::b_binless for semantics). // if (mod) { - if (r.second.owns_lock ()) + if (locked) { if (user_binless) r.first.assign (b_binless) = true; } else { - lookup l (r.first[b_binless]); + lookup l (cpt[b_binless]); if (user_binless ? !cast_false<bool> (l) : l.defined ()) fail << "synthesized dependency for prerequisite " << p << " would be incompatible with existing target " - << r.first << + << cpt << info << "incompatible bin.binless value"; } } - pt = &r.first; + // Binary-specific compile options. + // + // Propagate compile rule options (*.poptions and *.coptions) + // specified on the binary to the obj/bmi{} targets that we + // synthesize. + // + // The semantics we are aiming for is as-if they were set on the + // obj/bmi{} target at the end of the buildfile (to be precise, at + // the end of loading all buildfiles for the scope). + // + // While ideally we would like to prevent sharing such obj/bmi{} + // between multiple binaries or for the user to specify any compile + // options explicitly, this is not easy to do (since once the + // options are set, we don't know who set them: same binary on the + // previous operation batch, another binary, or user). Instead, we + // are going to approximate this by making sure the options (or + // their absence) match. + // + // Note also that we don't touch user-specified obj/bmi{} + // prerequisites (neither set nor verify). In particular, this + // allows customizing compile options for specific translation + // units. + // + // NOTE: keep last since unlocks the lock. + // + { + // If we have any value, then either set them (locked) or make + // sure they all match (unlocked). + // + // Note that the case where one lib/exe{} specifies a value while + // the other doesn't specify any (and they share a synthesized + // dependency) will be racy to verify. Getting rid of this race + // will be difficult because in the verify case we can't say who + // set the value on obj/bmi{}. We could set some sort of a marker + // variable but then it means we would need to look it up for + // every lib/exe{} (whether they use this feature or not, and most + // won't). However, we can handle the common case based on the + // target being newly created (as opposed to being mentioned in + // the buildfile; see target_decl). + // + auto check = [&bs, &p, &d] (const variable& var, + const target& et, + const strings* e, + const target& st) + { + // If the expected value is NULL, then just make sure this + // variable is not set on the target. Otherwise, compare the + // result of the normal lookup and if it matches, then we assume + // it's good regardless of where it comes from (this covers all + // the corner cases which we cannot verify precisely; see above + // for details). + // + const strings* v; + size_t vd (0); + + if (e == nullptr) + { + v = cast_null<strings> (st.vars[var]); + + if (v == nullptr) + return; + } + else + { + // Optimize the lookup for the common case. + // + pair<lookup, size_t> p ( + st.lookup_original ( + var, + d.empty () ? &bs : nullptr)); + + v = cast_null<strings> (p.first); + + if (v != nullptr && *v == *e) + return; + + vd = p.second; + } + + diag_record dr (fail); + + dr << "synthesized dependency for prerequisite " << p + << " would be incompatible with existing target " << st << + info << "variable " << var << " value mismatch"; + + if (e == nullptr) // Expected to be absent. + { + dr << info << st << " value: "; to_stream_quoted (dr.os, *v); + dr << info << et << " value is absent"; + } + else if (v == nullptr) // Expected to be present. + { + dr << info << st << " value is absent"; + dr << info << et << " value: "; to_stream_quoted (dr.os, *e); + } + else // Expected to match. + { + dr << info << st << " value: "; to_stream_quoted (dr.os, *v); + dr << info << et << " value: "; to_stream_quoted (dr.os, *e); + + if (vd > 1) + { + if (const target* g = st.group) + dr << info << st << " value came from group " << *g; + } + } + }; + + auto set = [&r] (const variable& var, const strings& v) + { + // One nuance here is that target type/pattern-specific + // append/prepend/assign specified for obj/bmi{} will not be + // in effect for options specified on lib/exe{}. For example: + // + // obj{*}: cc.poptions = -DFOO + // lib{bar}: cc.poptions += -DBAR + // + // It doesn't seem there is anything sensible we can do about + // it automatically other than documenting this nuance and + // suggesting the user adds lib/exe{} to such a pattern. + // + r.first.assign (var) = v; + }; + + bool absent (false); + if (cr_ops->c_p != nullptr || cr_ops->x_p != nullptr || + cr_ops->c_c != nullptr || cr_ops->x_c != nullptr || + (absent = !operator>= (cpt.decl, target_decl::implied))) // VC14 + { + if (locked) + { + if (!absent) + { + if (cr_ops->c_p != nullptr) set (c_poptions, *cr_ops->c_p); + if (cr_ops->x_p != nullptr) set (x_poptions, *cr_ops->x_p); + if (cr_ops->c_c != nullptr) set (c_coptions, *cr_ops->c_c); + if (cr_ops->x_c != nullptr) set (x_coptions, *cr_ops->x_c); + } + + // @@ PERF: maybe pass the lock to search_new_locked() below? + // + r.second.unlock (); + locked = false; + } + else + { + if (absent && cpt.vars.empty ()) + ; // Optimize for the common case. + else + { + // If the values came from the group, then use the group in + // diagnostics. + // + const target& et (group ? *t.group : t); + + check (c_poptions, et, cr_ops->c_p, cpt); + check (x_poptions, et, cr_ops->x_p, cpt); + check (c_coptions, et, cr_ops->c_c, cpt); + check (x_coptions, et, cr_ops->x_c, cpt); + } + } + + // Verify member/group consistency. + // + // The check above doesn't quite work for libraries where the + // one that specified any options will likely end up + // synthesizing obja/objs{} targets (see the group logic above) + // while the one that didn't -- obj{}. So we also need to check + // obj{} vs obj[as]{} consistency if both are synthesized. + // + // (This is actually even hairier than that: sometimes, the + // obj{} library will race ahead in its matching and manage to + // create the obja/objs{} prerequisite. In which case we will + // end up failing the above check, which will be quite confusing + // and which is the reason we have added the "value came from + // group" info above). + // + // Implementing this verification in this racing environment is + // challanging, to put it mildly. So what we are going to do is, + // in case of a group, also enter the member (which we will be + // doing anyway shortly). If we were the ones who created it, + // then we have "staked out" our view of its variables and any + // subsequent attempts to enter it will trigger the above + // check. If, however, this target was already there, then we + // verify that it's consistent with the values we expect. + // + if (group) + { + const target_type& tt (mod ? tts.bmi : tts.obj); + + // @@ PERF: maybe we should stash the member in pd and avoid + // another search when we pick the member? (Though obj{} + // might also not be synthesized.) + + pair<target&, ulock> r ( + search_new_locked ( + ctx, tt, d, dir_path (), *cp.tk.name, nullptr, cp.scope)); + + if (r.second.owns_lock ()) + r.second.unlock (); + else + { + const target& cmt (r.first); + + if (!absent || + !operator>= (cmt.decl, target_decl::implied)) // VC14 + { + if (absent && cmt.vars.empty ()) + ; // Optimize for the common case. + else + { + check (c_poptions, cpt, cr_ops->c_p, cmt); + check (x_poptions, cpt, cr_ops->x_p, cmt); + check (c_coptions, cpt, cr_ops->c_c, cmt); + check (x_coptions, cpt, cr_ops->x_c, cmt); + } + } + } + } + } + } + + if (locked) + r.second.unlock (); + + // If we shouldn't clean obj{}, then it is fair to assume we + // shouldn't clean the source either (generated source will be in + // the same directory as obj{} and if not, well, go find yourself + // another build system ;-)). + // + // Note: should be done after setting/verifying variables since can + // be an operation batch. + // + if (skip (cpt)) + { + pt = nullptr; + continue; + } + + pt = &cpt; + pd = group ? 1 : 0; mk = mod ? 2 : 1; } else if (p.is_a<libx> () || @@ -1310,7 +1621,7 @@ namespace build2 // if (const string* t = cast_null<string> ( ft->state[a].lookup_original ( - c_type, true /* target_only */).first)) + c_type, lookup_limit::target).first)) { if (recursively_binless (*t)) continue; @@ -1330,7 +1641,7 @@ namespace build2 { auto find = [&t, &bs] (const variable& v) -> lookup { - return t.lookup_original (v, false, &bs).first; + return t.lookup_original (v, &bs).first; }; auto has_simple = [] (lookup l) @@ -1729,9 +2040,9 @@ namespace build2 } } - // Process prerequisites, pass 2: finish rule chaining but don't start - // matching anything yet since that may trigger recursive matching of - // bmi{} targets we haven't completed yet. Hairy, I know. + // Process prerequisites, pass 2: finish dependency synthesis but don't + // start matching anything yet since that may trigger recursive matching + // of bmi{} targets we haven't completed yet. Hairy, I know. // // Parallel prerequisites/prerequisite_targets loop. @@ -1759,14 +2070,14 @@ namespace build2 // Note that if this is a library not to be cleaned, we keep it // marked for completion (see the next phase). } - else if (mk == 1 || mk == 2) // Source/module chain. + else if (mk == 1 || mk == 2) // Synthesized source/module dependency. { bool mod (mk == 2); // p is_a x_mod mk = 1; const target& rt (*pt); - bool group (!p.prerequisite.belongs (t)); // Group's prerequisite. + bool group (pd != 0); // Group's prerequisite (see pass 1). // If we have created a obj/bmi{} target group, pick one of its // members; the rest would be primarily concerned with it. @@ -1812,7 +2123,7 @@ namespace build2 } // Add our lib*{} (see the export.* machinery for details) and - // bmi*{} (both original and chained; see module search logic) + // bmi*{} (both original and synthesized; see module search logic) // prerequisites. // // Note that we don't resolve lib{} to liba{}/libs{} here @@ -1833,7 +2144,8 @@ namespace build2 size_t j (start); for (prerequisite_member p: group_prerequisite_members (a, t)) { - const target* pt (pts[j++]); + const target* pt (pts[j].target); + uintptr_t& pd (pts[j++].data); if (pt == nullptr) // Note: ad hoc is taken care of. continue; @@ -1847,7 +2159,8 @@ namespace build2 { ps.push_back (p.as_prerequisite ()); } - else if (x_mod != nullptr && p.is_a (*x_mod)) // Chained module. + else if (x_mod != nullptr && p.is_a (*x_mod)) // Synthesized + // module dependency. { // Searched during pass 1 but can be NULL or marked. // @@ -1857,7 +2170,7 @@ namespace build2 // was a group, then we would have picked up a member. So // here we may have to "unpick" it. // - bool group (j < i && !p.prerequisite.belongs (t)); + bool group (j < i && pd != 0); unmark (pt); ps.push_back (prerequisite (group ? *pt->group : *pt)); @@ -1975,7 +2288,7 @@ namespace build2 lookup l (p.prerequisite.vars[var]); if (!l.defined ()) - l = pt->lookup_original (var, true /* target_only */).first; + l = pt->lookup_original (var, lookup_limit::target).first; if (!l.defined ()) { @@ -1997,7 +2310,8 @@ namespace build2 mark (pt, mk); } - // Process prerequisites, pass 3: match everything and verify chains. + // Process prerequisites, pass 3: match everything and verify synthesized + // dependencies. // // Wait with unlocked phase to allow phase switching. @@ -2051,7 +2365,9 @@ namespace build2 i = start; for (prerequisite_member p: group_prerequisite_members (a, t)) { - const target*& pt (pts[i++]); + const target*& pt (pts[i].target); + uintptr_t& pd (pts[i++].data); + // Skipped or not marked for completion. // @@ -2088,7 +2404,7 @@ namespace build2 if (&tp != &tp1) { - bool group (!p.prerequisite.belongs (t)); + bool group (pd != 0); // See pass 1. const target_type& rtt (mod ? (group ? bmi::static_type : tts.bmi) @@ -2193,12 +2509,19 @@ namespace build2 // appear after the preceding static library of which this binless // library is a dependency. // + // Note that we omit the duplicate suppression if we are linking the + // whole archive since the previous instance may not necessarily do + // the same (see GH issue #411; we could have complicated things and + // stored the flag in appended_libraries but it doesn't feel + // worthwhile in this case). + // // From the process_libraries() semantics we know that this callback // is always called and always after the options callbacks. // - appended_library* al (l != nullptr - ? &d.ls.append (*l, d.args.size ()) - : d.ls.append (ns, d.args.size ())); + appended_library* al ( + f & lflag_whole ? nullptr : + l != nullptr ? &d.ls.append (*l, d.args.size ()) : + d.ls.append (ns, d.args.size ())); if (al != nullptr && al->end != appended_library::npos) // Closed. { @@ -2397,13 +2720,13 @@ namespace build2 else { d.args.push_back ("-Wl,--whole-archive"); - d.args.push_back (move (p)); + d.args.push_back (move (p)); p.clear (); d.args.push_back ("-Wl,--no-whole-archive"); - goto done; } } - d.args.push_back (move (p)); + if (!p.empty ()) + d.args.push_back (move (p)); } if (d.cs != nullptr) @@ -3199,10 +3522,6 @@ namespace build2 // args.push_back ("/NOLOGO"); - // Add /MACHINE. - // - args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); - // For utility libraries use thin archives if possible. // // LLVM's lib replacement had the /LLVMLIBTHIN option at least from @@ -3269,7 +3588,16 @@ namespace build2 { // Are we using the compiler or the linker (e.g., link.exe) directly? // - bool ldc (tsys != "win32-msvc"); + bool ldc; + + if (tsys == "win32-msvc") + { + args.push_back ("/NOLOGO"); + ldc = false; + } + else + ldc = true; + if (ldc) { @@ -3679,6 +4007,20 @@ namespace build2 if (lt.shared_library () && (tsys == "win32-msvc" || tsys == "mingw32")) reli = relative (find_adhoc_member<libi> (t)->path ()); + if (tsys == "win32-msvc") + { + // Add /MACHINE unless there is a custom value (/MACHINE:ARM64EC). + // + // Note that we don't bother hashing it since to change its value one + // would have to use a different MSVC toolchain (which means things + // would be rebuilt from scratch anyway). + // + if (!find_option_prefix ("/MACHINE:", args, true)) + { + args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); + } + } + const process_path* ld (nullptr); if (lt.static_library ()) { @@ -3702,15 +4044,10 @@ namespace build2 // Using link.exe directly. // ld = &cast<process_path> (rs["bin.ld.path"]); - args.push_back ("/NOLOGO"); if (ot == otype::s) args.push_back ("/DLL"); - // Add /MACHINE. - // - args.push_back (msvc_machine (cast<string> (rs[x_target_cpu]))); - // Unless explicitly enabled with /INCREMENTAL, disable incremental // linking (it is implicitly enabled if /DEBUG is specified). The // reason is the .ilk file: its name cannot be changed and if we diff --git a/libbuild2/cc/module.cxx b/libbuild2/cc/module.cxx index cf6c6e4..a3c64d9 100644 --- a/libbuild2/cc/module.cxx +++ b/libbuild2/cc/module.cxx @@ -22,6 +22,12 @@ namespace build2 { namespace cc { + // cc.core_module + // + const string core_module::name ("cc.core.config"); + + // x.config_module + // void config_module:: guess (scope& rs, const location& loc, const variable_map&) { @@ -891,6 +897,9 @@ namespace build2 config::save_environment (rs, xi.platform_environment); } + // x module + // + // Global cache of ad hoc importable headers. // // The key is a hash of the system header search directories diff --git a/libbuild2/cc/module.hxx b/libbuild2/cc/module.hxx index 4213516..d4e9a67 100644 --- a/libbuild2/cc/module.hxx +++ b/libbuild2/cc/module.hxx @@ -14,6 +14,8 @@ #include <libbuild2/cc/common.hxx> +#include <libbuild2/cc/compiledb.hxx> + #include <libbuild2/cc/compile-rule.hxx> #include <libbuild2/cc/link-rule.hxx> #include <libbuild2/cc/install-rule.hxx> @@ -27,6 +29,35 @@ namespace build2 { struct compiler_info; + // cc.core module + // + class core_module: public build2::module + { + public: + static const string name; + + explicit + core_module (const core_module* om) + : outer_module_ (om) + { + } + + public: + const core_module* outer_module_; + + strings cdb_names_; + + const compiledb_name_filter* cdb_filter_ = nullptr; + const compiledb_type_filter* cdb_filter_input_ = nullptr; + const compiledb_type_filter* cdb_filter_output_ = nullptr; + + compiledb_name_filter cdb_filter_storage_; + compiledb_type_filter cdb_filter_input_storage_; + compiledb_type_filter cdb_filter_output_storage_; + }; + + // x.config module + // class LIBBUILD2_CC_SYMEXPORT config_module: public build2::module, public config_data { @@ -92,13 +123,36 @@ namespace build2 // struct header_key { - path file; + // We used to use path comparison/hash which are case-insensitive on + // Windows. While this sounds right on the surface, the catch is that + // our target names are always case-sensitive, even on Windows. So + // what can happen is that the same header spelled in different case + // ends up with distinct targets but trying to occupy the same cache + // entry. See GH issue #390 for details. + // + // The conceptually correct way to fix this would be to actualize the + // header name. But that would be prohibitively expensive, especially + // on Windows. Plus the header may be generated and thus not yet + // exist. + // + // On the other hand, we can already end up with different targets + // mapped to the same filesystem entry in other situations (h{} vs + // hxx{} is the canonical example). And we are ok with that provided + // they have noop recipes. So it feels like the simplest solution here + // is to go along by having case-sensitive cache entries. This means + // that auto-generated headers will have to be included using the same + // spelling, but that seems like a sensible restriction (doing + // otherwise won't be portable). + // + path file; size_t hash; friend bool operator== (const header_key& x, const header_key& y) { - return x.file == y.file; // Note: hash was already compared. + // Note: hash was already compared. + // + return x.file.string () == y.file.string (); } }; @@ -130,6 +184,8 @@ namespace build2 msvc_library_search_dirs (const compiler_info&, scope&) const; }; + // x module + // class LIBBUILD2_CC_SYMEXPORT module: public build2::module, public virtual common, public link_rule, @@ -139,7 +195,6 @@ namespace build2 public predefs_rule { public: - explicit module (data&& d, const scope& rs) : common (move (d)), link_rule (move (d)), diff --git a/libbuild2/cc/msvc.cxx b/libbuild2/cc/msvc.cxx index d21969c..416df36 100644 --- a/libbuild2/cc/msvc.cxx +++ b/libbuild2/cc/msvc.cxx @@ -33,10 +33,10 @@ namespace build2 const char* msvc_cpu (const string& cpu) { - const char* m (cpu == "i386" || cpu == "i686" ? "x86" : - cpu == "x86_64" ? "x64" : + const char* m (cpu == "x86_64" ? "x64" : + cpu == "i386" || cpu == "i686" ? "x86" : + cpu == "aarch64" ? "arm64" : cpu == "arm" ? "arm" : - cpu == "arm64" ? "arm64" : nullptr); if (m == nullptr) @@ -51,10 +51,10 @@ namespace build2 const char* msvc_machine (const string& cpu) { - const char* m (cpu == "i386" || cpu == "i686" ? "/MACHINE:x86" : - cpu == "x86_64" ? "/MACHINE:x64" : + const char* m (cpu == "x86_64" ? "/MACHINE:x64" : + cpu == "i386" || cpu == "i686" ? "/MACHINE:x86" : + cpu == "aarch64" ? "/MACHINE:ARM64" : cpu == "arm" ? "/MACHINE:ARM" : - cpu == "arm64" ? "/MACHINE:ARM64" : nullptr); if (m == nullptr) diff --git a/libbuild2/cc/pkgconfig.cxx b/libbuild2/cc/pkgconfig.cxx index 046fbc8..79a38ea 100644 --- a/libbuild2/cc/pkgconfig.cxx +++ b/libbuild2/cc/pkgconfig.cxx @@ -1,6 +1,8 @@ // file : libbuild2/cc/pkgconfig.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file +#include <libbuild2/cc/pkgconfig.hxx> + #include <libbuild2/scope.hxx> #include <libbuild2/target.hxx> #include <libbuild2/context.hxx> @@ -18,7 +20,6 @@ #include <libbuild2/cc/utility.hxx> #include <libbuild2/cc/common.hxx> -#include <libbuild2/cc/pkgconfig.hxx> #include <libbuild2/cc/compile-rule.hxx> #include <libbuild2/cc/link-rule.hxx> @@ -707,6 +708,7 @@ namespace build2 cmp ("user32") || cmp ("userenv") || cmp ("uuid") || + cmp ("uxtheme") || cmp ("version") || cmp ("windowscodecs") || cmp ("winhttp") || @@ -2256,7 +2258,7 @@ namespace build2 const string& t ( cast<string> ( l.state[a].lookup_original ( - c_type, true /* target_only */).first)); + c_type, lookup_limit::target).first)); // If common, then only save the language (the rest could be // static/shared-specific; strictly speaking even the language could @@ -2276,7 +2278,7 @@ namespace build2 // if (cast_false<bool> (l.lookup_original ( ctx.var_pool["bin.whole"], - true /* target_only */).first)) + lookup_limit::target).first)) { os << endl << "bin.whole = true" << endl; diff --git a/libbuild2/cc/windows-manifest.cxx b/libbuild2/cc/windows-manifest.cxx index 14f4a53..ff77327 100644 --- a/libbuild2/cc/windows-manifest.cxx +++ b/libbuild2/cc/windows-manifest.cxx @@ -23,8 +23,9 @@ namespace build2 const char* windows_manifest_arch (const string& tcpu) { - const char* pa (tcpu == "i386" || tcpu == "i686" ? "x86" : - tcpu == "x86_64" ? "amd64" : + const char* pa (tcpu == "x86_64" ? "amd64" : + tcpu == "i386" || tcpu == "i686" ? "x86" : + tcpu == "aarch64" ? "arm64" : nullptr); if (pa == nullptr) |