// file : libbuild2/adhoc-rule-buildscript.cxx -*- C++ -*- // license : MIT; see accompanying LICENSE file #include #include #include // try_rm_file(), path_entry() #include #include #include #include #include #include #include // path_perms(), auto_rmfile #include #include #include // attributes #include #include using namespace std; namespace build2 { static inline void hash_script_vars (sha256& cs, const build::script::script& s, const scope& bs, const target& t, names& storage) { auto& vp (bs.var_pool ()); for (const string& n: s.vars) { cs.append (n); lookup l; if (const variable* var = vp.find (n)) l = t[var]; cs.append (!l.defined () ? '\x1' : l->null ? '\x2' : '\x3'); if (l) { storage.clear (); names_view ns (reverse (*l, storage, true /* reduce */)); for (const name& n: ns) to_checksum (cs, n); } } } // How should we hash target and prerequisite sets ($> and $<)? We could // hash them as target names (i.e., the same as the $>/< content) or as // paths (only for path-based targets). While names feel more general, they // are also more expensive to compute. And for path-based targets, path is // generally a good proxy for the target name. Since the bulk of the ad hoc // recipes will presumably be operating exclusively on path-based targets, // let's do it both ways. // static inline void hash_target (sha256& cs, const target& t, names& storage) { if (const path_target* pt = t.is_a ()) cs.append (pt->path ().string ()); else { storage.clear (); t.as_name (storage); for (const name& n: storage) to_checksum (cs, n); } }; // The script can reference a program in one of four ways: // // 1. As an (imported) target (e.g., $cli) // // 2. As a process_path_ex (e.g., $cxx.path). // // 3. As a builtin (e.g., sed) // // 4. As a program path/name. // // When it comes to change tracking, there is nothing we can do for (4) (the // user can track its environment manually with depdb-env) and there is // nothing to do for (3) (assuming builtin semantics is stable/backwards- // compatible). The (2) case is handled automatically by hashing all the // variable values referenced by the script (see below), which in case of // process_path_ex includes the checksums (both executable and environment), // if available. // // This leaves the (1) case, which itself splits into two sub-cases: the // target comes with the dependency information (e.g., imported from a // project via an export stub) or it does not (e.g., imported as installed). // We don't need to do anything extra for the first sub-case since the // target's state/mtime can be relied upon like any other prerequisite. // Which cannot be said about the second sub-case, where we reply on // checksum that may be included as part of the target metadata. // // So what we are going to do is hash checksum metadata of every executable // prerequisite target that has it (we do it here in order to include ad hoc // prerequisites, which feels like the right thing to do; the user may mark // tools as ad hoc in order to omit them from $<). // static inline void hash_prerequisite_target (sha256& cs, sha256& exe_cs, sha256& env_cs, const target& pt, names& storage) { hash_target (cs, pt, storage); if (const exe* et = pt.is_a ()) { if (const string* c = et->lookup_metadata ("checksum")) { exe_cs.append (*c); } if (const strings* e = et->lookup_metadata ("environment")) { hash_environment (env_cs, *e); } } } bool adhoc_buildscript_rule:: recipe_text (const scope& s, const target_type& tt, string&& t, attributes& as) { // Handle and erase recipe-specific attributes. // optional diag; for (auto i (as.begin ()); i != as.end (); ) { attribute& a (*i); const string& n (a.name); if (n == "diag") try { diag = convert (move (a.value)); } catch (const invalid_argument& e) { fail (as.loc) << "invalid " << n << " attribute value: " << e; } else { ++i; continue; } i = as.erase (i); } checksum = sha256 (t).string (); ttype = &tt; istringstream is (move (t)); build::script::parser p (s.ctx); script = p.pre_parse (s, tt, actions, is, loc.file, loc.line + 1, move (diag), as.loc); return false; } void adhoc_buildscript_rule:: dump_attributes (ostream& os) const { // For now we dump it as an attribute whether it was specified or derived // from the script. Maybe that's ok (we use this in tests)? // if (script.diag_name) { os << " ["; os << "diag="; to_stream (os, name (*script.diag_name), quote_mode::normal, '@'); os << ']'; } } void adhoc_buildscript_rule:: dump_text (ostream& os, string& ind) const { os << ind << string (braces, '{') << endl; ind += " "; if (script.depdb_clear) os << ind << "depdb clear" << endl; script::dump (os, ind, script.depdb_preamble); script::dump (os, ind, script.diag_preamble); script::dump (os, ind, script.body); ind.resize (ind.size () - 2); os << ind << string (braces, '}'); } bool adhoc_buildscript_rule:: reverse_fallback (action a, const target_type& tt) const { // We can provide clean for a file or group target if we are providing // update. // return (a == perform_clean_id && (tt.is_a () || tt.is_a ()) && find (actions.begin (), actions.end (), perform_update_id) != actions.end ()); } using dynamic_target = build::script::parser::dynamic_target; using dynamic_targets = build::script::parser::dynamic_targets; // Return true if the path exist and is a symlink. // static inline bool path_symlink (const path& p) { pair r ( butl::path_entry (p, false /* follow_symlinks */, true /* ignore_errors */)); return r.first && r.second.type == butl::entry_type::symlink; }; struct adhoc_buildscript_rule::match_data { match_data (action a, const target& t, const scope& bs, bool temp_dir) : env (a, t, bs, temp_dir) {} build::script::environment env; build::script::default_runner run; path dd; dynamic_targets dyn_targets; const scope* bs; timestamp mt; bool symlink; bool deferred_failure; }; struct adhoc_buildscript_rule::match_data_byproduct { match_data_byproduct (action a, const target& t, const scope& bs, bool temp_dir) : env (a, t, bs, temp_dir) {} build::script::environment env; build::script::default_runner run; build::script::parser::dyndep_byproduct byp; depdb::reopen_state dd; size_t skip_count = 0; size_t pts_n; // Number of static prerequisites in prerequisite_targets. const scope* bs; timestamp mt; bool symlink; }; bool adhoc_buildscript_rule:: match (action a, target& xt, const string& h, match_extra& me) const { const target& t (xt); // See adhoc_rule::match(). // We pre-parsed the script with the assumption it will be used on a // non/file-based (or file group-based) target. Note that this should not // be possible with patterns. // if (pattern == nullptr) { // Let's not allow mixing file/group. // if ((t.is_a () != nullptr) == ttype->is_a () || (t.is_a () != nullptr) == ttype->is_a ()) ; else fail (loc) << "incompatible target types used with shared recipe" << info << "all targets must be file- or file group-based or non"; } return adhoc_rule::match (a, xt, h, me); } recipe adhoc_buildscript_rule:: apply (action a, target& t, match_extra& me) const { return apply (a, t, me, nullopt); } recipe adhoc_buildscript_rule:: apply (action a, target& t, match_extra& me, const optional& deadline) const { tracer trace ("adhoc_buildscript_rule::apply"); // Handle matching group members (see adhoc_rule::match() for background). // if (const group* g = t.group != nullptr ? t.group->is_a () : nullptr) { // Note: this looks very similar to how we handle ad hoc group members. // match_sync (a, *g, 0 /* options */); return group_recipe; // Execute the group's recipe. } // We don't support deadlines for any of these cases (see below). // if (deadline && (a.outer () || me.fallback || (a == perform_update_id && (t.is_a () || t.is_a ())))) return empty_recipe; // If this is an outer operation (e.g., update-for-test), then delegate to // the inner. // if (a.outer ()) { match_inner (a, t); return inner_recipe; } context& ctx (t.ctx); const scope& bs (t.base_scope ()); group* g (t.is_a ()); // Explicit group. // Inject pattern's ad hoc group members, if any (explicit group members // are injected after reset below). // if (g == nullptr && pattern != nullptr) pattern->apply_group_members (a, t, bs, me); // Derive file names for the target and its static/ad hoc group members, // if any. // if (a == perform_update_id || a == perform_clean_id) { if (g != nullptr) { g->reset_members (a); // See group::group_members() for background. // Note that we rely on the fact that if the group has static members, // then they always come first in members and the first static member // is a file. // for (const target& m: g->static_members) g->members.push_back (&m); g->members_static = g->members.size (); if (pattern != nullptr) { pattern->apply_group_members (a, *g, bs, me); g->members_static = g->members.size (); } if (g->members_static == 0) { if (!script.depdb_dyndep_dyn_target) fail << "group " << *g << " has no static or dynamic members"; } else { if (!g->members.front ()->is_a ()) { // We use the first static member to derive depdb path, get mtime, // etc. So it must be file-based. // fail << "first static member " << g->members.front () << " of group " << *g << " is not a file"; } // Derive paths for all the static members. // for (const target* m: g->members) if (auto* p = m->is_a ()) p->derive_path (); } } else { for (target* m (&t); m != nullptr; m = m->adhoc_member) { if (auto* p = m->is_a ()) p->derive_path (); } } } else if (g != nullptr) { // This could be, for example, configure/dist update which could need a // "representative sample" of members (in order to be able to match the // rules). So add static members unless we already have something // cached. // if (g->group_members (a).members == nullptr) // Note: not g->member. { g->reset_members (a); for (const target& m: g->static_members) g->members.push_back (&m); g->members_static = g->members.size (); if (pattern != nullptr) { pattern->apply_group_members (a, *g, bs, me); g->members_static = g->members.size (); } } } // Inject dependency on the output directory. // // We do it always instead of only if one of the targets is path-based in // case the recipe creates temporary files or some such. // // Note that we disable the prerequisite search for fsdir{} because of the // prerequisites injected by the pattern. So we have to handle this ad hoc // below. // const fsdir* dir (inject_fsdir (a, t, true /*match*/, false /*prereq*/)); // Match prerequisites. // // This is essentially match_prerequisite_members() but with support // for update=unmatch|match. // auto& pts (t.prerequisite_targets[a]); { // Re-create the clean semantics as in match_prerequisite_members(). // bool clean (a.operation () == clean_id && !t.is_a ()); // Add target's prerequisites. // for (prerequisite_member p: group_prerequisite_members (a, t)) { // Note that we have to recognize update=unmatch|match for *(update), // not just perform(update). But only actually do anything about it // for perform(update). // lookup l; // The `update` variable value, if any. include_type pi ( include (a, t, p, a.operation () == update_id ? &l : nullptr)); // Use prerequisite_target::include to signal update during match or // unmatch. // uintptr_t mask (0); if (l) { const string& v (cast (l)); if (v == "match") { if (a == perform_update_id) mask = prerequisite_target::include_udm; } else if (v == "unmatch") { if (a == perform_update_id) mask = include_unmatch; } else if (v != "false" && v != "true" && v != "execute") { fail << "unrecognized update variable value '" << v << "' specified for prerequisite " << p.prerequisite; } } // Skip excluded. // if (!pi) continue; const target& pt (p.search (t)); if (&pt == dir) // Don't add injected fsdir{} twice. continue; if (clean && !pt.in (*bs.root_scope ())) continue; prerequisite_target pto (&pt, pi); if (mask != 0) pto.include |= mask; pts.push_back (move (pto)); } // Inject pattern's prerequisites, if any. // if (pattern != nullptr) pattern->apply_prerequisites (a, t, bs, me); // Start asynchronous matching of prerequisites. Wait with unlocked // phase to allow phase switching. // wait_guard wg (ctx, ctx.count_busy (), t[a].task_count, true); for (const prerequisite_target& pt: pts) { if (pt.target == dir) // Don't match injected fsdir{} twice. continue; match_async (a, *pt.target, ctx.count_busy (), t[a].task_count); } wg.wait (); // Finish matching all the targets that we have started. // for (prerequisite_target& pt: pts) { if (pt.target == dir) // See above. continue; // Handle update=unmatch. // unmatch um ((pt.include & include_unmatch) != 0 ? unmatch::safe : unmatch::none); pair mr (match_complete (a, *pt.target, um)); if (um != unmatch::none) { l6 ([&]{trace << "unmatch " << *pt.target << ": " << mr.first;}); // If we managed to unmatch, blank it out so that it's not executed, // etc. Otherwise, leave it as is (but we still automatically avoid // hashing it, updating it during match in exec_depdb_dyndep(), and // making us out of date in execute_update_prerequisites()). // // The hashing part is tricky: by not hashing it we won't detect the // case where it was removed as a prerequisite altogether. The // thinking is that it was added with update=unmatch to extract some // information (e.g., poptions from a library) and those will be // change-tracked. // // Note: set the include_target flag for the updated_during_match() // check. // if (mr.first) { pt.data = reinterpret_cast (pt.target); pt.target = nullptr; pt.include |= prerequisite_target::include_target; // Note that this prerequisite could also be ad hoc and we must // clear that flag if we managed to unmatch (failed that we will // treat it as ordinary ad hoc since it has the target pointer in // data). // // But that makes it impossible to distinguish ad hoc unmatch from // ordinary unmatch prerequisites later when setting $<. Another // flag to the rescue. // if ((pt.include & prerequisite_target::include_adhoc) != 0) { pt.include &= ~prerequisite_target::include_adhoc; pt.include |= include_unmatch_adhoc; } } } } } // Read the list of dynamic targets and, optionally, fsdir{} prerequisites // from depdb, if exists (used in a few depdb-dyndep --dyn-target handling // places below). // auto read_dyn_targets = [] (path ddp, bool fsdir) -> pair { depdb dd (move (ddp), true /* read_only */); pair r; while (dd.reading ()) // Breakout loop. { string* l; auto read = [&dd, &l] () -> bool { return (l = dd.read ()) != nullptr; }; if (!read ()) // Rule id. break; // We can omit this for as long as we don't break our blank line // anchors semantics. // #if 0 if (*l != rule_id_) fail << "unable to clean dynamic target group " << t << " with old depdb"; #endif // Note that we cannot read out expected lines since there can be // custom depdb builtins. So we use the blank lines as anchors to // skip to the parts we need. // // Skip until the first blank that separated custom depdb entries from // the prerequisites list. { bool g; while ((g = read ()) && !l->empty ()) ; if (!g) break; } // Next read the prerequisites, detecting fsdir{} entries if asked. // { bool g; while ((g = read ()) && !l->empty ()) { if (fsdir) { path p (*l); if (p.to_directory ()) r.second.push_back (path_cast (move (p))); } } if (!g) break; } // Read the dynamic target files. We should always end with a blank // line. // for (;;) { if (!read () || l->empty ()) break; // Split into type and path. // size_t p (l->find (' ')); if (p == string::npos || // Invalid format. p == 0 || // Empty type. p + 1 == l->size ()) // Empty path. break; r.first.push_back ( dynamic_target {string (*l, 0, p), path (*l, p + 1, string::npos)}); } break; } return r; }; // Target path to derive the depdb path, query mtime (if file), etc. // // To derive the depdb path for a group with at least one static member we // use the path of the first member. For a group without any static // members we use the group name with the target type name as the // second-level extension. // auto target_path = [&t, g, p = path ()] () mutable -> const path& { return g == nullptr ? t.as ().path () : g->members_static != 0 ? g->members.front ()->as ().path () : (p = g->dir / (g->name + '.' + g->type ().name)); }; // See if we are providing the standard clean as a fallback. // if (me.fallback) { // For depdb-dyndep --dyn-target use depdb to clean dynamic targets. // if (script.depdb_dyndep_dyn_target) { // Note that only removing the relevant filesystem entries is not // enough: we actually have to populate the group with members since // this information could be used to clean derived targets (for // example, object files). So we just do that and let the standard // clean logic take care of them the same as static members. // // NOTE that this logic should be consistent with what we have in // exec_depdb_dyndep(). // using dyndep = dyndep_rule; function filter; if (g != nullptr) { filter = [] (mtime_target& g, const build2::file& m) { auto& ms (g.as ().members); return find (ms.begin (), ms.end (), &m) == ms.end (); }; } pair p ( read_dyn_targets (target_path () + ".d", true)); for (dynamic_target& dt: p.first) { path& f (dt.path); // Resolve target type. Clean it as file if unable to. // const target_type* tt (bs.find_target_type (dt.type)); if (tt == nullptr) tt = &file::static_type; if (g != nullptr) { pair r ( dyndep::inject_group_member (a, bs, *g, move (f), *tt, filter)); if (r.second) g->members.push_back (&r.first); } else { // Note that here we don't bother cleaning any old dynamic targets // -- the more we can clean, the merrier. // dyndep::inject_adhoc_group_member (a, bs, t, move (f), *tt); } } // Enter fsdir{} prerequisites. // // See the add lambda in exec_depdb_dyndep() for background. // for (dir_path& d: p.second) { dir_path o; string n; // For GCC 13 -Wdangling-reference. const fsdir& dt (search (t, move (d), move (o), move (n), nullptr, nullptr)); match_sync (a, dt); pts.push_back (prerequisite_target (&dt, true /* adhoc */)); } } return g == nullptr ? perform_clean_file : perform_clean_group; } // If we have any update during match prerequisites, now is the time to // update them. // // Note that we ignore the result and whether it renders us out of date, // leaving it to the common execute logic in perform_update_*(). // // Note also that update_during_match_prerequisites() spoils // prerequisite_target::data. // if (a == perform_update_id) update_during_match_prerequisites (trace, a, t); // See if this is not update or not on a file/group-based target. // if (a != perform_update_id || !(g != nullptr || t.is_a ())) { // Make sure we get small object optimization. // if (deadline) { return [dv = *deadline, this] (action a, const target& t) { return default_action (a, t, dv); }; } else { return [this] (action a, const target& t) { return default_action (a, t, nullopt); }; } } // This is a perform update on a file or group target. // // See if this is the simple case with only static dependencies. // if (!script.depdb_dyndep) { return [this] (action a, const target& t) { return perform_update_file_or_group (a, t); }; } // This is a perform update on a file or group target with extraction of // dynamic dependency information either in the depdb preamble // (depdb-dyndep without --byproduct) or as a byproduct of the recipe body // execution (depdb-dyndep with --byproduct). // // For the former case, we may need to add additional prerequisites (or // even target group members). We also have to save any such additional // prerequisites in depdb so that we can check if any of them have changed // on subsequent updates. So all this means that we have to take care of // depdb here in apply() instead of perform_*() like we normally do. We // also do things in slightly different order due to the restrictions // impose by the match phase. // // The latter case (depdb-dyndep --byproduct) is sort of a combination // of the normal dyndep and the static case: we check the depdb during // match but save after executing the recipe. // // Note that the C/C++ header dependency extraction is the canonical // example and all this logic is based on the prior work in the cc module // where you can often find more detailed rationale for some of the steps // performed (like the fsdir update below). // Re-acquire fsdir{} specified by the user, similar to inject_fsdir() // (which we have disabled; see above). // if (dir == nullptr) { for (const target* pt: pts) { if (pt != nullptr) { if (const fsdir* dt = pt->is_a ()) { if (dt->dir == t.dir) { dir = dt; break; } } } } } if (dir != nullptr) fsdir_rule::perform_update_direct (a, *dir); // Because the depdb preamble can access $<, we have to blank out all the // ad hoc prerequisites. Since we will still need them later, we "move" // them to the auxiliary data member in prerequisite_target (see // execute_update_prerequisites() for details). // // Note: set the include_target flag for the updated_during_match() check. // for (prerequisite_target& p: pts) { // Note that fsdir{} injected above is adhoc. // if (p.target != nullptr && p.adhoc ()) { p.data = reinterpret_cast (p.target); p.target = nullptr; p.include |= prerequisite_target::include_target; } } const path& tp (target_path ()); // Note that while it's tempting to turn match_data* into recipes, some of // their members are not movable. And in the end we will have the same // result: one dynamic memory allocation. // unique_ptr md; unique_ptr mdb; dynamic_targets old_dyn_targets; if (script.depdb_dyndep_byproduct) { mdb.reset (new match_data_byproduct ( a, t, bs, script.depdb_preamble_temp_dir)); } else { md.reset (new match_data (a, t, bs, script.depdb_preamble_temp_dir)); // If the set of dynamic targets can change based on changes to the // inputs (say, each entity, such as a type, in the input file gets its // own output file), then we can end up with a large number of old // output files laying around because they are not part of the new // dynamic target set. So we try to clean them up based on the old depdb // information, similar to how we do it for perform_clean above (except // here we will just keep the list of old files). // // Note: do before opening depdb, which can start over-writing it. // // We also have to do this speculatively, without knowing whether we // will need to update. Oh, well, being dynamic ain't free. // if (script.depdb_dyndep_dyn_target) old_dyn_targets = read_dyn_targets (tp + ".d", false).first; } depdb dd (tp + ".d"); // NOTE: see the "static dependencies" version (with comments) below. // // NOTE: We use blank lines as anchors to skip directly to certain entries // (e.g., dynamic targets). So make sure none of the other entries // can be blank (for example, see `depdb string` builtin). // // NOTE: KEEP IN SYNC WITH read_dyn_targets ABOVE! // if (dd.expect (" 1") != nullptr) l4 ([&]{trace << "rule mismatch forcing update of " << t;}); if (dd.expect (checksum) != nullptr) l4 ([&]{trace << "recipe text change forcing update of " << t;}); if (!script.depdb_clear) { names storage; sha256 prq_cs, exe_cs, env_cs; for (const prerequisite_target& p: pts) { if (const target* pt = (p.target != nullptr ? p.target : p.adhoc () ? reinterpret_cast (p.data) : nullptr)) { if ((p.include & include_unmatch) != 0) // Skip update=unmatch. continue; hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage); } } { sha256 cs; hash_script_vars (cs, script, bs, t, storage); if (dd.expect (cs.string ()) != nullptr) l4 ([&]{trace << "recipe variable change forcing update of " << t;}); } // Static targets and prerequisites (there can also be dynamic targets; // see dyndep --dyn-target). // { sha256 tcs; if (g == nullptr) { // There is a nuance: in an operation batch (e.g., `b update // update`) we will already have the dynamic targets as members on // the subsequent operations and we need to make sure we don't treat // them as static. Using target_decl to distinguish the two seems // like a natural way. // for (const target* m (&t); m != nullptr; m = m->adhoc_member) { if (m->decl == target_decl::real) hash_target (tcs, *m, storage); } } else { // Feels like there is not much sense in hashing the group itself. // for (const target* m: g->members) hash_target (tcs, *m, storage); } if (dd.expect (tcs.string ()) != nullptr) l4 ([&]{trace << "target set change forcing update of " << t;}); if (dd.expect (prq_cs.string ()) != nullptr) l4 ([&]{trace << "prerequisite set change forcing update of " << t;}); } { if (dd.expect (exe_cs.string ()) != nullptr) l4 ([&]{trace << "program checksum change forcing update of " << t;}); if (dd.expect (env_cs.string ()) != nullptr) l4 ([&]{trace << "environment change forcing update of " << t;}); } } // Get ready to run the depdb preamble. // build::script::environment& env (mdb != nullptr ? mdb->env : md->env); build::script::default_runner& run (mdb != nullptr ? mdb->run : md->run); run.enter (env, script.start_loc); // Run the first half of the preamble (before depdb-dyndep). // { build::script::parser p (ctx); p.execute_depdb_preamble (a, bs, t, env, script, run, dd); // Write a blank line after the custom depdb entries and before // prerequisites, which we use as an anchor (see read_dyn_targets // above). We only do it for the new --dyn-target mode in order not to // invalidate the existing depdb instances. // if (script.depdb_dyndep_dyn_target) dd.expect (""); } // Determine if we need to do an update based on the above checks. // bool update (false); timestamp mt; // Support creating file symlinks using ad hoc recipes. // bool symlink (false); if (dd.writing ()) update = true; // Will re-query symlink. else { if (g == nullptr) { const file& ft (t.as ()); if ((mt = ft.mtime ()) == timestamp_unknown) ft.mtime (mt = mtime (tp)); // Cache. symlink = mt != timestamp_nonexistent && path_symlink (tp); } else { // Use static member, old dynamic, or force update. // const path* p ( g->members_static != 0 ? &tp /* first static member path */ : (!old_dyn_targets.empty () ? &old_dyn_targets.front ().path : nullptr)); if (p != nullptr) { mt = g->load_mtime (*p); symlink = mt != timestamp_nonexistent && path_symlink (*p); } else update = true; // Will re-query symlink. } if (!update) { // If this is a symlink, depdb mtime could be greater than the symlink // target. // update = dd.mtime > mt && !symlink; } } if (update) mt = timestamp_nonexistent; if (script.depdb_dyndep_byproduct) { // If we have the dynamic dependency information as byproduct of the // recipe body, then do the first part: verify the entries in depdb // unless we are already updating. Essentially, this is the `if(cache)` // equivalent of the restart loop in exec_depdb_dyndep(). using dyndep = dyndep_rule; // Update our prerequisite targets and extract the depdb-dyndep // command's information (we may also execute some variable // assignments). // // Do we really need to update our prerequisite targets in this case? // While it may seem like we should be able to avoid it by triggering // update on encountering any non-existent files in depbd, we may // actually incorrectly "validate" some number of depdb entires while // having an out-of-date main source file. We could probably avoid the // update if we are already updating (or not: there is pre-generation // to consider; see inject_existing_file() for details). // { build::script::parser p (ctx); mdb->byp = p.execute_depdb_preamble_dyndep_byproduct ( a, bs, t, env, script, run, dd, update, mt); } mdb->pts_n = pts.size (); if (!update) { const auto& byp (mdb->byp); const char* what (byp.what.c_str ()); const location& ll (byp.location); function map_ext ( [] (const scope& bs, const string& n, const string& e) { // NOTE: another version in exec_depdb_dyndep(). return dyndep::map_extension (bs, n, e, nullptr); }); // Similar to exec_depdb_dyndep()::add() but only for cache=true and // without support for generated files. // // Note that we have to update each file for the same reason as the // main source file -- if any of them changed, then we must assume the // subsequent entries are invalid. // size_t& skip_count (mdb->skip_count); auto add = [&trace, what, a, &bs, &t, pts_n = mdb->pts_n, &byp, &map_ext, &skip_count, mt] (path fp) -> optional { if (const build2::file* ft = dyndep::enter_file ( trace, what, a, bs, t, fp, true /* cache */, true /* normalized */, map_ext, *byp.default_type).first) { // Note: mark the injected prerequisite target as updated (see // execute_update_prerequisites() for details). // if (optional u = dyndep::inject_existing_file ( trace, what, a, t, pts_n, *ft, mt, false /* fail */, false /* adhoc */, 1 /* data */)) { skip_count++; return *u; } } return nullopt; }; auto df = make_diag_frame ( [&ll, &t] (const diag_record& dr) { if (verb != 0) dr << info (ll) << "while extracting dynamic dependencies for " << t; }); while (!update) { // We should always end with a blank line. // string* l (dd.read ()); // If the line is invalid, run the compiler. // if (l == nullptr) { update = true; break; } if (l->empty ()) // Done, nothing changed. break; if (optional r = add (path (move (*l)))) { if (*r) update = true; } else { // Invalidate this line and trigger update. // dd.write (); update = true; } if (update) l6 ([&]{trace << "restarting (cache)";}); } } // Note that in case of dry run we will have an incomplete (but valid) // database which will be updated on the next non-dry run. // if (!update || ctx.dry_run_option) dd.close (false /* mtime_check */); else mdb->dd = dd.close_to_reopen (); // Pass on base scope and update/mtime. // mdb->bs = &bs; mdb->mt = update ? timestamp_nonexistent : mt; mdb->symlink = symlink; return [this, md = move (mdb)] (action a, const target& t) { return perform_update_file_or_group_dyndep_byproduct (a, t, *md); }; } else { // Run the second half of the preamble (depdb-dyndep commands) to update // our prerequisite targets and extract dynamic dependencies (targets // and prerequisites). // // Note that this should be the last update to depdb (the invalidation // order semantics). // md->deferred_failure = false; { build::script::parser p (ctx); p.execute_depdb_preamble_dyndep (a, bs, t, env, script, run, dd, md->dyn_targets, update, mt, md->deferred_failure); } if (update && dd.reading () && !ctx.dry_run_option) dd.touch = timestamp_unknown; dd.close (false /* mtime_check */); // Remove previous dynamic targets since their set may change with // changes to the inputs. // // The dry-run mode complicates things: if we don't remove the old // files, then that information will be gone (since we update depdb even // in the dry-run mode). But if we remove everything in the dry-run // mode, then we may also remove some of the current files, which would // be incorrect. So let's always remove but only files that are not in // the current set. // // Note that we used to do this in perform_update_file_or_group_dyndep() // but that had a tricky issue: if we end up performing match but not // execute (e.g., via the resolve_members() logic), then we will not // cleanup old targets but loose this information (since the depdb has // be updated). So now we do it here, which is a bit strange, but it // sort of fits into that dry-run logic above. Note also that we do this // unconditionally, update or not, since if everything is up to date, // then old and new sets should be the same. // for (const dynamic_target& dt: old_dyn_targets) { const path& f (dt.path); if (find_if (md->dyn_targets.begin (), md->dyn_targets.end (), [&f] (const dynamic_target& dt) { return dt.path == f; }) == md->dyn_targets.end ()) { // This is an optimization so best effort. // if (optional s = butl::try_rmfile_ignore_error (f)) { if (s == rmfile_status::success && verb >= 2) text << "rm " << f; } } } // Pass on the base scope, depdb path, and update/mtime. // md->bs = &bs; md->dd = move (dd.path); md->mt = update ? timestamp_nonexistent : mt; md->symlink = symlink; return [this, md = move (md)] (action a, const target& t) { return perform_update_file_or_group_dyndep (a, t, *md); }; } } target_state adhoc_buildscript_rule:: perform_update_file_or_group_dyndep_byproduct ( action a, const target& t, match_data_byproduct& md) const { // Note: using shared function name among the three variants. // tracer trace ( "adhoc_buildscript_rule::perform_update_file_or_group_dyndep_byproduct"); context& ctx (t.ctx); // For a group we use the first (for now static) member as a source of // mtime. // // @@ TODO: expl: byproduct: Note that until we support dynamic targets in // the byproduct mode, we verify there is at least one static member in // apply() above. Once we do support this, we will need to verify after // the dependency extraction below. // const group* g (t.is_a ()); // Note that even if we've updated all our prerequisites in apply(), we // still need to execute them here to keep the dependency counts straight. // optional ps (execute_update_prerequisites (a, t, md.mt)); if (!ps) md.mt = timestamp_nonexistent; // Update. build::script::environment& env (md.env); build::script::default_runner& run (md.run); if (md.mt != timestamp_nonexistent) { run.leave (env, script.end_loc); return *ps; } const scope& bs (*md.bs); // Sequence start time for mtime checks below. // timestamp start (!ctx.dry_run && depdb::mtime_check () ? system_clock::now () : timestamp_unknown); if (!ctx.dry_run || verb != 0) { if (g == nullptr) execute_update_file (bs, a, t.as (), env, run); else { // Note: no dynamic members yet. // execute_update_group (bs, a, *g, env, run); } } // Extract the dynamic dependency information as byproduct of the recipe // body. Essentially, this is the `if(!cache)` equivalent of the restart // loop in exec_depdb_dyndep(). // if (!ctx.dry_run) { using dyndep = dyndep_rule; using dyndep_format = build::script::parser::dyndep_format; depdb dd (move (md.dd)); const auto& byp (md.byp); const location& ll (byp.location); const char* what (byp.what.c_str ()); const path& file (byp.file); env.clean ({build2::script::cleanup_type::always, file}, true /* implicit */); function map_ext ( [] (const scope& bs, const string& n, const string& e) { // NOTE: another version in exec_depdb_dyndep() and above. return dyndep::map_extension (bs, n, e, nullptr); }); // Analogous to exec_depdb_dyndep()::add() but only for cache=false. // The semantics is quite different, however: instead of updating the // dynamic prerequisites we verify they are not generated. // // Note that fp is expected to be absolute. // size_t skip (md.skip_count); const auto& pts (t.prerequisite_targets[a]); auto add = [&trace, what, a, &bs, &t, g, &pts, pts_n = md.pts_n, &byp, &map_ext, &dd, &skip] (path fp) { normalize_external (fp, what); // Note that unless we take into account dynamic targets, the skip // logic below falls apart since we neither see targets entered via // prerequsites (skip static prerequisites) nor by the cache=true code // above (skip depdb entries). // // If this turns out to be racy (which is the reason we would skip // dynamic targets; see the fine_file() implementation for details), // then the only answer for now is to not use the byproduct mode. // if (const build2::file* ft = dyndep::find_file ( trace, what, a, bs, t, fp, false /* cache */, true /* normalized */, true /* dynamic */, map_ext, *byp.default_type).first) { // Skip if this is one of the static prerequisites provided it was // updated. // for (size_t i (0); i != pts_n; ++i) { const prerequisite_target& p (pts[i]); if (const target* pt = (p.target != nullptr ? p.target : p.adhoc () ? reinterpret_cast (p.data) : nullptr)) { if (ft == pt && (p.adhoc () || p.data == 1)) return; } } // Skip if this is one of the targets (see the non-byproduct version // for background). // if (byp.drop_cycles) { if (g != nullptr) { auto& ms (g->members); if (find (ms.begin (), ms.end (), ft) != ms.end ()) return; } else { for (const target* m (&t); m != nullptr; m = m->adhoc_member) { if (ft == m) return; } } } // Skip until where we left off. // if (skip != 0) { --skip; return; } // Verify it has noop recipe. // // @@ Currently we will issue an imprecise diagnostics if this is // a static prerequisite that was not updated (see above). // dyndep::verify_existing_file (trace, what, a, t, pts_n, *ft); } dd.write (fp); }; auto df = make_diag_frame ( [&ll, &t] (const diag_record& dr) { if (verb != 0) dr << info (ll) << "while extracting dynamic dependencies for " << t; }); ifdstream is (ifdstream::badbit); try { is.open (file); } catch (const io_error& e) { fail (ll) << "unable to open file " << file << ": " << e; } location il (file, 1); // The way we parse things is format-specific. // // Note: similar code in exec_depdb_dyndep(). Except here we just add // the paths to depdb without entering them as targets. // switch (md.byp.format) { case dyndep_format::make: { using make_state = make_parser; using make_type = make_parser::type; make_parser make; for (string l;; ++il.line) // Reuse the buffer. { if (eof (getline (is, l))) { if (make.state != make_state::end) fail (il) << "incomplete make dependency declaration"; break; } size_t pos (0); do { // Note that we don't really need a diag frame that prints the // line being parsed since we are always parsing the file. // pair r (make.next (l, pos, il)); if (r.second.empty ()) continue; // Note: no support for dynamic targets in byproduct mode. // if (r.first == make_type::target) continue; path& f (r.second); if (f.relative ()) { if (!byp.cwd) fail (il) << "relative " << what << " prerequisite path '" << f << "' in make dependency declaration" << info << "consider using --cwd to specify relative path " << "base"; f = *byp.cwd / f; } add (move (f)); } while (pos != l.size ()); if (make.state == make_state::end) break; } break; } case dyndep_format::lines: { for (string l;; ++il.line) // Reuse the buffer. { if (eof (getline (is, l))) break; if (l.empty ()) fail (il) << "blank line in prerequisites list"; if (l.front () == ' ') fail (il) << "non-existent prerequisite in --byproduct mode"; path f; try { f = path (l); // fsdir{} prerequisites only make sense with dynamic targets. // if (f.to_directory ()) throw invalid_path (""); if (f.relative ()) { if (!byp.cwd) fail (il) << "relative " << what << " prerequisite path '" << f << "' in lines dependency declaration" << info << "consider using --cwd to specify " << "relative path base"; f = *byp.cwd / f; } } catch (const invalid_path&) { fail (il) << "invalid " << what << " prerequisite path '" << l << "'"; } add (move (f)); } break; } } // Add the terminating blank line. // dd.expect (""); dd.close (); //@@ TODO: expl: byproduct: verify have at least one member. md.dd.path = move (dd.path); // For mtime check below. } run.leave (env, script.end_loc); timestamp now (system_clock::now ()); const path* tp (nullptr); if (!ctx.dry_run) { // Only now we know for sure there must be a member in the group. // const file& ft ((g == nullptr ? t : *g->members.front ()).as ()); tp = &ft.path (); md.symlink = path_symlink (*tp); // Re-query. // Again, if this is a symlink, depdb mtime will be greater than // the symlink target. // if (!md.symlink) depdb::check_mtime (start, md.dd.path, *tp, now); } // Symlinks don't play well with dry-run (see full description in // perform_update_file_or_group()). // (g == nullptr ? static_cast (t.as ()) : static_cast (*g)).mtime ( md.symlink && tp != nullptr ? build2::mtime (*tp) : now); return target_state::changed; } target_state adhoc_buildscript_rule:: perform_update_file_or_group_dyndep ( action a, const target& t, match_data& md) const { tracer trace ( "adhoc_buildscript_rule::perform_update_file_or_group_dyndep"); context& ctx (t.ctx); // For a group we use the first (static or dynamic) member as a source of // mtime. Note that in this case there must be at least one since we fail // if we were unable to extract any dynamic members and there are no // static (see exec_depdb_dyndep()). // const group* g (t.is_a ()); // Note that even if we've updated all our prerequisites in apply(), we // still need to execute them here to keep the dependency counts straight. // optional ps (execute_update_prerequisites (a, t, md.mt)); if (!ps) md.mt = timestamp_nonexistent; // Update. build::script::environment& env (md.env); build::script::default_runner& run (md.run); // Force update in case of a deferred failure even if nothing changed. // if (md.mt != timestamp_nonexistent && !md.deferred_failure) { run.leave (env, script.end_loc); return *ps; } // Sequence start time for mtime checks below. // timestamp start (!ctx.dry_run && depdb::mtime_check () ? system_clock::now () : timestamp_unknown); if (!ctx.dry_run || verb != 0) { if (g == nullptr) execute_update_file ( *md.bs, a, t.as (), env, run, md.deferred_failure); else execute_update_group (*md.bs, a, *g, env, run, md.deferred_failure); } run.leave (env, script.end_loc); timestamp now (system_clock::now ()); const path* tp (nullptr); if (!ctx.dry_run) { // Note: in case of deferred failure we may not have any members. // const file& ft ((g == nullptr ? t : *g->members.front ()).as ()); tp = &ft.path (); md.symlink = path_symlink (*tp); // Re-query. // Again, if this is a symlink, depdb mtime will be greater than the // symlink target. // if (!md.symlink) depdb::check_mtime (start, md.dd, *tp, now); } // Symlinks don't play well with dry-run (see full description in // perform_update_file_or_group()). // (g == nullptr ? static_cast (t) : static_cast (*g)).mtime ( md.symlink && tp != nullptr ? build2::mtime (*tp) : now); return target_state::changed; } target_state adhoc_buildscript_rule:: perform_update_file_or_group (action a, const target& t) const { tracer trace ("adhoc_buildscript_rule::perform_update_file_or_group"); context& ctx (t.ctx); const scope& bs (t.base_scope ()); // For a group we use the first (static) member to derive depdb path, as a // source of mtime, etc. Note that in this case there must be a static // member since in this version of perform_update we don't extract dynamic // dependencies (see apply() details). // const group* g (t.is_a ()); const file& ft ((g == nullptr ? t : *g->members.front ()).as ()); const path& tp (ft.path ()); // Update prerequisites and determine if any of them render this target // out-of-date. // timestamp mt (g == nullptr ? ft.load_mtime () : g->load_mtime (tp)); // Support creating file symlinks using ad hoc recipes. // bool symlink (mt != timestamp_nonexistent && path_symlink (tp)); // This is essentially ps=execute_prerequisites(a, t, mt) which we // cannot use because we need to see ad hoc prerequisites. // optional ps (execute_update_prerequisites (a, t, mt)); // Calculate prerequisite checksums (that need to include ad hoc // prerequisites) unless the script tracks changes itself. // names storage; sha256 prq_cs, exe_cs, env_cs; if (!script.depdb_clear) { for (const prerequisite_target& p: t.prerequisite_targets[a]) { if (const target* pt = (p.target != nullptr ? p.target : p.adhoc () ? reinterpret_cast (p.data) : nullptr)) { if ((p.include & include_unmatch) != 0) // Skip update=unmatch. continue; hash_prerequisite_target (prq_cs, exe_cs, env_cs, *pt, storage); } } } bool update (!ps); // We use depdb to track changes to the script itself, input/output file // names, tools, etc. // // NOTE: see the "dynamic dependencies" version above. // depdb dd (tp + ".d"); // First should come the rule name/version. // if (dd.expect (" 1") != nullptr) l4 ([&]{trace << "rule mismatch forcing update of " << t;}); // Then the script checksum. // // Ideally, to detect changes to the script semantics, we would hash the // text with all the variables expanded but without executing any // commands. In practice, this is easier said than done (think the set // builtin that receives output of a command that modifies the // filesystem). // // So as the next best thing we are going to hash the unexpanded text as // well as values of all the variables expanded in it (which we get as a // side effect of pre-parsing the script). This approach has a number of // drawbacks: // // - We can't handle computed variable names (e.g., $($x ? X : Y)). // // - We may "overhash" by including variables that are actually // script-local. // // - There are functions like $install.resolve() with result based on // external (to the script) information. // if (dd.expect (checksum) != nullptr) l4 ([&]{trace << "recipe text change forcing update of " << t;}); // Track the variables, targets, and prerequisites changes, unless the // script tracks the dependency changes itself. // if (!script.depdb_clear) { // For each variable hash its name, undefined/null/non-null indicator, // and the value if non-null. // // Note that this excludes the special $< and $> variables which we // handle below. // // @@ TODO: maybe detect and decompose process_path_ex in order to // properly attribute checksum and environment changes? // { sha256 cs; hash_script_vars (cs, script, bs, t, storage); if (dd.expect (cs.string ()) != nullptr) l4 ([&]{trace << "recipe variable change forcing update of " << t;}); } // Target and prerequisite sets ($> and $<). // { sha256 tcs; if (g == nullptr) { for (const target* m (&t); m != nullptr; m = m->adhoc_member) hash_target (tcs, *m, storage); } else { // Feels like there is not much sense in hashing the group itself. // for (const target* m: g->members) hash_target (tcs, *m, storage); } if (dd.expect (tcs.string ()) != nullptr) l4 ([&]{trace << "target set change forcing update of " << t;}); if (dd.expect (prq_cs.string ()) != nullptr) l4 ([&]{trace << "prerequisite set change forcing update of " << t;}); } // Finally the programs and environment checksums. // { if (dd.expect (exe_cs.string ()) != nullptr) l4 ([&]{trace << "program checksum change forcing update of " << t;}); if (dd.expect (env_cs.string ()) != nullptr) l4 ([&]{trace << "environment change forcing update of " << t;}); } } // Execute the custom dependency change tracking commands, if present. // // Note that we share the environment between the execute_depdb_preamble() // and execute_body() calls, which is not merely an optimization since // variables set in the preamble must be available in the body. // // Creating the environment instance is not cheap so optimize for the // common case where we don't have the depdb preamble and nothing to // update. // bool depdb_preamble (!script.depdb_preamble.empty ()); if (!depdb_preamble) { // If this is a symlink, depdb mtime could be greater than the symlink // target. // if (dd.writing () || (dd.mtime > mt && !symlink)) update = true; if (!update) { dd.close (); return *ps; } } build::script::environment env (a, t, bs, false /* temp_dir */); build::script::default_runner run; if (depdb_preamble) { if (script.depdb_preamble_temp_dir) env.set_temp_dir_variable (); build::script::parser p (ctx); run.enter (env, script.start_loc); p.execute_depdb_preamble (a, bs, t, env, script, run, dd); } // Update if depdb mismatch. // if (dd.writing () || (dd.mtime > mt && !symlink)) update = true; dd.close (); // If nothing changed, then we are done. // if (!update) { // Note that if we execute the depdb preamble but not the script body, // we need to call the runner's leave() function explicitly (here and // below). // if (depdb_preamble) run.leave (env, script.end_loc); return *ps; } bool r (false); if (!ctx.dry_run || verb != 0) { // Prepare to execute the script diag preamble and/or body. // r = g == nullptr ? execute_update_file (bs, a, ft, env, run) : execute_update_group (bs, a, *g, env, run); if (r) { if (!ctx.dry_run) { symlink = path_symlink (tp); // Re-query. // Again, if this is a symlink, depdb mtime will be greater than // the symlink target. // if (!symlink) dd.check_mtime (tp); } } } if (r || depdb_preamble) run.leave (env, script.end_loc); // Symlinks don't play well with dry-run: we can't extract accurate target // timestamp without creating the symlink. Overriding the dry-run doesn't // seem to be an option since we don't know whether it will be a symlink // until it's created. At least we are being pessimistic rather than // optimistic here. // (g == nullptr ? static_cast (ft) : static_cast (*g)).mtime ( symlink ? build2::mtime (tp) : system_clock::now ()); return target_state::changed; } // Update prerequisite targets. // // Each (non-NULL) prerequisite target should be in one of the following // states: // // target adhoc data // -------------------- // !NULL false 0 - normal prerequisite to be updated // !NULL false 1 - normal prerequisite already updated // !NULL true 0 - ad hoc prerequisite to be updated and blanked // NULL true !NULL - ad hoc prerequisite already updated and blanked // NULL false !NULL - unmatched prerequisite (ignored by this function) // // Note that we still execute already updated prerequisites to keep the // dependency counts straight. But we don't consider them for the "renders // us out-of-date" check assuming this has already been done. // // See also environment::set_special_variables(). // // See also perform_execute() which has to deal with these shenanigans. // optional adhoc_buildscript_rule:: execute_update_prerequisites (action a, const target& t, timestamp mt) const { context& ctx (t.ctx); // This is essentially a customized execute_prerequisites(a, t, mt). // size_t busy (ctx.count_busy ()); target_state rs (target_state::unchanged); wait_guard wg (ctx, busy, t[a].task_count); auto& pts (t.prerequisite_targets[a]); for (const prerequisite_target& p: pts) { if (const target* pt = (p.target != nullptr ? p.target : p.adhoc () ? reinterpret_cast (p.data) : nullptr)) { target_state s (execute_async (a, *pt, busy, t[a].task_count)); assert (s != target_state::postponed); } } wg.wait (); bool e (mt == timestamp_nonexistent); for (prerequisite_target& p: pts) { if (const target* pt = (p.target != nullptr ? p.target : p.adhoc () ? reinterpret_cast (p.data) : nullptr)) { target_state s (execute_complete (a, *pt)); if (p.data == 0) { rs |= s; // Compare our timestamp to this prerequisite's skipping // update=unmatch. // if (!e && (p.include & include_unmatch) == 0) { // If this is an mtime-based target, then compare timestamps. // if (const mtime_target* mpt = pt->is_a ()) { if (mpt->newer (mt, s)) e = true; } else { // Otherwise we assume the prerequisite is newer if it was // changed. // if (s == target_state::changed) e = true; } } // Blank out adhoc. // // Note: set the include_target flag for the updated_during_match() // check. // if (p.adhoc ()) { p.data = reinterpret_cast (p.target); p.target = nullptr; p.include |= prerequisite_target::include_target; } } } } return e ? nullopt : optional (rs); } // Return true if execute_diag_preamble() and/or execute_body() were called // and thus the caller should call run.leave(). // bool adhoc_buildscript_rule:: execute_update_file (const scope& bs, action a, const file& t, build::script::environment& env, build::script::default_runner& run, bool deferred_failure) const { // NOTE: similar to execute_update_group() below. // context& ctx (t.ctx); const scope& rs (*bs.root_scope ()); // Note that it doesn't make much sense to use the temporary directory // variable ($~) in the 'diag' builtin call, so we postpone setting it // until the script body execution, that can potentially be omitted. // build::script::parser p (ctx); bool exec_body (!ctx.dry_run || verb >= 2); bool exec_diag (!script.diag_preamble.empty () && (exec_body || verb == 1)); bool exec_depdb (!script.depdb_preamble.empty ()); if (script.diag_name) { if (verb == 1) { // By default we print the first non-ad hoc prerequisite target as the // "main" prerequisite, unless there isn't any or it's not file-based, // in which case we fallback to the second form without the // prerequisite. Potential future improvements: // // - Somehow detect that the first prerequisite target is a tool being // executed and fallback to the second form. It's tempting to just // exclude all exe{} targets, but this could be a rule for something // like strip. // const file* pt (nullptr); for (const prerequisite_target& p: t.prerequisite_targets[a]) { // See execute_update_prerequisites(). // if (p.target != nullptr && !p.adhoc ()) { pt = p.target->is_a (); break; } } if (t.adhoc_member == nullptr) { if (pt != nullptr) print_diag (script.diag_name->c_str (), *pt, t); else print_diag (script.diag_name->c_str (), t); } else { vector ts; for (const target* m (&t); m != nullptr; m = m->adhoc_member) ts.push_back (m->key ()); if (pt != nullptr) print_diag (script.diag_name->c_str (), pt->key (), move (ts)); else print_diag (script.diag_name->c_str (), move (ts)); } } } else if (exec_diag) { if (script.diag_preamble_temp_dir && !script.depdb_preamble_temp_dir) env.set_temp_dir_variable (); pair diag ( p.execute_diag_preamble (rs, bs, env, script, run, verb == 1 /* diag */, !exec_depdb /* enter */, false /* leave */)); if (verb == 1) print_custom_diag (bs, move (diag.first), diag.second); } if (exec_body) { // On failure remove the target files that may potentially exist but // be invalid. // small_vector rms; if (!ctx.dry_run) { for (const target* m (&t); m != nullptr; m = m->adhoc_member) { if (auto* f = m->is_a ()) rms.emplace_back (f->path ()); } } if (script.body_temp_dir && !script.depdb_preamble_temp_dir && !script.diag_preamble_temp_dir) env.set_temp_dir_variable (); p.execute_body (rs, bs, env, script, run, !exec_depdb && !exec_diag /* enter */, false /* leave */); if (!ctx.dry_run) { if (deferred_failure) fail << "expected error exit status from recipe body"; // If this is an executable, let's be helpful to the user and set // the executable bit on POSIX. // #ifndef _WIN32 auto chmod = [] (const path& p) { path_perms (p, (path_perms (p) | permissions::xu | permissions::xg | permissions::xo)); }; for (const target* m (&t); m != nullptr; m = m->adhoc_member) { if (auto* p = m->is_a ()) chmod (p->path ()); } #endif for (auto& rm: rms) rm.cancel (); } } return exec_diag || exec_body; } bool adhoc_buildscript_rule:: execute_update_group (const scope& bs, action a, const group& g, build::script::environment& env, build::script::default_runner& run, bool deferred_failure) const { // Note: similar to execute_update_file() above (see there for comments). // // NOTE: when called from perform_update_file_or_group_dyndep_byproduct(), // the group does not contain dynamic members yet and thus could // have no members at all. // context& ctx (g.ctx); const scope& rs (*bs.root_scope ()); build::script::parser p (ctx); bool exec_body (!ctx.dry_run || verb >= 2); bool exec_diag (!script.diag_preamble.empty () && (exec_body || verb == 1)); bool exec_depdb (!script.depdb_preamble.empty ()); if (script.diag_name) { if (verb == 1) { const file* pt (nullptr); for (const prerequisite_target& p: g.prerequisite_targets[a]) { if (p.target != nullptr && !p.adhoc ()) { pt = p.target->is_a (); break; } } if (pt != nullptr) print_diag (script.diag_name->c_str (), *pt, g); else print_diag (script.diag_name->c_str (), g); } } else if (exec_diag) { if (script.diag_preamble_temp_dir && !script.depdb_preamble_temp_dir) env.set_temp_dir_variable (); pair diag ( p.execute_diag_preamble (rs, bs, env, script, run, verb == 1 /* diag */, !exec_depdb /* enter */, false /* leave */)); if (verb == 1) print_custom_diag (bs, move (diag.first), diag.second); } if (exec_body) { // On failure remove the target files that may potentially exist but // be invalid. // // Note: we may leave dynamic members if we don't know about them yet. // Feels natural enough. // small_vector rms; if (!ctx.dry_run) { for (const target* m: g.members) { if (auto* f = m->is_a ()) rms.emplace_back (f->path ()); } } if (script.body_temp_dir && !script.depdb_preamble_temp_dir && !script.diag_preamble_temp_dir) env.set_temp_dir_variable (); p.execute_body (rs, bs, env, script, run, !exec_depdb && !exec_diag /* enter */, false /* leave */); if (!ctx.dry_run) { if (deferred_failure) fail << "expected error exit status from recipe body"; // @@ TODO: expl: byproduct // // Note: will not work for dynamic members if we don't know about them // yet. Could probably fix by doing this later, after the dynamic // dependency extraction. // #ifndef _WIN32 auto chmod = [] (const path& p) { path_perms (p, (path_perms (p) | permissions::xu | permissions::xg | permissions::xo)); }; for (const target* m: g.members) { if (auto* p = m->is_a ()) chmod (p->path ()); } #endif for (auto& rm: rms) rm.cancel (); } } return exec_diag || exec_body; } target_state adhoc_buildscript_rule:: perform_clean_file (action a, const target& t) { // Besides .d (depdb) also clean .t which is customarily used as a // temporary file, such as make dependency output in depdb-dyndep. In // fact, initially the plan was to only clean it if we have dyndep but // there is no reason it cannot be used for something else. // // Note that the main advantage of using this file over something in the // temporary directory ($~) is that it's next to other output which makes // it easier to examine during recipe troubleshooting. // // Finally, we print the entire ad hoc group at verbosity level 1, similar // to the default update diagnostics. // // @@ TODO: .t may also be a temporary directory (and below). // return perform_clean_extra (a, t.as (), {".d", ".t"}, {}, true /* show_adhoc_members */); } target_state adhoc_buildscript_rule:: perform_clean_group (action a, const target& xt) { const group& g (xt.as ()); path d, t; if (g.members_static != 0) { const path& p (g.members.front ()->as ().path ()); d = p + ".d"; t = p + ".t"; } else { // See target_path lambda in apply(). // t = g.dir / (g.name + '.' + g.type ().name); d = t + ".d"; t += ".t"; } return perform_clean_group_extra (a, g, {d.string ().c_str (), t.string ().c_str ()}); } target_state adhoc_buildscript_rule:: default_action (action a, const target& t, const optional& deadline) const { tracer trace ("adhoc_buildscript_rule::default_action"); context& ctx (t.ctx); target_state ts (target_state::unchanged); if (ctx.current_mode == execution_mode::first) ts |= straight_execute_prerequisites (a, t); bool exec (!ctx.dry_run || verb != 0); // Special handling for fsdir{} (which is the recommended if somewhat // hackish way to represent directory symlinks). See fsdir_rule for // background. // // @@ Note that because there is no depdb, we cannot detect the target // directory change (or any other changes in the script). // if (exec && (a == perform_update_id || a == perform_clean_id) && t.is_a ()) { // For update we only want to skip if it's a directory. For clean we // want to (try) to clean up any filesystem entry, including a dangling // symlink. // exec = a == perform_update_id ? !exists (t.dir, true /* ignore_errors */) : build2::entry_exists (t.dir, false /* follow_symlinks */); } if (exec) { const scope& bs (t.base_scope ()); const scope& rs (*bs.root_scope ()); build::script::environment e (a, t, bs, false /* temp_dir */, deadline); build::script::parser p (ctx); build::script::default_runner r; bool exec_body (!ctx.dry_run || verb >= 2); bool exec_diag (!script.diag_preamble.empty () && (exec_body || verb == 1)); if (script.diag_name) { if (verb == 1) { // For operations other than update (as well as for non-file // targets), we default to the second form (without the // prerequisite). Think test. // if (t.adhoc_member == nullptr) print_diag (script.diag_name->c_str (), t); else { vector ts; for (const target* m (&t); m != nullptr; m = m->adhoc_member) ts.push_back (m->key ()); print_diag (script.diag_name->c_str (), move (ts)); } } } else if (exec_diag) { if (script.diag_preamble_temp_dir) e.set_temp_dir_variable (); pair diag ( p.execute_diag_preamble (rs, bs, e, script, r, verb == 1 /* diag */, true /* enter */, !exec_body /* leave */)); if (verb == 1) print_custom_diag (bs, move (diag.first), diag.second); } if (exec_body) { if (script.body_temp_dir && !script.diag_preamble_temp_dir) e.set_temp_dir_variable (); p.execute_body (rs, bs, e, script, r, !exec_diag /* enter */); } ts |= target_state::changed; } if (ctx.current_mode == execution_mode::last) ts |= reverse_execute_prerequisites (a, t); return ts; } void adhoc_buildscript_rule:: print_custom_diag (const scope& bs, names&& ns, const location& l) const { // The straightforward thing to do would be to just print the diagnostics // as specified by the user. But that will make some of the tidying up // done by print_diag() unavailable to custom diagnostics. Things like // omitting the out-qualification as well as compact printing of the // groups. Also, in the future we may want to support colorization of the // diagnostics, which will be difficult to achive with such a "just print" // approach. // // So instead we are going to parse the custom diagnostics, translate // names back to targets (where appropriate), and call one of the // print_diag() functions. Specifically, we expect the custom diagnostics // to be in one of the following two forms (which correspond to the two // forms of pring_diag()): // // diag

... // diag

... // // And the way we are going to disambiguate this is by analyzing name // types. Specifically, we expect to be a simple name that also // does not contain any directory separators (so we can distinguish it // from both target names as well as paths, which can be specified on // either side). We will also recognize `-` as the special stdout path // name (so cannot be `-`). Finally, (but not // ) can be a string (e.g., an argument) but that should not // pose an ambiguity. // // With this approach, the way to re-create the default diagnostics would // be: // // diag ($<[0]) -> $> // diag $> // auto i (ns.begin ()), e (ns.end ()); // // if (i == e) fail (l) << "missing program name in diag builtin"; if (!i->simple () || i->empty ()) fail (l) << "expected simple name as program name in diag builtin"; const char* prog (i->value.c_str ()); ++i; // // const target* l_t (nullptr); path l_p; string l_s; auto parse_target = [&bs, &l, &i, &e] () -> const target& { name& n (*i++); name o; if (n.pair) { if (i == e) fail (l) << "invalid target name pair in diag builtin"; o = move (*i++); } // Similar to to_target() in $target.*(). // if (const target* r = search_existing (n, bs, o.dir)) return *r; fail (l) << "target " << (n.pair ? names {move (n), move (o)} : names {move (n)}) << " not found in diag builtin" << endf; }; auto parse_first = [&l, &i, &e, &parse_target] (const target*& t, path& p, string& s, const char* after) { if (i == e) fail (l) << "missing target after " << after << " in diag builtin"; try { if (i->typed ()) { t = &parse_target (); return; // i is already incremented. } else if (!i->dir.empty ()) { p = move (i->dir); p /= i->value; } else if (path_traits::find_separator (i->value) != string::npos) { p = path (move (i->value)); } else if (!i->value.empty ()) { s = move (i->value); } else fail (l) << "expected target, path, or argument after " << after << " in diag builtin"; } catch (const invalid_path& e) { fail (l) << "invalid path '" << e.path << "' after " << after << " in diag builtin"; } ++i; }; parse_first (l_t, l_p, l_s, "program name"); // Now detect which form it is. // if (i != e && i->simple () && !i->empty () && path_traits::find_separator (i->value) == string::npos) { // The first form. // // const char* comb (i->value.c_str ()); ++i; // // const target* r_t (nullptr); path r_p; string r_s; parse_first (r_t, r_p, r_s, "combiner"); path_name r_pn; if (r_t != nullptr) ; else if (!r_p.empty ()) r_pn = path_name (&r_p); else { if (r_s != "-") fail (l) << "expected target or path instead of '" << r_s << "' after combiner in diag builtin"; r_pn = path_name (move (r_s)); } if (i == e) { if (r_t != nullptr) { if (l_t != nullptr) print_diag (prog, *l_t, *r_t, comb); else if (!l_p.empty ()) print_diag (prog, l_p, *r_t, comb); else print_diag (prog, l_s, *r_t, comb); } else { if (l_t != nullptr) print_diag (prog, *l_t, r_pn, comb); else if (!l_p.empty ()) print_diag (prog, l_p, r_pn, comb); else print_diag (prog, l_s, r_pn, comb); } return; } // We can only have multiple targets, not paths. // if (r_t == nullptr) fail (l) << "unexpected name after path in diag builtin"; // ... // vector r_ts {r_t->key ()}; do r_ts.push_back (parse_target ().key ()); while (i != e); if (l_t != nullptr) print_diag (prog, l_t->key (), move (r_ts), comb); else if (!l_p.empty ()) print_diag (prog, l_p, move (r_ts), comb); else print_diag (prog, l_s, move (r_ts), comb); } else { // The second form. // First "absorb" the l_* values as the first . // const target* r_t (nullptr); path_name r_pn; if (l_t != nullptr) r_t = l_t; else if (!l_p.empty ()) r_pn = path_name (&l_p); else { if (l_s != "-") { diag_record dr (fail (l)); dr << "expected target or path instead of '" << l_s << "' after program name in diag builtin"; if (i != e) dr << info << "alternatively, missing combiner after '" << l_s << "'"; } r_pn = path_name (move (l_s)); } if (i == e) { if (r_t != nullptr) print_diag (prog, *r_t); else print_diag (prog, r_pn); return; } // We can only have multiple targets, not paths. // if (r_t == nullptr) fail (l) << "unexpected name after path in diag builtin"; // ... // vector r_ts {r_t->key ()}; do r_ts.push_back (parse_target ().key ()); while (i != e); print_diag (prog, move (r_ts)); } } }