From 5267c62f8c1f483c5348d4a2e498088a41c2e945 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 12 Apr 2023 07:34:39 +0200 Subject: Implement in-process package configuration --- bpkg/pkg-configure.cxx | 402 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 373 insertions(+), 29 deletions(-) (limited to 'bpkg/pkg-configure.cxx') diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx index 59a6af0..d6c5446 100644 --- a/bpkg/pkg-configure.cxx +++ b/bpkg/pkg-configure.cxx @@ -3,6 +3,17 @@ #include +#include +#include +#include + +#include +#include +#include +#include + +#include // build2_init(), etc + #include #include #include @@ -275,6 +286,25 @@ namespace bpkg else od = sp->effective_out_root (pdb.config); + // We tried to use global overrides to recreate the original + // behavior of not warning about unused config.import.* + // variables (achived via the config.config.persist value in + // amalgamation). Even though it's probably misguided (we + // don't actually save the unused values anywhere, just + // don't warn about them). + // + // Can we somehow cause a clash, say if the same package + // comes from different configurations? Yeah, we probably + // can. So could add it as undermined (?), detect a clash, + // and "fallforward" to the correct behavior. + // + // But we can clash with an absent value -- that is, we + // force importing from a wrong configuration where without + // any import things would have been found in the same + // amalgamation. Maybe we could detect that (no import + // for the same package -- but it could be for a package + // we are not configuring). + // vars.push_back ("config.import." + sp->name.variable () + "='" + od.representation () + '\''); } @@ -345,13 +375,87 @@ namespace bpkg move (srcs)}; } + + unique_ptr + pkg_configure_context ( + const common_options& o, + strings&& cmd_vars, + const function& var_ovr_func) + { + using namespace build2; + + // Initialize the build system. + // + // Note that this takes into account --build-option and default options + // files (which may have global overrides and which end up in + // build2_cmd_vars). + // + if (!build2_sched.started ()) + build2_init (o); + + // Re-tune the scheduler for parallel execution (see build2_init() + // for details). + // + if (build2_sched.tuned ()) + build2_sched.tune (0); + + auto merge_cmd_vars = [&cmd_vars] () -> const strings& + { + if (cmd_vars.empty ()) + return build2_cmd_vars; + + if (!build2_cmd_vars.empty ()) + cmd_vars.insert (cmd_vars.begin (), + build2_cmd_vars.begin (), build2_cmd_vars.end ()); + + return cmd_vars; + }; + + // Shouldn't we shared the module context with package skeleton + // contexts? Maybe we don't have to since we don't build modules in + // them concurrently (in a sence, we didn't share it when we were + // invoking the build system driver). + // + unique_ptr ctx ( + new context (build2_sched, + build2_mutexes, + build2_fcache, + false /* match_only */, + false /* no_external_modules */, + false /* dry_run */, + false /* no_diag_buffer */, + false /* keep_going */, + merge_cmd_vars (), + context::reserves { + 30000 /* targets */, + 1100 /* variables */}, + nullptr /* module_context */, + nullptr /* inherited_mudules_lock */, + var_ovr_func)); + + // Set the current meta-operation once per context so that we don't reset + // ctx->current_on. Note that this function also sets ctx->current_mname + // and var_build_meta_operation on global scope. + // + ctx->current_meta_operation (config::mo_configure); + ctx->current_oname = string (); // default + + return ctx; + } + void pkg_configure (const common_options& o, database& db, transaction& t, const shared_ptr& p, configure_prerequisites_result&& cpr, - bool disfigured, +#ifndef BPKG_OUTPROC_CONFIGURE + const unique_ptr& pctx, + const build2::variable_overrides& ovrs, +#else + const unique_ptr&, + const build2::variable_overrides&, // Still in cpr.config_variables. +#endif bool simulate) { tracer trace ("pkg_configure"); @@ -361,15 +465,22 @@ namespace bpkg tracer_guard tg (db, trace); - const dir_path& c (db.config_orig); +#ifndef BPKG_OUTPROC_CONFIGURE + const dir_path& c (db.config); // Absolute. +#else + const dir_path& c (db.config_orig); // Relative. +#endif + dir_path src_root (p->effective_src_root (c)); // Calculate package's out_root. // // Note: see a version of this in pkg_configure_prerequisites(). // + bool external (p->external ()); + dir_path out_root ( - p->external () + external ? c / dir_path (p->name.string ()) : c / dir_path (p->name.string () + '-' + p->version.string ())); @@ -383,28 +494,18 @@ namespace bpkg // if (!simulate) { - // Unless this package has been completely disfigured, disfigure all the - // package configuration variables to reset all the old values to - // defaults (all the new user/dependent/reflec values, including old - // user, are returned by collect_config() and specified as overrides). - // Note that this semantics must be consistent with how we load things - // in the package skeleton during configuration negotiation. + // Original implementation that runs the standard build system driver. // - // Note also that this means we don't really use the dependent and - // reflect sources that we save in the database. But let's keep them - // for the completeness of information (maybe could be useful during - // configuration reset or some such). + // Note that the semantics doesn't match 100%. In particular, in the + // in-process implementation we enter overrides with global visibility + // in each project instead of the amalgamation (which is probably more + // accurate, since we don't re-configure the amalgamation nor some + // dependencies which could be affected by such overrides). In a sense, + // we enter them as if they were specified with the special .../ scope + // (but not with the % project visibility -- they must still be visible + // in subprojects). // - string dvar; - if (!disfigured) - { - // Note: must be quoted to preserve the pattern. - // - dvar = "config.config.disfigure='config."; - dvar += p->name.variable (); - dvar += "**'"; - } - +#ifdef BPKG_OUTPROC_CONFIGURE // Form the buildspec. // string bspec; @@ -422,11 +523,7 @@ namespace bpkg try { - run_b (o, - verb_b::quiet, - cpr.config_variables, - (!dvar.empty () ? dvar.c_str () : nullptr), - bspec); + run_b (o, verb_b::quiet, cpr.config_variables, bspec); } catch (const failed&) { @@ -437,6 +534,214 @@ namespace bpkg pkg_disfigure (o, db, t, p, true, true, false); throw; } +#else + // Print the out-process command line in the verbose mode. + // + if (verb >= 2) + { + string bspec; + + // Use path representation to get canonical trailing slash. + // + if (src_root == out_root) + bspec = "configure('" + out_root.representation () + "')"; + else + bspec = "configure('" + + src_root.representation () + "'@'" + + out_root.representation () + "')"; + + print_b (o, verb_b::quiet, cpr.config_variables, bspec); + } + + try + { + // Note: no bpkg::failed should be thrown from this block. + // + using namespace build2; + using build2::fail; + using build2::info; + using build2::endf; + using build2::location; + + // The build2_init() function initializes the build system verbosity + // as if running with verb_b::normal while we need verb_b::quiet. So + // we temporarily adjust the build2 verbosity (see map_verb_b() for + // details). + // + auto verbg (make_guard ([ov = build2::verb] () {build2::verb = ov;})); + if (bpkg::verb == 1) + build2::verb = 0; + + context& ctx (*pctx); + + // Bootstrap and load the project. + // + // Note: in many ways similar to package_skeleton code. + // + scope& rs (*create_root (ctx, out_root, src_root)->second.front ()); + + // If we are configuring in the dependency order (as we should), then + // it feels like the only situation where we can end up with an + // already bootstrapped project is an unspecified dependency. Note + // that this is a hard fail since it would have been loaded without + // the proper configuration. + // + if (bootstrapped (rs)) + { + fail << p->name << db << " loaded ahead of its dependents" << + info << "likely unspecified dependency on package " << p->name; + } + + optional altn; + value& v (bootstrap_out (rs, altn)); + + if (!v) + v = src_root; + else + { + dir_path& p (cast (v)); + + if (src_root != p) + { + // @@ Fuzzy if need this or can do as package skeleton (seeing + // that we know we are re-configuring). + // + ctx.new_src_root = src_root; + ctx.old_src_root = move (p); + p = src_root; + } + } + + setup_root (rs, false /* forwarded */); + + // Note: we already know our amalgamation. + // + bootstrap_pre (rs, altn); + bootstrap_src (rs, altn, + db.config.relative (out_root) /* amalgamation */, + true /* subprojects */); + + create_bootstrap_outer (rs, true /* subprojects */); + bootstrap_post (rs); + + values mparams; + const meta_operation_info& mif (config::mo_configure); + const operation_info& oif (op_default); + + // Skip configure_pre() and configure_operation_pre() calls since we + // don't pass any parameteres and pass default operation. We also know + // that op_default has no pre/post operations, naturally. + + // Find the root buildfile. Note that the implied buildfile logic does + // not apply (our target is the project root directory). + // + optional bf (find_buildfile (src_root, src_root, altn)); + + if (!bf) + fail << "no buildfile in " << src_root; + + // Enter project-wide overrides. + // + // Note that the use of the root scope as amalgamation makes sure + // scenarious like below work correctly (see above for background). + // + // bpkg create -d cfg cc config.cc.coptions=-Wall + // bpkg build { config.cc.coptions+=-g }+ libfoo + // { config.cc.coptions+=-O }+ libbar + // + ctx.enter_project_overrides (rs, out_root, ovrs, &rs); + + // The goal here is to be more or less semantically equivalent to + // configuring several projects at once. Except that here we have + // interleaving load/match instead of first all load then all + // match. But presumably this shouldn't be a problem (we can already + // have match interrupted by load and the "island append" requirement + // should hold here as well). + // + // Note that either way we will be potentially re-matching the same + // dependency targets multiple times (see build2::configure_execute() + // for details). + // + const path_name bsn (""); + const location loc (bsn, 0, 0); + + // out_root/dir{./} + // + target_key tk { + &dir::static_type, + &out_root, + &empty_dir_path, + &empty_string, + nullopt}; + + action_targets tgs; + mif.load (mparams, rs, *bf, out_root, src_root, loc); + mif.search (mparams, rs, rs, *bf, tk, loc, tgs); + + ctx.current_operation (oif, nullptr); + action a (ctx.current_action ()); + + mif.match (mparams, a, tgs, 2 /* diag */, true /* progress */); + mif.execute (mparams, a, tgs, 2 /* diag */, true /* progress */); + + // Note: no operation_post/meta_operation_post for configure. + + // Here is a tricky part: if this is a normal package, then it will be + // discovered as a subproject of the bpkg configuration when we load + // it for the first time (because they are all unpacked). However, if + // this is an external package, there could be no out_root directory + // for it in the bpkg configuration yet. As a result, we need to + // manually add it as a newly discovered subproject. + // + if (external) + { + scope* as (rs.parent_scope ()->root_scope ()); + assert (as != nullptr); // No bpkg configuration? + + // Kept NULL if there are no subprojects, so we may need to + // initialize it (see build2::bootstrap_src() for details). + // + subprojects* sp (*as->root_extra->subprojects); + if (sp == nullptr) + { + value& v (as->vars.assign (*ctx.var_subprojects)); + v = subprojects {}; + sp = *(as->root_extra->subprojects = &cast (v)); + } + + const project_name& n (**rs.root_extra->project); + + if (sp->find (n) == sp->end ()) + sp->emplace (n, out_root.leaf ()); + } + } + catch (const build2::failed&) + { + // Assume the diagnostics has already been issued. + + // If we failed to configure the package, make sure we revert + // it back to the unpacked state by running disfigure (it is + // valid to run disfigure on an un-configured build). And if + // disfigure fails as well, then the package will be set into + // the broken state. + + // Indicate to pkg_disfigure() we are partially configured. + // + p->out_root = out_root.leaf (); + p->state = package_state::broken; + + // Commits the transaction. + // + pkg_disfigure (o, db, t, + p, + true /* clean */, + true /* disfigure */, + false /* simulate */); + + + throw bpkg::failed (); + } +#endif p->config_variables = move (cpr.config_sources); } @@ -473,7 +778,46 @@ namespace bpkg fdb, nullptr)); - pkg_configure (o, db, t, p, move (cpr), disfigured, simulate); + if (!simulate) + { + // Unless this package has been completely disfigured, disfigure all the + // package configuration variables to reset all the old values to + // defaults (all the new user/dependent/reflec values, including old + // user, are returned by collect_config() and specified as overrides). + // Note that this semantics must be consistent with how we load things + // in the package skeleton during configuration negotiation. + // + // Note also that this means we don't really use the dependent and + // reflect sources that we save in the database. But let's keep them + // for the completeness of information (maybe could be useful during + // configuration reset or some such). + // + if (!disfigured) + { + // Note: must be quoted to preserve the pattern. + // + cpr.config_variables.push_back ( + "config.config.disfigure='config." + p->name.variable () + "**'"); + } + } + + unique_ptr ctx; + +#ifndef BPKG_OUTPROC_CONFIGURE + if (!simulate) + ctx = pkg_configure_context (o, move (cpr.config_variables)); +#endif + + pkg_configure (o, + db, + t, + p, + move (cpr), + ctx, + (ctx != nullptr + ? ctx->var_overrides + : build2::variable_overrides {}), + simulate); } shared_ptr -- cgit v1.1