aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2021-08-30 08:24:26 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2021-09-06 10:56:57 +0200
commit8f1a2ab7bd19e0f4bdf7f173c20078df96b0e12d (patch)
treecdfb1a28bb292aa8c7c0ff2c5c6256386c88c725
parentbb25e12f0e15438d738806aed77ab63dc02cbe3c (diff)
Synchronize entire linked configuration cluster in bdep-sync
-rw-r--r--bdep/sync.cli2
-rw-r--r--bdep/sync.cxx1174
-rw-r--r--bdep/types.hxx6
3 files changed, 883 insertions, 299 deletions
diff --git a/bdep/sync.cli b/bdep/sync.cli
index 86c6095..fcc1417 100644
--- a/bdep/sync.cli
+++ b/bdep/sync.cli
@@ -29,7 +29,7 @@ namespace bdep
<cfg-spec> = (\b{@}<cfg-name> | \b{--config}|\b{-c} <cfg-dir>)... | \b{--all}|\b{-a}\n
<pkg-spec> = (\b{--directory}|\b{-d} <pkg-dir>)... | <prj-spec>\n
<prj-spec> = \b{--directory}|\b{-d} <prj-dir>\n
- <pkg-args> = (<pkg> | <cfg-var>)...}
+ <pkg-args> = (\b{?}<pkg> | <cfg-var>)...}
\h|DESCRIPTION|
diff --git a/bdep/sync.cxx b/bdep/sync.cxx
index e4e302b..3d95777 100644
--- a/bdep/sync.cxx
+++ b/bdep/sync.cxx
@@ -3,6 +3,7 @@
#include <bdep/sync.hxx>
+#include <list>
#include <cstring> // strchr()
#include <libbpkg/manifest.hxx>
@@ -115,59 +116,149 @@ namespace bdep
return r;
}
- // Project to be synchronized.
+ // Configurations linked into a cluster.
//
- namespace
+ struct linked_config
{
- struct project
+ dir_path path;
+ bdep::uuid uuid;
+ };
+
+ class linked_configs: public small_vector<linked_config, 16>
+ {
+ public:
+ const linked_config*
+ find (const dir_path& p)
{
- dir_path path;
- shared_ptr<configuration> config;
+ auto i (find_if (begin (), end (),
+ [&p] (const linked_config& c)
+ {
+ return c.path == p;
+ }));
+ return i != end () ? &*i : nullptr;
+ }
+ };
+ // Configurations to be synced are passed either as configuration objects
+ // (if syncing from a project) or as directories (if syncing from a
+ // configuration, normally implicit/hook).
+ //
+ struct sync_config: shared_ptr<configuration>
+ {
+ dir_path path_; // Empty if not NULL.
+
+ const dir_path&
+ path () const {return *this != nullptr ? (*this)->path : path_;}
+
+ sync_config (shared_ptr<configuration> c)
+ : shared_ptr<configuration> (move (c)) {}
+ sync_config (dir_path p): path_ (move (p)) {}
+ };
+
+ using sync_configs = small_vector<sync_config, 16>;
+
+ // Projects and their configurations to be synchronized.
+ //
+ struct sync_project
+ {
+ struct config: shared_ptr<configuration>
+ {
+ bool origin;
bool implicit;
bool fetch;
+
+ config (shared_ptr<configuration> p, bool o, bool i, bool f)
+ : shared_ptr<configuration> (move (p)),
+ origin (o), implicit (i), fetch (f) {}
};
- using projects = small_vector<project, 1>;
+ dir_path path;
+ small_vector<config, 16> configs;
+
+ explicit
+ sync_project (dir_path p) : path (move (p)) {}
+ };
+
+ using sync_projects = small_vector<sync_project, 1>;
+
+ static inline ostream&
+ operator<< (ostream& o, const sync_config& sc)
+ {
+ if (const shared_ptr<configuration>& c = sc)
+ o << *c; // Prints as @name if available.
+ else
+ o << sc.path_;
+ return o;
}
- // Append the list of additional (to origin, if not empty) projects that are
- // using this configuration.
+ // Append the list of projects that are using this configuration. Note that
+ // if the project is already on the list, then add the configuration to the
+ // existing entry unless it's already there.
//
static void
load_implicit (const common_options& co,
const dir_path& cfg,
+ sync_projects& r,
const dir_path& origin_prj,
- projects& r)
+ transaction* origin_tr)
{
tracer trace ("load_implicit");
- for (dir_path& d: configuration_projects (co, cfg, origin_prj))
+ for (dir_path& d: configuration_projects (co, cfg))
{
+ // Do duplicate suppression before any heavy lifting.
+ //
+ auto i (find_if (r.begin (), r.end (),
+ [&d] (const sync_project& p)
+ {
+ return p.path == d;
+ }));
+
+ if (i != r.end ())
+ {
+ if (find_if (i->configs.begin (), i->configs.end (),
+ [&cfg] (const sync_project::config& c)
+ {
+ return c->path == cfg;
+ }) != i->configs.end ())
+ continue;
+ }
+
shared_ptr<configuration> c;
{
using query = bdep::query<configuration>;
- // Save and restore the current transaction, if any.
+ query q (query::path == cfg.string ());
+
+ // Reuse the transaction (if any) if this is origin project.
//
- transaction* ct (nullptr);
- if (transaction::has_current ())
+ if (origin_tr != nullptr && origin_prj == d)
{
- ct = &transaction::current ();
- transaction::reset_current ();
+ c = origin_tr->database ().query_one<configuration> (q);
}
+ else
+ {
+ // Save and restore the current transaction, if any.
+ //
+ transaction* ct (nullptr);
+ if (transaction::has_current ())
+ {
+ ct = &transaction::current ();
+ transaction::reset_current ();
+ }
- auto tg (make_guard ([ct] ()
- {
- if (ct != nullptr)
- transaction::current (*ct);
- }));
+ auto tg (make_guard ([ct] ()
+ {
+ if (ct != nullptr)
+ transaction::current (*ct);
+ }));
- {
- database db (open (d, trace));
- transaction t (db.begin ());
- c = db.query_one<configuration> (query::path == cfg.string ());
- t.commit ();
+ {
+ database db (open (d, trace));
+ transaction t (db.begin ());
+ c = db.query_one<configuration> (q);
+ t.commit ();
+ }
}
}
@@ -179,10 +270,17 @@ namespace bdep
if (c == nullptr)
continue;
- r.push_back (project {move (d),
- move (c),
- true /* implicit */,
- true /* fetch */});
+ if (i == r.end ())
+ {
+ r.push_back (sync_project (d));
+ i = r.end () - 1;
+ }
+
+ i->configs.push_back (
+ sync_project::config {move (c),
+ false /* origin */,
+ true /* implicit */,
+ true /* fetch */});
}
}
@@ -190,14 +288,13 @@ namespace bdep
//
static void
link_dependency_config (const common_options& co,
- const dir_path& cfg,
- const projects& prjs,
- const dir_path& origin_prj,
- const shared_ptr<configuration>& origin_config,
+ const dir_path& origin_prj, // Can be empty.
+ const sync_configs& origin_cfgs,
+ const sync_projects& prjs,
const strings& dep_chain,
bool create_host_config,
bool create_build2_config,
- transaction* tr,
+ transaction* origin_tr,
vector<pair<dir_path, string>>* created_cfgs,
tracer& trace)
{
@@ -297,18 +394,21 @@ namespace bdep
// project returning its configuration object if found and NULL
// otherwise.
//
- auto find_config = [&origin_config, &origin_prj] (database& db,
- const dir_path& prj,
- const dir_path& cfg)
+ auto find_config = [&origin_prj, &origin_cfgs] (
+ database& db,
+ const dir_path& prj,
+ const dir_path& cfg) -> shared_ptr<configuration>
{
// Note that this is not merely an optimization since the origin
// configuration can be changed but not updated in the database yet (see
// cmd_init() for a use case).
//
- if (origin_config != nullptr &&
- origin_config->path == cfg &&
- origin_prj == prj)
- return origin_config;
+ if (origin_prj == prj)
+ {
+ for (const sync_config& ocfg: origin_cfgs)
+ if (ocfg.path () == cfg)
+ return ocfg;
+ }
using query = bdep::query<configuration>;
return db.query_one<configuration> (query::path == cfg.string ());
@@ -382,13 +482,16 @@ namespace bdep
// Show how we got here (used for both info and text).
//
- auto add_info = [&dep, &dependents, &cfg] (const basic_mark& bm)
+ auto add_info = [&dep, &dependents, &origin_cfgs] (const basic_mark& bm)
{
const dependent& dpt (dependents[0]);
+
bm << "while searching for configuration for build-time dependency "
<< dep << " of package " << dpt.name << "/" << dpt.version
<< " [" << dpt.config << "]";
- bm << "while synchronizing configuration " << cfg;
+
+ for (const sync_config& cfg: origin_cfgs)
+ bm << "while synchronizing configuration " << cfg.path ();
};
// Show how we got here if things go wrong.
@@ -437,9 +540,9 @@ namespace bdep
for (const dependent& dpt: dependents)
{
- for (const project& prj: prjs)
+ for (const sync_project& prj: prjs)
{
- database_transaction t (prj.path == origin_prj ? tr : nullptr,
+ database_transaction t (prj.path == origin_prj ? origin_tr : nullptr,
prj.path,
trace);
@@ -459,8 +562,7 @@ namespace bdep
dpt.config));
if (dc != nullptr &&
- find_if (dc->packages.begin (),
- dc->packages.end (),
+ find_if (dc->packages.begin (), dc->packages.end (),
[&dpt] (const package_state& s)
{
return dpt.name == s.name;
@@ -503,7 +605,7 @@ namespace bdep
if (dep_cfg == nullptr) // Need to create the config?
{
- for (const project& p: prjs)
+ for (const sync_project& p: prjs)
{
if (p.path != prj.path)
{
@@ -622,7 +724,7 @@ namespace bdep
bool create (true);
for (const dir_path& prj: dpt_prjs)
{
- database_transaction t (prj == origin_prj ? tr : nullptr,
+ database_transaction t (prj == origin_prj ? origin_tr : nullptr,
prj,
trace);
@@ -685,11 +787,23 @@ namespace bdep
// If upgrade is not nullopt, then: If there are dep_pkgs, then we are
// upgrading specific dependency packages. Otherwise -- project packages.
//
+ // Note that if origin_prj is not empty, then origin_cfgs are specified as
+ // configurations (as opposed to paths). Also upgrade can only be specified
+ // with origin_prj.
+ //
+ // Note that pkg_args and dep_pkgs may contain groups.
+ //
+ struct config // VC14 doesn't like it inside the function.
+ {
+ reference_wrapper<const dir_path> path; // Reference to prjs.
+ strings reps;
+ };
+
static void
cmd_sync (const common_options& co,
- const dir_path& cfg,
const dir_path& origin_prj,
- const shared_ptr<configuration>& origin_config,
+ sync_configs&& origin_cfgs,
+ linked_configs&& linked_cfgs,
const strings& pkg_args,
bool implicit,
bool fetch,
@@ -701,86 +815,223 @@ namespace bdep
const strings& dep_pkgs,
bool create_host_config,
bool create_build2_config,
- transaction* tr = nullptr,
+ transaction* origin_tr = nullptr,
vector<pair<dir_path, string>>* created_cfgs = nullptr)
{
tracer trace ("cmd_sync");
- assert (origin_config == nullptr || !origin_config->packages.empty ());
+ // True if we have originating project.
+ //
+ bool origin (!origin_prj.empty ());
+
assert (prj_pkgs.empty () || dep_pkgs.empty ()); // Can't have both.
// If a transaction is specified, then it must be started on the origin
// project's database (which therefore must be specified) and it must be
// the current.
//
- if (tr != nullptr)
- assert (!origin_prj.empty () && tr == &transaction::current ());
+ if (origin_tr != nullptr)
+ assert (origin && origin_tr == &transaction::current ());
// Must both be either specified or not.
//
- assert ((tr == nullptr) == (created_cfgs == nullptr));
+ assert ((origin_tr == nullptr) == (created_cfgs == nullptr));
- projects prjs;
+ // Collect all the projects that will be involved in this synchronization
+ // (we synchronize everything at once).
+ //
+ sync_projects prjs;
- if (origin_config != nullptr)
- prjs.push_back (project {origin_prj, origin_config, implicit, fetch});
+ if (origin)
+ {
+ prjs.push_back (sync_project (origin_prj));
+
+ for (sync_config& c: origin_cfgs)
+ {
+ // If we have origin project then we should have origin config.
+ //
+ assert (c != nullptr);
+ prjs.back ().configs.push_back (
+ sync_project::config {c, true /* origin */, implicit, fetch});
+ }
+ }
- // Load other projects that might be using the same configuration -- we
- // have to synchronize everything at once.
+ // Load other projects that might be using the same configuration cluster.
+ //
+ // Note that this may add more (implicit) configurations to origin_prj's
+ // entry.
//
- load_implicit (co, cfg, origin_prj, prjs);
+ for (const linked_config& cfg: linked_cfgs)
+ load_implicit (co, cfg.path, prjs, origin_prj, origin_tr);
// Verify that no initialized package in any of the projects sharing this
// configuration is specified as a dependency.
//
- for (const string& n: dep_pkgs)
+ if (!dep_pkgs.empty ())
{
- for (const project& prj: prjs)
+ for (const sync_project& prj: prjs)
{
- auto& pkgs (prj.config->packages);
+ for (const sync_project::config& cfg: prj.configs)
+ {
+ auto& pkgs (cfg->packages);
- if (find_if (pkgs.begin (),
- pkgs.end (),
- [&n] (const package_state& ps)
- {
- return n == ps.name;
- }) != pkgs.end ())
- fail << "initialized package " << n << " specified as a dependency";
+ for (const string& n: dep_pkgs)
+ {
+ if (find_if (pkgs.begin (), pkgs.end (),
+ [&n] (const package_state& ps)
+ {
+ return n == ps.name;
+ }) != pkgs.end ())
+ fail << "initialized package " << n << " specified as dependency" <<
+ info << "package initialized in project " << prj.path;
+ }
+ }
}
}
- // Prepare the list of packages to build and repositories to fetch.
+ // Prepare the list of packages to build, configurations involved, and
+ // repositories to fetch in each such configuration.
//
strings args;
- strings reps;
+ small_vector<config, 16> cfgs;
+
+ // First collect configurations and their repositories. We do it as a
+ // separate (from the one below) pass in order to determine how many
+ // projects/configurations will be involved. If it's just one, then we
+ // often can have a simpler command line.
+ //
+ bool origin_only (dep_pkgs.empty ()); // Only origin packages on cmd line.
+ for (const sync_project& prj: prjs)
+ {
+ for (const sync_project::config& cfg: prj.configs)
+ {
+ bool empty (cfg->packages.empty ());
+
+ if (empty)
+ {
+ // Note that we keep empty origin configurations if we have any
+ // dependencies to upgrade (see below for details).
+ //
+ if (dep_pkgs.empty ())
+ continue;
+ else
+ {
+ if (find_if (origin_cfgs.begin (), origin_cfgs.end (),
+ [&cfg] (const sync_config& ocfg)
+ {
+ return ocfg.path () == cfg->path;
+ }) == origin_cfgs.end ())
+ continue;
+ }
+ }
+ else if (!cfg.origin)
+ origin_only = false;
+
+ auto i (find_if (cfgs.begin (), cfgs.end (),
+ [&cfg] (const config& c)
+ {
+ return cfg->path == c.path.get ();
+ }));
+
+ if (i == cfgs.end ())
+ {
+ cfgs.push_back (config {cfg->path, {}});
+ i = cfgs.end () - 1;
+ }
+
+ if (cfg.fetch && !empty)
+ i->reps.push_back (repository_name (prj.path));
+ }
+ }
+
+ bool multi_cfg (cfgs.size () != 1);
- // First add configuration variables from pkg_args, if any.
+ // Start by adding configuration variables from pkg_args, if any.
+ //
+ // If we have dep_pkgs (third form), then non-global configuration
+ // variables should only apply to them. Otherwise, if we have origin
+ // (first form), then they should only apply to packages from the origin
+ // project in origin configurations. They don't seem to make sense
+ // otherwise (second form, implicit).
//
+ bool dep_vars (false);
+ bool origin_vars (false);
+
+ if (!pkg_args.empty ())
{
+ if (origin_only)
+ {
+ for (const string& a: pkg_args)
+ {
+ if (a.find ('=') == string::npos)
+ {
+ origin_only = false;
+ break;
+ }
+ }
+ }
+
for (const string& a: pkg_args)
- if (a.find ('=') != string::npos)
- args.push_back (a);
+ {
+ size_t p (a.find ('='));
+ if (p == string::npos)
+ continue;
+
+ if (a.front () != '!')
+ {
+ if (!dep_pkgs.empty ())
+ dep_vars = true;
+ else if (origin)
+ {
+ // Simplify the command line if we only have origin on the command
+ // line.
+ //
+ if (!origin_only)
+ origin_vars = true;
+ }
+ else
+ fail << "non-global configuration variable " <<
+ string (a, 0, p) << " without packages or dependencies";
+
+ if (dep_vars || origin_vars)
+ continue;
+ }
+
+ args.push_back (a);
+ }
if (!args.empty ())
args.push_back ("--");
}
- for (const project& prj: prjs)
+ // Next collect init'ed packages.
+ //
+ for (const sync_project& prj: prjs)
{
- if (prj.fetch)
- reps.push_back (repository_name (prj.path));
-
- for (const package_state& pkg: prj.config->packages)
+ for (const sync_project::config& cfg: prj.configs)
{
- if (upgrade && !prj.implicit)
+ if (cfg->packages.empty ())
+ continue;
+
+ for (const package_state& pkg: cfg->packages)
{
- // We synchronize all the init'ed packages, including those from
- // other projects. But if the dependencies are not specified, we
- // only upgrade dependencies of the packages specified explicitly or
- // init'ed in the origin project.
- //
- if (dep_pkgs.empty ())
+ bool vars (origin_vars && cfg.origin);
+
+ bool g (multi_cfg || vars);
+ if (g)
+ args.push_back ("{");
+
+ if (multi_cfg)
+ args.push_back ("--config-uuid=" +
+ linked_cfgs.find (cfg->path)->uuid.string ());
+
+ if (upgrade && dep_pkgs.empty () && !cfg.implicit)
{
+ // We synchronize all the init'ed packages, including those from
+ // other projects. But if the dependencies are not specified, we
+ // only upgrade dependencies of the packages specified explicitly
+ // or init'ed in the origin project.
+ //
auto contains = [] (const auto& pkgs, const package_state& pkg)
{
return find_if (pkgs.begin (), pkgs.end (),
@@ -790,9 +1041,17 @@ namespace bdep
}) != pkgs.end ();
};
- if (prj_pkgs.empty () && origin_config != nullptr
- ? contains (origin_config->packages, pkg)
- : contains (prj_pkgs, pkg))
+ bool c (false);
+ if (prj_pkgs.empty () && origin)
+ {
+ for (const sync_config& cfg: origin_cfgs)
+ if ((c = contains (cfg->packages, pkg)))
+ break;
+ }
+ else
+ c = contains (prj_pkgs, pkg);
+
+ if (c)
{
// The project package itself must always be upgraded to the
// latest version/iteration. So we have to translate to
@@ -800,55 +1059,150 @@ namespace bdep
//
assert (recursive);
- args.push_back ("{");
+ if (!g)
+ {
+ args.push_back ("{");
+ g = true;
+ }
+
args.push_back (
*upgrade
? *recursive ? "--upgrade-recursive" : "--upgrade-immediate"
: *recursive ? "--patch-recursive" : "--patch-immediate");
- args.push_back ("}+");
}
}
- }
- // We need to add the explicit location qualification (@<rep-loc>)
- // since there is no guarantee a higher version isn't available from
- // another repository.
- //
- args.push_back (pkg.name.string () + '@' + prj.path.string ());
+ // Note: must come after options, if any.
+ //
+ if (vars)
+ {
+ for (const string& a: pkg_args)
+ if (a.find ('=') != string::npos && a.front () != '!')
+ args.push_back (a);
+ }
+
+ if (g)
+ args.push_back ("}+");
+
+ // We need to add the explicit location qualification (@<rep-loc>)
+ // since there is no guarantee a better version isn't available from
+ // another repository.
+ //
+ args.push_back (pkg.name.string () + '@' + prj.path.string ());
+ }
}
}
// Add dependencies to upgrade (if any).
//
+ // This gets quite fuzzy when we are dealing with multiple configurations.
+ // To start, we only want to upgrade dependencies in the configurations
+ // specified by the user (origin_cfgs). This part is pretty clear. But if
+ // we just qualify the dependencies with configurations, then that will be
+ // interpreted by bpkg as a request to move if any other configurations
+ // (of the same type) happened to also have this package as a dependency.
+ // So we pass --no-move below to prevent this (which basically means
+ // "upgrade there if present, build there if necessary, and otherwise
+ // ignore").
+ //
+ // The (admittedly still fuzzy) conceptual model behind this is that if
+ // the user wishes to partition dependencies into multiple configurations
+ // (of the same type; say base/target) then they do it manually with bpkg
+ // and bdep does not alter this partitioning of dependencies in any way.
+ //
+ // Note that in this model, while the user may use bpkg directly to
+ // upgrade/downgrate such dependencies, that feels a bit awkward (imagine
+ // host and base having the same dependency with one upgraded via bdep
+ // while the other -- via bpkg). Instead, we support associating such a
+ // base configuration with a bdep-managed project and using that to manage
+ // upgrades by not skipping empty origin configurations in this mode (see
+ // above). Note also that in this case we don't need to worry about
+ // fetching empty configuration's repository information since it won't be
+ // used for the upgrade anyway (instead, information from ultimate
+ // dependent's configuration will be used).
+ //
if (upgrade)
{
for (const string& n: dep_pkgs)
{
+ bool g (multi_cfg || dep_vars);
+ if (g)
+ args.push_back ("{");
+
+ if (multi_cfg)
+ for (const sync_config& cfg: origin_cfgs)
+ args.push_back ("--config-uuid=" +
+ linked_cfgs.find (cfg.path ())->uuid.string ());
+
// Unless this is the default "non-recursive upgrade" we need to add a
// group.
//
if (recursive || !*upgrade)
{
- args.push_back ("{");
+ if (!g)
+ {
+ args.push_back ("{");
+ g = true;
+ }
string o (*upgrade ? "-u" : "-p");
if (recursive) o += *recursive ? 'r' : 'i';
args.push_back (move (o));
+ }
- args.push_back ("}+");
+ // Note: must come after options, if any.
+ //
+ if (dep_vars)
+ {
+ for (const string& a: pkg_args)
+ if (a.find ('=') != string::npos && a.front () != '!')
+ args.push_back (a);
}
+ if (g)
+ args.push_back ("}+");
+
// Make sure it is treated as a dependency.
//
args.push_back ('?' + n);
}
}
- // Finally, add packages from pkg_args, if any.
+ // Finally, add packages (?<pkg>) from pkg_args, if any.
+ //
+ // Similar to the dep_pkgs case above, we restrict this to the origin
+ // configurations.
//
for (const string& a: pkg_args)
- if (a.find ('=') == string::npos)
- args.push_back (a);
+ {
+ if (a.find ('=') != string::npos)
+ continue;
+
+ if (multi_cfg)
+ {
+ args.push_back ("{");
+
+ // Note that here (unlike the dep_pkgs case above), we have to make
+ // sure the configuration is actually involved.
+ //
+ for (const sync_config& ocfg: origin_cfgs)
+ {
+ if (find_if (cfgs.begin (), cfgs.end (),
+ [&ocfg] (const config& cfg)
+ {
+ return ocfg.path () == cfg.path.get ();
+ }) == cfgs.end ())
+ continue;
+
+ args.push_back ("--config-uuid=" +
+ linked_cfgs.find (ocfg.path ())->uuid.string ());
+ }
+
+ args.push_back ("}+");
+ }
+
+ args.push_back (a);
+ }
// We do a separate fetch instead of letting pkg-build do it. This way we
// get better control of the diagnostics (no "fetching ..." for the
@@ -867,25 +1221,39 @@ namespace bdep
// "synchronizing <cfg-dir>:". Maybe rep-fetch also needs something
// like --plan but for progress? Plus there might be no sync at all.
//
- if (!reps.empty ())
- run_bpkg (3, co, "fetch", "-d", cfg, "--shallow", reps);
+ for (const config& cfg: cfgs)
+ {
+ if (cfg.reps.empty ())
+ continue;
- string plan ("synchronizing");
+ run_bpkg (3, co, "fetch", "-d", cfg.path.get (), "--shallow", cfg.reps);
+ }
+
+ string plan;
if (name_cfg)
{
- plan += ' ';
-
- // Use name if available, directory otherwise.
- //
- if (origin_config != nullptr && origin_config->name)
+ for (const sync_config& cfg: origin_cfgs)
{
- plan += '@';
- plan += *origin_config->name;
+ if (!plan.empty ())
+ plan += ",\n";
+
+ plan += "synchronizing ";
+
+ // Use name if available, directory otherwise.
+ //
+ if (cfg != nullptr && cfg->name)
+ {
+ plan += '@';
+ plan += *cfg->name;
+ }
+ else
+ plan += cfg.path ().representation ();
}
- else
- plan += cfg.representation ();
+
+ plan += ':';
}
- plan += ':';
+ else
+ plan = "synchronizing:";
// Now configure the requested packages, preventing bpkg-pkg-build from
// creating private configurations for build-time dependencies and
@@ -941,23 +1309,34 @@ namespace bdep
bool need_config (false);
strings dep_chain;
{
+ // -d options
+ //
+ small_vector<const char*, 32> d;
+ for (const config& cfg: cfgs)
+ {
+ d.push_back ("-d");
+ d.push_back (cfg.path.get ().string ().c_str ());
+ }
+
fdpipe pipe (open_pipe ()); // Text mode seems appropriate.
- process pr (start_bpkg (2,
- co,
- pipe /* stdout */,
- 2 /* stderr */,
- "build",
- "-d", cfg,
- "--no-fetch",
- "--no-refinement",
- "--no-private-config", 125,
- "--noop-exit", 124,
- "--configure-only",
- "--keep-out",
- "--plan", plan,
- (yes ? "--yes" : nullptr),
- args));
+ process pr (
+ start_bpkg (2,
+ co,
+ pipe /* stdout */,
+ 2 /* stderr */,
+ "build",
+ d,
+ (linked_cfgs.size () != 1 ? "--no-move" : nullptr),
+ "--no-fetch",
+ "--no-refinement",
+ "--no-private-config", 125,
+ "--noop-exit", 124,
+ "--configure-only",
+ "--keep-out",
+ "--plan", plan,
+ (yes ? "--yes" : nullptr),
+ args));
// Shouldn't throw, unless something is severely damaged.
//
@@ -1028,13 +1407,12 @@ namespace bdep
}
link_dependency_config (co,
- cfg,
+ origin_prj, origin_cfgs,
prjs,
- origin_prj, origin_config,
dep_chain,
create_host_config,
create_build2_config,
- tr,
+ origin_tr,
created_cfgs,
trace);
}
@@ -1052,81 +1430,84 @@ namespace bdep
//
if (!implicit || !noop)
{
- for (const project& prj: prjs)
+ for (const sync_project& prj: prjs)
{
package_locations pls (load_packages (prj.path));
- for (const package_state& pkg: prj.config->packages)
+ for (const sync_project::config& cfg: prj.configs)
{
- // If this is a forwarded configuration, make sure forwarding is
- // configured and is up-to-date. Otherwise, make sure it is
- // disfigured (the config set --no-forward case).
- //
- dir_path src (prj.path);
+ for (const package_state& pkg: cfg->packages)
{
- auto i (find_if (pls.begin (),
- pls.end (),
- [&pkg] (const package_location& pl)
- {
- return pkg.name == pl.name;
- }));
-
- if (i == pls.end ())
- fail << "package " << pkg.name << " is not listed in "
- << prj.path;
-
- src /= i->path;
- }
+ // If this is a forwarded configuration, make sure forwarding is
+ // configured and is up-to-date. Otherwise, make sure it is
+ // disfigured (the config set --no-forward case).
+ //
+ dir_path src (prj.path);
+ {
+ auto i (find_if (pls.begin (), pls.end (),
+ [&pkg] (const package_location& pl)
+ {
+ return pkg.name == pl.name;
+ }));
-#if 0
- // We could run 'b info' and used the 'forwarded' value but this is
- // both faster and simpler. Or at least it was until we got the
- // alternative naming scheme.
- //
- auto check = [&src] ()
- {
- path f (src / "build" / "bootstrap" / "out-root.build");
- bool e (exists (f));
+ if (i == pls.end ())
+ fail << "package " << pkg.name << " is not listed in "
+ << prj.path;
- if (!e)
- {
- f = src / "build2" / "bootstrap" / "out-root.build2";
- e = exists (f);
+ src /= i->path;
}
- return e;
- };
+#if 0
+ // We could run 'b info' and used the 'forwarded' value but this
+ // is both faster and simpler. Or at least it was until we got the
+ // alternative naming scheme.
+ //
+ auto check = [&src] ()
+ {
+ path f (src / "build" / "bootstrap" / "out-root.build");
+ bool e (exists (f));
+
+ if (!e)
+ {
+ f = src / "build2" / "bootstrap" / "out-root.build2";
+ e = exists (f);
+ }
+
+ return e;
+ };
#endif
- const char* o (nullptr);
- if (prj.config->forward)
- {
- o = "configure:";
- }
- else if (!prj.implicit) // Requires explicit sync.
- {
- //@@ This is broken: we will disfigure forwards to other configs.
- // Looks like we will need to test that the forward is to this
- // config. 'b info' here we come?
+ const char* o (nullptr);
+ if (cfg->forward)
+ {
+ o = "configure:";
+ }
+ else if (!cfg.implicit) // Requires explicit sync.
+ {
+ //@@ This is broken: we will disfigure forwards to other
+ // configs. Looks like we will need to test that the forward
+ // is to this config. 'b info' here we come?
#if 0
- if (check ())
- o = "disfigure:";
+ if (check ())
+ o = "disfigure:";
#endif
- }
+ }
- if (o != nullptr)
- {
- dir_path out (dir_path (cfg) /= pkg.name.string ());
+ if (o != nullptr)
+ {
+ dir_path out (dir_path (cfg->path) /= pkg.name.string ());
- // Note that --no-external-modules makes a difference for
- // developing build system modules that require bootstrapping
- // (which without that option would trigger a recursive sync).
- //
- run_b (co,
- "--no-external-modules",
- o,
- "'" + src.representation () + "'@'" + out.representation () +
- "',forward");
+ // Note that --no-external-modules makes a difference for
+ // developing build system modules that require bootstrapping
+ // (which without that option would trigger a recursive sync).
+ //
+ run_b (
+ co,
+ "--no-external-modules",
+ o,
+ "'" + src.representation () + "'@'" + out.representation () +
+ "',forward");
+ }
}
}
}
@@ -1134,60 +1515,66 @@ namespace bdep
// Add/remove auto-synchronization build system hook.
//
- if (origin_config != nullptr && !implicit)
+ // It feels right to only do this for origin_cfgs (remember, we require
+ // explicit sync after changing the auto-sync flag).
+ //
+ if (origin && !implicit)
{
- path f (cfg / hook_file);
- bool e (exists (f));
-
- if (origin_config->auto_sync)
+ for (const sync_config& cfg: origin_cfgs)
{
- if (!e)
- {
- mk (f.directory ());
+ path f (cfg->path / hook_file);
+ bool e (exists (f));
- try
+ if (cfg->auto_sync)
+ {
+ if (!e)
{
- ofdstream os (f);
+ mk (f.directory ());
- // Should we analyze BDEP_SYNCED_CONFIGS ourselves or should we
- // let bdep-sync do it for us? Doing it here instead of spawning a
- // process (which will load the database, etc) will be faster.
- // But, on the other hand, this is only an issue for commands like
- // update and test that do their own implicit sync.
- //
- // cfgs = $getenv(BDEP_SYNCED_CONFIGS)
- // if! $null($cfgs)
- // cfgs = [dir_paths] $regex.split($cfgs, ' *"([^"]*)" *', '\1')
- //
- // Also note that we try to avoid setting any variables in order
- // not to pollute the configuration's root scope.
- //
- os << "# Created automatically by bdep." << endl
- << "#" << endl
- << "if ($build.meta_operation != 'info' && \\" << endl
- << " $build.meta_operation != 'configure' && \\" << endl
- << " $build.meta_operation != 'disfigure')" << endl
- << "{" << endl
- << " if ($getenv('BDEP_SYNC') == [null] || \\" << endl
- << " $getenv('BDEP_SYNC') == true || \\" << endl
- << " $getenv('BDEP_SYNC') == 1)" << endl
- << " run '" << argv0 << "' sync --hook=1 " <<
- "--verbose $build.verbosity " <<
- "--config \"$out_root\"" << endl
- << "}" << endl;
-
- os.close ();
- }
- catch (const io_error& e)
- {
- fail << "unable to write to " << f << ": " << e;
+ try
+ {
+ ofdstream os (f);
+
+ // Should we analyze BDEP_SYNCED_CONFIGS ourselves or should we
+ // let bdep-sync do it for us? Doing it here instead of spawning
+ // a process (which will load the database, etc) will be faster.
+ // But, on the other hand, this is only an issue for commands
+ // like update and test that do their own implicit sync.
+ //
+ // cfgs = $getenv(BDEP_SYNCED_CONFIGS)
+ // if! $null($cfgs)
+ // cfgs = [dir_paths] $regex.split($cfgs, ' *"([^"]*)" *', '\1')
+ //
+ // Also note that we try to avoid setting any variables in order
+ // not to pollute the configuration's root scope.
+ //
+ os << "# Created automatically by bdep." << endl
+ << "#" << endl
+ << "if ($build.meta_operation != 'info' && \\" << endl
+ << " $build.meta_operation != 'configure' && \\" << endl
+ << " $build.meta_operation != 'disfigure')" << endl
+ << "{" << endl
+ << " if ($getenv('BDEP_SYNC') == [null] || \\" << endl
+ << " $getenv('BDEP_SYNC') == true || \\" << endl
+ << " $getenv('BDEP_SYNC') == 1)" << endl
+ << " run '" << argv0 << "' sync --hook=1 " <<
+ "--verbose $build.verbosity " <<
+ "--config \"$out_root\"" << endl
+ << "}" << endl;
+
+ os.close ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to write to " << f << ": " << e;
+ }
}
}
- }
- else
- {
- if (e)
- rm (f);
+ else
+ {
+ if (e)
+ rm (f);
+ }
}
}
}
@@ -1260,6 +1647,108 @@ namespace bdep
return false;
}
+ // Given the configuration directory, return absolute and normalized
+ // directories and UUIDs of the entire linked configuration cluster.
+ //
+ static linked_configs
+ find_config_cluster (const common_options& co, const dir_path& d)
+ {
+ linked_configs r;
+
+ // Run bpkg-cfg-info to get the list of linked configurations.
+ //
+ fdpipe pipe (open_pipe ()); // Text mode seems appropriate.
+
+ process pr (start_bpkg (3,
+ co,
+ pipe /* stdout */,
+ 2 /* stderr */,
+ "cfg-info",
+ "-d", d,
+ "--link",
+ "--backlink",
+ "--recursive"));
+
+ pipe.out.close (); // Shouldn't throw unless very broken.
+
+ bool io (false);
+ try
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit);
+
+ string l; // Reuse the buffer.
+ do
+ {
+ linked_config c;
+
+ optional<pair<bool /* path */, bool /* uuid */>> s;
+ while (!eof (getline (is, l)))
+ {
+ if (!s)
+ s = make_pair (false, false);
+
+ if (l.empty ())
+ break;
+
+ if (l.compare (0, 6, "path: ") == 0)
+ {
+ try
+ {
+ c.path = dir_path (string (l, 6));
+ s->first = true;
+ }
+ catch (const invalid_path&)
+ {
+ fail << "invalid bpkg-cfg-info output line '" << l
+ << "': invalid configuration path";
+ }
+ }
+ else if (l.compare (0, 6, "uuid: ") == 0)
+ {
+ try
+ {
+ c.uuid = uuid (string (l, 6));
+ s->second = true;
+ }
+ catch (const invalid_argument&)
+ {
+ fail << "invalid bpkg-cfg-info output line '" << l
+ << "': invalid configuration uuid";
+ }
+ }
+ }
+
+ if (s)
+ {
+ if (!s->first)
+ fail << "invalid bpkg-cfg-info output: missing configuration path";
+
+ if (!s->second)
+ fail << "invalid bpkg-cfg-info output: missing configuration uuid";
+
+ r.push_back (move (c));
+ }
+ }
+ while (!is.eof ());
+
+ is.close (); // Detect errors.
+ }
+ catch (const io_error&)
+ {
+ // Presumably the child process failed and issued diagnostics so let
+ // finish_bpkg() try to deal with that first.
+ //
+ io = true;
+ }
+
+ finish_bpkg (co, pr, io);
+
+ if (r.empty ()) // We should have at leas the main configuration.
+ fail << "invalid bpkg-cfg-info output: missing configuration information";
+
+ return r;
+ }
+
void
cmd_sync (const common_options& co,
const dir_path& prj,
@@ -1274,24 +1763,28 @@ namespace bdep
transaction* t,
vector<pair<dir_path, string>>* created_cfgs)
{
- if (!synced (c->path, implicit))
- cmd_sync (co,
- c->path,
- prj,
- c,
- pkg_args,
- implicit,
- fetch,
- yes,
- name_cfg,
- nullopt /* upgrade */,
- nullopt /* recursive */,
- package_locations () /* prj_pkgs */,
- strings () /* dep_pkgs */,
- create_host_config,
- create_build2_config,
- t,
- created_cfgs);
+ assert (!c->packages.empty ());
+
+ if (synced (c->path, implicit))
+ return;
+
+ cmd_sync (co,
+ prj,
+ {sync_config (c)},
+ find_config_cluster (co, c->path),
+ pkg_args,
+ implicit,
+ fetch,
+ yes,
+ name_cfg,
+ nullopt /* upgrade */,
+ nullopt /* recursive */,
+ package_locations () /* prj_pkgs */,
+ strings () /* dep_pkgs */,
+ create_host_config,
+ create_build2_config,
+ t,
+ created_cfgs);
}
void
@@ -1303,22 +1796,24 @@ namespace bdep
bool create_host_config,
bool create_build2_config)
{
- if (!synced (cfg, true /* implicit */))
- cmd_sync (co,
- cfg,
- dir_path (),
- nullptr,
- strings (),
- true /* implicit */,
- fetch,
- yes,
- name_cfg,
- nullopt /* upgrade */,
- nullopt /* recursive */,
- package_locations () /* prj_pkgs */,
- strings () /* dep_pkgs */,
- create_host_config,
- create_build2_config);
+ if (synced (cfg, true /* implicit */))
+ return;
+
+ cmd_sync (co,
+ dir_path () /* prj */,
+ {sync_config (cfg)},
+ find_config_cluster (co, cfg),
+ strings () /* pkg_args */,
+ true /* implicit */,
+ fetch,
+ yes,
+ name_cfg,
+ nullopt /* upgrade */,
+ nullopt /* recursive */,
+ package_locations () /* prj_pkgs */,
+ strings () /* dep_pkgs */,
+ create_host_config,
+ create_build2_config);
}
int
@@ -1404,8 +1899,12 @@ namespace bdep
dir_path prj; // Empty if we have no originating project.
package_locations prj_pkgs;
- configurations cfgs;
- dir_paths cfg_dirs;
+
+ // Note that all elements are either NULL/path or not (see initialization
+ // below for details).
+ //
+ list<sync_config> cfgs;
+
bool default_fallback (false);
// In the implicit mode we don't search the current working directory
@@ -1451,7 +1950,8 @@ namespace bdep
prj = move (pp.project);
prj_pkgs = move (pp.packages);
- cfgs = move (cs.first);
+ cfgs.assign (make_move_iterator (cs.first.begin ()),
+ make_move_iterator (cs.first.end ()));
default_fallback = cs.second;
}
else
@@ -1488,8 +1988,7 @@ namespace bdep
if (synced (d, o.implicit (), false /* add */))
continue;
- cfgs.push_back (nullptr);
- cfg_dirs.push_back (move (d));
+ cfgs.push_back (move (d));
}
if (cfgs.empty ())
@@ -1502,20 +2001,71 @@ namespace bdep
// Synchronize each configuration.
//
+ // A configuration can be part of a linked cluster which we sync all at
+ // once. And some configurations in cfgs can be part of earlier clusters
+ // that we have already sync'ed. Re-synchronizing them (actually the whole
+ // clusters) again would be strange. Plus, we want to treat them as
+ // "originating configurations" for dependency upgrade purposes.
+ //
+ // So what we are going to do is remove configurations from cfgs/cfg_dirs
+ // as we go along.
+ //
bool empty (true); // All configurations are empty.
- for (size_t i (0), n (cfgs.size ()); i != n; ++i)
+ for (size_t i (0), n (cfgs.size ()); !cfgs.empty (); )
{
- const shared_ptr<configuration>& c (cfgs[i]); // Can be NULL.
- const dir_path& cd (c != nullptr ? c->path : cfg_dirs[i]);
+ sync_configs ocfgs; // Originating configurations for this sync.
+ optional<size_t> m; // Number of packages in ocfgs.
+
+ ocfgs.push_back (move (cfgs.front ()));
+ cfgs.pop_front ();
+
+ if (const shared_ptr<configuration>& c = ocfgs.back ())
+ m = c->packages.size ();
+
+ const dir_path& cd (ocfgs.back ().path ());
// Check if this configuration is already (being) synchronized.
//
+ // Note that we should ignore the whole cluster but we can't run bpkg
+ // here. So we will just ignore the configurations one by one (we expect
+ // them all to be on the list, see below).
+ //
if (synced (cd, o.implicit ()))
{
empty = false;
continue;
}
+ // Get the linked configuration cluster and "pull out" of cfgs
+ // configurations that belong to this cluster. While at it also mark the
+ // entire cluster as being synced.
+ //
+ // Note: we have already deatl with the first configuration in lcfgs.
+ //
+ linked_configs lcfgs (find_config_cluster (o, cd));
+
+ for (auto j (lcfgs.begin () + 1); j != lcfgs.end (); ++j)
+ {
+ const linked_config& cfg (*j);
+
+ bool r (synced (cfg.path, true /* implicit */));
+ assert (!r); // Should have been skipped via the first above.
+
+ for (auto i (cfgs.begin ()); i != cfgs.end (); )
+ {
+ if (cfg.path == i->path ())
+ {
+ ocfgs.push_back (move (*i));
+ i = cfgs.erase (i);
+
+ if (const shared_ptr<configuration>& c = ocfgs.back ())
+ *m += c->packages.size ();
+ }
+ else
+ ++i;
+ }
+ }
+
// Skipping empty ones (part one).
//
// Note that we would normally be printing that for build-time
@@ -1524,23 +2074,48 @@ namespace bdep
// in case of the default configuration fallback (but also check and
// warn if all of them were empty below).
//
- if (c != nullptr && c->packages.empty () && default_fallback)
+ if (m && *m == 0 && default_fallback)
continue;
// If we are synchronizing multiple configurations, separate them with a
// blank line and print the configuration name/directory.
//
if (verb && n > 1)
- text << (i == 0 ? "" : "\n")
- << "in configuration " << *c << ':';
+ {
+ diag_record dr (text);
+
+ if (i++ != 0)
+ dr << '\n';
+
+ for (auto b (ocfgs.begin ()), j (b); j != ocfgs.end (); ++j)
+ {
+ if (j != b)
+ dr << ",\n";
+
+ dr << "in configuration " << *j;
+ }
+
+ dr << ':';
+ }
// Skipping empty ones (part two).
//
- if (c != nullptr && c->packages.empty ())
+ if (m && *m == 0)
{
if (verb)
- info << "no packages initialized in configuration " << *c
- << ", skipping";
+ {
+ diag_record dr (info);
+ dr << "no packages initialized in ";
+
+ // Note that in case of a cluster, we know we have printed the
+ // configurations (see above) and thus can omit mentioning them
+ // here.
+ //
+ if (ocfgs.size () == 0)
+ dr << "configuration " << *ocfgs.back () << ", skipping";
+ else
+ dr << "configuration cluster, skipping";
+ }
continue;
}
@@ -1550,7 +2125,10 @@ namespace bdep
bool fetch (o.fetch () || o.fetch_full ());
if (fetch)
- cmd_fetch (o, prj, c, o.fetch_full ());
+ {
+ for (const sync_config& c: ocfgs)
+ cmd_fetch (o, prj, c, o.fetch_full ());
+ }
if (!dep_pkgs.empty ())
{
@@ -1564,9 +2142,9 @@ namespace bdep
// Only prompt if upgrading their dependencies.
//
cmd_sync (o,
- cd,
prj,
- c,
+ move (ocfgs),
+ move (lcfgs),
pkg_args,
false /* implicit */,
!fetch,
@@ -1586,9 +2164,9 @@ namespace bdep
// (immediate by default, recursive if requested).
//
cmd_sync (o,
- cd,
prj,
- c,
+ move (ocfgs),
+ move (lcfgs),
pkg_args,
false /* implicit */,
!fetch,
@@ -1609,9 +2187,9 @@ namespace bdep
// time) add the configuration name/directory to the plan header.
//
cmd_sync (o,
- cd,
prj,
- c,
+ move (ocfgs),
+ move (lcfgs),
pkg_args,
o.implicit (),
!fetch,
diff --git a/bdep/types.hxx b/bdep/types.hxx
index 23ea3f2..2413382 100644
--- a/bdep/types.hxx
+++ b/bdep/types.hxx
@@ -23,6 +23,8 @@
#include <libbutl/url.mxx>
#include <libbutl/path.mxx>
+#include <libbutl/uuid.hxx>
+#include <libbutl/uuid-io.hxx>
#include <libbutl/process.mxx>
#include <libbutl/optional.mxx>
#include <libbutl/fdstream.mxx>
@@ -96,6 +98,10 @@ namespace bdep
using paths = vector<path>;
using dir_paths = vector<dir_path>;
+ // <libbutl/uuid.mxx>
+ //
+ using butl::uuid;
+
// <libbutl/process.mxx>
//
using butl::process;