diff options
Diffstat (limited to 'bpkg/pkg-checkout.cxx')
| -rw-r--r-- | bpkg/pkg-checkout.cxx | 394 |
1 files changed, 280 insertions, 114 deletions
diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx index 05875a4..81efdc2 100644 --- a/bpkg/pkg-checkout.cxx +++ b/bpkg/pkg-checkout.cxx @@ -10,23 +10,26 @@ #include <bpkg/package-odb.hxx> #include <bpkg/database.hxx> #include <bpkg/checksum.hxx> +#include <bpkg/rep-mask.hxx> #include <bpkg/diagnostics.hxx> #include <bpkg/manifest-utility.hxx> #include <bpkg/pkg-purge.hxx> #include <bpkg/pkg-verify.hxx> -#include <bpkg/pkg-configure.hxx> using namespace std; using namespace butl; namespace bpkg { + // pkg_checkout() + // static void checkout (const common_options& o, const repository_location& rl, const dir_path& dir, - const shared_ptr<available_package>& ap) + const shared_ptr<available_package>& ap, + database& db) { switch (rl.type ()) { @@ -41,9 +44,9 @@ namespace bpkg // Print the progress indicator to attribute the possible fetching // progress. // - if (verb && !o.no_progress ()) + if ((verb && !o.no_progress ()) || o.progress ()) text << "checking out " - << package_string (ap->id.name, ap->version); + << package_string (ap->id.name, ap->version) << db; git_checkout_submodules (o, rl, dir); } @@ -58,19 +61,23 @@ namespace bpkg // For some platforms/repository types the working tree needs to be // temporary "fixed up" for the build2 operations to work properly on it. // - static bool + static optional<bool> fixup (const common_options& o, const repository_location& rl, const dir_path& dir, - bool revert = false) + bool revert = false, + bool ie = false) { - bool r (false); + optional<bool> r; switch (rl.type ()) { case repository_type::git: { - r = git_fixup_worktree (o, dir, revert); + if (!revert && !ie) + git_verify_symlinks (o, dir); + + r = git_fixup_worktree (o, dir, revert, ie); break; } case repository_type::pkg: @@ -80,23 +87,30 @@ namespace bpkg return r; } - shared_ptr<selected_package> - pkg_checkout (const common_options& o, - const dir_path& c, + // Return the selected package object which may replace the existing one. + // + static shared_ptr<selected_package> + pkg_checkout (pkg_checkout_cache& cache, + const common_options& o, + database& pdb, + database& rdb, transaction& t, package_name n, version v, + const optional<dir_path>& output_root, bool replace, + bool purge, bool simulate) { tracer trace ("pkg_checkout"); - database& db (t.database ()); - tracer_guard tg (db, trace); + tracer_guard tg (pdb, trace); // NOTE: sets tracer for the whole cluster. + + const dir_path& c (pdb.config_orig); // See if this package already exists in this configuration. // - shared_ptr<selected_package> p (db.find<selected_package> (n)); + shared_ptr<selected_package> p (pdb.find<selected_package> (n)); if (p != nullptr) { @@ -117,13 +131,13 @@ namespace bpkg } } - check_any_available (c, t); + check_any_available (rdb, t); - // Note that here we compare including the revision (see pkg-fetch() + // Note that here we compare including the revision (see pkg_fetch() // implementation for more details). // shared_ptr<available_package> ap ( - db.find<available_package> (available_package_id (n, v))); + rdb.find<available_package> (available_package_id (n, v))); if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; @@ -135,14 +149,17 @@ namespace bpkg for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository_fragment.load ()->location); - - if (rl.version_control_based () && (pl == nullptr || rl.local ())) + if (!rep_masked_fragment (l.repository_fragment)) { - pl = &l; + const repository_location& rl (l.repository_fragment.load ()->location); - if (rl.local ()) - break; + if (rl.version_control_based () && (pl == nullptr || rl.local ())) + { + pl = &l; + + if (rl.local ()) + break; + } } } @@ -157,8 +174,8 @@ namespace bpkg const repository_location& rl (pl->repository_fragment->location); auto_rmdir rmd; - optional<string> mc; - dir_path d (c / dir_path (n.string () + '-' + v.string ())); + const dir_path& ord (output_root ? *output_root : c); + dir_path d (ord / dir_path (n.string () + '-' + v.string ())); // An incomplete checkout may result in an unusable repository state // (submodule fetch is interrupted, working tree fix up failed in the @@ -167,10 +184,7 @@ namespace bpkg // (or interruption) the user will need to run bpkg-rep-fetch to restore // the missing repository. // - bool fs_changed (false); - if (!simulate) - try { if (exists (d)) fail << "package directory " << d << " already exists"; @@ -179,57 +193,84 @@ namespace bpkg // if the previous checkout have failed or been interrupted. // dir_path sd (repository_state (rl)); - dir_path rd (c / repos_dir / sd); + dir_path rd (rdb.config_orig / repos_dir / sd); - if (!exists (rd)) - fail << "missing repository directory for package " << n << " " << v - << " in configuration " << c << - info << "run 'bpkg rep-fetch' to repair"; + // Use the temporary directory from the repository information source + // configuration, so that we can always move the repository into and out + // of it (note that if they appear on different filesystems that won't + // be possible). + // + auto ti (tmp_dirs.find (rdb.config_orig)); + assert (ti != tmp_dirs.end ()); + const dir_path& tdir (ti->second); - // The repository temporary directory. + // Try to reuse the cached repository (moved to the temporary directory + // with some fragment checked out and fixed up). // - auto_rmdir rmt (temp_dir / sd); - const dir_path& td (rmt.path); + pkg_checkout_cache::state_map& cm (cache.map_); + auto i (cm.find (rd)); + + if (i == cm.end () || i->second.rl.fragment () != rl.fragment ()) + { + // Restore the repository if some different fragment is checked out. + // + if (i != cm.end ()) + cache.erase (i); + + // Checkout and cache the fragment. + // + if (!exists (rd)) + fail << "missing repository directory for package " << n << " " << v + << " in its repository information configuration " + << rdb.config_orig << + info << "run 'bpkg rep-fetch' to repair"; + + // The repository temporary directory. + // + auto_rmdir rmt (tdir / sd, !keep_tmp); + + // Move the repository to the temporary directory. + // + { + const dir_path& td (rmt.path); + + if (exists (td)) + rm_r (td); + + mv (rd, td); + } + + // Pre-insert the incomplete repository entry into the cache and + // "finalize" it by setting the fixed up value later, after the + // repository fragment checkout succeeds. Until then the repository + // may not be restored in its permanent place. + // + using state = pkg_checkout_cache::state; - if (exists (td)) - rm_r (td); + i = cm.emplace (rd, state {move (rmt), rl, nullopt}).first; + + // Checkout the repository fragment and fix up the working tree. + // + state& s (i->second); + const dir_path& td (s.rmt.path); + + checkout (o, rl, td, ap, pdb); + s.fixedup = fixup (o, rl, td); + } // The temporary out of source directory that is required for the dist // meta-operation. // - auto_rmdir rmo (temp_dir / dir_path (n.string ())); + auto_rmdir rmo (tdir / dir_path (n.string ()), !keep_tmp); const dir_path& od (rmo.path); if (exists (od)) rm_r (od); - // Finally, move the repository to the temporary directory and proceed - // with the checkout. - // - mv (rd, td); - fs_changed = true; - - // Checkout the repository fragment and fix up the working tree. - // - checkout (o, rl, td, ap); - bool fixedup (fixup (o, rl, td)); - // Calculate the package path that points into the checked out fragment // directory. // - dir_path pd (td / path_cast<dir_path> (pl->location)); - - // Verify the package prerequisites are all configured since the dist - // meta-operation generally requires all imports to be resolvable. - // - package_manifest m (pkg_verify (pd, - true /* ignore_unknown */, - [&ap] (version& v) {v = ap->version;})); - - pkg_configure_prerequisites (o, - t, - convert (move (m.dependencies)), - m.name); + dir_path pd (i->second.rmt.path / path_cast<dir_path> (pl->location)); // Form the buildspec. // @@ -245,7 +286,11 @@ namespace bpkg // Distribute. // - // Note that on failure the package stays in the existing (working) + // Note that we are using the bootstrap distribution mode (and also skip + // bootstrapping external modules) to make sure a package can be checked + // out without its dependencies being present. + // + // Note also that on failure the package stays in the existing (working) // state. // // At first it may seem we have a problem: an existing package with the @@ -256,43 +301,18 @@ namespace bpkg // of our dependencies. // - // At verbosity level 1 we want our (nicer) progress header but the - // build system's actual progress. + // If the verbosity level is less than 2, then we want our (nicer) + // progress header but the build system's actual progress. // - if (verb == 1 && !o.no_progress ()) - text << "distributing " << n << '/' << v; + if ((verb == 1 && !o.no_progress ()) || (verb == 0 && o.progress ())) + text << "distributing " << n << '/' << v << pdb; run_b (o, verb_b::progress, - strings ({"config.dist.root='" + c.representation () + "'"}), + "--no-external-modules", + "!config.dist.bootstrap=true", + "config.dist.root='" + ord.representation () + '\'', bspec); - - // Revert the fix-ups. - // - if (fixedup) - fixup (o, rl, td, true /* revert */); - - // Manipulations over the repository are now complete, so we can return - // it to its permanent location. - // - mv (td, rd); - fs_changed = false; - - rmt.cancel (); - - mc = sha256 (o, d / manifest_file); - } - catch (const failed&) - { - if (fs_changed) - { - // We assume that the diagnostics has already been issued. - // - warn << "repository state is now broken" << - info << "run 'bpkg rep-fetch' to repair"; - } - - throw; } if (p != nullptr) @@ -301,7 +321,7 @@ namespace bpkg // replacing. Once this is done, there is no going back. If things go // badly, we can't simply abort the transaction. // - pkg_purge_fs (c, t, p, simulate); + pkg_purge_fs (pdb, t, p, simulate); // Note that if the package name spelling changed then we need to update // it, to make sure that the subsequent commands don't fail and the @@ -310,21 +330,34 @@ namespace bpkg // if (p->name.string () != n.string ()) { - db.erase (p); + pdb.erase (p); p = nullptr; } } + // Make the package path absolute and normalized. If the package is inside + // the configuration, use the relative path. This way we can move the + // configuration around. + // + normalize (d, "package"); + + if (d.sub (pdb.config)) + d = d.leaf (pdb.config); + if (p != nullptr) { + // Note: we can be replacing an external package and thus we reset the + // manifest/subprojects and buildfiles checksums. + // p->version = move (v); p->state = package_state::unpacked; p->repository_fragment = rl; - p->src_root = d.leaf (); - p->purge_src = true; - p->manifest_checksum = move (mc); + p->src_root = move (d); + p->purge_src = purge; + p->manifest_checksum = nullopt; + p->buildfiles_checksum = nullopt; - db.update (p); + pdb.update (p); } else { @@ -340,13 +373,14 @@ namespace bpkg rl, nullopt, // No archive false, - d.leaf (), // Source root. - true, // Purge directory. - move (mc), + move (d), // Source root. + purge, // Purge directory. + nullopt, // No manifest/subprojects checksum. + nullopt, // No buildfiles checksum. nullopt, // No output directory yet. {}}); // No prerequisites captured yet. - db.persist (p); + pdb.persist (p); } t.commit (); @@ -355,6 +389,56 @@ namespace bpkg return p; } + shared_ptr<selected_package> + pkg_checkout (pkg_checkout_cache& cache, + const common_options& o, + database& pdb, + database& rdb, + transaction& t, + package_name n, + version v, + const dir_path& d, + bool replace, + bool purge, + bool simulate) + { + return pkg_checkout (cache, + o, + pdb, + rdb, + t, + move (n), + move (v), + optional<dir_path> (d), + replace, + purge, + simulate); + } + + shared_ptr<selected_package> + pkg_checkout (pkg_checkout_cache& cache, + const common_options& o, + database& pdb, + database& rdb, + transaction& t, + package_name n, + version v, + bool replace, + bool simulate) + { + return pkg_checkout (cache, + o, + pdb, + rdb, + t, + move (n), + move (v), + nullopt /* output_root */, + replace, + true /* purge */, + simulate); + } + int pkg_checkout (const pkg_checkout_options& o, cli::scanner& args) { @@ -363,7 +447,7 @@ namespace bpkg dir_path c (o.directory ()); l4 ([&]{trace << "configuration: " << c;}); - database db (open (c, trace)); + database db (c, trace, true /* pre_attach */); transaction t (db); session s; @@ -381,19 +465,101 @@ namespace bpkg fail << "package version expected" << info << "run 'bpkg help pkg-checkout' for more information"; + pkg_checkout_cache checkout_cache (o); + // Commits the transaction. // - p = pkg_checkout (o, - c, - t, - move (n), - move (v), - o.replace (), - false /* simulate */); + if (o.output_root_specified ()) + p = pkg_checkout (checkout_cache, + o, + db /* pdb */, + db /* rdb */, + t, + move (n), + move (v), + o.output_root (), + o.replace (), + o.output_purge (), + false /* simulate */); + else + p = pkg_checkout (checkout_cache, + o, + db /* pdb */, + db /* rdb */, + t, + move (n), + move (v), + o.replace (), + false /* simulate */); + + checkout_cache.clear (); // Detect errors. if (verb && !o.no_result ()) text << "checked out " << *p; return 0; } + + // pkg_checkout_cache + // + pkg_checkout_cache:: + ~pkg_checkout_cache () + { + if (!map_.empty () && !clear (true /* ignore_errors */)) + { + // We assume that the diagnostics has already been issued. + // + warn << "repository state is now broken" << + info << "run 'bpkg rep-fetch' to repair"; + } + } + + bool pkg_checkout_cache:: + clear (bool ie) + { + while (!map_.empty ()) + { + if (!erase (map_.begin (), ie)) + return false; + } + + return true; + } + + bool pkg_checkout_cache:: + erase (state_map::iterator i, bool ie) + { + state& s (i->second); + + // Bail out if the entry is incomplete. + // + if (!s.fixedup) + { + assert (ie); // Only makes sense in the ignore errors mode. + return false; + } + + // Revert the fix-ups. + // + // But first make the entry incomplete, so on error we don't try to + // restore the partially restored repository later. + // + bool f (*s.fixedup); + + s.fixedup = nullopt; + + if (f && !fixup (options_, s.rl, s.rmt.path, true /* revert */, ie)) + return false; + + // Manipulations over the repository are now complete, so we can return it + // to the permanent location. + // + if (!mv (s.rmt.path, i->first, ie)) + return false; + + s.rmt.cancel (); + + map_.erase (i); + return true; + } } |