diff options
Diffstat (limited to 'bpkg/pkg-checkout.cxx')
-rw-r--r-- | bpkg/pkg-checkout.cxx | 317 |
1 files changed, 210 insertions, 107 deletions
diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx index 3b99496..81efdc2 100644 --- a/bpkg/pkg-checkout.cxx +++ b/bpkg/pkg-checkout.cxx @@ -10,23 +10,26 @@ #include <bpkg/package-odb.hxx> #include <bpkg/database.hxx> #include <bpkg/checksum.hxx> +#include <bpkg/rep-mask.hxx> #include <bpkg/diagnostics.hxx> #include <bpkg/manifest-utility.hxx> #include <bpkg/pkg-purge.hxx> #include <bpkg/pkg-verify.hxx> -#include <bpkg/pkg-configure.hxx> using namespace std; using namespace butl; namespace bpkg { + // pkg_checkout() + // static void checkout (const common_options& o, const repository_location& rl, const dir_path& dir, - const shared_ptr<available_package>& ap) + const shared_ptr<available_package>& ap, + database& db) { switch (rl.type ()) { @@ -41,9 +44,9 @@ namespace bpkg // Print the progress indicator to attribute the possible fetching // progress. // - if (verb && !o.no_progress ()) + if ((verb && !o.no_progress ()) || o.progress ()) text << "checking out " - << package_string (ap->id.name, ap->version); + << package_string (ap->id.name, ap->version) << db; git_checkout_submodules (o, rl, dir); } @@ -58,19 +61,23 @@ namespace bpkg // For some platforms/repository types the working tree needs to be // temporary "fixed up" for the build2 operations to work properly on it. // - static bool + static optional<bool> fixup (const common_options& o, const repository_location& rl, const dir_path& dir, - bool revert = false) + bool revert = false, + bool ie = false) { - bool r (false); + optional<bool> r; switch (rl.type ()) { case repository_type::git: { - r = git_fixup_worktree (o, dir, revert); + if (!revert && !ie) + git_verify_symlinks (o, dir); + + r = git_fixup_worktree (o, dir, revert, ie); break; } case repository_type::pkg: @@ -83,8 +90,10 @@ namespace bpkg // Return the selected package object which may replace the existing one. // static shared_ptr<selected_package> - pkg_checkout (const common_options& o, - dir_path c, + pkg_checkout (pkg_checkout_cache& cache, + const common_options& o, + database& pdb, + database& rdb, transaction& t, package_name n, version v, @@ -95,12 +104,13 @@ namespace bpkg { tracer trace ("pkg_checkout"); - database& db (t.database ()); - tracer_guard tg (db, trace); + tracer_guard tg (pdb, trace); // NOTE: sets tracer for the whole cluster. + + const dir_path& c (pdb.config_orig); // See if this package already exists in this configuration. // - shared_ptr<selected_package> p (db.find<selected_package> (n)); + shared_ptr<selected_package> p (pdb.find<selected_package> (n)); if (p != nullptr) { @@ -121,13 +131,13 @@ namespace bpkg } } - check_any_available (c, t); + check_any_available (rdb, t); - // Note that here we compare including the revision (see pkg-fetch() + // Note that here we compare including the revision (see pkg_fetch() // implementation for more details). // shared_ptr<available_package> ap ( - db.find<available_package> (available_package_id (n, v))); + rdb.find<available_package> (available_package_id (n, v))); if (ap == nullptr) fail << "package " << n << " " << v << " is not available"; @@ -139,14 +149,17 @@ namespace bpkg for (const package_location& l: ap->locations) { - const repository_location& rl (l.repository_fragment.load ()->location); - - if (rl.version_control_based () && (pl == nullptr || rl.local ())) + if (!rep_masked_fragment (l.repository_fragment)) { - pl = &l; + const repository_location& rl (l.repository_fragment.load ()->location); + + if (rl.version_control_based () && (pl == nullptr || rl.local ())) + { + pl = &l; - if (rl.local ()) - break; + if (rl.local ()) + break; + } } } @@ -161,8 +174,6 @@ namespace bpkg const repository_location& rl (pl->repository_fragment->location); auto_rmdir rmd; - optional<string> mc; - const dir_path& ord (output_root ? *output_root : c); dir_path d (ord / dir_path (n.string () + '-' + v.string ())); @@ -173,10 +184,7 @@ namespace bpkg // (or interruption) the user will need to run bpkg-rep-fetch to restore // the missing repository. // - bool fs_changed (false); - if (!simulate) - try { if (exists (d)) fail << "package directory " << d << " already exists"; @@ -185,45 +193,84 @@ namespace bpkg // if the previous checkout have failed or been interrupted. // dir_path sd (repository_state (rl)); - dir_path rd (c / repos_dir / sd); + dir_path rd (rdb.config_orig / repos_dir / sd); - if (!exists (rd)) - fail << "missing repository directory for package " << n << " " << v - << " in configuration " << c << - info << "run 'bpkg rep-fetch' to repair"; + // Use the temporary directory from the repository information source + // configuration, so that we can always move the repository into and out + // of it (note that if they appear on different filesystems that won't + // be possible). + // + auto ti (tmp_dirs.find (rdb.config_orig)); + assert (ti != tmp_dirs.end ()); + const dir_path& tdir (ti->second); - // The repository temporary directory. + // Try to reuse the cached repository (moved to the temporary directory + // with some fragment checked out and fixed up). // - auto_rmdir rmt (temp_dir / sd); - const dir_path& td (rmt.path); + pkg_checkout_cache::state_map& cm (cache.map_); + auto i (cm.find (rd)); + + if (i == cm.end () || i->second.rl.fragment () != rl.fragment ()) + { + // Restore the repository if some different fragment is checked out. + // + if (i != cm.end ()) + cache.erase (i); + + // Checkout and cache the fragment. + // + if (!exists (rd)) + fail << "missing repository directory for package " << n << " " << v + << " in its repository information configuration " + << rdb.config_orig << + info << "run 'bpkg rep-fetch' to repair"; + + // The repository temporary directory. + // + auto_rmdir rmt (tdir / sd, !keep_tmp); + + // Move the repository to the temporary directory. + // + { + const dir_path& td (rmt.path); - if (exists (td)) - rm_r (td); + if (exists (td)) + rm_r (td); + + mv (rd, td); + } + + // Pre-insert the incomplete repository entry into the cache and + // "finalize" it by setting the fixed up value later, after the + // repository fragment checkout succeeds. Until then the repository + // may not be restored in its permanent place. + // + using state = pkg_checkout_cache::state; + + i = cm.emplace (rd, state {move (rmt), rl, nullopt}).first; + + // Checkout the repository fragment and fix up the working tree. + // + state& s (i->second); + const dir_path& td (s.rmt.path); + + checkout (o, rl, td, ap, pdb); + s.fixedup = fixup (o, rl, td); + } // The temporary out of source directory that is required for the dist // meta-operation. // - auto_rmdir rmo (temp_dir / dir_path (n.string ())); + auto_rmdir rmo (tdir / dir_path (n.string ()), !keep_tmp); const dir_path& od (rmo.path); if (exists (od)) rm_r (od); - // Finally, move the repository to the temporary directory and proceed - // with the checkout. - // - mv (rd, td); - fs_changed = true; - - // Checkout the repository fragment and fix up the working tree. - // - checkout (o, rl, td, ap); - bool fixedup (fixup (o, rl, td)); - // Calculate the package path that points into the checked out fragment // directory. // - dir_path pd (td / path_cast<dir_path> (pl->location)); + dir_path pd (i->second.rmt.path / path_cast<dir_path> (pl->location)); // Form the buildspec. // @@ -254,45 +301,18 @@ namespace bpkg // of our dependencies. // - // At verbosity level 1 we want our (nicer) progress header but the - // build system's actual progress. + // If the verbosity level is less than 2, then we want our (nicer) + // progress header but the build system's actual progress. // - if (verb == 1 && !o.no_progress ()) - text << "distributing " << n << '/' << v; + if ((verb == 1 && !o.no_progress ()) || (verb == 0 && o.progress ())) + text << "distributing " << n << '/' << v << pdb; run_b (o, verb_b::progress, "--no-external-modules", "!config.dist.bootstrap=true", - "config.dist.root='" + ord.representation () + "'", + "config.dist.root='" + ord.representation () + '\'', bspec); - - // Revert the fix-ups. - // - if (fixedup) - fixup (o, rl, td, true /* revert */); - - // Manipulations over the repository are now complete, so we can return - // it to its permanent location. - // - mv (td, rd); - fs_changed = false; - - rmt.cancel (); - - mc = sha256 (o, d / manifest_file); - } - catch (const failed&) - { - if (fs_changed) - { - // We assume that the diagnostics has already been issued. - // - warn << "repository state is now broken" << - info << "run 'bpkg rep-fetch' to repair"; - } - - throw; } if (p != nullptr) @@ -301,7 +321,7 @@ namespace bpkg // replacing. Once this is done, there is no going back. If things go // badly, we can't simply abort the transaction. // - pkg_purge_fs (c, t, p, simulate); + pkg_purge_fs (pdb, t, p, simulate); // Note that if the package name spelling changed then we need to update // it, to make sure that the subsequent commands don't fail and the @@ -310,31 +330,34 @@ namespace bpkg // if (p->name.string () != n.string ()) { - db.erase (p); + pdb.erase (p); p = nullptr; } } - // Make the package and configuration paths absolute and normalized. - // If the package is inside the configuration, use the relative path. - // This way we can move the configuration around. + // Make the package path absolute and normalized. If the package is inside + // the configuration, use the relative path. This way we can move the + // configuration around. // - normalize (c, "configuration"); normalize (d, "package"); - if (d.sub (c)) - d = d.leaf (c); + if (d.sub (pdb.config)) + d = d.leaf (pdb.config); if (p != nullptr) { + // Note: we can be replacing an external package and thus we reset the + // manifest/subprojects and buildfiles checksums. + // p->version = move (v); p->state = package_state::unpacked; p->repository_fragment = rl; p->src_root = move (d); p->purge_src = purge; - p->manifest_checksum = move (mc); + p->manifest_checksum = nullopt; + p->buildfiles_checksum = nullopt; - db.update (p); + pdb.update (p); } else { @@ -352,11 +375,12 @@ namespace bpkg false, move (d), // Source root. purge, // Purge directory. - move (mc), + nullopt, // No manifest/subprojects checksum. + nullopt, // No buildfiles checksum. nullopt, // No output directory yet. {}}); // No prerequisites captured yet. - db.persist (p); + pdb.persist (p); } t.commit (); @@ -366,8 +390,10 @@ namespace bpkg } shared_ptr<selected_package> - pkg_checkout (const common_options& o, - const dir_path& c, + pkg_checkout (pkg_checkout_cache& cache, + const common_options& o, + database& pdb, + database& rdb, transaction& t, package_name n, version v, @@ -376,8 +402,10 @@ namespace bpkg bool purge, bool simulate) { - return pkg_checkout (o, - c, + return pkg_checkout (cache, + o, + pdb, + rdb, t, move (n), move (v), @@ -388,16 +416,20 @@ namespace bpkg } shared_ptr<selected_package> - pkg_checkout (const common_options& o, - const dir_path& c, + pkg_checkout (pkg_checkout_cache& cache, + const common_options& o, + database& pdb, + database& rdb, transaction& t, package_name n, version v, bool replace, bool simulate) { - return pkg_checkout (o, - c, + return pkg_checkout (cache, + o, + pdb, + rdb, t, move (n), move (v), @@ -415,7 +447,7 @@ namespace bpkg dir_path c (o.directory ()); l4 ([&]{trace << "configuration: " << c;}); - database db (open (c, trace)); + database db (c, trace, true /* pre_attach */); transaction t (db); session s; @@ -433,11 +465,15 @@ namespace bpkg fail << "package version expected" << info << "run 'bpkg help pkg-checkout' for more information"; + pkg_checkout_cache checkout_cache (o); + // Commits the transaction. // if (o.output_root_specified ()) - p = pkg_checkout (o, - c, + p = pkg_checkout (checkout_cache, + o, + db /* pdb */, + db /* rdb */, t, move (n), move (v), @@ -446,17 +482,84 @@ namespace bpkg o.output_purge (), false /* simulate */); else - p = pkg_checkout (o, - c, + p = pkg_checkout (checkout_cache, + o, + db /* pdb */, + db /* rdb */, t, move (n), move (v), o.replace (), false /* simulate */); + checkout_cache.clear (); // Detect errors. + if (verb && !o.no_result ()) text << "checked out " << *p; return 0; } + + // pkg_checkout_cache + // + pkg_checkout_cache:: + ~pkg_checkout_cache () + { + if (!map_.empty () && !clear (true /* ignore_errors */)) + { + // We assume that the diagnostics has already been issued. + // + warn << "repository state is now broken" << + info << "run 'bpkg rep-fetch' to repair"; + } + } + + bool pkg_checkout_cache:: + clear (bool ie) + { + while (!map_.empty ()) + { + if (!erase (map_.begin (), ie)) + return false; + } + + return true; + } + + bool pkg_checkout_cache:: + erase (state_map::iterator i, bool ie) + { + state& s (i->second); + + // Bail out if the entry is incomplete. + // + if (!s.fixedup) + { + assert (ie); // Only makes sense in the ignore errors mode. + return false; + } + + // Revert the fix-ups. + // + // But first make the entry incomplete, so on error we don't try to + // restore the partially restored repository later. + // + bool f (*s.fixedup); + + s.fixedup = nullopt; + + if (f && !fixup (options_, s.rl, s.rmt.path, true /* revert */, ie)) + return false; + + // Manipulations over the repository are now complete, so we can return it + // to the permanent location. + // + if (!mv (s.rmt.path, i->first, ie)) + return false; + + s.rmt.cancel (); + + map_.erase (i); + return true; + } } |