diff options
20 files changed, 979 insertions, 233 deletions
diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx index 69eb5a0..ff53318 100644 --- a/bpkg/fetch-git.cxx +++ b/bpkg/fetch-git.cxx @@ -5,12 +5,12 @@ #include <bpkg/fetch.hxx> #include <map> -#include <algorithm> // find(), find_if(), replace(), sort() +#include <algorithm> // find_if(), replace(), sort() #include <libbutl/git.mxx> #include <libbutl/utility.mxx> // digit(), xdigit() #include <libbutl/process.mxx> -#include <libbutl/filesystem.mxx> // path_match() +#include <libbutl/filesystem.mxx> // path_match(), path_entry() #include <libbutl/semantic-version.mxx> #include <libbutl/standard-version.mxx> // parse_standard_version() @@ -1490,65 +1490,50 @@ namespace bpkg return sort (move (r)); } - // Checkout the repository submodules (see git_checkout_submodules() - // description for details). + // Print diagnostics, optionally attributing it to a submodule with the + // specified (non-empty) directory prefix, and fail. // - static void - checkout_submodules (const common_options& co, - const dir_path& dir, - const dir_path& git_dir, - const dir_path& prefix) + [[noreturn]] static void + submodule_failure (const string& desc, + const dir_path& prefix, + const exception* e = nullptr) { - tracer trace ("checkout_submodules"); - - path mf (dir / path (".gitmodules")); - - if (!exists (mf)) - return; - - auto failure = [&prefix] (const char* desc) - { - diag_record dr (fail); - dr << desc; + diag_record dr (fail); + dr << desc; - if (!prefix.empty ()) - // Strips the trailing slash. - // - dr << " for submodule '" << prefix.string () << "'"; + if (!prefix.empty ()) + // Strips the trailing slash. + // + dr << " for submodule '" << prefix.string () << "'"; - dr << endg; - }; + if (e != nullptr) + dr << ": " << *e; - // Initialize submodules. - // - if (!run_git ( - co, - co.git_option (), - "-C", dir, + dr << endg; + } - // Note that older git versions don't recognize the --super-prefix - // option but seem to behave correctly without any additional - // efforts when it is omitted. - // - !prefix.empty () && git_ver >= semantic_version {2, 14, 0} - ? strings ({"--super-prefix", prefix.posix_representation ()}) - : strings (), + // Find submodules for a top repository or submodule directory. The prefix + // is only used for diagnostics (see submodule_failure() for details). + // + struct submodule + { + dir_path path; // Relative to the containing repository. + string commit; + }; + using submodules = vector<submodule>; - "submodule--helper", "init", - verb < 2 ? "-q" : nullptr)) - failure ("unable to initialize submodules"); + static submodules + find_submodules (const common_options& co, + const dir_path& dir, + const dir_path& prefix) + { + tracer trace ("find_submodules"); - repository_url orig_url (origin_url (co, dir)); + auto failure = [&prefix] (const string& d, const exception* e = nullptr) + { + submodule_failure (d, prefix, e); + }; - // Iterate over the registered submodules initializing/fetching them and - // recursively checking them out. - // - // Note that we don't expect submodules nesting be too deep and so recurse - // while reading the git process output. - // - // Also note that we don't catch the failed exception here, relying on the - // fact that the process destructor will wait for the process completion. - // fdpipe pipe (open_pipe ()); process pr (start_git (co, @@ -1563,6 +1548,7 @@ namespace bpkg try { + submodules r; ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit); for (string l; !eof (getline (is, l)); ) @@ -1578,156 +1564,220 @@ namespace bpkg l4 ([&]{trace << "submodule: " << l;}); if (!(l.size () > 50 && l[48] == '0' && l[49] == '\t')) - failure ("invalid submodule description"); - - string commit (l.substr (7, 40)); + throw runtime_error ("invalid submodule description '" + l + "'"); - // Submodule directory path, relative to the containing project. + // Submodule directory path is relative to the containing repository. // - dir_path sdir (l.substr (50)); + r.push_back (submodule {dir_path (string (l, 50)) /* path */, + string (l, 7, 40) /* commit */}); + } - // Submodule directory path, relative to the top project. - // - dir_path psdir (prefix / sdir); - string psd (psdir.posix_string ()); // For use in the diagnostics. + is.close (); - string nm (git_line (co, "submodule name", - co.git_option (), - "-C", dir, - "submodule--helper", "name", - sdir)); + if (pr.wait ()) + return r; - string uo ("submodule." + nm + ".url"); - string uv (config_get (co, dir, uo, "submodule URL")); + // Fall through. + } + catch (const invalid_path& e) + { + if (pr.wait ()) + failure ("invalid submodule path '" + e.path + "'"); - l4 ([&]{trace << "name: " << nm << ", URL: " << uv;}); + // Fall through. + } + catch (const io_error& e) + { + if (pr.wait ()) + failure ("unable to read submodules list", &e); - dir_path fsdir (dir / sdir); - bool initialized (git_repository (fsdir)); + // Fall through. + } + // Note that the io_error class inherits from the runtime_error class, so + // this catch-clause must go last. + // + catch (const runtime_error& e) + { + if (pr.wait ()) + failure (e.what ()); - // If the submodule is already initialized and its commit didn't - // change then we skip it. - // - if (initialized && git_line (co, "submodule commit", - co.git_option (), - "-C", fsdir, - "rev-parse", - "--verify", - "HEAD") == commit) - continue; - - // Note that the "submodule--helper init" command (see above) doesn't - // sync the submodule URL in .git/config file with the one in - // .gitmodules file, that is a primary URL source. Thus, we always - // calculate the URL using .gitmodules and update it in .git/config, if - // necessary. - // - repository_url url; + // Fall through. + } - try - { - url = from_git_url ( - config_get (co, mf, uo, "submodule original URL")); + // We should only get here if the child exited with an error status. + // + assert (!pr.wait ()); - // Complete the relative submodule URL against the containing - // repository origin URL. - // - if (url.scheme == repository_protocol::file && url.path->relative ()) - { - repository_url u (orig_url); - *u.path /= *url.path; + submodule_failure ("unable to list submodules", prefix); + } - // Note that we need to collapse 'example.com/a/..' to - // 'example.com/', rather than to 'example.com/.'. - // - u.path->normalize ( - false /* actual */, - orig_url.scheme != repository_protocol::file /* cur_empty */); + // Checkout the repository submodules (see git_checkout_submodules() + // description for details). + // + static void + checkout_submodules (const common_options& co, + const dir_path& dir, + const dir_path& git_dir, + const dir_path& prefix) + { + tracer trace ("checkout_submodules"); - url = move (u); - } + auto failure = [&prefix] (const string& d, const exception* e = nullptr) + { + submodule_failure (d, prefix, e); + }; - // Fix-up submodule URL in .git/config file, if required. + path mf (dir / path (".gitmodules")); + + if (!exists (mf)) + return; + + // Initialize submodules. + // + if (!run_git ( + co, + co.git_option (), + "-C", dir, + + // Note that older git versions don't recognize the --super-prefix + // option but seem to behave correctly without any additional + // efforts when it is omitted. // - if (url != from_git_url (move (uv))) - { - config_set (co, dir, uo, to_git_url (url)); + !prefix.empty () && git_ver >= semantic_version {2, 14, 0} + ? strings ({"--super-prefix", prefix.posix_representation ()}) + : strings (), - // We also need to fix-up submodule's origin URL, if its - // repository is already initialized. - // - if (initialized) - origin_url (co, fsdir, url); - } - } - catch (const invalid_path& e) - { - fail << "invalid repository path for submodule '" << psd << "': " - << e << endg; - } - catch (const invalid_argument& e) - { - fail << "invalid repository URL for submodule '" << psd << "': " - << e << endg; - } + "submodule--helper", "init", + verb < 2 ? "-q" : nullptr)) + failure ("unable to initialize submodules"); - // Initialize the submodule repository. - // - // Note that we initialize the submodule repository git directory out - // of the working tree, the same way as "submodule--helper clone" - // does. This prevents us from loosing the fetched data when switching - // the containing repository between revisions, that potentially - // contain different sets of submodules. - // - dir_path gdir (git_dir / dir_path ("modules") / sdir); + repository_url orig_url (origin_url (co, dir)); - if (!initialized) - { - mk_p (gdir); - init (co, fsdir, url, gdir); - } + // Iterate over the registered submodules initializing/fetching them and + // recursively checking them out. + // + for (const submodule& sm: find_submodules (co, dir, prefix)) + { + // Submodule directory path, relative to the top repository. + // + dir_path psdir (prefix / sm.path); + string psd (psdir.posix_string ()); // For use in the diagnostics. - // Fetch and checkout the submodule. - // - git_ref_filters rfs { - git_ref_filter {nullopt, commit, false /* exclusion */}}; + string nm (git_line (co, "submodule name", + co.git_option (), + "-C", dir, + "submodule--helper", "name", + sm.path)); + + string uo ("submodule." + nm + ".url"); + string uv (config_get (co, dir, uo, "submodule URL")); + + l4 ([&]{trace << "name: " << nm << ", URL: " << uv;}); + + dir_path fsdir (dir / sm.path); + bool initialized (git_repository (fsdir)); - fetch (co, fsdir, psdir, rfs); + // If the submodule is already initialized and its commit didn't + // change then we skip it. + // + if (initialized && git_line (co, "submodule commit", + co.git_option (), + "-C", fsdir, + "rev-parse", + "--verify", + "HEAD") == sm.commit) + continue; + + // Note that the "submodule--helper init" command (see above) doesn't + // sync the submodule URL in .git/config file with the one in + // .gitmodules file, that is a primary URL source. Thus, we always + // calculate the URL using .gitmodules and update it in .git/config, if + // necessary. + // + repository_url url; - git_checkout (co, fsdir, commit); + try + { + url = from_git_url ( + config_get (co, mf, uo, "submodule original URL")); - // Let's make the message match the git-submodule script output - // (again, except for capitalization). + // Complete the relative submodule URL against the containing + // repository origin URL. // - if (verb && !co.no_progress ()) - text << "submodule path '" << psd << "': checked out '" << commit - << "'"; + if (url.scheme == repository_protocol::file && url.path->relative ()) + { + repository_url u (orig_url); + *u.path /= *url.path; + + // Note that we need to collapse 'example.com/a/..' to + // 'example.com/', rather than to 'example.com/.'. + // + u.path->normalize ( + false /* actual */, + orig_url.scheme != repository_protocol::file /* cur_empty */); + + url = move (u); + } - // Check out the submodule submodules, recursively. + // Fix-up submodule URL in .git/config file, if required. // - checkout_submodules (co, fsdir, gdir, psdir); + if (url != from_git_url (move (uv))) + { + config_set (co, dir, uo, to_git_url (url)); + + // We also need to fix-up submodule's origin URL, if its + // repository is already initialized. + // + if (initialized) + origin_url (co, fsdir, url); + } + } + catch (const invalid_path& e) + { + failure ( + "invalid submodule '" + nm + "' repository path '" + e.path + "'"); + } + catch (const invalid_argument& e) + { + failure ("invalid submodule '" + nm + "' repository URL", &e); } - is.close (); + // Initialize the submodule repository. + // + // Note that we initialize the submodule repository git directory out of + // the working tree, the same way as "submodule--helper clone" does. + // This prevents us from loosing the fetched data when switching the + // containing repository between revisions, that potentially contain + // different sets of submodules. + // + dir_path gdir (git_dir / dir_path ("modules") / sm.path); - if (pr.wait ()) - return; + if (!initialized) + { + mk_p (gdir); + init (co, fsdir, url, gdir); + } - // Fall through. - } - catch (const io_error&) - { - if (pr.wait ()) - failure ("unable to read submodules list"); + // Fetch and checkout the submodule. + // + git_ref_filters rfs { + git_ref_filter {nullopt, sm.commit, false /* exclusion */}}; - // Fall through. - } + fetch (co, fsdir, psdir, rfs); - // We should only get here if the child exited with an error status. - // - assert (!pr.wait ()); + git_checkout (co, fsdir, sm.commit); - failure ("unable to list submodules"); + // Let's make the message match the git-submodule script output (again, + // except for capitalization). + // + if (verb && !co.no_progress ()) + text << "submodule path '" << psd << "': checked out '" << sm.commit + << "'"; + + // Check out the submodule submodules, recursively. + // + checkout_submodules (co, fsdir, gdir, psdir); + } } void @@ -1805,8 +1855,8 @@ namespace bpkg const string& commit) { // For some (probably valid) reason the hard reset command doesn't remove - // a submodule directory that is not plugged into the project anymore. It - // also prints the non-suppressible warning like this: + // a submodule directory that is not plugged into the repository anymore. + // It also prints the non-suppressible warning like this: // // warning: unable to rmdir libbar: Directory not empty // @@ -1851,4 +1901,391 @@ namespace bpkg dir / dir_path (".git"), dir_path () /* prefix */); } + +#ifndef _WIN32 + + // Noop on POSIX. + // + bool + git_fixup_worktree (const common_options&, const dir_path&, bool) + { + return false; + } + +#else + + // Find symlinks in the repository (non-recursive submodule-wise). + // + static paths + find_symlinks (const common_options& co, + const dir_path& dir, + const dir_path& prefix) + { + tracer trace ("find_symlinks"); + + auto failure = [&prefix] (const string& d, const exception* e = nullptr) + { + submodule_failure (d, prefix, e); + }; + + fdpipe pipe (open_pipe ()); + + // Note: -z tells git to print file paths literally (without escaping) and + // terminate lines with NUL character. + // + process pr (start_git (co, + pipe, 2 /* stderr */, + co.git_option (), + "-C", dir, + "ls-files", + "--stage", + "-z")); + + // Shouldn't throw, unless something is severely damaged. + // + pipe.out.close (); + + try + { + paths r; + ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit); + + for (string l; !eof (getline (is, l, '\0')); ) + { + // The line describing a file is NUL-terminated and has the following + // form: + // + // <mode><SPACE><object><SPACE><stage><TAB><path> + // + // The mode is a 6-digit octal representation of the file type and + // permission bits mask. For example: + // + // 100644 165b42ec7a10fb6dd4a60b756fa1966c1065ef85 0 README + // + l4 ([&]{trace << "file: " << l;}); + + if (!(l.size () > 50 && l[48] == '0' && l[49] == '\t')) + throw runtime_error ("invalid file description '" + l + "'"); + + // For symlinks permission bits are always zero, so we can match the + // mode as a string. + // + if (l.compare (0, 6, "120000") == 0) + r.push_back (path (string (l, 50))); + } + + is.close (); + + if (pr.wait ()) + return r; + + // Fall through. + } + catch (const invalid_path& e) + { + if (pr.wait ()) + failure ("invalid repository symlink path '" + e.path + "'"); + + // Fall through. + } + catch (const io_error& e) + { + if (pr.wait ()) + failure ("unable to read repository file list", &e); + + // Fall through. + } + // Note that the io_error class inherits from the runtime_error class, + // so this catch-clause must go last. + // + catch (const runtime_error& e) + { + if (pr.wait ()) + failure (e.what ()); + + // Fall through. + } + + // We should only get here if the child exited with an error status. + // + assert (!pr.wait ()); + + // Show the noreturn attribute to the compiler to avoid the 'end of + // non-void function' warning. + // + submodule_failure ("unable to list repository files", prefix); + } + + // Fix up or revert the previously made fixes in a working tree of a top + // repository or submodule (see git_fixup_worktree() description for + // details). Return nullopt if no changes are required (because real symlink + // are being used). + // + static optional<bool> + fixup_worktree (const common_options& co, + const dir_path& dir, + bool revert, + const dir_path& prefix) + { + bool r (false); + + auto failure = [&prefix] (const string& d, const exception* e = nullptr) + { + submodule_failure (d, prefix, e); + }; + + if (!revert) + { + // Fix up symlinks depth-first, so link targets in submodules exist by + // the time we potentially reference them from the containing + // repository. + // + for (const submodule& sm: find_submodules (co, dir, prefix)) + { + optional<bool> fixed ( + fixup_worktree (co, dir / sm.path, revert, prefix / sm.path)); + + // If no further fix up is required, then the repository contains a + // real symlink. If that's the case, bailout or fail if git's + // filesystem-agnostic symlinks are also present in the repository. + // + if (!fixed) + { + // Note that the error message is not precise as path for the + // symlink in question is no longer available. However, the case + // feels unusual, so let's not complicate things for now. + // + if (r) + failure ("unexpected real symlink in submodule '" + + sm.path.string () + "'"); + + return nullopt; + } + + if (*fixed) + r = true; + } + + // Note that the target belonging to the current repository can be + // unavailable at the time we create a link to it because its path may + // contain a not yet created link components. Also, an existing target + // can be a not yet replaced filesystem-agnostic symlink. + // + // First, we cache link/target paths and remove the filesystem-agnostic + // links from the filesystem in order not to end up hard-linking them as + // targets. Then, we create links (hardlinks and junctions) iteratively, + // skipping those with not-yet-existing target, unless no links were + // created at the previous run, in which case we fail. + // + paths ls (find_symlinks (co, dir, prefix)); + vector<pair<path, path>> links; // List of the link/target path pairs. + + // Cache/remove filesystem-agnostic symlinks. + // + for (auto& l: ls) + { + path lp (dir / l); // Absolute or relative to the current directory. + + // Check the symlink type to see if we need to replace it or can bail + // out/fail (see above). + // + // @@ Note that things are broken here if running in the Windows + // "elevated console mode": + // + // - file symlinks are currently not supported (see + // libbutl/filesystem.mxx for details). + // + // - git creates symlinks to directories, rather than junctions. This + // makes things to fall apart as Windows API seems to be unable to + // see through such directory symlinks. More research is required. + // + try + { + pair<bool, entry_stat> e (path_entry (lp)); + + if (!e.first) + failure ("symlink '" + l.string () + "' does not exist"); + + if (e.second.type == entry_type::symlink) + { + if (r) + failure ("unexpected real symlink '" + l.string () + "'"); + + return nullopt; + } + } + catch (const system_error& e) + { + failure ("unable to stat symlink '" + l.string () + "'", &e); + } + + // Read the symlink target path. + // + path t; + + try + { + ifdstream fs (lp); + t = path (fs.read_text ()); + } + catch (const invalid_path& e) + { + failure ("invalid target path '" + e.path + "' for symlink '" + + l.string () + "'", + &e); + } + catch (const io_error& e) + { + failure ("unable to read target path for symlink '" + l.string () + + "'", + &e); + } + + // Mark the symlink as unchanged and remove it. + // + if (!run_git (co, + co.git_option (), + "-C", dir, + "update-index", + "--assume-unchanged", + l)) + failure ("unable to mark symlink '" + l.string () + + "' as unchanged"); + + links.emplace_back (move (l), move (t)); + + rm (lp); + r = true; + } + + // Create real links (hardlinks and junctions). + // + while (!links.empty ()) + { + size_t n (links.size ()); + + for (auto i (links.cbegin ()); i != links.cend (); ) + { + const path& l (i->first); + const path& t (i->second); + + // Absolute or relative to the current directory. + // + path lp (dir / l); + path tp (lp.directory () / t); + + bool dir_target; + + try + { + pair<bool, entry_stat> pe (path_entry (tp)); + + // Skip the symlink that references a not-yet-existing target. + // + if (!pe.first) + { + ++i; + continue; + } + + dir_target = pe.second.type == entry_type::directory; + } + catch (const system_error& e) + { + failure ("unable to stat target '" + t.string () + + "' for symlink '" + l.string () + "'", + &e); + } + + // Create the hardlink for a file target and junction for a + // directory target. + // + try + { + if (dir_target) + mksymlink (tp, lp, true /* dir */); + else + mkhardlink (tp, lp); + } + catch (const system_error& e) + { + failure (string ("unable to create ") + + (dir_target ? "junction" : "hardlink") + " '" + + l.string () + "' with target '" + t.string () + "'", + &e); + } + + i = links.erase (i); + } + + // Fail if no links were created on this run. + // + if (links.size () == n) + { + assert (!links.empty ()); + + failure ("target '" + links[0].first.string () + "' for symlink '" + + links[0].second.string () + "' does not exist"); + } + } + } + else + { + // Revert the fixes we've made previously in the opposite, depth-last, + // order. + // + // For the directory junctions the git-checkout command (see below) + // removes the target directory content, rather then the junction + // filesystem entry. To prevent this, we remove all hardlinks/junctions + // ourselves first. + // + for (const path& l: find_symlinks (co, dir, prefix)) + { + try + { + try_rmfile (dir / l); + } + catch (const system_error& e) + { + failure ("unable to remove hardlink or junction '" + l.string () + + "'", + &e); + } + } + + if (!run_git (co, + co.git_option (), + "-C", dir, + "checkout", + "--", + "./")) + failure ("unable to revert '" + dir.string () + '"'); + + // Revert fixes in submodules. + // + for (const submodule& sm: find_submodules (co, dir, prefix)) + fixup_worktree (co, dir / sm.path, revert, prefix / sm.path); + + // Let's not complicate things detecting if we have reverted anything + // and always return true, assuming there wouldn't be a reason to revert + // if no fixes were made previously. + // + r = true; + } + + return r; + } + + bool + git_fixup_worktree (const common_options& co, + const dir_path& dir, + bool revert) + { + optional<bool> r ( + fixup_worktree (co, dir, revert, dir_path () /* prefix */)); + + return r ? *r : false; + } + +#endif } diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx index 2adbfe3..0ed473f 100644 --- a/bpkg/fetch.hxx +++ b/bpkg/fetch.hxx @@ -102,6 +102,19 @@ namespace bpkg const repository_location&, const dir_path&); + // Fix up or revert the fixes (including in submodules, recursively) in a + // working tree previously checked out by git_checkout() or + // git_checkout_submodules(). Return true if any changes have been made to + // the filesystem. + // + // Noop on POSIX. On Windows it may replace git's filesystem-agnostic + // symlinks with hardlinks for the file targets and junctions for the + // directory targets. Note that it still makes sure the working tree is + // being treated by git as "clean" despite the changes. + // + bool + git_fixup_worktree (const common_options&, const dir_path&, bool revert); + // Low-level fetch API (fetch.cxx). // diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx index 6802c36..764373e 100644 --- a/bpkg/pkg-checkout.cxx +++ b/bpkg/pkg-checkout.cxx @@ -56,6 +56,31 @@ namespace bpkg } } + // For some platforms/repository types the working tree needs to be + // temporary "fixed up" for the build2 operations to work properly on it. + // + static bool + fixup (const common_options& o, + const repository_location& rl, + const dir_path& dir, + bool revert = false) + { + bool r (false); + + switch (rl.type ()) + { + case repository_type::git: + { + r = git_fixup_worktree (o, dir, revert); + break; + } + case repository_type::pkg: + case repository_type::dir: assert (false); break; + } + + return r; + } + shared_ptr<selected_package> pkg_checkout (const common_options& o, const dir_path& c, @@ -136,29 +161,39 @@ namespace bpkg optional<string> mc; dir_path d (c / dir_path (n.string () + '-' + v.string ())); + // An incomplete checkout may result in an unusable repository state + // (submodule fetch is interrupted, working tree fix up failed in the + // middle, etc.). That's why we will move the repository into the + // temporary directory prior to manipulating it. In the case of a failure + // (or interruption) the user will need to run bpkg-rep-fetch to restore + // the missing repository. + // + bool fs_changed (false); + if (!simulate) + try { - // Checkout the repository fragment. - // - dir_path sd (c / repos_dir / repository_state (rl)); - checkout (o, rl, sd, ap); + if (exists (d)) + fail << "package directory " << d << " already exists"; - // Calculate the package path that points into the checked out fragment - // directory. + // Check that the repository directory exists, which may not be the case + // if the previous checkout have failed or been interrupted. // - sd /= path_cast<dir_path> (pl->location); + dir_path sd (repository_state (rl)); + dir_path rd (c / repos_dir / sd); - // Verify the package prerequisites are all configured since the dist - // meta-operation generally requires all imports to be resolvable. - // - package_manifest m (pkg_verify (sd, - true /* ignore_unknown */, - [&ap] (version& v) {v = ap->version;})); + if (!exists (rd)) + fail << "missing repository directory for package " << n << " " << v + << " in configuration " << c << + info << "run 'bpkg rep-fetch' to repair"; - pkg_configure_prerequisites (o, t, m.dependencies, m.name); + // The repository temporary directory. + // + auto_rmdir rmt (temp_dir / sd); + const dir_path& td (rmt.path); - if (exists (d)) - fail << "package directory " << d << " already exists"; + if (exists (td)) + rm_r (td); // The temporary out of source directory that is required for the dist // meta-operation. @@ -169,10 +204,35 @@ namespace bpkg if (exists (od)) rm_r (od); + // Finally, move the repository to the temporary directory and proceed + // with the checkout. + // + mv (rd, td); + fs_changed = true; + + // Checkout the repository fragment and fix up the working tree. + // + checkout (o, rl, td, ap); + bool fixedup (fixup (o, rl, td)); + + // Calculate the package path that points into the checked out fragment + // directory. + // + dir_path pd (td / path_cast<dir_path> (pl->location)); + + // Verify the package prerequisites are all configured since the dist + // meta-operation generally requires all imports to be resolvable. + // + package_manifest m (pkg_verify (pd, + true /* ignore_unknown */, + [&ap] (version& v) {v = ap->version;})); + + pkg_configure_prerequisites (o, t, m.dependencies, m.name); + // Form the buildspec. // string bspec ("dist("); - bspec += sd.representation (); + bspec += pd.representation (); bspec += '@'; bspec += od.representation (); bspec += ')'; @@ -205,8 +265,33 @@ namespace bpkg strings ({"config.dist.root=" + c.representation ()}), bspec); + // Revert the fix-ups. + // + if (fixedup) + fixup (o, rl, td, true /* revert */); + + // Manipulations over the repository are now complete, so we can return + // it to its permanent location. + // + mv (td, rd); + fs_changed = false; + + rmt.cancel (); + mc = sha256 (o, d / manifest_file); } + catch (const failed&) + { + if (fs_changed) + { + // We assume that the diagnostics has already been issued. + // + warn << "repository state is now broken" << + info << "run 'bpkg rep-fetch' to repair"; + } + + throw; + } if (p != nullptr) { diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index b64bbe2..0c2ed89 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -313,7 +313,7 @@ namespace bpkg dir_path sd (repository_state (rl)); auto_rmdir rm (temp_dir / sd); - dir_path& td (rm.path); + const dir_path& td (rm.path); if (exists (td)) rm_r (td); diff --git a/tests/common/git/init b/tests/common/git/init index 4fac21e..5153175 100755 --- a/tests/common/git/init +++ b/tests/common/git/init @@ -42,6 +42,18 @@ fi # cd state0 +rm -f -r links.git/.git +rm -f links.git/.gitmodules +rm -f links.git/bl +rm -f links.git/lc +rm -f links.git/pg +rm -f links.git/bs +rm -f links.git/bf +rm -f links.git/tl +rm -f links.git/td +rm -f links.git/ts +rm -f -r links.git/doc/style + rm -f -r libfoo.git/.git rm -f libfoo.git/.gitmodules rm -f libfoo.git/README @@ -156,6 +168,86 @@ git -C libfox.git submodule add ../libbar.git libbar git -C libfox.git submodule update --init --recursive # Recursive for safety. git -C libfox.git commit -am 'Create' +# Create master branch for links.git, adding style.git as a submodule. +# +git -C links.git init + +cat <<EOF >links.git/manifest +: 1 +name: links +version: 0.0.1 +summary: links +license: MIT +url: http://example.org +email: pkg@example.org +EOF + +git -C links.git add '*' +git -C links.git submodule add ../style.git doc/style +git -C links.git submodule update --init --recursive # Updates doc/style/basic. +git -C links.git commit -am 'Create' +git -C links.git tag -a 'v0.0.1' -m 'Tag version 0.0.1' + +# Increase links version and add symlinks. +# +cat <<EOF >links.git/manifest +: 1 +name: links +version: 1.0.0-a.0.z +summary: links +license: MIT +url: http://example.org +email: pkg@example.org +EOF + +ln -s tests links.git/ts # Directory symlink. +ln -s ts/TODO links.git/td # File symlink via directory symlink. +ln -s td links.git/tl # Symlink symlink. +ln -s doc/style/buildfile links.git/bf # Submodule file symlink. +ln -s doc/style/basic links.git/bs # Submodule directory symlink. +ln -s bs/page.css links.git/pg # Symlink via submodule directory symlink. + +git -C links.git add '*' +git -C links.git commit -am 'Add symlinks' +git -C links.git tag -a 'v1.0.0-alpha' -m 'Tag version 1.0.0-alpha' + +# Increase links version and add dangling symlink. +# +cat <<EOF >links.git/manifest +: 1 +name: links +version: 1.0.1 +summary: links +license: MIT +url: http://example.org +email: pkg@example.org +EOF + +ln -s lc links.git/bl # Dangling symlink. + +git -C links.git add '*' +git -C links.git commit -am 'Add dangling symlinks' +git -C links.git tag -a 'v1.0.1' -m 'Tag version 1.0.1' + +# Increase links version and add cyclic symlink. +# +cat <<EOF >links.git/manifest +: 1 +name: links +version: 1.0.2 +summary: links +license: MIT +url: http://example.org +email: pkg@example.org +EOF + +ln -s bl links.git/lc # Cyclic symlink. + +git -C links.git add '*' +git -C links.git commit -am 'Add cyclic symlinks' +git -C links.git tag -a 'v1.0.2' -m 'Tag version 1.0.2' + + # Create the modified state of the repositories, replacing libbar.git submodule # of libfoo with the newly created libbaz.git repository. Also advance master # branches and tags for libfoo.git and it's submodule style.git. @@ -169,6 +261,10 @@ for d in ../state0/*.git; do cp -r $d . done +# Drop the links.git repository. +# +rm -f -r links.git/ + # Create libbaz.git repository. # rm -f -r libbaz.git/.git diff --git a/tests/common/git/pack b/tests/common/git/pack index f9d9772..fd0b49c 100755 --- a/tests/common/git/pack +++ b/tests/common/git/pack @@ -15,6 +15,7 @@ function error () { info "$*"; exit 1; } projects=(\ state0/libfoo state0/libfox state0/libbar state0/style state0/style-basic \ + state0/links \ state1/libfoo state1/libfox state1/libbaz state1/style state1/style-basic) for p in "${projects[@]}"; do diff --git a/tests/common/git/state0/libbar.tar b/tests/common/git/state0/libbar.tar Binary files differindex 2e1a3ad..027112c 100644 --- a/tests/common/git/state0/libbar.tar +++ b/tests/common/git/state0/libbar.tar diff --git a/tests/common/git/state0/libfoo.tar b/tests/common/git/state0/libfoo.tar Binary files differindex 6a0cc8a..96a9f5b 100644 --- a/tests/common/git/state0/libfoo.tar +++ b/tests/common/git/state0/libfoo.tar diff --git a/tests/common/git/state0/libfox.tar b/tests/common/git/state0/libfox.tar Binary files differindex a73f460..fc2c391 100644 --- a/tests/common/git/state0/libfox.tar +++ b/tests/common/git/state0/libfox.tar diff --git a/tests/common/git/state0/links.tar b/tests/common/git/state0/links.tar Binary files differnew file mode 100644 index 0000000..33c5dbf --- /dev/null +++ b/tests/common/git/state0/links.tar diff --git a/tests/common/git/state0/style-basic.tar b/tests/common/git/state0/style-basic.tar Binary files differindex 36d0dcd..8b57bd0 100644 --- a/tests/common/git/state0/style-basic.tar +++ b/tests/common/git/state0/style-basic.tar diff --git a/tests/common/git/state0/style.tar b/tests/common/git/state0/style.tar Binary files differindex 7b48a9d..56e29f0 100644 --- a/tests/common/git/state0/style.tar +++ b/tests/common/git/state0/style.tar diff --git a/tests/common/git/state1/libbaz.tar b/tests/common/git/state1/libbaz.tar Binary files differindex 1b35ec4..7acf277 100644 --- a/tests/common/git/state1/libbaz.tar +++ b/tests/common/git/state1/libbaz.tar diff --git a/tests/common/git/state1/libfoo.tar b/tests/common/git/state1/libfoo.tar Binary files differindex ab694f7..532e974 100644 --- a/tests/common/git/state1/libfoo.tar +++ b/tests/common/git/state1/libfoo.tar diff --git a/tests/common/git/state1/libfox.tar b/tests/common/git/state1/libfox.tar Binary files differindex a6ef40e..ec49a86 100644 --- a/tests/common/git/state1/libfox.tar +++ b/tests/common/git/state1/libfox.tar diff --git a/tests/common/git/state1/style-basic.tar b/tests/common/git/state1/style-basic.tar Binary files differindex 7b7a6c5..1946606 100644 --- a/tests/common/git/state1/style-basic.tar +++ b/tests/common/git/state1/style-basic.tar diff --git a/tests/common/git/state1/style.tar b/tests/common/git/state1/style.tar Binary files differindex e6243f4..769b6d5 100644 --- a/tests/common/git/state1/style.tar +++ b/tests/common/git/state1/style.tar diff --git a/tests/pkg-checkout.testscript b/tests/pkg-checkout.testscript index 148dcff..a284f6b 100644 --- a/tests/pkg-checkout.testscript +++ b/tests/pkg-checkout.testscript @@ -11,6 +11,8 @@ # |-- libbar.git -> style-basic.git (prerequisite) # `-- style-basic.git +posix = ($cxx.target.class != 'windows') + # Prepare repositories used by tests if running in the local mode. # +if ($remote != true) @@ -19,6 +21,11 @@ $git_extract $src/git/libbar.tar $git_extract $src/git/style-basic0.tar &$out_git/state0/*** $git_extract $src/git/style-basic1.tar &$out_git/state1/*** + + if $posix + $git_extract $src/git/style.tar + $git_extract $src/git/links.tar + end end : git-rep @@ -39,67 +46,172 @@ else pkg_purge += -d cfg 2>! pkg_status += -d cfg - test.cleanups += &cfg/.bpkg/repos/*/*** + test.cleanups += &?cfg/.bpkg/repos/*/*** : unconfigured-dependency : - $clone_root_cfg; - $rep_add "$rep/libbar.git#master"; - $rep_fetch; - - $* libmbar/1.0.0 2>>EOE != 0 - error: no configured package satisfies dependency on style-basic >= 1.0.0 - EOE + { + $clone_root_cfg; + $rep_add "$rep/libbar.git#master"; + $rep_fetch; + + $* libmbar/1.0.0 2>>EOE != 0 + error: no configured package satisfies dependency on style-basic >= 1.0.0 + warning: repository state is now broken + info: run 'bpkg rep-fetch' to repair + EOE + } : configured-dependency : - $clone_root_cfg; - $rep_add "$rep/libbar.git#master" && $rep_add "$rep/style-basic.git#master"; - $rep_fetch; + { + $clone_root_cfg; + $rep_add "$rep/libbar.git#master" && $rep_add "$rep/style-basic.git#master"; + $rep_fetch; - $pkg_status style-basic | sed -n -e 's/style-basic available \[.+\] ([^ ]+)/\1/p' | set v; + $pkg_status style-basic | sed -n -e 's/style-basic available \[.+\] ([^ ]+)/\1/p' | set v; - $* "style-basic/$v" 2>>"EOE"; - distributing style-basic/$v - checked out style-basic/$v - EOE + $* "style-basic/$v" 2>>"EOE"; + distributing style-basic/$v + checked out style-basic/$v + EOE - $pkg_configure style-basic; + $pkg_configure style-basic; - $* libmbar/1.0.0 2>>EOE; - distributing libmbar/1.0.0 - checked out libmbar/1.0.0 - EOE + $* libmbar/1.0.0 2>>EOE; + distributing libmbar/1.0.0 + checked out libmbar/1.0.0 + EOE - $pkg_disfigure style-basic; + $pkg_disfigure style-basic; - $pkg_purge libmbar; - $pkg_purge style-basic + $pkg_purge libmbar; + $pkg_purge style-basic + } : replacement : - # @@ Reduce to a single repository when multiple revisions can be specified - # in the repository URL fragment. - # - rep0 = "$rep_git/state0"; - rep1 = "$rep_git/state1"; + { + # @@ Reduce to a single repository when multiple revisions can be specified + # in the repository URL fragment. + # + rep0 = "$rep_git/state0"; + rep1 = "$rep_git/state1"; + + $clone_root_cfg; + $rep_add "$rep0/style-basic.git#master"; + $rep_add "$rep1/style-basic.git#stable"; + $rep_fetch; - $clone_root_cfg; - $rep_add "$rep0/style-basic.git#master"; - $rep_add "$rep1/style-basic.git#stable"; - $rep_fetch; + $pkg_status style-basic | \ + sed -n -e 's/style-basic available ([^ ]+) +([^ ]+)/\1 \2/p' | set vs; - $pkg_status style-basic | \ - sed -n -e 's/style-basic available ([^ ]+) +([^ ]+)/\1 \2/p' | set vs; + echo "$vs" | sed -e 's/([^ ]+).+/\1/' | set v0; + echo "$vs" | sed -e 's/([^ ]+) +([^ ]+)/\2/' | set v1; - echo "$vs" | sed -e 's/([^ ]+).+/\1/' | set v0; - echo "$vs" | sed -e 's/([^ ]+) +([^ ]+)/\2/' | set v1; + $* "style-basic/$v0" 2>!; + $pkg_status style-basic >~"/style-basic unpacked $v0/"; - $* "style-basic/$v0" 2>!; - $pkg_status style-basic >~"/style-basic unpacked $v0/"; + $* --replace "style-basic/$v1" 2>!; + $pkg_status style-basic >~"/style-basic unpacked $v1 .+/"; - $* --replace "style-basic/$v1" 2>!; - $pkg_status style-basic >~"/style-basic unpacked $v1 .+/"; + $pkg_purge style-basic + } - $pkg_purge style-basic + : links + : + if ($remote == true || $posix) + { + $clone_root_cfg; + + $rep_fetch "$rep/links.git#v1.0.0-alpha"; + + $pkg_status links | sed -n -e 's/links available (.+)/\1/p' | set v; + + $* "links/$v" 2>>~%EOE%; + %.* + %checking out links/1.0.0-a.0.[^.]+.[^.]+%d + %.* + %distributing links/1.0.0-a.0.[^.]+.[^.]+%d + %checked out links/1.0.0-a.0.[^.]+.[^.]+%d + EOE + + d = "cfg/links-$v"; + + # See common/git/init script for the symlinks descriptions. + # + test -d $d/bs; + test -d $d/ts; + + cat $d/pg >'h1 {font-size: 3em;}'; + cat $d/bs/page.css >'h1 {font-size: 3em;}'; + cat $d/bf >'./: file{manifest}'; + cat $d/td >'@@'; + cat $d/tl >'@@'; + cat $d/ts/TODO >'@@'; + + $pkg_purge links; + $rep_fetch "$rep/links.git#v0.0.1"; + + $* links/0.0.1 2>>~%EOE%; + checking out links/0.0.1 + distributing links/0.0.1 + checked out links/0.0.1 + EOE + + d = cfg/links-0.0.1; + + test -d $d/bs == 1; + test -d $d/ts == 1; + test -f $d/pg == 1; + test -f $d/bf == 1; + test -f $d/td == 1; + test -f $d/tl == 1; + + $pkg_purge links; + + # Dangling symlink in the repository. + # + $rep_fetch "$rep/links.git#v1.0.1"; + + if $posix + $* links/1.0.1 2>>~%EOE% + checking out links/1.0.1 + distributing links/1.0.1 + checked out links/1.0.1 + EOE + + $pkg_purge links + else + $* links/1.0.1 2>>~%EOE% != 0 + checking out links/1.0.1 + error: target 'bl' for symlink 'lc' does not exist + info: re-run with -v for more information + warning: repository state is now broken + info: run 'bpkg rep-fetch' to repair + EOE + end; + + # Cyclic symlinks in the repository. + # + if $posix + $rep_fetch "$rep/links.git#v1.0.2" 2>>~%EOE% != 0 + %.* + %error: unable to iterate over .+% + warning: repository state is now broken and will be cleaned up + info: run 'bpkg rep-fetch' to update + EOE + else + $rep_fetch "$rep/links.git#v1.0.2" + + $* links/1.0.2 2>>~%EOE% != 0 + checking out links/1.0.2 + %.* + %error: target '..' for symlink '..' does not exist% + info: re-run with -v for more information + warning: repository state is now broken + info: run 'bpkg rep-fetch' to repair + EOE + end + } } diff --git a/tests/pkg-checkout/git/links.tar b/tests/pkg-checkout/git/links.tar new file mode 120000 index 0000000..63545fe --- /dev/null +++ b/tests/pkg-checkout/git/links.tar @@ -0,0 +1 @@ +../../common/git/state0/links.tar
\ No newline at end of file diff --git a/tests/pkg-checkout/git/style.tar b/tests/pkg-checkout/git/style.tar new file mode 120000 index 0000000..948d152 --- /dev/null +++ b/tests/pkg-checkout/git/style.tar @@ -0,0 +1 @@ +../../common/git/state0/style.tar
\ No newline at end of file |