aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2019-02-11 19:22:43 (GMT)
committerKaren Arutyunov <karen@codesynthesis.com>2019-02-15 12:13:45 (GMT)
commit982916a05ab73f8ca113d45a6ddabcd09f481de5 (patch)
tree98bfc8c17649459ea4dba487e92611da9cc7c534
parentf1c95d45bd86180ef64da018b657461c44d0236a (diff)
Implement git repository working tree fix up for package checkout on Windows
-rw-r--r--bpkg/fetch-git.cxx783
-rw-r--r--bpkg/fetch.hxx13
-rw-r--r--bpkg/pkg-checkout.cxx119
-rw-r--r--bpkg/rep-fetch.cxx2
-rwxr-xr-xtests/common/git/init96
-rwxr-xr-xtests/common/git/pack1
-rw-r--r--tests/common/git/state0/libbar.tarbin81920 -> 81920 bytes
-rw-r--r--tests/common/git/state0/libfoo.tarbin327680 -> 327680 bytes
-rw-r--r--tests/common/git/state0/libfox.tarbin143360 -> 143360 bytes
-rw-r--r--tests/common/git/state0/links.tarbin0 -> 276480 bytes
-rw-r--r--tests/common/git/state0/style-basic.tarbin71680 -> 71680 bytes
-rw-r--r--tests/common/git/state0/style.tarbin143360 -> 143360 bytes
-rw-r--r--tests/common/git/state1/libbaz.tarbin61440 -> 61440 bytes
-rw-r--r--tests/common/git/state1/libfoo.tarbin409600 -> 409600 bytes
-rw-r--r--tests/common/git/state1/libfox.tarbin143360 -> 143360 bytes
-rw-r--r--tests/common/git/state1/style-basic.tarbin71680 -> 71680 bytes
-rw-r--r--tests/common/git/state1/style.tarbin143360 -> 143360 bytes
-rw-r--r--tests/pkg-checkout.testscript196
l---------tests/pkg-checkout/git/links.tar1
l---------tests/pkg-checkout/git/style.tar1
20 files changed, 979 insertions, 233 deletions
diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
index 69eb5a0..ff53318 100644
--- a/bpkg/fetch-git.cxx
+++ b/bpkg/fetch-git.cxx
@@ -5,12 +5,12 @@
#include <bpkg/fetch.hxx>
#include <map>
-#include <algorithm> // find(), find_if(), replace(), sort()
+#include <algorithm> // find_if(), replace(), sort()
#include <libbutl/git.mxx>
#include <libbutl/utility.mxx> // digit(), xdigit()
#include <libbutl/process.mxx>
-#include <libbutl/filesystem.mxx> // path_match()
+#include <libbutl/filesystem.mxx> // path_match(), path_entry()
#include <libbutl/semantic-version.mxx>
#include <libbutl/standard-version.mxx> // parse_standard_version()
@@ -1490,65 +1490,50 @@ namespace bpkg
return sort (move (r));
}
- // Checkout the repository submodules (see git_checkout_submodules()
- // description for details).
+ // Print diagnostics, optionally attributing it to a submodule with the
+ // specified (non-empty) directory prefix, and fail.
//
- static void
- checkout_submodules (const common_options& co,
- const dir_path& dir,
- const dir_path& git_dir,
- const dir_path& prefix)
+ [[noreturn]] static void
+ submodule_failure (const string& desc,
+ const dir_path& prefix,
+ const exception* e = nullptr)
{
- tracer trace ("checkout_submodules");
-
- path mf (dir / path (".gitmodules"));
-
- if (!exists (mf))
- return;
-
- auto failure = [&prefix] (const char* desc)
- {
- diag_record dr (fail);
- dr << desc;
+ diag_record dr (fail);
+ dr << desc;
- if (!prefix.empty ())
- // Strips the trailing slash.
- //
- dr << " for submodule '" << prefix.string () << "'";
+ if (!prefix.empty ())
+ // Strips the trailing slash.
+ //
+ dr << " for submodule '" << prefix.string () << "'";
- dr << endg;
- };
+ if (e != nullptr)
+ dr << ": " << *e;
- // Initialize submodules.
- //
- if (!run_git (
- co,
- co.git_option (),
- "-C", dir,
+ dr << endg;
+ }
- // Note that older git versions don't recognize the --super-prefix
- // option but seem to behave correctly without any additional
- // efforts when it is omitted.
- //
- !prefix.empty () && git_ver >= semantic_version {2, 14, 0}
- ? strings ({"--super-prefix", prefix.posix_representation ()})
- : strings (),
+ // Find submodules for a top repository or submodule directory. The prefix
+ // is only used for diagnostics (see submodule_failure() for details).
+ //
+ struct submodule
+ {
+ dir_path path; // Relative to the containing repository.
+ string commit;
+ };
+ using submodules = vector<submodule>;
- "submodule--helper", "init",
- verb < 2 ? "-q" : nullptr))
- failure ("unable to initialize submodules");
+ static submodules
+ find_submodules (const common_options& co,
+ const dir_path& dir,
+ const dir_path& prefix)
+ {
+ tracer trace ("find_submodules");
- repository_url orig_url (origin_url (co, dir));
+ auto failure = [&prefix] (const string& d, const exception* e = nullptr)
+ {
+ submodule_failure (d, prefix, e);
+ };
- // Iterate over the registered submodules initializing/fetching them and
- // recursively checking them out.
- //
- // Note that we don't expect submodules nesting be too deep and so recurse
- // while reading the git process output.
- //
- // Also note that we don't catch the failed exception here, relying on the
- // fact that the process destructor will wait for the process completion.
- //
fdpipe pipe (open_pipe ());
process pr (start_git (co,
@@ -1563,6 +1548,7 @@ namespace bpkg
try
{
+ submodules r;
ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit);
for (string l; !eof (getline (is, l)); )
@@ -1578,156 +1564,220 @@ namespace bpkg
l4 ([&]{trace << "submodule: " << l;});
if (!(l.size () > 50 && l[48] == '0' && l[49] == '\t'))
- failure ("invalid submodule description");
-
- string commit (l.substr (7, 40));
+ throw runtime_error ("invalid submodule description '" + l + "'");
- // Submodule directory path, relative to the containing project.
+ // Submodule directory path is relative to the containing repository.
//
- dir_path sdir (l.substr (50));
+ r.push_back (submodule {dir_path (string (l, 50)) /* path */,
+ string (l, 7, 40) /* commit */});
+ }
- // Submodule directory path, relative to the top project.
- //
- dir_path psdir (prefix / sdir);
- string psd (psdir.posix_string ()); // For use in the diagnostics.
+ is.close ();
- string nm (git_line (co, "submodule name",
- co.git_option (),
- "-C", dir,
- "submodule--helper", "name",
- sdir));
+ if (pr.wait ())
+ return r;
- string uo ("submodule." + nm + ".url");
- string uv (config_get (co, dir, uo, "submodule URL"));
+ // Fall through.
+ }
+ catch (const invalid_path& e)
+ {
+ if (pr.wait ())
+ failure ("invalid submodule path '" + e.path + "'");
- l4 ([&]{trace << "name: " << nm << ", URL: " << uv;});
+ // Fall through.
+ }
+ catch (const io_error& e)
+ {
+ if (pr.wait ())
+ failure ("unable to read submodules list", &e);
- dir_path fsdir (dir / sdir);
- bool initialized (git_repository (fsdir));
+ // Fall through.
+ }
+ // Note that the io_error class inherits from the runtime_error class, so
+ // this catch-clause must go last.
+ //
+ catch (const runtime_error& e)
+ {
+ if (pr.wait ())
+ failure (e.what ());
- // If the submodule is already initialized and its commit didn't
- // change then we skip it.
- //
- if (initialized && git_line (co, "submodule commit",
- co.git_option (),
- "-C", fsdir,
- "rev-parse",
- "--verify",
- "HEAD") == commit)
- continue;
-
- // Note that the "submodule--helper init" command (see above) doesn't
- // sync the submodule URL in .git/config file with the one in
- // .gitmodules file, that is a primary URL source. Thus, we always
- // calculate the URL using .gitmodules and update it in .git/config, if
- // necessary.
- //
- repository_url url;
+ // Fall through.
+ }
- try
- {
- url = from_git_url (
- config_get (co, mf, uo, "submodule original URL"));
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
- // Complete the relative submodule URL against the containing
- // repository origin URL.
- //
- if (url.scheme == repository_protocol::file && url.path->relative ())
- {
- repository_url u (orig_url);
- *u.path /= *url.path;
+ submodule_failure ("unable to list submodules", prefix);
+ }
- // Note that we need to collapse 'example.com/a/..' to
- // 'example.com/', rather than to 'example.com/.'.
- //
- u.path->normalize (
- false /* actual */,
- orig_url.scheme != repository_protocol::file /* cur_empty */);
+ // Checkout the repository submodules (see git_checkout_submodules()
+ // description for details).
+ //
+ static void
+ checkout_submodules (const common_options& co,
+ const dir_path& dir,
+ const dir_path& git_dir,
+ const dir_path& prefix)
+ {
+ tracer trace ("checkout_submodules");
- url = move (u);
- }
+ auto failure = [&prefix] (const string& d, const exception* e = nullptr)
+ {
+ submodule_failure (d, prefix, e);
+ };
- // Fix-up submodule URL in .git/config file, if required.
+ path mf (dir / path (".gitmodules"));
+
+ if (!exists (mf))
+ return;
+
+ // Initialize submodules.
+ //
+ if (!run_git (
+ co,
+ co.git_option (),
+ "-C", dir,
+
+ // Note that older git versions don't recognize the --super-prefix
+ // option but seem to behave correctly without any additional
+ // efforts when it is omitted.
//
- if (url != from_git_url (move (uv)))
- {
- config_set (co, dir, uo, to_git_url (url));
+ !prefix.empty () && git_ver >= semantic_version {2, 14, 0}
+ ? strings ({"--super-prefix", prefix.posix_representation ()})
+ : strings (),
- // We also need to fix-up submodule's origin URL, if its
- // repository is already initialized.
- //
- if (initialized)
- origin_url (co, fsdir, url);
- }
- }
- catch (const invalid_path& e)
- {
- fail << "invalid repository path for submodule '" << psd << "': "
- << e << endg;
- }
- catch (const invalid_argument& e)
- {
- fail << "invalid repository URL for submodule '" << psd << "': "
- << e << endg;
- }
+ "submodule--helper", "init",
+ verb < 2 ? "-q" : nullptr))
+ failure ("unable to initialize submodules");
- // Initialize the submodule repository.
- //
- // Note that we initialize the submodule repository git directory out
- // of the working tree, the same way as "submodule--helper clone"
- // does. This prevents us from loosing the fetched data when switching
- // the containing repository between revisions, that potentially
- // contain different sets of submodules.
- //
- dir_path gdir (git_dir / dir_path ("modules") / sdir);
+ repository_url orig_url (origin_url (co, dir));
- if (!initialized)
- {
- mk_p (gdir);
- init (co, fsdir, url, gdir);
- }
+ // Iterate over the registered submodules initializing/fetching them and
+ // recursively checking them out.
+ //
+ for (const submodule& sm: find_submodules (co, dir, prefix))
+ {
+ // Submodule directory path, relative to the top repository.
+ //
+ dir_path psdir (prefix / sm.path);
+ string psd (psdir.posix_string ()); // For use in the diagnostics.
- // Fetch and checkout the submodule.
- //
- git_ref_filters rfs {
- git_ref_filter {nullopt, commit, false /* exclusion */}};
+ string nm (git_line (co, "submodule name",
+ co.git_option (),
+ "-C", dir,
+ "submodule--helper", "name",
+ sm.path));
+
+ string uo ("submodule." + nm + ".url");
+ string uv (config_get (co, dir, uo, "submodule URL"));
+
+ l4 ([&]{trace << "name: " << nm << ", URL: " << uv;});
+
+ dir_path fsdir (dir / sm.path);
+ bool initialized (git_repository (fsdir));
- fetch (co, fsdir, psdir, rfs);
+ // If the submodule is already initialized and its commit didn't
+ // change then we skip it.
+ //
+ if (initialized && git_line (co, "submodule commit",
+ co.git_option (),
+ "-C", fsdir,
+ "rev-parse",
+ "--verify",
+ "HEAD") == sm.commit)
+ continue;
+
+ // Note that the "submodule--helper init" command (see above) doesn't
+ // sync the submodule URL in .git/config file with the one in
+ // .gitmodules file, that is a primary URL source. Thus, we always
+ // calculate the URL using .gitmodules and update it in .git/config, if
+ // necessary.
+ //
+ repository_url url;
- git_checkout (co, fsdir, commit);
+ try
+ {
+ url = from_git_url (
+ config_get (co, mf, uo, "submodule original URL"));
- // Let's make the message match the git-submodule script output
- // (again, except for capitalization).
+ // Complete the relative submodule URL against the containing
+ // repository origin URL.
//
- if (verb && !co.no_progress ())
- text << "submodule path '" << psd << "': checked out '" << commit
- << "'";
+ if (url.scheme == repository_protocol::file && url.path->relative ())
+ {
+ repository_url u (orig_url);
+ *u.path /= *url.path;
+
+ // Note that we need to collapse 'example.com/a/..' to
+ // 'example.com/', rather than to 'example.com/.'.
+ //
+ u.path->normalize (
+ false /* actual */,
+ orig_url.scheme != repository_protocol::file /* cur_empty */);
+
+ url = move (u);
+ }
- // Check out the submodule submodules, recursively.
+ // Fix-up submodule URL in .git/config file, if required.
//
- checkout_submodules (co, fsdir, gdir, psdir);
+ if (url != from_git_url (move (uv)))
+ {
+ config_set (co, dir, uo, to_git_url (url));
+
+ // We also need to fix-up submodule's origin URL, if its
+ // repository is already initialized.
+ //
+ if (initialized)
+ origin_url (co, fsdir, url);
+ }
+ }
+ catch (const invalid_path& e)
+ {
+ failure (
+ "invalid submodule '" + nm + "' repository path '" + e.path + "'");
+ }
+ catch (const invalid_argument& e)
+ {
+ failure ("invalid submodule '" + nm + "' repository URL", &e);
}
- is.close ();
+ // Initialize the submodule repository.
+ //
+ // Note that we initialize the submodule repository git directory out of
+ // the working tree, the same way as "submodule--helper clone" does.
+ // This prevents us from loosing the fetched data when switching the
+ // containing repository between revisions, that potentially contain
+ // different sets of submodules.
+ //
+ dir_path gdir (git_dir / dir_path ("modules") / sm.path);
- if (pr.wait ())
- return;
+ if (!initialized)
+ {
+ mk_p (gdir);
+ init (co, fsdir, url, gdir);
+ }
- // Fall through.
- }
- catch (const io_error&)
- {
- if (pr.wait ())
- failure ("unable to read submodules list");
+ // Fetch and checkout the submodule.
+ //
+ git_ref_filters rfs {
+ git_ref_filter {nullopt, sm.commit, false /* exclusion */}};
- // Fall through.
- }
+ fetch (co, fsdir, psdir, rfs);
- // We should only get here if the child exited with an error status.
- //
- assert (!pr.wait ());
+ git_checkout (co, fsdir, sm.commit);
- failure ("unable to list submodules");
+ // Let's make the message match the git-submodule script output (again,
+ // except for capitalization).
+ //
+ if (verb && !co.no_progress ())
+ text << "submodule path '" << psd << "': checked out '" << sm.commit
+ << "'";
+
+ // Check out the submodule submodules, recursively.
+ //
+ checkout_submodules (co, fsdir, gdir, psdir);
+ }
}
void
@@ -1805,8 +1855,8 @@ namespace bpkg
const string& commit)
{
// For some (probably valid) reason the hard reset command doesn't remove
- // a submodule directory that is not plugged into the project anymore. It
- // also prints the non-suppressible warning like this:
+ // a submodule directory that is not plugged into the repository anymore.
+ // It also prints the non-suppressible warning like this:
//
// warning: unable to rmdir libbar: Directory not empty
//
@@ -1851,4 +1901,391 @@ namespace bpkg
dir / dir_path (".git"),
dir_path () /* prefix */);
}
+
+#ifndef _WIN32
+
+ // Noop on POSIX.
+ //
+ bool
+ git_fixup_worktree (const common_options&, const dir_path&, bool)
+ {
+ return false;
+ }
+
+#else
+
+ // Find symlinks in the repository (non-recursive submodule-wise).
+ //
+ static paths
+ find_symlinks (const common_options& co,
+ const dir_path& dir,
+ const dir_path& prefix)
+ {
+ tracer trace ("find_symlinks");
+
+ auto failure = [&prefix] (const string& d, const exception* e = nullptr)
+ {
+ submodule_failure (d, prefix, e);
+ };
+
+ fdpipe pipe (open_pipe ());
+
+ // Note: -z tells git to print file paths literally (without escaping) and
+ // terminate lines with NUL character.
+ //
+ process pr (start_git (co,
+ pipe, 2 /* stderr */,
+ co.git_option (),
+ "-C", dir,
+ "ls-files",
+ "--stage",
+ "-z"));
+
+ // Shouldn't throw, unless something is severely damaged.
+ //
+ pipe.out.close ();
+
+ try
+ {
+ paths r;
+ ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit);
+
+ for (string l; !eof (getline (is, l, '\0')); )
+ {
+ // The line describing a file is NUL-terminated and has the following
+ // form:
+ //
+ // <mode><SPACE><object><SPACE><stage><TAB><path>
+ //
+ // The mode is a 6-digit octal representation of the file type and
+ // permission bits mask. For example:
+ //
+ // 100644 165b42ec7a10fb6dd4a60b756fa1966c1065ef85 0 README
+ //
+ l4 ([&]{trace << "file: " << l;});
+
+ if (!(l.size () > 50 && l[48] == '0' && l[49] == '\t'))
+ throw runtime_error ("invalid file description '" + l + "'");
+
+ // For symlinks permission bits are always zero, so we can match the
+ // mode as a string.
+ //
+ if (l.compare (0, 6, "120000") == 0)
+ r.push_back (path (string (l, 50)));
+ }
+
+ is.close ();
+
+ if (pr.wait ())
+ return r;
+
+ // Fall through.
+ }
+ catch (const invalid_path& e)
+ {
+ if (pr.wait ())
+ failure ("invalid repository symlink path '" + e.path + "'");
+
+ // Fall through.
+ }
+ catch (const io_error& e)
+ {
+ if (pr.wait ())
+ failure ("unable to read repository file list", &e);
+
+ // Fall through.
+ }
+ // Note that the io_error class inherits from the runtime_error class,
+ // so this catch-clause must go last.
+ //
+ catch (const runtime_error& e)
+ {
+ if (pr.wait ())
+ failure (e.what ());
+
+ // Fall through.
+ }
+
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
+
+ // Show the noreturn attribute to the compiler to avoid the 'end of
+ // non-void function' warning.
+ //
+ submodule_failure ("unable to list repository files", prefix);
+ }
+
+ // Fix up or revert the previously made fixes in a working tree of a top
+ // repository or submodule (see git_fixup_worktree() description for
+ // details). Return nullopt if no changes are required (because real symlink
+ // are being used).
+ //
+ static optional<bool>
+ fixup_worktree (const common_options& co,
+ const dir_path& dir,
+ bool revert,
+ const dir_path& prefix)
+ {
+ bool r (false);
+
+ auto failure = [&prefix] (const string& d, const exception* e = nullptr)
+ {
+ submodule_failure (d, prefix, e);
+ };
+
+ if (!revert)
+ {
+ // Fix up symlinks depth-first, so link targets in submodules exist by
+ // the time we potentially reference them from the containing
+ // repository.
+ //
+ for (const submodule& sm: find_submodules (co, dir, prefix))
+ {
+ optional<bool> fixed (
+ fixup_worktree (co, dir / sm.path, revert, prefix / sm.path));
+
+ // If no further fix up is required, then the repository contains a
+ // real symlink. If that's the case, bailout or fail if git's
+ // filesystem-agnostic symlinks are also present in the repository.
+ //
+ if (!fixed)
+ {
+ // Note that the error message is not precise as path for the
+ // symlink in question is no longer available. However, the case
+ // feels unusual, so let's not complicate things for now.
+ //
+ if (r)
+ failure ("unexpected real symlink in submodule '" +
+ sm.path.string () + "'");
+
+ return nullopt;
+ }
+
+ if (*fixed)
+ r = true;
+ }
+
+ // Note that the target belonging to the current repository can be
+ // unavailable at the time we create a link to it because its path may
+ // contain a not yet created link components. Also, an existing target
+ // can be a not yet replaced filesystem-agnostic symlink.
+ //
+ // First, we cache link/target paths and remove the filesystem-agnostic
+ // links from the filesystem in order not to end up hard-linking them as
+ // targets. Then, we create links (hardlinks and junctions) iteratively,
+ // skipping those with not-yet-existing target, unless no links were
+ // created at the previous run, in which case we fail.
+ //
+ paths ls (find_symlinks (co, dir, prefix));
+ vector<pair<path, path>> links; // List of the link/target path pairs.
+
+ // Cache/remove filesystem-agnostic symlinks.
+ //
+ for (auto& l: ls)
+ {
+ path lp (dir / l); // Absolute or relative to the current directory.
+
+ // Check the symlink type to see if we need to replace it or can bail
+ // out/fail (see above).
+ //
+ // @@ Note that things are broken here if running in the Windows
+ // "elevated console mode":
+ //
+ // - file symlinks are currently not supported (see
+ // libbutl/filesystem.mxx for details).
+ //
+ // - git creates symlinks to directories, rather than junctions. This
+ // makes things to fall apart as Windows API seems to be unable to
+ // see through such directory symlinks. More research is required.
+ //
+ try
+ {
+ pair<bool, entry_stat> e (path_entry (lp));
+
+ if (!e.first)
+ failure ("symlink '" + l.string () + "' does not exist");
+
+ if (e.second.type == entry_type::symlink)
+ {
+ if (r)
+ failure ("unexpected real symlink '" + l.string () + "'");
+
+ return nullopt;
+ }
+ }
+ catch (const system_error& e)
+ {
+ failure ("unable to stat symlink '" + l.string () + "'", &e);
+ }
+
+ // Read the symlink target path.
+ //
+ path t;
+
+ try
+ {
+ ifdstream fs (lp);
+ t = path (fs.read_text ());
+ }
+ catch (const invalid_path& e)
+ {
+ failure ("invalid target path '" + e.path + "' for symlink '" +
+ l.string () + "'",
+ &e);
+ }
+ catch (const io_error& e)
+ {
+ failure ("unable to read target path for symlink '" + l.string () +
+ "'",
+ &e);
+ }
+
+ // Mark the symlink as unchanged and remove it.
+ //
+ if (!run_git (co,
+ co.git_option (),
+ "-C", dir,
+ "update-index",
+ "--assume-unchanged",
+ l))
+ failure ("unable to mark symlink '" + l.string () +
+ "' as unchanged");
+
+ links.emplace_back (move (l), move (t));
+
+ rm (lp);
+ r = true;
+ }
+
+ // Create real links (hardlinks and junctions).
+ //
+ while (!links.empty ())
+ {
+ size_t n (links.size ());
+
+ for (auto i (links.cbegin ()); i != links.cend (); )
+ {
+ const path& l (i->first);
+ const path& t (i->second);
+
+ // Absolute or relative to the current directory.
+ //
+ path lp (dir / l);
+ path tp (lp.directory () / t);
+
+ bool dir_target;
+
+ try
+ {
+ pair<bool, entry_stat> pe (path_entry (tp));
+
+ // Skip the symlink that references a not-yet-existing target.
+ //
+ if (!pe.first)
+ {
+ ++i;
+ continue;
+ }
+
+ dir_target = pe.second.type == entry_type::directory;
+ }
+ catch (const system_error& e)
+ {
+ failure ("unable to stat target '" + t.string () +
+ "' for symlink '" + l.string () + "'",
+ &e);
+ }
+
+ // Create the hardlink for a file target and junction for a
+ // directory target.
+ //
+ try
+ {
+ if (dir_target)
+ mksymlink (tp, lp, true /* dir */);
+ else
+ mkhardlink (tp, lp);
+ }
+ catch (const system_error& e)
+ {
+ failure (string ("unable to create ") +
+ (dir_target ? "junction" : "hardlink") + " '" +
+ l.string () + "' with target '" + t.string () + "'",
+ &e);
+ }
+
+ i = links.erase (i);
+ }
+
+ // Fail if no links were created on this run.
+ //
+ if (links.size () == n)
+ {
+ assert (!links.empty ());
+
+ failure ("target '" + links[0].first.string () + "' for symlink '" +
+ links[0].second.string () + "' does not exist");
+ }
+ }
+ }
+ else
+ {
+ // Revert the fixes we've made previously in the opposite, depth-last,
+ // order.
+ //
+ // For the directory junctions the git-checkout command (see below)
+ // removes the target directory content, rather then the junction
+ // filesystem entry. To prevent this, we remove all hardlinks/junctions
+ // ourselves first.
+ //
+ for (const path& l: find_symlinks (co, dir, prefix))
+ {
+ try
+ {
+ try_rmfile (dir / l);
+ }
+ catch (const system_error& e)
+ {
+ failure ("unable to remove hardlink or junction '" + l.string () +
+ "'",
+ &e);
+ }
+ }
+
+ if (!run_git (co,
+ co.git_option (),
+ "-C", dir,
+ "checkout",
+ "--",
+ "./"))
+ failure ("unable to revert '" + dir.string () + '"');
+
+ // Revert fixes in submodules.
+ //
+ for (const submodule& sm: find_submodules (co, dir, prefix))
+ fixup_worktree (co, dir / sm.path, revert, prefix / sm.path);
+
+ // Let's not complicate things detecting if we have reverted anything
+ // and always return true, assuming there wouldn't be a reason to revert
+ // if no fixes were made previously.
+ //
+ r = true;
+ }
+
+ return r;
+ }
+
+ bool
+ git_fixup_worktree (const common_options& co,
+ const dir_path& dir,
+ bool revert)
+ {
+ optional<bool> r (
+ fixup_worktree (co, dir, revert, dir_path () /* prefix */));
+
+ return r ? *r : false;
+ }
+
+#endif
}
diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx
index 2adbfe3..0ed473f 100644
--- a/bpkg/fetch.hxx
+++ b/bpkg/fetch.hxx
@@ -102,6 +102,19 @@ namespace bpkg
const repository_location&,
const dir_path&);
+ // Fix up or revert the fixes (including in submodules, recursively) in a
+ // working tree previously checked out by git_checkout() or
+ // git_checkout_submodules(). Return true if any changes have been made to
+ // the filesystem.
+ //
+ // Noop on POSIX. On Windows it may replace git's filesystem-agnostic
+ // symlinks with hardlinks for the file targets and junctions for the
+ // directory targets. Note that it still makes sure the working tree is
+ // being treated by git as "clean" despite the changes.
+ //
+ bool
+ git_fixup_worktree (const common_options&, const dir_path&, bool revert);
+
// Low-level fetch API (fetch.cxx).
//
diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx
index 6802c36..764373e 100644
--- a/bpkg/pkg-checkout.cxx
+++ b/bpkg/pkg-checkout.cxx
@@ -56,6 +56,31 @@ namespace bpkg
}
}
+ // For some platforms/repository types the working tree needs to be
+ // temporary "fixed up" for the build2 operations to work properly on it.
+ //
+ static bool
+ fixup (const common_options& o,
+ const repository_location& rl,
+ const dir_path& dir,
+ bool revert = false)
+ {
+ bool r (false);
+
+ switch (rl.type ())
+ {
+ case repository_type::git:
+ {
+ r = git_fixup_worktree (o, dir, revert);
+ break;
+ }
+ case repository_type::pkg:
+ case repository_type::dir: assert (false); break;
+ }
+
+ return r;
+ }
+
shared_ptr<selected_package>
pkg_checkout (const common_options& o,
const dir_path& c,
@@ -136,29 +161,39 @@ namespace bpkg
optional<string> mc;
dir_path d (c / dir_path (n.string () + '-' + v.string ()));
+ // An incomplete checkout may result in an unusable repository state
+ // (submodule fetch is interrupted, working tree fix up failed in the
+ // middle, etc.). That's why we will move the repository into the
+ // temporary directory prior to manipulating it. In the case of a failure
+ // (or interruption) the user will need to run bpkg-rep-fetch to restore
+ // the missing repository.
+ //
+ bool fs_changed (false);
+
if (!simulate)
+ try
{
- // Checkout the repository fragment.
- //
- dir_path sd (c / repos_dir / repository_state (rl));
- checkout (o, rl, sd, ap);
+ if (exists (d))
+ fail << "package directory " << d << " already exists";
- // Calculate the package path that points into the checked out fragment
- // directory.
+ // Check that the repository directory exists, which may not be the case
+ // if the previous checkout have failed or been interrupted.
//
- sd /= path_cast<dir_path> (pl->location);
+ dir_path sd (repository_state (rl));
+ dir_path rd (c / repos_dir / sd);
- // Verify the package prerequisites are all configured since the dist
- // meta-operation generally requires all imports to be resolvable.
- //
- package_manifest m (pkg_verify (sd,
- true /* ignore_unknown */,
- [&ap] (version& v) {v = ap->version;}));
+ if (!exists (rd))
+ fail << "missing repository directory for package " << n << " " << v
+ << " in configuration " << c <<
+ info << "run 'bpkg rep-fetch' to repair";
- pkg_configure_prerequisites (o, t, m.dependencies, m.name);
+ // The repository temporary directory.
+ //
+ auto_rmdir rmt (temp_dir / sd);
+ const dir_path& td (rmt.path);
- if (exists (d))
- fail << "package directory " << d << " already exists";
+ if (exists (td))
+ rm_r (td);
// The temporary out of source directory that is required for the dist
// meta-operation.
@@ -169,10 +204,35 @@ namespace bpkg
if (exists (od))
rm_r (od);
+ // Finally, move the repository to the temporary directory and proceed
+ // with the checkout.
+ //
+ mv (rd, td);
+ fs_changed = true;
+
+ // Checkout the repository fragment and fix up the working tree.
+ //
+ checkout (o, rl, td, ap);
+ bool fixedup (fixup (o, rl, td));
+
+ // Calculate the package path that points into the checked out fragment
+ // directory.
+ //
+ dir_path pd (td / path_cast<dir_path> (pl->location));
+
+ // Verify the package prerequisites are all configured since the dist
+ // meta-operation generally requires all imports to be resolvable.
+ //
+ package_manifest m (pkg_verify (pd,
+ true /* ignore_unknown */,
+ [&ap] (version& v) {v = ap->version;}));
+
+ pkg_configure_prerequisites (o, t, m.dependencies, m.name);
+
// Form the buildspec.
//
string bspec ("dist(");
- bspec += sd.representation ();
+ bspec += pd.representation ();
bspec += '@';
bspec += od.representation ();
bspec += ')';
@@ -205,8 +265,33 @@ namespace bpkg
strings ({"config.dist.root=" + c.representation ()}),
bspec);
+ // Revert the fix-ups.
+ //
+ if (fixedup)
+ fixup (o, rl, td, true /* revert */);
+
+ // Manipulations over the repository are now complete, so we can return
+ // it to its permanent location.
+ //
+ mv (td, rd);
+ fs_changed = false;
+
+ rmt.cancel ();
+
mc = sha256 (o, d / manifest_file);
}
+ catch (const failed&)
+ {
+ if (fs_changed)
+ {
+ // We assume that the diagnostics has already been issued.
+ //
+ warn << "repository state is now broken" <<
+ info << "run 'bpkg rep-fetch' to repair";
+ }
+
+ throw;
+ }
if (p != nullptr)
{
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index b64bbe2..0c2ed89 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -313,7 +313,7 @@ namespace bpkg
dir_path sd (repository_state (rl));
auto_rmdir rm (temp_dir / sd);
- dir_path& td (rm.path);
+ const dir_path& td (rm.path);
if (exists (td))
rm_r (td);
diff --git a/tests/common/git/init b/tests/common/git/init
index 4fac21e..5153175 100755
--- a/tests/common/git/init
+++ b/tests/common/git/init
@@ -42,6 +42,18 @@ fi
#
cd state0
+rm -f -r links.git/.git
+rm -f links.git/.gitmodules
+rm -f links.git/bl
+rm -f links.git/lc
+rm -f links.git/pg
+rm -f links.git/bs
+rm -f links.git/bf
+rm -f links.git/tl
+rm -f links.git/td
+rm -f links.git/ts
+rm -f -r links.git/doc/style
+
rm -f -r libfoo.git/.git
rm -f libfoo.git/.gitmodules
rm -f libfoo.git/README
@@ -156,6 +168,86 @@ git -C libfox.git submodule add ../libbar.git libbar
git -C libfox.git submodule update --init --recursive # Recursive for safety.
git -C libfox.git commit -am 'Create'
+# Create master branch for links.git, adding style.git as a submodule.
+#
+git -C links.git init
+
+cat <<EOF >links.git/manifest
+: 1
+name: links
+version: 0.0.1
+summary: links
+license: MIT
+url: http://example.org
+email: pkg@example.org
+EOF
+
+git -C links.git add '*'
+git -C links.git submodule add ../style.git doc/style
+git -C links.git submodule update --init --recursive # Updates doc/style/basic.
+git -C links.git commit -am 'Create'
+git -C links.git tag -a 'v0.0.1' -m 'Tag version 0.0.1'
+
+# Increase links version and add symlinks.
+#
+cat <<EOF >links.git/manifest
+: 1
+name: links
+version: 1.0.0-a.0.z
+summary: links
+license: MIT
+url: http://example.org
+email: pkg@example.org
+EOF
+
+ln -s tests links.git/ts # Directory symlink.
+ln -s ts/TODO links.git/td # File symlink via directory symlink.
+ln -s td links.git/tl # Symlink symlink.
+ln -s doc/style/buildfile links.git/bf # Submodule file symlink.
+ln -s doc/style/basic links.git/bs # Submodule directory symlink.
+ln -s bs/page.css links.git/pg # Symlink via submodule directory symlink.
+
+git -C links.git add '*'
+git -C links.git commit -am 'Add symlinks'
+git -C links.git tag -a 'v1.0.0-alpha' -m 'Tag version 1.0.0-alpha'
+
+# Increase links version and add dangling symlink.
+#
+cat <<EOF >links.git/manifest
+: 1
+name: links
+version: 1.0.1
+summary: links
+license: MIT
+url: http://example.org
+email: pkg@example.org
+EOF
+
+ln -s lc links.git/bl # Dangling symlink.
+
+git -C links.git add '*'
+git -C links.git commit -am 'Add dangling symlinks'
+git -C links.git tag -a 'v1.0.1' -m 'Tag version 1.0.1'
+
+# Increase links version and add cyclic symlink.
+#
+cat <<EOF >links.git/manifest
+: 1
+name: links
+version: 1.0.2
+summary: links
+license: MIT
+url: http://example.org
+email: pkg@example.org
+EOF
+
+ln -s bl links.git/lc # Cyclic symlink.
+
+git -C links.git add '*'
+git -C links.git commit -am 'Add cyclic symlinks'
+git -C links.git tag -a 'v1.0.2' -m 'Tag version 1.0.2'
+
+
# Create the modified state of the repositories, replacing libbar.git submodule
# of libfoo with the newly created libbaz.git repository. Also advance master
# branches and tags for libfoo.git and it's submodule style.git.
@@ -169,6 +261,10 @@ for d in ../state0/*.git; do
cp -r $d .
done
+# Drop the links.git repository.
+#
+rm -f -r links.git/
+
# Create libbaz.git repository.
#
rm -f -r libbaz.git/.git
diff --git a/tests/common/git/pack b/tests/common/git/pack
index f9d9772..fd0b49c 100755
--- a/tests/common/git/pack
+++ b/tests/common/git/pack
@@ -15,6 +15,7 @@ function error () { info "$*"; exit 1; }
projects=(\
state0/libfoo state0/libfox state0/libbar state0/style state0/style-basic \
+ state0/links \
state1/libfoo state1/libfox state1/libbaz state1/style state1/style-basic)
for p in "${projects[@]}"; do
diff --git a/tests/common/git/state0/libbar.tar b/tests/common/git/state0/libbar.tar
index 2e1a3ad..027112c 100644
--- a/tests/common/git/state0/libbar.tar
+++ b/tests/common/git/state0/libbar.tar
Binary files differ
diff --git a/tests/common/git/state0/libfoo.tar b/tests/common/git/state0/libfoo.tar
index 6a0cc8a..96a9f5b 100644
--- a/tests/common/git/state0/libfoo.tar
+++ b/tests/common/git/state0/libfoo.tar
Binary files differ
diff --git a/tests/common/git/state0/libfox.tar b/tests/common/git/state0/libfox.tar
index a73f460..fc2c391 100644
--- a/tests/common/git/state0/libfox.tar
+++ b/tests/common/git/state0/libfox.tar
Binary files differ
diff --git a/tests/common/git/state0/links.tar b/tests/common/git/state0/links.tar
new file mode 100644
index 0000000..33c5dbf
--- /dev/null
+++ b/tests/common/git/state0/links.tar
Binary files differ
diff --git a/tests/common/git/state0/style-basic.tar b/tests/common/git/state0/style-basic.tar
index 36d0dcd..8b57bd0 100644
--- a/tests/common/git/state0/style-basic.tar
+++ b/tests/common/git/state0/style-basic.tar
Binary files differ
diff --git a/tests/common/git/state0/style.tar b/tests/common/git/state0/style.tar
index 7b48a9d..56e29f0 100644
--- a/tests/common/git/state0/style.tar
+++ b/tests/common/git/state0/style.tar
Binary files differ
diff --git a/tests/common/git/state1/libbaz.tar b/tests/common/git/state1/libbaz.tar
index 1b35ec4..7acf277 100644
--- a/tests/common/git/state1/libbaz.tar
+++ b/tests/common/git/state1/libbaz.tar
Binary files differ
diff --git a/tests/common/git/state1/libfoo.tar b/tests/common/git/state1/libfoo.tar
index ab694f7..532e974 100644
--- a/tests/common/git/state1/libfoo.tar
+++ b/tests/common/git/state1/libfoo.tar
Binary files differ
diff --git a/tests/common/git/state1/libfox.tar b/tests/common/git/state1/libfox.tar
index a6ef40e..ec49a86 100644
--- a/tests/common/git/state1/libfox.tar
+++ b/tests/common/git/state1/libfox.tar
Binary files differ
diff --git a/tests/common/git/state1/style-basic.tar b/tests/common/git/state1/style-basic.tar
index 7b7a6c5..1946606 100644
--- a/tests/common/git/state1/style-basic.tar
+++ b/tests/common/git/state1/style-basic.tar
Binary files differ
diff --git a/tests/common/git/state1/style.tar b/tests/common/git/state1/style.tar
index e6243f4..769b6d5 100644
--- a/tests/common/git/state1/style.tar
+++ b/tests/common/git/state1/style.tar
Binary files differ
diff --git a/tests/pkg-checkout.testscript b/tests/pkg-checkout.testscript
index 148dcff..a284f6b 100644
--- a/tests/pkg-checkout.testscript
+++ b/tests/pkg-checkout.testscript
@@ -11,6 +11,8 @@
# |-- libbar.git -> style-basic.git (prerequisite)
# `-- style-basic.git
+posix = ($cxx.target.class != 'windows')
+
# Prepare repositories used by tests if running in the local mode.
#
+if ($remote != true)
@@ -19,6 +21,11 @@
$git_extract $src/git/libbar.tar
$git_extract $src/git/style-basic0.tar &$out_git/state0/***
$git_extract $src/git/style-basic1.tar &$out_git/state1/***
+
+ if $posix
+ $git_extract $src/git/style.tar
+ $git_extract $src/git/links.tar
+ end
end
: git-rep
@@ -39,67 +46,172 @@ else
pkg_purge += -d cfg 2>!
pkg_status += -d cfg
- test.cleanups += &cfg/.bpkg/repos/*/***
+ test.cleanups += &?cfg/.bpkg/repos/*/***
: unconfigured-dependency
:
- $clone_root_cfg;
- $rep_add "$rep/libbar.git#master";
- $rep_fetch;
-
- $* libmbar/1.0.0 2>>EOE != 0
- error: no configured package satisfies dependency on style-basic >= 1.0.0
- EOE
+ {
+ $clone_root_cfg;
+ $rep_add "$rep/libbar.git#master";
+ $rep_fetch;
+
+ $* libmbar/1.0.0 2>>EOE != 0
+ error: no configured package satisfies dependency on style-basic >= 1.0.0
+ warning: repository state is now broken
+ info: run 'bpkg rep-fetch' to repair
+ EOE
+ }
: configured-dependency
:
- $clone_root_cfg;
- $rep_add "$rep/libbar.git#master" && $rep_add "$rep/style-basic.git#master";
- $rep_fetch;
+ {
+ $clone_root_cfg;
+ $rep_add "$rep/libbar.git#master" && $rep_add "$rep/style-basic.git#master";
+ $rep_fetch;
- $pkg_status style-basic | sed -n -e 's/style-basic available \[.+\] ([^ ]+)/\1/p' | set v;
+ $pkg_status style-basic | sed -n -e 's/style-basic available \[.+\] ([^ ]+)/\1/p' | set v;
- $* "style-basic/$v" 2>>"EOE";
- distributing style-basic/$v
- checked out style-basic/$v
- EOE
+ $* "style-basic/$v" 2>>"EOE";
+ distributing style-basic/$v
+ checked out style-basic/$v
+ EOE
- $pkg_configure style-basic;
+ $pkg_configure style-basic;
- $* libmbar/1.0.0 2>>EOE;
- distributing libmbar/1.0.0
- checked out libmbar/1.0.0
- EOE
+ $* libmbar/1.0.0 2>>EOE;
+ distributing libmbar/1.0.0
+ checked out libmbar/1.0.0
+ EOE
- $pkg_disfigure style-basic;
+ $pkg_disfigure style-basic;
- $pkg_purge libmbar;
- $pkg_purge style-basic
+ $pkg_purge libmbar;
+ $pkg_purge style-basic
+ }
: replacement
:
- # @@ Reduce to a single repository when multiple revisions can be specified
- # in the repository URL fragment.
- #
- rep0 = "$rep_git/state0";
- rep1 = "$rep_git/state1";
+ {
+ # @@ Reduce to a single repository when multiple revisions can be specified
+ # in the repository URL fragment.
+ #
+ rep0 = "$rep_git/state0";
+ rep1 = "$rep_git/state1";
+
+ $clone_root_cfg;
+ $rep_add "$rep0/style-basic.git#master";
+ $rep_add "$rep1/style-basic.git#stable";
+ $rep_fetch;
- $clone_root_cfg;
- $rep_add "$rep0/style-basic.git#master";
- $rep_add "$rep1/style-basic.git#stable";
- $rep_fetch;
+ $pkg_status style-basic | \
+ sed -n -e 's/style-basic available ([^ ]+) +([^ ]+)/\1 \2/p' | set vs;
- $pkg_status style-basic | \
- sed -n -e 's/style-basic available ([^ ]+) +([^ ]+)/\1 \2/p' | set vs;
+ echo "$vs" | sed -e 's/([^ ]+).+/\1/' | set v0;
+ echo "$vs" | sed -e 's/([^ ]+) +([^ ]+)/\2/' | set v1;
- echo "$vs" | sed -e 's/([^ ]+).+/\1/' | set v0;
- echo "$vs" | sed -e 's/([^ ]+) +([^ ]+)/\2/' | set v1;
+ $* "style-basic/$v0" 2>!;
+ $pkg_status style-basic >~"/style-basic unpacked $v0/";
- $* "style-basic/$v0" 2>!;
- $pkg_status style-basic >~"/style-basic unpacked $v0/";
+ $* --replace "style-basic/$v1" 2>!;
+ $pkg_status style-basic >~"/style-basic unpacked $v1 .+/";
- $* --replace "style-basic/$v1" 2>!;
- $pkg_status style-basic >~"/style-basic unpacked $v1 .+/";
+ $pkg_purge style-basic
+ }
- $pkg_purge style-basic
+ : links
+ :
+ if ($remote == true || $posix)
+ {
+ $clone_root_cfg;
+
+ $rep_fetch "$rep/links.git#v1.0.0-alpha";
+
+ $pkg_status links | sed -n -e 's/links available (.+)/\1/p' | set v;
+
+ $* "links/$v" 2>>~%EOE%;
+ %.*
+ %checking out links/1.0.0-a.0.[^.]+.[^.]+%d
+ %.*
+ %distributing links/1.0.0-a.0.[^.]+.[^.]+%d
+ %checked out links/1.0.0-a.0.[^.]+.[^.]+%d
+ EOE
+
+ d = "cfg/links-$v";
+
+ # See common/git/init script for the symlinks descriptions.
+ #
+ test -d $d/bs;
+ test -d $d/ts;
+
+ cat $d/pg >'h1 {font-size: 3em;}';
+ cat $d/bs/page.css >'h1 {font-size: 3em;}';
+ cat $d/bf >'./: file{manifest}';
+ cat $d/td >'@@';
+ cat $d/tl >'@@';
+ cat $d/ts/TODO >'@@';
+
+ $pkg_purge links;
+ $rep_fetch "$rep/links.git#v0.0.1";
+
+ $* links/0.0.1 2>>~%EOE%;
+ checking out links/0.0.1
+ distributing links/0.0.1
+ checked out links/0.0.1
+ EOE
+
+ d = cfg/links-0.0.1;
+
+ test -d $d/bs == 1;
+ test -d $d/ts == 1;
+ test -f $d/pg == 1;
+ test -f $d/bf == 1;
+ test -f $d/td == 1;
+ test -f $d/tl == 1;
+
+ $pkg_purge links;
+
+ # Dangling symlink in the repository.
+ #
+ $rep_fetch "$rep/links.git#v1.0.1";
+
+ if $posix
+ $* links/1.0.1 2>>~%EOE%
+ checking out links/1.0.1
+ distributing links/1.0.1
+ checked out links/1.0.1
+ EOE
+
+ $pkg_purge links
+ else
+ $* links/1.0.1 2>>~%EOE% != 0
+ checking out links/1.0.1
+ error: target 'bl' for symlink 'lc' does not exist
+ info: re-run with -v for more information
+ warning: repository state is now broken
+ info: run 'bpkg rep-fetch' to repair
+ EOE
+ end;
+
+ # Cyclic symlinks in the repository.
+ #
+ if $posix
+ $rep_fetch "$rep/links.git#v1.0.2" 2>>~%EOE% != 0
+ %.*
+ %error: unable to iterate over .+%
+ warning: repository state is now broken and will be cleaned up
+ info: run 'bpkg rep-fetch' to update
+ EOE
+ else
+ $rep_fetch "$rep/links.git#v1.0.2"
+
+ $* links/1.0.2 2>>~%EOE% != 0
+ checking out links/1.0.2
+ %.*
+ %error: target '..' for symlink '..' does not exist%
+ info: re-run with -v for more information
+ warning: repository state is now broken
+ info: run 'bpkg rep-fetch' to repair
+ EOE
+ end
+ }
}
diff --git a/tests/pkg-checkout/git/links.tar b/tests/pkg-checkout/git/links.tar
new file mode 120000
index 0000000..63545fe
--- /dev/null
+++ b/tests/pkg-checkout/git/links.tar
@@ -0,0 +1 @@
+../../common/git/state0/links.tar \ No newline at end of file
diff --git a/tests/pkg-checkout/git/style.tar b/tests/pkg-checkout/git/style.tar
new file mode 120000
index 0000000..948d152
--- /dev/null
+++ b/tests/pkg-checkout/git/style.tar
@@ -0,0 +1 @@
+../../common/git/state0/style.tar \ No newline at end of file