aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bpkg/fetch-git.cxx1084
-rw-r--r--bpkg/fetch.hxx42
-rw-r--r--bpkg/package.cxx6
-rw-r--r--bpkg/package.hxx4
-rw-r--r--bpkg/pkg-checkout.cxx39
-rw-r--r--bpkg/rep-add.cli53
-rw-r--r--bpkg/rep-fetch.cxx156
-rw-r--r--bpkg/rep-remove.cxx9
-rw-r--r--tests/buildfile2
-rwxr-xr-xtests/common/git/init16
-rwxr-xr-xtests/common/git/pack5
-rw-r--r--tests/common/git/state0/libbar.tarbin71680 -> 71680 bytes
-rw-r--r--tests/common/git/state0/libfoo.tarbin296960 -> 296960 bytes
-rw-r--r--tests/common/git/state0/libfox.tarbin0 -> 133120 bytes
-rw-r--r--tests/common/git/state0/style-basic.tarbin71680 -> 71680 bytes
-rw-r--r--tests/common/git/state0/style.tarbin133120 -> 133120 bytes
-rw-r--r--tests/common/git/state1/libbaz.tarbin61440 -> 61440 bytes
-rw-r--r--tests/common/git/state1/libfoo.tarbin378880 -> 378880 bytes
-rw-r--r--tests/common/git/state1/libfox.tarbin0 -> 133120 bytes
-rw-r--r--tests/common/git/state1/style-basic.tarbin71680 -> 71680 bytes
-rw-r--r--tests/common/git/state1/style.tarbin133120 -> 133120 bytes
-rw-r--r--tests/rep-add.test10
-rw-r--r--tests/rep-fetch-git-branch.test156
-rw-r--r--tests/rep-fetch-git-commit.test77
-rw-r--r--tests/rep-fetch-git-refname.test148
-rw-r--r--tests/rep-fetch-git.test32
-rw-r--r--tests/rep-fetch.test57
27 files changed, 1174 insertions, 722 deletions
diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
index c22bafb..fba1c8e 100644
--- a/bpkg/fetch-git.cxx
+++ b/bpkg/fetch-git.cxx
@@ -4,9 +4,8 @@
#include <bpkg/fetch.hxx>
-#ifdef _WIN32
-# include <algorithm> // replace()
-#endif
+#include <map>
+#include <algorithm> // find(), find_if(), replace()
#include <libbutl/utility.mxx> // digit(), xdigit()
#include <libbutl/process.mxx>
@@ -340,7 +339,81 @@ namespace bpkg
u.insert (7, 1, '/');
#endif
- return repository_url (u);
+ repository_url r (u);
+
+ path& up (*r.path);
+
+ if (!up.to_directory ())
+ up = path_cast<dir_path> (move (up));
+
+ return r;
+ }
+
+ // Get/set the repository configuration option.
+ //
+ inline static string
+ config_get (const common_options& co,
+ const dir_path& dir,
+ const string& key,
+ const char* what)
+ {
+ return git_string (co,
+ what,
+ co.git_option (),
+ "-C", dir,
+ "config",
+ "--get",
+ key);
+ }
+
+ inline static void
+ config_set (const common_options& co,
+ const dir_path& dir,
+ const string& key,
+ const string& value)
+ {
+ run_git (co, co.git_option (), "-C", dir, "config", key, value);
+ }
+
+ // Get option from the specified configuration file.
+ //
+ inline static string
+ config_get (const common_options& co,
+ const path& file,
+ const string& key,
+ const char* what)
+ {
+ return git_string (co,
+ what,
+ co.git_option (),
+ "config",
+ "--file", file,
+ "--get",
+ key);
+ }
+
+ // Get/set the repository remote URL.
+ //
+ static repository_url
+ origin_url (const common_options& co, const dir_path& dir)
+ {
+ try
+ {
+ return from_git_url (
+ config_get (co, dir, "remote.origin.url", "repository remote URL"));
+ }
+ catch (const invalid_argument& e)
+ {
+ fail << "invalid remote.origin.url configuration value: " << e << endg;
+ }
+ }
+
+ inline static void
+ origin_url (const common_options& co,
+ const dir_path& dir,
+ const repository_url& url)
+ {
+ config_set (co, dir, "remote.origin.url", to_git_url (url));
}
// Sense the git protocol capabilities for a specified URL.
@@ -476,16 +549,121 @@ namespace bpkg
fail << "unable to fetch " << url << endg;
}
- // Return true if a commit is advertised by the remote repository. It is
- // assumed that sense_capabilities() function was already called for the URL.
+ // A git ref (tag, branch, etc) and its commit id (i.e., one line of the
+ // git-ls-remote output).
//
- static bool
- commit_advertized (const common_options& co,
- const repository_url& url,
- const string& commit)
+ struct ref
+ {
+ string name; // Note: without the peel marker ('^{}').
+ string commit;
+ bool peeled; // True for '...^{}' references.
+ };
+
+ // List of all refs and their commit ids advertized by a repository (i.e.,
+ // the git-ls-remote output).
+ //
+ class refs: public vector<ref>
+ {
+ public:
+ // Resolve a git refname (tag, branch, etc) returning NULL if not found.
+ //
+ const ref*
+ find_name (const string& n, bool abbr_commit) const
+ {
+ auto find = [this] (const string& n) -> const ref*
+ {
+ auto i (find_if (
+ begin (), end (),
+ [&n] (const ref& i)
+ {
+ return !i.peeled && i.name == n; // Note: skip peeled.
+ }));
+
+ return i != end () ? &*i : nullptr;
+ };
+
+ // Let's search in the order refs are disambiguated by git (see
+ // gitrevisions(7) for details).
+ //
+ // What should we do if the name already contains a slash, for example,
+ // tags/v1.2.3? Note that while it most likely is relative to refs/, it
+ // can also be relative to other subdirectories (say releases/v1.2.3 in
+ // tags/releases/v1.2.3). So we just check everywhere.
+
+ // This handles symbolic references like HEAD.
+ //
+ if (const ref* r = find (n))
+ return r;
+
+ if (const ref* r = find ("refs/" + n))
+ return r;
+
+ if (const ref* r = find ("refs/tags/" + n))
+ return r;
+
+ if (const ref* r = find ("refs/heads/" + n))
+ return r;
+
+ // See if this is an abbreviated commit id. We do this check last since
+ // this can be ambiguous. We also don't bother checking strings shorter
+ // than 7 characters (git default).
+ //
+ return abbr_commit && n.size () >= 7 ? find_commit (n) : nullptr;
+ }
+
+ // Resolve (potentially abbreviated) commit id returning NULL if not
+ // found and failing if the resolution is ambiguous.
+ //
+ const ref*
+ find_commit (const string& c) const
+ {
+ const ref* r (nullptr);
+ size_t n (c.size ());
+
+ for (const ref& rf: *this)
+ {
+ if (rf.commit.compare (0, n, c) == 0)
+ {
+ if (r == nullptr)
+ r = &rf;
+
+ // Note that different names can refer to the same commit.
+ //
+ else if (r->commit != rf.commit)
+ fail << "abbreviated commit id " << c << " is ambiguous" <<
+ info << "candidate: " << r->commit <<
+ info << "candidate: " << rf.commit;
+ }
+ }
+
+ return r;
+ }
+ };
+
+ // Map of repository URLs to their advertized refs/commits.
+ //
+ using repository_refs_map = map<string, refs>;
+
+ static repository_refs_map repository_refs;
+
+ // It is assumed that sense_capabilities() function was already called for
+ // the URL.
+ //
+ static const refs&
+ load_refs (const common_options& co, const repository_url& url)
{
- tracer trace ("commit_advertized");
+ tracer trace ("load_refs");
+ string u (url.string ());
+ auto i (repository_refs.find (u));
+
+ if (i != repository_refs.end ())
+ return i->second;
+
+ if (verb)
+ text << "querying " << url;
+
+ refs rs;
fdpipe pipe (open_pipe ());
process pr (start_git (co,
@@ -493,47 +671,128 @@ namespace bpkg
timeout_opts (co, url.scheme),
co.git_option (),
"ls-remote",
- "--refs",
to_git_url (url)));
- pipe.out.close (); // Shouldn't throw, unless something is severely damaged.
+ // Shouldn't throw, unless something is severely damaged.
+ //
+ pipe.out.close ();
- try
+ for (;;) // Breakout loop.
{
- bool r (false);
- ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit);
-
- for (string l; !eof (getline (is, l)); )
+ try
{
- l4 ([&]{trace << "ref: " << l;});
+ ifdstream is (move (pipe.in), fdstream_mode::skip, ifdstream::badbit);
- if (l.compare (0, commit.size (), commit) == 0)
+ for (string l; !eof (getline (is, l)); )
{
- r = true;
- break;
+ l4 ([&]{trace << "ref line: " << l;});
+
+ size_t n (l.find ('\t'));
+
+ if (n == string::npos)
+ fail << "unable to parse references for " << url << endg;
+
+ string cm (l, 0, n);
+ string nm (l, n + 1);
+
+ n = nm.rfind ("^{");
+ bool peeled (n != string::npos);
+
+ if (peeled)
+ nm.resize (n); // Strip the peel operation ('^{...}').
+
+ rs.push_back (ref {move (nm), move (cm), peeled});
}
+
+ is.close ();
+
+ if (pr.wait ())
+ break;
+
+ // Fall through.
}
+ catch (const io_error&)
+ {
+ if (pr.wait ())
+ fail << "unable to read references for " << url << endg;
- is.close ();
+ // Fall through.
+ }
- if (pr.wait ())
- return r;
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
- // Fall through.
+ fail << "unable to list references for " << url << endg;
}
- catch (const io_error&)
- {
- if (pr.wait ())
- fail << "unable to read references for " << url << endg;
- // Fall through.
- }
+ return repository_refs.emplace (move (u), move (rs)).first->second;
+ }
- // We should only get here if the child exited with an error status.
+ // Return true if a commit is advertised by the remote repository. It is
+ // assumed that sense_capabilities() function was already called for the URL.
+ //
+ static bool
+ commit_advertized (const common_options& co,
+ const repository_url& url,
+ const string& commit)
+ {
+ return load_refs (co, url).find_commit (commit) != nullptr;
+ }
+
+ // Return true if a commit is already fetched.
+ //
+ static bool
+ commit_fetched (const common_options& co,
+ const dir_path& dir,
+ const string& commit)
+ {
+ auto_fd dev_null (open_dev_null ());
+
+ process pr (start_git (co,
+ 1, // The output is suppressed by -e.
+ dev_null,
+ co.git_option (),
+ "-C", dir,
+ "cat-file",
+ "-e",
+ commit + "^{commit}"));
+
+ // Shouldn't throw, unless something is severely damaged.
//
- assert (!pr.wait ());
+ dev_null.close ();
+ return pr.wait ();
+ }
+
+ // Create an empty repository and configure the remote origin URL and the
+ // default fetch refspec. If requested, use a separate git directory,
+ // creating it if absent.
+ //
+ static void
+ init (const common_options& co,
+ const dir_path& dir,
+ const repository_url& url,
+ const dir_path& git_dir = dir_path ())
+ {
+ if (!run_git (
+ co,
+ co.git_option (),
+ "init",
+
+ !git_dir.empty ()
+ ? strings ({"--separate-git-dir=" + git_dir.string ()})
+ : strings (),
+
+ verb < 2 ? opt ("-q") : nullopt,
+ dir))
+ fail << "unable to init " << dir << endg;
+
+ origin_url (co, dir, url);
- fail << "unable to list references for " << url << endg;
+ config_set (co,
+ dir,
+ "remote.origin.fetch",
+ "+refs/heads/*:refs/remotes/origin/*");
}
// Return true if the shallow fetch is possible for the reference.
@@ -542,7 +801,7 @@ namespace bpkg
shallow_fetch (const common_options& co,
const repository_url& url,
capabilities cap,
- const git_reference& ref)
+ const git_ref_filter& rf)
{
switch (cap)
{
@@ -552,7 +811,7 @@ namespace bpkg
}
case capabilities::smart:
{
- return !ref.commit || commit_advertized (co, url, *ref.commit);
+ return !rf.commit || commit_advertized (co, url, *rf.commit);
}
case capabilities::unadv:
{
@@ -564,123 +823,177 @@ namespace bpkg
return false;
}
- // Return true if a commit is reachable from the tip(s).
+ // @@ Move to libbpkg when adding support for multiple fragments.
//
- // Can be used to avoid redundant fetches.
- //
- // Note that git-submodule script implements this check, so it is probably an
- // important optimization.
+ using git_ref_filters = vector<git_ref_filter>;
+
+ // Fetch the references and return their commit ids. If there are no
+ // reference filters specified, then fetch all the tags and branches.
//
- static bool
- commit_reachable (const common_options& co,
- const dir_path& dir,
- const string& commit)
+ static strings
+ fetch (const common_options& co,
+ const dir_path& dir,
+ const dir_path& submodule, // Used only for diagnostics.
+ const git_ref_filters& rfs)
{
- fdpipe pipe (open_pipe ());
- auto_fd dev_null (open_dev_null ());
+ strings r;
+ capabilities cap;
+ repository_url url;
- process pr (start_git (co,
- pipe,
- dev_null,
- co.git_option (),
- "-C", dir,
- "rev-list",
- "-n", "1",
- commit,
- "--not",
- "--all"));
+ strings scs; // Shallow fetch commits.
+ strings dcs; // Deep fetch commits.
- // Shouldn't throw, unless something is severely damaged.
- //
- pipe.out.close ();
- dev_null.close ();
+ bool fetch_repo (false);
- try
+ // Translate a name to the corresponding commit. Fail if the name is not
+ // known by the remote repository.
+ //
+ auto commit = [&co, &dir, &url] (const string& nm, bool abbr_commit)
+ -> const string&
{
- ifdstream is (move (pipe.in), fdstream_mode::skip);
+ if (url.empty ())
+ url = origin_url (co, dir);
- string s;
- if (is.peek () != ifdstream::traits_type::eof ())
- getline (is, s);
+ const ref* r (load_refs (co, url).find_name (nm, abbr_commit));
- is.close ();
- return pr.wait () && s.empty ();
- }
- catch (const io_error&) {}
- return false;
- }
+ if (r == nullptr)
+ fail << "unable to fetch " << nm << " from " << url <<
+ info << "name is not recognized" << endg;
- // Print warnings about non-shallow fetching.
- //
- static void
- fetch_warn (capabilities cap,
- const char* what,
- const dir_path& submodule = dir_path ())
- {
+ return r->commit;
+ };
+
+ // Add a commit to the list, suppressing duplicates.
+ //
+ auto add = [] (const string& c, strings& cs)
{
- diag_record dr (warn);
- dr << "fetching whole " << what << " history";
+ if (find (cs.begin (), cs.end (), c) == cs.end ())
+ cs.push_back (c);
+ };
- if (!submodule.empty ())
- dr << " for submodule '" << submodule.posix_string () << "'";
+ if (rfs.empty ())
+ {
+ url = origin_url (co, dir);
- dr << " ("
- << (cap == capabilities::dumb
- ? "dumb HTTP"
- : "unadvertised commit") // There are no other reasons so far.
- << ')';
+ for (const ref& rf: load_refs (co, url))
+ {
+ // Skip everything other than tags and branches.
+ //
+ if (rf.peeled ||
+ (rf.name.compare (0, 11, "refs/heads/") != 0 &&
+ rf.name.compare (0, 10, "refs/tags/") != 0))
+ continue;
- }
+ // Add an already fetched commit to the resulting list.
+ //
+ if (commit_fetched (co, dir, rf.commit))
+ {
+ add (rf.commit, r);
+ continue;
+ }
- if (cap == capabilities::dumb)
- warn << "fetching over dumb HTTP, no progress will be shown";
- }
+ // Evaluate if the commit can be obtained with the shallow fetch and
+ // add it to the proper list.
+ //
+ if (scs.empty () && dcs.empty ())
+ cap = sense_capabilities (co, url);
- // Update git index and working tree to match the reference. Fetch if
- // necessary.
- //
- static void
- update_tree (const common_options& co,
- const dir_path& dir,
- const dir_path& submodule, // Is relative to the top project.
- const git_reference& ref,
- capabilities cap,
- bool shallow,
- const strings& to)
- {
- // Don't fetch it the reference is a commit that is reachable from the
- // tip(s).
- //
- if (!(ref.commit && commit_reachable (co, dir, *ref.commit)))
+ // The commit is advertised, so we can fetch shallow, unless the
+ // protocol is dumb.
+ //
+ add (rf.commit, cap != capabilities::dumb ? scs : dcs);
+ }
+ }
+ else
{
- if (!shallow)
- fetch_warn (cap, ref.commit ? "repository" : "branch", submodule);
+ for (const git_ref_filter& rf: rfs)
+ {
+ // Add an already fetched commit to the resulting list.
+ //
+ if (rf.commit && commit_fetched (co, dir, *rf.commit))
+ {
+ add (*rf.commit, r);
+ continue;
+ }
- // The clone command prints the following line prior to the progress
- // lines:
- //
- // Cloning into '<dir>'...
- //
- // The fetch command doesn't print anything similar, for some reason.
- // This makes it hard to understand which superproject/submodule is
- // currently being fetched. Let's fix that.
- //
- // Note that we have "fixed" that capital letter nonsense (hoping that
- // git-clone will do the same at some point).
- //
- if (verb != 0)
- text << "fetching in '" << dir.posix_string () << "'...";
+ if (!rf.commit)
+ {
+ assert (rf.name);
+ const string& c (commit (*rf.name, true /* abbr_commit */));
+
+ if (commit_fetched (co, dir, c))
+ {
+ add (c, r);
+ continue;
+ }
+ }
+
+ // Evaluate if the commit can be obtained with the shallow fetch and
+ // add it to the proper list.
+ //
+ if (scs.empty () && dcs.empty ())
+ {
+ if (url.empty ()) // Can already be assigned by commit() lambda.
+ url = origin_url (co, dir);
+
+ cap = sense_capabilities (co, url);
+ }
+
+ bool shallow (shallow_fetch (co, url, cap, rf));
+ strings& commits (shallow ? scs : dcs);
+
+ // If commit is not specified, then we fetch the commit the refname
+ // translates to.
+ //
+ if (!rf.commit)
+ {
+ assert (rf.name);
+
+ add (commit (*rf.name, true /* abbr_commit */), commits);
+ }
+
+ // If commit is specified and the shallow fetch is possible, then we
+ // fetch the commit.
+ //
+ else if (shallow)
+ add (*rf.commit, commits);
+
+ // If commit is specified and the shallow fetch is not possible, but
+ // the refname containing the commit is specified, then we fetch the
+ // whole refname history.
+ //
+ else if (rf.name)
+ add (commit (*rf.name, false /* abbr_commit */), commits);
+
+ // Otherwise, if the refname is not specified and the commit is not
+ // advertised, we have to fetch the whole repository history.
+ //
+ else
+ {
+ add (*rf.commit, commits);
+ fetch_repo = !commit_advertized (co, url, *rf.commit);
+ }
+ }
+ }
+
+ // Bail out if all commits are already fetched.
+ //
+ if (scs.empty () && dcs.empty ())
+ return r;
+
+ auto fetch = [&co, &url, &dir] (const strings& refspecs, bool shallow)
+ {
// Note that we suppress the (too detailed) fetch command output if the
// verbosity level is 1. However, we still want to see the progress in
// this case, unless STDERR is not directed to a terminal.
//
// Also note that we don't need to specify --refmap option since we can
- // rely on the clone command that properly set the remote.origin.fetch
- // configuration option.
+ // rely on the init() function that properly sets the
+ // remote.origin.fetch configuration option.
//
if (!run_git (co,
- to,
+ timeout_opts (co, url.scheme),
co.git_option (),
"-C", dir,
"fetch",
@@ -689,50 +1002,113 @@ namespace bpkg
verb == 1 && fdterm (2) ? opt ( "--progress") : nullopt,
verb < 2 ? opt ("-q") : verb > 3 ? opt ("-v") : nullopt,
"origin",
- ref.commit ? *ref.commit : *ref.branch))
+ refspecs))
fail << "unable to fetch " << dir << endg;
+ };
+
+ // Print warnings prior to the deep fetching.
+ //
+ if (!dcs.empty ())
+ {
+ {
+ diag_record dr (warn);
+ dr << "fetching whole " << (fetch_repo ? "repository" : "reference")
+ << " history";
+
+ if (!submodule.empty ())
+ dr << " for submodule '" << submodule.posix_string () << "'";
+
+ dr << " ("
+ << (cap == capabilities::dumb
+ ? "dumb HTTP"
+ : "unadvertised commit") // There are no other reasons so far.
+ << ')';
+ }
+
+ if (cap == capabilities::dumb)
+ warn << "fetching over dumb HTTP, no progress will be shown";
}
- const string& commit (ref.commit ? *ref.commit : string ("FETCH_HEAD"));
+ // Print the progress indicator.
+ //
+ // Note that the clone command prints the following line prior to the
+ // progress lines:
+ //
+ // Cloning into '<dir>'...
+ //
+ // The fetch command doesn't print anything similar, for some reason.
+ // This makes it hard to understand which superproject/submodule is
+ // currently being fetched. Let's fix that.
+ //
+ // Also note that we have "fixed" that capital letter nonsense and stripped
+ // the trailing '...'.
+ //
+ if (verb)
+ {
+ diag_record dr (text);
+ dr << "fetching ";
+
+ if (!submodule.empty ())
+ dr << "submodule '" << submodule.posix_string () << "' ";
- // For some (probably valid) reason the hard reset command doesn't remove
- // a submodule directory that is not plugged into the project anymore. It
- // also prints the non-suppressible warning like this:
+ dr << "from " << url;
+
+ if (verb >= 2)
+ dr << " in '" << dir.posix_string () << "'"; // Is used by tests.
+ }
+
+ // Note that the shallow, deep and the resulting lists don't overlap.
+ // Thus, we will be moving commits between the lists not caring about
+ // suppressing duplicates.
//
- // warning: unable to rmdir libbar: Directory not empty
+
+ // First, we fetch deep commits.
//
- // That's why we run the clean command afterwards. It may also be helpful
- // if we produce any untracked files in the tree between fetches down the
- // road.
+ if (!dcs.empty ())
+ {
+ fetch (!fetch_repo ? dcs : strings (), false);
+
+ r.insert (r.end (),
+ make_move_iterator (dcs.begin ()),
+ make_move_iterator (dcs.end ()));
+ }
+
+ // After the deep fetching some of the shallow commits might also be
+ // fetched, so we move them to the resulting list.
//
- if (!run_git (
- co,
- co.git_option (),
- "-C", dir,
- "reset",
- "--hard",
- verb < 2 ? opt ("-q") : nullopt,
- commit))
- fail << "unable to reset to " << commit << endg;
+ strings cs;
+ for (auto& c: scs)
+ (commit_fetched (co, dir, c) ? r : cs).push_back (move (c));
- if (!run_git (
- co,
- co.git_option (),
- "-C", dir,
- "clean",
- "-d",
- "-x",
- "-ff",
- verb < 2 ? opt ("-q") : nullopt))
- fail << "unable to clean " << dir << endg;
+ // Finally, fetch shallow commits.
+ //
+ if (!cs.empty ())
+ {
+ fetch (cs, true);
+
+ r.insert (r.end (),
+ make_move_iterator (cs.begin ()),
+ make_move_iterator (cs.end ()));
+ }
+
+ return r;
}
+ // Checkout the repository submodules (see git_checkout_submodules()
+ // description for details).
+ //
static void
- update_submodules (const common_options& co,
- const dir_path& dir,
- const dir_path& prefix)
+ checkout_submodules (const common_options& co,
+ const dir_path& dir,
+ const dir_path& git_dir,
+ const dir_path& prefix)
{
- tracer trace ("update_submodules");
+ tracer trace ("checkout_submodules");
+
+ path mf (dir / path (".gitmodules"));
+
+ if (!exists (mf))
+ return;
auto failure = [&prefix] (const char* desc)
{
@@ -759,15 +1135,20 @@ namespace bpkg
: strings (),
"submodule--helper", "init",
- verb < 1 ? opt ("-q") : nullopt))
+ verb < 2 ? opt ("-q") : nullopt))
failure ("unable to initialize submodules");
- // Iterate over the registered submodules cloning/fetching them and
- // recursively updating their submodules.
+ repository_url orig_url (origin_url (co, dir));
+
+ // Iterate over the registered submodules initializing/fetching them and
+ // recursively checking them out.
//
// Note that we don't expect submodules nesting be too deep and so recurse
// while reading the git process output.
//
+ // Also note that we don't catch the failed exception here, relying on the
+ // fact that the process destructor will wait for the process completion.
+ //
fdpipe pipe (open_pipe ());
process pr (start_git (co,
@@ -776,7 +1157,9 @@ namespace bpkg
"-C", dir,
"submodule--helper", "list"));
- pipe.out.close (); // Shouldn't throw, unless something is severely damaged.
+ // Shouldn't throw, unless something is severely damaged.
+ //
+ pipe.out.close ();
try
{
@@ -808,92 +1191,119 @@ namespace bpkg
dir_path psdir (prefix / sdir);
string psd (psdir.posix_string ()); // For use in the diagnostics.
- string name (git_string (co, "submodule name",
- co.git_option (),
- "-C", dir,
- "submodule--helper", "name",
- sdir));
+ string nm (git_string (co, "submodule name",
+ co.git_option (),
+ "-C", dir,
+ "submodule--helper", "name",
+ sdir));
+
+ string uo ("submodule." + nm + ".url");
+ string uv (config_get (co, dir, uo, "submodule URL"));
+ l4 ([&]{trace << "name: " << nm << ", URL: " << uv;});
+
+ dir_path fsdir (dir / sdir);
+ bool initialized (exists (fsdir / path (".git")));
+
+ // If the submodule is already initialized and its commit didn't
+ // change then we skip it.
+ //
+ if (initialized && git_string (co, "submodule commit",
+ co.git_option (),
+ "-C", fsdir,
+ "rev-parse",
+ "--verify",
+ "HEAD") == commit)
+ continue;
+
+ // Note that the "submodule--helper init" command (see above) doesn't
+ // sync the submodule URL in .git/config file with the one in
+ // .gitmodules file, that is a primary URL source. Thus, we always
+ // calculate the URL using .gitmodules and update it in .git/config, if
+ // necessary.
+ //
repository_url url;
try
{
- url = from_git_url (git_string (co, "submodule URL",
- co.git_option (),
- "-C", dir,
- "config",
- "--get",
- "submodule." + name + ".url"));
+ url = from_git_url (
+ config_get (co, mf, uo, "submodule original URL"));
+
+ // Complete the relative submodule URL against the containing
+ // repository origin URL.
+ //
+ if (url.scheme == repository_protocol::file && url.path->relative ())
+ {
+ repository_url u (orig_url);
+ *u.path /= *url.path;
+
+ // Note that we need to collapse 'example.com/a/..' to
+ // 'example.com/', rather than to 'example.com/.'.
+ //
+ u.path->normalize (
+ false /* actual */,
+ orig_url.scheme != repository_protocol::file /* cur_empty */);
+
+ url = move (u);
+ }
+
+ // Fix-up submodule URL in .git/config file, if required.
+ //
+ if (url != from_git_url (move (uv)))
+ {
+ config_set (co, dir, uo, to_git_url (url));
+
+ // We also need to fix-up submodule's origin URL, if its
+ // repository is already initialized.
+ //
+ if (initialized)
+ origin_url (co, fsdir, url);
+ }
}
catch (const invalid_argument& e)
{
fail << "invalid repository URL for submodule '" << psd << "': "
<< e << endg;
}
+ catch (const invalid_path& e)
+ {
+ fail << "invalid repository path for submodule '" << psd << "': "
+ << e << endg;
+ }
- l4 ([&]{trace << "name: " << name << ", URL: " << url;});
-
- dir_path fsdir (dir / sdir);
- bool cloned (exists (fsdir / path (".git")));
-
- // If the submodule is already cloned and it's commit didn't change
- // then we skip it.
+ // Initialize the submodule repository.
//
- // Note that git-submodule script still recurse into it for some
- // unclear reason.
+ // Note that we initialize the submodule repository git directory out
+ // of the working tree, the same way as "submodule--helper clone"
+ // does. This prevents us from loosing the fetched data when switching
+ // the containing repository between revisions, that potentially
+ // contain different sets of submodules.
//
- if (cloned && git_string (co, "submodule commit",
- co.git_option (),
- "-C", fsdir,
- "rev-parse",
- "--verify",
- "HEAD") == commit)
- continue;
+ dir_path gdir (git_dir / dir_path ("modules") / sdir);
- git_reference ref {nullopt, commit};
- capabilities cap (sense_capabilities (co, url));
- bool shallow (shallow_fetch (co, url, cap, ref));
- strings to (timeout_opts (co, url.scheme));
-
- // Clone new submodule.
- //
- if (!cloned)
+ if (!initialized)
{
- if (!shallow)
- fetch_warn (cap, "repository", psdir);
-
- if (!run_git (co,
- to,
- co.git_option (),
- "-C", dir,
- "submodule--helper", "clone",
-
- "--name", name,
- "--path", sdir,
- "--url", to_git_url (url),
- shallow
- ? cstrings ({"--depth", "1"})
- : cstrings (),
- verb < 1 ? opt ("-q") : nullopt))
- fail << "unable to clone submodule '" << psd << "'" << endg;
+ mk_p (gdir);
+ init (co, fsdir, url, gdir);
}
- update_tree (co, fsdir, psdir, ref, cap, shallow, to);
+ // Fetch and checkout the submodule.
+ //
+ git_ref_filters rfs {git_ref_filter {nullopt, commit}};
+ fetch (co, fsdir, psdir, rfs);
+
+ git_checkout (co, fsdir, commit);
- // Not quite a checkout, but let's make the message match the
- // git-submodule script output (again, except for capitalization).
+ // Let's make the message match the git-submodule script output
+ // (again, except for capitalization).
//
- if (verb > 0)
+ if (verb)
text << "submodule path '" << psd << "': checked out '" << commit
<< "'";
- // Recurse.
+ // Check out the submodule submodules, recursively.
//
- // Can throw the failed exception that we don't catch here, relying on
- // the fact that the process destructor will wait for the process
- // completion.
- //
- update_submodules (co, fsdir, psdir);
+ checkout_submodules (co, fsdir, gdir, psdir);
}
is.close ();
@@ -918,169 +1328,107 @@ namespace bpkg
failure ("unable to list submodules");
}
- // Extract the git reference from the repository URL fragment. Set the URL
- // fragment to nullopt.
- //
- static git_reference
- parse_reference (repository_url& url, const char* what)
+ void
+ git_init (const common_options& co,
+ const repository_location& rl,
+ const dir_path& dir)
+ {
+ repository_url url (rl.url ());
+ url.fragment = nullopt;
+
+ init (co, dir, url);
+ }
+
+ strings
+ git_fetch (const common_options& co,
+ const repository_location& rl,
+ const dir_path& dir)
{
+ git_ref_filters refs;
+ repository_url url (rl.url ());
+
+ if (url.fragment)
try
{
- git_reference r (git_reference (url.fragment));
+ refs.emplace_back (*url.fragment);
url.fragment = nullopt;
- return r;
}
catch (const invalid_argument& e)
{
- fail << "unable to " << what << ' ' << url << ": " << e << endf;
+ fail << "unable to fetch " << url << ": " << e;
}
- }
-
- // Produce a repository directory name for the specified git reference.
- //
- // Truncate commit id-based directory names to shorten absolute directory
- // paths, lowering the probability of hitting the limit on Windows.
- //
- // Note that we can't truncate them for branches/tags as chances to clash
- // would be way higher than for commit ids. Though such names are normally
- // short anyway.
- //
- static inline dir_path
- repository_dir (const git_reference& ref)
- {
- return dir_path (ref.commit ? ref.commit->substr (0, 16) : *ref.branch);
- }
-
- dir_path
- git_clone (const common_options& co,
- const repository_location& rl,
- const dir_path& destdir)
- {
- repository_url url (rl.url ());
- git_reference ref (parse_reference (url, "clone"));
- // All protocols support single branch cloning, so we will always be
- // cloning a single branch if the branch is specified.
+ // Update the repository URL, if changed.
//
- bool single_branch (ref.branch);
- capabilities cap (sense_capabilities (co, url));
- bool shallow (shallow_fetch (co, url, cap, ref));
-
- if (shallow)
- single_branch = false; // Is implied for shallow cloning.
- else
- fetch_warn (cap, single_branch ? "branch" : "repository");
-
- dir_path r (repository_dir (ref));
- dir_path d (destdir / r);
+ repository_url u (origin_url (co, dir));
- strings to (timeout_opts (co, url.scheme));
-
- if (!run_git (
- co,
- to,
- "-c", "advice.detachedHead=false",
- co.git_option (),
- "clone",
-
- ref.branch ? strings ({"--branch", *ref.branch}) : strings (),
- single_branch ? opt ("--single-branch") : nullopt,
- shallow ? strings ({"--depth", "1"}) : strings (),
- ref.commit ? opt ("--no-checkout") : nullopt,
+ if (url != u)
+ {
+ // Note that the repository canonical name can not change under the
+ // legal scenarios that lead to the location change. Changed canonical
+ // name means that the repository was manually amended. We could fix-up
+ // such repositories as well but want to leave the backdoor for tests.
+ //
+ u.fragment = rl.url ().fragment; // Restore the fragment.
+ repository_location l (u, rl.type ());
- verb < 1 ? opt ("-q") : verb > 3 ? opt ("-v") : nullopt,
- to_git_url (url),
- d))
- fail << "unable to clone " << url << endg;
+ if (rl.canonical_name () == l.canonical_name ())
+ {
+ if (verb)
+ info << "location changed for " << rl.canonical_name () <<
+ info << "new location " << rl <<
+ info << "old location " << l;
- if (ref.commit)
- update_tree (co, d, dir_path (), ref, cap, shallow, to);
+ origin_url (co, dir, url);
+ }
+ }
- update_submodules (co, d, dir_path ());
- return r;
+ return fetch (co, dir, dir_path () /* submodule */, refs);
}
- dir_path
- git_fetch (const common_options& co,
- const repository_location& rl,
- const dir_path& destdir)
+ void
+ git_checkout (const common_options& co,
+ const dir_path& dir,
+ const string& commit)
{
- repository_url url (rl.url ());
- git_reference ref (parse_reference (url, "fetch"));
-
- dir_path r (repository_dir (ref));
-
- // Fetch is noop if the specific commit is checked out.
- //
- // What if the user replaces the repository URL with a one with a new
- // branch/tag/commit? These are not part of the repository name which
- // means such a repository will have the same hash. But then when we
- // remove the repository, we will also clean up its state. So seems like
- // this should work correctly automatically.
+ // For some (probably valid) reason the hard reset command doesn't remove
+ // a submodule directory that is not plugged into the project anymore. It
+ // also prints the non-suppressible warning like this:
//
- if (ref.commit)
- return r;
-
- assert (ref.branch);
-
- dir_path d (destdir);
- d /= dir_path (*ref.branch);
-
- // If the repository location differs from the one that was used to clone
- // the repository then we re-clone it from the new location.
+ // warning: unable to rmdir libbar: Directory not empty
//
- // Another (more hairy) way of doing this would be fixing up the remote
- // origin URLs recursively prior to fetching.
+ // That's why we run the clean command afterwards. It may also be helpful
+ // if we produce any untracked files in the tree between checkouts down
+ // the road.
//
- try
- {
- repository_url u (from_git_url (git_string (co, "remote repository URL",
- co.git_option (),
- "-C", d,
- "config",
- "--get",
- "remote.origin.url")));
- if (u != url)
- {
- // Note that the repository canonical name can not change under the
- // legal scenarios that lead to the location change. Changed canonical
- // name means that the repository was manually amended. We could
- // re-clone such repositories as well but want to leave the backdoor
- // for tests.
- //
- u.fragment = rl.url ().fragment; // Restore the fragment.
- repository_location l (u, rl.type ());
-
- if (rl.canonical_name () == l.canonical_name ())
- {
- if (verb)
- info << "re-cloning " << rl.canonical_name ()
- << " due to location change" <<
- info << "new location " << rl <<
- info << "old location " << l;
-
- rm_r (d);
- return git_clone (co, rl, destdir);
- }
- }
- }
- catch (const invalid_argument& e)
- {
- fail << "invalid remote.origin.url configuration value: " << e << endg;
- }
-
- capabilities cap (sense_capabilities (co, url));
- bool shallow (shallow_fetch (co, url, cap, ref));
+ if (!run_git (
+ co,
+ co.git_option (),
+ "-C", dir,
+ "reset",
+ "--hard",
+ verb < 2 ? opt ("-q") : nullopt,
+ commit))
+ fail << "unable to reset to " << commit << endg;
- update_tree (co,
- d,
- dir_path (),
- ref,
- cap,
- shallow,
- timeout_opts (co, url.scheme));
+ if (!run_git (
+ co,
+ co.git_option (),
+ "-C", dir,
+ "clean",
+ "-d",
+ "-x",
+ "-ff",
+ verb < 2 ? opt ("-q") : nullopt))
+ fail << "unable to clean " << dir << endg;
+ }
- update_submodules (co, d, dir_path ());
- return r;
+ void
+ git_checkout_submodules (const common_options& co, const dir_path& dir)
+ {
+ checkout_submodules (co,
+ dir,
+ dir / dir_path (".git"),
+ dir_path () /* prefix */);
}
}
diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx
index e784cb6..9530455 100644
--- a/bpkg/fetch.hxx
+++ b/bpkg/fetch.hxx
@@ -49,21 +49,41 @@ namespace bpkg
// Repository type git (fetch-git.cxx).
//
- // Clone git repository into destdir/<name>/. Return the cloned repository
- // directory name that was deduced from the repository URL fragment.
+ // Create a git repository in the specified directory and prepare it for
+ // fetching from the specified repository location. Note that the repository
+ // URL fragment is neither used nor validated.
//
- dir_path
- git_clone (const common_options&,
- const repository_location&,
- const dir_path& destdir);
-
- // Fetch git repository in destdir/<name>/. Return the fetched repository
- // directory name that was deduced from the repository URL fragment.
+ void
+ git_init (const common_options&,
+ const repository_location&,
+ const dir_path&);
+
+ // Fetch a git repository in the specifid directory (previously created by
+ // git_init()) for the references obtained from the repository URL fragment
+ // returning commit ids these references resolve to. After fetching the
+ // repository working tree state is unspecified (see git_checkout ()).
//
- dir_path
+ // Note that submodules are not fetched.
+ //
+ strings
git_fetch (const common_options&,
const repository_location&,
- const dir_path& destdir);
+ const dir_path&);
+
+ // Checkout the specified commit previously fetched by git_fetch().
+ //
+ // Note that submodules are not checked out.
+ //
+ void
+ git_checkout (const common_options&,
+ const dir_path&,
+ const string& commit);
+
+ // Fetch (if necessary) and checkout submodules, recursively, in a working
+ // tree previously checked out by git_checkout().
+ //
+ void
+ git_checkout_submodules (const common_options&, const dir_path&);
// Low-level fetch API (fetch.cxx).
//
diff --git a/bpkg/package.cxx b/bpkg/package.cxx
index 08e71f9..2880368 100644
--- a/bpkg/package.cxx
+++ b/bpkg/package.cxx
@@ -65,6 +65,12 @@ namespace bpkg
const auto& ps (r->prerequisites);
const auto& cs (r->complements);
+ // @@ The same repository can be present in the location set multiple times
+ // with different fragment values. Given that we may traverse the same
+ // repository tree multiple times, which is inefficient but harmless.
+ // Let's leave it this way for now as it likely to be changed with
+ // adding support for repository fragment objects.
+ //
for (const package_location& pl: ap->locations)
{
const lazy_shared_ptr<repository>& lr (pl.repository);
diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index f0bb6fd..5b0db8c 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -935,7 +935,7 @@ namespace bpkg
// Return a list of packages available from this repository.
//
- #pragma db view object(repository) \
+ #pragma db view object(repository) query(distinct) \
table("available_package_locations" = "pl" inner: \
"pl.repository = " + repository::name) \
object(available_package = package inner: \
@@ -956,7 +956,7 @@ namespace bpkg
// Return a list of repositories the packages come from.
//
- #pragma db view object(repository) \
+ #pragma db view object(repository) query(distinct) \
table("available_package_locations" = "pl" inner: \
"pl.repository = " + repository::name) \
object(available_package = package inner: \
diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx
index 6524730..1759991 100644
--- a/bpkg/pkg-checkout.cxx
+++ b/bpkg/pkg-checkout.cxx
@@ -6,6 +6,7 @@
#include <libbpkg/manifest.hxx>
+#include <bpkg/fetch.hxx> // git_checkout*()
#include <bpkg/package.hxx>
#include <bpkg/package-odb.hxx>
#include <bpkg/database.hxx>
@@ -22,6 +23,38 @@ using namespace butl;
namespace bpkg
{
+ static void
+ checkout (const common_options& o,
+ const repository_location& rl,
+ const dir_path& dir,
+ const string& fragment,
+ const shared_ptr<available_package>& ap)
+ {
+ switch (rl.type ())
+ {
+ case repository_type::git:
+ {
+ git_checkout (o, dir, fragment);
+
+ if (exists (dir / path (".gitmodules")))
+ {
+ // Print the progress indicator to attribute the possible fetching
+ // progress.
+ //
+ if (verb)
+ text << "checking out "
+ << package_string (ap->id.name, ap->version);
+
+ git_checkout_submodules (o, dir);
+ }
+
+ break;
+ }
+ case repository_type::pkg:
+ case repository_type::dir: assert (false); break;
+ }
+ }
+
shared_ptr<selected_package>
pkg_checkout (const common_options& o,
const dir_path& c,
@@ -106,10 +139,10 @@ namespace bpkg
// working tree so all we need to do is distribute it to the package
// directory.
//
- dir_path sd (c / repos_dir);
+ dir_path sd (c / repos_dir / repository_state (rl));
+
+ checkout (o, rl, sd, pl->fragment, ap);
- sd /= repository_state (rl);
- sd /= dir_path (pl->fragment);
sd /= path_cast<dir_path> (pl->location);
// Verify the package prerequisites are all configured since the dist
diff --git a/bpkg/rep-add.cli b/bpkg/rep-add.cli
index fc69f77..a5ede84 100644
--- a/bpkg/rep-add.cli
+++ b/bpkg/rep-add.cli
@@ -72,36 +72,45 @@ namespace bpkg
as there are commits. Practically, however, we are normally only
interested in a small subset of them while fetching and processing the
necessary information for all of them could be prohibitively expensive.
- As a result, a \cb{git} repository URL must include the fragment
- component that restricts the set of versions to consider as available.
- While in the future it will be possible to specify multiple available
- versions, currently the fragment must identify a single version using one
- of the following forms:
+ As a result, by default, only advertised tags (\cb{refs/tags/*}) and
+ branches (\cb{refs/heads/*}) are considered to be sources of useful
+ package versions (see \cb{git-ls-remote(1)} for details).
+
+ It is also possible to narrow down (or expand) the set of available
+ versions by specifying one or more references and/or commit ids in the
+ repository URL fragment. For example:
\
- #<tag>
- #<branch>
- #<commit-id>
- #<branch>@<commit-id>
+ https://example.com/foo.git#v1.2.3
+ https://example.com/foo.git#master
+ https://example.com/foo.git#af234f56
+ https://example.com/foo.git#HEAD,tags/v1.2.3,heads/feature1
\
- In the last form <branch> is only used to minimize the amount of data to
- fetch. If specified, then the commit must belong to its history.
+ A \cb{git} repository URL fragment must be a comma-separated list of
+ reference filters in the following form:
- No abbreviated commit ids are supported and <commit-id> must be the
- complete, 40-characters SHA1 string. A 40-characters fragment that
- consists of only hexadecimal digits is assumed to be a commit id and a
- branch/tag otherwise. In an unlikely case this produces an incorrect
- result, the last form with omitted <commit-id> can be used.
+ \c{[\i{refname}][\b{@}\i{commit}]}
- Below are some examples of \cb{git} repository URLs:
+ Either \ci{refname}, \ci{commit}, or both must be specified. If both are
+ specified then \ci{refname} is only used to minimize the amount of data
+ to fetch and \ci{commit} must belong to its history. For example:
\
- https://example.com/foo.git#v1.2.3
- https://example.com/foo.git#master
- https://example.com/foo.git#48fba3625d65941bb85a39061bcf795d4949c778
- https://example.com/foo.git#bar@48fba3625d65941bb85a39061bcf795d4949c778
- https://example.com/foo.git#deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@
+ .../foo.git#master@48fba3625d65941bb85a39061bcf795d4949c778
+ \
+
+ The \ci{refname} part can be a tag, branch, an abbreviated commit id, or
+ some other advertised reference under \cb{refs/}. While \ci{commit} must
+ be the complete, 40-characters SHA1 that need not be advertised. For
+ convenience, a 40-characters filter that consists of only hexadecimal
+ digits is assumed to be \ci{commit} even if not prefixed with \cb{@}. In
+ an unlikely event this produces an incorrect result, the \cb{@}-form with
+ omitted \ci{commit} can be used. For example:
+
+ \
+ .../foo.git#48fba3625d65941bb85a39061bcf795d4949c778 (commit id)
+ .../foo.git#deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@ (reference)
\
A \cb{git} repository has the same structure and manifest files as the
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 6d37f9f..26a1dec 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -281,45 +281,11 @@ namespace bpkg
const repository_location& rl,
bool ignore_unknown)
{
- // Plan:
- //
- // 1. Check repos_dir/<hash>/:
- //
- // 1.a If does not exist, git-clone into temp_dir/<hash>/<fragment>/.
- //
- // 1.a Otherwise, move as temp_dir/<hash>/ and git-fetch.
- //
- // 2. Move from temp_dir/<hash>/ to repos_dir/<hash>/<fragment>/
- //
- // 3. Check if repos_dir/<hash>/<fragment>/repositories.manifest exists:
- //
- // 3.a If exists, load.
- //
- // 3.b Otherwise, synthesize repository list with base repository.
- //
- // 4. Check if repos_dir/<hash>/<fragment>/packages.manifest exists:
- //
- // 4.a If exists, load. (into "skeleton" packages list to be filled?)
- //
- // 4.b Otherwise, synthesize as if single 'location: ./'.
- //
- // 5. For each package location obtained on step 4:
- //
- // 5.a Load repos_dir/<hash>/<fragment>/<location>/manifest.
- //
- // 5.b Run 'b info: repos_dir/<hash>/<fragment>/<location>/' and fix-up
- // package version.
- //
- // 6. Return repository and package manifests (certificate is NULL).
- //
-
if (conf != nullptr && conf->empty ())
conf = dir_exists (bpkg_dir) ? &current_dir : nullptr;
assert (conf == nullptr || !conf->empty ());
- // Clone or fetch the repository.
- //
dir_path sd (repository_state (rl));
auto_rmdir rm (temp_dir / sd);
@@ -329,17 +295,19 @@ namespace bpkg
rm_r (td);
// If the git repository directory already exists, then we are fetching
- // an already cloned repository. Move it to the temporary directory.
+ // an already existing repository, moved to the temporary directory first.
+ // Otherwise, we initialize the repository in the temporary directory.
//
- // In this case also set the filesystem_state_changed flag since we are
- // modifying the repository filesystem state.
+ // In the first case also set the filesystem_state_changed flag since we
+ // are modifying the repository filesystem state.
//
// In the future we can probably do something smarter about the flag,
// keeping it unset unless the repository state directory is really
// changed.
//
dir_path rd;
- bool fetch (false);
+ bool init (true);
+
if (conf != nullptr)
{
rd = *conf / repos_dir / sd;
@@ -348,34 +316,95 @@ namespace bpkg
{
mv (rd, td);
filesystem_state_changed = true;
- fetch = true;
+ init = false;
}
}
- dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td));
- dir_path fd (td / nm); // Full directory path.
+ // Initialize a new repository in the temporary directory.
+ //
+ if (init)
+ git_init (co, rl, td);
- // Parse manifests.
+ // Fetch the repository in the temporary directory.
//
- git_repository_manifests rms (
- parse_repository_manifests<git_repository_manifests> (
- fd / repositories_file,
- ignore_unknown,
- rl));
+ strings commits (git_fetch (co, rl, td));
+ assert (!commits.empty ());
- git_package_manifests pms (
- parse_directory_manifests<git_package_manifests> (
- fd / packages_file,
- ignore_unknown,
- rl));
+ // Go through fetched commits, checking them out and collecting the
+ // prerequisite repositories and packages.
+ //
+ // For each checked out commit:
+ //
+ // - If repositories.manifest file doesn't exist, then synthesize the
+ // repository list with just the base repository.
+ //
+ // - Save the repositories into the resulting list.
+ //
+ // @@ Currently we just save ones from the first commit, assuming them
+ // to be the same for others. However, this is not very practical
+ // and must be fixed.
+ //
+ // - If packages.manifest file exists then load it into the "skeleton"
+ // packages list. Otherwise, synthesize it with the single:
+ //
+ // location: ./
+ //
+ // - If any of the package locations point to non-existent directory, then
+ // assume it to be in a submodule and checkout submodules, recursively.
+ //
+ // - For each package location parse the package manifest and add it to
+ // the resulting list.
+ //
+ git_repository_manifests rms;
+ vector<rep_fetch_data::package> fps;
- vector<rep_fetch_data::package> fps (
- parse_package_manifests (co,
- fd,
- nm.string (),
- move (pms),
- ignore_unknown,
- rl));
+ for (const string& c: commits)
+ {
+ git_checkout (co, td, c);
+
+ // Parse repository manifests.
+ //
+ if (rms.empty ())
+ rms = parse_repository_manifests<git_repository_manifests> (
+ td / repositories_file,
+ ignore_unknown,
+ rl);
+
+ // Parse package skeleton manifests.
+ //
+ git_package_manifests pms (
+ parse_directory_manifests<git_package_manifests> (
+ td / packages_file,
+ ignore_unknown,
+ rl));
+
+ // Checkout submodules, if required.
+ //
+ for (const package_manifest& sm: pms)
+ {
+ dir_path d (td / path_cast<dir_path> (*sm.location));
+
+ if (!exists (d) || empty (d))
+ {
+ git_checkout_submodules (co, td);
+ break;
+ }
+ }
+
+ // Parse package manifests.
+ //
+ vector<rep_fetch_data::package> cps (
+ parse_package_manifests (co,
+ td,
+ c,
+ move (pms),
+ ignore_unknown,
+ rl));
+
+ fps.insert (fps.end (),
+ make_move_iterator (cps.begin ()),
+ make_move_iterator (cps.end ()));
+ }
// Move the state directory to its proper place.
//
@@ -389,7 +418,7 @@ namespace bpkg
filesystem_state_changed = true;
}
- return rep_fetch_data {move (rms), move (fps), nullptr};
+ return rep_fetch_data {move (rms), move (fps), nullptr /* certificate */};
}
rep_fetch_data
@@ -740,9 +769,8 @@ namespace bpkg
}
}
- // This repository shouldn't already be in the location set since
- // that would mean it has already been loaded and we shouldn't be
- // here.
+ // Note that the repository may already be present in the location set
+ // multiple times with different fragments.
//
p->locations.push_back (
package_location {lazy_shared_ptr<repository> (db, r),
@@ -799,7 +827,7 @@ namespace bpkg
rep_remove (conf, t, r);
// Finally, make sure that the external packages are available from a
- // single repository.
+ // single directory-based repository.
//
// Sort the packages by name and version. This way the external packages
// with the same upstream version and revision will be adjacent.
diff --git a/bpkg/rep-remove.cxx b/bpkg/rep-remove.cxx
index f161c71..fafe5a8 100644
--- a/bpkg/rep-remove.cxx
+++ b/bpkg/rep-remove.cxx
@@ -82,13 +82,12 @@ namespace bpkg
const shared_ptr<available_package>& p (rp);
vector<package_location>& ls (p->locations);
- for (auto i (ls.cbegin ()), e (ls.cend ()); i != e; ++i)
+ for (auto i (ls.cbegin ()); i != ls.cend (); )
{
if (i->repository.object_id () == name)
- {
- ls.erase (i);
- break;
- }
+ i = ls.erase (i);
+ else
+ ++i;
}
if (ls.empty ())
diff --git a/tests/buildfile b/tests/buildfile
index 4716658..d37b8f0 100644
--- a/tests/buildfile
+++ b/tests/buildfile
@@ -6,7 +6,7 @@ define common: file
common{*}: extension = test
commons = common config auth remote \
- remote-git rep-fetch-git rep-fetch-git-branch rep-fetch-git-commit
+ remote-git rep-fetch-git rep-fetch-git-refname rep-fetch-git-commit
# The common/ directory contains repositories that are reused, being symlinked
# in source repositories specific for testscripts.
diff --git a/tests/common/git/init b/tests/common/git/init
index 691799e..e80ebf4 100755
--- a/tests/common/git/init
+++ b/tests/common/git/init
@@ -37,8 +37,8 @@ if [ -n "$unpack" ]; then
done
fi
-# Create the initial state of the repositories libfoo.git, libbar.git,
-# style.git, and style-basic.git.
+# Create the initial state of the repositories libfoo.git, libfox.git,
+# libbar.git, style.git, and style-basic.git.
#
cd state0
@@ -48,6 +48,10 @@ rm -f libfoo.git/README
rm -f -r libfoo.git/libbar
rm -f -r libfoo.git/doc/style
+rm -f -r libfox.git/.git
+rm -f libfox.git/.gitmodules
+rm -f -r libfox.git/libbar
+
rm -f -r libbar.git/.git
rm -f -r style.git/.git
@@ -113,6 +117,14 @@ touch libfoo.git/README
git -C libfoo.git add README
git -C libfoo.git commit -am 'README'
+# Create master branch for libfox.git, adding libbar.git as a submodule.
+#
+git -C libfox.git init
+git -C libfox.git add '*'
+git -C libfox.git submodule add ../libbar.git libbar
+git -C libfox.git submodule update --init --recursive # Recursive for safety.
+git -C libfox.git commit -am 'Create'
+
# Create the modified state of the repositories, replacing libbar.git submodule
# of libfoo with the newly created libbaz.git repository. Also advance master
# branches and tags for libfoo.git and it's submodule style.git.
diff --git a/tests/common/git/pack b/tests/common/git/pack
index f53e794..f9d9772 100755
--- a/tests/common/git/pack
+++ b/tests/common/git/pack
@@ -13,8 +13,9 @@ set -o errtrace # Trap in functions.
function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }
-projects=('state0/libfoo' 'state0/libbar' 'state0/style' 'state0/style-basic' \
- 'state1/libfoo' 'state1/libbaz' 'state1/style' 'state1/style-basic')
+projects=(\
+ state0/libfoo state0/libfox state0/libbar state0/style state0/style-basic \
+ state1/libfoo state1/libfox state1/libbaz state1/style state1/style-basic)
for p in "${projects[@]}"; do
d=$p.git
diff --git a/tests/common/git/state0/libbar.tar b/tests/common/git/state0/libbar.tar
index bbb6f81..8be7c94 100644
--- a/tests/common/git/state0/libbar.tar
+++ b/tests/common/git/state0/libbar.tar
Binary files differ
diff --git a/tests/common/git/state0/libfoo.tar b/tests/common/git/state0/libfoo.tar
index 6f9af32..31b5519 100644
--- a/tests/common/git/state0/libfoo.tar
+++ b/tests/common/git/state0/libfoo.tar
Binary files differ
diff --git a/tests/common/git/state0/libfox.tar b/tests/common/git/state0/libfox.tar
new file mode 100644
index 0000000..b4037c5
--- /dev/null
+++ b/tests/common/git/state0/libfox.tar
Binary files differ
diff --git a/tests/common/git/state0/style-basic.tar b/tests/common/git/state0/style-basic.tar
index 6a7a1f2..80731dd 100644
--- a/tests/common/git/state0/style-basic.tar
+++ b/tests/common/git/state0/style-basic.tar
Binary files differ
diff --git a/tests/common/git/state0/style.tar b/tests/common/git/state0/style.tar
index 6295f34..1a88771 100644
--- a/tests/common/git/state0/style.tar
+++ b/tests/common/git/state0/style.tar
Binary files differ
diff --git a/tests/common/git/state1/libbaz.tar b/tests/common/git/state1/libbaz.tar
index ee8ad3e..298712c 100644
--- a/tests/common/git/state1/libbaz.tar
+++ b/tests/common/git/state1/libbaz.tar
Binary files differ
diff --git a/tests/common/git/state1/libfoo.tar b/tests/common/git/state1/libfoo.tar
index 4221cdd..2f2e3b4 100644
--- a/tests/common/git/state1/libfoo.tar
+++ b/tests/common/git/state1/libfoo.tar
Binary files differ
diff --git a/tests/common/git/state1/libfox.tar b/tests/common/git/state1/libfox.tar
new file mode 100644
index 0000000..d296973
--- /dev/null
+++ b/tests/common/git/state1/libfox.tar
Binary files differ
diff --git a/tests/common/git/state1/style-basic.tar b/tests/common/git/state1/style-basic.tar
index 8bc8b85..7391b66 100644
--- a/tests/common/git/state1/style-basic.tar
+++ b/tests/common/git/state1/style-basic.tar
Binary files differ
diff --git a/tests/common/git/state1/style.tar b/tests/common/git/state1/style.tar
index feedcaa..4be977b 100644
--- a/tests/common/git/state1/style.tar
+++ b/tests/common/git/state1/style.tar
Binary files differ
diff --git a/tests/rep-add.test b/tests/rep-add.test
index 8ee1b9d..9b52efd 100644
--- a/tests/rep-add.test
+++ b/tests/rep-add.test
@@ -45,13 +45,13 @@ rep_list += -d cfg
EOE
}
- : git-no-branch
+ : git-invalid-fragment
:
{
$clone_cfg;
- $* 'git://example.org/repo' 2>>EOE != 0
- error: invalid git repository location 'git://example.org/repo': missing branch/tag or commit id for git repository
+ $* 'git://example.org/repo#' 2>>EOE != 0
+ error: invalid git repository location 'git://example.org/repo#': missing reference name or commit id for git repository
EOE
}
@@ -108,8 +108,8 @@ rep_list += -d cfg
:
$clone_cfg && mkdir -p repo/.git;
- $* 'repo' 2>>~%EOE% != 0
- %error: invalid git repository location '.+repo': missing branch/tag or commit id for git repository%
+ $* 'repo' 2>>/~%EOE%
+ %added git:.+/repo%
EOE
: file-pkg
diff --git a/tests/rep-fetch-git-branch.test b/tests/rep-fetch-git-branch.test
deleted file mode 100644
index 6e23e20..0000000
--- a/tests/rep-fetch-git-branch.test
+++ /dev/null
@@ -1,156 +0,0 @@
-# file : tests/rep-fetch-git-branch.test
-# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
-# license : MIT; see accompanying LICENSE file
-
-reason_dumb = ' (dumb HTTP)'
-reason_unadv = ' (unadvertised commit)'
-
-warn_dumb='
-warning: fetching over dumb HTTP, no progress will be shown'
-
-: clone
-:
-{
- $clone_root_cfg && $rep_add "$rep/state0/libfoo.git#$branch";
-
- # Note that the commit for doc/style/basic submodule is not at the branch tip
- # and so is not advertised.
- #
- if ($git_protocol == 'local' || \
- $git_protocol == 'https-smart' || \
- $git_protocol == 'git')
- warn1 = '%.{0}'
- warn2 = '%.{0}'
- warn3 = "warning: fetching whole repository history for submodule 'doc/style/basic'$reason_unadv"
- fetch = '%.{0}'
- warn4 = '%.{0}'
- elif ($git_protocol == 'https-dumb')
- warn1 = "warning: fetching whole branch history$reason_dumb$warn_dumb"
- warn2 = "warning: fetching whole repository history for submodule 'doc/style'$reason_dumb$warn_dumb"
- warn3 = "warning: fetching whole repository history for submodule 'doc/style/basic'$reason_dumb$warn_dumb"
- fetch = '%.{0}'
- warn4 = "warning: fetching whole repository history for submodule 'libbar'$reason_dumb$warn_dumb"
- elif ($git_protocol == 'https-smart-unadv')
- warn1 = '%.{0}'
- warn2 = '%.{0}'
- warn3 = '%.{0}'
- fetch = "%fetching in '.+style/basic'.+%"
- warn4 = '%.{0}'
- end;
-
- $* 2>>~"%EOE%"
- %fetching git:.+libfoo#$branch%
- $warn1
- %Cloning into '.+$branch'.+%
- %Submodule 'doc/style' .+ registered for path 'doc/style'%
- %Submodule 'libbar' .+ registered for path 'libbar'%
- $warn2
- %Cloning into '.+doc/style'.+%
- %submodule path 'doc/style': checked out .+%
- %Submodule 'basic' .+ registered for path 'doc/style/basic'%
- $warn3
- %Cloning into '.+doc/style/basic'.+%
- $fetch
- %submodule path 'doc/style/basic': checked out .+%
- $warn4
- %Cloning into '.+libbar'.+%
- %submodule path 'libbar': checked out .+%
- 1 package\(s\) in 1 repository\(s\)
- EOE
-}
-
-: fetch
-:
-{
- : same
- :
- {
- $clone_root_cfg && $rep_add "$rep/state0/libfoo.git#$branch";
-
- # Convert specific warnings to infos as we expect them to appear. This, in
- # particular, prevents bbot workers to set task result status to warning.
- #
- $* 2>&1 | sed -e 's/warning: (fetching (over|whole) .*)/info: \1/' >&2 2>!;
-
- if ($git_protocol == 'https-dumb')
- warn = "warning: fetching whole branch history$reason_dumb$warn_dumb"
- else
- warn = '%.{0}'
- end;
-
- $* 2>>~"%EOE%"
- %fetching git:.+libfoo#$branch%
- $warn
- %fetching in '.+$branch'.+%
- 1 package\(s\) in 1 repository\(s\)
- EOE
- }
-
- : changed
- :
- {
- g = git -C
- u = "$rep_git/state1"
-
- $clone_root_cfg && $rep_add "$rep/state0/libfoo.git#$branch";
-
- # Extract the repository path from the output line like this:
- #
- # Cloning into 'cfg\.bpkg\tmp\f9be881264703b5d\master'...
- #
- $* 2>&1 | sed -n -e "s/Cloning into '\(.+$branch\)'\.{3}/\$1/p" | \
- sed -n -e 's%(.+[\\/])tmp([\\/].+)%$1repos$2%p' | \
- set r;
-
- $g "$r" config remote.origin.url "$u/libfoo.git";
-
- $g "$r" config submodule.libbar.url "$u/libbar.git";
- $g "$r" config submodule.doc/style.url "$u/style.git";
-
- $g "$r/libbar" config remote.origin.url "$u/libbar.git";
-
- $g "$r/doc/style" config remote.origin.url "$u/style.git";
- $g "$r/doc/style" config submodule.basic.url "$u/style-basic.git";
-
- $g "$r/doc/style/basic" config remote.origin.url "$u/style-basic.git";
-
- # Preconditions.
- #
- test -f $r/tests/TODO;
- test -f $r/libbar/libbar/manifest;
- test -f $r/doc/style/README != 0;
- test -d $r/libbaz != 0;
-
- if ($git_protocol == 'https-dumb')
- warn1 = "warning: fetching whole branch history$reason_dumb$warn_dumb"
- warn2 = "warning: fetching whole repository history for submodule 'doc/style'$reason_dumb$warn_dumb"
- warn3 = "warning: fetching whole repository history for submodule 'libbaz'$reason_dumb$warn_dumb"
- else
- warn1 = '%.{0}'
- warn2 = '%.{0}'
- warn3 = '%.{0}'
- end;
-
- $* 2>>~"%EOE%" 1>&2;
- %fetching git:.+libfoo#$branch%
- $warn1
- %fetching in '.+$branch'.+%
- %warning: unable to rmdir '?libbar'?:.+%
- %Submodule 'libbaz' .+ registered for path 'libbaz'%
- $warn2
- %fetching in '.+doc/style'.+%
- %submodule path 'doc/style': checked out .+%
- $warn3
- %Cloning into '.+libbaz'.+%
- %submodule path 'libbaz': checked out .+%
- 1 package\(s\) in 1 repository\(s\)
- EOE
-
- # Postconditions.
- #
- test -d $r/tests != 0;
- test -d $r/libbar != 0;
- test -f $r/doc/style/README;
- test -f $r/libbaz/manifest
- }
-}
diff --git a/tests/rep-fetch-git-commit.test b/tests/rep-fetch-git-commit.test
index 3169904..da08718 100644
--- a/tests/rep-fetch-git-commit.test
+++ b/tests/rep-fetch-git-commit.test
@@ -2,12 +2,6 @@
# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-reason_dumb = ' (dumb HTTP)'
-reason_unadv = ' (unadvertised commit)'
-
-warn_dumb='
-warning: fetching over dumb HTTP, no progress will be shown'
-
+git clone "$rep_git/state0/style-basic.git" 2>! &style-basic/***
: unadvertised
@@ -16,35 +10,22 @@ warning: fetching over dumb HTTP, no progress will be shown'
+git -C ../style-basic log '--pretty=format:%H' --all --grep='README' | \
set commit
- +git -C ../style-basic log '--pretty=format:%h' --all --grep='README' \
- --abbrev=16 | \
- set abbr_commit
-
: no-branch
:
{
$clone_root_cfg && $rep_add "$rep/state0/style-basic.git#@$commit";
- if ($git_protocol == 'https-smart-unadv')
- warn = '%.{0}'
- fetch = "%fetching in '.+$abbr_commit'.+%"
- else
- warn = "warning: fetching whole repository history"
-
- if ($git_protocol == 'https-dumb')
- warn = "$warn$reason_dumb$warn_dumb"
- else
- warn = "$warn$reason_unadv"
- end
-
- fetch = '%.{0}'
+ if ($git_protocol == 'https-dumb')
+ warn = "$warn_repo_hist$reason_dumb$warn_dumb"
+ elif ($git_protocol != 'https-smart-unadv')
+ warn = "$warn_repo_hist$reason_unadv"
end;
$* 2>>~"%EOE%"
%fetching git:.+style-basic#@$commit%
+ %querying .+style-basic\.git%?
$warn
- %Cloning into '.+$abbr_commit'.+%
- $fetch
+ %fetching from .+style-basic\.git%
1 package\(s\) in 1 repository\(s\)
EOE
}
@@ -54,26 +35,17 @@ warning: fetching over dumb HTTP, no progress will be shown'
{
$clone_root_cfg && $rep_add "$rep/state0/style-basic.git#stable@$commit";
- if ($git_protocol == 'https-smart-unadv')
- warn = '%.{0}'
- fetch = "%fetching in '.+$abbr_commit'.+%"
- else
- warn = "warning: fetching whole branch history"
-
- if ($git_protocol == 'https-dumb')
- warn = "$warn$reason_dumb$warn_dumb"
- else
- warn = "$warn$reason_unadv"
- end
-
- fetch = '%.{0}'
+ if ($git_protocol == 'https-dumb')
+ warn = "$warn_ref_hist$reason_dumb$warn_dumb"
+ elif ($git_protocol != 'https-smart-unadv')
+ warn = "$warn_ref_hist$reason_unadv"
end;
$* 2>>~"%EOE%"
%fetching git:.+style-basic#stable@$commit%
+ %querying .+style-basic\.git%?
$warn
- %Cloning into '.+$abbr_commit'.+%
- $fetch
+ %fetching from .+style-basic\.git%
1 package\(s\) in 1 repository\(s\)
EOE
}
@@ -85,24 +57,20 @@ warning: fetching over dumb HTTP, no progress will be shown'
+git -C ../style-basic log '--pretty=format:%H' --all --grep='INSTALL' | \
set commit
+ +if ($git_protocol == 'https-dumb')
+ warn = "$warn_ref_hist$reason_dumb$warn_dumb"
+ end
+
: no-branch
:
{
$clone_root_cfg && $rep_add "$rep/state0/style-basic.git#@$commit";
- if ($git_protocol == 'https-dumb')
- warn = "warning: fetching whole repository history$reason_dumb$warn_dumb"
- fetch = '%.{0}'
- else
- warn = '%.{0}'
- fetch = "%fetching in '.+$abbr_commit'.+%"
- end;
-
$* 2>>~"%EOE%"
%fetching git:.+style-basic#@$commit%
+ %querying .+style-basic\.git%?
$warn
- %Cloning into '.+$abbr_commit'.+%
- $fetch
+ %fetching from .+style-basic.\git%
1 package\(s\) in 1 repository\(s\)
EOE
}
@@ -112,16 +80,11 @@ warning: fetching over dumb HTTP, no progress will be shown'
{
$clone_root_cfg && $rep_add "$rep/state0/style-basic.git#stable@$commit";
- if ($git_protocol == 'https-dumb')
- warn ="warning: fetching whole branch history$reason_dumb$warn_dumb"
- else
- warn = '%.{0}'
- end;
-
$* 2>>~"%EOE%"
%fetching git:.+style-basic#stable@$commit%
+ %querying .+style-basic\.git%?
$warn
- %Cloning into '.+$abbr_commit'.+%
+ %fetching from .+style-basic\.git%
1 package\(s\) in 1 repository\(s\)
EOE
}
diff --git a/tests/rep-fetch-git-refname.test b/tests/rep-fetch-git-refname.test
new file mode 100644
index 0000000..ca1e0ff
--- /dev/null
+++ b/tests/rep-fetch-git-refname.test
@@ -0,0 +1,148 @@
+# file : tests/rep-fetch-git-refname.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+: fetch
+:
+{
+ $clone_root_cfg && $rep_add "$rep/state0/libfoo.git$fragment";
+
+ if ($git_protocol == 'https-dumb')
+ warn = "$warn_ref_hist$reason_dumb$warn_dumb"
+ end;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+libfoo$fragment%
+ %querying .+libfoo\.git%
+ $warn
+ %fetching from .+libfoo\.git%
+ 1 package\(s\) in 1 repository\(s\)
+ EOE
+}
+
+: re-fetch
+:
+{
+ : unchanged
+ :
+ {
+ $clone_root_cfg && $rep_add "$rep/state0/libfoo.git$fragment";
+
+ $* 2>!;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+libfoo$fragment%
+ %querying .+libfoo\.git%
+ 1 package\(s\) in 1 repository\(s\)
+ EOE
+ }
+
+ : changed
+ :
+ {
+ g = git -C
+ u = "$rep_git/state1"
+ d = cfg/libfoo-1.0.0
+
+ $clone_root_cfg && $rep_add "$rep/state0/libfoo.git$fragment";
+
+ # Extract the repository path from the output line like this:
+ #
+ # fetching from <url> in 'cfg/.bpkg/tmp/4bde15f59461'...
+ #
+ $* --verbose 2 2>&1 | \
+ sed -n -e "s/fetching from .+ in '\(.+\)'/\$1/p" | \
+ sed -n -e 's%(.+[\\/])tmp([\\/].+)%$1repos$2%p' | \
+ set r;
+
+ # Note that the commit for doc/style/basic submodule is not at the branch
+ # tip and so is not advertised.
+ #
+ warn1 = '%.{0}';
+ warn2 = '%.{0}';
+ warn3 = '%.{0}';
+
+ if ($git_protocol == 'local' || \
+ $git_protocol == 'https-smart' || \
+ $git_protocol == 'git')
+ warn2 = "$warn_repo_hist for submodule 'doc/style/basic'$reason_unadv"
+ elif ($git_protocol == 'https-dumb')
+ warn1 = "$warn_ref_hist for submodule 'doc/style'$reason_dumb$warn_dumb"
+ warn2 = "$warn_repo_hist for submodule 'doc/style/basic'$reason_dumb$warn_dumb"
+ warn3 = "$warn_ref_hist for submodule 'libbar'$reason_dumb$warn_dumb"
+ end;
+
+ $pkg_checkout libfoo/1.0.0 2>>~"%EOE%";
+ checking out libfoo/1.0.0
+ %querying .+style\.git%?
+ $warn1
+ %fetching submodule 'doc/style' from .+style\.git%
+ %submodule path 'doc/style': checked out .+%
+ %querying .+style-basic\.git%?
+ $warn2
+ %fetching submodule 'doc/style/basic' from .+style-basic\.git%
+ %submodule path 'doc/style/basic': checked out .+%
+ %querying .+libbar\.git%?
+ $warn3
+ %fetching submodule 'libbar' from .+libbar\.git%
+ %submodule path 'libbar': checked out .+%
+ distributing libfoo/1.0.0
+ checked out libfoo/1.0.0
+ EOE
+
+ $g "$r" config remote.origin.url "$u/libfoo.git";
+
+ # Preconditions.
+ #
+ test -f $d/tests/TODO;
+ test -f $d/libbar/libbar/manifest;
+ test -f $d/doc/style/README != 0;
+ test -d $d/libbaz != 0;
+
+ if ($git_protocol == 'https-dumb')
+ warn = "$warn_ref_hist$reason_dumb$warn_dumb"
+ end;
+
+ $* 2>>~"%EOE%" 1>&2;
+ %fetching git:.+libfoo$fragment%
+ %querying .+libfoo\.git%
+ $warn
+ %fetching from .+libfoo\.git%
+ %warning: unable to rmdir .?libbar.?: .+%
+ 1 package\(s\) in 1 repository\(s\)
+ EOE
+
+ $pkg_drop libfoo;
+
+ warn1 = '%.{0}';
+ warn2 = '%.{0}';
+
+ if ($git_protocol == 'https-dumb')
+ warn1 = "$warn_ref_hist for submodule 'doc/style'$reason_dumb$warn_dumb"
+ warn2 = "$warn_ref_hist for submodule 'libbaz'$reason_dumb$warn_dumb"
+ end;
+
+ $pkg_checkout libfoo/1.0.0 2>>~"%EOE%";
+ checking out libfoo/1.0.0
+ %querying .+style\.git%?
+ $warn1
+ %fetching submodule 'doc/style' from .+style\.git%
+ %submodule path 'doc/style': checked out .+%
+ %querying .+libbaz\.git%?
+ $warn2
+ %fetching submodule 'libbaz' from .+libbaz\.git%
+ %submodule path 'libbaz': checked out .+%
+ distributing libfoo/1.0.0
+ checked out libfoo/1.0.0
+ EOE
+
+ # Postconditions.
+ #
+ test -d $d/tests != 0;
+ test -d $d/libbar != 0;
+ test -f $d/doc/style/README;
+ test -f $d/libbaz/manifest;
+
+ $pkg_drop libfoo
+ }
+}
diff --git a/tests/rep-fetch-git.test b/tests/rep-fetch-git.test
index 1dbea3f..4b2a597 100644
--- a/tests/rep-fetch-git.test
+++ b/tests/rep-fetch-git.test
@@ -6,6 +6,17 @@
# the final states. See tests/common/git/init script for more details.
#
+reason_dumb = ' (dumb HTTP)'
+reason_unadv = ' (unadvertised commit)'
+
+warn_dumb='
+warning: fetching over dumb HTTP, no progress will be shown'
+
+warn_repo_hist = "warning: fetching whole repository history"
+warn_ref_hist = "warning: fetching whole reference history"
+
+warn = '%.{0}' # None.
+
test.cleanups += &cfg/.bpkg/repos/*/***
+if ($git_protocol == 'local')
@@ -33,25 +44,32 @@ else
rep_git = "$rep"
end
-: branch
+: none
+:
+{
+ fragment = ''
+ .include rep-fetch-git-refname.test
+}
+
+: refname
:
{
- branch = 'master'
- .include rep-fetch-git-branch.test
+ refname = '#master'
+ .include rep-fetch-git-refname.test
}
: ltag
:
{
- branch = 'ltag'
- .include rep-fetch-git-branch.test
+ fragment = '#ltag'
+ .include rep-fetch-git-refname.test
}
: atag
:
{
- branch = 'atag'
- .include rep-fetch-git-branch.test
+ fragment = '#atag'
+ .include rep-fetch-git-refname.test
}
: commit
diff --git a/tests/rep-fetch.test b/tests/rep-fetch.test
index db0f801..5028bf9 100644
--- a/tests/rep-fetch.test
+++ b/tests/rep-fetch.test
@@ -99,6 +99,7 @@
# Create git repositories.
#
$git_extract $src/git/state0/libfoo.tar
+ $git_extract $src/git/state0/libfox.tar
$git_extract $src/git/state0/libbar.tar
$git_extract $src/git/state0/style.tar
$git_extract $src/git/state0/style-basic.tar &$out_git/state0/***
@@ -115,8 +116,9 @@ rep_remove += -d cfg 2>!
pkg_status += -d cfg
pkg_fetch += -d cfg 2>!
pkg_unpack += -d cfg 2>!
-pkg_checkout += -d cfg 2>!
+pkg_checkout += -d cfg
pkg_purge += -d cfg
+pkg_drop += -d cfg --yes 2>!
: no-repos
:
@@ -514,15 +516,14 @@ if ($remote != true)
$clone_root_cfg;
+ $* "$rep/style.git#master" 2>! &cfg/.bpkg/repos/*/***;
+
# Convert specific warnings to infos as we expect them to appear. This,
# in particular, prevents bbot workers to set task result status to
# warning.
#
- $* "$rep/style.git#master" 2>&1 | \
- sed -e 's/warning: (fetching (over|whole) .*)/info: \1/' >&2 2>! \
- &cfg/.bpkg/repos/*/***;
-
- $pkg_checkout "style/1.0.0" 2>!;
+ $pkg_checkout "style/1.0.0" 2>&1 | \
+ sed -e 's/warning: (fetching (over|whole) .*)/info: \1/' >&2 2>!;
$rep_add $rep/style.git;
$* 2>!;
@@ -598,9 +599,31 @@ else
}
}
- : re-clone
+ : checkout-submodules
+ :
+ : Test that if some of the packages are located in the repository
+ : submodules directory, then rep-fetch checks out submoodules.
+ :
+ {
+ $clone_root_cfg && $rep_add "$rep_git/state0/libfox.git#master";
+
+ $* 2>>~%EOE% &cfg/.bpkg/repos/*/***
+ %fetching git:.+libfox#master%
+ %querying .+libfox\.git%
+ %.{0,2}
+ %fetching from .+libfox\.git%
+ %querying .+libbar\.git%?
+ %.{0,2}
+ %fetching submodule 'libbar' from .+libbar\.git%
+ %submodule path 'libbar': checked out '.+'%
+ 2 package(s) in 1 repository(s)
+ EOE
+
+ }
+
+ : re-fetching
:
- : Test that repository is re-cloned on the location change. Here this happens
+ : Test that repository is re-fetched on the location change. Here it happens
: due to the scheme change.
:
if ($remote == true)
@@ -609,14 +632,13 @@ else
$* 2>! &cfg/.bpkg/repos/*/***;
$rep_add "$rep_git_https_smart_unadv/state0/libfoo.git#master";
- $* 2>>~"%EOE%"
+ $* 2>>~%EOE%
%fetching git:.+libfoo#master%
- %info: re-cloning git:.+libfoo#master due to location change%
+ %info: location changed for git:.+libfoo#master%
% info: new location https://.+libfoo.git#master%
% info: old location git://.+libfoo.git#master%
- %.+
- %Submodule 'libbar' .+https://.+libbar.+%
- %.+
+ %querying .+libfoo\.git%
+ 1 package(s) in 1 repository(s)
EOE
}
@@ -630,11 +652,12 @@ else
# Extract the repository path from the output line like this:
#
- # Cloning into 'cfg\.bpkg\tmp\f9be881264703b5d\master'...
+ # fetching from <url> in 'cfg/.bpkg/tmp/4bde15f59461'...
#
- $* 2>&1 | sed -n -e "s/Cloning into '\(.+master\)'\.{3}/\$1/p" | \
- sed -n -e 's%(.+[\\/])tmp([\\/].+)%$1repos$2%p' | \
- set r;
+ $* --verbose 2 2>&1 | \
+ sed -n -e "s/fetching from .+ in '\(.+\)'/\$1/p" | \
+ sed -n -e 's%(.+[\\/])tmp([\\/].+)%$1repos$2%p' | \
+ set r;
# Break the repo (drop the remote repository URL) and try to re-fetch.
#