aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2018-02-12 17:30:16 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2018-02-12 17:34:55 +0300
commitb7763416f8a1e4940a10336d3a8b9fbbb879f414 (patch)
tree31f1bc7427181aa81984a639d998ca4e99058199
parentecb1efeebaa5597bee4cfdaab7bff4007b73127d (diff)
Clone and fetch git repositories
-rw-r--r--bpkg/auth.cxx11
-rw-r--r--bpkg/cfg-create.cxx8
-rw-r--r--bpkg/common.cli41
-rw-r--r--bpkg/fetch-bpkg.cxx270
-rw-r--r--bpkg/fetch-git.cxx986
-rw-r--r--bpkg/fetch.cxx263
-rw-r--r--bpkg/fetch.hxx64
-rw-r--r--bpkg/package.hxx24
-rw-r--r--bpkg/pkg-fetch.cxx4
-rw-r--r--bpkg/rep-create.cxx4
-rw-r--r--bpkg/rep-fetch.cxx110
-rw-r--r--bpkg/utility.cxx39
-rw-r--r--bpkg/utility.hxx11
-rw-r--r--tests/buildfile3
-rw-r--r--tests/common/git/README104
-rwxr-xr-xtests/common/git/init171
-rwxr-xr-xtests/common/git/pack29
-rw-r--r--tests/common/git/state0/libbar.tarbin0 -> 61440 bytes
-rw-r--r--tests/common/git/state0/libfoo.tarbin0 -> 256000 bytes
-rw-r--r--tests/common/git/state0/style-basic.tarbin0 -> 61440 bytes
-rw-r--r--tests/common/git/state0/style.tarbin0 -> 112640 bytes
-rw-r--r--tests/common/git/state1/libbaz.tarbin0 -> 61440 bytes
-rw-r--r--tests/common/git/state1/libfoo.tarbin0 -> 337920 bytes
-rw-r--r--tests/common/git/state1/style-basic.tarbin0 -> 61440 bytes
-rw-r--r--tests/common/git/state1/style.tarbin0 -> 112640 bytes
-rw-r--r--tests/common/libfoo-1.1.0/build/bootstrap.build2
-rw-r--r--tests/pkg-clean.test3
-rw-r--r--tests/pkg-configure.test3
-rw-r--r--tests/pkg-fetch.test3
-rw-r--r--tests/pkg-unpack.test2
-rw-r--r--tests/pkg-update.test3
-rwxr-xr-xtests/publish70
-rw-r--r--tests/remote-git.test35
-rw-r--r--tests/remote.test2
-rw-r--r--tests/rep-add.test28
-rw-r--r--tests/rep-auth.test9
-rw-r--r--tests/rep-fetch-git-branch.test152
-rw-r--r--tests/rep-fetch-git-commit.test124
-rw-r--r--tests/rep-fetch-git.test62
-rw-r--r--tests/rep-fetch.test192
l---------tests/rep-fetch/git/state01
l---------tests/rep-fetch/git/state11
42 files changed, 2428 insertions, 406 deletions
diff --git a/bpkg/auth.cxx b/bpkg/auth.cxx
index b60a8ee..06555e2 100644
--- a/bpkg/auth.cxx
+++ b/bpkg/auth.cxx
@@ -68,7 +68,8 @@ namespace bpkg
// use the location rather than the name prefix.
//
if (rl.remote ())
- return repository_location (p.posix_string (), rl).canonical_name ();
+ return repository_location (
+ repository_url (p.posix_string ()), rl).canonical_name ();
else
return (path_cast<dir_path> (rl.path ()) / p).normalize ().string ();
}
@@ -555,11 +556,7 @@ namespace bpkg
//
if (pem)
{
- dir_path d (conf / certs_dir);
- if (!dir_exists (d))
- mk (d);
-
- path f (d / path (fp + ".pem"));
+ path f (conf / certs_dir / path (fp + ".pem"));
try
{
@@ -576,8 +573,6 @@ namespace bpkg
return cert;
}
- static const dir_path current_dir (".");
-
shared_ptr<const certificate>
authenticate_certificate (const common_options& co,
const dir_path* conf,
diff --git a/bpkg/cfg-create.cxx b/bpkg/cfg-create.cxx
index acd6bb9..1a21cfa 100644
--- a/bpkg/cfg-create.cxx
+++ b/bpkg/cfg-create.cxx
@@ -83,9 +83,13 @@ namespace bpkg
true,
vars);
- // Create .bpkg/.
+ // Create .bpkg/ and its subdirectories.
//
- mk (c / bpkg_dir);
+ {
+ mk (c / bpkg_dir);
+ mk (c / certs_dir);
+ mk (c / repos_dir);
+ }
// Initialize tmp directory.
//
diff --git a/bpkg/common.cli b/bpkg/common.cli
index 45d2c55..5c3de83 100644
--- a/bpkg/common.cli
+++ b/bpkg/common.cli
@@ -122,13 +122,20 @@ namespace bpkg
size_t --fetch-timeout
{
"<sec>",
- "The fetch program timeout. While the exact semantics of the value
- depends on the fetch program used, at a minimum it specifies in
- seconds the maximum time that can be spent without any network
- activity. Specifically, it is translated to the \cb{--max-time}
- option for \cb{curl} and to the \cb{--timeout} option for \cb{wget}
- and \cb{fetch}. See \cb{--fetch} for more information on the fetch
- program."
+ "The fetch and fetch-like (for example, \cb{git}) program timeout.
+ While the exact semantics of the value depends on the program used,
+ at a minimum it specifies in seconds the maximum time that can be
+ spent without any network activity.
+
+ Specifically, it is translated to the \cb{--max-time} option for
+ \cb{curl} and to the \cb{--timeout} option for \cb{wget} and
+ \cb{fetch}. For \cb{git} over HTTP/HTTPS this semantics is achieved
+ using the \cb{http.lowSpeedLimit}=\i{1} \cb{http.lowSpeedTime}=\i{sec}
+ configuration values (the \cb{git://} protocol currently does not
+ support timeouts).
+
+ See \cb{--fetch} and \cb{--git} for more information on the fetch
+ programs."
}
strings --fetch-option
@@ -139,6 +146,26 @@ namespace bpkg
specify multiple fetch options."
}
+ path --git = "git"
+ {
+ "<path>",
+ "The git program to be used to fetch git repositories. You can also
+ specify additional options that should be passed to the git program with
+ \cb{--git-option}.
+
+ If the git program is not explicitly specified, then \cb{bpkg} will use
+ \cb{git} by default."
+ }
+
+ strings --git-option
+ {
+ "<opt>",
+ "Additional common option to be passed to the git program. Note that
+ the common options are the ones that precede the \cb{git} command.
+ See \cb{--git} for more information on the git program. Repeat this
+ option to specify multiple git options."
+ }
+
path --sha256
{
"<path>",
diff --git a/bpkg/fetch-bpkg.cxx b/bpkg/fetch-bpkg.cxx
new file mode 100644
index 0000000..39f84f9
--- /dev/null
+++ b/bpkg/fetch-bpkg.cxx
@@ -0,0 +1,270 @@
+// file : bpkg/fetch-bpkg.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <bpkg/fetch.hxx>
+
+#include <sstream>
+
+#include <libbutl/fdstream.mxx>
+#include <libbutl/filesystem.mxx> // cpfile ()
+#include <libbutl/manifest-parser.mxx>
+
+#include <bpkg/checksum.hxx>
+#include <bpkg/diagnostics.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace bpkg
+{
+ template <typename M>
+ static pair<M, string/*checksum*/>
+ fetch_manifest (const common_options& o,
+ const repository_url& u,
+ bool ignore_unknown)
+ {
+ string url (u.string ());
+ process pr (start_fetch (o, url));
+
+ try
+ {
+ // Unfortunately we cannot read from the original source twice as we do
+ // below for files. There doesn't seem to be anything better than reading
+ // the entire file into memory and then streaming it twice, once to
+ // calculate the checksum and the second time to actually parse. We need
+ // to read the original stream in the binary mode for the checksum
+ // calculation, then use the binary data to create the text stream for
+ // the manifest parsing.
+ //
+ ifdstream is (move (pr.in_ofd), fdstream_mode::binary);
+ stringstream bs (ios::in | ios::out | ios::binary);
+
+ // Note that the eof check is important: if the stream is at eof, write
+ // will fail.
+ //
+ if (is.peek () != ifdstream::traits_type::eof ())
+ bs << is.rdbuf ();
+
+ is.close ();
+
+ string s (bs.str ());
+ string sha256sum (sha256 (s.c_str (), s.size ()));
+
+ istringstream ts (s); // Text mode.
+
+ manifest_parser mp (ts, url);
+ M m (mp, ignore_unknown);
+
+ if (pr.wait ())
+ return make_pair (move (m), move (sha256sum));
+
+ // Child existed with an error, fall through.
+ }
+ // Ignore these exceptions if the child process exited with
+ // an error status since that's the source of the failure.
+ //
+ catch (const manifest_parsing& e)
+ {
+ if (pr.wait ())
+ fail (e.name, e.line, e.column) << e.description;
+ }
+ catch (const io_error&)
+ {
+ if (pr.wait ())
+ fail << "unable to read fetched " << url;
+ }
+
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
+
+ // While it is reasonable to assuming the child process issued
+ // diagnostics, some may not mention the URL.
+ //
+ fail << "unable to fetch " << url <<
+ info << "re-run with -v for more information" << endf;
+ }
+
+ static path
+ fetch_file (const common_options& o,
+ const repository_url& u,
+ const dir_path& d)
+ {
+ path r (d / u.path->leaf ());
+
+ if (exists (r))
+ fail << "file " << r << " already exists";
+
+ auto_rmfile arm (r);
+ process pr (start_fetch (o, u.string (), r));
+
+ if (!pr.wait ())
+ {
+ // While it is reasonable to assuming the child process issued
+ // diagnostics, some may not mention the URL.
+ //
+ fail << "unable to fetch " << u <<
+ info << "re-run with -v for more information";
+ }
+
+ arm.cancel ();
+ return r;
+ }
+
+ static path
+ fetch_file (const path& f, const dir_path& d)
+ {
+ path r (d / f.leaf ());
+
+ try
+ {
+ cpfile (f, r);
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to copy " << f << " to " << r << ": " << e;
+ }
+
+ return r;
+ }
+
+ // If o is nullptr, then don't calculate the checksum.
+ //
+ template <typename M>
+ static pair<M, string/*checksum*/>
+ fetch_manifest (const common_options* o,
+ const path& f,
+ bool ignore_unknown)
+ {
+ if (!exists (f))
+ fail << "file " << f << " does not exist";
+
+ try
+ {
+ // We can not use the same file stream for both calculating the checksum
+ // and reading the manifest. The file should be opened in the binary
+ // mode for the first operation and in the text mode for the second one.
+ //
+ string sha256sum;
+ if (o != nullptr)
+ sha256sum = sha256 (*o, f); // Read file in the binary mode.
+
+ ifdstream ifs (f); // Open file in the text mode.
+
+ manifest_parser mp (ifs, f.string ());
+ return make_pair (M (mp, ignore_unknown), move (sha256sum));
+ }
+ catch (const manifest_parsing& e)
+ {
+ fail (e.name, e.line, e.column) << e.description << endf;
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to read from " << f << ": " << e << endf;
+ }
+ }
+
+ static const path repositories ("repositories");
+
+ repository_manifests
+ bpkg_fetch_repositories (const dir_path& d, bool iu)
+ {
+ return fetch_manifest<repository_manifests> (
+ nullptr, d / repositories, iu).first;
+ }
+
+ pair<repository_manifests, string/*checksum*/>
+ bpkg_fetch_repositories (const common_options& o,
+ const repository_location& rl,
+ bool iu)
+ {
+ assert (rl.remote () || rl.absolute ());
+
+ repository_url u (rl.url ());
+
+ path& f (*u.path);
+ f /= repositories;
+
+ return rl.remote ()
+ ? fetch_manifest<repository_manifests> (o, u, iu)
+ : fetch_manifest<repository_manifests> (&o, f, iu);
+ }
+
+ static const path packages ("packages");
+
+ package_manifests
+ bpkg_fetch_packages (const dir_path& d, bool iu)
+ {
+ return fetch_manifest<package_manifests> (nullptr, d / packages, iu).first;
+ }
+
+ pair<package_manifests, string/*checksum*/>
+ bpkg_fetch_packages (const common_options& o,
+ const repository_location& rl,
+ bool iu)
+ {
+ assert (rl.remote () || rl.absolute ());
+
+ repository_url u (rl.url ());
+
+ path& f (*u.path);
+ f /= packages;
+
+ return rl.remote ()
+ ? fetch_manifest<package_manifests> (o, u, iu)
+ : fetch_manifest<package_manifests> (&o, f, iu);
+ }
+
+ static const path signature ("signature");
+
+ signature_manifest
+ bpkg_fetch_signature (const common_options& o,
+ const repository_location& rl,
+ bool iu)
+ {
+ assert (rl.remote () || rl.absolute ());
+
+ repository_url u (rl.url ());
+
+ path& f (*u.path);
+ f /= signature;
+
+ return rl.remote ()
+ ? fetch_manifest<signature_manifest> (o, u, iu).first
+ : fetch_manifest<signature_manifest> (nullptr, f, iu).first;
+ }
+
+ path
+ bpkg_fetch_archive (const common_options& o,
+ const repository_location& rl,
+ const path& a,
+ const dir_path& d)
+ {
+ assert (!a.empty () && a.relative ());
+ assert (rl.remote () || rl.absolute ());
+
+ repository_url u (rl.url ());
+
+ path& f (*u.path);
+ f /= a;
+
+ auto bad_loc = [&u] () {fail << "invalid archive location " << u;};
+
+ try
+ {
+ f.normalize ();
+
+ if (*f.begin () == "..") // Can be the case for the remote location.
+ bad_loc ();
+ }
+ catch (const invalid_path&)
+ {
+ bad_loc ();
+ }
+
+ return rl.remote ()
+ ? fetch_file (o, u, d)
+ : fetch_file (f, d);
+ }
+}
diff --git a/bpkg/fetch-git.cxx b/bpkg/fetch-git.cxx
new file mode 100644
index 0000000..1194178
--- /dev/null
+++ b/bpkg/fetch-git.cxx
@@ -0,0 +1,986 @@
+// file : bpkg/fetch-git.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license : MIT; see accompanying LICENSE file
+
+#include <bpkg/fetch.hxx>
+
+#ifdef _WIN32
+# include <algorithm> // replace()
+#endif
+
+#include <libbutl/process.mxx>
+#include <libbutl/fdstream.mxx>
+
+#include <bpkg/diagnostics.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace bpkg
+{
+ struct fail_git
+ {
+ [[noreturn]] void
+ operator() (const diag_record& r) const
+ {
+ if (verb < 2)
+ r << info << "re-run with -v for more information";
+
+ r << endf;
+ }
+ };
+
+ static const diag_noreturn_end<fail_git> endg;
+
+ static fdpipe
+ open_pipe ()
+ {
+ try
+ {
+ return fdopen_pipe ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to open pipe: " << e << endf;
+ }
+ }
+
+ static auto_fd
+ open_dev_null ()
+ {
+ try
+ {
+ return fdnull ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to open null device: " << e << endf;
+ }
+ }
+
+ using opt = optional<const char*>; // Program option.
+
+ static strings
+ timeout_opts (const common_options& co, repository_protocol proto)
+ {
+ if (!co.fetch_timeout_specified ())
+ return strings ();
+
+ switch (proto)
+ {
+ case repository_protocol::http:
+ case repository_protocol::https:
+ {
+ // Git doesn't support the connection timeout option. The options we
+ // use instead are just an approximation of the former, that, in
+ // particular, doesn't cover the connection establishing. Sensing
+ // HTTP(s) smart vs dumb protocol using a fetch utility prior to
+ // running git (see below) will probably mitigate this somewhat.
+ //
+ return strings ({
+ "-c", "http.lowSpeedLimit=1",
+ "-c", "http.lowSpeedTime=" + to_string (co.fetch_timeout ())});
+ }
+ case repository_protocol::git:
+ {
+ warn << "--fetch-timeout is not supported by the git protocol";
+ break;
+ }
+ case repository_protocol::file: return strings (); // Local communications.
+ }
+
+ assert (false); // Can't be here.
+ return strings ();
+ }
+
+ // Start git process.
+ //
+ // Note that git is executed in the "sanitized" environment, having the
+ // environment variables that are local to the repository being unset (all
+ // except GIT_CONFIG_PARAMETERS). We do the same as the git-submodule script
+ // does for commands executed for submodules. Though we do it for all
+ // commands (including the ones related to the top repository).
+ //
+ static optional<strings> unset_vars;
+
+ template <typename O, typename E, typename... A>
+ static process
+ start_git (const common_options& co,
+ O&& out,
+ E&& err,
+ A&&... args)
+ {
+ try
+ {
+ if (!unset_vars)
+ {
+ unset_vars = strings ();
+
+ for (;;) // Breakout loop.
+ {
+ fdpipe pipe (open_pipe ());
+
+ // We assume that non-sanitized git environment can't harm this call.
+ //
+ process pr (start_git (co,
+ pipe, 2 /* stderr */,
+ co.git_option (),
+ "rev-parse",
+ "--local-env-vars"));
+
+ // Shouldn't throw, unless something is severely damaged.
+ //
+ pipe.out.close ();
+
+ try
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::skip);
+
+ while (is.peek () != ifdstream::traits_type::eof ())
+ {
+ string v;
+ getline (is, v);
+
+ if (v != "GIT_CONFIG_PARAMETERS")
+ unset_vars->push_back (move (v));
+ }
+
+ is.close ();
+
+ if (pr.wait ())
+ break;
+
+ // Fall through.
+ }
+ catch (const io_error&)
+ {
+ if (pr.wait ())
+ fail << "unable to read git local environment variables" << endg;
+
+ // Fall through.
+ }
+
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
+
+ fail << "unable to list git local environment variables" << endg;
+ }
+ }
+
+ return process_start_callback ([] (const char* const args[], size_t n)
+ {
+ if (verb >= 2)
+ print_process (args, n);
+ },
+ 0 /* stdin */, out, err,
+ process_env (co.git (), *unset_vars),
+ forward<A> (args)...);
+ }
+ catch (const process_error& e)
+ {
+ fail << "unable to execute " << co.git () << ": " << e << endg;
+ }
+ }
+
+ // Run git process.
+ //
+ template <typename... A>
+ static process_exit
+ run_git (const common_options& co, A&&... args)
+ {
+ process pr (start_git (co, 1, 2, forward<A> (args)...));
+ pr.wait ();
+ return *pr.exit;
+ }
+
+ // Run git process and return it's output as a string. Fail if the output
+ // doesn't contain a single line.
+ //
+ template <typename... A>
+ static string
+ git_string (const common_options& co, const char* what, A&&... args)
+ {
+ fdpipe pipe (open_pipe ());
+ process pr (start_git (co, pipe, 2 /* stderr */, forward<A> (args)...));
+ pipe.out.close (); // Shouldn't throw, unless something is severely damaged.
+
+ try
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::skip);
+
+ optional<string> r;
+ if (is.peek () != ifdstream::traits_type::eof ())
+ {
+ string s;
+ getline (is, s);
+
+ if (!is.eof () && is.peek () == ifdstream::traits_type::eof ())
+ r = move (s);
+ }
+
+ is.close ();
+
+ if (pr.wait ())
+ {
+ if (r)
+ return *r;
+
+ fail << "invalid " << what << endg;
+ }
+
+ // Fall through.
+ }
+ catch (const io_error&)
+ {
+ if (pr.wait ())
+ fail << "unable to read " << what << endg;
+
+ // Fall through.
+ }
+
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
+
+ fail << "unable to obtain " << what << endg;
+ }
+
+ // Convert the URL object to string representation that is usable in the git
+ // commands. This, in particular, means using file:// (rather than local
+ // path) notation for local URLs.
+ //
+ // Note that cloning the local git repository using the local path notation
+ // disregards --depth option (and issues a warning), creating full copy of
+ // the source repository (copying some files and hard-linking others if
+ // possible). Using --no-local option overrides such an unwanted behavior.
+ // However, this options can not be propagated to submodule--helper's clone
+ // command that we use to clone submodules. So to truncate local submodule
+ // histories we will use the file URL notation for local repositories.
+ //
+ static string
+ git_url (const repository_url& url)
+ {
+ if (url.scheme != repository_protocol::file)
+ return url.string ();
+
+#ifndef _WIN32
+ // Enforce the 'file://' notation for local URLs (see libpkg/manifest.hxx).
+ //
+ repository_url u (url.scheme,
+ repository_url::authority_type (),
+ url.path,
+ url.query);
+
+ return u.string ();
+#else
+ // On Windows the appropriate file notations are:
+ //
+ // file://c:/...
+ // file://c:\...
+ //
+ // Note that none of them conforms to RFC3986. The proper one should be:
+ //
+ // file:///c:/...
+ //
+ // We choose to convert it to the "most conformant" (the first)
+ // representation to ease the fix-up before creating the URL object from
+ // it, when required.
+ //
+ string p (url.path->string ());
+ replace (p.begin (), p.end (), '\\', '/');
+ return "file://" + p;
+#endif
+ }
+
+ // Sense the git protocol capabilities for a specified URL.
+ //
+ // Protocols other than HTTP(S) are considered smart but without the
+ // unadvertised refs (note that this is a pessimistic assumption for
+ // git://).
+ //
+ // For HTTP(S) sense the protocol type by sending the first HTTP request of
+ // the fetch operation handshake and analyzing the first line of the
+ // response. Fail if connecting to the server failed, the response code
+ // differs from 200, or reading the response body failed.
+ //
+ // Note that, as a side-effect, this function checks the HTTP(S) server
+ // availability and so must be called prior to any git command that involves
+ // communication to the remote server. Not doing so may result in the command
+ // hanging indefinitely while trying to establish TCP/IP connection (see the
+ // timeout_opts() function for the gory details).
+ //
+ enum class capabilities
+ {
+ dumb, // No shallow clone support.
+ smart, // Support for shallow clone, but not for unadvertised refs fetch.
+ unadv // Support for shallow clone and for unadvertised refs fetch.
+ };
+
+ static capabilities
+ sense_capabilities (const common_options& co, repository_url url)
+ {
+ assert (url.path);
+
+ switch (url.scheme)
+ {
+ case repository_protocol::git:
+ case repository_protocol::file: return capabilities::smart;
+ case repository_protocol::http:
+ case repository_protocol::https: break; // Ask the server (see below).
+ }
+
+ path& up (*url.path);
+
+ if (!up.to_directory ())
+ up = path_cast<dir_path> (move (up));
+
+ up /= path ("info/refs");
+
+ if (url.query)
+ *url.query += "&service=git-upload-pack";
+ else
+ url.query = "service=git-upload-pack";
+
+ string u (url.string ());
+ process pr (start_fetch (co, u));
+
+ try
+ {
+ // We unset failbit to properly handle an empty response (no refs) from
+ // the dumb server.
+ //
+ ifdstream is (move (pr.in_ofd),
+ fdstream_mode::skip | fdstream_mode::binary,
+ ifdstream::badbit);
+
+ string l;
+ getline (is, l); // Is empty if no refs returned by the dumb server.
+
+ // If the first response line has the following form:
+ //
+ // XXXX# service=git-upload-pack"
+ //
+ // where XXXX is a sequence of 4 hex digits, then the server implements
+ // the smart protocol.
+ //
+ // Note that to consider the server to be "smart" it would make sense
+ // to also check that the response Content-Type header value is
+ // 'application/x-git-upload-pack-advertisement'. However, we will skip
+ // this check in order to not complicate the fetch API.
+ //
+ size_t n (l.size ());
+
+ capabilities r (
+ n >= 4 &&
+ xdigit (l[0]) && xdigit (l[1]) && xdigit (l[2]) && xdigit (l[3]) &&
+ l.compare (4, n - 4, "# service=git-upload-pack") == 0
+ ? capabilities::smart
+ : capabilities::dumb);
+
+ // If the transport is smart let's see it the server also supports
+ // unadvertised refs fetch.
+ //
+ if (r == capabilities::smart && !is.eof ())
+ {
+ getline (is, l);
+
+ // Parse the space-separated list of capabilities that follows the
+ // NULL character.
+ //
+ for (size_t p (l.find ('\0')); p != string::npos; )
+ {
+ size_t e (l.find (' ', ++p));
+ size_t n (e != string::npos ? e - p : e);
+
+ if (l.compare (p, n, "allow-reachable-sha1-in-want") == 0 ||
+ l.compare (p, n, "allow-tip-sha1-in-want") == 0)
+ {
+ r = capabilities::unadv;
+ break;
+ }
+
+ p = e;
+ }
+ }
+
+ is.close ();
+
+ if (pr.wait ())
+ return r;
+
+ // Fall through.
+ }
+ catch (const io_error&)
+ {
+ if (pr.wait ())
+ fail << "unable to read fetched " << url << endg;
+
+ // Fall through.
+ }
+
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
+
+ fail << "unable to fetch " << url << endg;
+ }
+
+ // Return true if a commit is advertised by the remote repository. It is
+ // assumed that sense_capabilities() function was already called for the URL.
+ //
+ static bool
+ commit_advertized (const common_options& co,
+ const repository_url& url,
+ const string& commit)
+ {
+ tracer trace ("commit_advertized");
+
+ fdpipe pipe (open_pipe ());
+
+ process pr (start_git (co,
+ pipe, 2 /* stderr */,
+ timeout_opts (co, url.scheme),
+ co.git_option (),
+ "ls-remote",
+ "--refs",
+ git_url (url)));
+
+ pipe.out.close (); // Shouldn't throw, unless something is severely damaged.
+
+ try
+ {
+ bool r (false);
+ ifdstream is (move (pipe.in), fdstream_mode::skip);
+
+ while (is.peek () != ifdstream::traits_type::eof ())
+ {
+ string s;
+ getline (is, s);
+
+ l4 ([&]{trace << "ref: " << s;});
+
+ if (s.compare (0, commit.size (), commit) == 0)
+ {
+ r = true;
+ break;
+ }
+ }
+
+ is.close ();
+
+ if (pr.wait ())
+ return r;
+
+ // Fall through.
+ }
+ catch (const io_error&)
+ {
+ if (pr.wait ())
+ fail << "unable to read references for " << url << endg;
+
+ // Fall through.
+ }
+
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
+
+ fail << "unable to list references for " << url << endg;
+ }
+
+ // Return true if the shallow fetch is possible for the reference.
+ //
+ static bool
+ shallow_fetch (const common_options& co,
+ const repository_url& url,
+ capabilities cap,
+ const git_reference& ref)
+ {
+ switch (cap)
+ {
+ case capabilities::dumb:
+ {
+ return false;
+ }
+ case capabilities::smart:
+ {
+ return !ref.commit || commit_advertized (co, url, *ref.commit);
+ }
+ case capabilities::unadv:
+ {
+ return true;
+ }
+ }
+
+ assert (false); // Can't be here.
+ return false;
+ }
+
+ // Return true if a commit is reachable from the tip(s).
+ //
+ // Can be used to avoid redundant fetches.
+ //
+ // Note that git-submodule script implements this check, so it is probably an
+ // important optimization.
+ //
+ static bool
+ commit_reachable (const common_options& co,
+ const dir_path& dir,
+ const string& commit)
+ {
+ fdpipe pipe (open_pipe ());
+ auto_fd dev_null (open_dev_null ());
+
+ process pr (start_git (co,
+ pipe,
+ dev_null,
+ co.git_option (),
+ "-C", dir,
+ "rev-list",
+ "-n", "1",
+ commit,
+ "--not",
+ "--all"));
+
+ // Shouldn't throw, unless something is severely damaged.
+ //
+ pipe.out.close ();
+ dev_null.close ();
+
+ try
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::skip);
+
+ string s;
+ if (is.peek () != ifdstream::traits_type::eof ())
+ getline (is, s);
+
+ is.close ();
+ return pr.wait () && s.empty ();
+ }
+ catch (const io_error&) {}
+ return false;
+ }
+
+ // Print warnings about non-shallow fetching.
+ //
+ static void
+ fetch_warn (capabilities cap,
+ const char* what,
+ const dir_path& submodule = dir_path ())
+ {
+ {
+ diag_record dr (warn);
+ dr << "fetching whole " << what << " history";
+
+ if (!submodule.empty ())
+ dr << " for submodule '" << submodule.posix_string () << "'";
+
+ dr << " ("
+ << (cap == capabilities::dumb
+ ? "dumb HTTP"
+ : "unadvertised commit") // There are no other reasons so far.
+ << ')';
+
+ }
+
+ if (cap == capabilities::dumb)
+ warn << "fetching over dumb HTTP, no progress will be displayed";
+ }
+
+ // Update git index and working tree to match the reference. Fetch if
+ // necessary.
+ //
+ static void
+ update_tree (const common_options& co,
+ const dir_path& dir,
+ const dir_path& submodule, // Is relative to the top project.
+ const git_reference& ref,
+ capabilities cap,
+ bool shallow,
+ const strings& to)
+ {
+ // Don't fetch it the reference is a commit that is reachable from the
+ // tip(s).
+ //
+ if (!(ref.commit && commit_reachable (co, dir, *ref.commit)))
+ {
+ if (!shallow)
+ fetch_warn (cap, ref.commit ? "repository" : "branch", submodule);
+
+ // The clone command prints the following line prior to the progress
+ // lines:
+ //
+ // Cloning into '<dir>'...
+ //
+ // The fetch command doesn't print anything similar, for some reason.
+ // This makes it hard to understand which superproject/submodule is
+ // currently being fetched. Let's fix that.
+ //
+ if (verb != 0)
+ text << "Fetching in '" << dir.posix_string () << "'...";
+
+ // Note that we suppress the (too detailed) fetch command output if the
+ // verbosity level is 1. However, we still want to see the progress in
+ // this case, unless STDERR is not directed to a terminal.
+ //
+ // Also note that we don't need to specify --refmap option since we can
+ // rely on the clone command that properly set the remote.origin.fetch
+ // configuration option.
+ //
+ if (!run_git (co,
+ to,
+ co.git_option (),
+ "-C", dir,
+ "fetch",
+ "--no-recurse-submodules",
+ shallow ? cstrings ({"--depth", "1"}) : cstrings (),
+ verb == 1 && fdterm (2) ? opt ( "--progress") : nullopt,
+ verb < 2 ? opt ("-q") : verb > 3 ? opt ("-v") : nullopt,
+ "origin",
+ ref.commit ? *ref.commit : *ref.branch))
+ fail << "unable to fetch " << dir << endg;
+ }
+
+ const string& commit (ref.commit ? *ref.commit : string ("FETCH_HEAD"));
+
+ // For some (probably valid) reason the hard reset command doesn't remove
+ // a submodule directory that is not plugged into the project anymore. It
+ // also prints the non-suppressible warning like this:
+ //
+ // warning: unable to rmdir libbar: Directory not empty
+ //
+ // That's why we run the clean command afterwards. It may also be helpful
+ // if we produce any untracked files in the tree between fetches down the
+ // road.
+ //
+ if (!run_git (
+ co,
+ co.git_option (),
+ "-C", dir,
+ "reset",
+ "--hard",
+ verb < 2 ? opt ("-q") : nullopt,
+ commit))
+ fail << "unable to reset to " << commit << endg;
+
+ if (!run_git (
+ co,
+ co.git_option (),
+ "-C", dir,
+ "clean",
+ "-d",
+ "-x",
+ "-ff",
+ verb < 2 ? opt ("-q") : nullopt))
+ fail << "unable to clean " << dir << endg;
+ }
+
+ static void
+ update_submodules (const common_options& co,
+ const dir_path& dir,
+ const dir_path& prefix)
+ {
+ tracer trace ("update_submodules");
+
+ auto failure = [&prefix] (const char* desc)
+ {
+ diag_record dr (fail);
+ dr << desc;
+
+ if (!prefix.empty ())
+ // Strips the trailing slash.
+ //
+ dr << " for submodule '" << prefix.string () << "'";
+
+ dr << endg;
+ };
+
+ // Initialize submodules.
+ //
+ if (!run_git (
+ co,
+ co.git_option (),
+ "-C", dir,
+
+ !prefix.empty ()
+ ? strings ({"--super-prefix", prefix.posix_representation ()})
+ : strings (),
+
+ "submodule--helper", "init",
+ verb < 1 ? opt ("-q") : nullopt))
+ failure ("unable to initialize submodules");
+
+ // Iterate over the registered submodules cloning/fetching them and
+ // recursively updating their submodules.
+ //
+ // Note that we don't expect submodules nesting be too deep and so recurse
+ // while reading the git process output.
+ //
+ fdpipe pipe (open_pipe ());
+
+ process pr (start_git (co,
+ pipe, 2 /* stderr */,
+ co.git_option (),
+ "-C", dir,
+ "submodule--helper", "list"));
+
+ pipe.out.close (); // Shouldn't throw, unless something is severely damaged.
+
+ try
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::skip);
+
+ while (is.peek () != ifdstream::traits_type::eof ())
+ {
+ // The line describing a submodule has the following form:
+ //
+ // <mode><SPACE><commit><SPACE><stage><TAB><path>
+ //
+ // For example:
+ //
+ // 160000 658436a9522b5a0d016c3da0253708093607f95d 0 doc/style
+ //
+ string s;
+ getline (is, s);
+
+ l4 ([&]{trace << "submodule: " << s;});
+
+ if (!(s.size () > 50 && s[48] == '0' && s[49] == '\t'))
+ failure ("invalid submodule description");
+
+ string commit (s.substr (7, 40));
+
+ // Submodule directory path, relative to the containing project.
+ //
+ dir_path sdir (s.substr (50));
+
+ // Submodule directory path, relative to the top project.
+ //
+ dir_path psdir (prefix / sdir);
+ string psd (psdir.posix_string ()); // For use in the diagnostics.
+
+ string name (git_string (co, "submodule name",
+ co.git_option (),
+ "-C", dir,
+ "submodule--helper", "name",
+ sdir));
+
+ repository_url url;
+
+ try
+ {
+ string u (git_string (co, "submodule URL",
+ co.git_option (),
+ "-C", dir,
+ "config",
+ "--get",
+ "submodule." + name + ".url"));
+
+ // Fix-up the broken Windows file URL notation (see the git_url()
+ // function for details).
+ //
+#ifdef _WIN32
+ if (casecmp (u, "file://", 7) == 0 && u[7] != '/')
+ u.insert (7, 1, '/');
+#endif
+ url = repository_url (u);
+ }
+ catch (const invalid_argument& e)
+ {
+ fail << "invalid repository URL for submodule '" << psd << "': "
+ << e << endg;
+ }
+
+ l4 ([&]{trace << "name: " << name << ", URL: " << url;});
+
+ dir_path fsdir (dir / sdir);
+ bool cloned (exists (fsdir / path (".git")));
+
+ // If the submodule is already cloned and it's commit didn't change
+ // then we skip it.
+ //
+ // Note that git-submodule script still recurse into it for some
+ // unclear reason.
+ //
+ if (cloned && git_string (co, "submodule commit",
+ co.git_option (),
+ "-C", fsdir,
+ "rev-parse",
+ "--verify",
+ "HEAD") == commit)
+ continue;
+
+ git_reference ref {nullopt, commit};
+ capabilities cap (sense_capabilities (co, url));
+ bool shallow (shallow_fetch (co, url, cap, ref));
+ strings to (timeout_opts (co, url.scheme));
+
+ // Clone new submodule.
+ //
+ if (!cloned)
+ {
+ if (!shallow)
+ fetch_warn (cap, "repository", psdir);
+
+ if (!run_git (co,
+ to,
+ co.git_option (),
+ "-C", dir,
+ "submodule--helper", "clone",
+
+ "--name", name,
+ "--path", sdir,
+ "--url", git_url (url),
+ shallow
+ ? cstrings ({"--depth", "1"})
+ : cstrings (),
+ verb < 1 ? opt ("-q") : nullopt))
+ fail << "unable to clone submodule '" << psd << "'" << endg;
+ }
+
+ update_tree (co, fsdir, psdir, ref, cap, shallow, to);
+
+ // Not quite a checkout, but let's make the message match the
+ // git-submodule script output.
+ //
+ if (verb > 0)
+ text << "Submodule path '" << psd << "': checked out '" << commit
+ << "'";
+
+ // Recurse.
+ //
+ // Can throw the failed exception that we don't catch here, relying on
+ // the fact that the process destructor will wait for the process
+ // completion.
+ //
+ update_submodules (co, fsdir, psdir);
+ }
+
+ is.close ();
+
+ if (pr.wait ())
+ return;
+
+ // Fall through.
+ }
+ catch (const io_error&)
+ {
+ if (pr.wait ())
+ failure ("unable to read submodules list");
+
+ // Fall through.
+ }
+
+ // We should only get here if the child exited with an error status.
+ //
+ assert (!pr.wait ());
+
+ failure ("unable to list submodules");
+ }
+
+ // Extract the git reference from the repository URL fragment. Set the URL
+ // fragment to nullopt.
+ //
+ static git_reference
+ parse_reference (repository_url& url, const char* what)
+ {
+ try
+ {
+ git_reference r (git_reference (url.fragment));
+ url.fragment = nullopt;
+ return r;
+ }
+ catch (const invalid_argument& e)
+ {
+ fail << "unable to " << what << ' ' << url << ": " << e << endf;
+ }
+ }
+
+ void
+ git_clone (const common_options& co,
+ const repository_location& rl,
+ const dir_path& destdir)
+ {
+ repository_url url (rl.url ());
+ git_reference ref (parse_reference (url, "clone"));
+
+ // All protocols support single branch cloning, so we will always be
+ // cloning a single branch if the branch is specified.
+ //
+ bool single_branch (ref.branch);
+ capabilities cap (sense_capabilities (co, url));
+ bool shallow (shallow_fetch (co, url, cap, ref));
+
+ if (shallow)
+ single_branch = false; // Is implied for shallow cloning.
+ else
+ fetch_warn (cap, single_branch ? "branch" : "repository");
+
+ dir_path d (destdir);
+ d /= dir_path (ref.branch ? *ref.branch : *ref.commit);
+
+ strings to (timeout_opts (co, url.scheme));
+
+ if (!run_git (
+ co,
+ to,
+ "-c", "advice.detachedHead=false",
+ co.git_option (),
+ "clone",
+
+ ref.branch ? strings ({"--branch", *ref.branch}) : strings (),
+ single_branch ? opt ("--single-branch") : nullopt,
+ shallow ? strings ({"--depth", "1"}) : strings (),
+ ref.commit ? opt ("--no-checkout") : nullopt,
+
+ verb < 1 ? opt ("-q") : verb > 3 ? opt ("-v") : nullopt,
+ git_url (url),
+ d))
+ fail << "unable to clone " << url << endg;
+
+ if (ref.commit)
+ update_tree (co, d, dir_path (), ref, cap, shallow, to);
+
+ update_submodules (co, d, dir_path ());
+ }
+
+ void
+ git_fetch (const common_options& co,
+ const repository_location& rl,
+ const dir_path& destdir)
+ {
+ repository_url url (rl.url ());
+ git_reference ref (parse_reference (url, "fetch"));
+
+ // Fetch is noop if the specific commit is checked out.
+ //
+ // What if the user replaces the repository URL with a one with a new
+ // branch/tag/commit? These are not part of the repository name which
+ // means such a repository will have the same hash. But then when we
+ // remove the repository, we will also clean up its state. So seems like
+ // this should work correctly automatically.
+ //
+ if (ref.commit)
+ return;
+
+ assert (ref.branch);
+
+ capabilities cap (sense_capabilities (co, url));
+ bool shallow (shallow_fetch (co, url, cap, ref));
+
+ dir_path d (destdir);
+ d /= dir_path (*ref.branch);
+
+ update_tree (co,
+ d,
+ dir_path (),
+ ref,
+ cap,
+ shallow,
+ timeout_opts (co, url.scheme));
+
+ update_submodules (co, d, dir_path ());
+ }
+}
diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx
index 5464c4a..c5366e3 100644
--- a/bpkg/fetch.cxx
+++ b/bpkg/fetch.cxx
@@ -4,14 +4,8 @@
#include <bpkg/fetch.hxx>
-#include <sstream>
-
-#include <libbutl/process.mxx>
#include <libbutl/fdstream.mxx>
-#include <libbutl/filesystem.mxx>
-#include <libbutl/manifest-parser.mxx>
-#include <bpkg/checksum.hxx>
#include <bpkg/diagnostics.hxx>
using namespace std;
@@ -510,11 +504,8 @@ namespace bpkg
return fetch_kind;
}
- // If out is empty, then fetch to STDOUT. In this case also don't
- // show any progress unless we are running verbose.
- //
- static process
- start (const common_options& o, const string& url, const path& out = path ())
+ process
+ start_fetch (const common_options& o, const string& url, const path& out)
{
process (*f) (const path&,
const optional<size_t>&,
@@ -547,254 +538,4 @@ namespace bpkg
throw failed ();
}
}
-
- static path
- fetch_file (const common_options& o,
- const repository_url& u,
- const dir_path& d)
- {
- path r (d / u.path->leaf ());
-
- if (exists (r))
- fail << "file " << r << " already exists";
-
- auto_rmfile arm (r);
- process pr (start (o, u.string (), r));
-
- if (!pr.wait ())
- {
- // While it is reasonable to assuming the child process issued
- // diagnostics, some may not mention the URL.
- //
- fail << "unable to fetch " << u <<
- info << "re-run with -v for more information";
- }
-
- arm.cancel ();
- return r;
- }
-
- template <typename M>
- static pair<M, string/*checksum*/>
- fetch_manifest (const common_options& o,
- const repository_url& u,
- bool ignore_unknown)
- {
- string url (u.string ());
- process pr (start (o, url));
-
- try
- {
- // Unfortunately we cannot read from the original source twice as we do
- // below for files. There doesn't seem to be anything better than reading
- // the entire file into memory and then streaming it twice, once to
- // calculate the checksum and the second time to actually parse. We need
- // to read the original stream in the binary mode for the checksum
- // calculation, then use the binary data to create the text stream for
- // the manifest parsing.
- //
- ifdstream is (move (pr.in_ofd), fdstream_mode::binary);
- stringstream bs (ios::in | ios::out | ios::binary);
-
- // Note that the eof check is important: if the stream is at eof, write
- // will fail.
- //
- if (is.peek () != ifdstream::traits_type::eof ())
- bs << is.rdbuf ();
-
- is.close ();
-
- string s (bs.str ());
- string sha256sum (sha256 (s.c_str (), s.size ()));
-
- istringstream ts (s); // Text mode.
-
- manifest_parser mp (ts, url);
- M m (mp, ignore_unknown);
-
- if (pr.wait ())
- return make_pair (move (m), move (sha256sum));
-
- // Child existed with an error, fall through.
- }
- // Ignore these exceptions if the child process exited with
- // an error status since that's the source of the failure.
- //
- catch (const manifest_parsing& e)
- {
- if (pr.wait ())
- fail (e.name, e.line, e.column) << e.description;
- }
- catch (const io_error&)
- {
- if (pr.wait ())
- fail << "unable to read fetched " << url;
- }
-
- // We should only get here if the child exited with an error status.
- //
- assert (!pr.wait ());
-
- // While it is reasonable to assuming the child process issued
- // diagnostics, some may not mention the URL.
- //
- fail << "unable to fetch " << url <<
- info << "re-run with -v for more information" << endf;
- }
-
- static path
- fetch_file (const path& f, const dir_path& d)
- {
- path r (d / f.leaf ());
-
- try
- {
- cpfile (f, r);
- }
- catch (const system_error& e)
- {
- fail << "unable to copy " << f << " to " << r << ": " << e;
- }
-
- return r;
- }
-
- // If o is nullptr, then don't calculate the checksum.
- //
- template <typename M>
- static pair<M, string/*checksum*/>
- fetch_manifest (const common_options* o,
- const path& f,
- bool ignore_unknown)
- {
- if (!exists (f))
- fail << "file " << f << " does not exist";
-
- try
- {
- // We can not use the same file stream for both calculating the checksum
- // and reading the manifest. The file should be opened in the binary
- // mode for the first operation and in the text mode for the second one.
- //
- string sha256sum;
- if (o != nullptr)
- sha256sum = sha256 (*o, f); // Read file in the binary mode.
-
- ifdstream ifs (f); // Open file in the text mode.
-
- manifest_parser mp (ifs, f.string ());
- return make_pair (M (mp, ignore_unknown), move (sha256sum));
- }
- catch (const manifest_parsing& e)
- {
- fail (e.name, e.line, e.column) << e.description << endf;
- }
- catch (const io_error& e)
- {
- fail << "unable to read from " << f << ": " << e << endf;
- }
- }
-
- static const path repositories ("repositories");
-
- repository_manifests
- fetch_repositories (const dir_path& d, bool iu)
- {
- return fetch_manifest<repository_manifests> (
- nullptr, d / repositories, iu).first;
- }
-
- pair<repository_manifests, string/*checksum*/>
- fetch_repositories (const common_options& o,
- const repository_location& rl,
- bool iu)
- {
- assert (rl.remote () || rl.absolute ());
-
- repository_url u (rl.url ());
-
- path& f (*u.path);
- f /= repositories;
-
- return rl.remote ()
- ? fetch_manifest<repository_manifests> (o, u, iu)
- : fetch_manifest<repository_manifests> (&o, f, iu);
- }
-
- static const path packages ("packages");
-
- package_manifests
- fetch_packages (const dir_path& d, bool iu)
- {
- return fetch_manifest<package_manifests> (nullptr, d / packages, iu).first;
- }
-
- pair<package_manifests, string/*checksum*/>
- fetch_packages (const common_options& o,
- const repository_location& rl,
- bool iu)
- {
- assert (rl.remote () || rl.absolute ());
-
- repository_url u (rl.url ());
-
- path& f (*u.path);
- f /= packages;
-
- return rl.remote ()
- ? fetch_manifest<package_manifests> (o, u, iu)
- : fetch_manifest<package_manifests> (&o, f, iu);
- }
-
- static const path signature ("signature");
-
- signature_manifest
- fetch_signature (const common_options& o,
- const repository_location& rl,
- bool iu)
- {
- assert (rl.remote () || rl.absolute ());
-
- repository_url u (rl.url ());
-
- path& f (*u.path);
- f /= signature;
-
- return rl.remote ()
- ? fetch_manifest<signature_manifest> (o, u, iu).first
- : fetch_manifest<signature_manifest> (nullptr, f, iu).first;
- }
-
- path
- fetch_archive (const common_options& o,
- const repository_location& rl,
- const path& a,
- const dir_path& d)
- {
- assert (!a.empty () && a.relative ());
- assert (rl.remote () || rl.absolute ());
-
- repository_url u (rl.url ());
-
- path& f (*u.path);
- f /= a;
-
- auto bad_loc = [&u] () {fail << "invalid archive location " << u;};
-
- try
- {
- f.normalize ();
-
- if (*f.begin () == "..") // Can be the case for the remote location.
- bad_loc ();
- }
- catch (const invalid_path&)
- {
- bad_loc ();
- }
-
- return rl.remote ()
- ? fetch_file (o, u, d)
- : fetch_file (f, d);
- }
}
diff --git a/bpkg/fetch.hxx b/bpkg/fetch.hxx
index 4e7b271..49e144c 100644
--- a/bpkg/fetch.hxx
+++ b/bpkg/fetch.hxx
@@ -5,6 +5,8 @@
#ifndef BPKG_FETCH_HXX
#define BPKG_FETCH_HXX
+#include <libbutl/process.mxx>
+
#include <libbpkg/manifest.hxx>
#include <bpkg/types.hxx>
@@ -14,32 +16,64 @@
namespace bpkg
{
+ // Repository type bpkg (fetch-bpkg.cxx).
+ //
+
repository_manifests
- fetch_repositories (const dir_path&, bool ignore_unknown);
+ bpkg_fetch_repositories (const dir_path&, bool ignore_unknown);
pair<repository_manifests, string /* checksum */>
- fetch_repositories (const common_options&,
- const repository_location&,
- bool ignore_unknown);
+ bpkg_fetch_repositories (const common_options&,
+ const repository_location&,
+ bool ignore_unknown);
package_manifests
- fetch_packages (const dir_path&, bool ignore_unknown);
+ bpkg_fetch_packages (const dir_path&, bool ignore_unknown);
pair<package_manifests, string /* checksum */>
- fetch_packages (const common_options&,
- const repository_location&,
- bool ignore_unknown);
+ bpkg_fetch_packages (const common_options&,
+ const repository_location&,
+ bool ignore_unknown);
signature_manifest
- fetch_signature (const common_options&,
- const repository_location&,
- bool ignore_unknown);
+ bpkg_fetch_signature (const common_options&,
+ const repository_location&,
+ bool ignore_unknown);
path
- fetch_archive (const common_options&,
- const repository_location&,
- const path& archive,
- const dir_path& destdir);
+ bpkg_fetch_archive (const common_options&,
+ const repository_location&,
+ const path& archive,
+ const dir_path& destdir);
+
+ // Repository type git (fetch-git.cxx).
+ //
+
+ // Clone git repository into destdir/<fragment>/.
+ //
+ void
+ git_clone (const common_options&,
+ const repository_location&,
+ const dir_path& destdir);
+
+ // Fetch git repository in destdir/<fragment>/.
+ //
+ void
+ git_fetch (const common_options&,
+ const repository_location&,
+ const dir_path& destdir);
+
+ // Low-level fetch API (fetch.cxx).
+ //
+
+ // Start the process of fetching the specified URL. If out is empty, then
+ // fetch to STDOUT. In this case also don't show any progress unless we are
+ // running verbose.
+ //
+ butl::process
+ start_fetch (const common_options& o,
+ const string& url,
+ const path& out = path ());
}
#endif // BPKG_FETCH_HXX
diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index 00a54d6..322fb2b 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -200,25 +200,28 @@ namespace bpkg
// repository_location
//
-
#pragma db value
struct _repository_location
{
- string url;
+ repository_url url;
repository_type type;
};
- // Note that the type() call fails for an empty repository location.
- //
- #pragma db map type(repository_location) as(_repository_location) \
- to({(?).string (), \
- (?).empty () ? bpkg::repository_type::bpkg : (?).type ()}) \
- from(bpkg::repository_location ((?).url, (?).type))
+ #pragma db map type(repository_url) as(string) \
+ to((?).string ()) \
+ from(bpkg::repository_url (?))
#pragma db map type(repository_type) as(string) \
to(to_string (?)) \
from(bpkg::to_repository_type (?))
+ // Note that the type() call fails for an empty repository location.
+ //
+ #pragma db map type(repository_location) as(_repository_location) \
+ to({(?).url (), \
+ (?).empty () ? bpkg::repository_type::bpkg : (?).type ()}) \
+ from(bpkg::repository_location (std::move ((?).url), (?).type))
+
// repository
//
#pragma db object pointer(shared_ptr) session
@@ -629,8 +632,9 @@ namespace bpkg
// certificate
//
// Information extracted from a repository X.509 certificate. The actual
- // certificate is stored on disk as .bpkg/certs/<fingerprint>.pem (we have
- // to store it as a file because that's the only way to pass it to openssl).
+ // certificate is stored on disk as .bpkg/certificates/<fingerprint>.pem (we
+ // have to store it as a file because that's the only way to pass it to
+ // openssl).
//
// If a repository is not authenticated (has no certificate/signature,
// called unauth from now on), then we ask for the user's confirmation and
diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx
index c2a6644..26f17d2 100644
--- a/bpkg/pkg-fetch.cxx
+++ b/bpkg/pkg-fetch.cxx
@@ -216,7 +216,9 @@ namespace bpkg
text << "fetching " << pl->location.leaf () << " "
<< "from " << pl->repository->name;
- path a (fetch_archive (co, pl->repository->location, pl->location, c));
+ path a (
+ bpkg_fetch_archive (co, pl->repository->location, pl->location, c));
+
auto_rmfile arm (a);
// We can't be fetching an archive for a transient object.
diff --git a/bpkg/rep-create.cxx b/bpkg/rep-create.cxx
index 173e2b0..60f7f59 100644
--- a/bpkg/rep-create.cxx
+++ b/bpkg/rep-create.cxx
@@ -185,7 +185,9 @@ namespace bpkg
// Load the 'repositories' file to make sure it is there and
// is valid.
//
- repository_manifests rms (fetch_repositories (d, o.ignore_unknown ()));
+ repository_manifests rms (
+ bpkg_fetch_repositories (d, o.ignore_unknown ()));
+
l4 ([&]{trace << rms.size () - 1 << " prerequisite repository(s)";});
// While we could have serialized as we go along, the order of
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index f53919f..5566114 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -4,6 +4,8 @@
#include <bpkg/rep-fetch.hxx>
+#include <libbutl/sha256.mxx>
+
#include <bpkg/auth.hxx>
#include <bpkg/fetch.hxx>
#include <bpkg/package.hxx>
@@ -26,7 +28,7 @@ namespace bpkg
// certificate.
//
pair<repository_manifests, string /* checksum */> rmc (
- fetch_repositories (co, rl, ignore_unknown));
+ bpkg_fetch_repositories (co, rl, ignore_unknown));
repository_manifests& rms (rmc.first);
@@ -46,7 +48,7 @@ namespace bpkg
// we just fetched.
//
pair<package_manifests, string /* checksum */> pmc (
- fetch_packages (co, rl, ignore_unknown));
+ bpkg_fetch_packages (co, rl, ignore_unknown));
package_manifests& pms (pmc.first);
@@ -58,7 +60,7 @@ namespace bpkg
if (a)
{
signature_manifest sm (
- fetch_signature (co, rl, true /* ignore_unknown */));
+ bpkg_fetch_signature (co, rl, true /* ignore_unknown */));
if (sm.sha256sum != pmc.second)
fail << "packages manifest file checksum mismatch for "
@@ -73,12 +75,104 @@ namespace bpkg
}
static rep_fetch_data
- rep_fetch_git (const common_options&,
- const dir_path*,
- const repository_location&,
- bool)
+ rep_fetch_git (const common_options& co,
+ const dir_path* conf,
+ const repository_location& rl,
+ bool /* ignore_unknown */)
{
- fail << "not implemented" << endf;
+ // Plan:
+ //
+ // 1. Check repos_dir/<hash>/:
+ //
+ // 1.a If does not exist, git-clone into temp_dir/<hash>/.
+ //
+ // 1.a Otherwise, move as temp_dir/<hash>/ and git-fetch.
+ //
+ // 2. Move from temp_dir/<hash>/ to repos_dir/<hash>/
+ //
+ // 3. Load manifest from repos_dir/<hash>/<fragment>/
+ //
+ // 4. Run 'b info' in repos_dir/<hash>/<fragment>/ and fix-up
+ // package version.
+ //
+ // 5. Synthesize repository manifest.
+ //
+ // 6. Return repository and package manifest (certificate is NULL).
+ //
+ // Notes:
+ //
+ // - Should we truncate sha256 hash? Maybe to 16 chars (this is what we
+ // use for abbreviated git commit id in the version module). Also in
+ // auth? Add abbreviated_string(size_t) to sha1 and sha256 classes?
+ //
+ // @@ If to truncate hash for auth, we would still need to store the full
+ // fingerprint in the certificate object as rep-info needs it to print.
+ // Leaving the certificate unchanged and truncating fingerprint on the
+ // fly for the file naming seems wrong (good to have the certificate
+ // file name to match the id). Probably it makes sense to make the
+ // certificate as follows:
+ //
+ // class certificate
+ // {
+ // public:
+ // string id; // SHA256 fingerprint truncated to 16 characters.
+ //
+ // string fingerprint; // Fingerprint canonical representation.
+ // ...
+ // };
+ //
+ // Yes, sounds good.
+ //
+ //
+ //
+
+ if (conf != nullptr && conf->empty ())
+ conf = dir_exists (bpkg_dir) ? &current_dir : nullptr;
+
+ assert (conf == nullptr || !conf->empty ());
+
+ dir_path h (sha256 (rl.canonical_name ()).abbreviated_string (16));
+
+ auto_rmdir rm (temp_dir / h);
+ dir_path& td (rm.path);
+
+ if (exists (td))
+ rm_r (td);
+
+ // If the git repository directory already exists, then we are fetching
+ // an already cloned repository. Move it to the temporary directory.
+ //
+ dir_path rd;
+ bool fetch (false);
+ if (conf != nullptr)
+ {
+ rd = *conf / repos_dir / h;
+
+ if (exists (rd))
+ {
+ mv (rd, td);
+ fetch = true;
+ }
+ }
+
+ if (fetch)
+ git_fetch (co, rl, td);
+ else
+ git_clone (co, rl, td);
+
+ if (!rd.empty ())
+ mv (td, rd);
+ else
+ // If there is no configuration directory then we leave the repository
+ // in the temporary directory.
+ //
+ rd = move (td);
+
+ rm.cancel ();
+
+ // @@ TODO
+ //
+ return rep_fetch_data ();
}
rep_fetch_data
diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx
index f32daa0..75ba102 100644
--- a/bpkg/utility.cxx
+++ b/bpkg/utility.cxx
@@ -22,22 +22,25 @@ namespace bpkg
const dir_path empty_dir_path;
const dir_path bpkg_dir (".bpkg");
- const dir_path certs_dir (dir_path (bpkg_dir) /= "certs");
+ const dir_path certs_dir (dir_path (bpkg_dir) /= "certificates");
+ const dir_path repos_dir (dir_path (bpkg_dir) /= "repositories");
- static dir_path tmp_dir_;
+ const dir_path current_dir (".");
+
+ dir_path temp_dir;
auto_rmfile
tmp_file (const string& p)
{
- assert (!tmp_dir_.empty ());
- return auto_rmfile (tmp_dir_ / path::traits::temp_name (p));
+ assert (!temp_dir.empty ());
+ return auto_rmfile (temp_dir / path::traits::temp_name (p));
}
auto_rmdir
tmp_dir (const string& p)
{
- assert (!tmp_dir_.empty ());
- return auto_rmdir (tmp_dir_ / dir_path (path::traits::temp_name (p)));
+ assert (!temp_dir.empty ());
+ return auto_rmdir (temp_dir / dir_path (path::traits::temp_name (p)));
}
void
@@ -56,16 +59,16 @@ namespace bpkg
mk (d); // We shouldn't need mk_p().
- tmp_dir_ = move (d);
+ temp_dir = move (d);
}
void
clean_tmp (bool ignore_error)
{
- if (!tmp_dir_.empty ())
+ if (!temp_dir.empty ())
{
- rm_r (tmp_dir_, true /* dir_itself */, 3, ignore_error);
- tmp_dir_.clear ();
+ rm_r (temp_dir, true /* dir_itself */, 3, ignore_error);
+ temp_dir.clear ();
}
}
@@ -213,6 +216,22 @@ namespace bpkg
}
}
+ void
+ mv (const dir_path& from, const dir_path& to)
+ {
+ if (verb >= 3)
+ text << "mv " << from << " to " << to; // Prints trailing slashes.
+
+ try
+ {
+ mvdir (from, to);
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to move directory " << from << " to " << to << ": " << e;
+ }
+ }
+
dir_path exec_dir;
void
diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx
index f4f8690..05835b0 100644
--- a/bpkg/utility.hxx
+++ b/bpkg/utility.hxx
@@ -52,8 +52,10 @@ namespace bpkg
// Widely-used paths.
//
- extern const dir_path bpkg_dir; // .bpkg/
- extern const dir_path certs_dir; // .bpkg/certs/
+ extern const dir_path bpkg_dir; // .bpkg/
+ extern const dir_path certs_dir; // .bpkg/certificates/
+ extern const dir_path repos_dir; // .bpkg/repositories/
+ extern const dir_path current_dir; // ./
// Temporary directory facility.
//
@@ -63,6 +65,8 @@ namespace bpkg
// you don't need to call init_tmp() explicitly except for certain special
// commands (like cfg-create).
//
+ extern dir_path temp_dir;
+
auto_rmfile
tmp_file (const string& prefix);
@@ -109,6 +113,9 @@ namespace bpkg
uint16_t verbosity = 3,
bool ignore_error = false);
+ void
+ mv (const dir_path& from, const dir_path& to);
+
// Process.
//
// By default the process command line is printed for verbosity >= 2
diff --git a/tests/buildfile b/tests/buildfile
index 1d7ede2..4716658 100644
--- a/tests/buildfile
+++ b/tests/buildfile
@@ -5,7 +5,8 @@
define common: file
common{*}: extension = test
-commons = common config auth remote
+commons = common config auth remote \
+ remote-git rep-fetch-git rep-fetch-git-branch rep-fetch-git-commit
# The common/ directory contains repositories that are reused, being symlinked
# in source repositories specific for testscripts.
diff --git a/tests/common/git/README b/tests/common/git/README
new file mode 100644
index 0000000..eac6765
--- /dev/null
+++ b/tests/common/git/README
@@ -0,0 +1,104 @@
+1. Local repositories.
+
+To modify the repositories that are used for git repository tests run
+
+$ ./init --unpack
+
+before modification, and
+
+$ ./pack
+
+afterwrds.
+
+Also note that config files under .git/ subdirectory refer to the submodule
+repositories using absolute paths. So prior to pulling in subproject directory
+(say in state0/libfoo.git/doc/style) you need to run the following commands,
+to make sure that the repository references match their current locations:
+
+$ git -C style.git submodule sync --recursive
+$ git -C libfoo.git submodule sync --recursive
+
+
+2. Remote repositories.
+
+To bootstrap the remote repositories run the following commands on build2.org
+host.
+
+$ cd /var/scm
+
+Create repositories, providing the proper project description:
+
+# bpkg test repository with doc basic style library (initial state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state0/style-basic.git
+
+# bpkg test repository with doc style library (initial state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state0/style.git
+
+# bpkg test repository with libbar library (initial state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state0/libbar.git
+
+# bpkg test repository with libfoo library (initial state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state0/libfoo.git
+
+# bpkg test repository with doc basic style library (final state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state1/style-basic.git
+
+# bpkg test repository with doc style library (final state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state1/style.git
+
+# bpkg test repository with libbaz library (final state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state1/libbaz.git
+
+# bpkg test repository with libfoo library (final state)
+#
+$ ./mkrepo testing/bpkg/unadv/rep-fetch/state1/libfoo.git
+
+
+# bpkg test repository with doc basic style library (advonly, initial state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state0/style-basic.git
+
+# bpkg test repository with doc style library (advonly, initial state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state0/style.git
+
+# bpkg test repository with libbar library (advonly, initial state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state0/libbar.git
+
+# bpkg test repository with libfoo library (advonly, initial state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state0/libfoo.git
+
+# bpkg test repository with doc basic style library (advonly, final state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state1/style-basic.git
+
+# bpkg test repository with doc style library (advonly, final state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state1/style.git
+
+# bpkg test repository with libbaz library (advonly, final state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state1/libbaz.git
+
+# bpkg test repository with libfoo library (advonly, final state)
+#
+$ ./mkrepo testing/bpkg/advonly/rep-fetch/state1/libfoo.git
+
+Add configuration options:
+
+$ for d in $(find . -type d -regex '\./testing/bpkg/.*/[^/]+\.git'); do \
+ git -C $d config receive.denyDeleteCurrent ignore \
+done
+
+$ for d in $(find . -type d -regex '\./testing/bpkg/advonly/.*/[^/]+\.git'); do \
+ git -C $d config uploadpack.allowAnySHA1InWant false \
+done
diff --git a/tests/common/git/init b/tests/common/git/init
new file mode 100755
index 0000000..c999347
--- /dev/null
+++ b/tests/common/git/init
@@ -0,0 +1,171 @@
+#! /bin/sh
+
+# Create git repositories from project directories/tarballs.
+#
+# Usage example:
+#
+# ./init [--unpack]
+#
+owd=`pwd`
+trap "{ cd $owd; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "error: $*"; exit 1; }
+function trace () { if [ "$verbose" == 'y' ]; then info "trace: $*"; fi }
+
+unpack=
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ --unpack)
+ unpack='y'
+ shift
+ ;;
+ *)
+ error "invalid option $1"
+ ;;
+ esac
+done
+
+# Unpack repositories if requested.
+#
+if [ -n "$unpack" ]; then
+ for f in */*.tar; do
+ rm -r -f ${f%.tar}.git
+ tar xf $f;
+ done
+fi
+
+# Create the initial state of the repositories libfoo.git, libbar.git,
+# style.git, and style-basic.git.
+#
+cd state0
+
+rm -f -r libfoo.git/.git
+rm -f libfoo.git/.gitmodules
+rm -f libfoo.git/README
+rm -f -r libfoo.git/libbar
+rm -f -r libfoo.git/doc/style
+
+rm -f -r libbar.git/.git
+
+rm -f -r style.git/.git
+rm -f -r style.git/basic
+
+rm -f -r style-basic.git/.git
+rm -f style-basic.git/README
+rm -f style-basic.git/INSTALL
+
+# Create master branch for style-basic.git.
+#
+git -C style-basic.git init
+git -C style-basic.git add '*'
+git -C style-basic.git commit -am 'Create'
+
+# Create stable branch for style-basic.
+#
+git -C style-basic.git branch stable
+git -C style-basic.git checkout stable
+touch style-basic.git/README
+git -C style-basic.git add README
+git -C style-basic.git commit -am 'README'
+
+# Create master branch for style.git, adding style-basic.git as a submodule.
+#
+git -C style.git init
+git -C style.git add '*'
+git -C style.git submodule add ../style-basic.git basic # The stable branch.
+git -C style.git commit -am 'Create'
+
+# Make style.git to refer an unadvertised reference, commiting into the stable
+# branch of style-basic.git.
+#
+touch style-basic.git/INSTALL
+git -C style-basic.git add INSTALL
+git -C style-basic.git commit -am 'INSTALL'
+git -C style-basic.git checkout master
+
+# Create master branch for libbar.git.
+#
+git -C libbar.git init
+git -C libbar.git add '*'
+git -C libbar.git commit -am 'Create'
+
+# Create master branch for libfoo.git, adding style.git and libbar.git as
+# submodules.
+#
+git -C libfoo.git init
+git -C libfoo.git add '*'
+git -C libfoo.git submodule add ../style.git doc/style
+git -C libfoo.git submodule add ../libbar.git libbar
+git -C libfoo.git submodule update --init --recursive # Updates doc/style/basic.
+git -C libfoo.git commit -am 'Create'
+
+# Add tags for libfoo.git.
+#
+git -C libfoo.git tag 'lightweight_tag'
+git -C libfoo.git tag -a 'annotated_tag' -m 'annotated_tag'
+
+# Advance the master branch to make tags not to mark a branch tip.
+#
+touch libfoo.git/README
+git -C libfoo.git add README
+git -C libfoo.git commit -am 'README'
+
+# Create the modified state of the repositories, replacing libbar.git submodule
+# of libfoo with the newly created libbaz.git repository. Also advance master
+# branches and tags for libfoo.git and it's submodule style.git.
+#
+cd ../state1
+
+# Copy repositories initial state.
+#
+for d in ../state0/*.git; do
+ rm -f -r $(basename $d)
+ cp -r $d .
+done
+
+# Create libbaz.git repository.
+#
+rm -f -r libbaz.git/.git
+
+git -C libbaz.git init
+git -C libbaz.git add '*'
+git -C libbaz.git commit -am 'Create'
+
+# Sync submodule references with their new locations.
+#
+for d in *.git; do
+ git -C $d submodule sync --recursive
+done
+
+# Advance style.git master branch.
+#
+touch style.git/README
+git -C style.git add README
+git -C style.git commit -am 'README'
+
+# Advance libfoo.git master branch.
+#
+git -C libfoo.git submodule update --init --remote # Pull style only.
+git -C libfoo.git commit -am 'Update style'
+
+git -C libfoo.git rm -r tests
+git -C libfoo.git commit -am 'Remove tests'
+
+git -C libfoo.git submodule deinit libbar
+git -C libfoo.git rm libbar
+git -C libfoo.git commit -am 'Remove libbar'
+rm -f -r libbar.git
+
+git -C libfoo.git submodule add ../libbaz.git libbaz
+git -C libfoo.git submodule update --init libbaz
+git -C libfoo.git commit -am 'Add libbaz'
+
+git -C libfoo.git tag -f 'lightweight_tag'
+git -C libfoo.git tag -f -a 'annotated_tag' -m 'annotated_tag'
+
+touch libfoo.git/INSTALL
+git -C libfoo.git add INSTALL
+git -C libfoo.git commit -am 'INSTALL'
diff --git a/tests/common/git/pack b/tests/common/git/pack
new file mode 100755
index 0000000..f53e794
--- /dev/null
+++ b/tests/common/git/pack
@@ -0,0 +1,29 @@
+#! /bin/sh
+
+# Move git projects to tar archives.
+#
+# Usage example:
+#
+# ./pack
+#
+owd=`pwd`
+trap "{ cd $owd; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+projects=('state0/libfoo' 'state0/libbar' 'state0/style' 'state0/style-basic' \
+ 'state1/libfoo' 'state1/libbaz' 'state1/style' 'state1/style-basic')
+
+for p in "${projects[@]}"; do
+ d=$p.git
+ if [ ! -d $d ]; then
+ error "$d directory not found"
+ fi
+
+ git -C $d submodule sync --recursive
+
+ tar cf $p.tar $d
+ rm -r -f $d
+done
diff --git a/tests/common/git/state0/libbar.tar b/tests/common/git/state0/libbar.tar
new file mode 100644
index 0000000..4eff7d2
--- /dev/null
+++ b/tests/common/git/state0/libbar.tar
Binary files differ
diff --git a/tests/common/git/state0/libfoo.tar b/tests/common/git/state0/libfoo.tar
new file mode 100644
index 0000000..3938070
--- /dev/null
+++ b/tests/common/git/state0/libfoo.tar
Binary files differ
diff --git a/tests/common/git/state0/style-basic.tar b/tests/common/git/state0/style-basic.tar
new file mode 100644
index 0000000..aabbbcd
--- /dev/null
+++ b/tests/common/git/state0/style-basic.tar
Binary files differ
diff --git a/tests/common/git/state0/style.tar b/tests/common/git/state0/style.tar
new file mode 100644
index 0000000..1821210
--- /dev/null
+++ b/tests/common/git/state0/style.tar
Binary files differ
diff --git a/tests/common/git/state1/libbaz.tar b/tests/common/git/state1/libbaz.tar
new file mode 100644
index 0000000..de6e3d9
--- /dev/null
+++ b/tests/common/git/state1/libbaz.tar
Binary files differ
diff --git a/tests/common/git/state1/libfoo.tar b/tests/common/git/state1/libfoo.tar
new file mode 100644
index 0000000..ac5d10d
--- /dev/null
+++ b/tests/common/git/state1/libfoo.tar
Binary files differ
diff --git a/tests/common/git/state1/style-basic.tar b/tests/common/git/state1/style-basic.tar
new file mode 100644
index 0000000..94dc12b
--- /dev/null
+++ b/tests/common/git/state1/style-basic.tar
Binary files differ
diff --git a/tests/common/git/state1/style.tar b/tests/common/git/state1/style.tar
new file mode 100644
index 0000000..9e997ae
--- /dev/null
+++ b/tests/common/git/state1/style.tar
Binary files differ
diff --git a/tests/common/libfoo-1.1.0/build/bootstrap.build b/tests/common/libfoo-1.1.0/build/bootstrap.build
index 54f267e..eb90fee 100644
--- a/tests/common/libfoo-1.1.0/build/bootstrap.build
+++ b/tests/common/libfoo-1.1.0/build/bootstrap.build
@@ -1,2 +1,2 @@
-project = fetch-libfoo
+project = libfoo
using config
diff --git a/tests/pkg-clean.test b/tests/pkg-clean.test
index 18c2a79..6f393af 100644
--- a/tests/pkg-clean.test
+++ b/tests/pkg-clean.test
@@ -67,7 +67,8 @@ $* 2>>EOE != 0
:
{
+$clone_cfg
- +$rep_add $rep/hello && $rep_fetch --trust $cert_fp &cfg/.bpkg/certs/***
+ +$rep_add $rep/hello
+ +$rep_fetch --trust $cert_fp &cfg/.bpkg/certificates/**
: no-such-package
:
diff --git a/tests/pkg-configure.test b/tests/pkg-configure.test
index d67e289..4253cf5 100644
--- a/tests/pkg-configure.test
+++ b/tests/pkg-configure.test
@@ -101,7 +101,8 @@ $* libhello libhello 2>>EOE != 0
:
{
+$clone_cfg
- +$rep_add $rep/hello && $rep_fetch --trust $cert_fp &cfg/.bpkg/certs/***
+ +$rep_add $rep/hello
+ +$rep_fetch --trust $cert_fp &cfg/.bpkg/certificates/**
: no-such-package
:
diff --git a/tests/pkg-fetch.test b/tests/pkg-fetch.test
index 3d5b40b..6ee2043 100644
--- a/tests/pkg-fetch.test
+++ b/tests/pkg-fetch.test
@@ -187,7 +187,8 @@ $* libfoo/1.0.0 2>>/EOE != 0
:
{
$clone_cfg;
- $rep_add $rep/hello && $rep_fetch --trust $cert_fp &cfg/.bpkg/certs/***;
+ $rep_add $rep/hello;
+ $rep_fetch --trust $cert_fp &cfg/.bpkg/certificates/**;
$* libhello/1.0.0 2>>~%EOE%;
%.*
diff --git a/tests/pkg-unpack.test b/tests/pkg-unpack.test
index 83fa9a6..236744f 100644
--- a/tests/pkg-unpack.test
+++ b/tests/pkg-unpack.test
@@ -185,7 +185,7 @@ $* 2>>EOE != 0
:
{
$clone_cfg;
- $rep_add $rep/hello && $rep_fetch --trust $cert_fp &cfg/.bpkg/certs/***;
+ $rep_add $rep/hello && $rep_fetch --trust $cert_fp &cfg/.bpkg/certificates/**;
$pkg_fetch libhello/1.0.0;
$* libhello 2>'unpacked libhello/1.0.0';
diff --git a/tests/pkg-update.test b/tests/pkg-update.test
index 4e92650..2d82934 100644
--- a/tests/pkg-update.test
+++ b/tests/pkg-update.test
@@ -71,7 +71,8 @@ $* 2>>EOE != 0
:
{
+$clone_cfg
- +$rep_add $rep/hello && $rep_fetch --trust $cert_fp &cfg/.bpkg/certs/***
+ +$rep_add $rep/hello
+ +$rep_fetch --trust $cert_fp &cfg/.bpkg/certificates/**
: no-such-package
:
diff --git a/tests/publish b/tests/publish
index 3a6b0d8..253c703 100755
--- a/tests/publish
+++ b/tests/publish
@@ -1,4 +1,4 @@
-#!/bin/sh
+#! /usr/bin/env bash
# Some commonly useful addtional options that can be specified via the
# command line:
@@ -6,6 +6,24 @@
# --dry-run
# --progress
#
+owd=`pwd`
+trap "{ cd $owd; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+echo_git=
+
+# Keep arguments intact for the future use with rsync.
+#
+for o in "$@"; do
+ case $o in
+ --dry-run)
+ echo_git=echo
+ ;;
+ esac
+done
+
+# Publish bpkg test repositories.
+#
rsync -v -rlpt --copy-unsafe-links \
--prune-empty-dirs --delete-after --delete-excluded $* \
--include '*/' \
@@ -15,3 +33,53 @@ rsync -v -rlpt --copy-unsafe-links \
--include 'signature' \
--exclude '*' \
test/*/pkg/1/build2.org/ build2.org:/var/pkg/1/
+
+# Publish git test repositories.
+#
+urls=('git.build2.org:/var/scm/testing/bpkg/unadv' \
+ 'git.build2.org:/var/scm/testing/bpkg/advonly')
+
+# Find git repository directories to publish.
+#
+for r in $(find test -type d -regex '.*/git/.*/[^/]+\.git'); do
+ br="${r/\/git\//\/git-bare\/}" # Bare repository directory.
+
+ # Make base repositories from the test ones.
+ #
+ rm -r -f $br
+ mkdir -p $(dirname $br)
+
+ git clone --bare $r $br
+
+ # Subdirectory that is relative to git-bare/.
+ #
+ d=$(echo $br | sed -n -e 's%.*/git-bare/\(.*\)%\1%p')
+
+ for u in "${urls[@]}"; do
+
+ # Point the bare repository origin to the remote repository.
+ #
+ git -C $br config remote.origin.url "$u/$d"
+
+ # Delete all remote branches and tags.
+ #
+ while read commit ref; do
+ $echo_git git -C $br push origin ":$ref"
+ done < <(git -C $br ls-remote --refs origin)
+
+ # Push local branches.
+ #
+ while read branch; do
+ $echo_git git -C $br push --tags origin "$branch:$branch"
+ done < <(git -C $br for-each-ref --format='%(refname:short)' 'refs/heads/')
+ done
+
+ # Prepare the bare repository for serving via the HTTPS dumb protocol.
+ #
+ git -C $br update-server-info --force
+done
+
+# Publish git repositories that are served via the HTTPS dumb protocol.
+#
+rsync -v -rlpt --copy-unsafe-links --delete-after $* \
+test/*/git-bare/ build2.org:/var/pkg/git/
diff --git a/tests/remote-git.test b/tests/remote-git.test
new file mode 100644
index 0000000..24b31e4
--- /dev/null
+++ b/tests/remote-git.test
@@ -0,0 +1,35 @@
+# file : tests/remote-git.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Tests for commands that accept git repository location must be able to run
+# regardless whether the repository is local or remote. They also must be able
+# to create the repository used for testing at the specified path, so being
+# published to build2.org it can be used for the remote testing. Note that
+# prior to publishing repositories tests must be performed with the
+# config.test.output=keep variable override, so their working directories (that
+# contain repositories produced) are not cleaned up.
+#
+
+# Output directory path that testscripts must use to prepare repositories
+# required by tests they contains.
+#
+out_git = $canonicalize([dir_path] $~/git/$cmd)
+
+# If $remote is true then remote repositories locations must be used for
+# tests.
+#
+remote = $config.bpkg.test.remote
+
++if ($remote != true)
+ rep_git_local = ($cxx.target.class != 'windows' \
+ ? "file://$out_git" \
+ : "file:/$regex.replace($out_git, '\\', '/')")
+
+ mkdir -p $out_git
+else
+ rep_git_https_dumb = "https://build2.org/bpkg/git/$cmd"
+ rep_git_https_smart = "https://git.build2.org/testing/bpkg/advonly/$cmd"
+ rep_git_https_smart_unadv = "https://git.build2.org/testing/bpkg/unadv/$cmd"
+ rep_git_git = "git://git.build2.org/testing/bpkg/unadv/$cmd"
+end
diff --git a/tests/remote.test b/tests/remote.test
index fdd3fbb..581d8be 100644
--- a/tests/remote.test
+++ b/tests/remote.test
@@ -2,7 +2,7 @@
# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-# Tests for commands that accept repository location must be able to run
+# Tests for commands that accept bpkg repository location must be able to run
# regardless whether the repository is local or remote. They also must be able
# to create the repository used for testing at the specified path, so being
# published to build2.org it can be used for the remote testing. Note that
diff --git a/tests/rep-add.test b/tests/rep-add.test
index 4e203cc..a84f725 100644
--- a/tests/rep-add.test
+++ b/tests/rep-add.test
@@ -38,7 +38,7 @@
: git-no-branch
:
$* 'git://example.org/repo' 2>>EOE != 0
- error: invalid git repository location 'git://example.org/repo': missing branch/tag for git repository
+ error: invalid git repository location 'git://example.org/repo': missing branch/tag or commit id for git repository
EOE
: bpkg-git-scheme
@@ -88,7 +88,7 @@
$clone_cfg && mkdir -p repo/.git;
$* 'repo' 2>>~%EOE% != 0
- %error: invalid git repository location '.+repo': missing branch/tag for git repository%
+ %error: invalid git repository location '.+repo': missing branch/tag or commit id for git repository%
EOE
: file-bpkg
@@ -107,11 +107,11 @@
$clone_cfg;
$* ./1/bar/stable 2>>/~%EOE%;
- %added repository .+/relative-path/bar/stable%
+ %added repository bpkg:.+/relative-path/bar/stable%
EOE
$* ./1/../1/bar/stable 2>>/~%EOE% != 0
- %error: .+/relative-path/bar/stable is already a repository of this configuration%
+ %error: bpkg:.+/relative-path/bar/stable is already a repository of this configuration%
EOE
}
@@ -121,17 +121,21 @@
$clone_cfg;
$* $~/1/foo/stable 2>>/~%EOE%;
- %added repository .+/absolute-path/foo/stable%
+ %added repository bpkg:.+/absolute-path/foo/stable%
EOE
$* $~/1/../1/foo/stable 2>>/~%EOE% != 0
- %error: .+/absolute-path/foo/stable is already a repository of this configuration%
+ %error: bpkg:.+/absolute-path/foo/stable is already a repository of this configuration%
EOE
}
: remote-url
:
{
+ +$clone_cfg
+
+ : bpkg
+ :
$clone_cfg;
$* 'http://pkg.example.org/1/testing' 2>>~%EOE%;
@@ -141,4 +145,16 @@
$* 'https://www.example.org/1/testing' 2>>~%EOE% != 0
%error: bpkg:example.org/testing is already a repository of this configuration%
EOE
+
+ : git
+ :
+ $clone_cfg;
+
+ $* 'git://example.org/testing#master' 2>>~%EOE%;
+ %added repository git:example.org/testing%
+ EOE
+
+ $* 'git://www.example.org/testing#master' 2>>~%EOE% != 0
+ %error: git:example.org/testing is already a repository of this configuration%
+ EOE
}
diff --git a/tests/rep-auth.test b/tests/rep-auth.test
index 5fa3568..b0c28cf 100644
--- a/tests/rep-auth.test
+++ b/tests/rep-auth.test
@@ -165,7 +165,7 @@ sc = " " # Space character to append to here-document line when required.
:
{
+$clone_root_cfg && $rep_add $rep/signed
- rep_fetch += --auth all &?cfg/.bpkg/certs/***
+ rep_fetch += --auth all &?cfg/.bpkg/certificates/**
: no-auth
:
@@ -312,8 +312,11 @@ sc = " " # Space character to append to here-document line when required.
$clone_root_cfg;
rep_info += -d cfg;
- $rep_info --trust "$cert_fp" &cfg/.bpkg/certs/*** >'name:build2.org';
- $rep_info >'name:build2.org'
+ $rep_info --trust "$cert_fp" &cfg/.bpkg/certificates/** >>EOO;
+ name:build2.org
+ EOO
+
+ $rep_info >'name:build2.org'
}
}
diff --git a/tests/rep-fetch-git-branch.test b/tests/rep-fetch-git-branch.test
new file mode 100644
index 0000000..94ca70e
--- /dev/null
+++ b/tests/rep-fetch-git-branch.test
@@ -0,0 +1,152 @@
+# file : tests/rep-fetch-git-branch.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+reason_dumb = ' (dumb HTTP)'
+reason_unadv = ' (unadvertised commit)'
+
+warn_dumb='
+warning: fetching over dumb HTTP, no progress will be displayed'
+
+: clone
+:
+{
+ $clone_root_cfg && $rep_add "$rep/state0/libfoo.git#$branch";
+
+ # Note that the commit for doc/style/basic submodule is not at the branch tip
+ # and so is not advertised.
+ #
+ if ($git_protocol == 'local' || \
+ $git_protocol == 'https-smart' || \
+ $git_protocol == 'git')
+ warn1 = '%.{0}'
+ warn2 = '%.{0}'
+ warn3 = "warning: fetching whole repository history for submodule 'doc/style/basic'$reason_unadv"
+ fetch = '%.{0}'
+ warn4 = '%.{0}'
+ elif ($git_protocol == 'https-dumb')
+ warn1 = "warning: fetching whole branch history$reason_dumb$warn_dumb"
+ warn2 = "warning: fetching whole repository history for submodule 'doc/style'$reason_dumb$warn_dumb"
+ warn3 = "warning: fetching whole repository history for submodule 'doc/style/basic'$reason_dumb$warn_dumb"
+ fetch = '%.{0}'
+ warn4 = "warning: fetching whole repository history for submodule 'libbar'$reason_dumb$warn_dumb"
+ elif ($git_protocol == 'https-smart-unadv')
+ warn1 = '%.{0}'
+ warn2 = '%.{0}'
+ warn3 = '%.{0}'
+ fetch = "%Fetching in '.+style/basic'.+%"
+ warn4 = '%.{0}'
+ end;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+libfoo%
+ $warn1
+ %Cloning into '.+$branch'.+%
+ %Submodule 'doc/style' .+ registered for path 'doc/style'%
+ %Submodule 'libbar' .+ registered for path 'libbar'%
+ $warn2
+ %Cloning into '.+doc/style'.+%
+ %Submodule path 'doc/style': checked out .+%
+ %Submodule 'basic' .+ registered for path 'doc/style/basic'%
+ $warn3
+ %Cloning into '.+doc/style/basic'.+%
+ $fetch
+ %Submodule path 'doc/style/basic': checked out .+%
+ $warn4
+ %Cloning into '.+libbar'.+%
+ %Submodule path 'libbar': checked out .+%
+ 0 package\(s\) in 1 repository\(s\)
+ EOE
+}
+
+: fetch
+:
+{
+ : unchanged
+ :
+ {
+ $clone_root_cfg && $rep_add "$rep/state0/libfoo.git#$branch";
+ $* >! 2>!;
+
+ if ($git_protocol == 'https-dumb')
+ warn = "warning: fetching whole branch history$reason_dumb$warn_dumb"
+ else
+ warn = '%.{0}'
+ end;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+libfoo%
+ $warn
+ %Fetching in '.+$branch'.+%
+ 0 package\(s\) in 1 repository\(s\)
+ EOE
+ }
+
+ : changed
+ :
+ {
+ g = git -C
+ u = "$rep_git/state1"
+
+ $clone_root_cfg && $rep_add "$rep/state0/libfoo.git#$branch";
+
+ # Extract the repository path from the output line like this:
+ #
+ # Cloning into 'cfg\.bpkg\tmp\f9be881264703b5d\master'...
+ #
+ $* 2>&1 | sed -n -e "s/Cloning into '\(.+$branch\)'\.{3}/\$1/p" | \
+ sed -n -e 's%(.+[\\/])tmp([\\/].+)%$1repositories$2%p' | \
+ set r;
+
+ $g "$r" config remote.origin.url "$u/libfoo.git";
+
+ $g "$r" config submodule.libbar.url "$u/libbar.git";
+ $g "$r" config submodule.doc/style.url "$u/style.git";
+
+ $g "$r/libbar" config remote.origin.url "$u/libbar.git";
+
+ $g "$r/doc/style" config remote.origin.url "$u/style.git";
+ $g "$r/doc/style" config submodule.basic.url "$u/style-basic.git";
+
+ $g "$r/doc/style/basic" config remote.origin.url "$u/style-basic.git";
+
+ # Preconditions.
+ #
+ test -f $r/tests/TODO;
+ test -f $r/libbar/manifest;
+ test -f $r/doc/style/README != 0;
+ test -d $r/libbaz != 0;
+
+ if ($git_protocol == 'https-dumb')
+ warn1 = "warning: fetching whole branch history$reason_dumb$warn_dumb"
+ warn2 = "warning: fetching whole repository history for submodule 'doc/style'$reason_dumb$warn_dumb"
+ warn3 = "warning: fetching whole repository history for submodule 'libbaz'$reason_dumb$warn_dumb"
+ else
+ warn1 = '%.{0}'
+ warn2 = '%.{0}'
+ warn3 = '%.{0}'
+ end;
+
+ $* 2>>~"%EOE%" 1>&2;
+ %fetching git:.+libfoo%
+ $warn1
+ %Fetching in '.+$branch'.+%
+ %warning: unable to rmdir '?libbar'?:.+%
+ %Submodule 'libbaz' .+ registered for path 'libbaz'%
+ $warn2
+ %Fetching in '.+doc/style'.+%
+ %Submodule path 'doc/style': checked out .+%
+ $warn3
+ %Cloning into '.+libbaz'.+%
+ %Submodule path 'libbaz': checked out .+%
+ 0 package\(s\) in 1 repository\(s\)
+ EOE
+
+ # Postconditions.
+ #
+ test -d $r/tests != 0;
+ test -d $r/libbar != 0;
+ test -f $r/doc/style/README;
+ test -f $r/libbaz/manifest
+ }
+}
diff --git a/tests/rep-fetch-git-commit.test b/tests/rep-fetch-git-commit.test
new file mode 100644
index 0000000..d0d757c
--- /dev/null
+++ b/tests/rep-fetch-git-commit.test
@@ -0,0 +1,124 @@
+# file : tests/rep-fetch-git-commit.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+reason_dumb = ' (dumb HTTP)'
+reason_unadv = ' (unadvertised commit)'
+
+warn_dumb='
+warning: fetching over dumb HTTP, no progress will be displayed'
+
+ +git clone "$rep_git/state0/style-basic.git" 2>! &style-basic/***
+
+: unadvertised
+:
+{
+ +git -C ../style-basic log '--pretty=format:%H' --all --grep='README' | \
+ set commit
+
+ : no-branch
+ :
+ {
+ $clone_root_cfg && $rep_add "$rep/state0/style-basic.git#@$commit";
+
+ if ($git_protocol == 'https-smart-unadv')
+ warn = '%.{0}'
+ fetch = "%Fetching in '.+$commit'.+%"
+ else
+ warn = "warning: fetching whole repository history"
+
+ if ($git_protocol == 'https-dumb')
+ warn = "$warn$reason_dumb$warn_dumb"
+ else
+ warn = "$warn$reason_unadv"
+ end
+
+ fetch = '%.{0}'
+ end;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+style-basic%
+ $warn
+ %Cloning into '.+$commit'.+%
+ $fetch
+ 0 package\(s\) in 1 repository\(s\)
+ EOE
+ }
+
+ : branch
+ :
+ {
+ $clone_root_cfg && $rep_add "$rep/state0/style-basic.git#stable@$commit";
+
+ if ($git_protocol == 'https-smart-unadv')
+ warn = '%.{0}'
+ fetch = "%Fetching in '.+stable'.+%"
+ else
+ warn = "warning: fetching whole branch history"
+
+ if ($git_protocol == 'https-dumb')
+ warn = "$warn$reason_dumb$warn_dumb"
+ else
+ warn = "$warn$reason_unadv"
+ end
+
+ fetch = '%.{0}'
+ end;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+style-basic%
+ $warn
+ %Cloning into '.+stable'.+%
+ $fetch
+ 0 package\(s\) in 1 repository\(s\)
+ EOE
+ }
+}
+
+: advertised
+:
+{
+ +git -C ../style-basic log '--pretty=format:%H' --all --grep='INSTALL' | \
+ set commit
+
+ : no-branch
+ :
+ {
+ $clone_root_cfg && $rep_add "$rep/state0/style-basic.git#@$commit";
+
+ if ($git_protocol == 'https-dumb')
+ warn = "warning: fetching whole repository history$reason_dumb$warn_dumb"
+ fetch = '%.{0}'
+ else
+ warn = '%.{0}'
+ fetch = "%Fetching in '.+$commit'.+%"
+ end;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+style-basic%
+ $warn
+ %Cloning into '.+$commit'.+%
+ $fetch
+ 0 package\(s\) in 1 repository\(s\)
+ EOE
+ }
+
+ : branch
+ :
+ {
+ $clone_root_cfg && $rep_add "$rep/state0/style-basic.git#stable@$commit";
+
+ if ($git_protocol == 'https-dumb')
+ warn ="warning: fetching whole branch history$reason_dumb$warn_dumb"
+ else
+ warn = '%.{0}'
+ end;
+
+ $* 2>>~"%EOE%"
+ %fetching git:.+style-basic%
+ $warn
+ %Cloning into '.+stable'.+%
+ 0 package\(s\) in 1 repository\(s\)
+ EOE
+ }
+}
diff --git a/tests/rep-fetch-git.test b/tests/rep-fetch-git.test
new file mode 100644
index 0000000..1330d20
--- /dev/null
+++ b/tests/rep-fetch-git.test
@@ -0,0 +1,62 @@
+# file : tests/rep-fetch-git.test
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# All tests use the same repository infrastructure present in the initial and
+# the final states. See tests/common/git/init script for more details.
+#
+
+rep_add += --type git -d cfg 2>!
+test.cleanups += &cfg/.bpkg/repositories/*/***
+
++if ($git_protocol == 'local')
+ rep = "$rep_git_local"
+elif ($git_protocol == 'https-dumb')
+ rep = "$rep_git_https_dumb"
+elif ($git_protocol == 'https-smart')
+ rep = "$rep_git_https_smart"
+elif ($git_protocol == 'https-smart-unadv')
+ rep = "$rep_git_https_smart_unadv"
+elif ($git_protocol == 'git')
+ rep = "$rep_git_git"
+else
+ exit "unexpected git protocol '$git_protocol'"
+end
+
+# Repository URL prefix for use with git commands.
+#
+# Note that git supports none of the standard 'file:' URL notations on Windows,
+# so we produce one that is acceptable for git.
+#
++if ($git_protocol == 'local' && $cxx.target.class == 'windows')
+ rep_git = "$regex.replace($rep, '^file:/', 'file://')"
+else
+ rep_git = "$rep"
+end
+
+: branch
+:
+{
+ branch = 'master'
+ .include rep-fetch-git-branch.test
+}
+
+: lightweight-tag
+:
+{
+ branch = 'lightweight_tag'
+ .include rep-fetch-git-branch.test
+}
+
+: annotated-tag
+:
+{
+ branch = 'annotated_tag'
+ .include rep-fetch-git-branch.test
+}
+
+: commit
+:
+{
+ .include rep-fetch-git-commit.test
+}
diff --git a/tests/rep-fetch.test b/tests/rep-fetch.test
index c9f026d..e18c8fa 100644
--- a/tests/rep-fetch.test
+++ b/tests/rep-fetch.test
@@ -2,7 +2,7 @@
# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
-.include common.test auth.test config.test remote.test
+.include common.test auth.test config.test remote.test remote-git.test
# Source repository:
#
@@ -19,6 +19,7 @@
# | | ../foo/testing (prerequisite)
# | |-- libbar-1.1.1.tar.gz -> libfoo >= 1.1.0
# | `-- repositories
+# |
# |-- foo
# | |-- stable
# | | |-- libfoo-1.0.0.tar.gz
@@ -26,9 +27,12 @@
# | `-- testing -> stable (complement)
# | |-- libfoo-1.1.0.tar.gz
# | `-- repositories
+# |
# `-- hello
-# |-- libhello-1.0.0.tar.gz
-# `-- repositories
+# | |-- libhello-1.0.0.tar.gz
+# | `-- repositories
+# |
+# `-- git/* (see rep-fetch-git.test)
# Prepare repositories used by tests if running in the local mode.
#
@@ -53,11 +57,27 @@
$rep_create $out/bar/stable &$out/bar/stable/packages
$rep_create $out/bar/testing &$out/bar/testing/packages
$rep_create $out/bar/unstable &$out/bar/unstable/packages
-end
-
-test.options += --auth all
-rep_add += -d cfg 2>!
+ # Create git repositories.
+ #
+ # Note that we can expect that the tar program is present on the platform. We
+ # will use the same options as we do for unpacking of bpkg packages (see
+ # pkg-unpack.cxx).
+ #
+ x = ($cxx.target.class != 'windows' \
+ ? tar -C $out_git -xf \
+ : tar -C $regex.replace($out_git, '\\', '/') --force-local -xf)
+
+ $x $src/git/state0/libfoo.tar
+ $x $src/git/state0/libbar.tar
+ $x $src/git/state0/style.tar
+ $x $src/git/state0/style-basic.tar &$out_git/state0/***
+
+ $x $src/git/state1/libfoo.tar
+ $x $src/git/state1/libbaz.tar
+ $x $src/git/state1/style.tar
+ $x $src/git/state1/style-basic.tar &$out_git/state1/***
+end
: no-repositories
:
@@ -67,68 +87,114 @@ $* 2>>/EOE != 0
info: use 'bpkg rep-add' to add a repository
EOE
-: hello
+: bpkg-repositories
:
{
- $clone_cfg && $rep_add $rep/hello;
-
- $* --trust $cert_fp 2>>EOE &cfg/.bpkg/certs/***;
- fetching bpkg:build2.org/rep-fetch/hello
- 1 package(s) in 1 repository(s)
- EOE
-
- $* 2>>EOE
- fetching bpkg:build2.org/rep-fetch/hello
- 1 package(s) in 1 repository(s)
- EOE
+ test.options += --auth all
+
+ rep_add += -d cfg 2>!
+
+ : hello
+ :
+ {
+ $clone_root_cfg && $rep_add $rep/hello;
+
+ $* --trust $cert_fp 2>>EOE &cfg/.bpkg/certificates/**;
+ fetching bpkg:build2.org/rep-fetch/hello
+ 1 package(s) in 1 repository(s)
+ EOE
+
+ $* 2>>EOE
+ fetching bpkg:build2.org/rep-fetch/hello
+ 1 package(s) in 1 repository(s)
+ EOE
+ }
+
+ : bar-unstable
+ :
+ {
+ $clone_root_cfg && $rep_add $rep/bar/unstable;
+
+ $* --trust-yes 2>>EOE;
+ fetching bpkg:build2.org/rep-fetch/bar/unstable
+ fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
+ fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
+ 5 package(s) in 5 repository(s)
+ EOE
+
+ $* 2>>EOE
+ fetching bpkg:build2.org/rep-fetch/bar/unstable
+ fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
+ fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
+ 5 package(s) in 5 repository(s)
+ EOE
+ }
+
+ : both
+ :
+ {
+ $clone_root_cfg && $rep_add $rep/hello && $rep_add $rep/bar/unstable;
+
+ $* --trust-yes 2>>EOE &cfg/.bpkg/certificates/**;
+ fetching bpkg:build2.org/rep-fetch/bar/unstable
+ fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
+ fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
+ fetching bpkg:build2.org/rep-fetch/hello
+ 6 package(s) in 6 repository(s)
+ EOE
+
+ $* 2>>EOE
+ fetching bpkg:build2.org/rep-fetch/bar/unstable
+ fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
+ fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
+ fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
+ fetching bpkg:build2.org/rep-fetch/hello
+ 6 package(s) in 6 repository(s)
+ EOE
+ }
}
-: bar-unstable
+: git-repositories
:
+if ($remote != true)
{
- $clone_cfg && $rep_add $rep/bar/unstable;
-
- $* --trust-yes 2>>EOE;
- fetching bpkg:build2.org/rep-fetch/bar/unstable
- fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
- fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
- 5 package(s) in 5 repository(s)
- EOE
-
- $* 2>>EOE
- fetching bpkg:build2.org/rep-fetch/bar/unstable
- fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
- fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
- 5 package(s) in 5 repository(s)
- EOE
+ git_protocol = 'local'
+ .include rep-fetch-git.test
}
-
-: both
-:
+else
{
- $clone_cfg && $rep_add $rep/hello && $rep_add $rep/bar/unstable;
-
- $* --trust-yes 2>>EOE &cfg/.bpkg/certs/***;
- fetching bpkg:build2.org/rep-fetch/bar/unstable
- fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
- fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
- fetching bpkg:build2.org/rep-fetch/hello
- 6 package(s) in 6 repository(s)
- EOE
-
- $* 2>>EOE
- fetching bpkg:build2.org/rep-fetch/bar/unstable
- fetching bpkg:build2.org/rep-fetch/foo/testing (prerequisite of bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/foo/stable (complements bpkg:build2.org/rep-fetch/foo/testing)
- fetching bpkg:build2.org/rep-fetch/bar/testing (complements bpkg:build2.org/rep-fetch/bar/unstable)
- fetching bpkg:build2.org/rep-fetch/bar/stable (complements bpkg:build2.org/rep-fetch/bar/testing)
- fetching bpkg:build2.org/rep-fetch/hello
- 6 package(s) in 6 repository(s)
- EOE
+ : https-dumb
+ :
+ {
+ git_protocol = 'https-dumb'
+ .include rep-fetch-git.test
+ }
+
+ : https-smart
+ :
+ {
+ git_protocol = 'https-smart'
+ .include rep-fetch-git.test
+ }
+
+ : https-smart-unadv
+ :
+ {
+ git_protocol = 'https-smart-unadv'
+ .include rep-fetch-git.test
+ }
+
+ : git
+ :
+ {
+ git_protocol = 'git'
+ .include rep-fetch-git.test
+ }
}
diff --git a/tests/rep-fetch/git/state0 b/tests/rep-fetch/git/state0
new file mode 120000
index 0000000..cfd06e4
--- /dev/null
+++ b/tests/rep-fetch/git/state0
@@ -0,0 +1 @@
+../../common/git/state0 \ No newline at end of file
diff --git a/tests/rep-fetch/git/state1 b/tests/rep-fetch/git/state1
new file mode 120000
index 0000000..7543de7
--- /dev/null
+++ b/tests/rep-fetch/git/state1
@@ -0,0 +1 @@
+../../common/git/state1 \ No newline at end of file