aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2015-09-25 15:19:16 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2015-09-25 15:19:16 +0200
commit8049a4de3a5d77d28f9dbc6b2d712b280548a08a (patch)
tree2ade1611947f617120380f22647cdf4d3df1c0ff
parent8aa9fd02fdbe7b2de9cbd02564431d74b620b1d9 (diff)
Implement archive fetching, complete pkg-fetch
-rw-r--r--bpkg/fetch6
-rw-r--r--bpkg/fetch.cxx260
-rw-r--r--bpkg/package4
-rw-r--r--bpkg/pkg-fetch.cxx62
-rw-r--r--bpkg/pkg-status.cxx2
-rw-r--r--bpkg/rep-fetch.cxx6
-rwxr-xr-xbpkg/test.sh41
7 files changed, 312 insertions, 69 deletions
diff --git a/bpkg/fetch b/bpkg/fetch
index 49d8cd9..9a58cc3 100644
--- a/bpkg/fetch
+++ b/bpkg/fetch
@@ -20,6 +20,12 @@ namespace bpkg
package_manifests fetch_packages (const dir_path&);
package_manifests fetch_packages (const common_options&,
const repository_location&);
+
+ path
+ fetch_archive (const common_options&,
+ const repository_location&,
+ const path& archive,
+ const dir_path& destdir);
}
#endif // BPKG_FETCH
diff --git a/bpkg/fetch.cxx b/bpkg/fetch.cxx
index 10acf1f..3ddd2e9 100644
--- a/bpkg/fetch.cxx
+++ b/bpkg/fetch.cxx
@@ -85,8 +85,12 @@ namespace bpkg
}
static process
- start_wget (const path& prog, const strings& ops, const string& url)
+ start_wget (const path& prog,
+ const strings& ops,
+ const string& url,
+ const path& out)
{
+ bool fo (!out.empty ()); // Output to file.
string ua (BPKG_USER_AGENT " wget/" + to_string (wget_major) + "."
+ to_string (wget_minor));
@@ -95,11 +99,12 @@ namespace bpkg
"-U", ua.c_str ()
};
- // Map verbosity level. If we are running quiet or at level 1,
- // then run wget quiet. At level 2 and 3 run it at the default
- // level (so we will print the command line and it will display
- // the progress, error messages, etc). Higher than that -- run
- // it with debug output.
+ // Map verbosity level. If we are running quiet or at level 1
+ // and the output is STDOUT, then run wget quiet. If at level
+ // 1 and the output is a file, then show the progress bar. At
+ // level 2 and 3 run it at the default level (so we will print
+ // the command line and it will display the progress, error
+ // messages, etc). Higher than that -- run it with debug output.
//
// In the wget world quiet means don't print anything, not even
// error messages. There is also the -nv mode (aka "non-verbose")
@@ -108,8 +113,20 @@ namespace bpkg
// If things go south, we suggest (in fetch_url()) below that the
// user re-runs the command with -v to see all the gory details.
//
- if (verb < 2)
+ if (verb < (fo ? 1 : 2))
args.push_back ("-q");
+ else if (fo && verb == 1)
+ {
+ // Wget 1.16 introduced the --show-progress option which in the
+ // quiet mode shows a nice and tidy progress bar (if only it also
+ // showed errors, then it would have been perfect).
+ //
+ if (wget_major > 1 || (wget_major == 1 && wget_minor >= 16))
+ {
+ args.push_back ("-q");
+ args.push_back ("--show-progress");
+ }
+ }
else if (verb > 3)
args.push_back ("-d");
@@ -119,15 +136,26 @@ namespace bpkg
for (const string& o: ops)
args.push_back (o.c_str ());
- args.push_back ("-O"); // Output to...
- args.push_back ("-"); // ...STDOUT.
+ // Output.
+ //
+ string o (fo ? out.leaf ().string () : "-");
+ args.push_back ("-O");
+ args.push_back (o.c_str ());
+
args.push_back (url.c_str ());
args.push_back (nullptr);
if (verb >= 2)
print_process (args);
- return process (args.data (), 0, -1); // Failure handled by the caller.
+ // If we are fetching into a file, change the wget's directory to
+ // that of the output file. We do it this way so that we end up with
+ // just the file name (rather than the whole path) in the progress
+ // report. Process exceptions must be handled by the caller.
+ //
+ return fo
+ ? process (out.directory ().string ().c_str (), args.data ())
+ : process (args.data (), 0, -1);
}
// curl
@@ -163,8 +191,13 @@ namespace bpkg
}
static process
- start_curl (const path& prog, const strings& ops, const string& url)
+ start_curl (const path& prog,
+ const strings& ops,
+ const string& url,
+ const path& out)
{
+ bool fo (!out.empty ()); // Output to file.
+
cstrings args {
prog.string ().c_str (),
"-f", // Fail on HTTP errors (e.g., 404).
@@ -172,16 +205,20 @@ namespace bpkg
"-A", (BPKG_USER_AGENT " curl")
};
- // Map verbosity level. If we are running quiet or at level 1,
- // then run curl quiet. At level 2 and 3 run it at the default
- // level (so we will print the command line and it will display
- // the progress). Higher than that -- run it verbose.
+ // Map verbosity level. If we are running quiet or at level 1
+ // and the output is STDOUT, then run curl quiet. If at level
+ // 1 and the output is a file, then show the progress bar. At
+ // level 2 and 3 run it at the default level (so we will print
+ // the command line and it will display its elaborate progress).
+ // Higher than that -- run it verbose.
//
- if (verb < 2)
+ if (verb < (fo ? 1 : 2))
{
args.push_back ("-s");
args.push_back ("-S"); // But show errors.
}
+ else if (fo && verb == 1)
+ args.push_back ("--progress-bar");
else if (verb > 3)
args.push_back ("-v");
@@ -191,13 +228,32 @@ namespace bpkg
for (const string& o: ops)
args.push_back (o.c_str ());
+ // Output. By default curl writes to STDOUT.
+ //
+ if (fo)
+ {
+ args.push_back ("-o");
+ args.push_back (out.string ().c_str ());
+ }
+
args.push_back (url.c_str ());
args.push_back (nullptr);
if (verb >= 2)
print_process (args);
+ else if (verb == 1 && fo)
+ //
+ // Unfortunately curl doesn't print the filename being fetched
+ // next to the progress bar. So the best we can do is print it
+ // on the previous line. Ugly, I know.
+ //
+ text << out.leaf () << ':';
- return process (args.data (), 0, -1); // Failure handled by the caller.
+ // Process exceptions must be handled by the caller.
+ //
+ return fo
+ ? process (args.data ())
+ : process (args.data (), 0, -1);
}
// fetch
@@ -235,22 +291,27 @@ namespace bpkg
}
static process
- start_fetch (const path& prog, const strings& ops, const string& url)
+ start_fetch (const path& prog,
+ const strings& ops,
+ const string& url,
+ const path& out)
{
- // -T|--timeout 120 seconds by default, leave it at that for now.
- // -n|--no-mtime
- //
+ bool fo (!out.empty ()); // Output to file.
+
cstrings args {
prog.string ().c_str (),
"--user-agent", (BPKG_USER_AGENT " fetch")
};
- // Map verbosity level. If we are running quiet or at level 1,
- // then run fetch quiet. At level 2 and 3 run it at the default
- // level (so we will print the command line and it will display
+ if (fo)
+ args.push_back ("--no-mtime"); // Use our own mtime.
+
+ // Map verbosity level. If we are running quiet then run fetch quiet.
+ // If we are at level 1 and we are fetching into a file or we are at
+ // level 2 or 3, then run it at the default level (so it will display
// the progress). Higher than that -- run it verbose.
//
- if (verb < 2)
+ if (verb < (fo ? 1 : 2))
args.push_back ("-q");
else if (verb > 3)
args.push_back ("-v");
@@ -261,15 +322,26 @@ namespace bpkg
for (const string& o: ops)
args.push_back (o.c_str ());
- args.push_back ("-o"); // Output to...
- args.push_back ("-"); // ...STDOUT.
+ // Output.
+ //
+ string o (fo ? out.leaf ().string () : "-");
+ args.push_back ("-o");
+ args.push_back (o.c_str ());
+
args.push_back (url.c_str ());
args.push_back (nullptr);
if (verb >= 2)
print_process (args);
- return process (args.data (), 0, -1); // Failure handled by the caller.
+ // If we are fetching into a file, change the fetch's directory to
+ // that of the output file. We do it this way so that we end up with
+ // just the file name (rather than the whole path) in the progress
+ // report. Process exceptions must be handled by the caller.
+ //
+ return fo
+ ? process (out.directory ().string ().c_str (), args.data ())
+ : process (args.data (), 0, -1);
}
// The dispatcher.
@@ -361,21 +433,25 @@ namespace bpkg
return fetch_kind;
}
+ // If out is empty, then fetch to STDOUT. In this case also don't
+ // show any progress unless we are running verbose.
+ //
static process
- start (const common_options& o, const string& url)
+ start (const common_options& o, const string& url, const path& out = path ())
{
- process (*start) (const path&, const strings&, const string&) = nullptr;
+ process (*f) (
+ const path&, const strings&, const string&, const path&) = nullptr;
switch (check (o))
{
- case wget: start = &start_wget; break;
- case curl: start = &start_curl; break;
- case fetch: start = &start_fetch; break;
+ case wget: f = &start_wget; break;
+ case curl: f = &start_curl; break;
+ case fetch: f = &start_fetch; break;
}
try
{
- return start (fetch_path, o.fetch_option (), url);
+ return f (fetch_path, o.fetch_option (), url, out);
}
catch (const process_error& e)
{
@@ -388,15 +464,9 @@ namespace bpkg
}
}
- template <typename M>
- static M
- fetch_url (const common_options& o,
- const string& host,
- uint16_t port,
- const path& file)
+ static string
+ to_url (const string& host, uint16_t port, const path& file)
{
- // Assemble the URL.
- //
//@@ Absolute path in URL: how is this going to work on Windows?
// Change to relative: watch for empty path.
//
@@ -409,7 +479,44 @@ namespace bpkg
url += ":" + to_string (port);
url += file.posix_string ();
+ return url;
+ }
+
+ static path
+ fetch_file (const common_options& o,
+ const string& host,
+ uint16_t port,
+ const path& f,
+ const dir_path& d)
+ {
+ path r (d / f.leaf ());
+
+ if (exists (r))
+ fail << "file " << r << " already exists";
+
+ string url (to_url (host, port, f));
+ process pr (start (o, url, r));
+
+ if (!pr.wait ())
+ {
+ // While it is reasonable to assuming the child process issued
+ // diagnostics, some may not mention the URL.
+ //
+ fail << "unable to fetch " << url <<
+ info << "re-run with -v for more information";
+ }
+ return r;
+ }
+
+ template <typename M>
+ static M
+ fetch_manifest (const common_options& o,
+ const string& host,
+ uint16_t port,
+ const path& f)
+ {
+ string url (to_url (host, port, f));
process pr (start (o, url));
try
@@ -452,9 +559,48 @@ namespace bpkg
throw failed ();
}
+ static path
+ fetch_file (const path& f, const dir_path& d)
+ {
+ if (!exists (f))
+ fail << "file " << f << " does not exist";
+
+ path r (d / f.leaf ());
+
+ if (exists (r))
+ fail << "file " << r << " already exists";
+
+ try
+ {
+ ifstream ifs (f.string (), ios::binary);
+ if (!ifs.is_open ())
+ fail << "unable to open " << f << " in read mode";
+
+ ofstream ofs (r.string (), ios::binary);
+ if (!ofs.is_open ())
+ fail << "unable to open " << r << " in write mode";
+
+ ifs.exceptions (ofstream::badbit | ofstream::failbit);
+ ofs.exceptions (ofstream::badbit | ofstream::failbit);
+
+ ofs << ifs.rdbuf();
+
+ // In case they throw.
+ //
+ ifs.close ();
+ ofs.close ();
+ }
+ catch (const iostream::failure&)
+ {
+ fail << "unable to copy " << f << " to " << r << ": io error";
+ }
+
+ return r;
+ }
+
template <typename M>
static M
- fetch_file (const path& f)
+ fetch_manifest (const path& f)
{
if (!exists (f))
fail << "file " << f << " does not exist";
@@ -485,7 +631,7 @@ namespace bpkg
repository_manifests
fetch_repositories (const dir_path& d)
{
- return fetch_file<repository_manifests> (d / repositories);
+ return fetch_manifest<repository_manifests> (d / repositories);
}
repository_manifests
@@ -496,8 +642,8 @@ namespace bpkg
path f (rl.path () / repositories);
return rl.remote ()
- ? fetch_url<repository_manifests> (o, rl.host (), rl.port (), f)
- : fetch_file<repository_manifests> (f);
+ ? fetch_manifest<repository_manifests> (o, rl.host (), rl.port (), f)
+ : fetch_manifest<repository_manifests> (f);
}
static const path packages ("packages");
@@ -505,7 +651,7 @@ namespace bpkg
package_manifests
fetch_packages (const dir_path& d)
{
- return fetch_file<package_manifests> (d / packages);
+ return fetch_manifest<package_manifests> (d / packages);
}
package_manifests
@@ -516,7 +662,23 @@ namespace bpkg
path f (rl.path () / packages);
return rl.remote ()
- ? fetch_url<package_manifests> (o, rl.host (), rl.port (), f)
- : fetch_file<package_manifests> (f);
+ ? fetch_manifest<package_manifests> (o, rl.host (), rl.port (), f)
+ : fetch_manifest<package_manifests> (f);
+ }
+
+ path
+ fetch_archive (const common_options& o,
+ const repository_location& rl,
+ const path& a,
+ const dir_path& d)
+ {
+ assert (a.relative ());
+ assert (rl.remote () || rl.absolute ());
+
+ path f (rl.path () / a);
+
+ return rl.remote ()
+ ? fetch_file (o, rl.host (), rl.port (), f, d)
+ : fetch_file (f, d);
}
}
diff --git a/bpkg/package b/bpkg/package
index f387d35..2601b7b 100644
--- a/bpkg/package
+++ b/bpkg/package
@@ -146,6 +146,8 @@ namespace bpkg
{
#pragma db column("count(" + repository::id.name + ")")
size_t result;
+
+ operator size_t () const {return result;}
};
// package_version_id
@@ -237,6 +239,8 @@ namespace bpkg
{
#pragma db column("count(" + available_package::id.data.name + ")")
size_t result;
+
+ operator size_t () const {return result;}
};
// state
diff --git a/bpkg/pkg-fetch.cxx b/bpkg/pkg-fetch.cxx
index 63dd31f..bfa6e54 100644
--- a/bpkg/pkg-fetch.cxx
+++ b/bpkg/pkg-fetch.cxx
@@ -6,6 +6,7 @@
#include <bpkg/manifest>
+#include <bpkg/fetch>
#include <bpkg/types>
#include <bpkg/package>
#include <bpkg/package-odb>
@@ -29,6 +30,8 @@ namespace bpkg
level4 ([&]{trace << "configuration: " << c;});
database db (open (c, trace));
+ transaction t (db.begin ());
+ session s;
path a;
bool purge;
@@ -52,21 +55,60 @@ namespace bpkg
fail << "package name argument expected" <<
info << "run 'bpkg help pkg-fetch' for more information";
- string name (args.next ());
+ string n (args.next ());
if (!args.more ())
fail << "package version argument expected" <<
info << "run 'bpkg help pkg-fetch' for more information";
- string ver (args.next ());
-
- // TODO:
+ //@@ Same code as in pkg-status. Similar problem to repo_location;
+ // need a place for such utilities.
//
- // - Will need to use some kind or auto_rm/exception guard on
- // fetched file.
+ version v;
+ {
+ const char* s (args.next ());
+ try
+ {
+ v = version (s);
+ }
+ catch (const invalid_argument& e)
+ {
+ fail << "invalid package version '" << s << "': " << e.what ();
+ }
+ }
+
+ if (db.query_value<repository_count> () == 0)
+ fail << "configuration " << c << " has no repositories" <<
+ info << "use 'bpkg rep-add' to add a repository";
+
+ if (db.query_value<available_package_count> () == 0)
+ fail << "configuration " << c << " has no available packages" <<
+ info << "use 'bpkg rep-fetch' to fetch available packages list";
+
+ shared_ptr<available_package> p (
+ db.find<available_package> (package_version_id (n, v)));
+
+ if (p == nullptr)
+ fail << "package " << n << " " << v << " is not available";
+
+ // Pick a repository. Prefer local ones over the remote.
//
-
- fail << "pkg-fetch " << name << " " << ver << " not yet implemented";
+ const package_location* pl (&p->locations.front ());
+
+ for (const package_location& l: p->locations)
+ {
+ if (!l.repository.load ()->location.remote ())
+ {
+ pl = &l;
+ break;
+ }
+ }
+
+ if (verb > 1)
+ text << "fetching " << pl->location.leaf () << " "
+ << "from " << pl->repository->name ();
+
+ a = fetch_archive (o, pl->repository->location, pl->location, c);
purge = true;
}
@@ -79,10 +121,6 @@ namespace bpkg
const auto& n (m.name);
- // Database time.
- //
- transaction t (db.begin ());
-
// See if this package already exists in this configuration.
//
if (shared_ptr<package> p = db.find<package> (n))
diff --git a/bpkg/pkg-status.cxx b/bpkg/pkg-status.cxx
index a5eab3e..8daf781 100644
--- a/bpkg/pkg-status.cxx
+++ b/bpkg/pkg-status.cxx
@@ -42,7 +42,7 @@ namespace bpkg
}
catch (const invalid_argument& e)
{
- fail << "invalid package version '" << s << "'";
+ fail << "invalid package version '" << s << "': " << e.what ();
}
}
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index e112bf7..641efb2 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -166,7 +166,7 @@ namespace bpkg
const auto& ua (root->complements); // User-added repositories.
if (ua.empty ())
- fail << "configuration has no repositories" <<
+ fail << "configuration " << c << " has no repositories" <<
info << "use 'bpkg rep-add' to add a repository";
// Clean repositories and available packages. At the end only
@@ -206,8 +206,8 @@ namespace bpkg
size_t rcount, pcount;
if (verb)
{
- rcount = db.query_value<repository_count> ().result;
- pcount = db.query_value<available_package_count> ().result;
+ rcount = db.query_value<repository_count> ();
+ pcount = db.query_value<available_package_count> ();
}
t.commit ();
diff --git a/bpkg/test.sh b/bpkg/test.sh
index d1ab6f2..63b6fb2 100755
--- a/bpkg/test.sh
+++ b/bpkg/test.sh
@@ -163,17 +163,50 @@ test cfg-create --wipe
test rep-add http://pkg.cppget.org/1/hello
test rep-fetch
-## @@
##
+## pkg-fetch
##
-test cfg-create --wipe config.cxx=g++-4.9 cxx config.install.root=/tmp/install
-stat unknown
+test cfg-create --wipe
+
+fail pkg-fetch -e # archive expected
+fail pkg-fetch -e ./no-such-file # archive does not exist
+
+fail pkg-fetch # package name expected
+fail pkg-fetch libhello # package version expected
+fail pkg-fetch libhello 1/2/3 # invalid package version
+fail pkg-fetch libhello 1.0.0 # no repositories
+test rep-add $rep
+fail pkg-fetch libhello 1.0.0 # no packages
+test rep-fetch
+fail pkg-fetch libhello 2+1.0.0 # not available
+
+# local
+#
+test cfg-create --wipe
+test rep-add $rep
+test rep-fetch
+test pkg-fetch libhello 1.0.0
+test pkg-unpack libhello
+test pkg-purge libhello
+
+# remote
+#
+test cfg-create --wipe
+test rep-add http://pkg.cppget.org/1/hello
+test rep-fetch
+#test pkg-fetch libheavy 1.0.0
+test pkg-fetch libhello 1.0.0
+test pkg-unpack libhello
+test pkg-purge libhello
+
+## @@
##
-## pkg-fetch
##
+test cfg-create --wipe config.cxx=g++-4.9 cxx config.install.root=/tmp/install
+stat unknown
# fetch existing archive
#