aboutsummaryrefslogtreecommitdiff
path: root/load/load.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'load/load.cxx')
-rw-r--r--load/load.cxx832
1 files changed, 593 insertions, 239 deletions
diff --git a/load/load.cxx b/load/load.cxx
index 83cc9e6..14b8374 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -1,13 +1,13 @@
// file : load/load.cxx -*- C++ -*-
-// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
// license : MIT; see accompanying LICENSE file
#include <signal.h> // signal()
#include <cerrno>
-#include <cstring> // strncmp()
+#include <chrono>
+#include <thread> // this_thread::sleep_for()
+#include <cstring> // strncmp()
#include <iostream>
-#include <algorithm> // find(), find_if()
#include <odb/session.hxx>
#include <odb/database.hxx>
@@ -17,13 +17,13 @@
#include <odb/pgsql/database.hxx>
-#include <libbutl/pager.mxx>
-#include <libbutl/sha256.mxx>
-#include <libbutl/process.mxx>
-#include <libbutl/fdstream.mxx>
-#include <libbutl/filesystem.mxx>
-#include <libbutl/tab-parser.mxx>
-#include <libbutl/manifest-parser.mxx>
+#include <libbutl/pager.hxx>
+#include <libbutl/sha256.hxx>
+#include <libbutl/process.hxx>
+#include <libbutl/fdstream.hxx>
+#include <libbutl/filesystem.hxx>
+#include <libbutl/tab-parser.hxx>
+#include <libbutl/manifest-parser.hxx>
#include <libbpkg/manifest.hxx>
@@ -37,6 +37,7 @@ using std::cout;
using std::cerr;
using std::endl;
+using namespace std::this_thread;
using namespace odb::core;
using namespace butl;
using namespace bpkg;
@@ -54,6 +55,17 @@ static const char* help_info (
static const path packages ("packages.manifest");
static const path repositories ("repositories.manifest");
+// Retry executing bpkg on recoverable errors for about 10 seconds.
+//
+// Should we just exit with some "bpkg recoverable" code instead and leave it
+// to the caller to perform retries? Feels like it's better to handle such
+// errors ourselves rather than to complicate every caller. Note that having
+// some frequently updated prerequisite repository can make these errors quite
+// probable, even if the internal repositories are rarely updated.
+//
+static const size_t bpkg_retries (10);
+static const std::chrono::seconds bpkg_retry_timeout (1);
+
struct internal_repository
{
repository_location location;
@@ -249,7 +261,7 @@ load_repositories (path p)
bad_line ("invalid buildable option value");
}
else
- bad_line ("invalid option '" + nv + "'");
+ bad_line ("invalid option '" + nv + '\'');
}
// For now cache option is mandatory.
@@ -353,22 +365,20 @@ repository_info (const options& lo, const string& rl, const cstrings& options)
//
static void
load_packages (const shared_ptr<repository>& rp,
+ const repository_location& cl,
database& db,
bool ignore_unknown,
- const manifest_name_values& overrides)
+ const manifest_name_values& overrides,
+ const string& overrides_name)
{
// packages_timestamp other than timestamp_nonexistent signals the
// repository packages are already loaded.
//
assert (rp->packages_timestamp == timestamp_nonexistent);
- // Only locally accessible repositories allowed until package manager API is
- // ready.
- //
- assert (!rp->cache_location.empty ());
-
vector<package_manifest> pms;
- const repository_location& cl (rp->cache_location);
+
+ assert (!cl.empty ());
path p (cl.path () / packages);
@@ -395,8 +405,8 @@ load_packages (const shared_ptr<repository>& rp,
mp,
move (nv),
ignore_unknown,
- false /* complete_depends */,
- package_manifest_flags::forbid_incomplete_dependencies);
+ false /* complete_values */,
+ package_manifest_flags::forbid_incomplete_values);
}
else
pms = pkg_package_manifests (mp, ignore_unknown);
@@ -408,6 +418,8 @@ load_packages (const shared_ptr<repository>& rp,
}
using brep::dependency;
+ using brep::dependency_alternative;
+ using brep::dependency_alternatives;
for (package_manifest& pm: pms)
{
@@ -423,108 +435,160 @@ load_packages (const shared_ptr<repository>& rp,
{
if (rp->internal)
{
+ if (!overrides.empty ())
try
{
- pm.override (overrides, "" /* name */);
+ pm.override (overrides, overrides_name);
}
- catch (const manifest_parsing&)
+ catch (const manifest_parsing& e)
{
- // Overrides are already validated (see below).
- //
- assert (false);
+ cerr << "error: unable to override " << p << " manifest: " << e
+ << endl;
+
+ throw failed ();
}
// Create internal package object.
//
- optional<string> dsc;
- optional<text_type> dst;
-
- if (pm.description)
+ // Return nullopt if the text is in a file (can happen if the
+ // repository is of a type other than pkg) or if the type is not
+ // recognized (can only happen in the "ignore unknown" mode).
+ //
+ auto to_typed_text = [&cl, ignore_unknown] (typed_text_file&& v)
{
+ optional<typed_text> r;
+
// The description value should not be of the file type if the
// package manifest comes from the pkg repository.
//
- assert (!pm.description->file || cl.type () != repository_type::pkg);
+ assert (!v.file || cl.type () != repository_type::pkg);
- if (!pm.description->file)
+ if (!v.file)
{
- dst = pm.effective_description_type (ignore_unknown);
+ // Cannot throw since the manifest parser has already verified the
+ // effective type in the same "ignore unknown" mode.
+ //
+ optional<text_type> t (v.effective_type (ignore_unknown));
// If the description type is unknown (which may be the case for
// some "transitional" period and only if --ignore-unknown is
// specified) we just silently drop the description.
//
- assert (dst || ignore_unknown);
+ assert (t || ignore_unknown);
- if (dst)
- dsc = move (pm.description->text);
+ if (t)
+ r = typed_text {move (v.text), *t};
}
- }
- string chn;
+ return r;
+ };
+
+ // Convert descriptions.
+ //
+ optional<typed_text> ds (
+ pm.description
+ ? to_typed_text (move (*pm.description))
+ : optional<typed_text> ());
+
+ optional<typed_text> pds (
+ pm.package_description
+ ? to_typed_text (move (*pm.package_description))
+ : optional<typed_text> ());
+
+ // Merge changes into a single typed text object.
+ //
+ // If the text type is not recognized for any changes entry or some
+ // entry refers to a file, then assume that no changes are specified.
+ //
+ optional<typed_text> chn;
+
for (auto& c: pm.changes)
{
- // The changes value should not be of the file type if the package
- // manifest comes from the pkg repository.
- //
- assert (!c.file || cl.type () != repository_type::pkg);
+ optional<typed_text> tc (to_typed_text (move (c)));
- if (!c.file)
+ if (!tc)
{
- if (chn.empty ())
- chn = move (c.text);
- else
- {
- if (chn.back () != '\n')
- chn += '\n'; // Always have a blank line as a separator.
-
- chn += "\n" + c.text;
- }
+ chn = nullopt;
+ break;
}
- }
-
- dependencies ds;
- for (auto& pda: pm.dependencies)
- {
- // Ignore special build2 and bpkg dependencies. We may not have
- // packages for them and also showing them for every package is
- // probably not very helpful.
- //
- if (pda.buildtime && !pda.empty ())
+ if (!chn)
{
- const package_name& n (pda.front ().name);
- if (n == "build2" || n == "bpkg")
- continue;
+ chn = move (*tc);
}
+ else
+ {
+ // Should have failed while parsing the manifest otherwise.
+ //
+ assert (tc->type == chn->type);
- ds.emplace_back (pda.conditional, pda.buildtime, move (pda.comment));
+ string& v (chn->text);
- for (auto& pd: pda)
- // The package member will be assigned during dependency
- // resolution procedure.
- //
- ds.back ().push_back (dependency {move (pd.name),
- move (pd.constraint),
- nullptr /* package */});
+ assert (!v.empty ()); // Changes manifest value cannot be empty.
+
+ if (v.back () != '\n')
+ v += '\n'; // Always have a blank line as a separator.
+
+ v += '\n';
+ v += tc->text;
+ }
}
- auto deps = [] (small_vector<bpkg::dependency, 1>&& ds)
+ dependencies tds;
+
+ for (auto& das: pm.dependencies)
{
- small_vector<dependency, 1> r;
+ dependency_alternatives tdas (das.buildtime, move (das.comment));
- if (!ds.empty ())
+ for (auto& da: das)
{
- r.reserve (ds.size ());
+ dependency_alternative tda (move (da.enable),
+ move (da.reflect),
+ move (da.prefer),
+ move (da.accept),
+ move (da.require));
+
+ for (auto& d: da)
+ {
+ package_name& n (d.name);
+
+ // Ignore special build2 and bpkg dependencies. We may not have
+ // packages for them and also showing them for every package is
+ // probably not very helpful.
+ //
+ if (das.buildtime && (n == "build2" || n == "bpkg"))
+ continue;
+
+ // The package member will be assigned during dependency
+ // resolution procedure.
+ //
+ tda.push_back (dependency {move (n),
+ move (d.constraint),
+ nullptr /* package */});
+ }
- for (bpkg::dependency& d: ds)
- r.push_back (dependency {move (d.name),
- move (d.constraint),
- nullptr /* package */});
+ if (!tda.empty ())
+ tdas.push_back (move (tda));
}
- return r;
- };
+ if (!tdas.empty ())
+ tds.push_back (move (tdas));
+ }
+
+ small_vector<brep::test_dependency, 1> ts;
+
+ if (!pm.tests.empty ())
+ {
+ ts.reserve (pm.tests.size ());
+
+ for (bpkg::test_dependency& td: pm.tests)
+ ts.emplace_back (move (td.name),
+ td.type,
+ td.buildtime,
+ move (td.constraint),
+ move (td.enable),
+ move (td.reflect));
+ }
// Cache before the package name is moved.
//
@@ -540,8 +604,8 @@ load_packages (const shared_ptr<repository>& rp,
move (pm.license_alternatives),
move (pm.topics),
move (pm.keywords),
- move (dsc),
- move (dst),
+ move (ds),
+ move (pds),
move (chn),
move (pm.url),
move (pm.doc_url),
@@ -552,13 +616,13 @@ load_packages (const shared_ptr<repository>& rp,
move (pm.build_email),
move (pm.build_warning_email),
move (pm.build_error_email),
- move (ds),
+ move (tds),
move (pm.requirements),
- deps (move (pm.tests)),
- deps (move (pm.examples)),
- deps (move (pm.benchmarks)),
+ move (ts),
move (pm.builds),
move (pm.build_constraints),
+ move (pm.build_auxiliaries),
+ move (pm.build_configs),
move (pm.location),
move (pm.fragment),
move (pm.sha256sum),
@@ -567,7 +631,13 @@ load_packages (const shared_ptr<repository>& rp,
else
// Create external package object.
//
- p = make_shared<package> (move (pm.name), move (pm.version), rp);
+ p = make_shared<package> (move (pm.name),
+ move (pm.version),
+ move (pm.builds),
+ move (pm.build_constraints),
+ move (pm.build_auxiliaries),
+ move (pm.build_configs),
+ rp);
db.persist (p);
}
@@ -601,9 +671,14 @@ load_packages (const shared_ptr<repository>& rp,
// A non-stub package is buildable if belongs to at least one
// buildable repository (see libbrep/package.hxx for details).
+ // Note that if this is an external test package it will be marked as
+ // unbuildable later (see resolve_dependencies() for details).
//
- if (!p->stub () && !p->buildable)
- p->buildable = rp->buildable;
+ if (rp->buildable && !p->buildable && !p->stub ())
+ {
+ p->buildable = true;
+ p->unbuildable_reason = nullopt;
+ }
}
p->other_repositories.push_back (rp);
@@ -620,7 +695,9 @@ load_packages (const shared_ptr<repository>& rp,
// changed members. Should be called once per persisted internal repository.
//
static void
-load_repositories (const shared_ptr<repository>& rp,
+load_repositories (const options& lo,
+ const shared_ptr<repository>& rp,
+ const repository_location& cl,
database& db,
bool ignore_unknown,
bool shallow)
@@ -630,11 +707,6 @@ load_repositories (const shared_ptr<repository>& rp,
//
assert (rp->repositories_timestamp == timestamp_nonexistent);
- // Only locally accessible repositories allowed until package manager API is
- // ready.
- //
- assert (!rp->cache_location.empty ());
-
const string& tenant (rp->tenant);
// Repository is already persisted by the load_packages() function call.
@@ -644,7 +716,9 @@ load_repositories (const shared_ptr<repository>& rp,
pkg_repository_manifests rpm;
- path p (rp->cache_location.path () / repositories);
+ assert (!cl.empty ());
+
+ path p (cl.path () / repositories);
try
{
@@ -653,6 +727,9 @@ load_repositories (const shared_ptr<repository>& rp,
manifest_parser mp (ifs, p.string ());
rpm = pkg_repository_manifests (mp, ignore_unknown);
+
+ if (rpm.empty ())
+ rpm.emplace_back (repository_manifest ()); // Add the base repository.
}
catch (const io_error& e)
{
@@ -779,15 +856,19 @@ load_repositories (const shared_ptr<repository>& rp,
pr = make_shared<repository> (tenant, move (rl));
- // If the prerequsite repository location is a relative path, then
- // calculate its cache location.
+ // If the base repository is internal and the prerequsite repository
+ // location is a relative path, then calculate its cache location.
//
- if (rm.location.relative ())
+ if (rp->internal && rm.location.relative ())
{
+ // For an internal repository the cache location always comes from the
+ // loadtab file.
+ //
+ assert (cl.path () == rp->cache_location.path ());
+
try
{
- pr->cache_location =
- repository_location (rm.location, rp->cache_location);
+ pr->cache_location = repository_location (rm.location, cl);
}
catch (const invalid_argument&)
{
@@ -795,21 +876,161 @@ load_repositories (const shared_ptr<repository>& rp,
<< "repository '" << rm.location << "'" << endl
<< " info: base (internal) repository location is "
<< rp->location << endl
- << " info: base repository cache location is "
- << rp->cache_location << endl;
+ << " info: base repository cache location is " << cl << endl;
throw failed ();
}
}
+ // If the (external) prerequisite repository cache location is empty, then
+ // check if the repository is local and, if that's the case, use its
+ // location as a cache location. Otherwise, fetch the repository
+ // information creating a temporary cache for it.
+ //
+ auto_rmdir cdr; // Remove the temporary cache after the repo load.
+ repository_location cl; // Repository temporary cache location.
+
+ if (pr->cache_location.empty ())
+ {
+ if (pr->location.local ())
+ {
+ pr->cache_location = pr->location;
+ }
+ else
+ {
+ dir_path cd;
+
+ try
+ {
+ cd = dir_path::temp_path ("brep-load-cache");
+ }
+ catch (const system_error& e)
+ {
+ cerr << "unable to obtain temporary directory: " << e;
+ throw failed ();
+ }
+
+ // It's highly unlikely but still possible that the temporary cache
+ // directory already exists. This can only happen due to the unclean
+ // loader termination. Let's remove it and retry.
+ //
+ try
+ {
+ if (try_mkdir (cd) == mkdir_status::already_exists)
+ {
+ try_rmdir_r (cd);
+
+ if (try_mkdir (cd) == mkdir_status::already_exists)
+ throw_generic_error (EEXIST);
+ }
+ }
+ catch (const system_error& e)
+ {
+ cerr << "unable to create directory '" << cd << "': " << e;
+ throw failed ();
+ }
+
+ cdr = auto_rmdir (cd);
+
+ path rf (cd / repositories);
+ path pf (cd / packages);
+
+ // Note that the fetch timeout can be overridden via --bpkg-option.
+ //
+ cstrings args {
+ "--fetch-timeout", "60", // 1 minute.
+ "--deep",
+ "--manifest",
+ "--repositories",
+ "--repositories-file", rf.string ().c_str (),
+ "--packages",
+ "--packages-file", pf.string ().c_str ()};
+
+ if (rm.trust)
+ {
+ args.push_back ("--trust");
+ args.push_back (rm.trust->c_str ());
+ }
+
+ // Always add it, so bpkg won't try to prompt for a certificate
+ // authentication if the fingerprint doesn't match.
+ //
+ args.push_back ("--trust-no");
+
+ // Retry bpkg-rep-info on recoverable errors, for a while.
+ //
+ for (size_t i (0);; ++i)
+ {
+ if (i != 0)
+ {
+ // Let's follow up the bpkg's diagnostics with the number of
+ // retries left.
+ //
+ cerr << bpkg_retries - i + 1 << " retries left" << endl;
+ sleep_for (bpkg_retry_timeout);
+ }
+
+ process p (repository_info (lo, pr->location.string (), args));
+
+ try
+ {
+ // Bail out from the retry loop on success.
+ //
+ if (p.wait ())
+ break;
+
+ // Assume the child issued diagnostics if terminated normally.
+ //
+ if (p.exit->normal ())
+ {
+ // Retry the manifests fetch on a recoverable error, unless the
+ // retries limit is reached.
+ //
+ if (p.exit->code () == 2 && i != bpkg_retries)
+ continue;
+ }
+ else
+ cerr << "process " << lo.bpkg () << " " << *p.exit << endl;
+
+ cerr << "error: unable to fetch manifests for "
+ << pr->canonical_name << endl
+ << " info: base repository location is "
+ << rp->location << endl;
+
+ throw failed ();
+ }
+ catch (const process_error& e)
+ {
+ cerr << "error: unable to fetch manifests for "
+ << pr->canonical_name << ": " << e << endl;
+
+ throw failed ();
+ }
+ }
+
+ // Note that this is a non-pkg repository cache and so we create the
+ // dir repository location (see load_repositories(path) for details).
+ //
+ cl = repository_location (repository_url (cd.string ()),
+ repository_type::dir);
+ }
+ }
+
// We don't apply overrides to the external packages.
//
load_packages (pr,
+ !pr->cache_location.empty () ? pr->cache_location : cl,
db,
ignore_unknown,
- manifest_name_values () /* overrides */);
-
- load_repositories (pr, db, ignore_unknown, false /* shallow */);
+ manifest_name_values () /* overrides */,
+ "" /* overrides_name */);
+
+ load_repositories (lo,
+ pr,
+ !pr->cache_location.empty () ? pr->cache_location : cl,
+ db,
+ ignore_unknown,
+ false /* shallow */);
}
db.update (rp);
@@ -850,29 +1071,36 @@ find (const lazy_shared_ptr<repository>& r,
return false;
}
-// Resolve package run-time dependencies, tests, examples, and benchmarks.
-// Make sure that the best matching dependency belongs to the package
-// repositories, their complements, recursively, or their immediate
-// prerequisite repositories (only for run-time dependencies). Should be
-// called once per internal package.
+// Resolve package regular dependencies and external tests. Make sure that the
+// best matching dependency belongs to the package repositories, their
+// complements, recursively, or their immediate prerequisite repositories
+// (only for regular dependencies). Set the buildable flag to false for the
+// resolved external tests packages. Fail if unable to resolve a regular
+// dependency, unless ignore_unresolved is true in which case leave this
+// dependency NULL. Fail if unable to resolve an external test, unless
+// ignore_unresolved or ignore_unresolved_tests is true in which case leave
+// this dependency NULL, if ignore_unresolved_tests is false, and remove the
+// respective tests manifest entry otherwise. Should be called once per
+// internal package.
//
static void
-resolve_dependencies (package& p, database& db)
+resolve_dependencies (package& p,
+ database& db,
+ bool ignore_unresolved,
+ bool ignore_unresolved_tests)
{
using brep::dependency;
+ using brep::dependency_alternative;
using brep::dependency_alternatives;
// Resolve dependencies for internal packages only.
//
assert (p.internal ());
- if (p.dependencies.empty () &&
- p.tests.empty () &&
- p.examples.empty () &&
- p.benchmarks.empty ())
+ if (p.dependencies.empty () && p.tests.empty ())
return;
- auto resolve = [&p, &db] (dependency& d, bool prereq)
+ auto resolve = [&p, &db] (dependency& d, bool test)
{
// Dependency should not be resolved yet.
//
@@ -934,9 +1162,26 @@ resolve_dependencies (package& p, database& db)
for (const auto& pp: db.query<package> (q + order_by_version_desc (vm)))
{
- if (find (p.internal_repository, pp, prereq))
+ if (find (p.internal_repository, pp, !test))
{
d.package.reset (db, pp.id);
+
+ // If the resolved dependency is an external test, then mark it as
+ // such, unless it is a stub.
+ //
+ if (test)
+ {
+ shared_ptr<package> dp (d.package.load ());
+
+ if (!dp->stub ())
+ {
+ dp->buildable = false;
+ dp->unbuildable_reason = unbuildable_reason::test;
+
+ db.update (dp);
+ }
+ }
+
return true;
}
}
@@ -944,49 +1189,50 @@ resolve_dependencies (package& p, database& db)
return false;
};
- auto bail = [&p] (const dependency& d, const char* what)
+ auto bail = [&p] (const dependency& d, const string& what)
{
- cerr << "error: can't resolve " << what << " " << d << " for the package "
- << p.name << " " << p.version << endl
+ cerr << "error: can't resolve " << what << ' ' << d << " for the package "
+ << p.name << ' ' << p.version << endl
<< " info: repository " << p.internal_repository.load ()->location
<< " appears to be broken" << endl;
throw failed ();
};
- for (dependency_alternatives& da: p.dependencies)
+ for (dependency_alternatives& das: p.dependencies)
{
- for (dependency& d: da)
+ // Practically it is enough to resolve at least one dependency alternative
+ // to build a package. Meanwhile here we consider an error specifying in
+ // the manifest file an alternative which can't be resolved, unless
+ // unresolved dependencies are allowed.
+ //
+ for (dependency_alternative& da: das)
{
- // Practically it is enough to resolve at least one dependency
- // alternative to build a package. Meanwhile here we consider an error
- // specifying in the manifest file an alternative which can't be
- // resolved.
- //
- if (!resolve (d, true /* prereq */))
- bail (d, "dependency");
+ for (dependency& d: da)
+ {
+ if (!resolve (d, false /* test */) && !ignore_unresolved)
+ bail (d, "dependency");
+ }
}
}
- // Should we allow tests, examples, and benchmarks packages to be
- // unresolvable? Let's forbid that until we see a use case for that.
- //
- for (dependency& d: p.tests)
+ for (auto i (p.tests.begin ()); i != p.tests.end (); )
{
- if (!resolve (d, false /* prereq */))
- bail (d, "tests");
- }
+ brep::test_dependency& td (*i);
- for (dependency& d: p.examples)
- {
- if (!resolve (d, false /* prereq */))
- bail (d, "examples");
- }
+ if (!resolve (td, true /* test */))
+ {
+ if (!ignore_unresolved && !ignore_unresolved_tests)
+ bail (td, to_string (td.type));
- for (dependency& d: p.benchmarks)
- {
- if (!resolve (d, false /* prereq */))
- bail (d, "benchmarks");
+ if (ignore_unresolved_tests)
+ {
+ i = p.tests.erase (i);
+ continue;
+ }
+ }
+
+ ++i;
}
db.update (p); // Update the package state.
@@ -1043,10 +1289,13 @@ detect_dependency_cycle (const package_id& id,
chain.push_back (id);
shared_ptr<package> p (db.load<package> (id));
- for (const auto& da: p->dependencies)
+ for (const auto& das: p->dependencies)
{
- for (const auto& d: da)
- detect_dependency_cycle (d.package.object_id (), chain, db);
+ for (const auto& da: das)
+ {
+ for (const auto& d: da)
+ detect_dependency_cycle (d.package.object_id (), chain, db);
+ }
}
chain.pop_back ();
@@ -1064,105 +1313,130 @@ certificate_info (const options& lo,
const repository_location& rl,
const optional<string>& fp)
{
- try
- {
- cstrings args {
- "--cert-fingerprint",
- "--cert-name",
- "--cert-organization",
- "--cert-email",
- "-q"}; // Don't print info messages.
+ cstrings args {
+ "--cert-fingerprint",
+ "--cert-name",
+ "--cert-organization",
+ "--cert-email",
+ "-q"}; // Don't print info messages.
- const char* trust ("--trust-no");
+ const char* trust ("--trust-no");
- if (fp)
+ if (fp)
+ {
+ if (!fp->empty ())
{
- if (!fp->empty ())
- {
- args.push_back ("--trust");
- args.push_back (fp->c_str ());
- }
- else
- trust = "--trust-yes";
+ args.push_back ("--trust");
+ args.push_back (fp->c_str ());
+ }
+ else
+ trust = "--trust-yes";
- if (!rl.remote ())
- {
- args.push_back ("--auth");
- args.push_back ("all");
- }
+ if (!rl.remote ())
+ {
+ args.push_back ("--auth");
+ args.push_back ("all");
}
+ }
- args.push_back (trust);
+ args.push_back (trust);
- process pr (repository_info (lo, rl.string (), args));
+ // Retry bpkg-rep-info on recoverable errors, for a while.
+ //
+ for (size_t i (0);; ++i)
+ {
+ if (i != 0)
+ {
+ // Let's follow up the bpkg's diagnostics with the number of retries
+ // left.
+ //
+ cerr << bpkg_retries - i + 1 << " retries left" << endl;
+ sleep_for (bpkg_retry_timeout);
+ }
try
{
- ifdstream is (
- move (pr.in_ofd),
- ifdstream::failbit | ifdstream::badbit | ifdstream::eofbit);
+ process pr (repository_info (lo, rl.string (), args));
- optional<certificate> cert;
+ try
+ {
+ ifdstream is (
+ move (pr.in_ofd),
+ ifdstream::failbit | ifdstream::badbit | ifdstream::eofbit);
- string fingerprint;
- getline (is, fingerprint);
+ optional<certificate> cert;
- if (!fingerprint.empty ())
- {
- cert = certificate ();
- cert->fingerprint = move (fingerprint);
- getline (is, cert->name);
- getline (is, cert->organization);
- getline (is, cert->email);
+ string fingerprint;
+ getline (is, fingerprint);
+
+ if (!fingerprint.empty ())
+ {
+ cert = certificate ();
+ cert->fingerprint = move (fingerprint);
+ getline (is, cert->name);
+ getline (is, cert->organization);
+ getline (is, cert->email);
+ }
+ else
+ {
+ // Read out empty lines.
+ //
+ string s;
+ getline (is, s); // Name.
+ getline (is, s); // Organization.
+ getline (is, s); // Email.
+ }
+
+ // Check that EOF is successfully reached.
+ //
+ is.exceptions (ifdstream::failbit | ifdstream::badbit);
+ if (is.peek () != ifdstream::traits_type::eof ())
+ throw io_error ("");
+
+ is.close ();
+
+ if (pr.wait ())
+ return cert;
+
+ // Fall through.
+ //
}
- else
+ catch (const io_error&)
{
- // Read out empty lines.
+ // Child exit status doesn't matter. Just wait for the process
+ // completion and fall through.
//
- string s;
- getline (is, s); // Name.
- getline (is, s); // Organization.
- getline (is, s); // Email.
+ pr.wait ();
}
- // Check that EOF is successfully reached.
+ // Assume the child issued diagnostics if terminated normally.
//
- is.exceptions (ifdstream::failbit | ifdstream::badbit);
- if (is.peek () != ifdstream::traits_type::eof ())
- throw io_error ("");
-
- is.close ();
+ if (pr.exit->normal ())
+ {
+ // Retry the certificate fetch on a recoverable error, unless the
+ // retries limit is reached.
+ //
+ if (pr.exit->code () == 2 && i != bpkg_retries)
+ continue;
+ }
+ else
+ cerr << "process " << lo.bpkg () << " " << *pr.exit << endl;
- if (pr.wait ())
- return cert;
+ cerr << "error: unable to fetch certificate information for "
+ << rl.canonical_name () << endl;
// Fall through.
- //
}
- catch (const io_error&)
+ catch (const process_error& e)
{
- // Child exit status doesn't matter. Just wait for the process
- // completion and fall through.
- //
- pr.wait ();
- }
+ cerr << "error: unable to fetch certificate information for "
+ << rl.canonical_name () << ": " << e << endl;
- // Assume the child issued diagnostics.
- //
- cerr << "error: unable to fetch certificate information for "
- << rl.canonical_name () << endl;
-
- // Fall through.
- }
- catch (const process_error& e)
- {
- cerr << "error: unable to fetch certificate information for "
- << rl.canonical_name () << ": " << e << endl;
+ // Fall through.
+ }
- // Fall through.
+ throw failed ();
}
-
- throw failed ();
}
int
@@ -1193,7 +1467,7 @@ try
<< "libbbot " << LIBBBOT_VERSION_ID << endl
<< "libbpkg " << LIBBPKG_VERSION_ID << endl
<< "libbutl " << LIBBUTL_VERSION_ID << endl
- << "Copyright (c) 2014-2019 Code Synthesis Ltd" << endl
+ << "Copyright (c) " << BREP_COPYRIGHT << "." << endl
<< "This is free software released under the MIT license." << endl;
return 0;
@@ -1241,8 +1515,46 @@ try
throw failed ();
}
+ // Verify the --service-* options.
+ //
+ if (ops.service_id_specified ())
+ {
+ if (!ops.tenant_specified ())
+ {
+ cerr << "error: --service-id requires --tenant" << endl;
+ throw failed ();
+ }
+
+ if (ops.service_type ().empty ())
+ {
+ cerr << "error: --service-id requires --service-type"
+ << endl;
+ throw failed ();
+ }
+ }
+ else
+ {
+ if (ops.service_type_specified ())
+ {
+ cerr << "error: --service-type requires --service-id"
+ << endl;
+ throw failed ();
+ }
+
+ if (ops.service_data_specified ())
+ {
+ cerr << "error: --service-data requires --service-id"
+ << endl;
+ throw failed ();
+ }
+ }
+
// Parse and validate overrides, if specified.
//
+ // Note that here we make sure that the overrides manifest is valid.
+ // Applying overrides to a specific package manifest may still fail (see
+ // package_manifest::validate_overrides() for details).
+ //
manifest_name_values overrides;
if (ops.overrides_file_specified ())
@@ -1277,7 +1589,7 @@ try
ops.db_port (),
"options='-c default_transaction_isolation=serializable'");
- // Prevent several brep-load/migrate instances from updating DB
+ // Prevent several brep utility instances from updating the package database
// simultaneously.
//
database_lock l (db);
@@ -1294,6 +1606,11 @@ try
throw failed ();
}
+ // Note: the interactive tenant implies private.
+ //
+ if (ops.interactive_specified ())
+ ops.private_ (true);
+
// Load the description of all the internal repositories from the
// configuration file.
//
@@ -1331,7 +1648,30 @@ try
// Persist the tenant.
//
- db.persist (tenant (tnt));
+ // Note that if the tenant service is specified and some tenant with the
+ // same service id and type is already persisted, then we will end up with
+ // the `object already persistent` error and terminate with the exit code
+ // 1 (fatal error). We could potentially dedicate a special exit code for
+ // such a case, so that the caller may recognize it and behave accordingly
+ // (CI request handler can treat it as a client error rather than an
+ // internal error, etc). However, let's first see if it ever becomes a
+ // problem.
+ //
+ optional<tenant_service> service;
+
+ if (ops.service_id_specified ())
+ service = tenant_service (ops.service_id (),
+ ops.service_type (),
+ (ops.service_data_specified ()
+ ? ops.service_data ()
+ : optional<string> ()));
+
+ db.persist (tenant (tnt,
+ ops.private_ (),
+ (ops.interactive_specified ()
+ ? ops.interactive ()
+ : optional<string> ()),
+ move (service)));
// On the first pass over the internal repositories we load their
// certificate information and packages.
@@ -1356,7 +1696,12 @@ try
ir.buildable,
priority++));
- load_packages (r, db, ops.ignore_unknown (), overrides);
+ load_packages (r,
+ r->cache_location,
+ db,
+ ops.ignore_unknown (),
+ overrides,
+ ops.overrides_file ().string ());
}
// On the second pass over the internal repositories we load their
@@ -1369,12 +1714,17 @@ try
db.load<repository> (
repository_id (tnt, ir.location.canonical_name ())));
- load_repositories (r, db, ops.ignore_unknown (), ops.shallow ());
+ load_repositories (ops,
+ r,
+ r->cache_location,
+ db,
+ ops.ignore_unknown (),
+ ops.shallow ());
}
- // Resolve internal packages dependencies unless this is a shallow load.
+ // Resolve internal packages dependencies and, unless this is a shallow
+ // load, make sure there are no package dependency cycles.
//
- if (!ops.shallow ())
{
session s;
using query = query<package>;
@@ -1383,16 +1733,20 @@ try
db.query<package> (
query::id.tenant == tnt &&
query::internal_repository.canonical_name.is_not_null ()))
- resolve_dependencies (p, db);
+ resolve_dependencies (p,
+ db,
+ ops.shallow (),
+ ops.ignore_unresolved_tests ());
- // Make sure there is no package dependency cycles.
- //
- package_ids chain;
- for (const auto& p:
- db.query<package> (
- query::id.tenant == tnt &&
- query::internal_repository.canonical_name.is_not_null ()))
- detect_dependency_cycle (p.id, chain, db);
+ if (!ops.shallow ())
+ {
+ package_ids chain;
+ for (const auto& p:
+ db.query<package> (
+ query::id.tenant == tnt &&
+ query::internal_repository.canonical_name.is_not_null ()))
+ detect_dependency_cycle (p.id, chain, db);
+ }
}
}