Update pkg-build

8 files changed, 324 insertions, 63 deletions

diff --git a/bpkg/manifest-utility.cxx b/bpkg/manifest-utility.cxx
index 8377bab..7b039c9 100644
--- a/bpkg/manifest-utility.cxx
+++ b/bpkg/manifest-utility.cxx
@@ -146,13 +146,12 @@ namespace bpkg
   bool
   repository_name (const string& s)
   {
-    size_t n (s.size ());
     size_t p (s.find (':'));
 
-    // If it has no scheme or starts with the URL scheme (followed by ://) then
+    // If it has no scheme or starts with the URL scheme (followed by :/) then
     // this is not a canonical name.
     //
-    if (p == string::npos || (p + 2 < n && s[p + 1] == '/' && s[p + 2] == '/'))
+    if (p == string::npos || s[p + 1] == '/')
       return false;
 
     // This is a canonical name if the scheme is convertible to the repository
diff --git a/bpkg/manifest-utility.hxx b/bpkg/manifest-utility.hxx
index a0966d4..5cc1c24 100644
--- a/bpkg/manifest-utility.hxx
+++ b/bpkg/manifest-utility.hxx
@@ -31,9 +31,21 @@ namespace bpkg
   string
   parse_package_name (const char*);
 
+  inline string
+  parse_package_name (const string& s)
+  {
+    return parse_package_name (s.c_str ());
+  }
+
   version
   parse_package_version (const char*);
 
+  inline version
+  parse_package_version (const string& s)
+  {
+    return parse_package_version (s.c_str ());
+  }
+
   // If the passed location is a relative local path, then assume this is a
   // relative path to the repository directory and complete it based on the
   // current working directory. Diagnose invalid locations and throw failed.
diff --git a/bpkg/package.cxx b/bpkg/package.cxx
index e9db6d6..2e5bffe 100644
--- a/bpkg/package.cxx
+++ b/bpkg/package.cxx
@@ -41,15 +41,19 @@ namespace bpkg
   static shared_ptr<repository>
   find (const shared_ptr<repository>& r,
         const shared_ptr<available_package>& ap,
-        repositories& chain)
+        repositories& chain,
+        bool prereq)
   {
+    // Prerequisites are not searched through recursively.
+    //
+    assert (!prereq || chain.empty ());
+
     auto pr = [&r] (const shared_ptr<repository>& i) -> bool {return i == r;};
     auto i (find_if (chain.begin (), chain.end (), pr));
 
     if (i != chain.end ())
       return nullptr;
 
-    bool prereq (chain.empty ()); // Check prerequisites in top-level only.
     chain.emplace_back (r);
 
     unique_ptr<repositories, void (*)(repositories*)> deleter (
@@ -81,7 +85,7 @@ namespace bpkg
         // Should we consider prerequisites of our complements as our
         // prerequisites? I'd say not.
         //
-        if (shared_ptr<repository> r = find (cr.load (), ap, chain))
+        if (shared_ptr<repository> r = find (cr.load (), ap, chain, false))
           return r;
       }
 
@@ -89,7 +93,7 @@ namespace bpkg
       {
         for (const lazy_weak_ptr<repository>& pr: ps)
         {
-          if (shared_ptr<repository> r = find (pr.load (), ap, chain))
+          if (shared_ptr<repository> r = find (pr.load (), ap, chain, false))
             return r;
         }
       }
@@ -100,20 +104,23 @@ namespace bpkg
 
   static inline shared_ptr<repository>
   find (const shared_ptr<repository>& r,
-        const shared_ptr<available_package>& ap)
+        const shared_ptr<available_package>& ap,
+        bool prereq)
   {
     repositories chain;
-    return find (r, ap, chain);
+    return find (r, ap, chain, prereq);
   }
 
   vector<shared_ptr<available_package>>
-  filter (const shared_ptr<repository>& r, result<available_package>&& apr)
+  filter (const shared_ptr<repository>& r,
+          result<available_package>&& apr,
+          bool prereq)
   {
     vector<shared_ptr<available_package>> aps;
 
     for (shared_ptr<available_package> ap: pointer_result (apr))
     {
-      if (find (r, ap) != nullptr)
+      if (find (r, ap, prereq) != nullptr)
         aps.push_back (move (ap));
     }
 
@@ -121,13 +128,15 @@ namespace bpkg
   }
 
   pair<shared_ptr<available_package>, shared_ptr<repository>>
-  filter_one (const shared_ptr<repository>& r, result<available_package>&& apr)
+  filter_one (const shared_ptr<repository>& r,
+              result<available_package>&& apr,
+              bool prereq)
   {
     using result = pair<shared_ptr<available_package>, shared_ptr<repository>>;
 
     for (shared_ptr<available_package> ap: pointer_result (apr))
     {
-      if (shared_ptr<repository> pr = find (r, ap))
+      if (shared_ptr<repository> pr = find (r, ap, prereq))
         return result (move (ap), move (pr));
     }
 
diff --git a/bpkg/package.hxx b/bpkg/package.hxx
index e864b59..35b9332 100644
--- a/bpkg/package.hxx
+++ b/bpkg/package.hxx
@@ -473,16 +473,20 @@ namespace bpkg
     operator size_t () const {return result;}
   };
 
-  // Only return packages that are in the specified repository or its
-  // complements, recursively. While you could maybe come up with a
-  // (barely comprehensible) view/query to achieve this, doing it on
-  // the "client side" is definitely more straightforward.
+  // Only return packages that are in the specified repository, its
+  // complements or prerequisites (if prereq is true), recursively. While you
+  // could maybe come up with a (barely comprehensible) view/query to achieve
+  // this, doing it on the "client side" is definitely more straightforward.
   //
   vector<shared_ptr<available_package>>
-  filter (const shared_ptr<repository>&, odb::result<available_package>&&);
+  filter (const shared_ptr<repository>&,
+          odb::result<available_package>&&,
+          bool prereq = true);
 
   pair<shared_ptr<available_package>, shared_ptr<repository>>
-  filter_one (const shared_ptr<repository>&, odb::result<available_package>&&);
+  filter_one (const shared_ptr<repository>&,
+              odb::result<available_package>&&,
+              bool prereq = true);
 
   // package_state
   //
@@ -791,24 +795,22 @@ namespace bpkg
 
   // Return a list of packages available from this repository.
   //
-  #pragma db view object(repository)                                    \
-    table("available_package_locations" = "pl" inner:                   \
-          "pl.repository = " + repository::name)                        \
-    object(available_package inner:                                     \
-           "pl.name = " + available_package::id.name + "AND" +          \
-           "pl.version_epoch = " +                                      \
-             available_package::id.version.epoch + "AND" +              \
-           "pl.version_canonical_upstream = " +                         \
-             available_package::id.version.canonical_upstream + "AND" + \
-           "pl.version_canonical_release = " +                          \
-             available_package::id.version.canonical_release + "AND" +  \
-           "pl.version_revision = " +                                   \
-             available_package::id.version.revision)
+  #pragma db view object(repository)                                   \
+    table("available_package_locations" = "pl" inner:                  \
+          "pl.repository = " + repository::name)                       \
+    object(available_package = package inner:                          \
+           "pl.name = " + package::id.name + "AND" +                   \
+           "pl.version_epoch = " + package::id.version.epoch + "AND" + \
+           "pl.version_canonical_upstream = " +                        \
+             package::id.version.canonical_upstream + "AND" +          \
+           "pl.version_canonical_release = " +                         \
+             package::id.version.canonical_release + "AND" +           \
+           "pl.version_revision = " +  package::id.version.revision)
   struct repository_package
   {
-    shared_ptr<available_package> object;
+    shared_ptr<available_package> package; // Must match the alias (see above).
 
-    operator const shared_ptr<available_package> () const {return object;}
+    operator const shared_ptr<available_package> () const {return package;}
   };
 
   // Version comparison operators.
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index fc6ff26..439aee7 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -9,7 +9,7 @@
 #include <list>
 #include <cstring>    // strlen()
 #include <iostream>   // cout
-#include <algorithm>  // find()
+#include <algorithm>  // find(), find_if()
 
 #include <bpkg/package.hxx>
 #include <bpkg/package-odb.hxx>
@@ -23,6 +23,7 @@
 #include <bpkg/pkg-drop.hxx>
 #include <bpkg/pkg-purge.hxx>
 #include <bpkg/pkg-fetch.hxx>
+#include <bpkg/rep-fetch.hxx>
 #include <bpkg/pkg-unpack.hxx>
 #include <bpkg/pkg-update.hxx>
 #include <bpkg/pkg-verify.hxx>
@@ -54,7 +55,8 @@ namespace bpkg
   find_available (database& db,
                   const string& name,
                   const shared_ptr<repository>& r,
-                  const optional<dependency_constraint>& c)
+                  const optional<dependency_constraint>& c,
+                  bool prereq = true)
   {
     using query = query<available_package>;
 
@@ -124,7 +126,7 @@ namespace bpkg
     // Filter the result based on the repository to which each version
     // belongs.
     //
-    return filter_one (r, db.query<available_package> (q));
+    return filter_one (r, db.query<available_package> (q), prereq);
   }
 
   // Create a transient (or fake, if you prefer) available_package object
@@ -1071,35 +1073,188 @@ namespace bpkg
       fail << "package name argument expected" <<
         info << "run 'bpkg help pkg-build' for more information";
 
+    // Check if the argument has the <packages>@<location> form. Return the
+    // delimiter position if that's the case. Otherwise return string::npos.
+    //
+    // We consider '@' to be such a delimiter if it comes before ':' (e.g., a
+    // URL which could contain its own '@').
+    //
+    auto location = [] (const string& arg) -> size_t
+    {
+      size_t p (arg.find_first_of ("@:"));
+      return p != string::npos && arg[p] == '@' ? p : string::npos;
+    };
+
+    // Collect repository locations from <packages>@<location> arguments,
+    // suppressing duplicates.
+    //
+    // Note that the last repository location overrides the previous ones with
+    // the same canonical name.
+    //
+    strings args;
+    vector<repository_location> locations;
+
+    while (a.more ())
+    {
+      string arg (a.next ());
+      size_t p (location (arg));
+
+      if (p != string::npos)
+      {
+        repository_location l (
+          parse_location (string (arg, p + 1), nullopt /* type */));
+
+        auto pr = [&l] (const repository_location& i) -> bool
+        {
+          return i.canonical_name () == l.canonical_name ();
+        };
+
+        auto i (find_if (locations.begin (), locations.end (), pr));
+
+        if (i != locations.end ())
+          *i = move (l);
+        else
+          locations.push_back (move (l));
+      }
+
+      args.push_back (move (arg));
+    }
+
+    database db (open (c, trace)); // Also populates the system repository.
+
+    // Note that the session spans all our transactions. The idea here is
+    // that selected_package objects in the build_packages list below will
+    // be cached in this session. When subsequent transactions modify any
+    // of these objects, they will modify the cached instance, which means
+    // our list will always "see" their updated state.
+    //
+    // Also note that rep_fetch() must be called in session.
+    //
+    session s;
+
+    if (!locations.empty ())
+      rep_fetch (o, c, db, locations);
+
+    // Expand <packages>@<location> arguments.
+    //
+    strings eargs;
+    {
+      transaction t (db.begin ());
+
+      for (string& arg: args)
+      {
+        size_t p (location (arg));
+
+        if (p == string::npos)
+        {
+          eargs.push_back (move (arg));
+          continue;
+        }
+
+        repository_location l (
+          parse_location (string (arg, p + 1), nullopt /* type */));
+
+        shared_ptr<repository> r (db.load<repository> (l.canonical_name ()));
+
+        // If no packages are specified explicitly (the argument starts with
+        // '@') then we select latest versions of all the packages from this
+        // repository. Otherwise, we search for the specified packages and
+        // versions (if specified) or latest versions (if unspecified) in the
+        // repository and its complements (recursively), failing if any of
+        // them are not found.
+        //
+        if (p == 0) // No packages are specified explicitly.
+        {
+          // Collect the latest package version.
+          //
+          map<string, version> pvs;
+
+          using query = query<repository_package>;
+
+          for (const auto& rp: db.query<repository_package> (
+                 (query::repository::name == r->name) +
+                 order_by_version_desc (query::package::id.version)))
+          {
+            const shared_ptr<available_package>& p (rp);
+            pvs.insert (make_pair (p->id.name, p->version));
+          }
+
+          // Populate the argument list with the latest package versions.
+          //
+          for (const auto& pv: pvs)
+            eargs.push_back (pv.first + '/' + pv.second.string ());
+        }
+        else // Packages with optional versions in the coma-separated list.
+        {
+          string ps (arg, 0, p);
+          for (size_t b (0); b != string::npos;)
+          {
+            // Extract the package.
+            //
+            p = ps.find (',', b);
+
+            string  s (ps, b, p != string::npos ? p - b : p);
+            string  n (parse_package_name (s));
+            version v (parse_package_version (s));
+
+            optional<dependency_constraint> c (
+              !v.empty () ? optional<dependency_constraint> (v) : nullopt);
+
+            // Check if the package is present in the repository and its
+            // complements, recursively.
+            //
+            shared_ptr<available_package> ap (
+              find_available (db, n, r, c, false /* prereq */).first);
+
+            if (ap == nullptr)
+            {
+              diag_record dr (fail);
+              dr << "package " << s << " is not found in " << r->name;
+
+              if (!r->complements.empty ())
+                dr << " nor its complements";
+            }
+
+            // Add the package/version to the argument list.
+            //
+            eargs.push_back (ap->id.name + '/' + ap->version.string ());
+
+            b = p != string::npos ? p + 1 : p;
+          }
+        }
+      }
+
+      t.commit ();
+    }
+
     map<string, string> package_arg;
 
     // Check if the package is a duplicate. Return true if it is but harmless.
     //
-    auto check_dup = [&package_arg] (const string& n, const char* arg) -> bool
+    auto check_dup = [&package_arg] (const string& n, const string& a) -> bool
     {
-      auto r (package_arg.emplace (n, arg));
+      auto r (package_arg.emplace (n, a));
 
-      if (!r.second && r.first->second != arg)
+      if (!r.second && r.first->second != a)
         fail << "duplicate package " << n <<
           info << "first mentioned as " << r.first->second <<
-          info << "second mentioned as " << arg;
+          info << "second mentioned as " << a;
 
       return !r.second;
     };
 
     // Pre-scan the arguments and sort them out into optional and mandatory.
     //
-    strings args;
-    while (a.more ())
+    strings pargs;
+    for (const string& arg: eargs)
     {
-      const char* arg (a.next ());
-      const char* s (arg);
+      const char* s (arg.c_str ());
 
       bool opt (s[0] == '?');
       if (opt)
         ++s;
       else
-        args.push_back (s);
+        pargs.emplace_back (s);
 
       if (parse_package_scheme (s) == package_scheme::sys)
       {
@@ -1113,7 +1268,10 @@ namespace bpkg
           v = wildcard_version;
 
         const system_package* sp (system_repository.find (n));
-        if (sp == nullptr) // Will deal with all the duplicates later.
+
+        // Will deal with all the duplicates later.
+        //
+        if (sp == nullptr || !sp->authoritative)
           system_repository.insert (n, v, true);
       }
       else if (opt)
@@ -1121,22 +1279,12 @@ namespace bpkg
           info << "package is ignored";
     }
 
-    if (args.empty ())
+    if (pargs.empty ())
     {
       warn << "nothing to build";
       return 0;
     }
 
-    database db (open (c, trace));
-
-    // Note that the session spans all our transactions. The idea here is
-    // that selected_package objects in the build_packages list below will
-    // be cached in this session. When subsequent transactions modify any
-    // of these objects, they will modify the cached instance, which means
-    // our list will always "see" their updated state.
-    //
-    session s;
-
     // Assemble the list of packages we will need to build.
     //
     build_packages pkgs;
@@ -1156,7 +1304,7 @@ namespace bpkg
       // diagnostics if the name/version guess doesn't pan out.
       //
       bool diag (false);
-      for (auto i (args.cbegin ()); i != args.cend (); )
+      for (auto i (pargs.cbegin ()); i != pargs.cend (); )
       {
         const char* package (i->c_str ());
 
diff --git a/tests/common/satisfy/t4d/repositories b/tests/common/satisfy/t4d/repositories
index f0e1983..6277925 100644
--- a/tests/common/satisfy/t4d/repositories
+++ b/tests/common/satisfy/t4d/repositories
@@ -1,3 +1,4 @@
 : 1
 location: ../t4c
+role: complement
 :
diff --git a/tests/pkg-build.test b/tests/pkg-build.test
index 0c9180a..2b54d12 100644
--- a/tests/pkg-build.test
+++ b/tests/pkg-build.test
@@ -48,7 +48,7 @@
 # |   |-- libfoo-1.0.0.tar.gz
 # |   `-- repositories
 # |
-# |-- t4d                     -> t4c (prerequisite)
+# |-- t4d                     -> t4c (complement)
 # |   |-- libbiz-1.0.0.tar.gz -> libfox, libfoo, libbaz
 # |   |-- libfox-1.0.0.tar.gz
 # |   `-- repositories
@@ -1305,6 +1305,96 @@ rep_fetch     += -d cfg --auth all --trust-yes 2>!
   $pkg_purge     libfoo 2>'purged libfoo/1.0.0'
 }
 
+: repository-location
+:
+{
+  test.arguments += --yes --auth all --trust-yes
+
+  : all-packages
+  :
+  {
+    $clone_root_cfg;
+
+    $* "@$rep/t4d" 2>>~%EOE%;
+      %.+
+      %info: .+libfox-1.0.0.+ is up to date%
+      %info: .+libbiz-1.0.0.+ is up to date%
+      updated libfox/1.0.0
+      updated libbiz/1.0.0
+      EOE
+
+    $pkg_disfigure libbiz 2>'disfigured libbiz/1.0.0';
+    $pkg_purge     libbiz 2>'purged libbiz/1.0.0';
+
+    $pkg_disfigure libfox 2>'disfigured libfox/1.0.0';
+    $pkg_purge     libfox 2>'purged libfox/1.0.0';
+
+    $pkg_disfigure libbaz 2>'disfigured libbaz/1.1.0';
+    $pkg_purge     libbaz 2>'purged libbaz/1.1.0';
+
+    $pkg_disfigure libbar 2>'disfigured libbar/1.1.0';
+    $pkg_purge     libbar 2>'purged libbar/1.1.0';
+
+    $pkg_disfigure libfoo 2>'disfigured libfoo/1.1.0';
+    $pkg_purge     libfoo 2>'purged libfoo/1.1.0'
+  }
+
+  : multiple-packages
+  :
+  {
+    $clone_root_cfg;
+
+    $* "libfox,libbiz/1.0.0@$rep/t4d" 2>>~%EOE%;
+      %.+
+      %info: .+libfox-1.0.0.+ is up to date%
+      %info: .+libbiz-1.0.0.+ is up to date%
+      updated libfox/1.0.0
+      updated libbiz/1.0.0
+      EOE
+
+    $pkg_disfigure libbiz 2>'disfigured libbiz/1.0.0';
+    $pkg_purge     libbiz 2>'purged libbiz/1.0.0';
+
+    $pkg_disfigure libfox 2>'disfigured libfox/1.0.0';
+    $pkg_purge     libfox 2>'purged libfox/1.0.0';
+
+    $pkg_disfigure libbaz 2>'disfigured libbaz/1.1.0';
+    $pkg_purge     libbaz 2>'purged libbaz/1.1.0';
+
+    $pkg_disfigure libbar 2>'disfigured libbar/1.1.0';
+    $pkg_purge     libbar 2>'purged libbar/1.1.0';
+
+    $pkg_disfigure libfoo 2>'disfigured libfoo/1.1.0';
+    $pkg_purge     libfoo 2>'purged libfoo/1.1.0'
+  }
+
+  : package-in-complement
+  :
+  {
+    $clone_root_cfg;
+
+    $* "libfoo@$rep/t4d" 2>>~%EOE%;
+      %.+
+      %info: .+libfoo-1.0.0.+ is up to date%
+      updated libfoo/1.0.0
+      EOE
+
+    $pkg_disfigure libfoo 2>'disfigured libfoo/1.0.0';
+    $pkg_purge     libfoo 2>'purged libfoo/1.0.0'
+  }
+
+  : non-existent-package
+  :
+  {
+    $clone_root_cfg;
+
+    $* "libbar@$rep/t4d" 2>>~%EOE% != 0
+      %.+
+      error: package libbar is not found in bpkg:build2.org/pkg-build/t4d nor its complements
+      EOE
+  }
+}
+
 : git-repos
 :
 if ($git_supported != true)
diff --git a/tests/pkg-drop.test b/tests/pkg-drop.test
index 07c1f93..9e63470 100644
--- a/tests/pkg-drop.test
+++ b/tests/pkg-drop.test
@@ -17,7 +17,7 @@
 # |   |-- libbaz-1.1.0.tar.gz -> libfoo, libbar
 # |   |-- libfoo-1.0.0.tar.gz
 # |   `-- repositories
-# `-- t4d                     -> t4c (prerequisite)
+# `-- t4d                     -> t4c (complement)
 #     |-- libbiz-1.0.0.tar.gz -> libfox, libfoo, libbaz
 #     |-- libfox-1.0.0.tar.gz
 #     `-- repositories
@@ -369,9 +369,9 @@ $* libfoo/1.0.0 2>>~%EOE% != 0
      EOE
 
    -$pkg_status libfox/1.0.0 >'available'
-   -$pkg_status libfoo/1.1.0 >'unknown'
+   -$pkg_status libfoo/1.1.0 >'available'
    -$pkg_status libbar/1.1.0 >'unknown'
-   -$pkg_status libbaz/1.1.0 >'unknown'
+   -$pkg_status libbaz/1.1.0 >'available'
    -$pkg_status libbiz/1.0.0 >'available'
 }