Fix unexpected 'no package available for dependency' error when building from archives (GH issue #303)

10 files changed, 282 insertions, 12 deletions

diff --git a/bpkg/package-query.cxx b/bpkg/package-query.cxx
index 9705579..0c6d459 100644
--- a/bpkg/package-query.cxx
+++ b/bpkg/package-query.cxx
@@ -7,6 +7,7 @@
 #include <bpkg/package-odb.hxx>
 #include <bpkg/database.hxx>
 #include <bpkg/rep-mask.hxx>
+#include <bpkg/satisfaction.hxx>
 
 using namespace std;
 
@@ -26,6 +27,34 @@ namespace bpkg
     return i != imaginary_stubs.end () ? *i : nullptr;
   }
 
+  vector<pair<reference_wrapper<database>,
+              shared_ptr<available_package>>> existing_packages;
+
+  pair<shared_ptr<available_package>, lazy_shared_ptr<repository_fragment>>
+  find_existing (const package_name& name,
+                 const optional<version_constraint>& c,
+                 const lazy_shared_ptr<repository_fragment>& rf)
+  {
+    database& db (rf.database ());
+
+    pair<shared_ptr<available_package>,
+         lazy_shared_ptr<repository_fragment>> r;
+
+    for (const auto& p: existing_packages)
+    {
+      if (p.first == db             &&
+          p.second->id.name == name &&
+          (!c || satisfies (p.second->version, *c)))
+      {
+        r.first = p.second;
+        r.second = lazy_shared_ptr<repository_fragment> (db, empty_string);
+        break;
+      }
+    }
+
+    return r;
+  }
+
   linked_databases repo_configs;
 
   linked_databases
diff --git a/bpkg/package-query.hxx b/bpkg/package-query.hxx
index 12456ce..be4e0be 100644
--- a/bpkg/package-query.hxx
+++ b/bpkg/package-query.hxx
@@ -226,6 +226,25 @@ namespace bpkg
   shared_ptr<available_package>
   find_imaginary_stub (const package_name&);
 
+  // Try to find an available package in the existing packages registry. Such
+  // a registry is configuration-specific and contains package versions
+  // specified by the user on the command line as archives or directories for
+  // specific configurations (see pkg-build for details on such packages).
+  //
+  // Note that semantically such a registry can be considered as an imaginary
+  // repository which complements all the real repositories fetched in the
+  // respective configuration. Also note that normally this repository is used
+  // first (by calling this function) when trying to resolve a dependency
+  // package, prior to searching in the real repositories.
+  //
+  extern vector<pair<reference_wrapper<database>,
+                     shared_ptr<available_package>>> existing_packages;
+
+  pair<shared_ptr<available_package>, lazy_shared_ptr<repository_fragment>>
+  find_existing (const package_name&,
+                 const optional<version_constraint>&,
+                 const lazy_shared_ptr<repository_fragment>&);
+
   // Configurations to use as the repository information sources.
   //
   // The list normally contains the current configurations and configurations
diff --git a/bpkg/pkg-build-collect.cxx b/bpkg/pkg-build-collect.cxx
index c581f5c..4006caa 100644
--- a/bpkg/pkg-build-collect.cxx
+++ b/bpkg/pkg-build-collect.cxx
@@ -2507,7 +2507,10 @@ namespace bpkg
               // recognized. An unrecognized package means the broken/stale
               // repository (see below).
               //
-              rp = find_available_one (dn, d.constraint, af);
+              rp = find_existing (dn, d.constraint, af);
+
+              if (dap == nullptr)
+                rp = find_available_one (dn, d.constraint, af);
 
               if (dap == nullptr && system && d.constraint)
                 rp = find_available_one (dn, nullopt, af);
@@ -2658,7 +2661,7 @@ namespace bpkg
                                    system,
                                    specified,
                                    force,
-                                    ru});
+                                   ru});
           }
 
           // Now, as we have pre-collected the dependency builds, go through
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index 9a73ea6..46569df 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -2946,6 +2946,8 @@ namespace bpkg
               af = root;
               ap = make_shared<available_package> (move (m));
               ap->locations.push_back (package_location {root, move (a)});
+
+              existing_packages.push_back (make_pair (ref (*pdb), ap));
             }
           }
           catch (const invalid_path&)
@@ -3037,6 +3039,8 @@ namespace bpkg
                 ap = make_shared<available_package> (move (m));
                 af = root;
                 ap->locations.push_back (package_location {root, move (d)});
+
+                existing_packages.push_back (make_pair (ref (*pdb), ap));
               }
             }
             catch (const invalid_path&)
diff --git a/bpkg/pkg-checkout.cxx b/bpkg/pkg-checkout.cxx
index d7b36e4..81efdc2 100644
--- a/bpkg/pkg-checkout.cxx
+++ b/bpkg/pkg-checkout.cxx
@@ -133,7 +133,7 @@ namespace bpkg
 
     check_any_available (rdb, t);
 
-    // Note that here we compare including the revision (see pkg-fetch()
+    // Note that here we compare including the revision (see pkg_fetch()
     // implementation for more details).
     //
     shared_ptr<available_package> ap (
diff --git a/bpkg/pkg-unpack.cxx b/bpkg/pkg-unpack.cxx
index 6e99b36..323f296 100644
--- a/bpkg/pkg-unpack.cxx
+++ b/bpkg/pkg-unpack.cxx
@@ -289,7 +289,7 @@ namespace bpkg
 
     check_any_available (rdb, t);
 
-    // Note that here we compare including the revision (see pkg-fetch()
+    // Note that here we compare including the revision (see pkg_fetch()
     // implementation for more details).
     //
     shared_ptr<available_package> ap (
diff --git a/tests/common/satisfy/libbar-1.2.0.tar.gz b/tests/common/satisfy/libbar-1.2.0.tar.gz
index f8fb1ec..51d997f 100644
--- a/tests/common/satisfy/libbar-1.2.0.tar.gz
+++ b/tests/common/satisfy/libbar-1.2.0.tar.gz
diff --git a/tests/pkg-build.testscript b/tests/pkg-build.testscript
index e018a1b..fe7f9e6 100644
--- a/tests/pkg-build.testscript
+++ b/tests/pkg-build.testscript
@@ -490,6 +490,8 @@ end
 
 config_cxx = [cmdline] config.cxx=$quote($recall($cxx.path) $cxx.config.mode, true)
 
+tar = [cmdline] tar (!$posix ? --force-local : )
+
 cfg_create    += 2>!
 cfg_link      += 2>!
 pkg_configure += -d cfg 2>!
@@ -4658,7 +4660,7 @@ test.arguments += --sys-no-query
       {
         +$clone_cfg
 
-        +tar (!$posix ? --force-local : ) -xzf $src/t8a/fax-1.0.0.tar.gz &fax-1.0.0/***
+        +$tar -xzf $src/t8a/fax-1.0.0.tar.gz &fax-1.0.0/***
         +mv fax-1.0.0 fax
 
         : change-manifest
@@ -18759,14 +18761,14 @@ else
       {
         $clone_root_cfg;
 
-        tar (!$posix ? --force-local : ) -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
+        $tar -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
         mv libfoo-1.1.0+2 libfoo;
 
         $rep_add --type dir libfoo/ && $rep_fetch;
 
         $* libfoo 2>!;
 
-        tar (!$posix ? --force-local : ) -xf $src/t2/libbar-1.0.0.tar.gz &libbar-1.0.0/***;
+        $tar -xf $src/t2/libbar-1.0.0.tar.gz &libbar-1.0.0/***;
         mv libbar-1.0.0 libbar;
 
         cat <<"EOI" >=libbar/repositories.manifest;
@@ -19131,7 +19133,7 @@ else
     : recursive
     :
     {
-      +tar (!$posix ? --force-local : ) -xf $src/t2/libbar-1.0.0.tar.gz &libbar-1.0.0/***
+      +$tar -xf $src/t2/libbar-1.0.0.tar.gz &libbar-1.0.0/***
       +mv libbar-1.0.0 libbar
 
       +cat <<"EOI" >=libbar/repositories.manifest
@@ -19387,7 +19389,7 @@ else
     : As above but uses 'deorphan all held packages form'.
     :
     {
-      +tar (!$posix ? --force-local : ) -xf $src/t2/libbar-1.0.0.tar.gz &libbar-1.0.0/***
+      +$tar -xf $src/t2/libbar-1.0.0.tar.gz &libbar-1.0.0/***
       +mv libbar-1.0.0 libbar
 
       +cat <<"EOI" >=libbar/repositories.manifest
@@ -19687,7 +19689,7 @@ else
     {
       $clone_root_cfg;
 
-      tar (!$posix ? --force-local : ) -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
+      $tar -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
       mv libfoo-1.1.0+2 libfoo;
 
       $rep_add --type dir libfoo/ && $rep_fetch;
@@ -19906,7 +19908,7 @@ else
     {
       $clone_root_cfg;
 
-      tar (!$posix ? --force-local : ) -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
+      $tar -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
       mv libfoo-1.1.0+2 libfoo;
 
       $rep_add --type dir libfoo/ && $rep_fetch;
@@ -20125,7 +20127,7 @@ else
     {
       $clone_root_cfg;
 
-      tar (!$posix ? --force-local : ) -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
+      $tar -xf $src/t14d/libfoo-1.1.0+2.tar.gz &libfoo-1.1.0+2/***;
       mv libfoo-1.1.0+2 libfoo;
 
       $rep_add --type dir libfoo/ && $rep_fetch;
@@ -20406,3 +20408,214 @@ else
     }
   }
 }
+
+: existing-package
+:
+{
+  +$clone_cfg
+
+  test.arguments += --yes
+
+  : archive
+  :
+  {
+    +$clone_cfg
+
+    : dependency-archive
+    :
+    : Test that libbar/1.0.0 specified as an archive is picked as a
+    : dependency for libbaz, despite the fact the repository contains
+    : libbar/1.2.0.
+    :
+    {
+      $clone_cfg;
+      $rep_add $rep/t5 && $rep_fetch;
+
+      $* $src/libbaz-1.1.0.tar.gz \
+         $src/libfoo-1.0.0.tar.gz \
+         $src/libbar-1.0.0.tar.gz 2>>~%EOE%;
+        fetched libfoo/1.0.0
+        unpacked libfoo/1.0.0
+        fetched libbar/1.0.0
+        unpacked libbar/1.0.0
+        fetched libbaz/1.1.0
+        unpacked libbaz/1.1.0
+        configured libfoo/1.0.0
+        configured libbar/1.0.0
+        configured libbaz/1.1.0
+        %info: .+libfoo-1.0.0.+ is up to date%
+        %info: .+libbar-1.0.0.+ is up to date%
+        %info: .+libbaz-1.1.0.+ is up to date%
+        updated libfoo/1.0.0
+        updated libbar/1.0.0
+        updated libbaz/1.1.0
+        EOE
+
+      $pkg_status -r libbaz >>EOO;
+        !libbaz configured !1.1.0
+          !libbar configured !1.0.0 available 1.2.0
+            !libfoo configured !1.0.0
+          !libfoo configured !1.0.0
+        EOO
+
+      $pkg_drop libbaz libfoo libbar
+    }
+
+    : dependency-repository
+    :
+    : Picks the libbar/1.2.0 dependency from the repository for the
+    : dependent libbaz/1.1.0 specified as an archive.
+    :
+    {
+      $clone_cfg;
+      $rep_add $rep/t5 && $rep_fetch;
+
+      $* $src/libbaz-1.1.0.tar.gz \
+         $src/libfoo-1.0.0.tar.gz 2>>~%EOE%;
+        fetched libbar/1.2.0
+        unpacked libbar/1.2.0
+        fetched libfoo/1.0.0
+        unpacked libfoo/1.0.0
+        fetched libbaz/1.1.0
+        unpacked libbaz/1.1.0
+        configured libbar/1.2.0
+        configured libfoo/1.0.0
+        configured libbaz/1.1.0
+        %info: .+libfoo-1.0.0.+ is up to date%
+        %info: .+libbaz-1.1.0.+ is up to date%
+        updated libfoo/1.0.0
+        updated libbaz/1.1.0
+        EOE
+
+      $pkg_status -r libbaz >>EOO;
+        !libbaz configured !1.1.0
+          libbar configured 1.2.0
+          !libfoo configured !1.0.0
+        EOO
+
+      $pkg_drop libbaz libfoo libbar
+    }
+
+    : unsatisfactory-dependency
+    :
+    {
+      $clone_cfg;
+      $rep_add $rep/t4b && $rep_fetch;
+
+      $* libbar $src/libfoo-1.0.0.tar.gz 2>>~%EOE% != 0
+        error: unable to satisfy constraints on package libfoo
+          info: command line depends on (libfoo == 1.0.0)
+          info: libbar depends on (libfoo == 1.1.0)
+          info: available libfoo/1.0.0
+          info: available libfoo/1.1.0
+          info: explicitly specify libfoo version to manually satisfy both constraints
+        info: while satisfying libbar/1.1.0
+        EOE
+    }
+
+    : unsatisfactory-dependency-archive-dependent
+    :
+    : Save as above but the dependent is specified as an archive.
+    :
+    {
+      $clone_cfg;
+      $rep_add $rep/t4a && $rep_fetch; # Note: libfoo/1.1.0 belongs to t4a.
+
+      $* $src/libbar-1.1.0.tar.gz $src/libfoo-1.0.0.tar.gz 2>>~%EOE% != 0
+        error: unable to satisfy constraints on package libfoo
+          info: command line depends on (libfoo == 1.0.0)
+          info: libbar depends on (libfoo == 1.1.0)
+          info: available libfoo/1.0.0
+          info: available libfoo/1.1.0
+          info: explicitly specify libfoo version to manually satisfy both constraints
+        info: while satisfying libbar/1.1.0
+        EOE
+    }
+
+    : dependency-alternative
+    :
+    : Note: by specifying an unsatisfactory dependency alternative as an
+    : archive we resolve the alternatives ambiguity here, building both libbar
+    : and libbaz packages as a result.
+    :
+    {
+      $clone_cfg;
+      $rep_add $rep/t8a && $rep_fetch;
+
+      $* fox 2>>EOE != 0;
+        error: unable to select dependency alternative for package fox/1.0.0
+          info: explicitly specify dependency packages to manually select the alternative
+          info: alternative: libbar
+          info: alternative: libbaz
+        info: while satisfying fox/1.0.0
+        EOE
+
+      $* fox $src/libbar-0.0.3.tar.gz 2>>~%EOE%;
+        fetched libbaz/1.1.0
+        unpacked libbaz/1.1.0
+        fetched fox/1.0.0
+        unpacked fox/1.0.0
+        fetched libbar/0.0.3
+        unpacked libbar/0.0.3
+        configured libbaz/1.1.0
+        configured fox/1.0.0
+        configured libbar/0.0.3
+        %info: .+fox-1.0.0.+ is up to date%
+        %info: .+libbar-0.0.3.+ is up to date%
+        updated fox/1.0.0
+        updated libbar/0.0.3
+        EOE
+
+      $pkg_status -r fox libbar >>EOO;
+        !fox configured 1.0.0
+          libbaz configured 1.1.0
+        !libbar configured !0.0.3 available 1.0.0
+          libbaz configured 1.1.0
+        EOO
+
+      $pkg_drop fox libbar
+    }
+  }
+
+  : directory
+  :
+  {
+    +$clone_cfg
+
+    : dependency-directory
+    :
+    {
+      $clone_cfg;
+
+      $tar -xf $src/libbaz-1.1.0.tar.gz &libbaz-1.1.0/***;
+      $tar -xf $src/libfoo-1.0.0.tar.gz &libfoo-1.0.0/***;
+      $tar -xf $src/libbar-1.0.0.tar.gz &libbar-1.0.0/***;
+
+      $* libbaz-1.1.0/ \
+         libfoo-1.0.0/ \
+         libbar-1.0.0/ 2>>~%EOE%;
+        using libfoo/1.0.0 (external)
+        using libbar/1.0.0 (external)
+        using libbaz/1.1.0 (external)
+        configured libfoo/1.0.0
+        configured libbar/1.0.0
+        configured libbaz/1.1.0
+        %info: .+libfoo.+ is up to date%
+        %info: .+libbar.+ is up to date%
+        %info: .+libbaz.+ is up to date%
+        updated libfoo/1.0.0
+        updated libbar/1.0.0
+        updated libbaz/1.1.0
+        EOE
+
+      $pkg_status -r libbaz >>EOO;
+        !libbaz configured !1.1.0
+          !libbar configured !1.0.0
+            !libfoo configured !1.0.0
+          !libfoo configured !1.0.0
+        EOO
+
+      $pkg_drop libbaz libfoo libbar
+    }
+  }
+}
diff --git a/tests/pkg-build/libbar-0.0.3.tar.gz b/tests/pkg-build/libbar-0.0.3.tar.gz
new file mode 120000
index 0000000..308e978
--- /dev/null
+++ b/tests/pkg-build/libbar-0.0.3.tar.gz
@@ -0,0 +1 @@
+../common/satisfy/libbar-0.0.3.tar.gz
\ No newline at end of file
diff --git a/tests/pkg-build/libbar-1.1.0.tar.gz b/tests/pkg-build/libbar-1.1.0.tar.gz
new file mode 120000
index 0000000..12ae746
--- /dev/null
+++ b/tests/pkg-build/libbar-1.1.0.tar.gz
@@ -0,0 +1 @@
+../common/satisfy/libbar-1.1.0.tar.gz
\ No newline at end of file