Create package skeleton filesystem state

4 files changed, 269 insertions, 75 deletions

diff --git a/bpkg/package-skeleton.cxx b/bpkg/package-skeleton.cxx
new file mode 100644
index 0000000..ebcc044
--- /dev/null
+++ b/bpkg/package-skeleton.cxx
@@ -0,0 +1,163 @@
+// file      : bpkg/package-skeleton.cxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#include <bpkg/package-skeleton.hxx>
+
+#include <libbutl/manifest-serializer.hxx>
+
+#include <bpkg/package.hxx>
+#include <bpkg/database.hxx>
+#include <bpkg/manifest-utility.hxx>
+
+using namespace std;
+using namespace butl;
+
+namespace bpkg
+{
+  package_skeleton::
+  package_skeleton (database& db,
+                    const available_package& ap,
+                    const strings& cvs,
+                    optional<dir_path> src_root)
+      : db_ (db),
+        available_ (ap),
+        config_vars_ (cvs),
+        src_root_ (move (src_root))
+  {
+    // Should not be created for stubs.
+    //
+    assert (available_.get ().bootstrap_build);
+
+    if (src_root_)
+      out_root_ = dir_path (db_.get ().config_orig) /= name ().string ();
+  }
+
+  void package_skeleton::
+  load ()
+  {
+    if (loaded_ && !dirty_)
+      return;
+
+    const available_package& ap (available_);
+
+    // The overall plan is as follows:
+    //
+    // 0. Create filesystem state if necessary (could have been created by
+    //    another instance, e.g., during simulation).
+    //
+    // 1. If loaded but dirty, save the accumulated reflect state, and
+    //    destroy the old state.
+    //
+    // 2. Load the state potentially with accumulated reflect state.
+
+    // Create the skeleton filesystem state, if it doesn't exist yet.
+    //
+    // Note that we create the skeleton directories in the skeletons/
+    // subdirectory of the configuration temporary directory to make sure they
+    // never clash with other temporary subdirectories (git repositories, etc).
+    //
+    if (!src_root_)
+    {
+      auto i (temp_dir.find (db_.get ().config_orig));
+      assert (i != temp_dir.end ());
+
+      dir_path d (i->second);
+      d /= "skeletons";
+      d /= name ().string () + '-' + ap.version.string ();
+
+      src_root_ = d;
+      out_root_ = move (d);
+    }
+
+    if (!exists (*src_root_))
+    {
+      // Create the buildfiles.
+      //
+      // Note that it's probably doesn't matter which naming scheme to use for
+      // the buildfiles, unless in the future we allow specifying additional
+      // files.
+      //
+      {
+        path bf (*src_root_ / std_bootstrap_file);
+
+        mk_p (bf.directory ());
+
+        // Save the {bootstrap,root}.build files.
+        //
+        auto save = [] (const string& s, const path& f)
+        {
+          try
+          {
+            ofdstream os (f);
+            os << s;
+            os.close ();
+          }
+          catch (const io_error& e)
+          {
+            fail << "unable to write to " << f << ": " << e;
+          }
+        };
+
+        save (*ap.bootstrap_build, bf);
+
+        if (ap.root_build)
+          save (*ap.root_build, *src_root_ / std_root_file);
+      }
+
+      // Create the manifest file containing the bare minimum of values
+      // which can potentially be required to load the build system state.
+      //
+      {
+        package_manifest m;
+        m.name = name ();
+        m.version = ap.version;
+
+        // Note that there is no guarantee that the potential build2
+        // constraint has already been verified. Thus, we also serialize the
+        // depends value, delegating the constraint verification to the
+        // version module. Also note that normally the toolchain build-time
+        // dependencies are specified first and, if that's the case, their
+        // constraints are already verified at this point and so build2 will
+        // not fail due to the constraint violation.
+        //
+        // Also note that the resulting file is not quite a valid package
+        // manifest, since it doesn't contain all the required values
+        // (summary, etc). It, however, is good enough for build2 which
+        // doesn't perform exhaustive manifest validation.
+        //
+        m.dependencies.reserve (ap.dependencies.size ());
+        for (const dependency_alternatives_ex& das: ap.dependencies)
+        {
+          // Skip the the special (inverse) test dependencies.
+          //
+          if (!das.type)
+            m.dependencies.push_back (das);
+        }
+
+        path mf (*src_root_ / manifest_file);
+
+        try
+        {
+          ofdstream os (mf);
+          manifest_serializer s (os, mf.string ());
+          m.serialize (s);
+          os.close ();
+        }
+        catch (const manifest_serialization&)
+        {
+          // We shouldn't be creating a non-serializable manifest, since it's
+          // crafted from the parsed values.
+          //
+          assert (false);
+        }
+        catch (const io_error& e)
+        {
+          fail << "unable to write to " << mf << ": " << e;
+        }
+      }
+    }
+
+    loaded_ = true;
+    dirty_ = false;
+  }
+}
diff --git a/bpkg/package-skeleton.hxx b/bpkg/package-skeleton.hxx
index 5fcb151..889a937 100644
--- a/bpkg/package-skeleton.hxx
+++ b/bpkg/package-skeleton.hxx
@@ -17,6 +17,14 @@ namespace bpkg
   class package_skeleton
   {
   public:
+    // If the package is external and will not be disfigured, then the
+    // existing package source root directory needs to be specified. In this
+    // case this source directory and the automatically deduced potentially
+    // non-existing out root directory will be used for build2 state loading
+    // instead of the newly created skeleton directory. This, in particular,
+    // allows to consider existing configuration variables while evaluating
+    // the dependency clauses.
+    //
     // Note that the database and available_package are expected to outlive
     // this object.
     //
@@ -32,10 +40,10 @@ namespace bpkg
     //    configuration from it, etc). Let's however keep it simple for now
     //    and just copy the configuration.
     //
-    package_skeleton (database& db,
-                      const available_package& ap,
-                      const strings& cvs)
-        : db_ (db), available_ (ap), config_vars_ (cvs) {}
+    package_skeleton (database&,
+                      const available_package&,
+                      const strings& cvs,
+                      optional<dir_path> src_root);
 
     // Evaluate the enable clause.
     //
@@ -91,27 +99,7 @@ namespace bpkg
     // Call this function before evaluating every clause.
     //
     void
-    load ()
-    {
-      if (loaded_ && !dirty_)
-        return;
-
-      // Plan:
-      //
-      // 0. Create filesystem state if necessary (could have been created by
-      //    another instance, e.g., during simulation).
-      //
-      //    @@ build/ vs build2/ -- probably doesn't matter unless in the
-      //    future we allow specifying additional files.
-      //
-      // 1. If loaded but dirty, save the accumulated reflect state, and
-      //    destroy the old state.
-      //
-      // 2. Load the state potentially with accumulated reflect state.
-
-      loaded_ = true;
-      dirty_ = false;
-    }
+    load ();
 
     // Mark the build system state as needing reloading.
     //
@@ -129,6 +117,9 @@ namespace bpkg
     reference_wrapper<const available_package> available_;
     strings config_vars_;
 
+    optional<dir_path> src_root_;
+    optional<dir_path> out_root_;
+
     bool loaded_ = false;
     bool dirty_ = false;
 
diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx
index 63910e8..f53c14c 100644
--- a/bpkg/pkg-build.cxx
+++ b/bpkg/pkg-build.cxx
@@ -712,6 +712,58 @@ namespace bpkg
         (*action == build && (flags & build_repoint) != 0);
     }
 
+    // Return true if this build replaces an external package with another
+    // external.
+    //
+    bool
+    external () const
+    {
+      if (selected == nullptr || !selected->external ())
+        return false;
+
+      assert (action);
+
+      if (*action == build_package::drop)
+        return false;
+
+      bool r (false);
+
+      // If adjustment or orphan, then new and old are the same.
+      //
+      if (available == nullptr || available->locations.empty ())
+      {
+        r = true;
+      }
+      else
+      {
+        const package_location& pl (available->locations[0]);
+
+        if (pl.repository_fragment.object_id () == "") // Special root.
+        {
+          r = !exists (pl.location); // Directory case.
+        }
+        else
+        {
+          // See if the package comes from the directory-based repository, and
+          // so is external.
+          //
+          // Note that such repository fragments are always preferred over
+          // others (see below).
+          //
+          for (const package_location& l: available->locations)
+          {
+            if (l.repository_fragment.load ()->location.directory_based ())
+            {
+              r = true;
+              break;
+            }
+          }
+        }
+      }
+
+      return r;
+    }
+
     const version&
     available_version () const
     {
@@ -774,6 +826,20 @@ namespace bpkg
         // options and variables are only saved into the pre-entered
         // dependencies, etc.).
         //
+        // Note that configuration can only be specified for packages on the
+        // command line and such packages get collected/pre-entered early,
+        // before any prerequisites get collected. Thus, it doesn't seem
+        // possible that a package configuration/options may change after we
+        // have created the package skeleton.
+        //
+        // Also note that if it wouldn't be true, we would potentially need to
+        // re-collect the package prerequisites, since configuration change
+        // could affect the enable condition evaluation and, as a result, the
+        // dependency alternative choice.
+        //
+        assert (!skeleton ||
+                (p.config_vars == config_vars && p.disfigure == disfigure));
+
         if (p.keep_out)
           keep_out = p.keep_out;
 
@@ -789,19 +855,6 @@ namespace bpkg
         if (p.checkout_purge)
           checkout_purge = p.checkout_purge;
 
-        // Note that configuration can only be specified for packages on the
-        // command line and such packages get collected/pre-entered early,
-        // before any prerequisites get collected. Thus, it doesn't seem
-        // possible that a package configuration may change after we have
-        // created the package skeleton.
-        //
-        // Also note that if it wouldn't be true, we would potentially need to
-        // re-collect the package prerequisites, since configuration change
-        // could affect the enable condition evaluation and, as a result, the
-        // dependency alternative choice.
-        //
-        assert (!skeleton || p.config_vars == config_vars);
-
         if (!p.config_vars.empty ())
           config_vars = move (p.config_vars);
 
@@ -1214,7 +1267,14 @@ namespace bpkg
         if (size_t n = deps.size ())
           pkg.dependencies->reserve (n);
 
-        pkg.skeleton = package_skeleton (pdb, *ap, pkg.config_vars);
+        optional<dir_path> src_root;
+        if (pkg.external () && !pkg.disfigure)
+          src_root = sp->src_root;
+
+        pkg.skeleton = package_skeleton (pdb,
+                                         *ap,
+                                         pkg.config_vars,
+                                         move (src_root));
       }
 
       dependencies& sdeps (*pkg.dependencies);
@@ -7589,39 +7649,7 @@ namespace bpkg
       bool external (false);
       if (!simulate)
       {
-        if (sp->external () && *p.action != build_package::drop)
-        {
-          const shared_ptr<available_package>& ap (p.available);
-
-          // If adjustment or orphan, then new and old are the same.
-          //
-          if (ap == nullptr || ap->locations.empty ())
-            external = true;
-          else
-          {
-            const package_location& pl (ap->locations[0]);
-
-            if (pl.repository_fragment.object_id () == "") // Special root.
-              external = !exists (pl.location); // Directory case.
-            else
-            {
-              // See if the package comes from the directory-based repository,
-              // and so is external.
-              //
-              // Note that such repository fragments are always preferred over
-              // others (see below).
-              //
-              for (const package_location& l: ap->locations)
-              {
-                if (l.repository_fragment.load ()->location.directory_based ())
-                {
-                  external = true;
-                  break;
-                }
-              }
-            }
-          }
-        }
+        external = p.external ();
 
         // Reset the keep_out flag if the package being unpacked is not
         // external.
@@ -8071,7 +8099,11 @@ namespace bpkg
         }
         else
         {
-          package_skeleton ps (pdb, *ap, p.config_vars);
+          optional<dir_path> src_root;
+          if (p.external () && !p.disfigure)
+            src_root = sp->src_root;
+
+          package_skeleton ps (pdb, *ap, p.config_vars, move (src_root));
 
           pkg_configure (o,
                          pdb,
@@ -8102,7 +8134,11 @@ namespace bpkg
         if (dap == nullptr)
           dap = make_available (o, pdb, sp);
 
-        package_skeleton ps (pdb, *dap, p.config_vars);
+        optional<dir_path> src_root;
+        if (p.external () && !p.disfigure)
+          src_root = sp->src_root;
+
+        package_skeleton ps (pdb, *dap, p.config_vars, move (src_root));
 
         // @@ Note that on reconfiguration the dependent looses the potential
         //    configuration variables specified by the user on some previous
diff --git a/bpkg/pkg-configure.cxx b/bpkg/pkg-configure.cxx
index 23cd85c..ff6fde6 100644
--- a/bpkg/pkg-configure.cxx
+++ b/bpkg/pkg-configure.cxx
@@ -429,7 +429,11 @@ namespace bpkg
       //
       shared_ptr<available_package> ap (make_available (o, db, p));
 
-      package_skeleton ps (db, *ap, vars);
+      optional<dir_path> src_root;
+      if (p->external ())
+        src_root = p->src_root;
+
+      package_skeleton ps (db, *ap, vars, move (src_root));
 
       pkg_configure (o,
                      db,