WIP

5 files changed, 446 insertions, 127 deletions

diff --git a/bpkg/pkg-bindist.cli b/bpkg/pkg-bindist.cli
index 5686ed6..ad743d9 100644
--- a/bpkg/pkg-bindist.cli
+++ b/bpkg/pkg-bindist.cli
@@ -98,6 +98,19 @@ namespace bpkg
        documentation in the build system manual for details. This option
        only makes sense together with \cb{--recursive}."
     }
+
+    bool --wipe-out
+    {
+      "Wipe the output directory (\ci{out-dir}) clean before using it to
+       generate the binary package."
+    }
+
+    bool --keep-out
+    {
+      "Keep intermediate files in the output directory (\ci{out-dir}) that
+       were used to generate the binary package. This is primarily useful
+       for troubleshooting."
+    }
   };
 
   "
diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx
index 47df65d..37844f1 100644
--- a/bpkg/system-package-manager-debian.cxx
+++ b/bpkg/system-package-manager-debian.cxx
@@ -3,8 +3,12 @@
 
 #include <bpkg/system-package-manager-debian.hxx>
 
+#include <libbutl/filesystem.hxx> // permissions
+
 #include <bpkg/diagnostics.hxx>
 
+#include <bpkg/pkg-bindist-options.hxx>
+
 using namespace butl;
 
 namespace bpkg
@@ -1455,6 +1459,311 @@ namespace bpkg
     }
   }
 
+  // Map non-system bpkg package to system package name(s) and version.
+  //
+  // This is used both to map the package being generated and its
+  // dependencies. What should we do with extras returned in package_status?
+  // We can't really generate any of them (which files would we place in
+  // them?) nor can we list them as dependencies (we don't know their system
+  // versions). So it feels like the only sensible choice is to ignore extras.
+  //
+  // In a sense, we have a parallel arrangement going on here: binary packages
+  // that we generate don't have extras (i.e., they include everything
+  // necessary in the "standard" packages from the main group) and when we
+  // punch a system dependency based on a non-system bpkg package, we assume
+  // it was generated by us and thus doesn't have any extras. Or, to put it
+  // another way, if you want the system dependency to refer to a "native"
+  // system package with extras you need to configure it as a system bpkg
+  // package.
+  //
+  // In fact, this extends to package names. For example, unless custom
+  // mapping is specified, we will generate libsqlite3 and libsqlite3-dev
+  // while native names are libsqlite3-0 and libsqlite3-dev. While this
+  // duality is not ideal, presumably we will normally only be producing our
+  // binary packages if there are no suitable native packages. And for a few
+  // exception (e.g., our package is "better" in some way, such as configured
+  // differently or fixes a critical bug), we will just have to provide
+  // appropriate manual mapping that makes sure the names match (the extras is
+  // still a potential problem though -- we will only have them as
+  // dependencies if we build against a native system package).
+  //
+  // @@ TODO: test, especially distribution version logic.
+  //
+  package_status system_package_manager_debian::
+  map_package (const package_name& pn,
+               const version& pv,
+               const available_packages& aps)
+  {
+    // We should only have one available package corresponding to this package
+    // name/version.
+    //
+    assert (aps.size () == 1);
+
+    strings ns (system_package_names (aps,
+                                      os_release.name_id,
+                                      os_release.version_id,
+                                      os_release.like_ids));
+    package_status r;
+    if (ns.empty ())
+    {
+      // Automatically translate our package name similar to the consumption
+      // case above. Except here we don't attempt to deduce main from -dev,
+      // naturally.
+      //
+      const string& n (pn.string ());
+
+      // The best we can do in trying to detect whether this is a library is
+      // to check for the lib prefix. Libraries without the lib prefix and
+      // non-libraries with the lib prefix (both of which we do not
+      // recomment) will have to provide a manual mapping.
+      //
+      if (n.compare (0, 3, "lib") == 0 && n.size () > 3)
+      {
+        r = package_status (n, n + "-dev");
+      }
+      else
+        r = package_status (n);
+    }
+    else
+    {
+      // Even though we only pass one available package, we may still end up
+      // with multiple mappings. In this case we take the first, per the
+      // documentation.
+      //
+      r = parse_name_value (pn,
+                            ns.front (),
+                            false /* need_doc */,
+                            false /* need_dbg */);
+    }
+
+    // Map the version.
+    //
+    // To recap, a Debian package version has the following form:
+    //
+    // [<epoch>:]<upstream>[-<revision>]
+    //
+    // For details on the ordering semantics, see the Version control file
+    // field documentation in the Debian Policy Manual. While overall
+    // unsurprising, one notable exception is `~`, which sorts before anything
+    // else and is commonly used for upstream pre-releases. For example,
+    // 1.0~beta1~svn1245 sorts earlier than 1.0~beta1, which sorts earlier
+    // than 1.0.
+    //
+    // There are also various special version conventions (such as all the
+    // revision components in 1.4-5+deb10u1~bpo9u1) but they all appear to
+    // express relationships between native packages and/or their upstream and
+    // thus do not apply to our case.
+    //
+    // Ok, so how do we map our version to that? To recap, the bpkg version
+    // has the following form:
+    //
+    // [+<epoch>-]<upstream>[-<prerel>][+<revision>]
+    //
+    // Let's start with the case where neither distribution nor upstream
+    // version is specified and we need to derive everything from the bpkg
+    // version.
+    //
+    // <epoch>
+    //
+    //   On one hand, if we keep the epoch, it won't necessarily match
+    //   Debian's native package epoch. But on the other it will allow our
+    //   binary packages form different epochs to co-exist. Seeing that this
+    //   can be easily overridden with a custom distribution version, let's
+    //   keep it.
+    //
+    //   Note that while the Debian start/default epoch is 0, ours is 1 (we
+    //   use the 0 epoch for stub packages). So we will need to shift this
+    //   value range.
+    //
+    //
+    // <upstream>[-<prerel>]
+    //
+    //   Our upstream version maps naturally to Debian's. That is, our
+    //   upstream version format/semantics is a subset of Debian's.
+    //
+    //   If this is a pre-release, then we could fail (that is, don't allow
+    //   pre-releases) but then we won't be able to test on pre-release
+    //   packages, for example, to make sure the name mapping is correct.
+    //   Plus sometimes it's useful to publish pre-releases. We could ignore
+    //   it, but then such packages will be indistinguishable from each other
+    //   and the final release, which is not ideal. On the other hand, Debian
+    //   has the mechanism (`~`) which is essentially meant for this, so let's
+    //   use it. We will use <prerel> as is since its format is the same as
+    //   upstream and thus should map naturally.
+    //
+    //
+    // <revision>
+    //
+    //   Similar to epoch, our revision won't necessarily match Debian's
+    //   native package revision. But on the other hand it will allow us to
+    //   establish a correspondence between source and binary packages.  Plus,
+    //   upgrades between binary package revisions will be handled naturally.
+    //   Seeing that we allow overriding the revision with a custom
+    //   distribution version (see below), let's keep it.
+    //
+    //   Note also that both Debian and our revision start/default is 0.
+    //   However, it is Debian's convention to start revision from 1. But it
+    //   doesn't seem worth it for us to do any shifting here and so we will
+    //   use our revision as is.
+    //
+    //   Another related question is whether we should also include some
+    //   metadata that identifies the distribution and its version that this
+    //   package is for. The strongest precedent here is probably Ubuntu's
+    //   PPA. While there doesn't appear to be a consistent approach, one can
+    //   often see versions like these:
+    //
+    //   2.1.0-1~ppa0~ubuntu14.04.1,
+    //   1.4-5-1.2.1~ubuntu20.04.1~ppa1
+    //   22.12.2-0ubuntu1~ubuntu23.04~ppa1
+    //
+    //   Seeing that this is a non-sortable component (what in semver would be
+    //   called "build metadata"), using `~` is probably not the worst choice.
+    //
+    //   So we follow this lead and add the ~<name_id><version_id> component
+    //   to revision. Note that this also means we will have to make the 0
+    //   revision explicit. For example:
+    //
+    //   1.2.3-0~ubuntu20.04
+    //   1.2.3-1~debian10
+    //
+    // The next case to consider is when we have the upstream version
+    // (upstream-version manifest value). After some rumination it feels
+    // correct to use it instead of the <epoch>-<upstream> components in the
+    // above mapping (upstream version itself cannot have epoch). In other
+    // words, we will add the pre-release and revision components from the
+    // bpkg version. If this is not the desired semantics, then it can always
+    // be overrided with the distribution version.
+    //
+    // Finally, we have the distribution version. The <epoch> and <upstream>
+    // components are straightforward: they should be specified by the
+    // distribution version as required. This leaves pre-release and
+    // revision. It feels like in most cases we would want these copied over
+    // from the bpkg version automatically -- it's too tedious and error-
+    // prone to maintain them manually. However, we want the user to have the
+    // full override ability. So instead, if empty revision is specified, as
+    // in 1.2.3-, then we automatically add bpkg revision. Similarly, if empty
+    // pre-release is specified, as in 1.2.3~, then we add bpkg pre-release.
+    // To add both automatically, we would specify 1.2.3~- (other combinations
+    // are 1.2.3~b.1- and 1.2.3~-1).
+    //
+    // Note also that per the Debian version specification, if upstream
+    // contains `:` and/or `-`, then epoch and/or revision must be specified
+    // explicitly, respectively. Note that the bpkg upstream version may not
+    // contain either.
+    //
+    const shared_ptr<available_package>&        ap (aps.front ().first);
+    const lazy_shared_ptr<repository_fragment>& rf (aps.front ().second);
+
+    string& sv (r.system_version);
+
+    if (optional<string> ov = system_package_version (ap,
+                                                      rf,
+                                                      os_release.name_id,
+                                                      os_release.version_id,
+                                                      os_release.like_ids))
+    {
+      string& dv (*ov);
+      size_t n (dv.size ());
+
+      // Find the revision and pre-release positions, if any.
+      //
+      size_t rp (dv.rfind ('-'));
+      size_t pp (dv.rfind ('~', rp));
+
+      // Copy over the [<epoch>:]<upstream> part.
+      //
+      sv.assign (dv, 0, pp < rp ? pp : rp);
+
+      // Add pre-release copying over the bpkg version value if empty.
+      //
+      if (pp != string::npos)
+      {
+        if (size_t pn = (rp != string::npos ? rp : n) - (pp + 1))
+        {
+          sv.append (dv, pp, pn + 1);
+        }
+        else
+        {
+          if (pv.release)
+          {
+            assert (!pv.release->empty ()); // Cannot be earliest special.
+            sv += '~';
+            sv += *pv.release;
+          }
+        }
+      }
+
+      // Add revision copying over the bpkg version value if empty.
+      //
+      if (rp != string::npos)
+      {
+        if (size_t rn = n - (rp + 1))
+        {
+          sv.append (dv, rp, rn + 1);
+        }
+        else
+        {
+          sv += '-';
+          sv += to_string (pv.revision ? *pv.revision : 0);
+        }
+      }
+      else
+        sv += "-0"; // Default revision (for build metadata; see below).
+    }
+    else
+    {
+      if (ap->upstream_version)
+      {
+        const string& uv (*ap->upstream_version);
+
+        // Add explicit epoch if upstream contains `:`.
+        //
+        // Note that we don't need to worry about `-` since we always add
+        // revision (see below).
+        //
+        if (uv.find (':') != string::npos)
+          sv = "0:";
+
+        sv += uv;
+      }
+      else
+      {
+        // Add epoch unless maps to 0.
+        //
+        assert (pv.epoch != 0); // Cannot be a stub.
+        if (pv.epoch != 1)
+        {
+          sv = to_string (pv.epoch - 1);
+          sv += ':';
+        }
+
+        sv += pv.upstream;
+      }
+
+      // Add pre-release.
+      //
+      if (pv.release)
+      {
+        assert (!pv.release->empty ()); // Cannot be earliest special.
+        sv += '~';
+        sv += *pv.release;
+      }
+
+      // Add revision.
+      //
+      sv += '-';
+      sv += to_string (pv.revision ? *pv.revision : 0);
+    }
+
+    // Add build matadata.
+    //
+    sv += '~';
+    sv += os_release.name_id;
+    sv += os_release.version_id; // Could be empty.
+
+    return r;
+  }
+
   // Some background on creating Debian packages (for a bit more detailed
   // overview see the Debian Packaging Tutorial).
   //
@@ -1524,125 +1833,9 @@ namespace bpkg
   generate (packages&& pkgs,
             packages&& deps,
             strings&&,
-            const dir_path&,
+            const dir_path& out,
             optional<recursive_mode>)
   {
-    // Map non-system bpkg package to system package name(s) and version.
-    //
-    // This is used both to map the package being generated and its
-    // dependencies. What should we do with extras returned in package_status?
-    // We can't really generate any of them (which files would we place in
-    // them?) and we can't list them as dependencies (we don't know their
-    // system versions). So it feels like the only sensible choice is to
-    // ignore extras.
-    //
-    // In a sense, we have a parallel arrangement going on here: binary
-    // packages that we generate don't have extras (i.e., they include
-    // everything necessary in the "standard" packages from the main group)
-    // and when we punch a system dependency based on a non-system bpkg
-    // package, we assume it was generated by us and thus doesn't have any
-    // extras. Or, to put it another way, if you want the system dependency to
-    // refer to a "native" system package with extras you need to configure it
-    // as a system bpkg package.
-    //
-    // In fact, this extends to package names. For example, unless custom
-    // mapping is specified, we will generate libsqlite3 and libsqlite3-dev
-    // while native names are libsqlite3-0 and libsqlite3-dev. While this
-    // duality is not ideal, presumably we will normally only be producing
-    // our binary packages if there are no suitable native packages. And for a
-    // few exception (e.g., our package is "better" in some way, such as
-    // configured differently or fixes a critical bug), we will just have to
-    // provide appropriate manual mapping that makes sure the names match (the
-    // extras is still a potential problem though -- we will only have them as
-    // dependencies if we build against a native system package).
-    //
-    auto map_package = [this] (const shared_ptr<selected_package>& sp,
-                               const available_packages& aps) -> package_status
-    {
-      // We should only have one available package corresponding to the
-      // selected package.
-      //
-      assert (sp->substate != package_substate::system && aps.size () == 1);
-
-      strings ns (system_package_names (aps,
-                                        os_release.name_id,
-                                        os_release.version_id,
-                                        os_release.like_ids));
-      package_status r;
-      if (ns.empty ())
-      {
-        // Automatically translate our package name similar to the consumption
-        // case above. Except here we don't attempt to deduce main from -dev,
-        // naturally.
-        //
-        const string& pn (sp->name.string ());
-
-        // The best we can do in trying to detect whether this is a library is
-        // to check for the lib prefix. Libraries without the lib prefix and
-        // non-libraries with the lib prefix (both of which we do not
-        // recomment) will have to provide a manual mapping.
-        //
-        if (pn.compare (0, 3, "lib") == 0 && pn.size () > 3)
-        {
-          r = package_status (pn, pn + "-dev");
-        }
-        else
-          r = package_status (pn);
-      }
-      else
-      {
-        // Even though we only pass one available package, we may still end up
-        // with multiple mappings. In this case we take the first, per the
-        // documentation.
-        //
-        r = parse_name_value (sp->name,
-                              ns.front (),
-                              false /* need_doc */,
-                              false /* need_dbg */);
-      }
-
-      // Map the version.
-      //
-      // @@ Should we fail if either is a pre-release (we cannot represent
-      //    it in Debian version). But then won't be able to test on
-      //    pre-release packages. maybe just strip/ignore? Can we translate
-      //    it to something appropriate in Debian version (something in the
-      //    revision)?
-      //
-      // @@ Should we include our revision as Debian revistion even if the
-      //    version is mapped? Feels natural.
-      //
-      const shared_ptr<available_package>&        ap (aps.front ().first);
-      const lazy_shared_ptr<repository_fragment>& rf (aps.front ().second);
-
-
-      if (optional<string> v = system_package_version (ap,
-                                                       rf,
-                                                       os_release.name_id,
-                                                       os_release.version_id,
-                                                       os_release.like_ids))
-      {
-        r.system_version = move (*v);
-      }
-      else if (ap->upstream_version)
-      {
-        r.system_version = *ap->upstream_version;
-      }
-      else
-      {
-        // Strip or keep epoch? On one hand it won't necessarily match Debian
-        // but on the other it will allow packages form different epochs to
-        // co-exist.
-        //
-        // @@ Also need to make sure have explicit epoch/revision if upstream
-        //    contains `:` or `-`. Can our upstream contain them?
-        //
-        r.system_version = sp->version.upstream;
-      }
-
-      return r;
-    };
-
     // As a first step, figure out the system names and version of the package
     // we are generating and all the dependencies, diagnosing anything fishy.
     //
@@ -1650,8 +1843,9 @@ namespace bpkg
     // the status cache.
     //
     const shared_ptr<selected_package>& sp (pkgs.front ().first);
-    const available_packages& aps (pkgs.front ().second);
-    package_status s (map_package (sp, aps));
+    const package_name& pn (sp->name);
+    const version& pv (sp->version);
+    package_status s (map_package (pn, pv, pkgs.front ().second));
 
     vector<package_status> sdeps;
     sdeps.reserve (deps.size ());
@@ -1665,18 +1859,118 @@ namespace bpkg
       {
         optional<package_status> os (status (sp->name, aps));
 
-        if (!os)
-          fail << "bad boy";
+        if (!os || os->status != package_status::installed)
+          fail << os_release.name_id << " package for " << sp->name
+               << " system package is no longer installed";
 
-        // @@ We should confirm configured version matches mapped back.
-        //    Can be `*`!
+        // For good measure verify the mapped back version still matches
+        // configured. Note that besides the normal case (queried by the
+        // system package manager), it could have also been specified by the
+        // user as an actual version or a wildcard. Ignoring this check for a
+        // wildcard feels consistent with the overall semantics.
+        //
+        if (sp->version != wildcard_version && sp->version != os->version)
+        {
+          fail << "current " << os_release.name_id << " package version for "
+               << sp->name << " system package does not match configured" <<
+            info << "configured version: " << sp->version <<
+            info << "current version: " << os->version << " ("
+               << os->system_version << ')';
+        }
 
         s = move (*os);
       }
       else
-        s = map_package (sp, aps);
+        s = map_package (sp->name, sp->version, aps);
 
       sdeps.push_back (move (s));
     }
+
+    {
+      auto print_status = [] (diag_record& dr, const package_status& s)
+      {
+        dr << s.main
+           << (s.dev.empty () ? "" : " ") << s.dev
+           << (s.doc.empty () ? "" : " ") << s.doc
+           << (s.dbg.empty () ? "" : " ") << s.dbg
+           << (s.common.empty () ? "" : " ") << s.common
+           << ' ' << s.system_version;
+      };
+
+      diag_record dr (text);
+      print_status (dr, s);
+
+      for (const package_status& ds: sdeps)
+      {
+        dr << "\n  ";
+        print_status (dr, ds);
+      }
+    }
+
+    // Start assembling the package "source" directory.
+    //
+    // It's hard to predict all the files that will be generated (and
+    // potentially read), so we will just require a clean output directory.
+    //
+    // Also, by default, we are going to keep all the intermediate files on
+    // failure for troubleshooting.
+    //
+    if (exists (out))
+    {
+      if (!empty (out))
+      {
+        if (!ops_->wipe_out ())
+          fail << "directory " << out << " is not empty" <<
+            info << "use --wipe to clean it up but be careful";
+
+        rm_r (out, false);
+      }
+    }
+
+    // Normally the source directory is called <name>-<upstream-version>
+    // (e.g., as unpacked from the source archive).
+    //
+    dir_path src (out / dir_path (pn.string () + '-' + pv.string ()));
+    dir_path deb (src / dir_path ("debian"));
+    mk_p (deb);
+
+    // The rules makefile. Note that it must be executable.
+    //
+    path rules (deb / "rules");
+    try
+    {
+      // See fdopen() for details (umask, etc).
+      //
+      permissions ps (permissions::ru | permissions::wu | permissions::xu |
+                      permissions::rg | permissions::wg | permissions::xg |
+                      permissions::ro | permissions::wo | permissions::xo);
+      ofdstream os (fdopen (rules,
+                            fdopen_mode::out | fdopen_mode::create,
+                            ps));
+
+      os << "#!/usr/bin/make -f\n"
+         << "# -*- makefile -*-\n"
+         << '\n';
+
+      // See debhelper(7) for details on these.
+      //
+      if (verb == 0)
+        os << "export DH_QUIET=1\n";
+      else if (verb >= 2)
+        os << "export DH_VERBOSE=1\n";
+
+      os.close ();
+    }
+    catch (const io_error& e)
+    {
+      fail << "unable to write to " << rules << ": " << e;
+    }
+
+    // Cleanup intermediate files unless requested not to.
+    //
+    if (!ops_->keep_out ())
+    {
+      rm_r (src);
+    }
   }
 }
diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx
index 1e53e38..8fccb55 100644
--- a/bpkg/system-package-manager-debian.hxx
+++ b/bpkg/system-package-manager-debian.hxx
@@ -164,11 +164,13 @@ namespace bpkg
     system_package_manager_debian (bpkg::os_release&& osr,
                                    const target_triplet& h,
                                    string a,
-                                   optional<bool> progress)
+                                   optional<bool> progress,
+                                   const pkg_bindist_options& ops)
         : system_package_manager (move (osr),
                                   h,
                                   a.empty () ? arch_from_target (h) : move (a),
-                                  progress) {}
+                                  progress),
+          ops_ (&ops) {}
 
     // Implementation details exposed for testing (see definitions for
     // documentation).
@@ -201,6 +203,11 @@ namespace bpkg
     static string
     arch_from_target (const target_triplet&);
 
+    package_status
+    map_package (const package_name&,
+                 const version&,
+                 const available_packages&);
+
     // If simulate is not NULL, then instead of executing the actual apt-cache
     // and apt-get commands simulate their execution: (1) for apt-cache by
     // printing their command lines and reading the results from files
@@ -242,6 +249,8 @@ namespace bpkg
     bool installed_ = false; // True if already installed.
 
     std::map<package_name, optional<system_package_status_debian>> status_cache_;
+
+    const pkg_bindist_options* ops_ = nullptr; // Only for production.
   };
 }
 
diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx
index 899d390..a04c213 100644
--- a/bpkg/system-package-manager.cxx
+++ b/bpkg/system-package-manager.cxx
@@ -154,7 +154,7 @@ namespace bpkg
             os.like_ids.push_back ("debian");
 
           r.reset (new system_package_manager_debian (
-                     move (os), host, arch, progress));
+                     move (os), host, arch, progress, o));
         }
         else if (is_or_like (os, "fedora") ||
                  is_or_like (os, "rhel")   ||
diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx
index 305a00e..91f9a49 100644
--- a/bpkg/system-package-manager.hxx
+++ b/bpkg/system-package-manager.hxx
@@ -346,6 +346,9 @@ namespace bpkg
                                            bool yes,
                                            const string& sudo);
 
+  // Note that the reference to options is expected to outlive the returned
+  // instance.
+  //
   class pkg_bindist_options;
 
   unique_ptr<system_package_manager>