aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bpkg/pkg-bindist.cli13
-rw-r--r--bpkg/system-package-manager-debian.cxx542
-rw-r--r--bpkg/system-package-manager-debian.hxx13
-rw-r--r--bpkg/system-package-manager.cxx2
-rw-r--r--bpkg/system-package-manager.hxx3
5 files changed, 446 insertions, 127 deletions
diff --git a/bpkg/pkg-bindist.cli b/bpkg/pkg-bindist.cli
index 5686ed6..ad743d9 100644
--- a/bpkg/pkg-bindist.cli
+++ b/bpkg/pkg-bindist.cli
@@ -98,6 +98,19 @@ namespace bpkg
documentation in the build system manual for details. This option
only makes sense together with \cb{--recursive}."
}
+
+ bool --wipe-out
+ {
+ "Wipe the output directory (\ci{out-dir}) clean before using it to
+ generate the binary package."
+ }
+
+ bool --keep-out
+ {
+ "Keep intermediate files in the output directory (\ci{out-dir}) that
+ were used to generate the binary package. This is primarily useful
+ for troubleshooting."
+ }
};
"
diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx
index 47df65d..37844f1 100644
--- a/bpkg/system-package-manager-debian.cxx
+++ b/bpkg/system-package-manager-debian.cxx
@@ -3,8 +3,12 @@
#include <bpkg/system-package-manager-debian.hxx>
+#include <libbutl/filesystem.hxx> // permissions
+
#include <bpkg/diagnostics.hxx>
+#include <bpkg/pkg-bindist-options.hxx>
+
using namespace butl;
namespace bpkg
@@ -1455,6 +1459,311 @@ namespace bpkg
}
}
+ // Map non-system bpkg package to system package name(s) and version.
+ //
+ // This is used both to map the package being generated and its
+ // dependencies. What should we do with extras returned in package_status?
+ // We can't really generate any of them (which files would we place in
+ // them?) nor can we list them as dependencies (we don't know their system
+ // versions). So it feels like the only sensible choice is to ignore extras.
+ //
+ // In a sense, we have a parallel arrangement going on here: binary packages
+ // that we generate don't have extras (i.e., they include everything
+ // necessary in the "standard" packages from the main group) and when we
+ // punch a system dependency based on a non-system bpkg package, we assume
+ // it was generated by us and thus doesn't have any extras. Or, to put it
+ // another way, if you want the system dependency to refer to a "native"
+ // system package with extras you need to configure it as a system bpkg
+ // package.
+ //
+ // In fact, this extends to package names. For example, unless custom
+ // mapping is specified, we will generate libsqlite3 and libsqlite3-dev
+ // while native names are libsqlite3-0 and libsqlite3-dev. While this
+ // duality is not ideal, presumably we will normally only be producing our
+ // binary packages if there are no suitable native packages. And for a few
+ // exception (e.g., our package is "better" in some way, such as configured
+ // differently or fixes a critical bug), we will just have to provide
+ // appropriate manual mapping that makes sure the names match (the extras is
+ // still a potential problem though -- we will only have them as
+ // dependencies if we build against a native system package).
+ //
+ // @@ TODO: test, especially distribution version logic.
+ //
+ package_status system_package_manager_debian::
+ map_package (const package_name& pn,
+ const version& pv,
+ const available_packages& aps)
+ {
+ // We should only have one available package corresponding to this package
+ // name/version.
+ //
+ assert (aps.size () == 1);
+
+ strings ns (system_package_names (aps,
+ os_release.name_id,
+ os_release.version_id,
+ os_release.like_ids));
+ package_status r;
+ if (ns.empty ())
+ {
+ // Automatically translate our package name similar to the consumption
+ // case above. Except here we don't attempt to deduce main from -dev,
+ // naturally.
+ //
+ const string& n (pn.string ());
+
+ // The best we can do in trying to detect whether this is a library is
+ // to check for the lib prefix. Libraries without the lib prefix and
+ // non-libraries with the lib prefix (both of which we do not
+ // recomment) will have to provide a manual mapping.
+ //
+ if (n.compare (0, 3, "lib") == 0 && n.size () > 3)
+ {
+ r = package_status (n, n + "-dev");
+ }
+ else
+ r = package_status (n);
+ }
+ else
+ {
+ // Even though we only pass one available package, we may still end up
+ // with multiple mappings. In this case we take the first, per the
+ // documentation.
+ //
+ r = parse_name_value (pn,
+ ns.front (),
+ false /* need_doc */,
+ false /* need_dbg */);
+ }
+
+ // Map the version.
+ //
+ // To recap, a Debian package version has the following form:
+ //
+ // [<epoch>:]<upstream>[-<revision>]
+ //
+ // For details on the ordering semantics, see the Version control file
+ // field documentation in the Debian Policy Manual. While overall
+ // unsurprising, one notable exception is `~`, which sorts before anything
+ // else and is commonly used for upstream pre-releases. For example,
+ // 1.0~beta1~svn1245 sorts earlier than 1.0~beta1, which sorts earlier
+ // than 1.0.
+ //
+ // There are also various special version conventions (such as all the
+ // revision components in 1.4-5+deb10u1~bpo9u1) but they all appear to
+ // express relationships between native packages and/or their upstream and
+ // thus do not apply to our case.
+ //
+ // Ok, so how do we map our version to that? To recap, the bpkg version
+ // has the following form:
+ //
+ // [+<epoch>-]<upstream>[-<prerel>][+<revision>]
+ //
+ // Let's start with the case where neither distribution nor upstream
+ // version is specified and we need to derive everything from the bpkg
+ // version.
+ //
+ // <epoch>
+ //
+ // On one hand, if we keep the epoch, it won't necessarily match
+ // Debian's native package epoch. But on the other it will allow our
+ // binary packages form different epochs to co-exist. Seeing that this
+ // can be easily overridden with a custom distribution version, let's
+ // keep it.
+ //
+ // Note that while the Debian start/default epoch is 0, ours is 1 (we
+ // use the 0 epoch for stub packages). So we will need to shift this
+ // value range.
+ //
+ //
+ // <upstream>[-<prerel>]
+ //
+ // Our upstream version maps naturally to Debian's. That is, our
+ // upstream version format/semantics is a subset of Debian's.
+ //
+ // If this is a pre-release, then we could fail (that is, don't allow
+ // pre-releases) but then we won't be able to test on pre-release
+ // packages, for example, to make sure the name mapping is correct.
+ // Plus sometimes it's useful to publish pre-releases. We could ignore
+ // it, but then such packages will be indistinguishable from each other
+ // and the final release, which is not ideal. On the other hand, Debian
+ // has the mechanism (`~`) which is essentially meant for this, so let's
+ // use it. We will use <prerel> as is since its format is the same as
+ // upstream and thus should map naturally.
+ //
+ //
+ // <revision>
+ //
+ // Similar to epoch, our revision won't necessarily match Debian's
+ // native package revision. But on the other hand it will allow us to
+ // establish a correspondence between source and binary packages. Plus,
+ // upgrades between binary package revisions will be handled naturally.
+ // Seeing that we allow overriding the revision with a custom
+ // distribution version (see below), let's keep it.
+ //
+ // Note also that both Debian and our revision start/default is 0.
+ // However, it is Debian's convention to start revision from 1. But it
+ // doesn't seem worth it for us to do any shifting here and so we will
+ // use our revision as is.
+ //
+ // Another related question is whether we should also include some
+ // metadata that identifies the distribution and its version that this
+ // package is for. The strongest precedent here is probably Ubuntu's
+ // PPA. While there doesn't appear to be a consistent approach, one can
+ // often see versions like these:
+ //
+ // 2.1.0-1~ppa0~ubuntu14.04.1,
+ // 1.4-5-1.2.1~ubuntu20.04.1~ppa1
+ // 22.12.2-0ubuntu1~ubuntu23.04~ppa1
+ //
+ // Seeing that this is a non-sortable component (what in semver would be
+ // called "build metadata"), using `~` is probably not the worst choice.
+ //
+ // So we follow this lead and add the ~<name_id><version_id> component
+ // to revision. Note that this also means we will have to make the 0
+ // revision explicit. For example:
+ //
+ // 1.2.3-0~ubuntu20.04
+ // 1.2.3-1~debian10
+ //
+ // The next case to consider is when we have the upstream version
+ // (upstream-version manifest value). After some rumination it feels
+ // correct to use it instead of the <epoch>-<upstream> components in the
+ // above mapping (upstream version itself cannot have epoch). In other
+ // words, we will add the pre-release and revision components from the
+ // bpkg version. If this is not the desired semantics, then it can always
+ // be overrided with the distribution version.
+ //
+ // Finally, we have the distribution version. The <epoch> and <upstream>
+ // components are straightforward: they should be specified by the
+ // distribution version as required. This leaves pre-release and
+ // revision. It feels like in most cases we would want these copied over
+ // from the bpkg version automatically -- it's too tedious and error-
+ // prone to maintain them manually. However, we want the user to have the
+ // full override ability. So instead, if empty revision is specified, as
+ // in 1.2.3-, then we automatically add bpkg revision. Similarly, if empty
+ // pre-release is specified, as in 1.2.3~, then we add bpkg pre-release.
+ // To add both automatically, we would specify 1.2.3~- (other combinations
+ // are 1.2.3~b.1- and 1.2.3~-1).
+ //
+ // Note also that per the Debian version specification, if upstream
+ // contains `:` and/or `-`, then epoch and/or revision must be specified
+ // explicitly, respectively. Note that the bpkg upstream version may not
+ // contain either.
+ //
+ const shared_ptr<available_package>& ap (aps.front ().first);
+ const lazy_shared_ptr<repository_fragment>& rf (aps.front ().second);
+
+ string& sv (r.system_version);
+
+ if (optional<string> ov = system_package_version (ap,
+ rf,
+ os_release.name_id,
+ os_release.version_id,
+ os_release.like_ids))
+ {
+ string& dv (*ov);
+ size_t n (dv.size ());
+
+ // Find the revision and pre-release positions, if any.
+ //
+ size_t rp (dv.rfind ('-'));
+ size_t pp (dv.rfind ('~', rp));
+
+ // Copy over the [<epoch>:]<upstream> part.
+ //
+ sv.assign (dv, 0, pp < rp ? pp : rp);
+
+ // Add pre-release copying over the bpkg version value if empty.
+ //
+ if (pp != string::npos)
+ {
+ if (size_t pn = (rp != string::npos ? rp : n) - (pp + 1))
+ {
+ sv.append (dv, pp, pn + 1);
+ }
+ else
+ {
+ if (pv.release)
+ {
+ assert (!pv.release->empty ()); // Cannot be earliest special.
+ sv += '~';
+ sv += *pv.release;
+ }
+ }
+ }
+
+ // Add revision copying over the bpkg version value if empty.
+ //
+ if (rp != string::npos)
+ {
+ if (size_t rn = n - (rp + 1))
+ {
+ sv.append (dv, rp, rn + 1);
+ }
+ else
+ {
+ sv += '-';
+ sv += to_string (pv.revision ? *pv.revision : 0);
+ }
+ }
+ else
+ sv += "-0"; // Default revision (for build metadata; see below).
+ }
+ else
+ {
+ if (ap->upstream_version)
+ {
+ const string& uv (*ap->upstream_version);
+
+ // Add explicit epoch if upstream contains `:`.
+ //
+ // Note that we don't need to worry about `-` since we always add
+ // revision (see below).
+ //
+ if (uv.find (':') != string::npos)
+ sv = "0:";
+
+ sv += uv;
+ }
+ else
+ {
+ // Add epoch unless maps to 0.
+ //
+ assert (pv.epoch != 0); // Cannot be a stub.
+ if (pv.epoch != 1)
+ {
+ sv = to_string (pv.epoch - 1);
+ sv += ':';
+ }
+
+ sv += pv.upstream;
+ }
+
+ // Add pre-release.
+ //
+ if (pv.release)
+ {
+ assert (!pv.release->empty ()); // Cannot be earliest special.
+ sv += '~';
+ sv += *pv.release;
+ }
+
+ // Add revision.
+ //
+ sv += '-';
+ sv += to_string (pv.revision ? *pv.revision : 0);
+ }
+
+ // Add build matadata.
+ //
+ sv += '~';
+ sv += os_release.name_id;
+ sv += os_release.version_id; // Could be empty.
+
+ return r;
+ }
+
// Some background on creating Debian packages (for a bit more detailed
// overview see the Debian Packaging Tutorial).
//
@@ -1524,125 +1833,9 @@ namespace bpkg
generate (packages&& pkgs,
packages&& deps,
strings&&,
- const dir_path&,
+ const dir_path& out,
optional<recursive_mode>)
{
- // Map non-system bpkg package to system package name(s) and version.
- //
- // This is used both to map the package being generated and its
- // dependencies. What should we do with extras returned in package_status?
- // We can't really generate any of them (which files would we place in
- // them?) and we can't list them as dependencies (we don't know their
- // system versions). So it feels like the only sensible choice is to
- // ignore extras.
- //
- // In a sense, we have a parallel arrangement going on here: binary
- // packages that we generate don't have extras (i.e., they include
- // everything necessary in the "standard" packages from the main group)
- // and when we punch a system dependency based on a non-system bpkg
- // package, we assume it was generated by us and thus doesn't have any
- // extras. Or, to put it another way, if you want the system dependency to
- // refer to a "native" system package with extras you need to configure it
- // as a system bpkg package.
- //
- // In fact, this extends to package names. For example, unless custom
- // mapping is specified, we will generate libsqlite3 and libsqlite3-dev
- // while native names are libsqlite3-0 and libsqlite3-dev. While this
- // duality is not ideal, presumably we will normally only be producing
- // our binary packages if there are no suitable native packages. And for a
- // few exception (e.g., our package is "better" in some way, such as
- // configured differently or fixes a critical bug), we will just have to
- // provide appropriate manual mapping that makes sure the names match (the
- // extras is still a potential problem though -- we will only have them as
- // dependencies if we build against a native system package).
- //
- auto map_package = [this] (const shared_ptr<selected_package>& sp,
- const available_packages& aps) -> package_status
- {
- // We should only have one available package corresponding to the
- // selected package.
- //
- assert (sp->substate != package_substate::system && aps.size () == 1);
-
- strings ns (system_package_names (aps,
- os_release.name_id,
- os_release.version_id,
- os_release.like_ids));
- package_status r;
- if (ns.empty ())
- {
- // Automatically translate our package name similar to the consumption
- // case above. Except here we don't attempt to deduce main from -dev,
- // naturally.
- //
- const string& pn (sp->name.string ());
-
- // The best we can do in trying to detect whether this is a library is
- // to check for the lib prefix. Libraries without the lib prefix and
- // non-libraries with the lib prefix (both of which we do not
- // recomment) will have to provide a manual mapping.
- //
- if (pn.compare (0, 3, "lib") == 0 && pn.size () > 3)
- {
- r = package_status (pn, pn + "-dev");
- }
- else
- r = package_status (pn);
- }
- else
- {
- // Even though we only pass one available package, we may still end up
- // with multiple mappings. In this case we take the first, per the
- // documentation.
- //
- r = parse_name_value (sp->name,
- ns.front (),
- false /* need_doc */,
- false /* need_dbg */);
- }
-
- // Map the version.
- //
- // @@ Should we fail if either is a pre-release (we cannot represent
- // it in Debian version). But then won't be able to test on
- // pre-release packages. maybe just strip/ignore? Can we translate
- // it to something appropriate in Debian version (something in the
- // revision)?
- //
- // @@ Should we include our revision as Debian revistion even if the
- // version is mapped? Feels natural.
- //
- const shared_ptr<available_package>& ap (aps.front ().first);
- const lazy_shared_ptr<repository_fragment>& rf (aps.front ().second);
-
-
- if (optional<string> v = system_package_version (ap,
- rf,
- os_release.name_id,
- os_release.version_id,
- os_release.like_ids))
- {
- r.system_version = move (*v);
- }
- else if (ap->upstream_version)
- {
- r.system_version = *ap->upstream_version;
- }
- else
- {
- // Strip or keep epoch? On one hand it won't necessarily match Debian
- // but on the other it will allow packages form different epochs to
- // co-exist.
- //
- // @@ Also need to make sure have explicit epoch/revision if upstream
- // contains `:` or `-`. Can our upstream contain them?
- //
- r.system_version = sp->version.upstream;
- }
-
- return r;
- };
-
// As a first step, figure out the system names and version of the package
// we are generating and all the dependencies, diagnosing anything fishy.
//
@@ -1650,8 +1843,9 @@ namespace bpkg
// the status cache.
//
const shared_ptr<selected_package>& sp (pkgs.front ().first);
- const available_packages& aps (pkgs.front ().second);
- package_status s (map_package (sp, aps));
+ const package_name& pn (sp->name);
+ const version& pv (sp->version);
+ package_status s (map_package (pn, pv, pkgs.front ().second));
vector<package_status> sdeps;
sdeps.reserve (deps.size ());
@@ -1665,18 +1859,118 @@ namespace bpkg
{
optional<package_status> os (status (sp->name, aps));
- if (!os)
- fail << "bad boy";
+ if (!os || os->status != package_status::installed)
+ fail << os_release.name_id << " package for " << sp->name
+ << " system package is no longer installed";
- // @@ We should confirm configured version matches mapped back.
- // Can be `*`!
+ // For good measure verify the mapped back version still matches
+ // configured. Note that besides the normal case (queried by the
+ // system package manager), it could have also been specified by the
+ // user as an actual version or a wildcard. Ignoring this check for a
+ // wildcard feels consistent with the overall semantics.
+ //
+ if (sp->version != wildcard_version && sp->version != os->version)
+ {
+ fail << "current " << os_release.name_id << " package version for "
+ << sp->name << " system package does not match configured" <<
+ info << "configured version: " << sp->version <<
+ info << "current version: " << os->version << " ("
+ << os->system_version << ')';
+ }
s = move (*os);
}
else
- s = map_package (sp, aps);
+ s = map_package (sp->name, sp->version, aps);
sdeps.push_back (move (s));
}
+
+ {
+ auto print_status = [] (diag_record& dr, const package_status& s)
+ {
+ dr << s.main
+ << (s.dev.empty () ? "" : " ") << s.dev
+ << (s.doc.empty () ? "" : " ") << s.doc
+ << (s.dbg.empty () ? "" : " ") << s.dbg
+ << (s.common.empty () ? "" : " ") << s.common
+ << ' ' << s.system_version;
+ };
+
+ diag_record dr (text);
+ print_status (dr, s);
+
+ for (const package_status& ds: sdeps)
+ {
+ dr << "\n ";
+ print_status (dr, ds);
+ }
+ }
+
+ // Start assembling the package "source" directory.
+ //
+ // It's hard to predict all the files that will be generated (and
+ // potentially read), so we will just require a clean output directory.
+ //
+ // Also, by default, we are going to keep all the intermediate files on
+ // failure for troubleshooting.
+ //
+ if (exists (out))
+ {
+ if (!empty (out))
+ {
+ if (!ops_->wipe_out ())
+ fail << "directory " << out << " is not empty" <<
+ info << "use --wipe to clean it up but be careful";
+
+ rm_r (out, false);
+ }
+ }
+
+ // Normally the source directory is called <name>-<upstream-version>
+ // (e.g., as unpacked from the source archive).
+ //
+ dir_path src (out / dir_path (pn.string () + '-' + pv.string ()));
+ dir_path deb (src / dir_path ("debian"));
+ mk_p (deb);
+
+ // The rules makefile. Note that it must be executable.
+ //
+ path rules (deb / "rules");
+ try
+ {
+ // See fdopen() for details (umask, etc).
+ //
+ permissions ps (permissions::ru | permissions::wu | permissions::xu |
+ permissions::rg | permissions::wg | permissions::xg |
+ permissions::ro | permissions::wo | permissions::xo);
+ ofdstream os (fdopen (rules,
+ fdopen_mode::out | fdopen_mode::create,
+ ps));
+
+ os << "#!/usr/bin/make -f\n"
+ << "# -*- makefile -*-\n"
+ << '\n';
+
+ // See debhelper(7) for details on these.
+ //
+ if (verb == 0)
+ os << "export DH_QUIET=1\n";
+ else if (verb >= 2)
+ os << "export DH_VERBOSE=1\n";
+
+ os.close ();
+ }
+ catch (const io_error& e)
+ {
+ fail << "unable to write to " << rules << ": " << e;
+ }
+
+ // Cleanup intermediate files unless requested not to.
+ //
+ if (!ops_->keep_out ())
+ {
+ rm_r (src);
+ }
}
}
diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx
index 1e53e38..8fccb55 100644
--- a/bpkg/system-package-manager-debian.hxx
+++ b/bpkg/system-package-manager-debian.hxx
@@ -164,11 +164,13 @@ namespace bpkg
system_package_manager_debian (bpkg::os_release&& osr,
const target_triplet& h,
string a,
- optional<bool> progress)
+ optional<bool> progress,
+ const pkg_bindist_options& ops)
: system_package_manager (move (osr),
h,
a.empty () ? arch_from_target (h) : move (a),
- progress) {}
+ progress),
+ ops_ (&ops) {}
// Implementation details exposed for testing (see definitions for
// documentation).
@@ -201,6 +203,11 @@ namespace bpkg
static string
arch_from_target (const target_triplet&);
+ package_status
+ map_package (const package_name&,
+ const version&,
+ const available_packages&);
+
// If simulate is not NULL, then instead of executing the actual apt-cache
// and apt-get commands simulate their execution: (1) for apt-cache by
// printing their command lines and reading the results from files
@@ -242,6 +249,8 @@ namespace bpkg
bool installed_ = false; // True if already installed.
std::map<package_name, optional<system_package_status_debian>> status_cache_;
+
+ const pkg_bindist_options* ops_ = nullptr; // Only for production.
};
}
diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx
index 899d390..a04c213 100644
--- a/bpkg/system-package-manager.cxx
+++ b/bpkg/system-package-manager.cxx
@@ -154,7 +154,7 @@ namespace bpkg
os.like_ids.push_back ("debian");
r.reset (new system_package_manager_debian (
- move (os), host, arch, progress));
+ move (os), host, arch, progress, o));
}
else if (is_or_like (os, "fedora") ||
is_or_like (os, "rhel") ||
diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx
index 305a00e..91f9a49 100644
--- a/bpkg/system-package-manager.hxx
+++ b/bpkg/system-package-manager.hxx
@@ -346,6 +346,9 @@ namespace bpkg
bool yes,
const string& sudo);
+ // Note that the reference to options is expected to outlive the returned
+ // instance.
+ //
class pkg_bindist_options;
unique_ptr<system_package_manager>