diff options
| author | Boris Kolpackov <boris@codesynthesis.com> | 2023-02-09 15:46:32 +0200 |
|---|---|---|
| committer | Boris Kolpackov <boris@codesynthesis.com> | 2023-03-09 12:31:18 +0200 |
| commit | 854c668b5e63e26a9d7a6e55226a0940638e0453 (patch) | |
| tree | 86b03f5db9d65c925941fc872aaa76d24248e0d3 | |
| parent | eaebfcff492cf7f707b44a3d28620e786116faf1 (diff) | |
Add pkg-bindist command (generate binary distribution package)
This commit includes an implementation for Debian and alike.
26 files changed, 3927 insertions, 278 deletions
diff --git a/bpkg/bpkg.cli b/bpkg/bpkg.cli index 17ac927..6edea97 100644 --- a/bpkg/bpkg.cli +++ b/bpkg/bpkg.cli @@ -257,6 +257,11 @@ namespace bpkg "\l{bpkg-pkg-clean(1)} \- clean package" } + bool pkg-bindist|bindist + { + "\l{bpkg-pkg-bindist(1)} \- generate binary distribution package" + } + bool pkg-verify { "\l{bpkg-pkg-verify(1)} \- verify package archive" diff --git a/bpkg/bpkg.cxx b/bpkg/bpkg.cxx index 28ba75f..21cbefc 100644 --- a/bpkg/bpkg.cxx +++ b/bpkg/bpkg.cxx @@ -49,6 +49,7 @@ #include <bpkg/cfg-link.hxx> #include <bpkg/cfg-unlink.hxx> +#include <bpkg/pkg-bindist.hxx> #include <bpkg/pkg-build.hxx> #include <bpkg/pkg-checkout.hxx> #include <bpkg/pkg-clean.hxx> @@ -767,6 +768,7 @@ try // These commands need the '--' separator to be kept in args. // + PKG_COMMAND (bindist, true, true); PKG_COMMAND (build, true, false); PKG_COMMAND (clean, true, true); PKG_COMMAND (configure, true, true); diff --git a/bpkg/buildfile b/bpkg/buildfile index ca78218..3ba9ea6 100644 --- a/bpkg/buildfile +++ b/bpkg/buildfile @@ -34,6 +34,7 @@ cfg-unlink-options \ common-options \ configuration-options \ help-options \ +pkg-bindist-options \ pkg-build-options \ pkg-checkout-options \ pkg-clean-options \ @@ -148,6 +149,7 @@ if $cli.configured # pkg-* command. # + cli.cxx{pkg-bindist-options}: cli{pkg-bindist} cli.cxx{pkg-build-options}: cli{pkg-build} cli.cxx{pkg-checkout-options}: cli{pkg-checkout} cli.cxx{pkg-clean-options}: cli{pkg-clean} @@ -213,6 +215,9 @@ if $cli.configured cli.cxx{pkg-build-options}: cli.options += --class-doc \ bpkg::pkg_build_pkg_options=exclude-base --generate-modifier + cli.cxx{pkg-bindist-options}: cli.options += --class-doc \ +bpkg::pkg_bindist_debian_options=exclude-base + # Avoid generating CLI runtime and empty inline file for help topics. # cli.cxx{repository-signing repository-types argument-grouping \ diff --git a/bpkg/package.hxx b/bpkg/package.hxx index e5e70ad..1c70676 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -790,6 +790,18 @@ namespace bpkg bool stub () const {return version.compare (wildcard_version, true) == 0;} + string + effective_type () const + { + return package_manifest::effective_type (type, id.name); + } + + small_vector<language, 1> + effective_languages () const + { + return package_manifest::effective_languages (languages, id.name); + } + // Return package system version if one has been discovered. Note that // we do not implicitly assume a wildcard version. // @@ -1319,8 +1331,7 @@ namespace bpkg return src_root->absolute () ? *src_root : configuration / *src_root; } - // Return the output directory using the configuration directory. Note - // that the output directory is always relative. + // Return the output directory using the configuration directory. // dir_path effective_out_root (const dir_path& configuration) const @@ -1328,6 +1339,9 @@ namespace bpkg // Cast for compiling with ODB (see above). // assert (static_cast<bool> (out_root)); + + // Note that out_root is always relative. + // return configuration / *out_root; } diff --git a/bpkg/pkg-bindist.cli b/bpkg/pkg-bindist.cli new file mode 100644 index 0000000..dbbf9c3 --- /dev/null +++ b/bpkg/pkg-bindist.cli @@ -0,0 +1,290 @@ +// file : bpkg/pkg-bindist.cli +// license : MIT; see accompanying LICENSE file + +include <bpkg/configuration.cli>; + +"\section=1" +"\name=bpkg-pkg-bindist" +"\summary=generate binary distribution package" + +namespace bpkg +{ + { + "<options> <dir> <vars> <pkg>", + + "\h|SYNOPSIS| + + \c{\b{bpkg pkg-bindist}|\b{bindist} [\b{--output-root}|\b{-o} <dir>] [<options>] [<vars>] <pkg>...} + + \h|DESCRIPTION| + + The \cb{pkg-bindist} command generates a binary distribution package for + the specified package. If additional packages are specified, then they + are bundled in the same distribution package. All the specified packages + must have been previously configured with \l{bpkg-pkg-build(1)} or + \l{bpkg-pkg-configure(1)}. For some system package managers a directory + for intermediate files and subdirectories as well as the resulting binary + package may have to be specified explicitly with the + \c{\b{--output-root}|\b{-o}} option. + + Underneath, this command roughly performs the following steps: First it + installs the specified packages similar to the \l{bpkg-pkg-install(1)} + command except that it may override the installation locations (via the + \cb{config.install.*} variables) to match the distribution's layout. Then + it generates any necessary distribution package metadata files based on + the information from the package \cb{manifest} files. Finally, it invokes + the distribution-specified command to produce the binary package. Unless + overrident with the \cb{--architecture} and \cb{--distribution} options, + the binary package is generated for the host architecture using the + host's standard system package manager. Additional command line variables + (<vars>, normally \cb{config.*}) can be passed to the build system during + the installation step. See distribution-specific description sections + below for details and invocation examples. + + The specified packages may have dependencies and the default behavior is + to not bundle them but rather to specify them as dependencies in the + corresponding distribution package metadata, if applicable. This default + behavior can be overridden with the \cb{--recursive} option (see the + option description for the available modes). Note, however, that + dependencies that are satisfied by system packages are always specified + as dependencies in the distribution package metadata. + " + } + + // Place distribution-specific options into separate classes in case one day + // we want to only pass their own options to each implementation. + // + class pkg_bindist_debian_options + { + "\h|DEBIAN DESCRIPTION| + + The Debian binary packages are generated by producing the standard + \cb{debian/control}, \cb{debian/rules}, and other package metadata files + and then invoking \cb{dpkg-buildpackage(1)} to build the binary package + from that. In particular, the \cb{debian/rules} implemenation is based on + the \cb{dh(1)} command sequencer. While this approach is normally used to + build packages from source, this implementation \"pretends\" that this is + what's happening by overriding a number of \cb{dh} targets to invoke the + \cb{build2} build system on the required packages directly in their + \cb{bpkg} configuration locations. Typical invocation: + + \ + bpkg build libhello + bpkg test libhello + bpkg bindist -o /tmp/output/ libhello + \ + + Note that the \cb{dpkg-dev} (or \cb{build-essential}) and \cb{debhelper} + Debian packages must be installed before invocation. + + See \l{bpkg#bindist-mapping-debian-produce Debian Package Mapping for + Production} for details on \cb{bpkg} to Debian package name and version + mapping. + " + + "\h|PKG-BINDIST DEBIAN OPTIONS|" + + bool --debian-prepare-only + { + "Prepare all the package metadata files (\cb{control}, \cb{rules}, etc) + but do not invoke \cb{bpkg-buildpackage} to generate the binary + package, printing its command line instead unless requested to be + quiet. Implies \cb{--keep-output}." + } + + string --debian-buildflags = "assign" + { + "<mode>", + "Package build flags (\cb{dpkg-buildflags}) usage mode. Valid <mode> + values are \cb{assign} (use the build flags instead of configured), + \cb{append} (use the build flags in addition to configured, putting + them last), \cb{prepend} (use the build flags in addition to + configured, putting them first), and \cb{ignore} (ignore build + flags). The default mode is \cb{assign}. Note that compiler mode + options, if any, are used as configured." + } + + strings --debian-maint-option + { + "<o>", + "Alternative options to specify in the \cb{DEB_BUILD_MAINT_OPTIONS} + variable of the \cb{rules} file. To specify multiple maintainer options + repeat this option and/or specify them as a single value separated + with spaces." + } + + strings --debian-build-option + { + "<o>", + "Additional option to pass to the \cb{dpkg-buildpackage} program. Repeat + this option to specify multiple build options." + } + + string --debian-build-meta + { + "<data>", + "Alternative build metadata to include in the binary package version. + If empty value is specified, then no build metadata is included. By + default, the build metadata is the \cb{ID} and \cb{VERSION_ID} + components from \cb{os-release(5)}, for example, \cb{debian10} in + version \cb{1.2.3-0~debian10}." + } + + string --debian-section + { + "<v>", + "Alternative \cb{Section} \cb{control} file field value for the main + binary package. The default is either \cb{libs} or \cb{devel}, + depending on the package type." + } + + string --debian-priority + { + "<v>", + "Alternative \cb{Priority} \cb{control} file field value. The default + is \cb{optional}." + } + + string --debian-maintainer + { + "<v>", + "Alternative \cb{Maintainer} \cb{control} file field value. The + default is the \cb{package-email} value from package \cb{manifest}." + } + + string --debian-architecture + { + "<v>", + "Alternative \cb{Architecture} \cb{control} file field value for + the main binary package. The default is \cb{any}." + } + + string --debian-main-langdep + { + "<v>", + "Override the language runtime dependencies (such as \cb{libc6}, + \cb{libstdc++6}, etc) in the \cb{Depends} \cb{control} file field + value of the main binary package." + } + + string --debian-dev-langdep + { + "<v>", + "Override the language runtime dependencies (such as \cb{libc-dev}, + \cb{libstdc++-dev}, etc) in the \cb{Depends} \cb{control} file field + value of the development (\cb{-dev}) binary package." + } + + string --debian-main-extradep + { + "<v>", + "Extra dependencies to add to the \cb{Depends} \cb{control} file field + value of the main binary package." + } + + string --debian-dev-extradep + { + "<v>", + "Extra dependencies to add to the \cb{Depends} \cb{control} file field + value of the development (\cb{-dev}) binary package." + } + }; + + // NOTE: remember to add the corresponding `--class-doc ...=exclude-base` + // (both in bpkg/ and doc/) if adding a new base class. + // + class pkg_bindist_options: configuration_options, + pkg_bindist_debian_options + { + "\h|PKG-BINDIST COMMON OPTIONS|" + + string --distribution + { + "<name>", + "Alternative system/distribution package manager to generate the binary + package for. The valid <name> values are \cb{debian} (Debian and + alike, such as Ubuntu, etc) and \cb{fedora} (Fedora and alike, + such as RHEL, CentOS, etc). Note that some package managers may + only be supported when running on certain host operating systems." + } + + string --architecture + { + "<name>", + "Alternative architecture to generate the binary package for. The + valid <name> values are system/distribution package manager-specific. + If unspecified, the host architecture is used." + } + + string --recursive + { + "<mode>", + "Bundle dependencies of the specified packages. The <mode> value can be + either \cb{auto}, in which case only the required files from each + dependency package are bundled, or \cb{full}, in which case all the + files are bundled. Specifically, in the \cb{auto} mode any required + files, for example, shared libraries, are pulled implicitly by the + \cb{install} build system operation, for example, as part of + installing an executable from one of the specified packages. In + contrast, in the \cb{full} mode, each dependency package is + installed explicitly and completely, as if they were specified + as additional package on the command line. See also the \cb{--private} + option." + } + + bool --private + { + "Enable the private installation subdirectory functionality using the + package name as the private subdirectory. This is primarily useful + when bundling dependencies, such as shared libraries, of an executable + that is being installed into a shared location, such as \cb{/usr/}. + See the \cb{config.install.private} configuration variable + documentation in the build system manual for details. This option only + makes sense together with \cb{--recursive}." + } + + dir_path --output-root|-o + { + "<dir>", + "Directory for intermediate files and subdirectories as well as the + resulting binary package. Note that this option may be required for + some system package managers and may not be specified for others." + } + + bool --wipe-output + { + "Wipe the output root directory (either specified with \ci{--output-root} + or system package manager-specific) clean before using it to generate + the binary package." + } + + bool --keep-output + { + "Keep intermediate files in the output root directory (either specified + with \ci{--output-root} or system package manager-specific) that were + used to generate the binary package. This is primarily useful for + troubleshooting." + } + }; + + " + \h|DEFAULT OPTIONS FILES| + + See \l{bpkg-default-options-files(1)} for an overview of the default + options files. For the \cb{pkg-bindist} command the search start + directory is the configuration directory. The following options files are + searched for in each directory and, if found, loaded in the order listed: + + \ + bpkg.options + bpkg-pkg-bindist.options + \ + + The following \cb{pkg-bindist} command options cannot be specified in the + default options files: + + \ + --directory|-d + \ + " +} diff --git a/bpkg/pkg-bindist.cxx b/bpkg/pkg-bindist.cxx new file mode 100644 index 0000000..e3ec9fa --- /dev/null +++ b/bpkg/pkg-bindist.cxx @@ -0,0 +1,448 @@ +// file : bpkg/pkg-bindist.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <bpkg/pkg-bindist.hxx> + +#include <bpkg/package.hxx> +#include <bpkg/package-odb.hxx> +#include <bpkg/package-query.hxx> +#include <bpkg/database.hxx> +#include <bpkg/pkg-verify.hxx> +#include <bpkg/diagnostics.hxx> +#include <bpkg/system-package-manager.hxx> + +using namespace std; +using namespace butl; + +namespace bpkg +{ + using package = system_package_manager::package; + using packages = system_package_manager::packages; + using recursive_mode = system_package_manager::recursive_mode; + + // Find the available package(s) for the specified selected package. + // + // Specifically, for non-system packages we look for a single available + // package. For system packages we look for all the available packages + // analogous to pkg-build. If none are found then we assume the + // --sys-no-stub option was used to configure this package and return an + // empty list. @@ What if it was configured with a specific bpkg version or + // `*`? + // + static available_packages + find_available_packages (const common_options& co, + database& db, + const shared_ptr<selected_package>& p) + { + assert (p->state == package_state::configured); + + available_packages r; + if (p->substate == package_substate::system) + { + r = find_available_all (repo_configs, p->name); + } + else + { + pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>> ap ( + find_available_fragment (co, db, p)); + + if (ap.second.loaded () && ap.second == nullptr) + { + // This is an orphan. We used to fail but there is no reason we cannot + // just load its manifest and make an available package out of that. + // And it's handy to be able to run this command on packages built + // from archives. + // + package_manifest m ( + pkg_verify (co, + p->effective_src_root (db.config_orig), + true /* ignore_unknown */, + false /* ignore_toolchain */, + false /* load_buildfiles */, + // Copy potentially fixed up version from selected package. + [&p] (version& v) {v = p->version;})); + + // Fake the buildfile information (not used). + // + m.alt_naming = false; + m.bootstrap_build = "project = " + p->name.string () + '\n'; + + ap.first = make_shared<available_package> (move (m)); + + // Fake the location (only used for diagnostics). + // + ap.second = make_shared<repository_fragment> ( + repository_location ( + p->effective_src_root (db.config).representation (), + repository_type::dir)); + + ap.first->locations.push_back ( + package_location {ap.second, current_dir}); + } + + r.push_back (move (ap)); + } + + return r; + } + + // Merge dependency languages for the (ultimate) dependent of the specified + // type. + // + static void + merge_languages (const string& type, + small_vector<language, 1>& langs, + const available_package& ap) + { + for (const language& l: ap.effective_languages ()) + { + // Unless both the dependent and dependency types are libraries, the + // interface/implementation distinction does not apply. + // + bool lib (type == "lib" && ap.effective_type () == "lib"); + + auto i (find_if (langs.begin (), langs.end (), + [&l] (const language& x) + { + return x.name == l.name; + })); + + if (i == langs.end ()) + { + // If this is an implementation language for a dependency, then it is + // also an implementation language for a dependent. The converse, + // howevere, depends on whether this dependency is an interface or + // imlementation of this dependent, which we do not know. So we have + // to assume it's interface. + // + langs.push_back (language {l.name, lib && l.impl}); + } + else + { + i->impl = i->impl && (lib && l.impl); // Merge. + } + } + } + + // Collect dependencies of the specified package, potentially recursively. + // System dependencies go to deps, non-system -- to pkgs, which could be the + // same as deps or NULL, depending on the desired semantics (see the call + // site for details). Find available packages for pkgs and deps and merge + // languages. + // + static void + collect_dependencies (const common_options& co, + database& db, + packages* pkgs, + packages& deps, + const string& type, + small_vector<language, 1>& langs, + const selected_package& p, + bool recursive) + { + for (const auto& pr: p.prerequisites) + { + const lazy_shared_ptr<selected_package>& ld (pr.first); + + // We only consider dependencies from target configurations, similar + // to pkg-install. + // + database& pdb (ld.database ()); + if (pdb.type == host_config_type || pdb.type == build2_config_type) + continue; + + shared_ptr<selected_package> d (ld.load ()); + + // Packaging stuff that is spread over multiple configurations is just + // to hairy so we don't support it. Specifically, it becomes tricky to + // override build options since using a global override will also affect + // host/build2 configurations. + // + if (db != pdb) + fail << "dependency package " << *d << " belongs to different " + << "configuration " << pdb.config_orig; + + // The selected package can only be configured if all its dependencies + // are configured. + // + assert (d->state == package_state::configured); + + bool sys (d->substate == package_substate::system); + packages* ps (sys ? &deps : pkgs); + + // Skip duplicates. + // + if (ps == nullptr || find_if (ps->begin (), ps->end (), + [&d] (const package& p) + { + return p.selected == d; + }) == ps->end ()) + { + const selected_package& p (*d); + + if (ps != nullptr || (recursive && !sys)) + { + available_packages aps (find_available_packages (co, db, d)); + + // Load and merge languages. + // + if (recursive && !sys) + { + const shared_ptr<available_package>& ap (aps.front ().first); + db.load (*ap, ap->languages_section); + merge_languages (type, langs, *ap); + } + + if (ps != nullptr) + { + dir_path out; + if (ps != &deps) + out = p.effective_out_root (db.config); + + ps->push_back (package {move (d), move (aps), move (out)}); + } + } + + if (recursive && !sys) + collect_dependencies (co, db, pkgs, deps, type, langs, p, recursive); + } + } + } + + int + pkg_bindist (const pkg_bindist_options& o, cli::scanner& args) + { + tracer trace ("pkg_bindist"); + + dir_path c (o.directory ()); + l4 ([&]{trace << "configuration: " << c;}); + + // Verify options. + // + optional<recursive_mode> rec; + { + diag_record dr; + + if (o.recursive_specified ()) + { + const string& m (o.recursive ()); + + if (m == "auto") rec = recursive_mode::auto_; + else if (m == "full") rec = recursive_mode::full; + else + dr << fail << "unknown mode '" << m << "' specified with --recursive"; + } + else if (o.private_ ()) + dr << fail << "--private specified without --recursive"; + + if (!dr.empty ()) + dr << info << "run 'bpkg help pkg-bindist' for more information"; + } + + // Sort arguments into package names and configuration variables. + // + vector<package_name> pns; + strings vars; + { + bool sep (false); // Seen `--`. + + while (args.more ()) + { + string a (args.next ()); + + // If we see the `--` separator, then we are done parsing variables + // (while they won't clash with package names, we may be given a + // directory path that contains `=`). + // + if (!sep && a == "--") + { + sep = true; + continue; + } + + if (a.find ('=') != string::npos) + vars.push_back (move (trim (a))); + else + { + try + { + pns.push_back (package_name (move (a))); // Not moved on failure. + } + catch (const invalid_argument& e) + { + fail << "invalid package name '" << a << "': " << e; + } + } + } + + if (pns.empty ()) + fail << "package name argument expected" << + info << "run 'bpkg help pkg-bindist' for more information"; + } + + database db (c, trace, true /* pre_attach */); + + // Similar to pkg-install we disallow generating packages from the + // host/build2 configurations. + // + if (db.type == host_config_type || db.type == build2_config_type) + { + fail << "unable to generate distribution package from " << db.type + << " configuration" << + info << "use target configuration instead"; + } + + // Prepare for the find_available_*() calls. + // + repo_configs.push_back (db); + + transaction t (db); + + // We need to suppress duplicate dependencies for the recursive mode. + // + session ses; + + // Resolve package names to selected packages and verify they are all + // configured. While at it collect their available packages and + // dependencies as well as figure out type and languages. + // + packages pkgs, deps; + string type; + small_vector<language, 1> langs; + + for (const package_name& n: pns) + { + shared_ptr<selected_package> p (db.find<selected_package> (n)); + + if (p == nullptr) + fail << "package " << n << " does not exist in configuration " << c; + + if (p->state != package_state::configured) + fail << "package " << n << " is " << p->state << + info << "expected it to be configured"; + + if (p->substate == package_substate::system) + fail << "package " << n << " is configured as system"; + + // Load the available package for type/languages as well as the mapping + // information. + // + available_packages aps (find_available_packages (o, db, p)); + const shared_ptr<available_package>& ap (aps.front ().first); + db.load (*ap, ap->languages_section); + + if (pkgs.empty ()) // First. + { + type = ap->effective_type (); + langs = ap->effective_languages (); + } + else + merge_languages (type, langs, *ap); + + const selected_package& r (*p); + pkgs.push_back ( + package {move (p), move (aps), r.effective_out_root (db.config)}); + + // If --recursive is not specified then we want all the immediate + // (system and non-) dependecies in deps. Otherwise, if the recursive + // mode is full, then we want all the transitive non-system dependecies + // in pkgs. In both recursive modes we also want all the transitive + // system dependecies in deps. + // + // Note also that in the auto recursive mode it's possible that some of + // the system dependencies are not really needed. But there is no way + // for us to detect this and it's better to over- than under-specify. + // + collect_dependencies (o, + db, + (rec + ? *rec == recursive_mode::full ? &pkgs : nullptr + : &deps), + deps, + type, + langs, + r, + rec.has_value ()); + } + + t.commit (); + + // Load the package manifest (source of extra metadata). This should be + // always possible since the package is configured and is not system. + // + const shared_ptr<selected_package>& sp (pkgs.front ().selected); + + package_manifest pm ( + pkg_verify (o, + sp->effective_src_root (db.config_orig), + true /* ignore_unknown */, + false /* ignore_toolchain */, + false /* load_buildfiles */, + // Copy potentially fixed up version from selected package. + [&sp] (version& v) {v = sp->version;})); + + // Note that we shouldn't need to install anything or use sudo. + // + unique_ptr<system_package_manager> spm ( + make_production_system_package_manager (o, + host_triplet, + o.distribution (), + o.architecture ())); + if (spm == nullptr) + { + fail << "no standard distribution package manager for this host " + << "or it is not yet supported" << + info << "consider specifying alternative distribution package " + << "manager with --distribution"; + } + + // Note that we pass type from here in case one day we want to provide an + // option to specify/override it (along with languages). Note that there + // will probably be no way to override type for dependencies. + // + paths r (spm->generate (pkgs, deps, vars, db.config, pm, type, langs, rec)); + + if (r.empty ()) + return 0; // Assume prepare-only mode or similar. + + if (verb && !o.no_result ()) + { + const selected_package& p (*pkgs.front ().selected); + + diag_record dr (text); + + dr << "generated " << spm->os_release.name_id << " package for " + << p.name << '/' << p.version << ':'; + + for (const path& p: r) + dr << "\n " << p; + } + + return 0; + } + + pkg_bindist_options + merge_options (const default_options<pkg_bindist_options>& defs, + const pkg_bindist_options& cmd) + { + // NOTE: remember to update the documentation if changing anything here. + + return merge_default_options ( + defs, + cmd, + [] (const default_options_entry<pkg_bindist_options>& e, + const pkg_bindist_options&) + { + const pkg_bindist_options& o (e.options); + + auto forbid = [&e] (const char* opt, bool specified) + { + if (specified) + fail (e.file) << opt << " in default options file"; + }; + + forbid ("--directory|-d", o.directory_specified ()); + }); + } +} diff --git a/bpkg/pkg-bindist.hxx b/bpkg/pkg-bindist.hxx new file mode 100644 index 0000000..3a756f8 --- /dev/null +++ b/bpkg/pkg-bindist.hxx @@ -0,0 +1,27 @@ +// file : bpkg/pkg-bindist.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_PKG_BINDIST_HXX +#define BPKG_PKG_BINDIST_HXX + +#include <bpkg/types.hxx> +#include <bpkg/utility.hxx> + +#include <bpkg/pkg-command.hxx> +#include <bpkg/pkg-bindist-options.hxx> + +namespace bpkg +{ + // Note that for now it doesn't seem we need to bother with package- + // specific configuration variables so it's scanner instead of + // group_scanner. + // + int + pkg_bindist (const pkg_bindist_options&, cli::scanner&); + + pkg_bindist_options + merge_options (const default_options<pkg_bindist_options>&, + const pkg_bindist_options&); +} + +#endif // BPKG_PKG_BINDIST_HXX diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx index b541541..747d037 100644 --- a/bpkg/system-package-manager-debian.cxx +++ b/bpkg/system-package-manager-debian.cxx @@ -3,8 +3,15 @@ #include <bpkg/system-package-manager-debian.hxx> +#include <locale> + +#include <libbutl/timestamp.hxx> +#include <libbutl/filesystem.hxx> // permissions + #include <bpkg/diagnostics.hxx> +#include <bpkg/pkg-bindist-options.hxx> + using namespace butl; namespace bpkg @@ -24,7 +31,8 @@ namespace bpkg c; } - // Parse the debian-name (or alike) value. + // Parse the debian-name (or alike) value. The first argument is the package + // type. // // Note that for now we treat all the packages from the non-main groups as // extras omitting the -common package (assuming it's pulled by the main @@ -32,7 +40,7 @@ namespace bpkg // extra_{doc,dbg} arguments. // package_status system_package_manager_debian:: - parse_name_value (const package_name& pn, + parse_name_value (const string& pt, const string& nv, bool extra_doc, bool extra_dbg) @@ -52,8 +60,7 @@ namespace bpkg return nn > sn && n.compare (nn - sn, sn, s) == 0; }; - auto parse_group = [&split, &suffix] (const string& g, - const package_name* pn) + auto parse_group = [&split, &suffix] (const string& g, const string* pt) { strings ns (split (g, ' ')); @@ -64,8 +71,6 @@ namespace bpkg // Handle the "dev instead of main" special case for libraries. // - // Note: the lib prefix check is based on the bpkg package name. - // // Check that the following name does not end with -dev. This will be // the only way to disambiguate the case where the library name happens // to end with -dev (e.g., libfoo-dev libfoo-dev-dev). @@ -73,10 +78,9 @@ namespace bpkg { string& m (ns[0]); - if (pn != nullptr && - pn->string ().compare (0, 3, "lib") == 0 && - pn->string ().size () > 3 && - suffix (m, "-dev") && + if (pt != nullptr && + *pt == "lib" && + suffix (m, "-dev") && !(ns.size () > 1 && suffix (ns[1], "-dev"))) { r = package_status ("", move (m)); @@ -117,7 +121,7 @@ namespace bpkg for (size_t i (0); i != gs.size (); ++i) { if (i == 0) // Main group. - r = parse_group (gs[i], &pn); + r = parse_group (gs[i], &pt); else { package_status g (parse_group (gs[i], nullptr)); @@ -894,14 +898,6 @@ namespace bpkg optional<const system_package_status*> system_package_manager_debian:: pkg_status (const package_name& pn, const available_packages* aps) { - // For now we ignore -doc and -dbg package components (but we may want to - // have options controlling this later). Note also that we assume -common - // is pulled automatically by the main package so we ignore it as well - // (see equivalent logic in parse_name_value()). - // - bool need_doc (false); - bool need_dbg (false); - // First check the cache. // { @@ -914,6 +910,25 @@ namespace bpkg return nullopt; } + optional<package_status> r (status (pn, *aps)); + + // Cache. + // + auto i (status_cache_.emplace (pn, move (r)).first); + return i->second ? &*i->second : nullptr; + } + + optional<package_status> system_package_manager_debian:: + status (const package_name& pn, const available_packages& aps) + { + // For now we ignore -doc and -dbg package components (but we may want to + // have options controlling this later). Note also that we assume -common + // is pulled automatically by the main package so we ignore it as well + // (see equivalent logic in parse_name_value()). + // + bool need_doc (false); + bool need_dbg (false); + vector<package_status> candidates; // Translate our package name to the Debian package names. @@ -926,12 +941,26 @@ namespace bpkg << os_release.name_id << " package name"; }); + // Without explicit type, the best we can do in trying to detect whether + // this is a library is to check for the lib prefix. Libraries without + // the lib prefix and non-libraries with the lib prefix (both of which + // we do not recomment) will have to provide a manual mapping (or + // explicit type). + // + // Note that using the first (latest) available package as a source of + // type information seems like a reasonable choice. + // + const string& pt (!aps.empty () + ? aps.front ().first->effective_type () + : package_manifest::effective_type (nullopt, pn)); + strings ns; - if (!aps->empty ()) - ns = system_package_names (*aps, + if (!aps.empty ()) + ns = system_package_names (aps, os_release.name_id, os_release.version_id, - os_release.like_ids); + os_release.like_ids, + true /* native */); if (ns.empty ()) { // Attempt to automatically translate our package name (see above for @@ -939,12 +968,7 @@ namespace bpkg // const string& n (pn.string ()); - // The best we can do in trying to detect whether this is a library is - // to check for the lib prefix. Libraries without the lib prefix and - // non-libraries with the lib prefix (both of which we do not - // recomment) will have to provide a manual mapping. - // - if (n.compare (0, 3, "lib") == 0 && n.size () > 3) + if (pt == "lib") { // Keep the main package name empty as an indication that it is to // be discovered. @@ -960,7 +984,7 @@ namespace bpkg // for (const string& n: ns) { - package_status s (parse_name_value (pn, n, need_doc, need_dbg)); + package_status s (parse_name_value (pt, n, need_doc, need_dbg)); // Suppress duplicates for good measure based on the main package // name (and falling back to -dev if empty). @@ -1269,9 +1293,9 @@ namespace bpkg string sv (r->system_version, 0, r->system_version.rfind ('-')); optional<version> v; - if (!aps->empty ()) + if (!aps.empty ()) v = downstream_package_version (sv, - *aps, + aps, os_release.name_id, os_release.version_id, os_release.like_ids); @@ -1304,10 +1328,7 @@ namespace bpkg r->version = move (*v); } - // Cache. - // - auto i (status_cache_.emplace (pn, move (r)).first); - return i->second ? &*i->second : nullptr; + return r; } void system_package_manager_debian:: @@ -1447,6 +1468,339 @@ namespace bpkg } } + // Map non-system bpkg package to system package name(s) and version. + // + // This is used both to map the package being generated and its + // dependencies. What should we do with extras returned in package_status? + // We can't really generate any of them (which files would we place in + // them?) nor can we list them as dependencies (we don't know their system + // versions). So it feels like the only sensible choice is to ignore extras. + // + // In a sense, we have a parallel arrangement going on here: binary packages + // that we generate don't have extras (i.e., they include everything + // necessary in the "standard" packages from the main group) and when we + // punch a system dependency based on a non-system bpkg package, we assume + // it was generated by us and thus doesn't have any extras. Or, to put it + // another way, if you want the system dependency to refer to a "native" + // system package with extras you need to configure it as a system bpkg + // package. + // + // In fact, this extends to package names. For example, unless custom + // mapping is specified, we will generate libsqlite3 and libsqlite3-dev + // while native names are libsqlite3-0 and libsqlite3-dev. While this + // duality is not ideal, presumably we will normally only be producing our + // binary packages if there are no suitable native packages. And for a few + // exception (e.g., our package is "better" in some way, such as configured + // differently or fixes a critical bug), we will just have to provide + // appropriate manual mapping that makes sure the names match (the extras is + // still a potential problem though -- we will only have them as + // dependencies if we build against a native system package; maybe we can + // add them manually with an option). + // + package_status system_package_manager_debian:: + map_package (const package_name& pn, + const version& pv, + const available_packages& aps, + const optional<string>& build_metadata) const + { + // We should only have one available package corresponding to this package + // name/version. + // + assert (aps.size () == 1); + + const shared_ptr<available_package>& ap (aps.front ().first); + const lazy_shared_ptr<repository_fragment>& rf (aps.front ().second); + + // Without explicit type, the best we can do in trying to detect whether + // this is a library is to check for the lib prefix. Libraries without the + // lib prefix and non-libraries with the lib prefix (both of which we do + // not recomment) will have to provide a manual mapping (or explicit + // type). + // + const string& pt (ap->effective_type ()); + + strings ns (system_package_names (aps, + os_release.name_id, + os_release.version_id, + os_release.like_ids, + false /* native */)); + package_status r; + if (ns.empty ()) + { + // Automatically translate our package name similar to the consumption + // case above. Except here we don't attempt to deduce main from -dev, + // naturally. + // + const string& n (pn.string ()); + + if (pt == "lib") + r = package_status (n, n + "-dev"); + else + r = package_status (n); + } + else + { + // Even though we only pass one available package, we may still end up + // with multiple mappings. In this case we take the first, per the + // documentation. + // + r = parse_name_value (pt, + ns.front (), + false /* need_doc */, + false /* need_dbg */); + + // If this is -dev without main, then derive main by stripping the -dev + // suffix. This feels tighter than just using the bpkg package name. + // + if (r.main.empty ()) + { + assert (!r.dev.empty ()); + r.main.assign (r.dev, 0, r.dev.size () - 4); + } + } + + // Map the version. + // + // NOTE: THE BELOW DESCRIPTION IS ALSO REPRODUCED IN THE BPKG MANUAL. + // + // To recap, a Debian package version has the following form: + // + // [<epoch>:]<upstream>[-<revision>] + // + // For details on the ordering semantics, see the Version control file + // field documentation in the Debian Policy Manual. While overall + // unsurprising, one notable exception is `~`, which sorts before anything + // else and is commonly used for upstream pre-releases. For example, + // 1.0~beta1~svn1245 sorts earlier than 1.0~beta1, which sorts earlier + // than 1.0. + // + // There are also various special version conventions (such as all the + // revision components in 1.4-5+deb10u1~bpo9u1) but they all appear to + // express relationships between native packages and/or their upstream and + // thus do not apply to our case. + // + // Ok, so how do we map our version to that? To recap, the bpkg version + // has the following form: + // + // [+<epoch>-]<upstream>[-<prerel>][+<revision>] + // + // Let's start with the case where neither distribution nor upstream + // version is specified and we need to derive everything from the bpkg + // version. + // + // <epoch> + // + // On one hand, if we keep the epoch, it won't necessarily match + // Debian's native package epoch. But on the other it will allow our + // binary packages from different epochs to co-exist. Seeing that this + // can be easily overridden with a custom distribution version, let's + // keep it. + // + // Note that while the Debian start/default epoch is 0, ours is 1 (we + // use the 0 epoch for stub packages). So we will need to shift this + // value range. + // + // + // <upstream>[-<prerel>] + // + // Our upstream version maps naturally to Debian's. That is, our + // upstream version format/semantics is a subset of Debian's. + // + // If this is a pre-release, then we could fail (that is, don't allow + // pre-releases) but then we won't be able to test on pre-release + // packages, for example, to make sure the name mapping is correct. + // Plus sometimes it's useful to publish pre-releases. We could ignore + // it, but then such packages will be indistinguishable from each other + // and the final release, which is not ideal. On the other hand, Debian + // has the mechanism (`~`) which is essentially meant for this, so let's + // use it. We will use <prerel> as is since its format is the same as + // upstream and thus should map naturally. + // + // + // <revision> + // + // Similar to epoch, our revision won't necessarily match Debian's + // native package revision. But on the other hand it will allow us to + // establish a correspondence between source and binary packages. Plus, + // upgrades between binary package revisions will be handled naturally. + // Seeing that we allow overriding the revision with a custom + // distribution version (see below), let's keep it. + // + // Note also that both Debian and our revision start/default is 0. + // However, it is Debian's convention to start revision from 1. But it + // doesn't seem worth it for us to do any shifting here and so we will + // use our revision as is. + // + // Another related question is whether we should also include some + // metadata that identifies the distribution and its version that this + // package is for. The strongest precedent here is probably Ubuntu's + // PPA. While there doesn't appear to be a consistent approach, one can + // often see versions like these: + // + // 2.1.0-1~ppa0~ubuntu14.04.1, + // 1.4-5-1.2.1~ubuntu20.04.1~ppa1 + // 22.12.2-0ubuntu1~ubuntu23.04~ppa1 + // + // Seeing that this is a non-sortable component (what in semver would be + // called "build metadata"), using `~` is probably not the worst choice. + // + // So we follow this lead and add the ~<name_id><version_id> component + // to revision. Note that this also means we will have to make the 0 + // revision explicit. For example: + // + // 1.2.3-1~debian10 + // 1.2.3-0~ubuntu20.04 + // + // The next case to consider is when we have the upstream version + // (upstream-version manifest value). After some rumination it feels + // correct to use it in place of the <epoch>-<upstream> components in the + // above mapping (upstream version itself cannot have epoch). In other + // words, we will add the pre-release and revision components from the + // bpkg version. If this is not the desired semantics, then it can always + // be overrided with the distribution version. + // + // Finally, we have the distribution version. The Debian <epoch> and + // <upstream> components are straightforward: they should be specified by + // the distribution version as required. This leaves pre-release and + // revision. It feels like in most cases we would want these copied over + // from the bpkg version automatically -- it's too tedious and error- + // prone to maintain them manually. However, we want the user to have the + // full override ability. So instead, if empty revision is specified, as + // in 1.2.3-, then we automatically add the bpkg revision. Similarly, if + // empty pre-release is specified, as in 1.2.3~, then we add the bpkg + // pre-release. To add both automatically, we would specify 1.2.3~- (other + // combinations are 1.2.3~b.1- and 1.2.3~-1). + // + // Note also that per the Debian version specification, if upstream + // contains `:` and/or `-`, then epoch and/or revision must be specified + // explicitly, respectively. Note that the bpkg upstream version may not + // contain either. + // + string& sv (r.system_version); + + bool no_build_metadata (build_metadata && build_metadata->empty ()); + + if (optional<string> ov = system_package_version (ap, + rf, + os_release.name_id, + os_release.version_id, + os_release.like_ids)) + { + string& dv (*ov); + size_t n (dv.size ()); + + // Find the revision and pre-release positions, if any. + // + size_t rp (dv.rfind ('-')); + size_t pp (dv.rfind ('~', rp)); + + // Copy over the [<epoch>:]<upstream> part. + // + sv.assign (dv, 0, pp < rp ? pp : rp); + + // Add pre-release copying over the bpkg version value if empty. + // + if (pp != string::npos) + { + if (size_t pn = (rp != string::npos ? rp : n) - (pp + 1)) + { + sv.append (dv, pp, pn + 1); + } + else + { + if (pv.release) + { + assert (!pv.release->empty ()); // Cannot be earliest special. + sv += '~'; + sv += *pv.release; + } + } + } + + // Add revision copying over the bpkg version value if empty. + // + // Omit the default -0 revision if we have no build metadata. + // + if (rp != string::npos) + { + if (size_t rn = n - (rp + 1)) + { + sv.append (dv, rp, rn + 1); + } + else if (pv.revision || !no_build_metadata) + { + sv += '-'; + sv += to_string (pv.revision ? *pv.revision : 0); + } + } + else if (!no_build_metadata) + sv += "-0"; // Default revision (for build metadata; see below). + } + else + { + if (ap->upstream_version) + { + const string& uv (*ap->upstream_version); + + // Add explicit epoch if upstream contains `:`. + // + // Note that we don't need to worry about `-` since we always add + // revision (see below). + // + if (uv.find (':') != string::npos) + sv = "0:"; + + sv += uv; + } + else + { + // Add epoch unless maps to 0. + // + assert (pv.epoch != 0); // Cannot be a stub. + if (pv.epoch != 1) + { + sv = to_string (pv.epoch - 1); + sv += ':'; + } + + sv += pv.upstream; + } + + // Add pre-release. + // + if (pv.release) + { + assert (!pv.release->empty ()); // Cannot be earliest special. + sv += '~'; + sv += *pv.release; + } + + // Add revision. + // + if (pv.revision || !no_build_metadata) + { + sv += '-'; + sv += to_string (pv.revision ? *pv.revision : 0); + } + } + + // Add build matadata. + // + if (!no_build_metadata) + { + sv += '~'; + if (build_metadata) + sv += *build_metadata; + else + { + sv += os_release.name_id; + sv += os_release.version_id; // Could be empty. + } + } + + return r; + } + // Some background on creating Debian packages (for a bit more detailed // overview see the Debian Packaging Tutorial). // @@ -1455,8 +1809,8 @@ namespace bpkg // create the package completely manually without using any of the Debian // tools and while some implementations (for example, cargo-deb) do it this // way, we are not going to go this route because it does not scale well to - // more complex packages which may require additional functionality, such as - // managing systemd files, and which is covered by the Debian tools (for an + // more complex packages which may require additional functionality (such as + // managing systemd files) and which is covered by the Debian tools (for an // example of where this leads, see the partial debhelper re-implementation // in cargo-deb). Another issues with this approach is that it's not // amenable to customizations, at least not in a way familiar to Debian @@ -1464,24 +1818,24 @@ namespace bpkg // // At the lowest level of the Debian tools for creating packages sits the // dpkg-deb --build|-b command (also accessible as dpkg --build|-b). Given a - // directory with all the binary contents (including the package metadata, - // such as the control file, in the debian/ subdirectory) this command will - // pack everything up into a .deb file. While an improvement over the fully - // manual packaging, this approach has essentially the same drawbacks. In - // particular, this command generates a single package which means we will - // have to manually sort out things into -dev, -doc, etc. + // directory with all the binary package contents (including the package + // metadata, such as the control file, in the debian/ subdirectory) this + // command will pack everything up into a .deb file. While an improvement + // over the fully manual packaging, this approach has essentially the same + // drawbacks. In particular, this command generates a single package which + // means we will have to manually sort out things into -dev, -doc, etc. // // Next up the stack is dpkg-buildpackage. This tool expects the package to - // follow the Debian way, that is, to provide the debian/rules makefile with - // a number of required targets which it then invokes to build, install, and - // pack a package from source (and somewhere in this process it calls - // dpkg-deb --build). The dpkg-buildpackage(1) man page has an overview of - // all the steps that this command performs and it is the recommended, - // lower-level, way to build packages on Debian. + // follow the Debian way of packaging, that is, to provide the debian/rules + // makefile with a number of required targets which it then invokes to + // build, install, and pack a package from source (and sometime during this + // process it calls dpkg-deb --build). The dpkg-buildpackage(1) man page has + // an overview of all the steps that this command performs and it is the + // recommended, lower-level, way to build packages on Debian. // // At the top of the stack sits debuild which calls dpkg-buildpackage, then - // lintian and finally design (though signing can also be performed by - // dpkg-buildpackage). + // lintian, and finally design (though signing can also be performed by + // dpkg-buildpackage itself). // // Based on this our plan is to use dpkg-buildpackage which brings us to the // Debian way of packaging with debian/rules at its core. As it turns out, @@ -1498,26 +1852,1478 @@ namespace bpkg // While debhelper tools definitely simplify debian/rules, there is often // still a lot of boilerplate code. So second-level helpers are often used, // with the dominant option being the dh(1) command sequencer (there is also - // CDBS but it appears to be mostly obsolete). + // CDBS but it appears to be fading into obsolescence). // // Based on that our options appear to be classic debhelper and dh. Looking // at the statistics, it's clear that the majority of packages (including // fairly complex ones) tend to prefer dh and there is no reason for us to // try to buck this trend. // + // NOTE: THE BELOW DESCRIPTION IS ALSO REWORDED IN BPKG-PKG-BINDIST(1). + // // So, to sum up, the plan is to produce debian/rules that uses the dh // command sequencer and then invoke dpkg-buildpackage to produce the binary // package from that. While this approach is normally used to build things - // from source, it feels like we should be able to pretend that we are by, - // for example, overriding the install target to invoke the build system to - // install all the packages directly from their bpkg locations. + // from source, it feels like we should be able to pretend that we are. + // Specifially, we can override the install target to invoke the build + // system and install all the packages directly from their bpkg locations. // - void system_package_manager_debian:: - generate (packages&&, - packages&&, - strings&&, - const dir_path&, - optional<recursive_mode>) + // Note that the -dbgsym packages are generated by default and all we need + // to do from our side is to compile with debug information (-g), failed + // which we get a warning from debhelper. + // + // Note: this setup requires dpkg-dev (or build-essential) and debhelper + // packages. + // + paths system_package_manager_debian:: + generate (const packages& pkgs, + const packages& deps, + const strings& vars, + const dir_path& cfg_dir, + const package_manifest& pm, + const string& pt, + const small_vector<language, 1>& langs, + optional<recursive_mode> recur) { + tracer trace ("system_package_manager_debian::generate"); + + assert (!langs.empty ()); // Should be effective. + + // We require explicit output root. + // + if (!ops_->output_root_specified ()) + fail << "output root directory must be specified explicitly with " + << "--output-root|-o"; + + const dir_path& out (ops_->output_root ()); // Cannot be empty. + + optional<string> build_metadata; + if (ops_->debian_build_meta_specified ()) + build_metadata = ops_->debian_build_meta (); + + const shared_ptr<selected_package>& sp (pkgs.front ().selected); + const package_name& pn (sp->name); + const version& pv (sp->version); + + const available_packages& aps (pkgs.front ().available); + + bool lib (pt == "lib"); + bool priv (ops_->private_ ()); // Private installation. + + // For now we only know how to handle libraries with C-common interface + // languages. But we allow other implementation languages. + // + if (lib) + { + for (const language& l: langs) + if (!l.impl && l.name != "c" && l.name != "c++" && l.name != "cc") + fail << l.name << " libraries are not yet supported"; + } + + // Return true if this package uses the specified language, only as + // interface language if intf_only is true. + // + auto lang = [&langs] (const char* n, bool intf_only = false) -> bool + { + return find_if (langs.begin (), langs.end (), + [n, intf_only] (const language& l) + { + return (!intf_only || !l.impl) && l.name == n; + }) != langs.end (); + }; + + // As a first step, figure out the system names and version of the package + // we are generating and all the dependencies, diagnosing anything fishy. + // + // Note that there should be no duplicate dependencies and we can sidestep + // the status cache. + // + package_status st (map_package (pn, pv, aps, build_metadata)); + + vector<package_status> sdeps; + sdeps.reserve (deps.size ()); + for (const package& p: deps) + { + const shared_ptr<selected_package>& sp (p.selected); + const available_packages& aps (p.available); + + package_status s; + if (sp->substate == package_substate::system) + { + optional<package_status> os (status (sp->name, aps)); + + if (!os || os->status != package_status::installed) + fail << os_release.name_id << " package for " << sp->name + << " system package is no longer installed"; + + // For good measure verify the mapped back version still matches + // configured. Note that besides the normal case (queried by the + // system package manager), it could have also been specified by the + // user as an actual version or a wildcard. Ignoring this check for a + // wildcard feels consistent with the overall semantics. + // + if (sp->version != wildcard_version && sp->version != os->version) + { + fail << "current " << os_release.name_id << " package version for " + << sp->name << " system package does not match configured" << + info << "configured version: " << sp->version << + info << "current version: " << os->version << " (" + << os->system_version << ')'; + } + + s = move (*os); + } + else + s = map_package (sp->name, sp->version, aps, build_metadata); + + sdeps.push_back (move (s)); + } + + if (verb >= 3) + { + auto print_status = [] (diag_record& dr, const package_status& s) + { + dr << s.main + << (s.dev.empty () ? "" : " ") << s.dev + << (s.doc.empty () ? "" : " ") << s.doc + << (s.dbg.empty () ? "" : " ") << s.dbg + << (s.common.empty () ? "" : " ") << s.common + << ' ' << s.system_version; + }; + + { + diag_record dr (trace); + dr << "package: "; + print_status (dr, st); + } + + for (const package_status& st: sdeps) + { + diag_record dr (trace); + dr << "dependency: "; + print_status (dr, st); + } + } + + if (!st.dbg.empty ()) + fail << "generation of obsolete manual -dbg packages not supported" << + info << "use automatic -dbgsym packages instead"; + + // We override every config.install.* variable in order not to pick + // anything configured. Note that we add some more in the rules file + // below. + // + // We make use of the <project> substitution since in the recursive mode + // we may be installing multiple projects. Note that the <private> + // directory component is automatically removed if this functionality is + // not enabled. One side-effect of using <project> is that we will be + // using the bpkg package name instead of the main Debian package name. + // But perhaps that's correct: on Debian it's usually the source package + // name, which is the same. To keep things consistent we use the bpkg + // package name for <private> as well. + // + // Note that some libraries have what looks like architecture-specific + // configuration files in /usr/include/$(DEB_HOST_MULTIARCH)/ which is + // what we use for our config.install.include_arch location. + // + // Note: we need to quote values that contain `$` so that they don't get + // expanded as build2 variables in the installed_entries() call. + // + // NOTE: make sure to update .install files below if changing anyting + // here. + // + strings config { + "config.install.root=/usr/", + "config.install.data_root=root/", + "config.install.exec_root=root/", + + "config.install.bin=exec_root/bin/", + "config.install.sbin=exec_root/sbin/", + + // On Debian shared libraries should not be executable. Also, + // libexec/ is the same as lib/ (note that executables that get + // installed there will still have the executable bit set). + // + "config.install.lib='exec_root/lib/$(DEB_HOST_MULTIARCH)/<private>/'", + "config.install.lib.mode=644", + "config.install.libexec=lib/<project>/", + "config.install.pkgconfig=lib/pkgconfig/", + + "config.install.etc=/etc/", + "config.install.include=data_root/include/<private>/", + "config.install.include_arch='data_root/include/$(DEB_HOST_MULTIARCH)/<private>/'", + "config.install.share=data_root/share/", + "config.install.data=share/<private>/<project>/", + + "config.install.doc=share/doc/<private>/<project>/", + "config.install.legal=doc/", + "config.install.man=share/man/", + "config.install.man1=man/man1/", + "config.install.man2=man/man2/", + "config.install.man3=man/man3/", + "config.install.man4=man/man4/", + "config.install.man5=man/man5/", + "config.install.man6=man/man6/", + "config.install.man7=man/man7/", + "config.install.man8=man/man8/"}; + + config.push_back ("config.install.private=" + + (priv ? pn.string () : "[null]")); + + // Add user-specified configuration variables last to allow them to + // override anything. + // + for (const string& v: vars) + config.push_back (v); + + // Note that we can use weak install scope for the auto recursive mode + // since we know dependencies cannot be spread over multiple linked + // configurations. + // + string scope (!recur || *recur == recursive_mode::full + ? "project" + : "weak"); + + // Get the map of files that will end up in the binary packages. + // + // Note that we are passing quoted values with $(DEB_HOST_MULTIARCH) which + // will be treated literally. + // + installed_entry_map ies (installed_entries (*ops_, pkgs, config, scope)); + + if (ies.empty ()) + fail << "specified package(s) do not install any files"; + + if (verb >= 4) + { + for (const auto& p: ies) + { + diag_record dr (trace); + dr << "installed entry: " << p.first; + + if (p.second.target != nullptr) + dr << " -> " << p.second.target->first; // Symlink. + else + dr << " " << p.second.mode; + } + } + + // Start assembling the package "source" directory. + // + // It's hard to predict all the files that will be generated (and + // potentially read), so we will just require a clean output directory. + // + // Also, by default, we are going to keep all the intermediate files on + // failure for troubleshooting. + // + if (exists (out)) + { + if (!empty (out)) + { + if (!ops_->wipe_output ()) + fail << "output root directory " << out << " is not empty" << + info << "use --wipe-output to clean it up but be careful"; + + rm_r (out, false); + } + } + + // Normally the source directory is called <name>-<upstream-version> + // (e.g., as unpacked from the source archive). + // + dir_path src (out / dir_path (pn.string () + '-' + pv.string ())); + dir_path deb (src / dir_path ("debian")); + mk_p (deb); + + // The control file. + // + // See the "Control files and their fields" chapter in the Debian Policy + // Manual for details (for example, which fields are mandatory). + // + // Note that we try to do a reasonably thorough job (e.g., filling in + // sections, etc) with the view that this can be used as a starting point + // for manual packaging (and perhaps we could add a mode for this in the + // future, call it "starting point" mode). + // + // Also note that this file supports variable substitutions (for example, + // ${binary:Version}) as described in deb-substvars(5). While we could do + // without, it is widely used in manual packages so we do the same. Note, + // however, that we don't use the shlibs:Depends/misc:Depends mechanism + // (which automatically detects dependencies) since we have an accurate + // set and some of them may not be system packages. + // + string homepage (pm.package_url ? pm.package_url->string () : + pm.url ? pm.url->string () : + string ()); + + string maintainer; + if (ops_->debian_maintainer_specified ()) + maintainer = ops_->debian_maintainer (); + else + { + const email* e (pm.package_email ? &*pm.package_email : + pm.email ? &*pm.email : + nullptr); + + if (e == nullptr) + fail << "unable to determine package maintainer from manifest" << + info << "specify explicitly with --debian-maintainer"; + + // In certain places (e.g., changelog), Debian expect this to be in the + // `John Doe <john@example.org>` form while we often specify just the + // email address (e.g., to the mailing list). Try to detect such a case + // and complete it to the desired format. + // + if (e->find (' ') == string::npos && e->find ('@') != string::npos) + { + // Try to use comment as name, if any. + // + if (!e->comment.empty ()) + maintainer = e->comment; + else + maintainer = pn.string () + " package maintainer"; + + maintainer += " <" + *e + '>'; + } + else + maintainer = *e; + } + + path ctrl (deb / "control"); + try + { + ofdstream os (ctrl); + + // First comes the general (source package) stanza. + // + // Note that the Priority semantics is not the same as our priority. + // Rather it should reflect the overall importance of the package. Our + // priority is more appropriately mapped to urgency in the changelog. + // + // If this is not a library, then by default we assume its some kind of + // a development tool and use the devel section. + // + // Note also that we require the debhelper compatibility level 13 which + // has more advanced features that we rely on. Such as: + // + // - Variable substitutions in the debhelper config files. + // + string section ( + ops_->debian_section_specified () ? ops_->debian_section () : + lib ? "libs" : + "devel"); + + string priority ( + ops_->debian_priority_specified () ? ops_->debian_priority () : + "optional"); + + os << "Source: " << pn << '\n' + << "Section: " << section << '\n' + << "Priority: " << priority << '\n' + << "Maintainer: " << maintainer << '\n' + << "Standards-Version: " << "4.6.2" << '\n' + << "Build-Depends: " << "debhelper-compat (= 13)" << '\n' + << "Rules-Requires-Root: " << "no" << '\n'; + if (!homepage.empty ()) + os << "Homepage: " << homepage << '\n'; + if (pm.src_url) + os << "Vcs-Browser: " << pm.src_url->string () << '\n'; + + // Then we have one or more binary package stanzas. + // + // Note that values from the source package stanza (such as Section, + // Priority) are used as defaults for the binary packages. + // + // We cannot easily detect architecture-independent packages (think + // libbutl.bash) and providing an option feels like the best we can do. + // Note that the value `any` means architecture-dependent while `all` + // means architecture-independent. + // + // The Multi-Arch hint can be `same` or `foreign`. The former means that + // a separate copy of the package may be installed for each architecture + // (e.g., library) while the latter -- that a single copy may be used by + // all architectures (e.g., executable, -doc, -common). Not that for + // some murky reasons Multi-Arch:foreign needs to be explicitly + // specified for Architecture:all. + // + // The Description field is quite messy: it requires both the short + // description (our summary) as a first line and a long description (our + // description) as the following lines in the multiline format. + // Converting our description to the Debian format is not going to be + // easy: it can be arbitrarily long and may not even be plain text (it's + // commonly the contents of the README.md file). So for now we fake it + // with a description of the package component. Note also that + // traditionally the Description field comes last. + // + string arch (ops_->debian_architecture_specified () + ? ops_->debian_architecture () + : "any"); + + string march (arch == "all" || !lib ? "foreign" : "same"); + + { + string depends; + + if (!st.common.empty ()) + depends = st.common + " (= ${binary:Version})"; + + for (const package_status& st: sdeps) + { + if (!depends.empty ()) + depends += ", "; + + // Note that the constraints will include build metadata (e.g., + // ~debian10). While it may be tempting to strip it, we cannot since + // the order is inverse. We could just make it empty `~`, though + // that will look a bit strange. But keeping it shouldn't cause any + // issues. Also note that the build metadata is part of the revision + // so we could strip the whole thing. + // + depends += st.main + " (>= " + st.system_version + ')'; + } + + if (ops_->debian_main_langdep_specified ()) + { + if (!ops_->debian_main_langdep ().empty ()) + { + if (!depends.empty ()) + depends += ", "; + + depends += ops_->debian_main_langdep (); + } + } + else + { + // Note that we are not going to add dependencies on libcN + // (currently libc6) or libstdc++N (currently libstdc++6) because + // it's not easy to determine N and they both are normally part of + // the base system. + // + // What about other language runtimes? Well, it doesn't seem like we + // can deduce those automatically so we will either have to add ad + // hoc support or the user will have to provide them manually with + // --debian-main-depends. + } + + if (!ops_->debian_main_extradep ().empty ()) + { + if (!depends.empty ()) + depends += ", "; + + depends += ops_->debian_main_extradep (); + } + + os << '\n' + << "Package: " << st.main << '\n' + << "Architecture: " << arch << '\n' + << "Multi-Arch: " << march << '\n'; + if (!depends.empty ()) + os << "Depends: " << depends << '\n'; + os << "Description: " << pm.summary << '\n' + << " This package contains the runtime files." << '\n'; + } + + if (!st.dev.empty ()) + { + string depends (st.main + " (= ${binary:Version})"); + + for (const package_status& st: sdeps) + { + // Doesn't look like we can distinguish between interface and + // implementation dependencies here. So better to over- than + // under-specify. + // + if (!st.dev.empty ()) + depends += ", " + st.dev + " (>= " + st.system_version + ')'; + } + + if (ops_->debian_dev_langdep_specified ()) + { + if (!ops_->debian_dev_langdep ().empty ()) + { + depends += ", " + ops_->debian_dev_langdep (); + } + } + else + { + // Add dependency on libcN-dev and libstdc++-N-dev. + // + // Note: libcN-dev provides libc-dev and libstdc++N-dev provides + // libstdc++-dev. While it would be better to depend on the exact + // versions, determining N is not easy (and in case of listdc++ + // there could be multiple installed at the same time). + // + // Note that we haven't seen just libc-dev in any native packages, + // it's always either libc6-dev or libc6-dev|libc-dev. So we will + // see how it goes. + // + // If this is an undetermined C-common library, we assume it may be + // C++ (better to over- than under-specify). + // + bool cc (lang ("cc", true)); + if (cc || (cc = lang ("c++", true))) depends += ", libstdc++-dev"; + if (cc || (cc = lang ("c", true))) depends += ", libc-dev"; + } + + if (!ops_->debian_dev_extradep ().empty ()) + { + depends += ", " + ops_->debian_dev_extradep (); + } + + // Feels like the architecture should be the same as for the main + // package. + // + os << '\n' + << "Package: " << st.dev << '\n' + << "Section: " << (lib ? "libdevel" : "devel") << '\n' + << "Architecture: " << arch << '\n' + << "Multi-Arch: " << march << '\n'; + if (!st.doc.empty ()) + os << "Suggests: " << st.doc << '\n'; + if (!depends.empty ()) + os << "Depends: " << depends << '\n'; + os << "Description: " << pm.summary << '\n' + << " This package contains the development files." << '\n'; + } + + if (!st.doc.empty ()) + { + os << '\n' + << "Package: " << st.doc << '\n' + << "Section: " << "doc" << '\n' + << "Architecture: " << "all" << '\n' + << "Multi-Arch: " << "foreign" << '\n' + << "Description: " << pm.summary << '\n' + << " This package contains the documentation." << '\n'; + } + + // Keep this in case we want to support it in the "starting point" mode. + // + if (!st.dbg.empty ()) + { + string depends (st.main + " (= ${binary:Version})"); + + os << '\n' + << "Package: " << st.dbg << '\n' + << "Section: " << "debug" << '\n' + << "Priority: " << "extra" << '\n' + << "Architecture: " << arch << '\n' + << "Multi-Arch: " << march << '\n'; + if (!depends.empty ()) + os << "Depends: " << depends << '\n'; + os << "Description: " << pm.summary << '\n' + << " This package contains the debugging information." << '\n'; + } + + if (!st.common.empty ()) + { + // Generally, this package is not necessarily architecture-independent + // (for example, it could contain something shared between multiple + // binary packages produced from the same source package rather than + // something shared between all the architectures of a binary + // package). But seeing that we always generate one binary package, + // for us it only makes sense as architecture-independent. + // + // It's also not clear what dependencies we can deduce for this + // package. Assuming that it depends on all the dependency -common + // packages is probably unreasonable. + // + os << '\n' + << "Package: " << st.common << '\n' + << "Architecture: " << "all" << '\n' + << "Multi-Arch: " << "foreign" << '\n' + << "Description: " << pm.summary << '\n' + << " This package contains the architecture-independent files." << '\n'; + } + + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to " << ctrl << ": " << e; + } + + // The changelog file. + // + // See the "Debian changelog" section in the Debian Policy Manual for + // details. + // + // In particular, this is the sole source of the package version. + // + timestamp now (system_clock::now ()); + + path chlog (deb / "changelog"); + try + { + ofdstream os (chlog); + + // The first line has the following format: + // + // <src-package> (<version>) <distribution>; urgency=<urgency> + // + // Note that <distribution> doesn't end up in the binary package. + // Normally all Debian packages start in unstable or experimental. + // + string urgency; + switch (pm.priority ? pm.priority->value : priority::low) + { + case priority::low: urgency = "low"; break; + case priority::medium: urgency = "medium"; break; + case priority::high: urgency = "high"; break; + case priority::security: urgency = "critical"; break; + } + + os << pn << " (" << st.system_version << ") " + << (pv.release ? "experimental" : "unstable") << "; " + << "urgency=" << urgency << '\n'; + + // Next we have a bunch of "change details" lines that start with `*` + // indented with two spaces. They are traditionally seperated from the + // first and last lines with blank lines. + // + os << '\n' + << " * New bpkg package release " << pv.string () << '.' << '\n' + << '\n'; + + // The last line is the "maintainer signoff" and has the following + // form: + // + // -- <name> <email> <date> + // + // The <date> component shall have the following form in the English + // locale (Mon, Jan, etc): + // + // <day-of-week>, <dd> <month> <yyyy> <hh>:<mm>:<ss> +<zzzz> + // + timestamp now (system_clock::now ()); + os << " -- " << maintainer << " "; + std::locale l (os.imbue (std::locale ("C"))); + to_stream (os, + now, + "%a, %d %b %Y %T %z", + false /* special */, + true /* local */); + os.imbue (l); + os << '\n'; + + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to " << chlog << ": " << e; + } + + // The copyright file. + // + // See the "Machine-readable debian/copyright file" document for + // details. + // + // Note that while not entirely clear, it looks like there should be at + // least one Files stanza. + // + // Note also that there is currently no way for us to get accurate + // copyright information. + // + // @@ TODO: Strictly speaking, in the recursive mode, we should collect + // licenses of all the dependencies we are bundling. + // + path copyr (deb / "copyright"); + try + { + ofdstream os (copyr); + + string license; + for (const licenses& ls: pm.license_alternatives) + { + if (!license.empty ()) + license += " or "; + + for (auto b (ls.begin ()), i (b); i != ls.end (); ++i) + { + if (i != b) + license += " and "; + + license += *i; + } + } + + os << "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/" << '\n' + << "Upstream-Name: " << pn << '\n' + << "Upstream-Contact: " << maintainer << '\n'; + if (!homepage.empty ()) + os << "Source: " << homepage << '\n'; + os << "License: " << license << '\n' + << "Comment: See accompanying files for exact copyright information" << '\n' + << " and full license text(s)." << '\n'; + + // Note that for licenses mentioned in the Files stanza we either have + // to provide the license text(s) inline or as separate License stanzas. + // + os << '\n' + << "Files: *" << '\n' + << "Copyright: "; + to_stream (os, now, "%Y", false /* special */, true /* local */); + os << " the " << pn << " authors (see accompanying files for details)" << '\n' + << "License: " << license << '\n' + << " See accompanying files for full license text(s)." << '\n'; + + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to " << copyr << ": " << e; + } + + // The source/format file. + // + dir_path deb_src (deb / dir_path ("source")); + mk (deb_src); + + path format (deb_src / "format"); + try + { + ofdstream os (format); + os << "3.0 (quilt)\n"; + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to " << format << ": " << e; + } + + // The rules makefile. Note that it must be executable. + // + // This file is executed by dpkg-buildpackage(1) which expects it to + // provide the following "API" make targets: + // + // clean + // + // build -- configure and build for all package + // build-arch -- configure and build for Architecture:any packages + // build-indep -- configure and build for Architecture:all packages + // + // binary -- make all binary packages + // binary-arch -- make Architecture:any binary packages + // binary-indep -- make Architecture:all binary packages + // + // The dh command sequencer provides the standard implementation of these + // API targets with the following customization point targets (for an + // overview of dh, start with the slides from the "Not Your Grandpa's + // Debhelper" presentation at DebConf 9 followed by the dh(1) man page): + // + // override_dh_auto_configure # ./configure --prefix=/usr + // override_dh_auto_build # make + // override_dh_auto_test # make test + // override_dh_auto_install # make install + // override_dh_auto_clean # make distclean + // + // Note that pretty much any dh_xxx command invoked by dh in order to + // implement the API targets can be customized with the corresponding + // override_dh_xxx target. To see what commands are executed for an API + // target, run `dh <target> --no-act`. + // + path rules (deb / "rules"); + try + { + bool lang_c (lang ("c")); + bool lang_cxx (lang ("c++")); + bool lang_cc (lang ("cc")); + + // See fdopen() for details (umask, etc). + // + permissions ps (permissions::ru | permissions::wu | permissions::xu | + permissions::rg | permissions::wg | permissions::xg | + permissions::ro | permissions::wo | permissions::xo); + ofdstream os (fdopen (rules, + fdopen_mode::out | fdopen_mode::create, + ps)); + + os << "#!/usr/bin/make -f\n" + << "# -*- makefile -*-\n" + << '\n'; + + // See debhelper(7) for details on these. + // + // Note that there is also the DEB_BUILD_OPTIONS=terse option. Perhaps + // for the "starting point" mode we should base DH_* values as well as + // the build system verbosity below on that value. See debian/rules in + // upstream mariadb for what looks like a sensible setup. + // + if (verb == 0) + os << "export DH_QUIET := 1\n" + << '\n'; + else if (verb == 1) + os << "# Uncomment this to turn on verbose mode.\n" + << "#export DH_VERBOSE := 1\n" + << '\n'; + else + os << "export DH_VERBOSE := 1\n" + << '\n'; + + // We could have instead called dpkg-architecture directly but seeing + // that we are also include buildflags.mk below, might as well use + // architecture.mk (in the packages that we sampled you see both + // approaches). Note that these come in the dpkg-dev package, the same + // as dpkg-buildpackage. + // + os << "# DEB_HOST_* (DEB_HOST_MULTIARCH, etc)" << '\n' + << "#" << '\n' + << "include /usr/share/dpkg/architecture.mk" << '\n' + << '\n'; + + if (ops_->debian_buildflags () != "ignore") + { + // While we could have called dpkg-buildflags directly, including + // buildflags.mk instead appears to be the standard practice. + // + // Note that theses flags are not limited to C-based languages (for + // example, they also cover Assembler, Fortran, and potentially others + // in the future). + // + string mo; // Include leading space if not empty. + if (ops_->debian_maint_option_specified ()) + { + for (const string& o: ops_->debian_maint_option ()) + { + if (!o.empty ()) + { + mo += ' '; + mo += o; + } + } + } + else + mo = " hardening=+all"; + + os << "# *FLAGS (CFLAGS, CXXFLAGS, etc)" << '\n' + << "#" << '\n' + << "export DEB_BUILD_MAINT_OPTIONS :=" << mo << '\n' + << "include /usr/share/dpkg/buildflags.mk" << '\n' + << '\n'; + + // Fixup -ffile-prefix-map option (if specified) which is used to + // strip source file path prefix in debug information (besides other + // places). By default it points to the source directory. We change it + // to point to the bpkg configuration directory. Note that this won't + // work for external packages with source out of configuration (e.g., + // managed by bdep). + // + if (lang_c || lang_cc) + { + // @@ TODO: OBJCFLAGS. + + os << "CFLAGS := $(patsubst -ffile-prefix-map=%,-ffile-prefix-map=" + << cfg_dir.string () << "=.,$(CFLAGS))" << '\n' + << '\n'; + } + + if (lang_cxx || lang_cc) + { + // @@ TODO: OBJCXXFLAGS. + + os << "CXXFLAGS := $(patsubst -ffile-prefix-map=%,-ffile-prefix-map=" + << cfg_dir.string () << "=.,$(CXXFLAGS))" << '\n' + << '\n'; + } + } + + // The debian/tmp/ subdirectory appears to be the canonical destination + // directory (see dh_auto_install(1) for details). + // + os << "DESTDIR := $(CURDIR)/debian/tmp" << '\n' + << '\n'; + + // Let's use absolute path to the build system driver in case we are + // invoked with altered environment or some such. + // + // See --jobs documentation in dpkg-buildpackage(1) for details on + // parallel=N. + // + // Note: should be consistent with the invocation in installed_entries() + // above. + // + cstrings verb_args; string verb_arg; + map_verb_b (*ops_, verb_b::normal, verb_args, verb_arg); + + os << "b := " << search_b (*ops_).effect_string (); + for (const char* o: verb_args) os << ' ' << o; + for (const string& o: ops_->build_option ()) os << ' ' << o; + os << '\n' + << '\n' + << "parallel := $(filter parallel=%,$(DEB_BUILD_OPTIONS))" << '\n' + << "ifneq ($(parallel),)" << '\n' + << " parallel := $(patsubst parallel=%,%,$(parallel))" << '\n' + << " ifeq ($(parallel),1)" << '\n' + << " b += --serial-stop" << '\n' + << " else" << '\n' + << " b += --jobs=$(parallel)" << '\n' + << " endif" << '\n' + << "endif" << '\n' + << '\n'; + + // Configuration variables. + // + // Note: we need to quote values that contain `<>`, `[]`, since they + // will be passed through shell. For simplicity, let's just quote + // everything. + // + os << "config := config.install.chroot='$(DESTDIR)/'" << '\n' + << "config += config.install.sudo='[null]'" << '\n'; + + // If this is a C-based language, add rpath for private installation. + // + if (priv && (lang_c || lang_cxx || lang_cc)) + os << "config += config.bin.rpath='/usr/lib/$(DEB_HOST_MULTIARCH)/" + << pn << "/'" << '\n'; + + // Add build flags. + // + if (ops_->debian_buildflags () != "ignore") + { + const string& m (ops_->debian_buildflags ()); + + string o (m == "assign" ? "=" : + m == "append" ? "+=" : + m == "prepend" ? "=+" : ""); + + if (o.empty ()) + fail << "unknown --debian-buildflags option value '" << m << "'"; + + // Note that config.cc.* doesn't play well with the append/prepend + // modes because the orders are: + // + // x.poptions cc.poptions + // cc.coptions x.coptions + // cc.loptions x.loptions + // + // Oh, well, hopefully it will be close enough for most cases. + // + // Note also that there are compiler mode options that are not + // overridden. + // + if (o == "=" && (lang_c || lang_cxx || lang_cc)) + { + os << "config += config.cc.poptions='[null]'" << '\n' + << "config += config.cc.coptions='[null]'" << '\n' + << "config += config.cc.loptions='[null]'" << '\n'; + } + + if (lang_c || lang_cc) + { + // @@ TODO: OBJCFLAGS (we currently don't have separate options). + // Also see -ffile-prefix-map fixup above. + + os << "config += config.c.poptions" << o << "'$(CPPFLAGS)'" << '\n' + << "config += config.c.coptions" << o << "'$(CFLAGS)'" << '\n' + << "config += config.c.loptions" << o << "'$(LDFLAGS)'" << '\n'; + } + + if (lang_cxx || lang_cc) + { + // @@ TODO: OBJCXXFLAGS (we currently don't have separate options). + // Also see -ffile-prefix-map fixup above. + + os << "config += config.cxx.poptions" << o << "'$(CPPFLAGS)'" << '\n' + << "config += config.cxx.coptions" << o << "'$(CXXFLAGS)'" << '\n' + << "config += config.cxx.loptions" << o << "'$(LDFLAGS)'" << '\n'; + } + + // @@ TODO: ASFLAGS (when we have assembler support). + } + + // Keep last to allow user-specified configuration variables to override + // anything. + // + for (const string& c: config) + { + // Quote the value unless already quoted (see above). Presense of + // potentially-quoted user variables complicates things a bit (can + // be partially quoted, double-quoted, etc). + // + size_t p (c.find_first_of ("=+ \t")); // End of name. + if (p != string::npos) + { + p = c.find_first_not_of ("=+ \t", p); // Beginning of value. + if (p != string::npos) + { + if (c.find_first_of ("'\"", p) == string::npos) // Not quoted. + { + os << "config += " << string (c, 0, p) << '\'' + << string (c, p) << "'\n"; + continue; + } + } + } + + os << "config += " << c << '\n'; + } + + os << '\n'; + + // List of packages we need to install. + // + for (auto b (pkgs.begin ()), i (b); i != pkgs.end (); ++i) + { + os << "packages" << (i == b ? " := " : " += ") + << i->out_root.representation () << '\n'; + } + os << '\n'; + + // Disable synchronization hooks for good measure. + // + os << "export BDEP_SYNC := 0\n" + << '\n'; + + // Default to the dh command sequencer. + // + // Note that passing --buildsystem=none doesn't seem to make any + // difference (other than add some noise). + // + os << "%:\n" + << '\t' << "dh $@" << '\n' + << '\n'; + + // Override dh_auto_configure. + // + os << "# Everything is already configured.\n" + << "#\n" + << "override_dh_auto_configure:\n" + << '\n'; + + // Override dh_auto_build. + // + os << "override_dh_auto_build:\n" + << '\t' << "$b $(config) update-for-install: $(packages)" << '\n' + << '\n'; + + // Override dh_auto_test. + // + // Note that running tests after update-for-install may cause rebuild + // (e.g., relinking without rpath, etc) before tests and again before + // install. So doesn't seem worth the trouble. + // + os << "# Assume any testing has already been done.\n" + << "#\n" + << "override_dh_auto_test:\n" + << '\n'; + + // Override dh_auto_install. + // + os << "override_dh_auto_install:\n" + << '\t' << "$b $(config) '!config.install.scope=" << scope << "' " + << "install: $(packages)" << '\n' + << '\n'; + + // Override dh_auto_clean. + // + os << "# This is not a real source directory so nothing to clean.\n" + << "#\n" + << "override_dh_auto_clean:\n" + << '\n'; + + // Override dh_shlibdeps. + // + // Failed that we get a warning about calculated ${shlibs:Depends} being + // unused. + // + // Note that there is also dh_makeshlibs which is invoked just before + // but we shouldn't override it because (quoting its man page): "It will + // also ensure that ldconfig is invoked during install and removal when + // it finds shared libraries." + // + os << "# Disable dh_shlibdeps since we don't use ${shlibs:Depends}.\n" + << "#\n" + << "override_dh_shlibdeps:\n" + << '\n'; + + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to " << rules << ": " << e; + } + + // Generate the dh_install (.install) config files for each package in + // order to sort out which files belong where. + // + // For documentation of the config file format see debhelper(1) and + // dh_install(1). But the summary is: + // + // - Supports only simple wildcards (?, *, [...]; no recursive/**). + // - But can install whole directories recursively. + // - An entry that doesn't match anything is an error (say, /usr/sbin/*). + // - Supports variable substitutions (${...}; since compat level 13). + // + // Keep in mind that wherever there is <project> in the config.install.* + // variable, we can end up with multiple different directories (bundled + // packages). + // + path main_install; + path dev_install; + path doc_install; + path dbg_install; + path common_install; + + const path* cur_install (nullptr); // File being opened/written to. + try + { + pair<path&, ofdstream> main (main_install, auto_fd ()); + pair<path&, ofdstream> dev (dev_install, auto_fd ()); + pair<path&, ofdstream> doc (doc_install, auto_fd ()); + pair<path&, ofdstream> dbg (dbg_install, auto_fd ()); + pair<path&, ofdstream> com (common_install, auto_fd ()); + + auto open = [&deb, &cur_install] (pair<path&, ofdstream>& os, + const string& n) + { + if (!n.empty ()) + { + cur_install = &(os.first = deb / (n + ".install")); + os.second.open (os.first); + } + }; + + open (main, st.main); + open (dev, st.dev); + open (doc, st.doc); + open (dbg, st.dbg); + open (com, st.common); + + auto is_open = [] (pair<path&, ofdstream>& os) + { + return os.second.is_open (); + }; + + auto add = [&cur_install] (pair<path&, ofdstream>& os, const path& p) + { + // Strip root. + // + string s (p.leaf (p.root_directory ()).string ()); + + // Replace () with {}. + // + for (char& c: s) + { + if (c == '(') c = '{'; + if (c == ')') c = '}'; + } + + cur_install = &os.first; + os.second << s << '\n'; + }; + + // Let's tighten things up and only look in <private>/ (if specified) to + // make sure there is nothing stray. + // + string pd (priv ? pn.string () + '/' : ""); + + // NOTE: keep consistent with the config.install.* values above. + // + dir_path bindir ("/usr/bin/"); + dir_path sbindir ("/usr/sbin/"); + dir_path etcdir ("/etc/"); + dir_path incdir ("/usr/include/" + pd); + dir_path incarchdir ("/usr/include/$(DEB_HOST_MULTIARCH)/" + pd); + dir_path libdir ("/usr/lib/$(DEB_HOST_MULTIARCH)/" + pd); + dir_path pkgdir (libdir / dir_path ("pkgconfig")); + dir_path sharedir ("/usr/share/" + pd); + dir_path docdir ("/usr/share/doc/" + pd); + dir_path mandir ("/usr/share/man/"); + + // The main package contains everything that doesn't go to another + // packages. + // + if (ies.contains_sub (bindir)) add (main, bindir / "*"); + if (ies.contains_sub (sbindir)) add (main, sbindir / "*"); + + // This could potentially go to -common but it could also be target- + // specific, who knows. So let's keep it in main for now. + // + if (ies.contains_sub (etcdir)) add (main, etcdir / "*"); + + if (!is_open (dev)) + { + if (ies.contains_sub (incdir)) add (main, incdir / "*"); + if (ies.contains_sub (incarchdir)) add (main, incarchdir / "*"); + if (ies.contains_sub (libdir)) add (main, libdir / "*"); + } + else + { + if (ies.contains_sub (incdir)) add (dev, incdir / "*"); + if (ies.contains_sub (incarchdir)) add (dev, incarchdir / "*"); + + // Ok, time for things to get hairy: we need to split the contents + // of lib/ into the main and -dev packages. The -dev package should + // contain three things: + // + // 1. Static libraries (.a). + // 2. Non-versioned shared library symlinks (.so). + // 3. Contents of the pkgconfig/ subdirectory. + // + // Everything else should go into the main package. In particular, we + // assume any subdirectories other than pkgconfig/ are the libexec + // stuff or similar. + // + // The (2) case (shared library) is tricky. Here we can have three + // plausible arrangements: + // + // A. Portably-versioned library: + // + // libfoo-1.2.so + // libfoo.so -> libfoo-1.2.so + // + // B. Natively-versioned library: + // + // libfoo.so.1.2.3 + // libfoo.so.1.2 -> libfoo.so.1.2.3 + // libfoo.so.1 -> libfoo.so.1.2 + // libfoo.so -> libfoo.so.1 + // + // C. Non-versioned library: + // + // libfoo.so + // + // Note that in the (C) case the library should go into the main + // package. Based on this, the criteria appears to be straightforwrad: + // the extension is .so and it's a symlink. For good measure we also + // check that there is the `lib` prefix (plugins, etc). + // + for (auto p (ies.find_sub (libdir)); p.first != p.second; ) + { + const path& f (p.first->first); + const installed_entry& ie ((p.first++)->second); + + path l (f.leaf (libdir)); + + if (l.simple ()) + { + string e (l.extension ()); + const string& n (l.string ()); + + bool d (n.size () > 3 && n.compare (0, 3, "lib") == 0 && + ((e == "a" ) || + (e == "so" && ie.target != nullptr))); + + add (d ? dev : main, libdir / l); + } + else + { + // Let's keep things tidy and use a wildcard rather than listing + // all the entries in subdirectories verbatim. + // + dir_path d (libdir / dir_path (*l.begin ())); + + add (d == pkgdir ? dev : main, d / "*"); + + // Skip all the other entries in this subdirectory (in the prefix + // map they will all be in a contiguous range). + // + while (p.first != p.second && p.first->first.sub (d)) + ++p.first; + } + } + } + + // We cannot just do usr/share/* since it will clash with doc/ and man/ + // below. So we have to list all the top-level entries in usr/share/ + // that are not doc/ or man/. + // + for (auto p (ies.find_sub (sharedir)); p.first != p.second; ) + { + const path& f ((p.first++)->first); + + if (f.sub (docdir) || f.sub (mandir)) + continue; + + path l (f.leaf (sharedir)); + + if (l.simple ()) + add (is_open (com) ? com : main, sharedir / l); + else + { + // Let's keep things tidy and use a wildcard rather than listing all + // the entries in subdirectories verbatim. + // + dir_path d (sharedir / dir_path (*l.begin ())); + + add (is_open (com) ? com : main, d / "*"); + + // Skip all the other entries in this subdirectory (in the prefix + // map they will all be in a contiguous range). + // + while (p.first != p.second && p.first->first.sub (d)) + ++p.first; + } + } + + // Should we put the documentation into -common if there is no -doc? + // While there doesn't seem to be anything explicit in the policy, there + // are packages that do it this way (e.g., libao, libaudit). And the + // same logic seems to apply to -dev (e.g., zlib). + // + { + auto& os (is_open (doc) ? doc : + is_open (com) ? com : + is_open (dev) ? dev : + main); + + if (ies.contains_sub (docdir)) add (os, docdir / "*"); + if (ies.contains_sub (mandir)) add (os, mandir / "*"); + } + + // Close. + // + auto close = [&cur_install] (pair<path&, ofdstream>& os) + { + if (os.second.is_open ()) + { + cur_install = &os.first; + os.second.close (); + } + }; + + close (main); + close (dev); + close (doc); + close (dbg); + close (com); + } + catch (const io_error& e) + { + fail << "unable to write to " << *cur_install << ": " << e; + } + + // Run dpkg-buildpackage. + // + // Note that there doesn't seem to be any way to control its verbosity or + // progress. + // + // Note also that dpkg-buildpackage causes recompilation on every run by + // changing the SOURCE_DATE_EPOCH environment variable (which we track for + // changes since it affects GCC). Note that since we don't have this + // SOURCE_DATE_EPOCH during dry-run caused by installed_entries(), there + // would be a recompilation even if the value weren't changing. + // + cstrings args { + "dpkg-buildpackage", + "--build=binary", // Only build binary packages. + "--no-sign", // Do not sign anything. + "--target-arch", arch.c_str ()}; + + // Pass our --jobs value, if any. + // + string jobs_arg; + if (size_t n = ops_->jobs_specified () ? ops_->jobs () : 0) + { + // Note: only accepts the --jobs=N form. + // + args.push_back ((jobs_arg = "--jobs=" + to_string (n)).c_str ()); + } + + // Pass any additional options specified by the user. + // + for (const string& o: ops_->debian_build_option ()) + args.push_back (o.c_str ()); + + args.push_back (nullptr); + + if (ops_->debian_prepare_only ()) + { + if (verb >= 1) + { + diag_record dr (text); + + dr << "prepared " << src << + text << "command line: "; + + print_process (dr, args); + } + + return paths {}; + } + + try + { + process_path pp (process::path_search (args[0])); + process_env pe (pp, src /* cwd */); + + // There is going to be quite a bit of diagnostics so print the command + // line unless quiet. + // + if (verb >= 1) + print_process (pe, args); + + // Redirect stdout to stderr since half of dpkg-buildpackage diagnostics + // goes there. For good measure also redirect stdin to /dev/null to make + // sure there are no prompts of any kind. + // + process pr (pp, + args, + -2 /* stdin */, + 2 /* stdout */, + 2 /* stderr */, + pe.cwd->string ().c_str (), + pe.vars); + + if (!pr.wait ()) + { + // Let's repeat the command line even if it was printed at the + // beginning to save the user a rummage through the logs. + // + diag_record dr (fail); + dr << args[0] << " exited with non-zero code" << + info << "command line: "; print_process (dr, pe, args); + } + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + // Cleanup intermediate files unless requested not to. + // + if (!ops_->keep_output ()) + { + rm_r (src); + } + + // Collect and return the binary package paths. + // + paths r; + auto add = [&out, &r] (const string& n, bool opt = false) + { + path p (out / n); + + if (exists (p)) + r.push_back (move (p)); + else if (!opt) + fail << "expected output file " << p << " does not exist"; + }; + + // The resulting .deb file names have the <name>_<version>_<arch>.deb + // form. If the package is architecture-independent, then <arch> is the + // special `all` value. + // + const string& ver (st.system_version); + + add (st.main + '_' + ver + '_' + arch + ".deb"); + add (st.main + "-dbgsym_" + ver + '_' + arch + ".deb", true); + if (!st.dev.empty ()) add (st.dev + '_' + ver + '_' + arch + ".deb"); + if (!st.doc.empty ()) add (st.doc + '_' + ver + "_all.deb"); + if (!st.common.empty ()) add (st.common + '_' + ver + "_all.deb"); + + // Besides the binary packages (.deb) we also get the .buildinfo and + // .changes files, which could be useful. Note that their names are based + // on the source package name. + // + add (pn.string () + '_' + ver + '_' + arch + ".buildinfo"); + add (pn.string () + '_' + ver + '_' + arch + ".changes"); + + return r; } } diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx index 0186b76..cf7b1c3 100644 --- a/bpkg/system-package-manager-debian.hxx +++ b/bpkg/system-package-manager-debian.hxx @@ -18,15 +18,15 @@ namespace bpkg // consumption and the dpkg-buildpackage/debhelper/dh tooling for // production. // - // NOTE: the below description is also reproduced in the bpkg manual. + // NOTE: THE BELOW DESCRIPTION IS ALSO REPRODUCED IN THE BPKG MANUAL. // // For background, a library in Debian is normally split up into several // packages: the shared library package (e.g., libfoo1 where 1 is the ABI // version), the development files package (e.g., libfoo-dev), the - // documentation files package (e.g., libfoo-doc), the debug symbols - // package (e.g., libfoo1-dbg), and the architecture-independent files - // (e.g., libfoo1-common). All the packages except -dev are optional - // and there is quite a bit of variability here. Here are a few examples: + // documentation files package (e.g., libfoo-doc), the debug symbols package + // (e.g., libfoo1-dbg), and the (usually) architecture-independent files + // (e.g., libfoo1-common). All the packages except -dev are optional and + // there is quite a bit of variability here. Here are a few examples: // // libsqlite3-0 libsqlite3-dev // @@ -39,6 +39,10 @@ namespace bpkg // Note that while most library package names in Debian start with lib (per // the policy), there are exceptions (e.g., zlib1g zlib1g-dev). // + // Also note that manual -dbg packages are obsolete in favor of automatic + // -dbgsym packages from Debian 9. So while we support -dbg for consumption, + // we only generate -dbgsym. + // // Based on that, it seems our best bet when trying to automatically map our // library package name to Debian package names is to go for the -dev // package first and figure out the shared library package from that based @@ -129,11 +133,14 @@ namespace bpkg virtual void pkg_install (const vector<package_name>&) override; - virtual void - generate (packages&&, - packages&&, - strings&&, + virtual paths + generate (const packages&, + const packages&, + const strings&, const dir_path&, + const package_manifest&, + const string&, + const small_vector<language, 1>&, optional<recursive_mode>) override; public: @@ -161,14 +168,19 @@ namespace bpkg yes, move (sudo)) {} + // Note: options can only be NULL when testing functions that don't need + // them. + // system_package_manager_debian (bpkg::os_release&& osr, const target_triplet& h, string a, - optional<bool> progress) + optional<bool> progress, + const pkg_bindist_options* ops) : system_package_manager (move (osr), h, a.empty () ? arch_from_target (h) : move (a), - progress) {} + progress), + ops_ (ops) {} // Implementation details exposed for testing (see definitions for // documentation). @@ -193,7 +205,7 @@ namespace bpkg apt_get_common (const char*, strings& args_storage); static package_status - parse_name_value (const package_name&, const string&, bool, bool); + parse_name_value (const string&, const string&, bool, bool); static string main_from_dev (const string&, const string&, const string&); @@ -201,6 +213,12 @@ namespace bpkg static string arch_from_target (const target_triplet&); + package_status + map_package (const package_name&, + const version&, + const available_packages&, + const optional<string>&) const; + // If simulate is not NULL, then instead of executing the actual apt-cache // and apt-get commands simulate their execution: (1) for apt-cache by // printing their command lines and reading the results from files @@ -233,11 +251,17 @@ namespace bpkg const simulation* simulate_ = nullptr; - protected: + private: + optional<system_package_status_debian> + status (const package_name&, const available_packages&); + + private: bool fetched_ = false; // True if already fetched metadata. bool installed_ = false; // True if already installed. std::map<package_name, optional<system_package_status_debian>> status_cache_; + + const pkg_bindist_options* ops_ = nullptr; // Only for production. }; } diff --git a/bpkg/system-package-manager-debian.test.cxx b/bpkg/system-package-manager-debian.test.cxx index 3e71ad2..df5275d 100644 --- a/bpkg/system-package-manager-debian.test.cxx +++ b/bpkg/system-package-manager-debian.test.cxx @@ -36,6 +36,8 @@ namespace bpkg // // main-from-dev <dev-pkg> <dev-ver> depends comes from stdin // + // map-package [<build-metadata>] manifest comes from stdin + // // build <query-pkg>... [--install [--no-fetch] <install-pkg>...] // // The stdin of the build command is used to read the simulation description @@ -135,12 +137,13 @@ namespace bpkg assert (argc == 3); // <pkg> package_name pn (argv[2]); + string pt (package_manifest::effective_type (nullopt, pn)); string v; getline (cin, v); package_status s ( - system_package_manager_debian::parse_name_value (pn, v, false, false)); + system_package_manager_debian::parse_name_value (pt, v, false, false)); if (!s.main.empty ()) cout << "main: " << s.main << '\n'; if (!s.dev.empty ()) cout << "dev: " << s.dev << '\n'; @@ -166,6 +169,35 @@ namespace bpkg cout << system_package_manager_debian::main_from_dev (n, v, d) << '\n'; } + else if (cmd == "map-package") + { + assert (argc >= 2 && argc <= 3); // [<build-metadata>] + + optional<string> bm; + if (argc > 2) + bm = argv[2]; + + available_packages aps; + aps.push_back (make_available_from_manifest ("", "-")); + + const package_name& n (aps.front ().first->id.name); + const version& v (aps.front ().first->version); + + system_package_manager_debian m (move (osr), + host_triplet, + "" /* arch */, + nullopt /* progress */, + nullptr /* options */); + + package_status s (m.map_package (n, v, aps, bm)); + + cout << "version: " << s.system_version << '\n' + << "main: " << s.main << '\n'; + if (!s.dev.empty ()) cout << "dev: " << s.dev << '\n'; + if (!s.doc.empty ()) cout << "doc: " << s.doc << '\n'; + if (!s.dbg.empty ()) cout << "dbg: " << s.dbg << '\n'; + if (!s.common.empty ()) cout << "common: " << s.common << '\n'; + } else if (cmd == "build") { assert (argc >= 3); // <query-pkg>... diff --git a/bpkg/system-package-manager-debian.test.testscript b/bpkg/system-package-manager-debian.test.testscript index b1a0030..56c6785 100644 --- a/bpkg/system-package-manager-debian.test.testscript +++ b/bpkg/system-package-manager-debian.test.testscript @@ -222,6 +222,196 @@ EOI } +: map-package +: +{ + test.arguments += map-package + + : default-name + : + $* <<EOI >>EOO + : 1 + name: byacc + version: 20210808 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808-0~debian10 + main: byacc + EOO + + : default-name-lib + : + $* <<EOI >>EOO + : 1 + name: libsqlite3 + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite3 + dev: libsqlite3-dev + EOO + + : custom-name + : + $* <<EOI >>EOO + : 1 + name: libsqlite3 + debian_9-name: libsqlite3-0 libsqlite3-dev + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite3-0 + dev: libsqlite3-dev + EOO + + : custom-name-dev-only + : + $* <<EOI >>EOO + : 1 + name: libsqlite3 + debian_9-name: libsqlite3-0-dev + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite3-0 + dev: libsqlite3-0-dev + EOO + + : custom-name-non-native + : + $* <<EOI >>EOO + : 1 + name: libsqlite3 + debian_0-name: libsqlite libsqlite-dev + debian_9-name: libsqlite3-0 libsqlite3-dev + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite + dev: libsqlite-dev + EOO + + : version-upstream + : + $* <<EOI >>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + upstream-version: 20210808 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-3~debian10 + main: byacc + EOO + + : version-distribution + : + $* <<EOI >>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~beta.1 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-0~debian10 + main: byacc + EOO + + : version-distribution-epoch-revision + : + $* <<EOI >>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 1:1.2.3-2 + summary: yacc parser generator + license: other: public domain + EOI + version: 1:1.2.3-2~debian10 + main: byacc + EOO + + : version-distribution-empty-release + : + $* <<EOI >>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~-4 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-4~debian10 + main: byacc + EOO + + : version-distribution-empty-revision + : + $* <<EOI >>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~b.1- + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~b.1-3~debian10 + main: byacc + EOO + + : version-distribution-empty-release-revision + : + $* <<EOI >>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~- + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-3~debian10 + main: byacc + EOO + + : version-no-build-metadata + : + $* '' <<EOI >>EOO + : 1 + name: byacc + version: 1.2.3 + summary: yacc parser generator + license: other: public domain + EOI + version: 1.2.3 + main: byacc + EOO + + : version-distribution-no-build-metadata + : + $* '' <<EOI >>EOO + : 1 + name: byacc + version: 1.2.3 + debian-version: 20210808 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808 + main: byacc + EOO +} + : build : { diff --git a/bpkg/system-package-manager-fedora.cxx b/bpkg/system-package-manager-fedora.cxx index 3178e4e..576e0ef 100644 --- a/bpkg/system-package-manager-fedora.cxx +++ b/bpkg/system-package-manager-fedora.cxx @@ -22,7 +22,8 @@ namespace bpkg c; } - // Parse the fedora-name (or alike) value. + // Parse the fedora-name (or alike) value. The first argument is the package + // type. // // Note that for now we treat all the packages from the non-main groups as // extras omitting the -common package (assuming it's pulled by the main @@ -31,7 +32,7 @@ namespace bpkg // we can't know whether the static library is needed or not). // package_status system_package_manager_fedora:: - parse_name_value (const package_name& pn, + parse_name_value (const string& pt, const string& nv, bool extra_doc, bool extra_debuginfo, @@ -52,8 +53,7 @@ namespace bpkg return nn > sn && n.compare (nn - sn, sn, s) == 0; }; - auto parse_group = [&split, &suffix] (const string& g, - const package_name* pn) + auto parse_group = [&split, &suffix] (const string& g, const string* pt) { strings ns (split (g, ' ')); @@ -64,8 +64,6 @@ namespace bpkg // Handle the "devel instead of main" special case for libraries. // - // Note: the lib prefix check is based on the bpkg package name. - // // Check that the following name does not end with -devel. This will be // the only way to disambiguate the case where the library name happens // to end with -devel (e.g., libfoo-devel libfoo-devel-devel). @@ -73,10 +71,9 @@ namespace bpkg { string& m (ns[0]); - if (pn != nullptr && - pn->string ().compare (0, 3, "lib") == 0 && - pn->string ().size () > 3 && - suffix (m, "-devel") && + if (pt != nullptr && + *pt == "lib" && + suffix (m, "-devel") && !(ns.size () > 1 && suffix (ns[1], "-devel"))) { r = package_status ("", move (m)); @@ -120,7 +117,7 @@ namespace bpkg for (size_t i (0); i != gs.size (); ++i) { if (i == 0) // Main group. - r = parse_group (gs[i], &pn); + r = parse_group (gs[i], &pt); else { package_status g (parse_group (gs[i], nullptr)); @@ -1079,15 +1076,6 @@ namespace bpkg optional<const system_package_status*> system_package_manager_fedora:: pkg_status (const package_name& pn, const available_packages* aps) { - // For now we ignore -doc and -debug* package components (but we may want - // to have options controlling this later). Note also that we assume - // -common is pulled automatically by the base package so we ignore it as - // well (see equivalent logic in parse_name_value()). - // - bool need_doc (false); - bool need_debuginfo (false); - bool need_debugsource (false); - // First check the cache. // { @@ -1100,6 +1088,26 @@ namespace bpkg return nullopt; } + optional<package_status> r (status (pn, *aps)); + + // Cache. + // + auto i (status_cache_.emplace (pn, move (r)).first); + return i->second ? &*i->second : nullptr; + } + + optional<package_status> system_package_manager_fedora:: + status (const package_name& pn, const available_packages& aps) + { + // For now we ignore -doc and -debug* package components (but we may want + // to have options controlling this later). Note also that we assume + // -common is pulled automatically by the base package so we ignore it as + // well (see equivalent logic in parse_name_value()). + // + bool need_doc (false); + bool need_debuginfo (false); + bool need_debugsource (false); + vector<package_status> candidates; // Translate our package name to the Fedora package names. @@ -1112,12 +1120,25 @@ namespace bpkg << " package name"; }); + // Without explicit type, the best we can do in trying to detect whether + // this is a library is to check for the lib prefix. Libraries without + // the lib prefix and non-libraries with the lib prefix (both of which + // we do not recomment) will have to provide a manual mapping. + // + // Note that using the first (latest) available package as a source of + // type information seems like a reasonable choice. + // + const string& pt (!aps.empty () + ? aps.front ().first->effective_type () + : package_manifest::effective_type (nullopt, pn)); + strings ns; - if (!aps->empty ()) - ns = system_package_names (*aps, + if (!aps.empty ()) + ns = system_package_names (aps, os_release.name_id, os_release.version_id, - os_release.like_ids); + os_release.like_ids, + true /* native */); if (ns.empty ()) { // Attempt to automatically translate our package name. Failed that we @@ -1126,23 +1147,18 @@ namespace bpkg const string& n (pn.string ()); // Note that theoretically different available packages can have - // different project names. But taking it form the latest version + // different project names. But taking it from the latest version // feels good enough. // - const shared_ptr<available_package>& ap (!aps->empty () - ? aps->front ().first + const shared_ptr<available_package>& ap (!aps.empty () + ? aps.front ().first : nullptr); string f (ap != nullptr && ap->project && *ap->project != pn ? ap->project->string () : empty_string); - // The best we can do in trying to detect whether this is a library is - // to check for the lib prefix. Libraries without the lib prefix and - // non-libraries with the lib prefix (both of which we do not - // recomment) will have to provide a manual mapping. - // - if (n.compare (0, 3, "lib") == 0 && n.size () > 3) + if (pt == "lib") { // If there is no project name let's try to use the package name // with the lib prefix stripped as a fallback. Note that naming @@ -1168,7 +1184,7 @@ namespace bpkg // for (const string& n: ns) { - package_status s (parse_name_value (pn, + package_status s (parse_name_value (pt, n, need_doc, need_debuginfo, @@ -1583,9 +1599,9 @@ namespace bpkg string sv (r->system_version, 0, r->system_version.rfind ('-')); optional<version> v; - if (!aps->empty ()) + if (!aps.empty ()) v = downstream_package_version (sv, - *aps, + aps, os_release.name_id, os_release.version_id, os_release.like_ids); @@ -1618,10 +1634,7 @@ namespace bpkg r->version = move (*v); } - // Cache. - // - auto i (status_cache_.emplace (pn, move (r)).first); - return i->second ? &*i->second : nullptr; + return r; } void system_package_manager_fedora:: @@ -1782,12 +1795,20 @@ namespace bpkg } } - void system_package_manager_fedora:: - generate (packages&&, - packages&&, - strings&&, + paths system_package_manager_fedora:: + generate (const packages&, + const packages&, + const strings&, const dir_path&, + const package_manifest&, + const string&, + const small_vector<language, 1>&, optional<recursive_mode>) { + // @@ TODO: make sure --output-root is not specified or matched the + // rpm standard directory. + + paths r; + return r; } } diff --git a/bpkg/system-package-manager-fedora.hxx b/bpkg/system-package-manager-fedora.hxx index 6c72b81..8da8863 100644 --- a/bpkg/system-package-manager-fedora.hxx +++ b/bpkg/system-package-manager-fedora.hxx @@ -16,7 +16,7 @@ namespace bpkg // The system package manager implementation for Fedora and alike (Red Hat // Enterprise Linux, CentOS, etc) using the DNF frontend. // - // NOTE: the below description is also reproduced in the bpkg manual. + // NOTE: THE BELOW DESCRIPTION IS ALSO REPRODUCED IN THE BPKG MANUAL. // // For background, a library in Fedora is normally split up into several // packages: the shared library package (e.g., libfoo), the development @@ -196,11 +196,14 @@ namespace bpkg virtual void pkg_install (const vector<package_name>&) override; - virtual void - generate (packages&&, - packages&&, - strings&&, + virtual paths + generate (const packages&, + const packages&, + const strings&, const dir_path&, + const package_manifest&, + const string&, + const small_vector<language, 1>&, optional<recursive_mode>) override; public: @@ -263,7 +266,7 @@ namespace bpkg strings& args_storage); static package_status - parse_name_value (const package_name&, const string&, bool, bool, bool); + parse_name_value (const string&, const string&, bool, bool, bool); static string main_from_devel (const string&, @@ -327,7 +330,11 @@ namespace bpkg const simulation* simulate_ = nullptr; - protected: + private: + optional<system_package_status_fedora> + status (const package_name&, const available_packages&); + + private: bool fetched_ = false; // True if already fetched metadata. bool installed_ = false; // True if already installed. diff --git a/bpkg/system-package-manager-fedora.test.cxx b/bpkg/system-package-manager-fedora.test.cxx index 11969b3..75b4642 100644 --- a/bpkg/system-package-manager-fedora.test.cxx +++ b/bpkg/system-package-manager-fedora.test.cxx @@ -151,13 +151,14 @@ namespace bpkg assert (argc == 3); // <pkg> package_name pn (argv[2]); + string pt (package_manifest::effective_type (nullopt, pn)); string v; getline (cin, v); package_status s ( system_package_manager_fedora::parse_name_value ( - pn, v, false, false, false)); + pt, v, false, false, false)); if (!s.main.empty ()) cout << "main: " << s.main << '\n'; if (!s.devel.empty ()) cout << "devel: " << s.devel << '\n'; diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx index 2ec7a60..793dec6 100644 --- a/bpkg/system-package-manager.cxx +++ b/bpkg/system-package-manager.cxx @@ -7,12 +7,15 @@ #include <libbutl/regex.hxx> #include <libbutl/semantic-version.hxx> +#include <libbutl/json/parser.hxx> #include <bpkg/package.hxx> #include <bpkg/package-odb.hxx> #include <bpkg/database.hxx> #include <bpkg/diagnostics.hxx> +#include <bpkg/pkg-bindist-options.hxx> + #include <bpkg/system-package-manager-debian.hxx> #include <bpkg/system-package-manager-fedora.hxx> @@ -122,15 +125,15 @@ namespace bpkg } unique_ptr<system_package_manager> - make_production_system_package_manager (const common_options& co, + make_production_system_package_manager (const pkg_bindist_options& o, const target_triplet& host, const string& name, const string& arch) { // Note: similar to make_production_system_package_manager() above. - optional<bool> progress (co.progress () ? true : - co.no_progress () ? false : + optional<bool> progress (o.progress () ? true : + o.no_progress () ? false : optional<bool> ()); unique_ptr<system_package_manager> r; @@ -152,7 +155,7 @@ namespace bpkg os.like_ids.push_back ("debian"); r.reset (new system_package_manager_debian ( - move (os), host, arch, progress)); + move (os), host, arch, progress, &o)); } else if (is_or_like (os, "fedora") || is_or_like (os, "rhel") || @@ -210,18 +213,19 @@ namespace bpkg // Parse the <distribution> component of the specified <distribution>-* // value into the distribution name and version (return as "0" if not - // present). Issue diagnostics and fail on parsing errors. + // present). Leave in the d argument the string representation of the + // version (used to detect the special non-native <name>_0). Issue + // diagnostics and fail on parsing errors. // // Note: the value_name, ap, and af arguments are only used for diagnostics. // static pair<string, semantic_version> - parse_distribution (string&& d, + parse_distribution (string& d, // <name>[_<version>] const string& value_name, const shared_ptr<available_package>& ap, const lazy_shared_ptr<repository_fragment>& af) { - string dn (move (d)); // <name>[_<version>] - size_t p (dn.rfind ('_')); // Version-separating underscore. + size_t p (d.rfind ('_')); // Version-separating underscore. // If the '_' separator is present, then make sure that the right-hand // part looks like a version (not empty and only contains digits and @@ -229,11 +233,11 @@ namespace bpkg // if (p != string::npos) { - if (p != dn.size () - 1) + if (p != d.size () - 1) { - for (size_t i (p + 1); i != dn.size (); ++i) + for (size_t i (p + 1); i != d.size (); ++i) { - if (!digit (dn[i]) && dn[i] != '.') + if (!digit (d[i]) && d[i] != '.') { p = string::npos; break; @@ -246,36 +250,43 @@ namespace bpkg // Parse the distribution version if present and leave it "0" otherwise. // + string dn; semantic_version dv (0, 0, 0); if (p != string::npos) - try { - dv = semantic_version (dn, - p + 1, - semantic_version::allow_omit_minor); + dn.assign (d, 0, p); + d.erase (0, p + 1); - dn.resize (p); - } - catch (const invalid_argument& e) - { - // Note: the repository fragment may have no database associated when - // used in tests. - // - shared_ptr<repository_fragment> f (af.get_eager ()); - database* db (!(f != nullptr && !af.loaded ()) // Not transient? - ? &af.database () - : nullptr); + try + { + dv = semantic_version (d, semantic_version::allow_omit_minor); + } + catch (const invalid_argument& e) + { + // Note: the repository fragment may have no database associated when + // used in tests. + // + shared_ptr<repository_fragment> f (af.get_eager ()); + database* db (!(f != nullptr && !af.loaded ()) // Not transient? + ? &af.database () + : nullptr); - diag_record dr (fail); - dr << "invalid distribution version '" << string (dn, p + 1) - << "' in value " << value_name << " for package " << ap->id.name - << ' ' << ap->version; + diag_record dr (fail); + dr << "invalid distribution version '" << d << "' in value " + << value_name << " for package " << ap->id.name << ' ' + << ap->version; - if (db != nullptr) - dr << *db; + if (db != nullptr) + dr << *db; - dr << " in repository " << (f != nullptr ? f : af.load ())->location - << ": " << e; + dr << " in repository " << (f != nullptr ? f : af.load ())->location + << ": " << e; + } + } + else + { + dn = move (d); + d.clear (); } return make_pair (move (dn), move (dv)); @@ -285,7 +296,8 @@ namespace bpkg system_package_names (const available_packages& aps, const string& name_id, const string& version_id, - const vector<string>& like_ids) + const vector<string>& like_ids, + bool native) { assert (!aps.empty ()); @@ -297,7 +309,8 @@ namespace bpkg // if not present) is less or equal the specified distribution version. // Suppress duplicate values. // - auto name_values = [&aps] (const string& n, const semantic_version& v) + auto name_values = [&aps, native] (const string& n, + const semantic_version& v) { strings r; @@ -319,13 +332,32 @@ namespace bpkg if (optional<string> d = dv.distribution ("-name")) { pair<string, semantic_version> dnv ( - parse_distribution (move (*d), dv.name, ap, a.second)); + parse_distribution (*d, dv.name, ap, a.second)); - if (dnv.first == n && dnv.second <= v) + // Skip <name>_0 if we are only interested in the native mappings. + // If we are interested in the non-native mapping, then we treat + // <name>_0 as the matching version. + // + bool nn (*d == "0"); + if (nn && native) + continue; + + semantic_version& dvr (dnv.second); + + if (dnv.first == n && (nn || dvr <= v)) { // Add the name/version pair to the sorted vector. // - name_version nv (make_pair (dv.value, move (dnv.second))); + // If this is the non-native mapping, then return just that. + // + if (nn) + { + r.clear (); // Drop anything we have accumulated so far. + r.push_back (move (dv.value)); + return r; + } + + name_version nv (make_pair (dv.value, move (dvr))); nvs.insert (upper_bound (nvs.begin (), nvs.end (), nv, [] (const name_version& x, @@ -374,6 +406,89 @@ namespace bpkg return r; } + optional<string> system_package_manager:: + system_package_version (const shared_ptr<available_package>& ap, + const lazy_shared_ptr<repository_fragment>& af, + const string& name_id, + const string& version_id, + const vector<string>& like_ids) + { + semantic_version vid (parse_version_id (version_id, name_id)); + + // Iterate over the <name>[_<version>]-version distribution values of the + // passed available package. Only consider those values whose <name> + // component matches the specified distribution name and the <version> + // component (assumed as "0" if not present) is less or equal the + // specified distribution version. Return the system package version if + // the distribution version is equal to the specified one. Otherwise (the + // version is less), continue iterating while preferring system version + // candidates for greater distribution versions. Note that here we are + // trying to pick the system version with distribution version closest to + // (but never greater than) the specified distribution version, similar to + // what we do in downstream_package_version() (see its + // downstream_version() lambda for details). + // + auto system_version = [&ap, &af] (const string& n, + const semantic_version& v) + -> optional<string> + { + optional<string> r; + semantic_version rv; + + for (const distribution_name_value& dv: ap->distribution_values) + { + if (optional<string> d = dv.distribution ("-version")) + { + pair<string, semantic_version> dnv ( + parse_distribution (*d, dv.name, ap, af)); + + semantic_version& dvr (dnv.second); + + if (dnv.first == n && dvr <= v) + { + // If the distribution version is equal to the specified one, then + // we are done. Otherwise, save the system version if it is + // preferable and continue iterating. + // + if (dvr == v) + return move (dv.value); + + if (!r || rv < dvr) + { + r = move (dv.value); + rv = move (dvr); + } + } + } + } + + return r; + }; + + // Try to deduce the system package version using the + // <distribution>-version values that match the name id and refer to the + // version which is less or equal than the version id. + // + optional<string> r (system_version (name_id, vid)); + + // If the system package version is not deduced and the like ids are + // specified, then re-try but now using the like id and "0" version id + // instead. + // + if (!r) + { + for (const string& like_id: like_ids) + { + r = system_version (like_id, semantic_version (0, 0, 0)); + if (r) + break; + } + } + + return r; + + } + optional<version> system_package_manager:: downstream_package_version (const string& system_version, const available_packages& aps, @@ -397,7 +512,7 @@ namespace bpkg // specified one. Otherwise (the version is less), continue iterating // while preferring downstream version candidates for greater distribution // versions. Note that here we are trying to use a version mapping for the - // distribution version closest (but never greater) to the specified + // distribution version closest to (but never greater than) the specified // distribution version. So, for example, if both following values contain // a matching mapping, then for debian 11 we prefer the downstream version // produced by the debian_10-to-downstream-version value: @@ -421,9 +536,11 @@ namespace bpkg if (optional<string> d = nv.distribution ("-to-downstream-version")) { pair<string, semantic_version> dnv ( - parse_distribution (move (*d), nv.name, ap, a.second)); + parse_distribution (*d, nv.name, ap, a.second)); + + semantic_version& dvr (dnv.second); - if (dnv.first == n && dnv.second <= v) + if (dnv.first == n && dvr <= v) { auto bad_value = [&nv, &ap, &a] (const string& d) { @@ -502,21 +619,21 @@ namespace bpkg version ver (dv); // If the distribution version is equal to the specified one, - // then we are done. Otherwise, save the version if it is - // preferable and continue iterating. + // then we are done. Otherwise, save the downstream version if + // it is preferable and continue iterating. // // Note that bailing out immediately in the former case is // essential. Otherwise, we can potentially fail later on, for // example, some ill-formed regex which is already fixed in // some newer package. // - if (dnv.second == v) + if (dvr == v) return ver; - if (!r || rv < dnv.second) + if (!r || rv < dvr) { r = move (ver); - rv = move (dnv.second); + rv = move (dvr); } } catch (const invalid_argument& e) @@ -554,4 +671,244 @@ namespace bpkg return r; } + + auto system_package_manager:: + installed_entries (const common_options& co, + const packages& pkgs, + const strings& vars, + const string& scope) -> installed_entry_map + { + process_path pp (search_b (co)); + + // Note that we don't use start_b() here since we want to be consistent + // with how things will be run when building the package. + // + cstrings args { + pp.recall_string (), + "--quiet", // Note: implies --no-progress. + "--dry-run"}; + + // Pass our --jobs value, if any. + // + string jobs; + if (size_t n = co.jobs_specified () ? co.jobs () : 0) + { + jobs = to_string (n); + args.push_back ("--jobs"); + args.push_back (jobs.c_str ()); + } + + // Pass any --build-option. + // + for (const string& o: co.build_option ()) args.push_back (o.c_str ()); + + // Configuration variables. + // + for (const string& v: vars) args.push_back (v.c_str ()); + + string scope_arg; + args.push_back ((scope_arg = "!config.install.scope=" + scope).c_str ()); + + args.push_back ("!config.install.manifest=-"); + + // Package directories to install. + // + strings dirs; + for (const package& p: pkgs) dirs.push_back (p.out_root.representation ()); + args.push_back ("install:"); + for (const string& d: dirs) args.push_back (d.c_str ()); + + args.push_back (nullptr); + + installed_entry_map r; + try + { + if (verb >= 2) + print_process (args); + else if (verb == 1) + text << "determining filesystem entries that would be installed..."; + + // Redirect stdout to a pipe. + // + process pr (pp, + args, + 0 /* stdin */, + -1 /* stdout */, + 2 /* stderr */); + try + { + ifdstream is (move (pr.in_ofd), fdstream_mode::skip); + + json::parser p (is, + args[0] /* input_name */, + true /* multi_value */, + "\n" /* value_separators */); + + using event = json::event; + + // Note: recursive lambda. + // + auto parse_entry = [&r, &p] (const auto& parse_entry) -> void + { + optional<event> e (p.next ()); + + // @@ This is really ugly, need to add next_expect() helpers to JSON + // parser (similar to libstudxml). + + if (*e != event::begin_object) + fail << "entry object expected"; + + // type + // + if (!(e = p.next ()) || *e != event::name || p.name () != "type") + fail << "type member expected"; + + if (!(e = p.next ()) || *e != event::string) + fail << "type member string value expected"; + + string t (p.value ()); // Note: value invalidated after p.next(). + + if (t == "target") + { + // name + // + if (!(e = p.next ()) || *e != event::name || p.name () != "name") + fail << "name member expected"; + + if (!(e = p.next ()) || *e != event::string) + fail << "name member string value expected"; + + // entries + // + if (!(e = p.next ()) || *e != event::name || p.name () != "entries") + fail << "entries member expected"; + + if (!(e = p.next ()) || *e != event::begin_array) + fail << "entries member array value expected"; + + while ((e = p.peek ()) && *e != event::end_array) + parse_entry (parse_entry); + + if (!(e = p.next ()) || *e != event::end_array) + fail << "entries member array value end expected"; + } + else if (t == "file" || t == "symlink" || t == "directory") + { + // path + // + if (!(e = p.next ()) || *e != event::name || p.name () != "path") + fail << "path member expected"; + + if (!(e = p.next ()) || *e != event::string) + fail << "path member string value expected"; + + path ep (p.value ()); + assert (ep.absolute () && ep.normalized (false /* separators */)); + + if (t == "file" || t == "directory") + { + // mode + // + if (!(e = p.next ()) || *e != event::name || p.name () != "mode") + fail << "mode member expected"; + + if (!(e = p.next ()) || *e != event::string) + fail << "mode member string value expected"; + + string em (p.value ()); + + if (t == "file") + { + auto p ( + r.emplace ( + move (ep), installed_entry {move (em), nullptr})); + + if (!p.second) + fail << p.first->first << " is installed multiple times"; + } + } + else + { + // target + // + if (!(e = p.next ()) || *e != event::name || p.name () != "target") + fail << "target member expected"; + + if (!(e = p.next ()) || *e != event::string) + fail << "target member string value expected"; + + path et (p.value ()); + if (et.relative ()) + { + et = ep.directory () / et; + et.normalize (); + } + + auto i (r.find (et)); + if (i == r.end ()) + fail << "symlink " << ep << " target " << et << " does not " + << "refer to previously installed entry"; + + auto p (r.emplace (move (ep), installed_entry {"", &*i})); + + if (!p.second) + fail << p.first->first << " is installed multiple times"; + } + } + else + fail << "unknown entry type '" << t << "'"; + + if (!(e = p.next ()) || *e != event::end_object) + fail << "entry object end expected"; + }; + + while (p.peek ()) // More values. + { + parse_entry (parse_entry); + + if (p.next ()) // Consume value-terminating nullopt. + fail << "unexpected data after entry object"; + } + + is.close (); + } + catch (const json::invalid_json_input& e) + { + if (pr.wait ()) + fail << "invalid " << args[0] << " json input: " << e; + + // Fall through. + } + catch (const io_error& e) + { + if (pr.wait ()) + fail << "unable to read " << args[0] << " output: " << e; + + // Fall through. + } + + if (!pr.wait ()) + { + diag_record dr (fail); + dr << args[0] << " exited with non-zero code"; + + if (verb < 2) + { + dr << info << "command line: "; + print_process (dr, args); + } + } + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + return r; + } } diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx index 941c981..4549fba 100644 --- a/bpkg/system-package-manager.hxx +++ b/bpkg/system-package-manager.hxx @@ -4,12 +4,11 @@ #ifndef BPKG_SYSTEM_PACKAGE_MANAGER_HXX #define BPKG_SYSTEM_PACKAGE_MANAGER_HXX -#include <libbpkg/manifest.hxx> // version -#include <libbpkg/package-name.hxx> - #include <bpkg/types.hxx> #include <bpkg/utility.hxx> +#include <libbutl/path-map.hxx> + #include <bpkg/package.hxx> #include <bpkg/common-options.hxx> #include <bpkg/host-os-release.hxx> @@ -155,23 +154,53 @@ namespace bpkg virtual void pkg_install (const vector<package_name>&) = 0; - // Generate a binary distribution package. + // Generate a binary distribution package. See the pkg-bindist(1) man page + // for background and the pkg_bindist() function implementation for + // details. + // + // The available packages are loaded for all the packages in pkgs and + // deps. For non-system packages (so for all in pkgs) there is always a + // single available package that corresponds to the selected package. The + // out_root is only set for packages in pkgs. Note also that all the + // packages in pkgs and deps are guaranteed to belong to the same build + // configuration (as opposed to being spread over multiple linked + // configurations). Its absolute path is bassed in cfg_dir. // - // @@ TODO: doc + // The passed package manifest corresponds to the first package in pkgs + // (normally used as a source of additional package metadata such as + // summary, emails, urls, etc). // - // See the pkg-bindist(1) man page and the pkg_bindist() function - // implementation for background and details. + // The passed package type corresponds to the first package in pkgs while + // the languages -- to all the packages in pkgs plus, in the recursive + // mode, to all the non-system dependencies. In other words, the languages + // list contains every language that is used by anything that ends up in + // the package. // - using packages = - vector<pair<shared_ptr<selected_package>, available_packages>>; + // Return the list of paths to binary packages and any other associated + // files (build metadata, etc) that could be useful for consumption of + // binary packages. If the result is empty, assume the prepare-only mode + // (or similar) with appropriate result diagnostics having been already + // issued. + // + struct package + { + shared_ptr<selected_package> selected; + available_packages available; + dir_path out_root; // Absolute and normalized. + }; + + using packages = vector<package>; enum class recursive_mode {auto_, full}; - virtual void - generate (packages&& pkgs, - packages&& deps, - strings&& vars, - const dir_path& out, + virtual paths + generate (const packages& pkgs, + const packages& deps, + const strings& vars, + const dir_path& cfg_dir, + const package_manifest&, + const string& type, + const small_vector<language, 1>&, optional<recursive_mode>) = 0; public: @@ -243,7 +272,7 @@ namespace bpkg // is a semver-like version (e.g, 10, 10.15, or 10.15.1) and return all // the values that are equal or less than the specified version_id // (include the value with the absent <version>). In a sense, absent - // <version> can be treated as a 0 semver-like version. + // <version> is treated as a 0 semver-like version. // // If no value is found then repeat the above process for every like_ids // entry (from left to right) instead of name_id with version_id equal 0. @@ -259,6 +288,15 @@ namespace bpkg // debian_10-name: libcurl4 libcurl4-doc libcurl4-openssl-dev // debian_10-name: libcurl3-gnutls libcurl4-gnutls-dev (yes, 3 and 4) // + // The <distribution> value in the <name>_0 form is the special "non- + // native" name mapping. If the native argument is false, then such a + // mapping is preferred over any other mapping. If it is true, then such a + // mapping is ignored. The purpose of this special value is to allow + // specifying different package names for production compared to + // consumption. Note, however, that such a deviation may make it + // impossible to use native and non-native binary packages + // interchangeably, for example, to satisfy dependencies. + // // Note also that the values are returned in the "override order", that is // from the newest package version to oldest and then from the highest // distribution version to lowest. @@ -267,7 +305,31 @@ namespace bpkg system_package_names (const available_packages&, const string& name_id, const string& version_id, - const vector<string>& like_ids); + const vector<string>& like_ids, + bool native); + + // Given the available package and the repository fragment it belongs to, + // return the system package version as mapped by one of the + // <distribution>-version values. + // + // The rest of the arguments as well as the overalls semantics is the same + // as in system_package_names() above. That is, first consider + // <distribution>-version values corresponding to name_id. If none match, + // then repeat the above process for every like_ids entry with version_id + // equal 0. If still no match, then return nullopt (in which case the + // caller may choose to fallback to the upstream/bpkg package version or + // do something more elaborate). + // + // Note that lazy_shared_ptr<repository_fragment> is used only for + // diagnostics and conveys the database the available package object + // belongs to. + // + static optional<string> + system_package_version (const shared_ptr<available_package>&, + const lazy_shared_ptr<repository_fragment>&, + const string& name_id, + const string& version_id, + const vector<string>& like_ids); // Given the system package version and available packages (as returned by // find_available_all()) return the downstream package version as mapped @@ -287,6 +349,48 @@ namespace bpkg const string& name_id, const string& version_id, const vector<string>& like_ids); + + // Return the map of filesystem entries (files and symlinks) that would be + // installed for the specified packages with the specified configuration + // variables. + // + // In essence, this function runs: + // + // b --dry-run --quiet <vars> !config.install.scope=<scope> + // !config.install.manifest=- install: <pkgs> + // + // And converts the printed installation manifest into the path map. + // + // Note that this function prints an appropriate progress indicator since + // even in the dry-run mode it may take some time (see the --dry-run + // option documentation for details). + // + struct installed_entry + { + string mode; // Empty if symlink. + const pair<const path, installed_entry>* target; // Target if symlink. + }; + + class installed_entry_map: public butl::path_map<installed_entry> + { + public: + // Return true if there are filesystem entries in the specified + // directory or its subdirectories. + // + bool + contains_sub (const dir_path& d) + { + auto p (find_sub (d)); + return p.first != p.second; + } + }; + + installed_entry_map + installed_entries (const common_options&, + const packages& pkgs, + const strings& vars, + const string& scope); + protected: optional<bool> progress_; // --[no]-progress (see also stderr_term) optional<size_t> fetch_timeout_; // --fetch-timeout @@ -321,8 +425,13 @@ namespace bpkg bool yes, const string& sudo); + // Note that the reference to options is expected to outlive the returned + // instance. + // + class pkg_bindist_options; + unique_ptr<system_package_manager> - make_production_system_package_manager (const common_options&, + make_production_system_package_manager (const pkg_bindist_options&, const target_triplet&, const string& name, const string& arch); diff --git a/bpkg/system-package-manager.test.cxx b/bpkg/system-package-manager.test.cxx index 1a669da..f0d7c8f 100644 --- a/bpkg/system-package-manager.test.cxx +++ b/bpkg/system-package-manager.test.cxx @@ -21,7 +21,11 @@ namespace bpkg // // Where <command> is one of: // - // system-package-names <name-id> <ver-id> [<like-id>...] -- <pkg> <file>... + // system-package-names <name-id> <ver-id> [<like-id>...] -- [--non-native] <pkg> <file>... + // + // Where <pkg> is a package name, <file> is a package manifest file. + // + // system-package-version <name-id> <ver-id> [<like-id>...] -- <pkg> <file> // // Where <pkg> is a package name, <file> is a package manifest file. // @@ -41,6 +45,7 @@ namespace bpkg os_release osr; if (cmd == "system-package-names" || + cmd == "system-package-version" || cmd == "downstream-package-version") { assert (argc >= 4); // <name-id> <ver-id> @@ -65,6 +70,14 @@ namespace bpkg string a (argv[argi++]); assert (a == "--"); + assert (argi != argc); + bool native (true); + if ((a = argv[argi]) == "--non-native") + { + native = false; + argi++; + } + assert (argi != argc); // <pkg> string pn (argv[argi++]); @@ -76,11 +89,34 @@ namespace bpkg strings ns ( system_package_manager::system_package_names ( - aps, osr.name_id, osr.version_id, osr.like_ids)); + aps, osr.name_id, osr.version_id, osr.like_ids, native)); for (const string& n: ns) cout << n << '\n'; } + else if (cmd == "system-package-version") + { + assert (argi != argc); // -- + string a (argv[argi++]); + assert (a == "--"); + + assert (argi != argc); // <pkg> + string pn (argv[argi++]); + + assert (argi != argc); // <file> + pair<shared_ptr<available_package>, + lazy_shared_ptr<repository_fragment>> apf ( + make_available_from_manifest (pn, argv[argi++])); + + assert (argi == argc); // No trailing junk. + + if (optional<string> v = + system_package_manager::system_package_version ( + apf.first, apf.second, osr.name_id, osr.version_id, osr.like_ids)) + { + cout << *v << '\n'; + } + } else if (cmd == "downstream-package-version") { assert (argi != argc); // -- diff --git a/bpkg/system-package-manager.test.hxx b/bpkg/system-package-manager.test.hxx index 0eb6717..688eb72 100644 --- a/bpkg/system-package-manager.test.hxx +++ b/bpkg/system-package-manager.test.hxx @@ -20,25 +20,33 @@ namespace bpkg { // Parse the manifest as if it comes from a git repository with a single - // package and make an available package out of it. + // package and make an available package out of it. If the file name is + // `-` then read fro stdin. If the package name is empty, then take the + // name from the manifest. Otherwise, assert they match. // inline pair<shared_ptr<available_package>, lazy_shared_ptr<repository_fragment>> - make_available_from_manifest (const string& n, const string& f) + make_available_from_manifest (const string& pn, const string& f) { using butl::manifest_parser; using butl::manifest_parsing; + path fp (f); + path_name fn (fp); + try { - ifdstream ifs (f); - manifest_parser mp (ifs, f); + ifdstream ifds; + istream& ifs (butl::open_file_or_stdin (fn, ifds)); + + manifest_parser mp (ifs, fn.name ? *fn.name : fn.path->string ()); package_manifest m (mp, false /* ignore_unknown */, true /* complete_values */); - assert (m.name.string () == n); + const string& n (m.name.string ()); + assert (pn.empty () || n == pn); m.alt_naming = false; m.bootstrap_build = "project = " + n + '\n'; @@ -61,7 +69,7 @@ namespace bpkg } catch (const io_error& e) { - fail << "unable to read from " << f << ": " << e << endf; + fail << "unable to read from " << fn << ": " << e << endf; } } diff --git a/bpkg/system-package-manager.test.testscript b/bpkg/system-package-manager.test.testscript index dc672f5..74c6ad2 100644 --- a/bpkg/system-package-manager.test.testscript +++ b/bpkg/system-package-manager.test.testscript @@ -43,6 +43,63 @@ $* ubuntu 16.04 debian -- libcurl libcurl7.64.manifest libcurl7.84.manifest >>EOO libcurl2 libcurl2-dev EOO + + : native + : + cat <<EOI >=libcurl.manifest; + : 1 + name: libcurl + version: 7.84.0 + debian-name: libcurl4 libcurl4-openssl-dev + debian_0-name: libcurl libcurl-dev + summary: curl + license: curl + EOI + $* debian 10 -- libcurl libcurl.manifest >>EOO; + libcurl4 libcurl4-openssl-dev + EOO + $* debian 10 -- --non-native libcurl libcurl.manifest >>EOO + libcurl libcurl-dev + EOO +} + +: system-package-version +: +{ + test.arguments += system-package-version + + : basics + : + cat <<EOI >=libssl1.1.1+19.manifest; + : 1 + name: libssl + version: 1.1.1+19 + fedora-name: openssl-libs + fedora-version: 1:1.1.1q-1 + fedora_35-version: 1:1.1.1q-1.fc35 + fedora_36-version: 1:1.1.1q-1.fc36 + summary: openssl + license: openssl + EOI + + $* fedora 34 -- libssl libssl1.1.1+19.manifest >>EOO; + 1:1.1.1q-1 + EOO + $* fedora 35 -- libssl libssl1.1.1+19.manifest >>EOO; + 1:1.1.1q-1.fc35 + EOO + $* fedora 36 -- libssl libssl1.1.1+19.manifest >>EOO; + 1:1.1.1q-1.fc36 + EOO + $* fedora 37 -- libssl libssl1.1.1+19.manifest >>EOO; + 1:1.1.1q-1.fc36 + EOO + $* fedora '' -- libssl libssl1.1.1+19.manifest >>EOO; + 1:1.1.1q-1 + EOO + $* rhel 7.8 fedora -- libssl libssl1.1.1+19.manifest >>EOO + 1:1.1.1q-1 + EOO } : downstream-package-version diff --git a/bpkg/system-repository.hxx b/bpkg/system-repository.hxx index 6fc04f9..d524ee4 100644 --- a/bpkg/system-repository.hxx +++ b/bpkg/system-repository.hxx @@ -12,10 +12,10 @@ #include <bpkg/types.hxx> #include <bpkg/utility.hxx> -#include <bpkg/system-package-manager.hxx> - namespace bpkg { + struct system_package_status; // <bpkg/system-package-manager.hxx> + // A map of discovered system package versions. The information can be // authoritative (i.e., it was provided by the user or auto-discovered // on this run) or non-authoritative (i.e., comes from selected packages diff --git a/bpkg/types.hxx b/bpkg/types.hxx index 7a7b2c7..80e5a7d 100644 --- a/bpkg/types.hxx +++ b/bpkg/types.hxx @@ -88,6 +88,8 @@ namespace bpkg // <libbutl/path.hxx> // using butl::path; + using butl::path_name; + using butl::path_name_view; using butl::dir_path; using butl::basic_path; using butl::invalid_path; @@ -233,6 +235,14 @@ namespace std ::butl::path::traits_type::canonicalize (r); return os << r; } + + inline ostream& + operator<< (ostream& os, const ::butl::path_name_view& v) + { + assert (!v.empty ()); + + return v.name != nullptr && *v.name ? (os << **v.name) : (os << *v.path); + } } #endif // BPKG_TYPES_HXX diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx index b79c85b..96fda0f 100644 --- a/bpkg/utility.cxx +++ b/bpkg/utility.cxx @@ -369,6 +369,26 @@ namespace bpkg : BPKG_EXE_PREFIX "b" BPKG_EXE_SUFFIX; } + process_path + search_b (const common_options& co) + { + const char* b (name_b (co)); + + try + { + // Use our executable directory as a fallback search since normally the + // entire toolchain is installed into one directory. This way, for + // example, if we installed into /opt/build2 and run bpkg with absolute + // path (and without PATH), then bpkg will be able to find "its" b. + // + return process::path_search (b, exec_dir); + } + catch (const process_error& e) + { + fail << "unable to execute " << b << ": " << e << endf; + } + } + void dump_stderr (auto_fd&& fd) { diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx index 69a02d3..bb264ba 100644 --- a/bpkg/utility.hxx +++ b/bpkg/utility.hxx @@ -244,9 +244,16 @@ namespace bpkg normal // Run normally (at verbosity 1). }; + template <typename V> + void + map_verb_b (const common_options&, verb_b, V& args, string& verb_arg); + const char* name_b (const common_options&); + process_path + search_b (const common_options&); + template <typename O, typename E, typename... A> process start_b (const common_options&, O&& out, E&& err, verb_b, A&&... args); diff --git a/bpkg/utility.txx b/bpkg/utility.txx index 6113e4e..a21325c 100644 --- a/bpkg/utility.txx +++ b/bpkg/utility.txx @@ -7,6 +7,56 @@ namespace bpkg { // *_b() // + template <typename V> + void + map_verb_b (const common_options& co, verb_b v, V& ops, string& verb_arg) + { + // Map verbosity level. If we are running quiet or at level 1, + // then run build2 quiet. Otherwise, run it at the same level + // as us. + // + bool progress (co.progress ()); + bool no_progress (co.no_progress ()); + + if (verb == 0) + { + ops.push_back ("-q"); + no_progress = false; // Already suppressed with -q. + } + else if (verb == 1) + { + if (v != verb_b::normal) + { + ops.push_back ("-q"); + + if (!no_progress) + { + if (v == verb_b::progress && stderr_term) + { + ops.push_back ("--progress"); + progress = false; // The option is already added. + } + } + else + no_progress = false; // Already suppressed with -q. + } + } + else if (verb == 2) + ops.push_back ("-v"); + else + { + verb_arg = to_string (verb); + ops.push_back ("--verbose"); + ops.push_back (verb_arg.c_str ()); + } + + if (progress) + ops.push_back ("--progress"); + + if (no_progress) + ops.push_back ("--no-progress"); + } + template <typename O, typename E, typename... A> process start_b (const common_options& co, @@ -15,64 +65,17 @@ namespace bpkg verb_b v, A&&... args) { - const char* b (name_b (co)); + process_path pp (search_b (co)); try { - // Use our executable directory as a fallback search since normally the - // entire toolchain is installed into one directory. This way, for - // example, if we installed into /opt/build2 and run bpkg with absolute - // path (and without PATH), then bpkg will be able to find "its" b. - // - process_path pp (process::path_search (b, exec_dir)); - small_vector<const char*, 1> ops; - // Map verbosity level. If we are running quiet or at level 1, - // then run build2 quiet. Otherwise, run it at the same level - // as us. - // - string vl; - bool progress (co.progress ()); - bool no_progress (co.no_progress ()); - - if (verb == 0) - { - ops.push_back ("-q"); - no_progress = false; // Already suppressed with -q. - } - else if (verb == 1) - { - if (v != verb_b::normal) - { - ops.push_back ("-q"); - - if (!no_progress) - { - if (v == verb_b::progress && stderr_term) - { - ops.push_back ("--progress"); - progress = false; // The option is already added. - } - } - else - no_progress = false; // Already suppressed with -q. - } - } - else if (verb == 2) - ops.push_back ("-v"); - else - { - vl = to_string (verb); - ops.push_back ("--verbose"); - ops.push_back (vl.c_str ()); - } - - if (progress) - ops.push_back ("--progress"); + // NOTE: see custom versions in system_package_manager* if adding + // anything new here (search for search_b()). - if (no_progress) - ops.push_back ("--no-progress"); + string verb_arg; + map_verb_b (co, v, ops, verb_arg); // Forward our --[no]diag-color options. // @@ -98,7 +101,7 @@ namespace bpkg } catch (const process_error& e) { - fail << "unable to execute " << b << ": " << e << endf; + fail << "unable to execute " << pp.recall_string () << ": " << e << endf; } } @@ -78,14 +78,16 @@ compile "bpkg" $o --output-prefix "" --class-doc bpkg::commands=short --class-do compile "pkg-build" $o --class-doc bpkg::pkg_build_pkg_options=exclude-base +compile "pkg-bindist" $o --class-doc bpkg::pkg_bindist_debian_options=exclude-base + # NOTE: remember to update a similar list in buildfile and bpkg.cli as well as # the help topics sections in bpkg/buildfile and help.cxx. # pages="cfg-create cfg-info cfg-link cfg-unlink help pkg-clean pkg-configure \ -pkg-disfigure pkg-drop pkg-fetch pkg-checkout pkg-install pkg-purge \ -pkg-status pkg-test pkg-uninstall pkg-unpack pkg-update pkg-verify rep-add \ -rep-remove rep-list rep-create rep-fetch rep-info repository-signing \ -repository-types argument-grouping default-options-files" +pkg-disfigure pkg-drop pkg-fetch pkg-checkout pkg-install pkg-purge pkg-status \ +pkg-test pkg-uninstall pkg-unpack pkg-update pkg-verify rep-add rep-remove \ +rep-list rep-create rep-fetch rep-info repository-signing repository-types \ +argument-grouping default-options-files" for p in $pages; do compile $p $o diff --git a/doc/manual.cli b/doc/manual.cli index d417e92..48592fd 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -1387,8 +1387,8 @@ need not be repeated in this value. \ The detailed description of the package. It can be provided either inline as a -text fragment or by referring to a file within a package (e.g., \c{README}), -but not both. +text fragment or by referring to a file within a package (for example, +\c{README}), but not both. In the web interface (\c{brep}) the description is displayed according to its type. Currently, pre-formatted plain text, \l{https://github.github.com/gfm @@ -2458,7 +2458,7 @@ unspecified, then appropriate name(s) are automatically derived from the \c{bpkg} package name (\l{#manifest-package-name \c{name}}). Similarly, the \c{-version} value specifies the distribution package version. If unspecified, then the \c{upstream-version} value is used if specified and the \c{bpkg} -version otherwise (\l{#manifest-package-version \c{version}}). While the +version (\l{#manifest-package-version \c{version}}) otherwise. While the \c{-to-downstream-version} values specify the reverse mapping, that is, from the distribution version to the \c{bpkg} version. If unspecified or none match, then the appropriate part of the distribution version is used. For @@ -2503,11 +2503,32 @@ Note also that some distributions are like others (for example, \c{ubuntu} is like \c{debian}) and the corresponding \"base\" distribution values are considered if no \"derived\" values are specified. -The exact format of the \c{-name} value and the distribution version part that -is matched against the \c{-to-downstream-version} pattern are -distribution-specific. For details, see -\l{#bindist-mapping-debian Debian Package Mapping} and -\l{#bindist-mapping-fedora Fedora Package Mapping}. +The \c{-name} value is used both during package consumption as a system +package and production with the \l{bpkg-pkg-bindist(1)} command. During +production, if multiple mappings match, then the value with the highest +matching distribution version from the package \c{manifest} with the latest +version is used. If it's necessary to use different names for the generated +binary packages (called \"non-native packages\" in contrast to \"native +packages\" that come from the distribution), the special \c{0} distribution +version can be used to specify such a mapping. For example: + +\ +name: libsqlite3 +debian_9-name: libsqlite3-0 libsqlite3-dev +debian_0-name: libsqlite3 libsqlite3-dev +\ + +Note that this special non-native mapping is ignored during consumption and a +deviation in the package names that it introduces may make it impossible to +use native and non-native binary packages interchangeably, for example, to +satisfy dependencies. + + +The exact format of the \c{-name} and \c{-version} values and the distribution +version part that is matched against the \c{-to-downstream-version} pattern +are distribution-specific. For details, see \l{#bindist-mapping-debian Debian +Package Mapping} and \l{#bindist-mapping-fedora Fedora Package Mapping}. + \h#manifest-package-list-pkg|Package List Manifest for \cb{pkg} Repositories| @@ -2972,6 +2993,8 @@ private key and then \c{base64}-encoding the result. This section describes the distribution package mapping for Debian and alike (Ubuntu, etc). +\h2#bindist-mapping-debian-consume|Debian Package Mapping for Consumption| + A library in Debian is normally split up into several packages: the shared library package (e.g., \c{libfoo1} where \c{1} is the ABI version), the development files package (e.g., \c{libfoo-dev}), the documentation files @@ -2981,7 +3004,7 @@ package (e.g., \c{libfoo-doc}), the debug symbols package (e.g., is quite a bit of variability. Here are a few examples: \ -libz3-4 libz3-dev +libsqlite3-0 libsqlite3-dev libssl1.1 libssl-dev libssl-doc libssl3 libssl-dev libssl-doc @@ -2990,6 +3013,11 @@ libcurl4 libcurl4-openssl-dev libcurl4-doc libcurl3-gnutls libcurl4-gnutls-dev libcurl4-doc \ +Note that while most library package names in Debian start with \c{lib} (per +the policy), there are exceptions (e.g., \c{zlib1g} \c{zlib1g-dev}). Also note +that manual \c{-dbg} packages are obsolete in favor of automatic \c{-dbgsym} +packages from Debian 9. + For executable packages there is normally no \c{-dev} packages but \c{-dbg}, \c{-doc}, and \c{-common} are plausible. @@ -3023,11 +3051,11 @@ group is called the main package. Note that all the groups are consumed (installed) but only the main group is produced (packaged). We allow/recommend specifying the \c{-dev} package instead of the main package -for libraries (the \c{bpkg} package name starts with \c{lib}), seeing that we -are capable of detecting the main package automatically (see above). If the -library name happens to end with \c{-dev} (which poses an ambiguity), then the -\c{-dev} package should be specified explicitly as the second package to -disambiguate this situation. +for libraries (see \l{#manifest-package-type-language \c{type}} for details), +seeing that we are capable of detecting the main package automatically (see +above). If the library name happens to end with \c{-dev} (which poses an +ambiguity), then the \c{-dev} package should be specified explicitly as the +second package to disambiguate this situation. The Debian package version has the \c{[<epoch>:]<upstream>[-<revision>]} form (see \cb{deb-version(5)} for details). If no explicit mapping to the \c{bpkg} @@ -3037,11 +3065,147 @@ part as the \c{bpkg} version. If explicit mapping is specified, then we match it against the \c{[<epoch>:]<upstream>} parts ignoring \c{<revision>}. +\h2#bindist-mapping-debian-produce|Debian Package Mapping for Production| + +The same \c{debian-name} (or alike) manifest values as used for consumption +are also used to derive the package names for production except here we have +the option to specify alternative non-native package names using the special +\c{debian_0-name} (or alike) value. If only the \c{-dev} package is specified, +then the main package name is derived from that by removing the \c{-dev} +suffix. + +The generated binary package version can be specified with the +\c{debian-version} (or alike) manifest value. If it's not specified, then the +\c{upstream-version} is used if specified. Otherwise, the \c{bpkg} version +is translated to the Debian version as described next. + +To recap, a Debian package version has the following form: + +\ +[<epoch>:]<upstream>[-<revision>] +\ + +For details on the ordering semantics, see the \c{Version} \c{control} file +field documentation in the Debian Policy Manual. While overall unsurprising, +one notable exception is \c{~}, which sorts before anything else and is +commonly used for upstream pre-releases. For example, \c{1.0~beta1~svn1245} +sorts earlier than \c{1.0~beta1}, which sorts earlier than \c{1.0}. + +There are also various special version conventions (such as all the revision +components in \c{1.4-5+deb10u1~bpo9u1}) but they all appear to express +relationships between native packages and/or their upstream and thus do not +apply to our case. + +To recap, the \c{bpkg} version has the following form (see +\l{#package-version Package Version} for details): + +\ +[+<epoch>-]<upstream>[-<prerel>][+<revision>] +\ + +Let's start with the case where neither distribution (\c{debian-version}) nor +upstream version (\c{upstream-version}) is specified and we need to derive +everything from the \c{bpkg} version (what follows is as much description as +rationale). + +\dl| + +\li|\c{<epoch>} + + On one hand, if we keep our (as in, \c{bpkg}) epoch, it won't necessarily + match Debian's native package epoch. But on the other it will allow our + binary packages from different epochs to co-exist. Seeing that this can be + easily overridden with a custom distribution version (see below), we keep + it. + + Note that while the Debian start/default epoch is 0, ours is 1 (we use the 0 + epoch for stub packages). So we shift this value range.| + +\li|\c{<upstream>[-<prerel>]} + + Our upstream version maps naturally to Debian's. That is, our upstream + version format/semantics is a subset of Debian's. + + If this is a pre-release, then we could fail (that is, don't allow + pre-releases) but then we won't be able to test on pre-release packages, for + example, to make sure the name mapping is correct. Plus sometimes it's + useful to publish pre-releases. We could ignore it, but then such packages + will be indistinguishable from each other and the final release, which is + not ideal. On the other hand, Debian has the mechanism (\c{~}) which is + essentially meant for this, so we use it. We will use \c{<prerel>} as is + since its format is the same as upstream and thus should map naturally.| + +\li|\c{<revision>} + + Similar to epoch, our revision won't necessarily match Debian's native + package revision. But on the other hand it will allow us to establish a + correspondence between source and binary packages. Plus, upgrades between + binary package revisions will be handled naturally. Seeing that we allow + overriding the revision with a custom distribution version (see below), + we keep it. + + Note also that both Debian and our revision start/default is 0. However, it + is Debian's convention to start revision from 1. But it doesn't seem worth + it for us to do any shifting here and so we will use our revision as is. + + Another related question is whether we should also include some metadata + that identifies the distribution and its version that this package is + for. The strongest precedent here is probably Ubuntu's PPA. While there + doesn't appear to be a consistent approach, one can often see versions like + these: + + \ + 2.1.0-1~ppa0~ubuntu14.04.1, + 1.4-5-1.2.1~ubuntu20.04.1~ppa1 + 22.12.2-0ubuntu1~ubuntu23.04~ppa1 + \ + + Seeing that this is a non-sortable component (what in semver would be called + \"build metadata\"), using \c{~} is probably not the worst choice. + + So we follow this lead and add the \c{~<ID><VERSION_ID>} \c{os-release(5)} + component to revision. Note that this also means we will have to make the 0 + revision explicit. For example: + + \ + 1.2.3-1~debian10 + 1.2.3-0~ubuntu20.04 + \ + +|| + +The next case to consider is when we have the upstream version +(\c{upstream-version} manifest value). After some rumination it feels correct +to use it in place of the \c{<epoch>-<upstream>} components in the above +mapping (upstream version itself cannot have epoch). In other words, we will +add the pre-release and revision components from the \c{bpkg} version. If this +is not the desired semantics, then it can always be overridden with the +distribution version (see below). + +Finally, we have the distribution version. The Debian \c{<epoch>} and +\c{<upstream>} components are straightforward: they should be specified by the +distribution version as required. This leaves pre-release and revision. It +feels like in most cases we would want these copied over from the \c{bpkg} +version automatically \- it's too tedious and error-prone to maintain them +manually. However, we want the user to have the full override ability. So +instead, if empty revision is specified, as in \c{1.2.3-}, then we +automatically add the \c{bpkg} revision. Similarly, if empty pre-release is +specified, as in \c{1.2.3~}, then we add the \c{bpkg} pre-release. To add both +automatically, we would specify \c{1.2.3~-} (other combinations are +\c{1.2.3~b.1-} and \c{1.2.3~-1}). + +Note also that per the Debian version specification, if upstream contains +\c{:} and/or \c{-}, then epoch and/or revision must be specified explicitly, +respectively. Note that the \c{bpkg} upstream version may not contain either. + + \h#bindist-mapping-fedora|Fedora Package Mapping| This section describes the distribution package mapping for Fedora and alike (Red Hat Enterprise Linux, Centos, etc). +\h2#bindist-mapping-fedora-consume|Fedora Package Mapping for Consumption| + A library in Fedora is normally split up into several packages: the shared library package (e.g., \c{libfoo}), the development files package (e.g., \c{libfoo-devel}), the static library package (e.g., \c{libfoo-static}; may @@ -3135,11 +3299,11 @@ package\" since the main package may not be the base package, for example being the \c{-libs} subpackage.) We allow/recommend specifying the \c{-devel} package instead of the main -package for libraries (the \c{bpkg} package name starts with \c{lib}), seeing -that we are capable of detecting the main package automatically (see -above). If the library name happens to end with \c{-devel} (which poses an -ambiguity), then the \c{-devel} package should be specified explicitly as the -second package to disambiguate this situation. +package for libraries (see \l{#manifest-package-type-language \c{type}} for +details), seeing that we are capable of detecting the main package +automatically (see above). If the library name happens to end with \c{-devel} +(which poses an ambiguity), then the \c{-devel} package should be specified +explicitly as the second package to disambiguate this situation. The Fedora package version has the \c{[<epoch>:]<version>-<release>} form (see Fedora Package Versioning Guidelines for details). If no explicit mapping @@ -3149,6 +3313,10 @@ to the \c{bpkg} version is specified with the \c{fedora-to-downstream-version} then we match it against the \c{[<epoch>:]<version>} parts ignoring \c{<release>}. +\h2#bindist-mapping-fedora-produce|Fedora Package Mapping for Production| + +@@ TODO + " //@@ TODO items (grep). |