From 5c02ef198e539fc4a054ca442a56d89a7b796098 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 8 Mar 2023 09:28:05 +0200 Subject: Add non-native _0-name mapping, some more tests --- bpkg/system-package-manager-debian.cxx | 31 ++-- bpkg/system-package-manager-debian.hxx | 9 +- bpkg/system-package-manager-debian.test.cxx | 31 ++++ bpkg/system-package-manager-debian.test.testscript | 190 +++++++++++++++++++++ bpkg/system-package-manager-fedora.cxx | 3 +- bpkg/system-package-manager.cxx | 101 +++++++---- bpkg/system-package-manager.hxx | 14 +- bpkg/system-package-manager.test.cxx | 12 +- bpkg/system-package-manager.test.hxx | 20 ++- bpkg/system-package-manager.test.testscript | 18 ++ bpkg/types.hxx | 10 ++ 11 files changed, 374 insertions(+), 65 deletions(-) diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx index 83c28e6..5c8ace0 100644 --- a/bpkg/system-package-manager-debian.cxx +++ b/bpkg/system-package-manager-debian.cxx @@ -944,7 +944,8 @@ namespace bpkg // Without explicit type, the best we can do in trying to detect whether // this is a library is to check for the lib prefix. Libraries without // the lib prefix and non-libraries with the lib prefix (both of which - // we do not recomment) will have to provide a manual mapping. + // we do not recomment) will have to provide a manual mapping (or + // explicit type). // // Note that using the first (latest) available package as a source of // type information seems like a reasonable choice. @@ -958,7 +959,8 @@ namespace bpkg ns = system_package_names (aps, os_release.name_id, os_release.version_id, - os_release.like_ids); + os_release.like_ids, + true /* native */); if (ns.empty ()) { // Attempt to automatically translate our package name (see above for @@ -1492,15 +1494,14 @@ namespace bpkg // differently or fixes a critical bug), we will just have to provide // appropriate manual mapping that makes sure the names match (the extras is // still a potential problem though -- we will only have them as - // dependencies if we build against a native system package). - // - // @@ TODO: test, especially distribution version logic. + // dependencies if we build against a native system package; maybe we can + // add them manually with an option). // package_status system_package_manager_debian:: map_package (const package_name& pn, const version& pv, const available_packages& aps, - const optional& build_metadata) + const optional& build_metadata) const { // We should only have one available package corresponding to this package // name/version. @@ -1513,14 +1514,16 @@ namespace bpkg // Without explicit type, the best we can do in trying to detect whether // this is a library is to check for the lib prefix. Libraries without the // lib prefix and non-libraries with the lib prefix (both of which we do - // not recomment) will have to provide a manual mapping. + // not recomment) will have to provide a manual mapping (or explicit + // type). // const string& pt (ap->effective_type ()); strings ns (system_package_names (aps, os_release.name_id, os_release.version_id, - os_release.like_ids)); + os_release.like_ids, + false /* native */)); package_status r; if (ns.empty ()) { @@ -1616,7 +1619,7 @@ namespace bpkg // // Similar to epoch, our revision won't necessarily match Debian's // native package revision. But on the other hand it will allow us to - // establish a correspondence between source and binary packages. Plus, + // establish a correspondence between source and binary packages. Plus, // upgrades between binary package revisions will be handled naturally. // Seeing that we allow overriding the revision with a custom // distribution version (see below), let's keep it. @@ -2904,11 +2907,11 @@ namespace bpkg const path* cur_install (nullptr); // File being opened/written to. try { - pair main (main_install, nullfd); - pair dev (dev_install, nullfd); - pair doc (doc_install, nullfd); - pair dbg (dbg_install, nullfd); - pair com (common_install, nullfd); + pair main (main_install, auto_fd ()); + pair dev (dev_install, auto_fd ()); + pair doc (doc_install, auto_fd ()); + pair dbg (dbg_install, auto_fd ()); + pair com (common_install, auto_fd ()); auto open = [&deb, &cur_install] (pair& os, const string& n) diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx index 429b14f..eb9d91b 100644 --- a/bpkg/system-package-manager-debian.hxx +++ b/bpkg/system-package-manager-debian.hxx @@ -167,16 +167,19 @@ namespace bpkg yes, move (sudo)) {} + // Note: options can only be NULL when testing functions that don't need + // them. + // system_package_manager_debian (bpkg::os_release&& osr, const target_triplet& h, string a, optional progress, - const pkg_bindist_options& ops) + const pkg_bindist_options* ops) : system_package_manager (move (osr), h, a.empty () ? arch_from_target (h) : move (a), progress), - ops_ (&ops) {} + ops_ (ops) {} // Implementation details exposed for testing (see definitions for // documentation). @@ -213,7 +216,7 @@ namespace bpkg map_package (const package_name&, const version&, const available_packages&, - const optional&); + const optional&) const; // If simulate is not NULL, then instead of executing the actual apt-cache // and apt-get commands simulate their execution: (1) for apt-cache by diff --git a/bpkg/system-package-manager-debian.test.cxx b/bpkg/system-package-manager-debian.test.cxx index 0890dd6..df5275d 100644 --- a/bpkg/system-package-manager-debian.test.cxx +++ b/bpkg/system-package-manager-debian.test.cxx @@ -36,6 +36,8 @@ namespace bpkg // // main-from-dev depends comes from stdin // + // map-package [] manifest comes from stdin + // // build ... [--install [--no-fetch] ...] // // The stdin of the build command is used to read the simulation description @@ -167,6 +169,35 @@ namespace bpkg cout << system_package_manager_debian::main_from_dev (n, v, d) << '\n'; } + else if (cmd == "map-package") + { + assert (argc >= 2 && argc <= 3); // [] + + optional bm; + if (argc > 2) + bm = argv[2]; + + available_packages aps; + aps.push_back (make_available_from_manifest ("", "-")); + + const package_name& n (aps.front ().first->id.name); + const version& v (aps.front ().first->version); + + system_package_manager_debian m (move (osr), + host_triplet, + "" /* arch */, + nullopt /* progress */, + nullptr /* options */); + + package_status s (m.map_package (n, v, aps, bm)); + + cout << "version: " << s.system_version << '\n' + << "main: " << s.main << '\n'; + if (!s.dev.empty ()) cout << "dev: " << s.dev << '\n'; + if (!s.doc.empty ()) cout << "doc: " << s.doc << '\n'; + if (!s.dbg.empty ()) cout << "dbg: " << s.dbg << '\n'; + if (!s.common.empty ()) cout << "common: " << s.common << '\n'; + } else if (cmd == "build") { assert (argc >= 3); // ... diff --git a/bpkg/system-package-manager-debian.test.testscript b/bpkg/system-package-manager-debian.test.testscript index b1a0030..56c6785 100644 --- a/bpkg/system-package-manager-debian.test.testscript +++ b/bpkg/system-package-manager-debian.test.testscript @@ -222,6 +222,196 @@ EOI } +: map-package +: +{ + test.arguments += map-package + + : default-name + : + $* <>EOO + : 1 + name: byacc + version: 20210808 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808-0~debian10 + main: byacc + EOO + + : default-name-lib + : + $* <>EOO + : 1 + name: libsqlite3 + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite3 + dev: libsqlite3-dev + EOO + + : custom-name + : + $* <>EOO + : 1 + name: libsqlite3 + debian_9-name: libsqlite3-0 libsqlite3-dev + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite3-0 + dev: libsqlite3-dev + EOO + + : custom-name-dev-only + : + $* <>EOO + : 1 + name: libsqlite3 + debian_9-name: libsqlite3-0-dev + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite3-0 + dev: libsqlite3-0-dev + EOO + + : custom-name-non-native + : + $* <>EOO + : 1 + name: libsqlite3 + debian_0-name: libsqlite libsqlite-dev + debian_9-name: libsqlite3-0 libsqlite3-dev + version: 3.40.1 + summary: database library + license: other: public domain + EOI + version: 3.40.1-0~debian10 + main: libsqlite + dev: libsqlite-dev + EOO + + : version-upstream + : + $* <>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + upstream-version: 20210808 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-3~debian10 + main: byacc + EOO + + : version-distribution + : + $* <>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~beta.1 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-0~debian10 + main: byacc + EOO + + : version-distribution-epoch-revision + : + $* <>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 1:1.2.3-2 + summary: yacc parser generator + license: other: public domain + EOI + version: 1:1.2.3-2~debian10 + main: byacc + EOO + + : version-distribution-empty-release + : + $* <>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~-4 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-4~debian10 + main: byacc + EOO + + : version-distribution-empty-revision + : + $* <>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~b.1- + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~b.1-3~debian10 + main: byacc + EOO + + : version-distribution-empty-release-revision + : + $* <>EOO + : 1 + name: byacc + version: +2-1.2.3-beta.1+3 + debian-version: 20210808~- + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808~beta.1-3~debian10 + main: byacc + EOO + + : version-no-build-metadata + : + $* '' <>EOO + : 1 + name: byacc + version: 1.2.3 + summary: yacc parser generator + license: other: public domain + EOI + version: 1.2.3 + main: byacc + EOO + + : version-distribution-no-build-metadata + : + $* '' <>EOO + : 1 + name: byacc + version: 1.2.3 + debian-version: 20210808 + summary: yacc parser generator + license: other: public domain + EOI + version: 20210808 + main: byacc + EOO +} + : build : { diff --git a/bpkg/system-package-manager-fedora.cxx b/bpkg/system-package-manager-fedora.cxx index 02c2f59..7cffcf7 100644 --- a/bpkg/system-package-manager-fedora.cxx +++ b/bpkg/system-package-manager-fedora.cxx @@ -1137,7 +1137,8 @@ namespace bpkg ns = system_package_names (aps, os_release.name_id, os_release.version_id, - os_release.like_ids); + os_release.like_ids, + true /* native */); if (ns.empty ()) { // Attempt to automatically translate our package name. Failed that we diff --git a/bpkg/system-package-manager.cxx b/bpkg/system-package-manager.cxx index 4dc9bf7..793dec6 100644 --- a/bpkg/system-package-manager.cxx +++ b/bpkg/system-package-manager.cxx @@ -155,7 +155,7 @@ namespace bpkg os.like_ids.push_back ("debian"); r.reset (new system_package_manager_debian ( - move (os), host, arch, progress, o)); + move (os), host, arch, progress, &o)); } else if (is_or_like (os, "fedora") || is_or_like (os, "rhel") || @@ -213,18 +213,19 @@ namespace bpkg // Parse the component of the specified -* // value into the distribution name and version (return as "0" if not - // present). Issue diagnostics and fail on parsing errors. + // present). Leave in the d argument the string representation of the + // version (used to detect the special non-native _0). Issue + // diagnostics and fail on parsing errors. // // Note: the value_name, ap, and af arguments are only used for diagnostics. // static pair - parse_distribution (string&& d, + parse_distribution (string& d, // [_] const string& value_name, const shared_ptr& ap, const lazy_shared_ptr& af) { - string dn (move (d)); // [_] - size_t p (dn.rfind ('_')); // Version-separating underscore. + size_t p (d.rfind ('_')); // Version-separating underscore. // If the '_' separator is present, then make sure that the right-hand // part looks like a version (not empty and only contains digits and @@ -232,11 +233,11 @@ namespace bpkg // if (p != string::npos) { - if (p != dn.size () - 1) + if (p != d.size () - 1) { - for (size_t i (p + 1); i != dn.size (); ++i) + for (size_t i (p + 1); i != d.size (); ++i) { - if (!digit (dn[i]) && dn[i] != '.') + if (!digit (d[i]) && d[i] != '.') { p = string::npos; break; @@ -249,36 +250,43 @@ namespace bpkg // Parse the distribution version if present and leave it "0" otherwise. // + string dn; semantic_version dv (0, 0, 0); if (p != string::npos) - try { - dv = semantic_version (dn, - p + 1, - semantic_version::allow_omit_minor); + dn.assign (d, 0, p); + d.erase (0, p + 1); - dn.resize (p); - } - catch (const invalid_argument& e) - { - // Note: the repository fragment may have no database associated when - // used in tests. - // - shared_ptr f (af.get_eager ()); - database* db (!(f != nullptr && !af.loaded ()) // Not transient? - ? &af.database () - : nullptr); + try + { + dv = semantic_version (d, semantic_version::allow_omit_minor); + } + catch (const invalid_argument& e) + { + // Note: the repository fragment may have no database associated when + // used in tests. + // + shared_ptr f (af.get_eager ()); + database* db (!(f != nullptr && !af.loaded ()) // Not transient? + ? &af.database () + : nullptr); - diag_record dr (fail); - dr << "invalid distribution version '" << string (dn, p + 1) - << "' in value " << value_name << " for package " << ap->id.name - << ' ' << ap->version; + diag_record dr (fail); + dr << "invalid distribution version '" << d << "' in value " + << value_name << " for package " << ap->id.name << ' ' + << ap->version; - if (db != nullptr) - dr << *db; + if (db != nullptr) + dr << *db; - dr << " in repository " << (f != nullptr ? f : af.load ())->location - << ": " << e; + dr << " in repository " << (f != nullptr ? f : af.load ())->location + << ": " << e; + } + } + else + { + dn = move (d); + d.clear (); } return make_pair (move (dn), move (dv)); @@ -288,7 +296,8 @@ namespace bpkg system_package_names (const available_packages& aps, const string& name_id, const string& version_id, - const vector& like_ids) + const vector& like_ids, + bool native) { assert (!aps.empty ()); @@ -300,7 +309,8 @@ namespace bpkg // if not present) is less or equal the specified distribution version. // Suppress duplicate values. // - auto name_values = [&aps] (const string& n, const semantic_version& v) + auto name_values = [&aps, native] (const string& n, + const semantic_version& v) { strings r; @@ -322,14 +332,31 @@ namespace bpkg if (optional d = dv.distribution ("-name")) { pair dnv ( - parse_distribution (move (*d), dv.name, ap, a.second)); + parse_distribution (*d, dv.name, ap, a.second)); + + // Skip _0 if we are only interested in the native mappings. + // If we are interested in the non-native mapping, then we treat + // _0 as the matching version. + // + bool nn (*d == "0"); + if (nn && native) + continue; semantic_version& dvr (dnv.second); - if (dnv.first == n && dvr <= v) + if (dnv.first == n && (nn || dvr <= v)) { // Add the name/version pair to the sorted vector. // + // If this is the non-native mapping, then return just that. + // + if (nn) + { + r.clear (); // Drop anything we have accumulated so far. + r.push_back (move (dv.value)); + return r; + } + name_version nv (make_pair (dv.value, move (dvr))); nvs.insert (upper_bound (nvs.begin (), nvs.end (), nv, @@ -413,7 +440,7 @@ namespace bpkg if (optional d = dv.distribution ("-version")) { pair dnv ( - parse_distribution (move (*d), dv.name, ap, af)); + parse_distribution (*d, dv.name, ap, af)); semantic_version& dvr (dnv.second); @@ -509,7 +536,7 @@ namespace bpkg if (optional d = nv.distribution ("-to-downstream-version")) { pair dnv ( - parse_distribution (move (*d), nv.name, ap, a.second)); + parse_distribution (*d, nv.name, ap, a.second)); semantic_version& dvr (dnv.second); diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx index c661919..4fd8ba1 100644 --- a/bpkg/system-package-manager.hxx +++ b/bpkg/system-package-manager.hxx @@ -270,7 +270,7 @@ namespace bpkg // is a semver-like version (e.g, 10, 10.15, or 10.15.1) and return all // the values that are equal or less than the specified version_id // (include the value with the absent ). In a sense, absent - // can be treated as a 0 semver-like version. + // is treated as a 0 semver-like version. // // If no value is found then repeat the above process for every like_ids // entry (from left to right) instead of name_id with version_id equal 0. @@ -286,6 +286,15 @@ namespace bpkg // debian_10-name: libcurl4 libcurl4-doc libcurl4-openssl-dev // debian_10-name: libcurl3-gnutls libcurl4-gnutls-dev (yes, 3 and 4) // + // The value in the _0 form is the special "non- + // native" name mapping. If the native argument is false, then such a + // mapping is preferred over any other mapping. If it is true, then such a + // mapping is ignored. The purpose of this special value is to allow + // specifying different package names for production compared to + // consumption. Note, however, that such a deviation may make it + // impossible to use native and non-native binary packages + // interchangeably, for example, to satisfy dependencies. + // // Note also that the values are returned in the "override order", that is // from the newest package version to oldest and then from the highest // distribution version to lowest. @@ -294,7 +303,8 @@ namespace bpkg system_package_names (const available_packages&, const string& name_id, const string& version_id, - const vector& like_ids); + const vector& like_ids, + bool native); // Given the available package and the repository fragment it belongs to, // return the system package version as mapped by one of the diff --git a/bpkg/system-package-manager.test.cxx b/bpkg/system-package-manager.test.cxx index d2d02cb..f0d7c8f 100644 --- a/bpkg/system-package-manager.test.cxx +++ b/bpkg/system-package-manager.test.cxx @@ -21,7 +21,7 @@ namespace bpkg // // Where is one of: // - // system-package-names [...] -- ... + // system-package-names [...] -- [--non-native] ... // // Where is a package name, is a package manifest file. // @@ -70,6 +70,14 @@ namespace bpkg string a (argv[argi++]); assert (a == "--"); + assert (argi != argc); + bool native (true); + if ((a = argv[argi]) == "--non-native") + { + native = false; + argi++; + } + assert (argi != argc); // string pn (argv[argi++]); @@ -81,7 +89,7 @@ namespace bpkg strings ns ( system_package_manager::system_package_names ( - aps, osr.name_id, osr.version_id, osr.like_ids)); + aps, osr.name_id, osr.version_id, osr.like_ids, native)); for (const string& n: ns) cout << n << '\n'; diff --git a/bpkg/system-package-manager.test.hxx b/bpkg/system-package-manager.test.hxx index 0eb6717..688eb72 100644 --- a/bpkg/system-package-manager.test.hxx +++ b/bpkg/system-package-manager.test.hxx @@ -20,25 +20,33 @@ namespace bpkg { // Parse the manifest as if it comes from a git repository with a single - // package and make an available package out of it. + // package and make an available package out of it. If the file name is + // `-` then read fro stdin. If the package name is empty, then take the + // name from the manifest. Otherwise, assert they match. // inline pair, lazy_shared_ptr> - make_available_from_manifest (const string& n, const string& f) + make_available_from_manifest (const string& pn, const string& f) { using butl::manifest_parser; using butl::manifest_parsing; + path fp (f); + path_name fn (fp); + try { - ifdstream ifs (f); - manifest_parser mp (ifs, f); + ifdstream ifds; + istream& ifs (butl::open_file_or_stdin (fn, ifds)); + + manifest_parser mp (ifs, fn.name ? *fn.name : fn.path->string ()); package_manifest m (mp, false /* ignore_unknown */, true /* complete_values */); - assert (m.name.string () == n); + const string& n (m.name.string ()); + assert (pn.empty () || n == pn); m.alt_naming = false; m.bootstrap_build = "project = " + n + '\n'; @@ -61,7 +69,7 @@ namespace bpkg } catch (const io_error& e) { - fail << "unable to read from " << f << ": " << e << endf; + fail << "unable to read from " << fn << ": " << e << endf; } } diff --git a/bpkg/system-package-manager.test.testscript b/bpkg/system-package-manager.test.testscript index 4744a82..74c6ad2 100644 --- a/bpkg/system-package-manager.test.testscript +++ b/bpkg/system-package-manager.test.testscript @@ -43,6 +43,24 @@ $* ubuntu 16.04 debian -- libcurl libcurl7.64.manifest libcurl7.84.manifest >>EOO libcurl2 libcurl2-dev EOO + + : native + : + cat <=libcurl.manifest; + : 1 + name: libcurl + version: 7.84.0 + debian-name: libcurl4 libcurl4-openssl-dev + debian_0-name: libcurl libcurl-dev + summary: curl + license: curl + EOI + $* debian 10 -- libcurl libcurl.manifest >>EOO; + libcurl4 libcurl4-openssl-dev + EOO + $* debian 10 -- --non-native libcurl libcurl.manifest >>EOO + libcurl libcurl-dev + EOO } : system-package-version diff --git a/bpkg/types.hxx b/bpkg/types.hxx index 7a7b2c7..80e5a7d 100644 --- a/bpkg/types.hxx +++ b/bpkg/types.hxx @@ -88,6 +88,8 @@ namespace bpkg // // using butl::path; + using butl::path_name; + using butl::path_name_view; using butl::dir_path; using butl::basic_path; using butl::invalid_path; @@ -233,6 +235,14 @@ namespace std ::butl::path::traits_type::canonicalize (r); return os << r; } + + inline ostream& + operator<< (ostream& os, const ::butl::path_name_view& v) + { + assert (!v.empty ()); + + return v.name != nullptr && *v.name ? (os << **v.name) : (os << *v.path); + } } #endif // BPKG_TYPES_HXX -- cgit v1.1