From 546391dab6173660acceba6404136e9411ce1388 Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 1 Feb 2023 11:42:31 +0200 Subject: Implement system package manager query and install support for Debian --- bpkg/bpkg.cxx | 1 + bpkg/buildfile | 8 +- bpkg/database.cxx | 1 - bpkg/diagnostics.cxx | 2 +- bpkg/diagnostics.hxx | 31 +- bpkg/package-query.cxx | 100 +- bpkg/package-query.hxx | 25 +- bpkg/package.hxx | 11 +- bpkg/pkg-build-collect.cxx | 29 + bpkg/pkg-build-collect.hxx | 23 +- bpkg/pkg-build.cli | 64 +- bpkg/pkg-build.cxx | 643 ++++++--- bpkg/system-package-manager-debian.cxx | 1411 ++++++++++++++++++++ bpkg/system-package-manager-debian.hxx | 201 +++ bpkg/system-package-manager-debian.test.cxx | 348 +++++ bpkg/system-package-manager-debian.test.testscript | 987 ++++++++++++++ bpkg/system-package-manager.cxx | 454 +++++++ bpkg/system-package-manager.hxx | 270 ++++ bpkg/system-package-manager.test.cxx | 124 ++ bpkg/system-package-manager.test.hxx | 104 ++ bpkg/system-package-manager.test.testscript | 101 ++ bpkg/system-repository.cxx | 8 +- bpkg/system-repository.hxx | 15 +- bpkg/types.hxx | 5 + bpkg/utility.cxx | 2 + bpkg/utility.hxx | 10 +- 26 files changed, 4732 insertions(+), 246 deletions(-) create mode 100644 bpkg/system-package-manager-debian.cxx create mode 100644 bpkg/system-package-manager-debian.hxx create mode 100644 bpkg/system-package-manager-debian.test.cxx create mode 100644 bpkg/system-package-manager-debian.test.testscript create mode 100644 bpkg/system-package-manager.cxx create mode 100644 bpkg/system-package-manager.hxx create mode 100644 bpkg/system-package-manager.test.cxx create mode 100644 bpkg/system-package-manager.test.hxx create mode 100644 bpkg/system-package-manager.test.testscript (limited to 'bpkg') diff --git a/bpkg/bpkg.cxx b/bpkg/bpkg.cxx index 3ede99e..76b2533 100644 --- a/bpkg/bpkg.cxx +++ b/bpkg/bpkg.cxx @@ -611,6 +611,7 @@ try cout << "bpkg " << BPKG_VERSION_ID << endl << "libbpkg " << LIBBPKG_VERSION_ID << endl << "libbutl " << LIBBUTL_VERSION_ID << endl + << "host " << host_triplet << endl << "Copyright (c) " << BPKG_COPYRIGHT << "." << endl << "This is free software released under the MIT license." << endl; return 0; diff --git a/bpkg/buildfile b/bpkg/buildfile index b3b2ba7..ca78218 100644 --- a/bpkg/buildfile +++ b/bpkg/buildfile @@ -97,8 +97,10 @@ for t: cxx{**.test...} # Build options. # -obj{utility}: cxx.poptions += -DBPKG_EXE_PREFIX='"'$bin.exe.prefix'"' \ --DBPKG_EXE_SUFFIX='"'$bin.exe.suffix'"' +obj{utility}: cxx.poptions += \ +"-DBPKG_EXE_PREFIX=\"$bin.exe.prefix\"" \ +"-DBPKG_EXE_SUFFIX=\"$bin.exe.suffix\"" \ +"-DBPKG_HOST_TRIPLET=\"$cxx.target\"" # Pass the copyright notice extracted from the LICENSE file. # @@ -107,7 +109,7 @@ copyright = $process.run_regex( \ 'Copyright \(c\) (.+) \(see the AUTHORS and LEGAL files\)\.', \ '\1') -obj{bpkg}: cxx.poptions += -DBPKG_COPYRIGHT=\"$copyright\" +obj{bpkg}: cxx.poptions += "-DBPKG_COPYRIGHT=\"$copyright\"" # Disable "unknown pragma" warnings. # diff --git a/bpkg/database.cxx b/bpkg/database.cxx index 6b135fc..65e3af8 100644 --- a/bpkg/database.cxx +++ b/bpkg/database.cxx @@ -349,7 +349,6 @@ namespace bpkg else config_orig = config; - string = '[' + config_orig.representation () + ']'; try diff --git a/bpkg/diagnostics.cxx b/bpkg/diagnostics.cxx index ef87cab..9232d93 100644 --- a/bpkg/diagnostics.cxx +++ b/bpkg/diagnostics.cxx @@ -129,7 +129,7 @@ namespace bpkg const basic_mark error ("error"); const basic_mark warn ("warning"); const basic_mark info ("info"); - const basic_mark text (nullptr); + const basic_mark text (nullptr, nullptr, nullptr, nullptr); // No frame. const fail_mark fail ("error"); const fail_end endf; } diff --git a/bpkg/diagnostics.hxx b/bpkg/diagnostics.hxx index e8d9f0a..a01d90c 100644 --- a/bpkg/diagnostics.hxx +++ b/bpkg/diagnostics.hxx @@ -118,9 +118,37 @@ namespace bpkg // using butl::diag_stream; using butl::diag_epilogue; + using butl::diag_frame; // Diagnostic facility, project specifics. // + + // Note: diag frames are not applied to text/trace diagnostics. + // + template + struct diag_frame_impl: diag_frame + { + explicit + diag_frame_impl (F f): diag_frame (&thunk), func_ (move (f)) {} + + private: + static void + thunk (const diag_frame& f, const butl::diag_record& r) + { + static_cast (f).func_ ( + const_cast (static_cast (r))); + } + + const F func_; + }; + + template + inline diag_frame_impl + make_diag_frame (F f) + { + return diag_frame_impl (move (f)); + } + struct simple_prologue_base { explicit @@ -184,7 +212,7 @@ namespace bpkg basic_mark_base (const char* type, const char* name = nullptr, const void* data = nullptr, - diag_epilogue* epilogue = nullptr) + diag_epilogue* epilogue = &diag_frame::apply) : type_ (type), name_ (name), data_ (data), epilogue_ (epilogue) {} simple_prologue @@ -288,6 +316,7 @@ namespace bpkg data, [](const diag_record& r, butl::diag_writer* w) { + diag_frame::apply (r); r.flush (w); throw failed (); }) {} diff --git a/bpkg/package-query.cxx b/bpkg/package-query.cxx index 8d7b652..ea64e71 100644 --- a/bpkg/package-query.cxx +++ b/bpkg/package-query.cxx @@ -311,11 +311,11 @@ namespace bpkg } // Sort the available package fragments in the package version descending - // order and suppress duplicate packages. + // order and suppress duplicate packages and, optionally, older package + // revisions. // static void - sort_dedup (vector, - lazy_shared_ptr>>& pfs) + sort_dedup (available_packages& pfs, bool suppress_older_revisions = false) { sort (pfs.begin (), pfs.end (), [] (const auto& x, const auto& y) @@ -323,22 +323,22 @@ namespace bpkg return x.first->version > y.first->version; }); - pfs.erase (unique (pfs.begin(), pfs.end(), - [] (const auto& x, const auto& y) - { - return x.first->version == y.first->version; - }), - pfs.end ()); + pfs.erase ( + unique (pfs.begin(), pfs.end(), + [suppress_older_revisions] (const auto& x, const auto& y) + { + return x.first->version.compare (y.first->version, + suppress_older_revisions) == 0; + }), + pfs.end ()); } - vector, - lazy_shared_ptr>> + available_packages find_available (const linked_databases& dbs, const package_name& name, const optional& c) { - vector, - lazy_shared_ptr>> r; + available_packages r; for (database& db: dbs) { @@ -376,15 +376,13 @@ namespace bpkg return r; } - vector, - lazy_shared_ptr>> + available_packages find_available (const package_name& name, const optional& c, const config_repo_fragments& rfs, bool prereq) { - vector, - lazy_shared_ptr>> r; + available_packages r; for (const auto& dfs: rfs) { @@ -546,6 +544,74 @@ namespace bpkg return make_pair (find_available (options, db, sp), nullptr); } + available_packages + find_available_all (const linked_databases& dbs, + const package_name& name, + bool suppress_older_revisions) + { + // Collect all the databases linked explicitly and implicitly to the + // specified databases, recursively. + // + // Note that this is a superset of the database cluster, since we descend + // into the database links regardless of their types (see + // cluster_configs() for details). + // + linked_databases all_dbs; + all_dbs.reserve (dbs.size ()); + + auto add = [&all_dbs] (database& db, const auto& add) + { + if (find (all_dbs.begin (), all_dbs.end (), db) != all_dbs.end ()) + return; + + all_dbs.push_back (db); + + { + const linked_configs& cs (db.explicit_links ()); + for (auto i (cs.begin_linked ()); i != cs.end (); ++i) + add (i->db, add); + } + + { + const linked_databases& cs (db.implicit_links ()); + for (auto i (cs.begin_linked ()); i != cs.end (); ++i) + add (*i, add); + } + }; + + for (database& db: dbs) + add (db, add); + + // Collect all the available packages from all the collected databases. + // + available_packages r; + + for (database& db: all_dbs) + { + for (shared_ptr ap: + pointer_result ( + query_available (db, name, nullopt /* version_constraint */))) + { + // An available package should come from at least one fetched + // repository fragment. + // + assert (!ap->locations.empty ()); + + // All repository fragments the package comes from are equally good, so + // we pick the first one. + // + r.emplace_back (move (ap), ap->locations[0].repository_fragment); + } + } + + // Sort the result in the package version descending order and suppress + // duplicates and, if requested, older package revisions. + // + sort_dedup (r, suppress_older_revisions); + + return r; + } + pair, lazy_shared_ptr> make_available_fragment (const common_options& options, diff --git a/bpkg/package-query.hxx b/bpkg/package-query.hxx index ebe92ac..ee9b595 100644 --- a/bpkg/package-query.hxx +++ b/bpkg/package-query.hxx @@ -75,14 +75,13 @@ namespace bpkg // Try to find packages that optionally satisfy the specified version // constraint in multiple databases, suppressing duplicates. Return the list // of packages and repository fragments in which each was found in the - // package version descending or empty list if none were found. Note that a - // stub satisfies any constraint. + // package version descending order or empty list if none were found. Note + // that a stub satisfies any constraint. // // Note that we return (loaded) lazy_shared_ptr in order to also convey // the database to which it belongs. // - vector, - lazy_shared_ptr>> + available_packages find_available (const linked_databases&, const package_name&, const optional&); @@ -94,8 +93,8 @@ namespace bpkg using config_repo_fragments = database_map>>; - vector, - lazy_shared_ptr>> + + available_packages find_available (const package_name&, const optional&, const config_repo_fragments&, @@ -173,6 +172,20 @@ namespace bpkg database&, const shared_ptr&); + // Try to find packages in multiple databases, traversing the explicitly and + // implicitly linked databases recursively and suppressing duplicates and, + // optionally, older package revisions. Return the list of packages and + // repository fragments in which each was found in the package version + // descending order or empty list if none were found. + // + // Note that we return (loaded) lazy_shared_ptr in order to also convey + // the database to which it belongs. + // + available_packages + find_available_all (const linked_databases&, + const package_name&, + bool suppress_older_revisions = true); + // Create a transient (or fake, if you prefer) available_package object // corresponding to the specified selected object. Note that the package // locations list is left empty and that the returned repository fragment diff --git a/bpkg/package.hxx b/bpkg/package.hxx index 8796036..e811e62 100644 --- a/bpkg/package.hxx +++ b/bpkg/package.hxx @@ -847,7 +847,7 @@ namespace bpkg // #pragma db member(tests) id_column("") value_column("test_") - // distributions + // distribution_values // #pragma db member(distribution_values) id_column("") value_column("dist_") @@ -888,6 +888,15 @@ namespace bpkg available_package () = default; }; + // The available packages together with the repository fragments they belong + // to. + // + // Note that lazy_shared_ptr is used to also convey the databases the + // objects belong to. + // + using available_packages = vector, + lazy_shared_ptr>>; + #pragma db view object(available_package) struct available_package_count { diff --git a/bpkg/pkg-build-collect.cxx b/bpkg/pkg-build-collect.cxx index 12db3ba..8442b15 100644 --- a/bpkg/pkg-build-collect.cxx +++ b/bpkg/pkg-build-collect.cxx @@ -29,6 +29,35 @@ namespace bpkg { // build_package // + const system_package_status* build_package:: + system_status () const + { + assert (action); + + if (*action != build_package::drop && system) + { + const optional& sys_rep (db.get ().system_repository); + assert (sys_rep); + + if (const system_package* sys_pkg = sys_rep->find (name ())) + return sys_pkg->system_status; + } + + return nullptr; + } + + const system_package_status* build_package:: + system_install () const + { + if (const system_package_status* s = system_status ()) + return s->status == system_package_status::partially_installed || + s->status == system_package_status::not_installed + ? s + : nullptr; + + return nullptr; + } + bool build_package:: user_selection () const { diff --git a/bpkg/pkg-build-collect.hxx b/bpkg/pkg-build-collect.hxx index e47d9fa..6c79abe 100644 --- a/bpkg/pkg-build-collect.hxx +++ b/bpkg/pkg-build-collect.hxx @@ -19,8 +19,9 @@ #include #include -#include // find_database_function() +#include // find_database_function() #include +#include namespace bpkg { @@ -210,6 +211,26 @@ namespace bpkg // bool system; + // Return the system/distribution package status if this is a system + // package (re-)configuration and the package is being managed by the + // system package manager (as opposed to user/fallback). Otherwise, return + // NULL (so can be used as bool). + // + // Note on terminology: We call the bpkg package that is being configured + // as available from the system as "system package" and we call the + // underlying package managed by the system/distribution package manager + // as "system/distribution package". See system-package-manager.hxx for + // background. + // + const system_package_status* + system_status () const; + + // As above but only return the status if the package needs to be + // installed. + // + const system_package_status* + system_install () const; + // If this flag is set and the external package is being replaced with an // external one, then keep its output directory between upgrades and // downgrades. diff --git a/bpkg/pkg-build.cli b/bpkg/pkg-build.cli index 493ffbe..740764b 100644 --- a/bpkg/pkg-build.cli +++ b/bpkg/pkg-build.cli @@ -95,12 +95,19 @@ namespace bpkg A package name () can be prefixed with a package scheme (). Currently the only recognized scheme is \cb{sys} which instructs \cb{pkg-build} to configure the package as available from the - system rather than building it from source. If the system package version - () is not specified or is '\cb{/*}', then it is considered to - be unknown but satisfying any version constraint. If specified, - may not be a version constraint. If the version is not - explicitly specified, then at least a stub package must be available from - one of the repositories. + system rather than building it from source. + + The system package version () may not be a version constraint + but may be the special '\cb{/*}' value, which indicates that the version + should be considered unknown but satisfying any version constraint. If + unspecified, then \cb{pkg-build} will attempt to query the system package + manager for the installed version unless the system package manager is + unsupported or this functionality is disabled with \cb{--sys-no-query}, + in which case the '\cb{/*}' is assumed. If the system package + manager is supported, then the automatic installation of an available + package can be requested with the \cb{--sys-install} option. Note that if + the version is not explicitly specified, then at least a stub package + must be available from one of the repositories. Finally, a package can be specified as either the path to the package archive () or to the package directory (\cb{/}; note that it @@ -293,7 +300,8 @@ namespace bpkg bool --yes|-y { - "Assume the answer to all prompts is \cb{yes}." + "Assume the answer to all prompts is \cb{yes}. Note that this excludes + the system package manager prompts; see \cb{--sys-yes} for details." } string --for|-f @@ -408,6 +416,48 @@ namespace bpkg See \l{bpkg-cfg-create(1)} for details on linked configurations." } + bool --sys-no-query + { + "Do not query the system package manager for the installed versions of + packages specified with the \cb{sys} scheme." + } + + bool --sys-install + { + "Instruct the system package manager to install available versions of + packages specified with the \cb{sys} scheme that are not already + installed. See also the \cb{--sys-no-fetch}, \cb{--sys-yes}, and + \cb{--sys-sudo} options." + } + + bool --sys-no-fetch + { + "Do not fetch the system package manager metadata before querying for + available versions of packages specified with the \cb{sys} scheme. + This option only makes sense together with \cb{--sys-install}." + } + + bool --sys-yes + { + "Assume the answer to the system package manager prompts is \cb{yes}. + Note that system package manager interactions may break your system + and you should normally only use this option on throw-away setups + (test virtual machines, etc)." + } + + string --sys-sudo = "sudo" + { + "", + + "The \cb{sudo} program to use for system package manager interactions + that normally require administrative privileges (fetch package + metadata, install packages, etc). If unspecified, \cb{sudo} is used + by default. Pass empty or the special \cb{false} value to disable the + use of the \cb{sudo} program. Note that the \cb{sudo} program is + normally only needed if the system package installation is enabled + with the \cb{--sys-install} option." + } + dir_paths --directory|-d { "", diff --git a/bpkg/pkg-build.cxx b/bpkg/pkg-build.cxx index aa34594..4d90df2 100644 --- a/bpkg/pkg-build.cxx +++ b/bpkg/pkg-build.cxx @@ -32,7 +32,9 @@ #include #include #include + #include +#include #include @@ -41,10 +43,10 @@ using namespace butl; namespace bpkg { - // @@ Overall TODO: - // - // - Configuration vars (both passed and preserved) + // System package manager. Resolved lazily if and when needed. Present NULL + // value means no system package manager is available for this host. // + static optional> sys_pkg_mgr; // Current configurations as specified with --directory|-d (or the current // working directory if none specified). @@ -171,6 +173,7 @@ namespace bpkg optional checkout_root; bool checkout_purge; strings config_vars; // Only if not system. + const system_package_status* system_status; // See struct pkg_arg. }; using dependency_packages = vector; @@ -536,9 +539,7 @@ namespace bpkg else if (!dsys || !wildcard (*dvc)) c = dvc; - vector, - lazy_shared_ptr>> afs ( - find_available (nm, c, rfs)); + available_packages afs (find_available (nm, c, rfs)); if (afs.empty () && dsys && c) afs = find_available (nm, nullopt, rfs); @@ -1071,6 +1072,10 @@ namespace bpkg << "specified" << info << "run 'bpkg help pkg-build' for more information"; + if (o.sys_no_query () && o.sys_install ()) + fail << "both --sys-no-query and --sys-install specified" << + info << "run 'bpkg help pkg-build' for more information"; + if (!args.more () && !o.upgrade () && !o.patch ()) fail << "package name argument expected" << info << "run 'bpkg help pkg-build' for more information"; @@ -1537,6 +1542,12 @@ namespace bpkg string value; pkg_options options; strings config_vars; + + // If schema is sys then this member indicates whether the constraint + // came from the system package manager (not NULL) or user/fallback + // (NULL). + // + const system_package_status* system_status; }; auto arg_parsed = [] (const pkg_arg& a) {return !a.name.empty ();}; @@ -1652,23 +1663,132 @@ namespace bpkg return r; }; - // Add the system package authoritative information to the database's - // system repository, unless it already contains authoritative information - // for this package. + // Figure out the system package version unless explicitly specified and + // add the system package authoritative information to the database's + // system repository unless the database is NULL or it already contains + // authoritative information for this package. Return the figured out + // system package version as constraint. // // Note that it is assumed that all the possible duplicates are handled // elsewhere/later. // - auto add_system_package = [] (database& db, - const package_name& nm, - const version& v) + auto add_system_package = [&o] (database* db, + const package_name& nm, + optional vc, + const system_package_status* sps, + vector>* stubs) + -> pair { - assert (db.system_repository); + if (!vc) + { + assert (sps == nullptr); + + // See if we should query the system package manager. + // + if (!sys_pkg_mgr) + sys_pkg_mgr = o.sys_no_query () + ? nullptr + : make_system_package_manager (o, + host_triplet, + o.sys_install (), + !o.sys_no_fetch (), + o.sys_yes (), + o.sys_sudo (), + "" /* name */); + + if (*sys_pkg_mgr != nullptr) + { + system_package_manager& spm (**sys_pkg_mgr); + + // First check the cache. + // + optional os ( + spm.pkg_status (nm, nullptr)); - const system_package* sp (db.system_repository->find (nm)); + available_packages aps; + if (!os) + { + // If no cache hit, then collect the available packages for the + // mapping information. + // + aps = find_available_all (current_configs, nm); + + // If no source/stub for the package (and thus no mapping), issue + // diagnostics consistent with other such places. + // + if (aps.empty ()) + fail << "unknown package " << nm << + info << "consider specifying " << nm << "/*"; + } + + // This covers both our diagnostics below as well as anything that + // might be issued by pkg_status(). + // + auto df = make_diag_frame ( + [&nm] (diag_record& dr) + { + dr << info << "specify " << nm << "/* if package is not " + << "installed with system package manager"; + + dr << info << "specify --sys-no-query to disable system " + << "package manager interactions"; + }); + + if (!os) + { + os = spm.pkg_status (nm, &aps); + assert (os); + } - if (sp == nullptr || !sp->authoritative) - db.system_repository->insert (nm, v, true /* authoritative */); + if ((sps = *os) != nullptr) + vc = version_constraint (sps->version); + else + { + diag_record dr (fail); + + dr << "no installed " << (o.sys_install () ? " or available " : "") + << "system package for " << nm; + + if (!o.sys_install ()) + dr << info << "specify --sys-install to try to install it"; + } + } + else + vc = version_constraint (wildcard_version); + } + else + { + // The system package may only have an exact/wildcard version + // specified. + // + assert (vc->min_version == vc->max_version); + + // For system packages not associated with a specific repository + // location add the stub package to the imaginary system repository + // (see below for details). + // + if (stubs != nullptr) + stubs->push_back (make_shared (nm)); + } + + if (db != nullptr) + { + assert (db->system_repository); + + const system_package* sp (db->system_repository->find (nm)); + + // Note that we don't check for the version match here since that's + // handled by check_dup() lambda at a later stage, which covers both + // db and no-db cases consistently. + // + if (sp == nullptr || !sp->authoritative) + db->system_repository->insert (nm, + *vc->min_version, + true /* authoritative */, + sps); + } + + return make_pair (move (*vc), sps); }; // Create the parsed package argument. Issue diagnostics and fail if the @@ -1680,15 +1800,23 @@ namespace bpkg package_name nm, optional vc, pkg_options os, - strings vs) -> pkg_arg + strings vs, + vector>* stubs = nullptr) + -> pkg_arg { assert (!vc || !vc->empty ()); // May not be empty if present. if (db == nullptr) assert (sc == package_scheme::sys && os.dependency ()); - pkg_arg r { - db, sc, move (nm), move (vc), string (), move (os), move (vs)}; + pkg_arg r {db, + sc, + move (nm), + move (vc), + string () /* value */, + move (os), + move (vs), + nullptr /* system_status */}; // Verify that the package database is specified in the multi-config // mode, unless this is a system dependency package. @@ -1707,17 +1835,16 @@ namespace bpkg { case package_scheme::sys: { - if (!r.constraint) - r.constraint = version_constraint (wildcard_version); + assert (stubs != nullptr); - // The system package may only have an exact/wildcard version - // specified. - // - assert (r.constraint->min_version == r.constraint->max_version); - - if (db != nullptr) - add_system_package (*db, r.name, *r.constraint->min_version); + auto sp (add_system_package (db, + r.name, + move (r.constraint), + nullptr /* system_package_status */, + stubs)); + r.constraint = move (sp.first); + r.system_status = sp.second; break; } case package_scheme::none: break; // Nothing to do. @@ -1739,7 +1866,8 @@ namespace bpkg nullopt /* constraint */, move (v), move (os), - move (vs)}; + move (vs), + nullptr /* system_status */}; }; vector pkg_args; @@ -1802,13 +1930,6 @@ namespace bpkg parse_package_version_constraint ( s, sys, version_flags (sc), version_only (sc))); - // For system packages not associated with a specific repository - // location add the stub package to the imaginary system - // repository (see above for details). - // - if (sys && vc) - stubs.push_back (make_shared (n)); - pkg_options& o (ps.options); // Disregard the (main) database for a system dependency with @@ -1825,7 +1946,8 @@ namespace bpkg move (n), move (vc), move (o), - move (ps.config_vars))); + move (ps.config_vars), + &stubs)); } else // Add unparsed. pkg_args.push_back (arg_raw (ps.db, @@ -2059,7 +2181,8 @@ namespace bpkg move (n), move (vc), ps.options, - ps.config_vars)); + ps.config_vars, + &stubs)); } } } @@ -2132,7 +2255,7 @@ namespace bpkg !compare_options (a.options, pa.options) || a.config_vars != pa.config_vars)) fail << "duplicate package " << pa.name << - info << "first mentioned as " << arg_string (r.first->second) << + info << "first mentioned as " << arg_string (a) << info << "second mentioned as " << arg_string (pa); return !r.second; @@ -2503,7 +2626,8 @@ namespace bpkg ? move (pa.options.checkout_root ()) : optional ()), pa.options.checkout_purge (), - move (pa.config_vars)}); + move (pa.config_vars), + pa.system_status}); continue; } @@ -2563,7 +2687,7 @@ namespace bpkg // if (pa.constraint) { - for (;;) + for (;;) // Breakout loop. { if (ap != nullptr) // Must be that version, see above. break; @@ -3180,12 +3304,11 @@ namespace bpkg // The system package may only have an exact/wildcard version // specified. // - add_system_package (db, + add_system_package (&db, p.name, - (p.constraint - ? *p.constraint->min_version - : wildcard_version)); - + p.constraint, + p.system_status, + nullptr /* stubs */); enter (db, p); }; @@ -4278,10 +4401,11 @@ namespace bpkg bool update_dependents (false); // We need the plan and to ask for the user's confirmation only if some - // implicit action (such as building prerequisite or reconfiguring - // dependent package) is to be taken or there is a selected package which - // version must be changed. But if the user explicitly requested it with - // --plan, then we print it as long as it is not empty. + // implicit action (such as building prerequisite, reconfiguring dependent + // package, or installing system/distribution packages) is to be taken or + // there is a selected package which version must be changed. But if the + // user explicitly requested it with --plan, then we print it as long as + // it is not empty. // string plan; sha256 csum; @@ -4292,6 +4416,31 @@ namespace bpkg o.plan_specified () || o.rebuild_checksum_specified ()) { + // Map the main system/distribution packages that need to be installed + // to the system packages which caused their installation (see + // build_package::system_install() for details). + // + using package_names = vector>; + using system_map = map; + + system_map sys_map; + + // Iterate in the reverse order as we will do for printing the action + // lines. This way a system-install action line will be printed right + // before the bpkg action line of a package which appears first in the + // system-install action's 'required by' list. + // + for (const build_package& p: reverse_iterate (pkgs)) + { + if (const system_package_status* s = p.system_install ()) + { + package_names& ps (sys_map[s->system_name]); + + if (find (ps.begin (), ps.end (), p.name ()) == ps.end ()) + ps.push_back (p.name ()); + } + } + // Start the transaction since we may query available packages for // skeleton initializations. // @@ -4299,200 +4448,253 @@ namespace bpkg bool first (true); // First entry in the plan. - for (build_package& p: reverse_iterate (pkgs)) + // Print the bpkg package action lines. + // + // Also print the system-install action lines for system/distribution + // packages which require installation by the system package manager. + // Print them before the respective system package action lines, but + // only once per (main) system/distribution package. For example: + // + // system-install libssl1.1/1.1.1l (required by sys:libssl, sys:libcrypto) + // configure sys:libssl/1.1.1 (required by foo) + // configure sys:libcrypto/1.1.1 (required by bar) + // + for (auto i (pkgs.rbegin ()); i != pkgs.rend (); ) { + build_package& p (*i); assert (p.action); - database& pdb (p.db); - const shared_ptr& sp (p.selected); - string act; - if (*p.action == build_package::drop) + const system_package_status* s; + system_map::iterator j; + + if ((s = p.system_install ()) != nullptr && + (j = sys_map.find (s->system_name)) != sys_map.end ()) { - act = "drop " + sp->string (pdb) + " (unused)"; + act = "system-install "; + act += s->system_name; + act += '/'; + act += s->system_version; + act += " (required by "; + + bool first (true); + for (const package_name& n: j->second) + { + if (first) + first = false; + else + act += ", "; + + act += "sys:"; + act += n.string (); + } + + act += ')'; + need_prompt = true; + + // Make sure that we print this system-install action just once. + // + sys_map.erase (j); + + // Note that we don't increment i in order to re-iterate this pkgs + // entry. } else { - // Print configuration variables. - // - // The idea here is to only print configuration for those packages - // for which we call pkg_configure*() in execute_plan(). - // - package_skeleton* cfg (nullptr); + ++i; - string cause; - if (*p.action == build_package::adjust) - { - assert (sp != nullptr && (p.reconfigure () || p.unhold ())); + database& pdb (p.db); + const shared_ptr& sp (p.selected); - // This is a dependent needing reconfiguration. + if (*p.action == build_package::drop) + { + act = "drop " + sp->string (pdb) + " (unused)"; + need_prompt = true; + } + else + { + // Print configuration variables. // - // This is an implicit reconfiguration which requires the plan to - // be printed. Will flag that later when composing the list of - // prerequisites. + // The idea here is to only print configuration for those packages + // for which we call pkg_configure*() in execute_plan(). // - if (p.reconfigure ()) - { - act = "reconfigure"; - cause = "dependent of"; + package_skeleton* cfg (nullptr); - if (!o.configure_only ()) - update_dependents = true; - } - - // This is a held package needing unhold. - // - if (p.unhold ()) + string cause; + if (*p.action == build_package::adjust) { - if (act.empty ()) - act = "unhold"; - else - act += "/unhold"; - } - - act += ' ' + sp->name.string (); + assert (sp != nullptr && (p.reconfigure () || p.unhold ())); - const string& s (pdb.string); - if (!s.empty ()) - act += ' ' + s; + // This is a dependent needing reconfiguration. + // + // This is an implicit reconfiguration which requires the plan + // to be printed. Will flag that later when composing the list + // of prerequisites. + // + if (p.reconfigure ()) + { + act = "reconfigure"; + cause = "dependent of"; - // This is an adjustment and so there is no available package - // specified for the build package object and thus the skeleton - // cannot be present. - // - assert (p.available == nullptr && !p.skeleton); + if (!o.configure_only ()) + update_dependents = true; + } - // We shouldn't be printing configurations for plain unholds. - // - if (p.reconfigure ()) - { - // Since there is no available package specified we need to find - // it (or create a transient one). + // This is a held package needing unhold. // - cfg = &p.init_skeleton (o, find_available (o, pdb, sp)); - } - } - else - { - assert (p.available != nullptr); // This is a package build. + if (p.unhold ()) + { + if (act.empty ()) + act = "unhold"; + else + act += "/unhold"; + } - // Even if we already have this package selected, we have to - // make sure it is configured and updated. - // - if (sp == nullptr) - { - act = p.system ? "configure" : "new"; + act += ' ' + sp->name.string (); - // For a new non-system package the skeleton must already be - // initialized. + const string& s (pdb.string); + if (!s.empty ()) + act += ' ' + s; + + // This is an adjustment and so there is no available package + // specified for the build package object and thus the skeleton + // cannot be present. // - assert (p.system || p.skeleton.has_value ()); + assert (p.available == nullptr && !p.skeleton); - // Initialize the skeleton if it is not initialized yet. + // We shouldn't be printing configurations for plain unholds. // - cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + if (p.reconfigure ()) + { + // Since there is no available package specified we need to + // find it (or create a transient one). + // + cfg = &p.init_skeleton (o, find_available (o, pdb, sp)); + } } - else if (sp->version == p.available_version ()) + else { - // If this package is already configured and is not part of the - // user selection (or we are only configuring), then there is - // nothing we will be explicitly doing with it (it might still - // get updated indirectly as part of the user selection update). + assert (p.available != nullptr); // This is a package build. + + // Even if we already have this package selected, we have to + // make sure it is configured and updated. // - if (!p.reconfigure () && - sp->state == package_state::configured && - (!p.user_selection () || - o.configure_only () || - p.configure_only ())) - continue; + if (sp == nullptr) + { + act = p.system ? "configure" : "new"; - act = p.system - ? "reconfigure" - : (p.reconfigure () - ? (o.configure_only () || p.configure_only () - ? "reconfigure" - : "reconfigure/update") - : "update"); + // For a new non-system package the skeleton must already be + // initialized. + // + assert (p.system || p.skeleton.has_value ()); - if (p.reconfigure ()) - { // Initialize the skeleton if it is not initialized yet. // cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); } - } - else - { - act = p.system - ? "reconfigure" - : sp->version < p.available_version () - ? "upgrade" - : "downgrade"; - - // For a non-system package up/downgrade the skeleton must - // already be initialized. - // - assert (p.system || p.skeleton.has_value ()); + else if (sp->version == p.available_version ()) + { + // If this package is already configured and is not part of + // the user selection (or we are only configuring), then there + // is nothing we will be explicitly doing with it (it might + // still get updated indirectly as part of the user selection + // update). + // + if (!p.reconfigure () && + sp->state == package_state::configured && + (!p.user_selection () || + o.configure_only () || + p.configure_only ())) + continue; - // Initialize the skeleton if it is not initialized yet. - // - cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + act = p.system + ? "reconfigure" + : (p.reconfigure () + ? (o.configure_only () || p.configure_only () + ? "reconfigure" + : "reconfigure/update") + : "update"); - need_prompt = true; - } + if (p.reconfigure ()) + { + // Initialize the skeleton if it is not initialized yet. + // + cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); + } + } + else + { + act += p.system + ? "reconfigure" + : sp->version < p.available_version () + ? "upgrade" + : "downgrade"; + + // For a non-system package up/downgrade the skeleton must + // already be initialized. + // + assert (p.system || p.skeleton.has_value ()); - if (p.unhold ()) - act += "/unhold"; + // Initialize the skeleton if it is not initialized yet. + // + cfg = &(p.skeleton ? *p.skeleton : p.init_skeleton (o)); - act += ' ' + p.available_name_version_db (); - cause = p.required_by_dependents ? "required by" : "dependent of"; + need_prompt = true; + } - if (p.configure_only ()) - update_dependents = true; - } + if (p.unhold ()) + act += "/unhold"; - // Also list dependents for the newly built user-selected - // dependencies. - // - bool us (p.user_selection ()); - string rb; - if (!us || (!p.user_selection (hold_pkgs) && sp == nullptr)) - { - // Note: if we are ever tempted to truncate this, watch out for - // the --rebuild-checksum functionality which uses this. But then - // it's not clear this information is actually important: can a - // dependent-dependency structure change without any of the - // package versions changing? Doesn't feel like it should. + act += ' ' + p.available_name_version_db (); + cause = p.required_by_dependents ? "required by" : "dependent of"; + + if (p.configure_only ()) + update_dependents = true; + } + + // Also list dependents for the newly built user-selected + // dependencies. // - for (const package_key& pk: p.required_by) + bool us (p.user_selection ()); + string rb; + if (!us || (!p.user_selection (hold_pkgs) && sp == nullptr)) { - // Skip the command-line dependent. + // Note: if we are ever tempted to truncate this, watch out for + // the --rebuild-checksum functionality which uses this. But + // then it's not clear this information is actually important: + // can a dependent-dependency structure change without any of + // the package versions changing? Doesn't feel like it should. // - if (!pk.name.empty ()) - rb += (rb.empty () ? " " : ", ") + pk.string (); + for (const package_key& pk: p.required_by) + { + // Skip the command-line dependent. + // + if (!pk.name.empty ()) + rb += (rb.empty () ? " " : ", ") + pk.string (); + } + + // If not user-selected, then there should be another (implicit) + // reason for the action. + // + assert (!rb.empty ()); } - // If not user-selected, then there should be another (implicit) - // reason for the action. - // - assert (!rb.empty ()); - } + if (!rb.empty ()) + act += " (" + cause + rb + ')'; - if (!rb.empty ()) - act += " (" + cause + rb + ')'; + if (cfg != nullptr && !cfg->empty_print ()) + { + ostringstream os; + cfg->print_config (os, o.print_only () ? " " : " "); + act += '\n'; + act += os.str (); + } - if (cfg != nullptr && !cfg->empty_print ()) - { - ostringstream os; - cfg->print_config (os, o.print_only () ? " " : " "); - act += '\n'; - act += os.str (); + if (!us) + need_prompt = true; } - - if (!us) - need_prompt = true; } if (first) @@ -4554,13 +4756,14 @@ namespace bpkg // Ok, we have "all systems go". The overall action plan is as follows. // - // 1. disfigure up/down-graded, reconfigured [left to right] - // 2. purge up/down-graded [right to left] - // 3.a fetch/unpack new, up/down-graded - // 3.b checkout new, up/down-graded - // 4. configure all - // 5. unhold unheld - // 6. build user selection [right to left] + // 1. system-install not installed system/distribution + // 2. disfigure up/down-graded, reconfigured [left to right] + // 3. purge up/down-graded [right to left] + // 4.a fetch/unpack new, up/down-graded + // 4.b checkout new, up/down-graded + // 5. configure all + // 6. unhold unheld + // 7. build user selection [right to left] // // Note that for some actions, e.g., purge or fetch, the order is not // really important. We will, however, do it right to left since that @@ -4668,6 +4871,40 @@ namespace bpkg size_t prog_i, prog_n, prog_percent; + // system-install + // + // Install the system/distribution packages required by the respective + // system packages (see build_package::system_install() for details). + // + if (!simulate && o.sys_install ()) + { + // Collect the names of all the system packages being managed by the + // system package manager (as opposed to user/fallback), suppressing + // duplicates. + // + vector ps; + + for (build_package& p: build_pkgs) + { + if (p.system_status () && + find (ps.begin (), ps.end (), p.name ()) == ps.end ()) + { + ps.push_back (p.name ()); + } + } + + // Install the system/distribution packages. + // + if (!ps.empty ()) + { + // Otherwise, we wouldn't get any package statuses. + // + assert (sys_pkg_mgr && *sys_pkg_mgr != nullptr); + + (*sys_pkg_mgr)->pkg_install (ps); + } + } + // disfigure // // Note: similar code in pkg-drop. diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx new file mode 100644 index 0000000..06b5060 --- /dev/null +++ b/bpkg/system-package-manager-debian.cxx @@ -0,0 +1,1411 @@ +// file : bpkg/system-package-manager-debian.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include + +using namespace butl; + +namespace bpkg +{ + using package_status = system_package_status_debian; + + // Parse the debian-name (or alike) value. + // + // Note that for now we treat all the packages from the non-main groups as + // extras omitting the -common package (assuming it's pulled by the main + // package) as well as -doc and -dbg unless requested with the + // extra_{doc,dbg} arguments. + // + package_status system_package_manager_debian:: + parse_name_value (const package_name& pn, + const string& nv, + bool extra_doc, + bool extra_dbg) + { + auto split = [] (const string& s, char d) -> strings + { + strings r; + for (size_t b (0), e (0); next_word (s, b, e, d); ) + r.push_back (string (s, b, e - b)); + return r; + }; + + auto suffix = [] (const string& n, const string& s) -> bool + { + size_t nn (n.size ()); + size_t sn (s.size ()); + return nn > sn && n.compare (nn - sn, sn, s) == 0; + }; + + auto parse_group = [&split, &suffix] (const string& g, + const package_name* pn) + { + strings ns (split (g, ' ')); + + if (ns.empty ()) + fail << "empty package group"; + + package_status r; + + // Handle the "dev instead of main" special case for libraries. + // + // Note: the lib prefix check is based on the bpkg package name. + // + // Check that the following name does not end with -dev. This will be + // the only way to disambiguate the case where the library name happens + // to end with -dev (e.g., libfoo-dev libfoo-dev-dev). + // + { + string& m (ns[0]); + + if (pn != nullptr && + pn->string ().compare (0, 3, "lib") == 0 && + suffix (m, "-dev") && + !(ns.size () > 1 && suffix (ns[1], "-dev"))) + { + r = package_status ("", move (m)); + } + else + r = package_status (move (m)); + } + + // Handle the rest. + // + for (size_t i (1); i != ns.size (); ++i) + { + string& n (ns[i]); + + const char* w; + if (string* v = (suffix (n, (w = "-dev")) ? &r.dev : + suffix (n, (w = "-doc")) ? &r.doc : + suffix (n, (w = "-dbg")) ? &r.dbg : + suffix (n, (w = "-common")) ? &r.common : nullptr)) + { + if (!v->empty ()) + fail << "multiple " << w << " package names in '" << g << "'" << + info << "did you forget to separate package groups with comma?"; + + *v = move (n); + } + else + r.extras.push_back (move (n)); + } + + return r; + }; + + strings gs (split (nv, ',')); + assert (!gs.empty ()); // *-name value cannot be empty. + + package_status r; + for (size_t i (0); i != gs.size (); ++i) + { + if (i == 0) // Main group. + r = parse_group (gs[i], &pn); + else + { + package_status g (parse_group (gs[i], nullptr)); + + if (!g.main.empty ()) r.extras.push_back (move (g.main)); + if (!g.dev.empty ()) r.extras.push_back (move (g.dev)); + if (!g.doc.empty () && extra_doc) r.extras.push_back (move (g.doc)); + if (!g.dbg.empty () && extra_dbg) r.extras.push_back (move (g.dbg)); + if (!g.common.empty () && false) r.extras.push_back (move (g.common)); + if (!g.extras.empty ()) r.extras.insert ( + r.extras.end (), + make_move_iterator (g.extras.begin ()), + make_move_iterator (g.extras.end ())); + } + } + + return r; + } + + // Attempt to determine the main package name from its -dev package based on + // the extracted Depends value. Return empty string if unable to. + // + string system_package_manager_debian:: + main_from_dev (const string& dev_name, + const string& dev_ver, + const string& depends) + { + // The format of the Depends value is a comma-seperated list of dependency + // expressions. For example: + // + // Depends: libssl3 (= 3.0.7-1), libc6 (>= 2.34), libfoo | libbar + // + // For the main package we look for a dependency in the form: + // + // * (= ) + // + // Usually it is the first one. + // + string dev_stem (dev_name, 0, dev_name.rfind ("-dev")); + + string r; + for (size_t b (0), e (0); next_word (depends, b, e, ','); ) + { + string d (depends, b, e - b); + trim (d); + + size_t p (d.find (' ')); + if (p != string::npos) + { + if (d.compare (0, dev_stem.size (), dev_stem) == 0) // * + { + size_t q (d.find ('(', p + 1)); + if (q != string::npos && d.back () == ')') // (...) + { + if (d[q + 1] == '=' && d[q + 2] == ' ') // Equal. + { + string v (d, q + 3, d.size () - q - 3 - 1); + trim (v); + + if (v == dev_ver) + { + r.assign (d, 0, p); + break; + } + } + } + } + } + } + + return r; + } + + // Do we use apt or apt-get? From apt(8): + // + // "The apt(8) commandline is designed as an end-user tool and it may change + // behavior between versions. [...] + // + // All features of apt(8) are available in dedicated APT tools like + // apt-get(8) and apt-cache(8) as well. [...] So you should prefer using + // these commands (potentially with some additional options enabled) in + // your scripts as they keep backward compatibility as much as possible." + // + // Note also that for some reason both apt-cache and apt-get exit with 100 + // code on error. + // + static process_path apt_cache_path; + static process_path apt_get_path; + static process_path sudo_path; + + // Obtain the installed and candidate versions for the specified list of + // Debian packages by executing `apt-cache policy`. + // + // If the n argument is not 0, then only query the first n packages. + // + void system_package_manager_debian:: + apt_cache_policy (vector& pps, size_t n) + { + if (n == 0) + n = pps.size (); + + assert (n != 0 && n <= pps.size ()); + + // The --quiet option makes sure we don't get a noice (N) printed to + // stderr if the package is unknown. It does not appear to affect error + // diagnostics (try temporarily renaming /var/lib/dpkg/status). + // + cstrings args {"apt-cache", "policy", "--quiet"}; + + for (size_t i (0); i != n; ++i) + { + package_policy& pp (pps[i]); + + const string& n (pp.name); + assert (!n.empty ()); + + pp.installed_version.clear (); + pp.candidate_version.clear (); + + args.push_back (n.c_str ()); + } + + args.push_back (nullptr); + + // Run with the C locale to make sure there is no localization. Note that + // this is not without potential drawbacks, see Debian bug #643787. But + // for now it seems to work and feels like the least of two potential + // evils. + // + const char* evars[] = {"LC_ALL=C", nullptr}; + + try + { + if (apt_cache_path.empty () && !simulate_) + apt_cache_path = process::path_search (args[0]); + + process_env pe (apt_cache_path, evars); + + if (verb >= 3) + print_process (pe, args); + + // Redirect stdout to a pipe. For good measure also redirect stdin to + // /dev/null to make sure there are no prompts of any kind. + // + process pr; + if (!simulate_) + pr = process (apt_cache_path, + args, + -2 /* stdin */, + -1 /* stdout */, + 2 /* stderr */, + nullptr /* cwd */, + evars); + else + { + strings k; + for (size_t i (0); i != n; ++i) + k.push_back (pps[i].name); + + const path* f (nullptr); + if (installed_) + { + auto i (simulate_->apt_cache_policy_installed_.find (k)); + if (i != simulate_->apt_cache_policy_installed_.end ()) + f = &i->second; + } + if (f == nullptr && fetched_) + { + auto i (simulate_->apt_cache_policy_fetched_.find (k)); + if (i != simulate_->apt_cache_policy_fetched_.end ()) + f = &i->second; + } + if (f == nullptr) + { + auto i (simulate_->apt_cache_policy_.find (k)); + if (i != simulate_->apt_cache_policy_.end ()) + f = &i->second; + } + + diag_record dr (text); + print_process (dr, pe, args); + dr << " <" << (f == nullptr || f->empty () ? "/dev/null" : f->string ()); + + pr = process (process_exit (0)); + pr.in_ofd = f == nullptr || f->empty () + ? fdopen_null () + : (f->string () == "-" + ? fddup (stdin_fd ()) + : fdopen (*f, fdopen_mode::in)); + } + + try + { + ifdstream is (move (pr.in_ofd), fdstream_mode::skip, ifdstream::badbit); + + // The output of `apt-cache policy ...` are blocks of + // lines in the following form: + // + // : + // Installed: 1.2.3-1 + // Candidate: 1.3.0-2 + // Version table: + // <...> + // : + // Installed: (none) + // Candidate: 1.3.0+dfsg-2+b1 + // Version table: + // <...> + // + // Where <...> are further lines indented with at least one space. If + // a package is unknown, then the entire block (including the first + // : line) is omitted. The blocks appear in the same order as + // packages on the command line and multiple entries for the same + // package result in multiple corresponding blocks. It looks like + // there should be not blank lines but who really knows. + // + // Note also that if Installed version is not (none), then the + // Candidate version will be that version of better. + // + { + auto df = make_diag_frame ( + [&pe, &args] (diag_record& dr) + { + dr << info << "while parsing output of "; + print_process (dr, pe, args); + }); + + size_t i (0); + + string l; + for (getline (is, l); !eof (is); ) + { + // Parse the first line of the block. + // + if (l.empty () || l.front () == ' ' || l.back () != ':') + fail << "expected package name instead of '" << l << "'"; + + l.pop_back (); + + // Skip until this package. + // + for (; i != n && pps[i].name != l; ++i) ; + + if (i == n) + fail << "unexpected package name '" << l << "'"; + + package_policy& pp (pps[i]); + + auto parse_version = [&l] (const string& n) -> string + { + size_t s (n.size ()); + + if (l[0] == ' ' && + l[1] == ' ' && + l.compare (2, s, n) == 0 && + l[2 + s] == ':') + { + string v (l, 2 + s + 1); + trim (v); + + if (!v.empty ()) + return v == "(none)" ? string () : move (v); + } + + fail << "invalid " << n << " version line '" << l << "'" << endf; + }; + + // Get the installed version line. + // + if (eof (getline (is, l))) + fail << "expected Installed version line after package name"; + + pp.installed_version = parse_version ("Installed"); + + // Get the candidate version line. + // + if (eof (getline (is, l))) + fail << "expected Candidate version line after Installed version"; + + pp.candidate_version = parse_version ("Candidate"); + + // Candidate should fallback to Installed. + // + assert (pp.installed_version.empty () || + !pp.candidate_version.empty ()); + + // Skip the rest of the indented lines (or blanks, just in case). + // + while (!eof (getline (is, l)) && (l.empty () || l.front () == ' ')) ; + } + } + + is.close (); + } + catch (const io_error& e) + { + if (pr.wait ()) + fail << "unable to read " << args[0] << " policy output: " << e; + + // Fall through. + } + + if (!pr.wait ()) + { + diag_record dr (fail); + dr << args[0] << " policy exited with non-zero code"; + + if (verb < 3) + { + dr << info << "command line: "; + print_process (dr, pe, args); + } + } + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + } + + // Execute `apt-cache show` and return the Depends value, if any, for the + // specified package and version. Fail if either package or version is + // unknown. + // + string system_package_manager_debian:: + apt_cache_show (const string& name, const string& ver) + { + assert (!name.empty () && !ver.empty ()); + + string spec (name + '=' + ver); + + // In particular, --quiet makes sure we don't get noices (N) printed to + // stderr. It does not appear to affect error diagnostics (try showing + // information for an unknown package). + // + const char* args[] = { + "apt-cache", "show", "--quiet", spec.c_str (), nullptr}; + + // Note that for this command there seems to be no need to run with the C + // locale since the output is presumably not localizable. But let's do it + // for good measure and also seeing that we try to backfit some + // diagnostics into apt-cache (see no_version below). + // + const char* evars[] = {"LC_ALL=C", nullptr}; + + string r; + try + { + if (apt_cache_path.empty () && !simulate_) + apt_cache_path = process::path_search (args[0]); + + process_env pe (apt_cache_path, evars); + + if (verb >= 3) + print_process (pe, args); + + // Redirect stdout to a pipe. For good measure also redirect stdin to + // /dev/null to make sure there are no prompts of any kind. + // + process pr; + if (!simulate_) + pr = process (apt_cache_path, + args, + -2 /* stdin */, + -1 /* stdout */, + 2 /* stderr */, + nullptr /* cwd */, + evars); + else + { + pair k (name, ver); + + const path* f (nullptr); + if (fetched_) + { + auto i (simulate_->apt_cache_show_fetched_.find (k)); + if (i != simulate_->apt_cache_show_fetched_.end ()) + f = &i->second; + } + if (f == nullptr) + { + auto i (simulate_->apt_cache_show_.find (k)); + if (i != simulate_->apt_cache_show_.end ()) + f = &i->second; + } + + diag_record dr (text); + print_process (dr, pe, args); + dr << " <" << (f == nullptr || f->empty () ? "/dev/null" : f->string ()); + + if (f == nullptr || f->empty ()) + { + text << "E: No packages found"; + pr = process (process_exit (100)); + } + else + { + pr = process (process_exit (0)); + pr.in_ofd = f->string () == "-" + ? fddup (stdin_fd ()) + : fdopen (*f, fdopen_mode::in); + } + } + + bool no_version (false); + try + { + ifdstream is (move (pr.in_ofd), fdstream_mode::skip, ifdstream::badbit); + + // The output of `apt-cache show =` appears to be a single + // Debian control file in the RFC 822 encoding followed by a blank + // line. See deb822(5) for details. Here is a representative example: + // + // Package: libcurl4 + // Version: 7.85.0-1 + // Depends: libbrotli1 (>= 0.6.0), libc6 (>= 2.34), ... + // Description-en: easy-to-use client-side URL transfer library + // libcurl is an easy-to-use client-side URL transfer library. + // + // Note that if the package is unknown, then we get an error but if + // the version is unknown, we get no output (and a note if running + // without --quiet). + // + string l; + if (eof (getline (is, l))) + { + // The unknown version case. Issue diagnostics consistent with the + // unknown package case, at least for the English locale. + // + text << "E: No package version found"; + no_version = true; + } + else + { + auto df = make_diag_frame ( + [&pe, &args] (diag_record& dr) + { + dr << info << "while parsing output of "; + print_process (dr, pe, args); + }); + + do + { + // This line should be the start of a field unless it's a comment + // or the terminating blank line. According to deb822(5), there + // can be no leading whitespaces before `#`. + // + if (l.empty ()) + break; + + if (l[0] == '#') + { + getline (is, l); + continue; + } + + size_t p (l.find (':')); + + if (p == string::npos) + fail << "expected field name instead of '" << l << "'"; + + // Extract the field name. Note that field names are case- + // insensitive. + // + string n (l, 0, p); + trim (n); + + // Extract the field value. + // + string v (l, p + 1); + trim (v); + + // If we have more lines see if the following line is part of this + // value. + // + while (!eof (getline (is, l)) && (l[0] == ' ' || l[0] == '\t')) + { + // This can either be a "folded" or a "multiline" field and + // which one it is depends on the field semantics. Here we only + // care about Depends and so treat them all as folded (it's + // unclear whether Depends must be a simple field). + // + trim (l); + v += ' '; + v += l; + } + + // See if this is a field of interest. + // + if (icasecmp (n, "Package") == 0) + { + assert (v == name); // Sanity check. + } + else if (icasecmp (n, "Version") == 0) + { + assert (v == ver); // Sanity check. + } + else if (icasecmp (n, "Depends") == 0) + { + r = move (v); + + // Let's not waste time reading any further. + // + break; + } + } + while (!eof (is)); + } + + is.close (); + } + catch (const io_error& e) + { + if (pr.wait ()) + fail << "unable to read " << args[0] << " show output: " << e; + + // Fall through. + } + + if (!pr.wait () || no_version) + { + diag_record dr (fail); + dr << args[0] << " show exited with non-zero code"; + + if (verb < 3) + { + dr << info << "command line: "; + print_process (dr, pe, args); + } + } + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + return r; + } + + // Prepare the common `apt-get ` options. + // + pair system_package_manager_debian:: + apt_get_common (const char* command) + { + cstrings args; + + if (!sudo_.empty ()) + args.push_back (sudo_.c_str ()); + + args.push_back ("apt-get"); + args.push_back (command); + + // Map our verbosity/progress to apt-get --quiet[=]. The levels + // appear to have the following behavior: + // + // 1 -- shows URL being downloaded but no percentage progress is shown. + // + // 2 -- only shows diagnostics (implies --assume-yes which cannot be + // overriden with --assume-no). + // + // It also appears to automatically use level 1 if stderr is not a + // terminal. This can be overrident with --quiet=0. + // + // Note also that --show-progress does not apply to apt-get update. For + // apt-get install it shows additionally progress during unpacking which + // looks quite odd. + // + if (progress_ && *progress_) + { + args.push_back ("--quiet=0"); + } + else if (verb == 0) + { + // Only use level 2 if assuming yes. + // + args.push_back (yes_ ? "--quiet=2" : "--quiet"); + } + else if (progress_ && !*progress_) + { + args.push_back ("--quiet"); + } + + if (yes_) + { + args.push_back ("--assume-yes"); + } + else if (!stderr_term) + { + // Suppress any prompts if stderr is not a terminal for good measure. + // + args.push_back ("--assume-no"); + } + + try + { + const process_path* pp (nullptr); + + if (!sudo_.empty ()) + { + if (sudo_path.empty () && !simulate_) + sudo_path = process::path_search (args[0]); + + pp = &sudo_path; + } + else + { + if (apt_get_path.empty () && !simulate_) + apt_get_path = process::path_search (args[0]); + + pp = &apt_get_path; + } + + return pair (move (args), *pp); + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + } + + // Execute `apt-get update` to update the package index. + // + void system_package_manager_debian:: + apt_get_update () + { + pair args_pp (apt_get_common ("update")); + + cstrings& args (args_pp.first); + const process_path& pp (args_pp.second); + + args.push_back (nullptr); + + try + { + if (verb >= 2) + print_process (args); + else if (verb == 1) + text << "updating " << os_release_.name_id << " package index..."; + + process pr; + if (!simulate_) + pr = process (pp, args); + else + { + print_process (args); + pr = process (process_exit (simulate_->apt_get_update_fail_ ? 100 : 0)); + } + + if (!pr.wait ()) + { + diag_record dr (fail); + dr << "apt-get update exited with non-zero code"; + + if (verb < 2) + { + dr << info << "command line: "; + print_process (dr, args); + } + } + + if (verb == 1) + text << "updated " << os_release_.name_id << " package index"; + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + } + + // Execute `apt-get install` to install the specified packages/versions + // (e.g., libfoo or libfoo=1.2.3). + // + void system_package_manager_debian:: + apt_get_install (const strings& pkgs) + { + assert (!pkgs.empty ()); + + pair args_pp (apt_get_common ("install")); + + cstrings& args (args_pp.first); + const process_path& pp (args_pp.second); + + for (const string& p: pkgs) + args.push_back (p.c_str ()); + + args.push_back (nullptr); + + try + { + if (verb >= 2) + print_process (args); + else if (verb == 1) + text << "installing " << os_release_.name_id << " packages..."; + + process pr; + if (!simulate_) + pr = process (pp, args); + else + { + print_process (args); + pr = process (process_exit (simulate_->apt_get_install_fail_ ? 100 : 0)); + } + + if (!pr.wait ()) + { + diag_record dr (fail); + dr << "apt-get install exited with non-zero code"; + + if (verb < 2) + { + dr << info << "command line: "; + print_process (dr, args); + } + + dr << info << "consider resolving the issue manually and retrying " + << "the bpkg command"; + } + + if (verb == 1) + text << "installed " << os_release_.name_id << " packages"; + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + } + + optional system_package_manager_debian:: + pkg_status (const package_name& pn, const available_packages* aps) + { + // For now we ignore -doc and -dbg package components (but we may want to + // have options controlling this later). Note also that we assume -common + // is pulled automatically by the main package so we ignore it as well + // (see equivalent logic in parse_name_value()). + // + bool need_doc (false); + bool need_dbg (false); + + // First check the cache. + // + { + auto i (status_cache_.find (pn)); + + if (i != status_cache_.end ()) + return i->second ? &*i->second : nullptr; + + if (aps == nullptr) + return nullopt; + } + + vector candidates; + + // Translate our package name to the Debian package names. + // + { + auto df = make_diag_frame ( + [this, &pn] (diag_record& dr) + { + dr << info << "while mapping " << pn << " to " + << os_release_.name_id << " package name"; + }); + + strings ns (system_package_names (*aps, + os_release_.name_id, + os_release_.version_id, + os_release_.like_ids)); + if (ns.empty ()) + { + // Attempt to automatically translate our package name (see above for + // details). + // + const string& n (pn.string ()); + + // The best we can do in trying to detect whether this is a library is + // to check for the lib prefix. Libraries without the lib prefix and + // non-libraries with the lib prefix (both of which we do not + // recomment) will have to provide a manual mapping. + // + if (n.compare (0, 3, "lib") == 0) + { + // Keep the main package name empty as an indication that it is to + // be discovered. + // + candidates.push_back (package_status ("", n + "-dev")); + } + else + candidates.push_back (package_status (n)); + } + else + { + // Parse each manual mapping. + // + for (const string& n: ns) + { + package_status s (parse_name_value (pn, n, need_doc, need_dbg)); + + // Suppress duplicates for good measure based on the main package + // name (and falling back to -dev if empty). + // + auto i (find_if (candidates.begin (), candidates.end (), + [&s] (const package_status& x) + { + // Note that it's possible for one mapping to be + // specified as -dev only while the other as main + // and -dev. + // + return s.main.empty () || x.main.empty () + ? s.dev == x.dev + : s.main == x.main; + })); + if (i == candidates.end ()) + candidates.push_back (move (s)); + else + { + // Should we verify the rest matches for good measure? But what if + // we need to override, as in: + // + // debian_10-name: libcurl4 libcurl4-openssl-dev + // debian_9-name: libcurl4 libcurl4-dev + // + // Note that for this to work we must get debian_10 values before + // debian_9, which is the semantics guaranteed by + // system_package_names(). + } + } + } + } + + // Guess unknown main package given the -dev package and its version. + // + auto guess_main = [this, &pn] (package_status& s, const string& ver) + { + string depends (apt_cache_show (s.dev, ver)); + + s.main = main_from_dev (s.dev, ver, depends); + + if (s.main.empty ()) + { + fail << "unable to guess main " << os_release_.name_id + << " package for " << s.dev << ' ' << ver << + info << s.dev << " Depends value: " << depends << + info << "consider specifying explicit mapping in " << pn + << " package manifest"; + } + }; + + // Calculate the package status from individual package components. + // Return nullopt if there is a component without installed or candidate + // version (which means the package cannot be installed). + // + // The main argument specifies the size of the main group. Only components + // from this group are considered for partially_installed determination. + // + // @@ TODO: we should probably prioritize partially installed with fully + // installed main group. Add almost_installed next to partially_installed? + // + using status_type = package_status::status_type; + + auto status = [] (const vector& pps, size_t main) + -> optional + { + bool i (false), u (false); + + for (size_t j (0); j != pps.size (); ++j) + { + const package_policy& pp (pps[j]); + + if (pp.installed_version.empty ()) + { + if (pp.candidate_version.empty ()) + return nullopt; + + u = true; + } + else if (j < main) + i = true; + } + + return (!u ? package_status::installed : + !i ? package_status::not_installed : + package_status::partially_installed); + }; + + // First look for an already fully installed package. + // + optional r; + + { + diag_record dr; // Ambiguity diagnostics. + + for (package_status& ps: candidates) + { + vector& pps (ps.package_policies); + + if (!ps.main.empty ()) pps.emplace_back (ps.main); + if (!ps.dev.empty ()) pps.emplace_back (ps.dev); + if (!ps.doc.empty () && need_doc) pps.emplace_back (ps.doc); + if (!ps.dbg.empty () && need_dbg) pps.emplace_back (ps.dbg); + if (!ps.common.empty () && false) pps.emplace_back (ps.common); + ps.package_policies_main = pps.size (); + for (const string& n: ps.extras) pps.emplace_back (n); + + apt_cache_policy (pps); + + // Handle the unknown main package. + // + if (ps.main.empty ()) + { + const package_policy& dev (pps.front ()); + + // Note that at this stage we can only use the installed -dev + // package (since the candidate version may change after fetch). + // + if (dev.installed_version.empty ()) + continue; + + guess_main (ps, dev.installed_version); + pps.emplace (pps.begin (), ps.main); + ps.package_policies_main++; + apt_cache_policy (pps, 1); + } + + optional s (status (pps, ps.package_policies_main)); + + if (!s || *s != package_status::installed) + continue; + + const package_policy& main (pps.front ()); + + ps.status = *s; + ps.system_name = main.name; + ps.system_version = main.installed_version; + + if (!r) + { + r = move (ps); + continue; + } + + if (dr.empty ()) + { + dr << fail << "multiple installed " << os_release_.name_id + << " packages for " << pn << + info << "candidate: " << r->main << " " << r->system_version; + } + + dr << info << "candidate: " << ps.main << " " << ps.system_version; + } + + if (!dr.empty ()) + dr << info << "consider specifying the desired version manually"; + } + + // Next look for available versions if we are allowed to install. + // + if (!r && install_) + { + // If we weren't instructed to fetch or we already fetched, then we + // don't need to re-run apt_cache_policy(). + // + bool requery; + if ((requery = fetch_ && !fetched_)) + { + apt_get_update (); + fetched_ = true; + } + + { + diag_record dr; // Ambiguity diagnostics. + + for (package_status& ps: candidates) + { + vector& pps (ps.package_policies); + + if (requery) + apt_cache_policy (pps); + + // Handle the unknown main package. + // + if (ps.main.empty ()) + { + const package_policy& dev (pps.front ()); + + // Note that this time we use the candidate version. + // + if (dev.candidate_version.empty ()) + continue; // Not installable. + + guess_main (ps, dev.candidate_version); + pps.emplace (pps.begin (), ps.main); + ps.package_policies_main++; + apt_cache_policy (pps, 1); + } + + optional s (status (pps, ps.package_policies_main)); + + if (!s) + { + ps.main.clear (); // Not installable. + continue; + } + + assert (*s != package_status::installed); // Sanity check. + + const package_policy& main (pps.front ()); + + // Note that if we are installing something for this main package, + // then we always go for the candidate version even though it may + // have an installed version that may be good enough (especially if + // what we are installing are extras). The reason is that it may as + // well not be good enough (especially if we are installing the -dev + // package) and there is no straightforward way to change our mind. + // + ps.status = *s; + ps.system_name = main.name; + ps.system_version = main.candidate_version; + + // Prefer partially installed to not installed. This makes detecting + // ambiguity a bit trickier so we handle partially installed here + // and not installed in a separate loop below. + // + if (ps.status != package_status::partially_installed) + continue; + + if (!r) + { + r = move (ps); + continue; + } + + auto print_missing = [&dr] (const package_status& s) + { + for (const package_policy& pp: s.package_policies) + if (pp.installed_version.empty ()) + dr << ' ' << pp.name; + }; + + if (dr.empty ()) + { + dr << fail << "multiple partially installed " + << os_release_.name_id << " packages for " << pn; + + dr << info << "candidate: " << r->main << " " << r->system_version + << ", missing components:"; + print_missing (*r); + } + + dr << info << "candidate: " << ps.main << " " << ps.system_version + << ", missing components:"; + print_missing (ps); + } + + if (!dr.empty ()) + dr << info << "consider fully installing the desired package " + << "manually and retrying the bpkg command"; + } + + if (!r) + { + diag_record dr; // Ambiguity diagnostics. + + for (package_status& ps: candidates) + { + if (ps.main.empty ()) + continue; + + assert (ps.status == package_status::not_installed); // Sanity check. + + if (!r) + { + r = move (ps); + continue; + } + + if (dr.empty ()) + { + dr << fail << "multiple available " << os_release_.name_id + << " packages for " << pn << + info << "candidate: " << r->main << " " << r->system_version; + } + + dr << info << "candidate: " << ps.main << " " << ps.system_version; + } + + if (!dr.empty ()) + dr << info << "consider installing the desired package manually and " + << "retrying the bpkg command"; + } + } + + if (r) + { + // Map the Debian version to the bpkg version. But first strip the + // revision from Debian version ([:][-]), if + // any. + // + // Note that according to deb-version(5), may contain `:`/`-` + // but in these cases / must be specified explicitly, + // respectively. + // + string sv (r->system_version, 0, r->system_version.rfind ('-')); + + optional v ( + downstream_package_version (sv, + *aps, + os_release_.name_id, + os_release_.version_id, + os_release_.like_ids)); + + if (!v) + { + // Fallback to using system version as downstream version. But first + // strip the epoch, if any. + // + size_t p (sv.find (':')); + if (p != string::npos) + sv.erase (0, p + 1); + + try + { + v = version (sv); + } + catch (const invalid_argument& e) + { + fail << "unable to map " << os_release_.name_id << " package " + << r->system_name << " version " << sv << " to bpkg package " + << pn << " version" << + info << os_release_.name_id << " version is not a valid bpkg " + << "version: " << e.what () << + info << "consider specifying explicit mapping in " << pn + << " package manifest"; + } + } + + r->version = move (*v); + } + + // Cache. + // + auto i (status_cache_.emplace (pn, move (r)).first); + return i->second ? &*i->second : nullptr; + } + + void system_package_manager_debian:: + pkg_install (const vector& pns) + { + assert (!pns.empty ()); + + assert (install_ && !installed_); + installed_ = true; + + // Collect and merge all the Debian packages/version for the specified + // bpkg packages. + // + struct package + { + string name; + string version; // Empty if unspecified. + }; + vector pkgs; + + for (const package_name& pn: pns) + { + auto it (status_cache_.find (pn)); + assert (it != status_cache_.end () && it->second); + + const package_status& ps (*it->second); + + // At first it may seem we don't need to do anything for already fully + // installed packages. But it's possible some of them were automatically + // installed, meaning that they can be automatically removed if they no + // longer have any dependents (see apt-mark(8) for details). Which in + // turn means that things may behave differently depending on whether + // we've installed a package ourselves or if it was already installed. + // So instead we are going to also pass the already fully installed + // packages which will make sure they are all set to manually installed. + // But we must be careful not to force their upgrade. To achieve this + // we will specify the installed version as the desired version. + // + // Note also that for partially/not installed we don't specify the + // version, expecting the candidate version to be installed. + // + bool fi (ps.status == package_status::installed); + + for (const package_policy& pp: ps.package_policies) + { + string n (pp.name); + string v (fi ? pp.installed_version : string ()); + + auto i (find_if (pkgs.begin (), pkgs.end (), + [&n] (const package& p) + { + return p.name == n; + })); + + if (i != pkgs.end ()) + { + if (i->version.empty ()) + i->version = move (v); + else + // Feels like this cannot happen since we always use the installed + // version of the package. + // + assert (i->version == v); + } + else + pkgs.push_back (package {move (n), move (v)}); + } + } + + // Install. + // + { + // Convert to the `apt-get install` [=] form. + // + strings specs; + specs.reserve (pkgs.size ()); + for (const package& p: pkgs) + { + string s (p.name); + if (!p.version.empty ()) + { + s += '='; + s += p.version; + } + specs.push_back (move (s)); + } + + apt_get_install (specs); + } + + // Verify that versions we have promised in pkg_status() match what + // actually got installed. + // + { + vector pps; + + // Here we just check the main package component of each package. + // + for (const package_name& pn: pns) + { + const package_status& ps (*status_cache_.find (pn)->second); + + if (find_if (pps.begin (), pps.end (), + [&ps] (const package_policy& pp) + { + return pp.name == ps.system_name; + }) == pps.end ()) + { + pps.push_back (package_policy (ps.system_name)); + } + } + + apt_cache_policy (pps); + + for (const package_name& pn: pns) + { + const package_status& ps (*status_cache_.find (pn)->second); + + auto i (find_if (pps.begin (), pps.end (), + [&ps] (const package_policy& pp) + { + return pp.name == ps.system_name; + })); + assert (i != pps.end ()); + + const package_policy& pp (*i); + + if (pp.installed_version != ps.system_version) + { + fail << "unexpected " << os_release_.name_id << " package version " + << "for " << ps.system_name << + info << "expected: " << ps.system_version << + info << "installed: " << pp.installed_version << + info << "consider retrying the bpkg command"; + } + } + } + } +} diff --git a/bpkg/system-package-manager-debian.hxx b/bpkg/system-package-manager-debian.hxx new file mode 100644 index 0000000..9fb93c7 --- /dev/null +++ b/bpkg/system-package-manager-debian.hxx @@ -0,0 +1,201 @@ +// file : bpkg/system-package-manager-debian.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_SYSTEM_PACKAGE_MANAGER_DEBIAN_HXX +#define BPKG_SYSTEM_PACKAGE_MANAGER_DEBIAN_HXX + +#include + +#include +#include + +#include + +namespace bpkg +{ + // The system package manager implementation for Debian and alike (Ubuntu, + // etc) using the APT frontend. + // + // NOTE: the below description is also reproduced in the bpkg manual. + // + // For background, a library in Debian is normally split up into several + // packages: the shared library package (e.g., libfoo1 where 1 is the ABI + // version), the development files package (e.g., libfoo-dev), the + // documentation files package (e.g., libfoo-doc), the debug symbols + // package (e.g., libfoo1-dbg), and the architecture-independent files + // (e.g., libfoo1-common). All the packages except -dev are optional + // and there is quite a bit of variability here. Here are a few examples: + // + // libz3-4 libz3-dev + // + // libssl1.1 libssl-dev libssl-doc + // libssl3 libssl-dev libssl-doc + // + // libcurl4 libcurl4-openssl-dev libcurl4-doc + // libcurl3-gnutls libcurl4-gnutls-dev libcurl4-doc (yes, 3 and 4) + // + // Based on that, it seems our best bet when trying to automatically map our + // library package name to Debian package names is to go for the -dev + // package first and figure out the shared library package from that based + // on the fact that the -dev package should have the == dependency on the + // shared library package with the same version and its name should normally + // start with the -dev package's stem. + // + // For executable packages there is normally no -dev packages but -dbg, + // -doc, and -common are plausible. + // + // The format of the debian-name (or alike) manifest value is a comma- + // separated list of one or more package groups: + // + // [, ...] + // + // Where each is the space-separated list of one or more + // package names: + // + // [ ...] + // + // All the packages in the group should be "package components" (for the + // lack of a better term) of the same "logical package", such as -dev, -doc, + // -common packages. They normally have the same version. + // + // The first group is called the main group and the first package in the + // group is called the main package. Note that all the groups are consumed + // (installed) but only the main group is produced (packaged). + // + // We allow/recommend specifying the -dev package instead of the main + // package for libraries (the bpkg package name starts with lib), seeing + // that we are capable of detecting the main package automatically. If the + // library name happens to end with -dev (which poses an ambiguity), then + // the -dev package should be specified explicitly as the second package to + // disambiguate this situation (if a non-library name happened to start with + // lib and end with -dev, well, you are out of luck, I guess). + // + // Note also that for now we treat all the packages from the non-main groups + // as extras but in the future we may decide to sort them out like the main + // group (see parse_name_value() for details). + // + // The Debian package version has the [:][-] form + // (see deb-version(5) for details). If no explicit mapping to the bpkg + // version is specified with the debian-to-downstream-version (or alike) + // manifest values or none match, then we fallback to using the + // part as the bpkg version. If explicit mapping is specified, then we match + // it against the [:] parts ignoring . + // + struct system_package_status_debian: system_package_status + { + string main; + string dev; + string doc; + string dbg; + string common; + strings extras; + + // The `apt-cache policy` output. + // + struct package_policy + { + string name; + string installed_version; // Empty if none. + string candidate_version; // Empty if none and no installed_version. + + explicit + package_policy (string n): name (move (n)) {} + }; + + vector package_policies; + size_t package_policies_main = 0; // Size of the main group. + + explicit + system_package_status_debian (string m, string d = {}) + : main (move (m)), dev (move (d)) + { + assert (!main.empty () || !dev.empty ()); + } + + system_package_status_debian () = default; + }; + + class system_package_manager_debian: public system_package_manager + { + public: + virtual optional + pkg_status (const package_name&, const available_packages*) override; + + virtual void + pkg_install (const vector&) override; + + public: + // Expects os_release::name_id to be "debian" or os_release::like_ids to + // contain "debian". + // + using system_package_manager::system_package_manager; + + // Implementation details exposed for testing (see definitions for + // documentation). + // + public: + using package_status = system_package_status_debian; + using package_policy = package_status::package_policy; + + void + apt_cache_policy (vector&, size_t = 0); + + string + apt_cache_show (const string&, const string&); + + void + apt_get_update (); + + void + apt_get_install (const strings&); + + pair + apt_get_common (const char*); + + static package_status + parse_name_value (const package_name&, const string&, bool, bool); + + static string + main_from_dev (const string&, const string&, const string&); + + // If simulate is not NULL, then instead of executing the actual apt-cache + // and apt-get commands simulate their execution: (1) for apt-cache by + // printing their command lines and reading the results from files + // specified in the below apt_cache_* maps and (2) for apt-get by printing + // their command lines and failing if requested. + // + // In the (1) case if the corresponding map entry does not exist or the + // path is empty, then act as if the specified package/version is + // unknown. If the path is special "-" then read from stdin. For apt-cache + // different post-fetch and (for policy) post-install results can be + // specified (if the result is not found in one of the later maps, the + // previous map is used as a fallback). Note that the keys in the + // apt_cache_policy_* maps are the package sets and the corresponding + // result file is expected to contain (or not) the results for all of + // them. See apt_cache_policy() and apt_cache_show() implementations for + // details on the expected results. + // + struct simulation + { + std::map apt_cache_policy_; + std::map apt_cache_policy_fetched_; + std::map apt_cache_policy_installed_; + + std::map, path> apt_cache_show_; + std::map, path> apt_cache_show_fetched_; + + bool apt_get_update_fail_ = false; + bool apt_get_install_fail_ = false; + }; + + const simulation* simulate_ = nullptr; + + protected: + bool fetched_ = false; // True if already fetched metadata. + bool installed_ = false; // True if already installed. + + std::map> status_cache_; + }; +} + +#endif // BPKG_SYSTEM_PACKAGE_MANAGER_DEBIAN_HXX diff --git a/bpkg/system-package-manager-debian.test.cxx b/bpkg/system-package-manager-debian.test.cxx new file mode 100644 index 0000000..a033400 --- /dev/null +++ b/bpkg/system-package-manager-debian.test.cxx @@ -0,0 +1,348 @@ +// file : bpkg/system-package-manager-debian.test.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include + +#include +#include + +#undef NDEBUG +#include + +#include + +using namespace std; + +namespace bpkg +{ + using package_status = system_package_status_debian; + using package_policy = package_status::package_policy; + + using butl::manifest_parser; + using butl::manifest_parsing; + + // Usage: args[0] ... + // + // Where is one of: + // + // apt-cache-policy ... result comes from stdin + // + // apt-cache-show result comes from stdin + // + // parse-name-value debian-name value from stdin + // + // main-from-dev depends comes from stdin + // + // build ... [--install [--no-fetch] ...] + // + // The stdin of the build command is used to read the simulation description + // which consists of lines in the following forms (blanks are ignored): + // + // manifest: + // + // Available package manifest for one of . If none is + // specified, then a stub is automatically added. + // + // apt-cache-policy[-{fetched,installed}]: ... + // + // Values for simulation::apt_cache_policy_*. If is the special `!` + // value, then make the entry empty. + // + // apt-cache-show[-fetched]: + // + // Values for simulation::apt_cache_show_*. If is the special `!` + // value, then make the entry empty. + // + // apt-get-update-fail: true + // apt-get-install-fail: true + // + // Values for simulation::apt_get_{update,install}_fail_. + // + int + main (int argc, char* argv[]) + try + { + assert (argc >= 2); // + + string cmd (argv[1]); + + // @@ TODO: add option to customize? Maybe option before command? + // + os_release osr {"debian", {}, "10", "", "Debian", "", ""}; + + if (cmd == "apt-cache-policy") + { + assert (argc >= 3); // ... + + strings key; + vector pps; + for (int i (2); i != argc; ++i) + { + key.push_back (argv[i]); + pps.push_back (package_policy (argv[i])); + } + + system_package_manager_debian::simulation s; + s.apt_cache_policy_.emplace (move (key), path ("-")); + + system_package_manager_debian m (move (osr), + host_triplet, + false /* install */, + false /* fetch */, + nullopt /* progress */, + false /* yes */, + "sudo"); + m.simulate_ = &s; + + m.apt_cache_policy (pps); + + for (const package_policy& pp: pps) + { + cout << pp.name << " '" + << pp.installed_version << "' '" + << pp.candidate_version << "'\n"; + } + } + else if (cmd == "apt-cache-show") + { + assert (argc == 4); // + + pair key (argv[2], argv[3]); + + system_package_manager_debian::simulation s; + s.apt_cache_show_.emplace (key, path ("-")); + + system_package_manager_debian m (move (osr), + host_triplet, + false /* install */, + false /* fetch */, + nullopt /* progress */, + false /* yes */, + "sudo"); + m.simulate_ = &s; + + cout << m.apt_cache_show (key.first, key.second) << '\n'; + } + else if (cmd == "parse-name-value") + { + assert (argc == 3); // + + package_name pn (argv[2]); + + string v; + getline (cin, v); + + package_status s ( + system_package_manager_debian::parse_name_value (pn, v, false, false)); + + if (!s.main.empty ()) cout << "main: " << s.main << '\n'; + if (!s.dev.empty ()) cout << "dev: " << s.dev << '\n'; + if (!s.doc.empty ()) cout << "doc: " << s.doc << '\n'; + if (!s.dbg.empty ()) cout << "dbg: " << s.dbg << '\n'; + if (!s.common.empty ()) cout << "common: " << s.common << '\n'; + if (!s.extras.empty ()) + { + cout << "extras:"; + for (const string& e: s.extras) + cout << ' ' << e; + cout << '\n'; + } + } + else if (cmd == "main-from-dev") + { + assert (argc == 4); // + + string n (argv[2]); + string v (argv[3]); + string d; + getline (cin, d); + + cout << system_package_manager_debian::main_from_dev (n, v, d) << '\n'; + } + else if (cmd == "build") + { + assert (argc >= 3); // ... + + strings qps; + map aps; + + // Parse ... + // + int argi (2); + for (; argi != argc; ++argi) + { + string a (argv[argi]); + + if (a.compare (0, 2, "--") == 0) + break; + + aps.emplace (a, available_packages {}); + qps.push_back (move (a)); + } + + // Parse --install [--no-fetch] + // + bool install (false); + bool fetch (true); + + for (; argi != argc; ++argi) + { + string a (argv[argi]); + + if (a == "--install") install = true; + else if (a == "--no-fetch") fetch = false; + else break; + } + + // Parse the description. + // + system_package_manager_debian::simulation s; + + for (string l; !eof (getline (cin, l)); ) + { + if (l.empty ()) + continue; + + size_t p (l.find (':')); assert (p != string::npos); + string k (l, 0, p); + + if (k == "manifest") + { + size_t q (l.rfind (' ')); assert (q != string::npos); + string n (l, p + 2, q - p - 2); trim (n); + string f (l, q + 1); trim (f); + + auto i (aps.find (n)); + if (i == aps.end ()) + fail << "unknown package " << n << " in '" << l << "'"; + + i->second.push_back (make_available_from_manifest (n, f)); + } + else if ( + map* policy = + k == "apt-cache-policy" ? &s.apt_cache_policy_ : + k == "apt-cache-policy-fetched" ? &s.apt_cache_policy_fetched_ : + k == "apt-cache-policy-installed" ? &s.apt_cache_policy_installed_ : + nullptr) + { + size_t q (l.rfind (' ')); assert (q != string::npos); + string n (l, p + 2, q - p - 2); trim (n); + string f (l, q + 1); trim (f); + + strings ns; + for (size_t b (0), e (0); next_word (n, b, e); ) + ns.push_back (string (n, b, e - b)); + + if (f == "!") + f.clear (); + + policy->emplace (move (ns), path (move (f))); + } + else if (map, path>* show = + k == "apt-cache-show" ? &s.apt_cache_show_ : + k == "apt-cache-show-fetched" ? &s.apt_cache_show_fetched_ : + nullptr) + { + size_t q (l.rfind (' ')); assert (q != string::npos); + string n (l, p + 2, q - p - 2); trim (n); + string f (l, q + 1); trim (f); + + q = n.find (' '); assert (q != string::npos); + pair nv (string (n, 0, q), string (n, q + 1)); + trim (nv.second); + + if (f == "!") + f.clear (); + + show->emplace (move (nv), path (move (f))); + } + else if (k == "apt-get-update-fail") + { + s.apt_get_update_fail_ = true; + } + else if (k == "apt-get-install-fail") + { + s.apt_get_install_fail_ = true; + } + else + fail << "unknown keyword '" << k << "' in simulation description"; + } + + // Fallback to stubs and sort in the version descending order. + // + for (pair& p: aps) + { + if (p.second.empty ()) + p.second.push_back (make_available_stub (p.first)); + + sort_available (p.second); + } + + system_package_manager_debian m (move (osr), + host_triplet, + install, + fetch, + nullopt /* progress */, + false /* yes */, + "sudo"); + m.simulate_ = &s; + + // Query each package. + // + for (const string& n: qps) + { + package_name pn (n); + + const system_package_status* s (*m.pkg_status (pn, &aps[n])); + + assert (*m.pkg_status (pn, nullptr) == s); // Test caching. + + if (s == nullptr) + fail << "no installed " << (install ? "or available " : "") + << "system package for " << pn; + + cout << pn << ' ' << s->version + << " (" << s->system_name << ' ' << s->system_version << ") "; + + switch (s->status) + { + case package_status::installed: cout << "installed"; break; + case package_status::partially_installed: cout << "part installed"; break; + case package_status::not_installed: cout << "not installed"; break; + } + + cout << '\n'; + } + + // Install if requested. + // + if (install) + { + assert (argi != argc); // ... + + vector ips; + for (; argi != argc; ++argi) + ips.push_back (package_name (argv[argi])); + + m.pkg_install (ips); + } + } + else + fail << "unknown command '" << cmd << "'"; + + return 0; + } + catch (const failed&) + { + return 1; + } +} + +int +main (int argc, char* argv[]) +{ + return bpkg::main (argc, argv); +} diff --git a/bpkg/system-package-manager-debian.test.testscript b/bpkg/system-package-manager-debian.test.testscript new file mode 100644 index 0000000..b1a0030 --- /dev/null +++ b/bpkg/system-package-manager-debian.test.testscript @@ -0,0 +1,987 @@ +# file : bpkg/system-package-manager-debian.test.testscript +# license : MIT; see accompanying LICENSE file + +: apt-cache-policy +: +{ + test.arguments += apt-cache-policy + + : basics + : + $* libssl3 libssl1.1 libssl-dev libsqlite5 libxerces-c-dev <>EOE >>EOO + libssl3: + Installed: 3.0.7-1 + Candidate: 3.0.7-2 + Version table: + 3.0.7-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 3.0.7-1 100 + 100 /var/lib/dpkg/status + libssl1.1: + Installed: 1.1.1n-0+deb11u3 + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 100 + 100 /var/lib/dpkg/status + libssl-dev: + Installed: 3.0.7-1 + Candidate: 3.0.7-2 + Version table: + 3.0.7-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 3.0.7-1 100 + 100 /var/lib/dpkg/status + libxerces-c-dev: + Installed: (none) + Candidate: 3.2.4+debian-1 + Version table: + 3.2.4+debian-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + LC_ALL=C apt-cache policy --quiet libssl3 libssl1.1 libssl-dev libsqlite5 libxerces-c-dev <- + EOE + libssl3 '3.0.7-1' '3.0.7-2' + libssl1.1 '1.1.1n-0+deb11u3' '1.1.1n-0+deb11u3' + libssl-dev '3.0.7-1' '3.0.7-2' + libsqlite5 '' '' + libxerces-c-dev '' '3.2.4+debian-1' + EOO + + : empty + : + $* libsqlite5 <:'' 2>>EOE >>EOO + LC_ALL=C apt-cache policy --quiet libsqlite5 <- + EOE + libsqlite5 '' '' + EOO + + : none-none + : + $* pulseaudio <>EOE >>EOO + pulseaudio: + Installed: (none) + Candidate: (none) + Version table: + 1:11.1-1ubuntu7.5 -1 + 500 http://au.archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages + 1:11.1-1ubuntu7 -1 + 500 http://au.archive.ubuntu.com/ubuntu bionic/main amd64 Packages + EOI + LC_ALL=C apt-cache policy --quiet pulseaudio <- + EOE + pulseaudio '' '' + EOO +} + +: apt-cache-show +: +{ + test.arguments += apt-cache-show + + # Note: put Depends last to test folded/multiline parsing. + # + : basics + : + $* libssl1.1 1.1.1n-0+deb11u3 <>EOE >>EOO + Package: libssl1.1 + Status: install ok installed + Priority: optional + Section: libs + Installed-Size: 4120 + Maintainer: Debian OpenSSL Team + Architecture: amd64 + Multi-Arch: same + Source: openssl + Version: 1.1.1n-0+deb11u3 + Breaks: isync (<< 1.3.0-2), lighttpd (<< 1.4.49-2), python-boto (<< 2.44.0-1.1), python-httplib2 (<< 0.11.3-1), python-imaplib2 (<< 2.57-5), python3-boto (<< 2.44.0-1.1), python3-imaplib2 (<< 2.57-5) + Description: Secure Sockets Layer toolkit - shared libraries + This package is part of the OpenSSL project's implementation of the SSL + and TLS cryptographic protocols for secure communication over the + Internet. + . + It provides the libssl and libcrypto shared libraries. + Description-md5: 88547c6206c7fbc4fcc7d09ce100d210 + Homepage: https://www.openssl.org/ + Depends: libc6 (>= 2.25), debconf (>= 0.5) | debconf-2.0 + + EOI + LC_ALL=C apt-cache show --quiet libssl1.1=1.1.1n-0+deb11u3 <- + EOE + libc6 (>= 2.25), debconf (>= 0.5) | debconf-2.0 + EOO + + : no-depends + : + $* libssl1.1 1.1.1n-0+deb11u3 <>EOE >'' + Package: libssl1.1 + Status: install ok installed + Priority: optional + Section: libs + Installed-Size: 4120 + Maintainer: Debian OpenSSL Team + Architecture: amd64 + Multi-Arch: same + Source: openssl + Version: 1.1.1n-0+deb11u3 + Breaks: isync (<< 1.3.0-2), lighttpd (<< 1.4.49-2), python-boto (<< 2.44.0-1.1), python-httplib2 (<< 0.11.3-1), python-imaplib2 (<< 2.57-5), python3-boto (<< 2.44.0-1.1), python3-imaplib2 (<< 2.57-5) + Description: Secure Sockets Layer toolkit - shared libraries + This package is part of the OpenSSL project's implementation of the SSL + and TLS cryptographic protocols for secure communication over the + Internet. + . + It provides the libssl and libcrypto shared libraries. + Description-md5: 88547c6206c7fbc4fcc7d09ce100d210 + Homepage: https://www.openssl.org/ + + EOI + LC_ALL=C apt-cache show --quiet libssl1.1=1.1.1n-0+deb11u3 <- + EOE +} + +: parse-name-value +: +{ + test.arguments += parse-name-value + + : basics + : + $* libssl <>EOO + libssl3 libssl-common libssl-doc libssl-dev libssl-dbg libssl-extras, libc6 libc-dev libc-common libc-doc, libz-dev + EOI + main: libssl3 + dev: libssl-dev + doc: libssl-doc + dbg: libssl-dbg + common: libssl-common + extras: libssl-extras libc6 libc-dev libz-dev + EOO + + : non-lib + : + $* sqlite3 <>EOO + sqlite3 sqlite3-common sqlite3-doc + EOI + main: sqlite3 + doc: sqlite3-doc + common: sqlite3-common + EOO + + : lib-dev + : + $* libssl <>EOO + libssl-dev + EOI + dev: libssl-dev + EOO + + : non-lib-dev + : + $* ssl-dev <>EOO + ssl-dev + EOI + main: ssl-dev + EOO + + : lib-custom-dev + : + $* libfoo-dev <>EOO + libfoo-dev libfoo-dev-dev + EOI + main: libfoo-dev + dev: libfoo-dev-dev + EOO +} + +: main-from-dev +: +{ + test.arguments += main-from-dev + + : first + : + $* libssl-dev 3.0.7-1 <'libssl3' + libssl3 (= 3.0.7-1), debconf (>= 0.5) | debconf-2.0 + EOI + + : not-first + : + $* libxerces-c-dev 3.2.4+debian-1 <'libxerces-c3.2' + libc6-dev | libc-dev, libicu-dev, libxerces-c3.2 (= 3.2.4+debian-1) + EOI + + : exact + : + $* libexpat1-dev 2.5.0-1 <'libexpat1' + libexpat1 (= 2.5.0-1), libc6-dev | libc-dev + EOI + + : not-stem + : + $* libcurl4-openssl-dev 7.87.0-2 <'' + libcurl4 (= 7.87.0-2) + EOI +} + +: build +: +{ + test.arguments += build + + : libsqlite3 + : + { + : installed + : + cat <=libsqlite3-dev.policy; + libsqlite3-dev: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + cat <=libsqlite3-dev.show; + Package: libsqlite3-dev + Version: 3.40.1-1 + Depends: libsqlite3-0 (= 3.40.1-1), libc-dev + EOI + cat <=libsqlite3-0.policy; + libsqlite3-0: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* libsqlite3 --install libsqlite3 <>EOE >>EOO + apt-cache-policy: libsqlite3-dev libsqlite3-dev.policy + apt-cache-show: libsqlite3-dev 3.40.1-1 libsqlite3-dev.show + apt-cache-policy: libsqlite3-0 libsqlite3-0.policy + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev =libsqlite3-dev.policy; + libsqlite3-dev: + Installed: (none) + Candidate: 3.40.1-1 + Version table: + 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libsqlite3-dev.show; + Package: libsqlite3-dev + Version: 3.40.1-1 + Depends: libsqlite3-0 (= 3.40.1-1), libc-dev + EOI + cat <=libsqlite3-0.policy; + libsqlite3-0: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* libsqlite3 --install libsqlite3 <>EOE >>EOO + apt-cache-policy: libsqlite3-dev libsqlite3-dev.policy + apt-cache-show: libsqlite3-dev 3.40.1-1 libsqlite3-dev.show + apt-cache-policy: libsqlite3-0 libsqlite3-0.policy + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev =libsqlite3-dev.policy; + libsqlite3-dev: + Installed: (none) + Candidate: 3.39.4-1 + Version table: + 3.39.4-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libsqlite3-dev.policy-fetched; + libsqlite3-dev: + Installed: (none) + Candidate: 3.40.1-1 + Version table: + 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libsqlite3-dev.show-fetched; + Package: libsqlite3-dev + Version: 3.40.1-1 + Depends: libsqlite3-0 (= 3.40.1-1), libc-dev + EOI + cat <=libsqlite3-0.policy-fetched; + libsqlite3-0: + Installed: 3.39.4-1 + Candidate: 3.40.1-1 + Version table: + 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 3.39.4-1 100 + 100 /var/lib/dpkg/status + EOI + cat <=libsqlite3-0.policy-installed; + libsqlite3-0: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* libsqlite3 --install libsqlite3 <>EOE >>EOO + apt-cache-policy: libsqlite3-dev libsqlite3-dev.policy + apt-cache-policy-fetched: libsqlite3-dev libsqlite3-dev.policy-fetched + apt-cache-show: libsqlite3-dev 3.40.1-1 libsqlite3-dev.show-fetched + apt-cache-policy-fetched: libsqlite3-0 libsqlite3-0.policy-fetched + apt-cache-policy-installed: libsqlite3-0 libsqlite3-0.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev =libsqlite3-dev.policy; + libsqlite3-dev: + Installed: (none) + Candidate: 3.39.4-1 + Version table: + 3.39.4-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libsqlite3-dev.show; + Package: libsqlite3-dev + Version: 3.39.4-1 + Depends: libsqlite3-0 (= 3.39.4-1), libc-dev + EOI + cat <=libsqlite3-0.policy; + libsqlite3-0: + Installed: 3.39.4-1 + Candidate: 3.39.4-1 + Version table: + *** 3.39.4-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + cat <=libsqlite3-0.policy-installed; + libsqlite3-0: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* libsqlite3 --install --no-fetch libsqlite3 <>EOE >>EOO != 0 + apt-cache-policy: libsqlite3-dev libsqlite3-dev.policy + apt-cache-show: libsqlite3-dev 3.39.4-1 libsqlite3-dev.show + apt-cache-policy: libsqlite3-0 libsqlite3-0.policy + apt-cache-policy-installed: libsqlite3-0 libsqlite3-0.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev =libsqlite3-dev.policy; + libsqlite3-dev: + Installed: (none) + Candidate: 3.40.1-1 + Version table: + 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libsqlite3-dev.show; + Package: libsqlite3-dev + Version: 3.40.1-1 + Depends: libsqlite3-0 (= 3.40.1-1), libc-dev + EOI + cat <=libsqlite3-0.policy; + libsqlite3-0: + Installed: (none) + Candidate: 3.40.1-1 + Version table: + 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libsqlite3-0.policy-installed; + libsqlite3-0: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* libsqlite3 --install libsqlite3 <>EOE >>EOO + apt-cache-policy: libsqlite3-dev libsqlite3-dev.policy + apt-cache-show: libsqlite3-dev 3.40.1-1 libsqlite3-dev.show + apt-cache-policy: libsqlite3-0 libsqlite3-0.policy + apt-cache-policy-installed: libsqlite3-0 libsqlite3-0.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev =libsqlite3-dev.policy; + libsqlite3-dev: + Installed: (none) + Candidate: 3.40.1-1 + Version table: + 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + $* libsqlite3 <>EOE != 0 + apt-cache-policy: libsqlite3-dev libsqlite3-dev.policy + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev >EOE != 0 + apt-cache-policy: libsqlite3-dev ! + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev >EOE != 0 + apt-cache-policy: libsqlite3-dev ! + EOI + LC_ALL=C apt-cache policy --quiet libsqlite3-dev =sqlite3.policy; + sqlite3: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* sqlite3 --install sqlite3 <>EOE >>EOO + apt-cache-policy: sqlite3 sqlite3.policy + EOI + LC_ALL=C apt-cache policy --quiet sqlite3 =sqlite3.policy; + sqlite3: + Installed: (none) + Candidate: 3.39.4-1 + Version table: + 3.39.4-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=sqlite3.policy-fetched; + sqlite3: + Installed: (none) + Candidate: 3.40.1-1 + Version table: + 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=sqlite3.policy-installed; + sqlite3: + Installed: 3.40.1-1 + Candidate: 3.40.1-1 + Version table: + *** 3.40.1-1 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* sqlite3 --install sqlite3 <>EOE >>EOO + apt-cache-policy: sqlite3 sqlite3.policy + apt-cache-policy-fetched: sqlite3 sqlite3.policy-fetched + apt-cache-policy-installed: sqlite3 sqlite3.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet sqlite3 =libcrypto.manifest + : 1 + name: libcrypto + version: 1.1.1+18 + upstream-version: 1.1.1n + debian-name: libssl1.1 libssl-dev + debian-to-downstream-version: /1\.1\.1[a-z]/1.1.1/ + summary: OpenSSL libcrypto + license: OpenSSL + EOI + +cat <=libssl.manifest + : 1 + name: libssl + version: 1.1.1+18 + upstream-version: 1.1.1n + debian-name: libssl1.1 libssl-dev + debian-to-downstream-version: /1\.1\.1[a-z]/1.1.1/ + summary: OpenSSL libssl + license: OpenSSL + EOI + + : installed + : + ln -s ../libcrypto.manifest ./; + ln -s ../libssl.manifest ./; + cat <=libssl1.1+libssl-dev.policy; + libssl1.1: + Installed: 1.1.1n-0+deb11u3 + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 100 + 100 /var/lib/dpkg/status + libssl-dev: + Installed: 1.1.1n-0+deb11u3 + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 100 + 100 /var/lib/dpkg/status + EOI + cat <=libssl1.1.policy-installed; + libssl1.1: + Installed: 1.1.1n-0+deb11u3 + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 100 + 100 /var/lib/dpkg/status + EOI + $* libcrypto libssl --install libcrypto libssl <>EOE >>EOO + manifest: libcrypto libcrypto.manifest + manifest: libssl libssl.manifest + + apt-cache-policy: libssl1.1 libssl-dev libssl1.1+libssl-dev.policy + apt-cache-policy-installed: libssl1.1 libssl1.1.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libssl1.1 libssl-dev =libssl1.1+libssl-dev.policy; + libssl1.1: + Installed: 1.1.1n-0+deb11u3 + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 100 + 100 /var/lib/dpkg/status + libssl-dev: + Installed: (none) + Candidate: 1.1.1n-0+deb11u3 + Version table: + 1.1.1n-0+deb11u3 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libssl1.1.policy-installed; + libssl1.1: + Installed: 1.1.1n-0+deb11u3 + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 100 + 100 /var/lib/dpkg/status + EOI + $* libcrypto libssl --install libcrypto libssl <>EOE >>EOO + manifest: libcrypto libcrypto.manifest + manifest: libssl libssl.manifest + + apt-cache-policy: libssl1.1 libssl-dev libssl1.1+libssl-dev.policy + apt-cache-policy-installed: libssl1.1 libssl1.1.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libssl1.1 libssl-dev =libssl1.1+libssl-dev.policy; + libssl1.1: + Installed: (none) + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + libssl-dev: + Installed: (none) + Candidate: 1.1.1n-0+deb11u3 + Version table: + 1.1.1n-0+deb11u3 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libssl1.1.policy-installed; + libssl1.1: + Installed: 1.1.1n-0+deb11u3 + Candidate: 1.1.1n-0+deb11u3 + Version table: + *** 1.1.1n-0+deb11u3 100 + 100 /var/lib/dpkg/status + EOI + $* libcrypto libssl --install libcrypto libssl <>EOE >>EOO + manifest: libcrypto libcrypto.manifest + manifest: libssl libssl.manifest + + apt-cache-policy: libssl1.1 libssl-dev libssl1.1+libssl-dev.policy + apt-cache-policy-installed: libssl1.1 libssl1.1.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libssl1.1 libssl-dev =libcurl.manifest + : 1 + name: libcurl + version: 7.84.0 + debian-name: libcurl4 libcurl4-openssl-dev libcurl4-doc + debian-name: libcurl3-gnutls libcurl4-gnutls-dev libcurl4-doc + summary: C library for transferring data with URLs + license: curl + EOI + + + : one-full-installed + : + ln -s ../libcurl.manifest ./; + cat <=libcurl4+libcurl4-openssl-dev.policy; + libcurl4: + Installed: 7.85.0-1 + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 7.85.0-1 100 + 100 /var/lib/dpkg/status + libcurl4-openssl-dev: + Installed: 7.85.0-1 + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 7.85.0-1 100 + 100 /var/lib/dpkg/status + EOI + cat <=libcurl3-gnutls+libcurl4-gnutls-dev.policy; + libcurl3-gnutls: + Installed: 7.85.0-1 + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 7.85.0-1 100 + 100 /var/lib/dpkg/status + libcurl4-gnutls-dev: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libcurl4.policy-installed; + libcurl4: + Installed: 7.85.0-1 + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 7.85.0-1 100 + 100 /var/lib/dpkg/status + EOI + $* libcurl --install libcurl <>EOE >>EOO + manifest: libcurl libcurl.manifest + + apt-cache-policy: libcurl4 libcurl4-openssl-dev libcurl4+libcurl4-openssl-dev.policy + apt-cache-policy: libcurl3-gnutls libcurl4-gnutls-dev libcurl3-gnutls+libcurl4-gnutls-dev.policy + apt-cache-policy-installed: libcurl4 libcurl4.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libcurl4 libcurl4-openssl-dev =libcurl4+libcurl4-openssl-dev.policy; + libcurl4: + Installed: 7.85.0-1 + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 7.85.0-1 100 + 100 /var/lib/dpkg/status + libcurl4-openssl-dev: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libcurl3-gnutls+libcurl4-gnutls-dev.policy; + libcurl3-gnutls: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + libcurl4-gnutls-dev: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libcurl4.policy-installed; + libcurl4: + Installed: 7.87.0-2 + Candidate: 7.87.0-2 + Version table: + *** 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + 100 /var/lib/dpkg/status + EOI + $* libcurl --install libcurl <>EOE >>EOO + manifest: libcurl libcurl.manifest + + apt-cache-policy: libcurl4 libcurl4-openssl-dev libcurl4+libcurl4-openssl-dev.policy + apt-cache-policy: libcurl3-gnutls libcurl4-gnutls-dev libcurl3-gnutls+libcurl4-gnutls-dev.policy + apt-cache-policy-installed: libcurl4 libcurl4.policy-installed + EOI + LC_ALL=C apt-cache policy --quiet libcurl4 libcurl4-openssl-dev =libcurl4+libcurl4-openssl-dev.policy; + libcurl4: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + libcurl4-openssl-dev: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libcurl3-gnutls+libcurl4-gnutls-dev.policy; + libcurl3-gnutls: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + libcurl4-gnutls-dev: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + $* libcurl --install libcurl <>EOE != 0 + manifest: libcurl libcurl.manifest + + apt-cache-policy: libcurl4 libcurl4-openssl-dev libcurl4+libcurl4-openssl-dev.policy + apt-cache-policy: libcurl3-gnutls libcurl4-gnutls-dev libcurl3-gnutls+libcurl4-gnutls-dev.policy + EOI + LC_ALL=C apt-cache policy --quiet libcurl4 libcurl4-openssl-dev =libcurl4+libcurl4-openssl-dev.policy; + libcurl4: + Installed: 7.85.0-1 + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 7.85.0-1 100 + 100 /var/lib/dpkg/status + libcurl4-openssl-dev: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + cat <=libcurl3-gnutls+libcurl4-gnutls-dev.policy; + libcurl3-gnutls: + Installed: 7.85.0-1 + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + *** 7.85.0-1 100 + 100 /var/lib/dpkg/status + libcurl4-gnutls-dev: + Installed: (none) + Candidate: 7.87.0-2 + Version table: + 7.87.0-2 500 + 500 http://deb.debian.org/debian bookworm/main amd64 Packages + EOI + $* libcurl --install libcurl <>EOE != 0 + manifest: libcurl libcurl.manifest + + apt-cache-policy: libcurl4 libcurl4-openssl-dev libcurl4+libcurl4-openssl-dev.policy + apt-cache-policy: libcurl3-gnutls libcurl4-gnutls-dev libcurl3-gnutls+libcurl4-gnutls-dev.policy + EOI + LC_ALL=C apt-cache policy --quiet libcurl4 libcurl4-openssl-dev + +#include + +#include +#include + +#include +#include +#include +#include + +#include + +using namespace std; +using namespace butl; + +namespace bpkg +{ + system_package_manager:: + ~system_package_manager () + { + // vtable + } + + unique_ptr + make_system_package_manager (const common_options& co, + const target_triplet& host, + bool install, + bool fetch, + bool yes, + const string& sudo, + const string& name) + { + optional progress (co.progress () ? true : + co.no_progress () ? false : + optional ()); + + unique_ptr r; + + if (optional osr = host_os_release (host)) + { + auto is_or_like = [&osr] (const char* id) + { + return (osr->name_id == id || + find_if (osr->like_ids.begin (), osr->like_ids.end (), + [id] (const string& n) + { + return n == id; + }) != osr->like_ids.end ()); + }; + + if (host.class_ == "linux") + { + if (is_or_like ("debian") || + is_or_like ("ubuntu")) + { + if (!name.empty () && name != "debian") + fail << "unsupported package manager '" << name << "' for " + << osr->name_id << " host"; + + // If we recognized this as Debian-like in an ad hoc manner, then + // add debian to like_ids. + // + if (osr->name_id != "debian" && !is_or_like ("debian")) + osr->like_ids.push_back ("debian"); + + r.reset (new system_package_manager_debian ( + move (*osr), host, install, fetch, progress, yes, sudo)); + } + } + } + + if (r == nullptr) + { + if (!name.empty ()) + fail << "unsupported package manager '" << name << "' for host " + << host; + } + + return r; + } + + // Return the version id parsed as a semantic version if it is not empty and + // the "0" semantic version otherwise. Issue diagnostics and fail on parsing + // errors. + // + // Note: the name_id argument is only used for diagnostics. + // + static inline semantic_version + parse_version_id (const string& version_id, const string& name_id) + { + if (version_id.empty ()) + return semantic_version (0, 0, 0); + + try + { + return semantic_version (version_id, semantic_version::allow_omit_minor); + } + catch (const invalid_argument& e) + { + fail << "invalid version '" << version_id << "' for " << name_id + << " host: " << e << endf; + } + } + + // Parse the component of the specified -* + // value into the distribution name and version (return as "0" if not + // present). Issue diagnostics and fail on parsing errors. + // + // Note: the value_name, ap, and af arguments are only used for diagnostics. + // + static pair + parse_distribution (string&& d, + const string& value_name, + const shared_ptr& ap, + const lazy_shared_ptr& af) + { + string dn (move (d)); // [_] + size_t p (dn.rfind ('_')); // Version-separating underscore. + + // If the '_' separator is present, then make sure that the right-hand + // part looks like a version (not empty and only contains digits and + // dots). + // + if (p != string::npos) + { + if (p != dn.size () - 1) + { + for (size_t i (p + 1); i != dn.size (); ++i) + { + if (!digit (dn[i]) && dn[i] != '.') + { + p = string::npos; + break; + } + } + } + else + p = string::npos; + } + + // Parse the distribution version if present and leave it "0" otherwise. + // + semantic_version dv (0, 0, 0); + if (p != string::npos) + try + { + dv = semantic_version (dn, + p + 1, + semantic_version::allow_omit_minor); + + dn.resize (p); + } + catch (const invalid_argument& e) + { + // Note: the repository fragment may have no database associated when + // used in tests. + // + shared_ptr f (af.get_eager ()); + database* db (!(f != nullptr && !af.loaded ()) // Not transient? + ? &af.database () + : nullptr); + + diag_record dr (fail); + dr << "invalid distribution version '" << string (dn, p + 1) + << "' in value " << value_name << " for package " << ap->id.name + << ' ' << ap->version; + + if (db != nullptr) + dr << *db; + + dr << " in repository " << (f != nullptr ? f : af.load ())->location + << ": " << e; + } + + return make_pair (move (dn), move (dv)); + } + + strings system_package_manager:: + system_package_names (const available_packages& aps, + const string& name_id, + const string& version_id, + const vector& like_ids) + { + assert (!aps.empty ()); + + semantic_version vid (parse_version_id (version_id, name_id)); + + // Return those [_]-name distribution values of the + // specified available packages whose component matches the + // specified distribution name and the component (assumed as "0" + // if not present) is less or equal the specified distribution version. + // Suppress duplicate values. + // + auto name_values = [&aps] (const string& n, const semantic_version& v) + { + strings r; + + // For each available package sort the system package names in the + // distribution version descending order and then append them to the + // resulting list, keeping this order and suppressing duplicates. + // + using name_version = pair; + vector nvs; // Reuse the buffer. + + for (const auto& a: aps) + { + nvs.clear (); + + const shared_ptr& ap (a.first); + + for (const distribution_name_value& dv: ap->distribution_values) + { + if (optional d = dv.distribution ("-name")) + { + pair dnv ( + parse_distribution (move (*d), dv.name, ap, a.second)); + + if (dnv.first == n && dnv.second <= v) + { + // Add the name/version pair to the sorted vector. + // + name_version nv (make_pair (dv.value, move (dnv.second))); + + nvs.insert (upper_bound (nvs.begin (), nvs.end (), nv, + [] (const name_version& x, + const name_version& y) + {return x.second > y.second;}), + move (nv)); + } + } + } + + // Append the sorted names to the resulting list. + // + for (name_version& nv: nvs) + { + if (find_if (r.begin (), r.end (), + [&nv] (const string& n) {return nv.first == n;}) == + r.end ()) + { + r.push_back (move (nv.first)); + } + } + } + + return r; + }; + + // Collect distribution values for those -name names which + // match the name id and refer to the version which is less or equal than + // the version id. + // + strings r (name_values (name_id, vid)); + + // If the resulting list is empty and the like ids are specified, then + // re-collect but now using the like id and "0" version id instead. + // + if (r.empty ()) + { + for (const string& like_id: like_ids) + { + r = name_values (like_id, semantic_version (0, 0, 0)); + if (!r.empty ()) + break; + } + } + + return r; + } + + optional system_package_manager:: + downstream_package_version (const string& system_version, + const available_packages& aps, + const string& name_id, + const string& version_id, + const vector& like_ids) + { + semantic_version vid (parse_version_id (version_id, name_id)); + + // Iterate over the passed available packages (in version descending + // order) and over the [_]-to-downstream-version + // distribution values they contain. Only consider those values whose + // component matches the specified distribution name and the + // component (assumed as "0" if not present) is less or equal + // the specified distribution version. For such values match the regex + // pattern against the passed system version and if it matches consider + // the replacement as the resulting downstream version candidate. Return + // this downstream version if the distribution version is equal to the + // specified one. Otherwise (the version is less), continue iterating + // while preferring downstream version candidates for greater distribution + // versions. Note that here we are trying to use a version mapping for the + // distribution version closest (but never greater) to the specified + // distribution version. So, for example, if both following values contain + // a matching mapping, then for debian 11 we prefer the downstream version + // produced by the debian_10-to-downstream-version value: + // + // debian_9-to-downstream-version + // debian_10-to-downstream-version + // + auto downstream_version = [&aps, &system_version] + (const string& n, + const semantic_version& v) -> optional + { + optional r; + semantic_version rv; + + for (const auto& a: aps) + { + const shared_ptr& ap (a.first); + + for (const distribution_name_value& nv: ap->distribution_values) + { + if (optional d = nv.distribution ("-to-downstream-version")) + { + pair dnv ( + parse_distribution (move (*d), nv.name, ap, a.second)); + + if (dnv.first == n && dnv.second <= v) + { + auto bad_value = [&nv, &ap, &a] (const string& d) + { + // Note: the repository fragment may have no database + // associated when used in tests. + // + const lazy_shared_ptr& af (a.second); + shared_ptr f (af.get_eager ()); + database* db (!(f != nullptr && !af.loaded ()) // Not transient? + ? &af.database () + : nullptr); + + diag_record dr (fail); + dr << "invalid distribution value '" << nv.name << ": " + << nv.value << "' for package " << ap->id.name << ' ' + << ap->version; + + if (db != nullptr) + dr << *db; + + dr << " in repository " + << (f != nullptr ? f : af.load ())->location << ": " << d; + }; + + // Parse the distribution value into the regex pattern and the + // replacement. + // + // Note that in the future we may add support for some regex + // flags. + // + pair rep; + try + { + size_t end; + const string& val (nv.value); + rep = regex_replace_parse (val.c_str (), val.size (), end); + } + catch (const invalid_argument& e) + { + bad_value (e.what ()); + } + + // Match the regex pattern against the system version and skip + // the value if it doesn't match or proceed to parsing the + // downstream version resulting from the regex replacement + // otherwise. + // + string dv; + try + { + regex re (rep.first, regex::ECMAScript); + + pair rr ( + regex_replace_match (system_version, re, rep.second)); + + // Skip the regex if it doesn't match. + // + if (!rr.second) + continue; + + dv = move (rr.first); + } + catch (const regex_error& e) + { + // Print regex_error description if meaningful (no space). + // + ostringstream os; + os << "invalid regex pattern '" << rep.first << "'" << e; + bad_value (os.str ()); + } + + // Parse the downstream version. + // + try + { + version ver (dv); + + // If the distribution version is equal to the specified one, + // then we are done. Otherwise, save the version if it is + // preferable and continue iterating. + // + // Note that bailing out immediately in the former case is + // essential. Otherwise, we can potentially fail later on, for + // example, some ill-formed regex which is already fixed in + // some newer package. + // + if (dnv.second == v) + return ver; + + if (!r || rv < dnv.second) + { + r = move (ver); + rv = move (dnv.second); + } + } + catch (const invalid_argument& e) + { + bad_value ("resulting downstream version '" + dv + + "' is invalid: " + e.what ()); + } + } + } + } + } + + return r; + }; + + // Try to deduce the downstream version using the + // -to-downstream-version values that match the name id and + // refer to the version which is less or equal than the version id. + // + optional r (downstream_version (name_id, vid)); + + // If the downstream version is not deduced and the like ids are + // specified, then re-try but now using the like id and "0" version id + // instead. + // + if (!r) + { + for (const string& like_id: like_ids) + { + r = downstream_version (like_id, semantic_version (0, 0, 0)); + if (r) + break; + } + } + + return r; + } +} diff --git a/bpkg/system-package-manager.hxx b/bpkg/system-package-manager.hxx new file mode 100644 index 0000000..9a9c443 --- /dev/null +++ b/bpkg/system-package-manager.hxx @@ -0,0 +1,270 @@ +// file : bpkg/system-package-manager.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_SYSTEM_PACKAGE_MANAGER_HXX +#define BPKG_SYSTEM_PACKAGE_MANAGER_HXX + +#include // version +#include + +#include +#include + +#include +#include +#include + +namespace bpkg +{ + // The system/distribution package manager interface. Used by both pkg-build + // (to query and install system packages) and by pkg-bindist (to build + // them). + // + // Note that currently the result of a query is a single available version. + // While some package managers may support having multiple available + // versions and may even allow installing multiple versions in parallel, + // supporting this on our side will complicate things quite a bit. While we + // can probably plug multiple available versions into our constraint + // satisfaction machinery, the rabbit hole goes deeper than that since, for + // example, different bpkg packages can be mapped to the same system + // package, as is the case for libcrypto/libssl which are both mapped to + // libssl on Debian. This means we will need to somehow coordinate (and + // likely backtrack) version selection between unrelated bpkg packages + // because only one underlying system version can be selected. (One + // simplified way to handle this would be to detect that different versions + // we selected and fail asking the user to resolve this manually.) + // + // Additionally, parallel installation is unlikely to be suppored for the + // packages we are interested in due to the underlying limitations. + // Specifically, the packages that we are primarily interested in are + // libraries with headers and executables (tools). While most package + // managers (e.g., Debian, Fedora) are able to install multiple libraries in + // parallel, they normally can only install a single set of headers, static + // libraries, pkg-config files, etc., (e.g., -dev/-devel package) at a time + // due to them being installed into the same location (e.g., /usr/include). + // The same holds for executables, which are installed into the same + // location (e.g., /usr/bin). + // + // It is possible that a certain library has made arrangements for + // multiple of its versions to co-exist. For example, hypothetically, our + // libssl package could be mapped to both libssl1.1 libssl1.1-dev and + // libssl3 libssl3-dev which could be installed at the same time (note + // that it is not the case in reality; there is only libssl-dev). However, + // in this case, we should probably also have two packages with separate + // names (e.g., libssl and libssl3) that can also co-exist. An example of + // this would be libQt5Core and libQt6Core. (Note that strictly speaking + // there could be different degrees of co-existence: for the system + // package manager it is sufficient for different versions not to clobber + // each other's files while for us we may also need the ability to use + // different versions in the base build). + // + // Note also that the above reasoning is quite C/C++-centric and it's + // possible that multiple versions of libraries (or equivalent) for other + // languages (e.g., Rust) can always co-exist. Plus, even in the case of + // C/C++ libraries, there is still the plausible case of picking one of + // the multiple available version. + // + // On the other hand, the ultimate goal of system package managers, at least + // traditional ones like Debian and Fedora, is to end up with a single, + // usually the latest available, version of the package that is used by + // everyone. In fact, if one looks at a stable distributions of Debian and + // Fedora, they normally provide only a single version of each package. This + // decision will also likely simplify the implementation. For example, on + // Debian, it's straightforward to get the installed and candidate versions + // (e.g., from apt-cache policy). But getting all the possible versions that + // can be installed without having to specify the release explicitly is a + // lot less straightforward (see the apt-cache command documentation in The + // Debian Administrator's Handbook for background). + // + // So for now we keep it simple and pick a single available version but can + // probably revise this decision later. + // + struct system_package_status + { + // Downstream (as in, bpkg package) version. + // + bpkg::version version; + + // System (as in, distribution) package name and version for diagnostics. + // + // Note that this status may represent multiple system packages (for + // example, libfoo and libfoo-dev) and here we have only the + // main/representative package name (for example, libfoo). + // + string system_name; + string system_version; + + // The system package can be either "available already installed", + // "available partially installed" (for example, libfoo but not + // libfoo-dev is installed) or "available not yet installed". + // + enum status_type {installed, partially_installed, not_installed}; + + status_type status = not_installed; + }; + + class system_package_manager + { + public: + // Query the system package status. + // + // This function has two modes: cache-only (available_packages is NULL) + // and full (available_packages is not NULL). In the cache-only mode this + // function returns the status of this package if it has already been + // queried and nullopt otherwise. This allows the caller to only collect + // all the available packages (for the name/version mapping information) + // if really necessary. + // + // The returned status can be NULL, which indicates that no such package + // is available from the system package manager. Note that NULL is also + // returned if no fully installed package is available from the system and + // package installation is not enabled (see the constructor below). + // + // Note also that the implementation is expected to issue appropriate + // progress and diagnostics if fetching package metadata (again see the + // constructor below). + // + virtual optional + pkg_status (const package_name&, const available_packages*) = 0; + + // Install the specified subset of the previously-queried packages. + // Should only be called if installation is enabled (see the constructor + // below). + // + // Note that this function should be called only once after the final set + // of the required system packages has been determined. And the specified + // subset should contain all the selected packages, including the already + // fully installed. This allows the implementation to merge and de- + // duplicate the system package set to be installed (since some bpkg + // packages may be mapped to the same system package), perform post- + // installation verifications (such as making sure the versions of already + // installed packages have not changed due to upgrades), change properties + // of already installed packages (e.g., mark them as manually installed in + // Debian), etc. + // + // Note also that the implementation is expected to issue appropriate + // progress and diagnostics. + // + virtual void + pkg_install (const vector&) = 0; + + public: + // If install is true, then enable package installation. + // + // If fetch is false, then do not re-fetch the system package repository + // metadata (that is, available packages/versions) before querying for the + // available version of the not yet installed or partially installed + // packages. + // + system_package_manager (os_release&& osr, + const target_triplet& host, + bool install, + bool fetch, + optional progress, + bool yes, + string sudo) + : os_release_ (osr), + host_ (host), + progress_ (progress), + install_ (install), + fetch_ (fetch), + yes_ (yes), + sudo_ (sudo != "false" ? move (sudo) : string ()) {} + + virtual + ~system_package_manager (); + + // Implementation details. + // + public: + // Given the available packages (as returned by find_available_all()) + // return the list of system package names as mapped by the + // -name values. + // + // The name_id, version_id, and like_ids are the values from os_release + // (refer there for background). If version_id is empty, then it's treated + // as "0". + // + // First consider -name values corresponding to name_id. + // Assume has the [_] form, where + // is a semver-like version (e.g, 10, 10.15, or 10.15.1) and return all + // the values that are equal or less than the specified version_id + // (include the value with the absent ). In a sense, absent + // can be treated as a 0 semver-like version. + // + // If no value is found then repeat the above process for every like_ids + // entry (from left to right) instead of name_id with version_id equal 0. + // + // If still no value is found, then return empty list (in which case the + // caller may choose to fallback to the downstream package name or do + // something more elaborate, like translate version_id to one of the + // like_id's version and try that). + // + // Note that multiple -name values per same distribution can be returned + // as, for example, for the following distribution values: + // + // debian_10-name: libcurl4 libcurl4-doc libcurl4-openssl-dev + // debian_10-name: libcurl3-gnutls libcurl4-gnutls-dev (yes, 3 and 4) + // + // Note also that the values are returned in the "override order", that is + // from the newest package version to oldest and then from the highest + // distribution version to lowest. + // + static strings + system_package_names (const available_packages&, + const string& name_id, + const string& version_id, + const vector& like_ids); + + // Given the system package version and available packages (as returned by + // find_available_all()) return the downstream package version as mapped + // by one of the -to-downstream-version values. + // + // The rest of the arguments as well as the overalls semantics is the same + // as in system_package_names() above. That is, first consider + // -to-downstream-version values corresponding to + // name_id. If none match, then repeat the above process for every + // like_ids entry with version_id equal 0. If still no match, then return + // nullopt (in which case the caller may choose to fallback to the system + // package version or do something more elaborate). + // + static optional + downstream_package_version (const string& system_version, + const available_packages&, + const string& name_id, + const string& version_id, + const vector& like_ids); + protected: + os_release os_release_; + target_triplet host_; + optional progress_; // --[no]-progress (see also stderr_term) + + // The --sys-* option values. + // + bool install_; + bool fetch_; + bool yes_; + string sudo_; + }; + + // Create a package manager instance corresponding to the specified host + // target and optional manager name. If name is empty, return NULL if there + // is no support for this platform. Currently recognized names: + // + // debian -- Debian and alike (Ubuntu, etc) using the APT frontend. + // fedora -- Fedora and alike (RHEL, Centos, etc) using the DNF frontend. + // + // Note: the name can be used to select an alternative package manager + // implementation on platforms that support multiple. + // + unique_ptr + make_system_package_manager (const common_options&, + const target_triplet&, + bool install, + bool fetch, + bool yes, + const string& sudo, + const string& name); +} + +#endif // BPKG_SYSTEM_PACKAGE_MANAGER_HXX diff --git a/bpkg/system-package-manager.test.cxx b/bpkg/system-package-manager.test.cxx new file mode 100644 index 0000000..1a669da --- /dev/null +++ b/bpkg/system-package-manager.test.cxx @@ -0,0 +1,124 @@ +// file : bpkg/system-package-manager.test.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include + +#include +#include + +#undef NDEBUG +#include + +#include + +using namespace std; + +namespace bpkg +{ + // Usage: args[0] ... + // + // Where is one of: + // + // system-package-names [...] -- ... + // + // Where is a package name, is a package manifest file. + // + // downstream-package-version [...] -- ... + // + // Where is a system version to translate, is a package + // name, and is a package manifest file. + // + int + main (int argc, char* argv[]) + try + { + assert (argc >= 2); // + + int argi (1); + string cmd (argv[argi++]); + + os_release osr; + if (cmd == "system-package-names" || + cmd == "downstream-package-version") + { + assert (argc >= 4); // + + osr.name_id = argv[argi++]; + osr.version_id = argv[argi++]; + + for (; argi != argc; ++argi) + { + string a (argv[argi]); + + if (a == "--") + break; + + osr.like_ids.push_back (move (a)); + } + } + + if (cmd == "system-package-names") + { + assert (argi != argc); // -- + string a (argv[argi++]); + assert (a == "--"); + + assert (argi != argc); // + string pn (argv[argi++]); + + assert (argi != argc); // + available_packages aps; + for (; argi != argc; ++argi) + aps.push_back (make_available_from_manifest (pn, argv[argi])); + sort_available (aps); + + strings ns ( + system_package_manager::system_package_names ( + aps, osr.name_id, osr.version_id, osr.like_ids)); + + for (const string& n: ns) + cout << n << '\n'; + } + else if (cmd == "downstream-package-version") + { + assert (argi != argc); // -- + string a (argv[argi++]); + assert (a == "--"); + + assert (argi != argc); // + string sv (argv[argi++]); + + assert (argi != argc); // + string pn (argv[argi++]); + + assert (argi != argc); // + available_packages aps; + for (; argi != argc; ++argi) + aps.push_back (make_available_from_manifest (pn, argv[argi])); + sort_available (aps); + + optional v ( + system_package_manager::downstream_package_version ( + sv, aps, osr.name_id, osr.version_id, osr.like_ids)); + + if (v) + cout << *v << '\n'; + } + else + fail << "unknown command '" << cmd << "'"; + + return 0; + } + catch (const failed&) + { + return 1; + } +} + +int +main (int argc, char* argv[]) +{ + return bpkg::main (argc, argv); +} diff --git a/bpkg/system-package-manager.test.hxx b/bpkg/system-package-manager.test.hxx new file mode 100644 index 0000000..0eb6717 --- /dev/null +++ b/bpkg/system-package-manager.test.hxx @@ -0,0 +1,104 @@ +// file : bpkg/system-package-manager.test.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef BPKG_SYSTEM_PACKAGE_MANAGER_TEST_HXX +#define BPKG_SYSTEM_PACKAGE_MANAGER_TEST_HXX + +#include + +#include // sort() + +#include +#include + +#include + +#include + +#include + +namespace bpkg +{ + // Parse the manifest as if it comes from a git repository with a single + // package and make an available package out of it. + // + inline + pair, lazy_shared_ptr> + make_available_from_manifest (const string& n, const string& f) + { + using butl::manifest_parser; + using butl::manifest_parsing; + + try + { + ifdstream ifs (f); + manifest_parser mp (ifs, f); + + package_manifest m (mp, + false /* ignore_unknown */, + true /* complete_values */); + + assert (m.name.string () == n); + + m.alt_naming = false; + m.bootstrap_build = "project = " + n + '\n'; + + shared_ptr ap ( + make_shared (move (m))); + + lazy_shared_ptr af ( + make_shared ( + repository_location ("https://example.com/" + n, + repository_type::git))); + + ap->locations.push_back (package_location {af, current_dir}); + + return make_pair (move (ap), move (af)); + } + catch (const manifest_parsing& e) + { + fail (e.name, e.line, e.column) << e.description << endf; + } + catch (const io_error& e) + { + fail << "unable to read from " << f << ": " << e << endf; + } + } + + // Make an available stub package as if it comes from git repository with + // a single package. + // + inline + pair, lazy_shared_ptr> + make_available_stub (const string& n) + { + shared_ptr ap ( + make_shared (package_name (n))); + + lazy_shared_ptr af ( + make_shared ( + repository_location ("https://example.com/" + n, + repository_type::git))); + + ap->locations.push_back (package_location {af, current_dir}); + + return make_pair (move (ap), move (af)); + } + + // Sort available packages in the version descending order. + // + inline void + sort_available (available_packages& aps) + { + using element_type = + pair, lazy_shared_ptr>; + + std::sort (aps.begin (), aps.end (), + [] (const element_type& x, const element_type& y) + { + return x.first->version > y.first->version; + }); + } +} + +#endif // BPKG_SYSTEM_PACKAGE_MANAGER_TEST_HXX diff --git a/bpkg/system-package-manager.test.testscript b/bpkg/system-package-manager.test.testscript new file mode 100644 index 0000000..dc672f5 --- /dev/null +++ b/bpkg/system-package-manager.test.testscript @@ -0,0 +1,101 @@ +# file : bpkg/system-package-manager.test.testscript +# license : MIT; see accompanying LICENSE file + +: system-package-names +: +{ + test.arguments += system-package-names + + : basics + : + cat <=libcurl7.64.manifest; + : 1 + name: libcurl + version: 7.64.0 + debian-name: libcurl2 libcurl2-dev + summary: curl + license: curl + EOI + cat <=libcurl7.84.manifest; + : 1 + name: libcurl + version: 7.84.0 + debian_9-name: libcurl2 libcurl2-dev libcurl2-doc + debian_10-name: libcurl4 libcurl4-openssl-dev + debian_10-name: libcurl3-gnutls libcurl4-gnutls-dev + summary: curl + license: curl + EOI + + $* debian 10 -- libcurl libcurl7.64.manifest libcurl7.84.manifest >>EOO; + libcurl4 libcurl4-openssl-dev + libcurl3-gnutls libcurl4-gnutls-dev + libcurl2 libcurl2-dev libcurl2-doc + libcurl2 libcurl2-dev + EOO + $* debian 9 -- libcurl libcurl7.64.manifest libcurl7.84.manifest >>EOO; + libcurl2 libcurl2-dev libcurl2-doc + libcurl2 libcurl2-dev + EOO + $* debian '' -- libcurl libcurl7.64.manifest libcurl7.84.manifest >>EOO; + libcurl2 libcurl2-dev + EOO + $* ubuntu 16.04 debian -- libcurl libcurl7.64.manifest libcurl7.84.manifest >>EOO + libcurl2 libcurl2-dev + EOO +} + +: downstream-package-version +: +{ + test.arguments += downstream-package-version + + : basics + : + cat <=libssl1.manifest; + : 1 + name: libssl + version: 1.1.1 + upstream-version: 1.1.1n + debian-to-downstream-version: /1\.1\.1[a-z]/1.1.1/ + summary: openssl + license: openssl + EOI + cat <=libssl3.manifest; + : 1 + name: libssl + version: 3.0.0 + debian-to-downstream-version: /([3-9])\.([0-9]+)\.([0-9]+)/\1.\2.\3/ + summary: openssl + license: openssl + EOI + $* debian 10 -- 1.1.1l libssl libssl1.manifest libssl3.manifest >'1.1.1'; + $* debian 10 -- 3.0.7 libssl libssl1.manifest libssl3.manifest >'3.0.7'; + $* debian '' -- 1.1.1l libssl libssl1.manifest libssl3.manifest >'1.1.1'; + $* debian '' -- 3.0.7 libssl libssl1.manifest libssl3.manifest >'3.0.7'; + $* ubuntu 16.04 debian -- 1.1.1l libssl libssl1.manifest libssl3.manifest >'1.1.1'; + $* ubuntu 16.05 debian -- 3.0.7 libssl libssl1.manifest libssl3.manifest >'3.0.7' + + : order + : + cat <=libssl1.manifest; + : 1 + name: libssl + version: 1.1.1 + debian-to-downstream-version: /.*/0/ + summary: openssl + license: openssl + EOI + cat <=libssl3.manifest; + : 1 + name: libssl + version: 3.0.0 + debian_9-to-downstream-version: /.*/9/ + debian_10-to-downstream-version: /.*/10/ + summary: openssl + license: openssl + EOI + $* debian 10 -- 1 libssl libssl1.manifest libssl3.manifest >'10'; + $* debian 9 -- 1 libssl libssl1.manifest libssl3.manifest >'9'; + $* debian 8 -- 1 libssl libssl1.manifest libssl3.manifest >'0' +} diff --git a/bpkg/system-repository.cxx b/bpkg/system-repository.cxx index d7a47b7..c308ddb 100644 --- a/bpkg/system-repository.cxx +++ b/bpkg/system-repository.cxx @@ -6,9 +6,12 @@ namespace bpkg { const version& system_repository:: - insert (const package_name& name, const version& v, bool authoritative) + insert (const package_name& name, + const version& v, + bool authoritative, + const system_package_status* s) { - auto p (map_.emplace (name, system_package {v, authoritative})); + auto p (map_.emplace (name, system_package {v, authoritative, s})); if (!p.second) { @@ -22,6 +25,7 @@ namespace bpkg { sp.authoritative = authoritative; sp.version = v; + sp.system_status = s; } } diff --git a/bpkg/system-repository.hxx b/bpkg/system-repository.hxx index f33d622..31e14d1 100644 --- a/bpkg/system-repository.hxx +++ b/bpkg/system-repository.hxx @@ -12,6 +12,8 @@ #include #include +#include + namespace bpkg { // A map of discovered system package versions. The information can be @@ -30,16 +32,25 @@ namespace bpkg version_type version; bool authoritative; + + // If the information is authoritative then this member indicates whether + // the version came from the system package manager (not NULL) or + // user/fallback (NULL). + // + const system_package_status* system_status; }; class system_repository { public: const version& - insert (const package_name& name, const version&, bool authoritative); + insert (const package_name& name, + const version&, + bool authoritative, + const system_package_status* = nullptr); const system_package* - find (const package_name& name) + find (const package_name& name) const { auto i (map_.find (name)); return i != map_.end () ? &i->second : nullptr; diff --git a/bpkg/types.hxx b/bpkg/types.hxx index 2b6a1f8..7a7b2c7 100644 --- a/bpkg/types.hxx +++ b/bpkg/types.hxx @@ -33,6 +33,7 @@ #include #include #include +#include #include namespace bpkg @@ -127,6 +128,10 @@ namespace bpkg using butl::ofdstream; using butl::fdstream_mode; + // + // + using butl::target_triplet; + // // using butl::default_options_files; diff --git a/bpkg/utility.cxx b/bpkg/utility.cxx index 52114df..b79c85b 100644 --- a/bpkg/utility.cxx +++ b/bpkg/utility.cxx @@ -46,6 +46,8 @@ namespace bpkg const dir_path current_dir ("."); + const target_triplet host_triplet (BPKG_HOST_TRIPLET); + map tmp_dirs; bool keep_tmp; diff --git a/bpkg/utility.hxx b/bpkg/utility.hxx index 8e7260a..69a02d3 100644 --- a/bpkg/utility.hxx +++ b/bpkg/utility.hxx @@ -10,7 +10,7 @@ #include // strcmp(), strchr() #include // move(), forward(), declval(), make_pair() #include // assert() -#include // make_move_iterator() +#include // make_move_iterator(), back_inserter() #include // * #include @@ -33,6 +33,7 @@ namespace bpkg using std::make_pair; using std::make_shared; using std::make_move_iterator; + using std::back_inserter; using std::to_string; using std::strcmp; @@ -51,6 +52,7 @@ namespace bpkg using butl::trim; using butl::trim_left; using butl::trim_right; + using butl::next_word; using butl::make_guard; using butl::make_exception_guard; @@ -59,6 +61,8 @@ namespace bpkg using butl::setenv; using butl::unsetenv; + using butl::eof; + // // using butl::process_start_callback; @@ -99,6 +103,10 @@ namespace bpkg extern const dir_path current_dir; // ./ + // Host target triplet for which we were built. + // + extern const target_triplet host_triplet; + // Temporary directory facility. // // An entry normally maps to /.bpkg/tmp/ but can also map -- cgit v1.1