From 692ac2f6123c341d8cb5203568d78bef51fe044a Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Mon, 23 Jan 2023 12:57:59 +0200 Subject: More work on Debian implementation --- bpkg/system-package-manager-debian.cxx | 529 ++++++++++++++++++++++++++++++--- 1 file changed, 488 insertions(+), 41 deletions(-) diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx index f1e78a1..d294416 100644 --- a/bpkg/system-package-manager-debian.cxx +++ b/bpkg/system-package-manager-debian.cxx @@ -41,7 +41,7 @@ namespace bpkg // libssl3 libssl-dev libssl-doc // // libcurl4 libcurl4-doc libcurl4-openssl-dev - // libcurl3-gnutls libcurl4-gnutls-dev + // libcurl3-gnutls libcurl4-gnutls-dev (yes, 3 and 4) // // Based on that, it seems our best bet when trying to automatically map our // library package name to Debian package names is to go for the -dev @@ -56,6 +56,17 @@ namespace bpkg // For executable packages there is normally no -dev packages but -dbg, // -doc, and -common are plausible. // + struct package_policy // apt-cache policy output + { + reference_wrapper name; + + string installed_version; // Empty if none. + string candidate_version; // Empty if none. + + explicit + package_policy (const string& n): name (n) {} + }; + class system_package_status_debian: public system_package_status { public: @@ -66,6 +77,8 @@ namespace bpkg string common; strings extras; + vector package_policies; + explicit system_package_status_debian (string m, string d = {}) : main (move (m)), dev (move (d)) @@ -117,11 +130,15 @@ namespace bpkg // well, you are out of luck, I guess). // // Note also that for now we treat all the packages from the non-main groups - // as extras. But in the future we may decide to sort them out like the main - // group. + // as extras (but in the future we may decide to sort them out like the main + // group). For now we omit the -common package (assuming it's pulled by the + // main package) as well as -doc and -dbg unless requested (the + // extra_{doc,dbg} arguments). // static unique_ptr - parse_debian_name (const string& nv) + parse_debian_name (const string& nv, + bool extra_doc, + bool extra_dbg) { auto split = [] (const string& s, char d) -> strings { @@ -203,14 +220,12 @@ namespace bpkg { unique_ptr g (parse_group (gs[i])); - // @@ Shouldn't we filter some based on what we are installing? - - if (!g->main.empty ()) r->extras.push_back (move (g->main)); - if (!g->dev.empty ()) r->extras.push_back (move (g->dev)); - if (!g->doc.empty ()) r->extras.push_back (move (g->doc)); - if (!g->dbg.empty ()) r->extras.push_back (move (g->dbg)); - if (!g->common.empty ()) r->extras.push_back (move (g->common)); - if (!g->extras.empty ()) r->extras.insert ( + if (!g->main.empty ()) r->extras.push_back (move (g->main)); + if (!g->dev.empty ()) r->extras.push_back (move (g->dev)); + if (!g->doc.empty () && extra_doc) r->extras.push_back (move (g->doc)); + if (!g->dbg.empty () && extra_dbg) r->extras.push_back (move (g->dbg)); + if (!g->common.empty () && false) r->extras.push_back (move (g->common)); + if (!g->extras.empty ()) r->extras.insert ( r->extras.end (), make_move_iterator (g->extras.begin ()), make_move_iterator (g->extras.end ())); @@ -220,22 +235,20 @@ namespace bpkg return r; } + static process_path apt_cache; + // Obtain the installed and candidate versions for the specified list // of Debian packages by executing apt-cache policy. // - struct package_policy - { - string name; - string installed_version; // Empty if none. - string candidate_version; // Empty if none. - }; - - static process_path apt_cache; - + // If the n argument is not 0, then only query the first n packages. + // static void - apt_cache_policy (vector& pps) + apt_cache_policy (vector& pps, size_t n = 0) { - assert (!pps.empty ()); + if (n == 0) + n = pps.size (); + + assert (n != 0 && n <= pps.size ()); // In particular, --quite makes sure we don't get a noice (N) printed to // stderr if the package is unknown. It does not appear to affect error @@ -243,13 +256,17 @@ namespace bpkg // cstrings args {"apt-cache", "policy", "--quiet"}; - for (const package_policy& pp: pps) + for (size_t i (0); i != n; ++i) { - assert (!pp.name.empty () && - pp.installed_version.empty () && - pp.candidate_version.empty ()); + package_policy& pp (pps[i]); + + const string& n (pp.name); + assert (!n.empty ()); + + pp.installed_version.clear (); + pp.candidate_version.clear (); - args.push_back (pp.name.c_str ()); + args.push_back (n.c_str ()); } args.push_back (nullptr); @@ -314,7 +331,7 @@ namespace bpkg print_process (dr, pe, args); }); - auto i (pps.begin ()); + size_t i (0); string l; for (getline (is, l); !eof (is); ) @@ -328,11 +345,13 @@ namespace bpkg // Skip until this package. // - for (; i != pps.end () && i->name != l; ++i) ; + for (; i != n && pps[i].name.get () != l; ++i) ; - if (i == pps.end ()) + if (i == n) fail << "unexpected package name '" << l << "'"; + package_policy& pp (pps[i]); + auto parse_version = [&l] (const string& n) -> string { size_t s (n.size ()); @@ -357,14 +376,14 @@ namespace bpkg if (eof (getline (is, l))) fail << "expected Installed version line after package name"; - i->installed_version = parse_version ("Installed"); + pp.installed_version = parse_version ("Installed"); // Get the candidate version line. // if (eof (getline (is, l))) fail << "expected Candidate version line after Installed version"; - i->installed_version = parse_version ("Candidate"); + pp.candidate_version = parse_version ("Candidate"); // Skip the rest of the indented lines (or blanks, just in case). // @@ -405,6 +424,246 @@ namespace bpkg } } + // Return the Depends value, if any, for the specified package and version. + // Fail if either package or version is unknown. + // + static string + apt_cache_show (const string& name, const string& ver) + { + assert (!name.empty () && !ver.empty ()); + + string spec (name + '=' + ver); + + // In particular, --quite makes sure we don't get noices (N) printed to + // stderr. It does not appear to affect error diagnostics (try showing + // information for an unknown package). + // + const char* args[] = { + "apt-cache", "show", "--quiet", spec.c_str (), nullptr}; + + // Note that for this command there seems to be no need to run with the C + // locale since the output is presumably not localizable. But let's do it + // for good measure and also seeing that we try to backfit some + // diagnostics into apt-cache (see no_version below). + // + const char* evars[] = {"LC_ALL=C", nullptr}; + + string r; + try + { + if (apt_cache.empty ()) + apt_cache = process::path_search (args[0]); + + process_env pe (apt_cache, evars); + + if (verb >= 3) + print_process (pe, args); + + // Redirect stdout to a pipe. For good measure also redirect stdin to + // /dev/null to make sure there are no prompts of any kind. + // + process pr (apt_cache, + args, + -2 /* stdin */, + -1 /* stdout */, + 2 /* stderr */, + nullptr /* cwd */, + evars); + + bool no_version (false); + try + { + ifdstream is (move (pr.in_ofd), fdstream_mode::skip, ifdstream::badbit); + + // The output of `apt-cache show =` appears to be a single + // Debian control file in the RFC 822 encoding followed by a blank + // line. See deb822(5) for details. Here is a representative example: + // + // Package: libcurl4 + // Version: 7.85.0-1 + // Depends: libbrotli1 (>= 0.6.0), libc6 (>= 2.34), ... + // Description-en: easy-to-use client-side URL transfer library + // libcurl is an easy-to-use client-side URL transfer library. + // + // Note that if the package is unknown, then we get an error but if + // the version is unknown, we get no output (and a note if running + // without --quiet). + // + string l; + if (eof (getline (is, l))) + { + // The unknown version case. Issue diagnostics consistent with the + // unknown package case, at least for the English locale. + // + text << "E: No package version found"; + no_version = true; + } + else + { + auto df = make_diag_frame ( + [&pe, &args] (diag_record& dr) + { + dr << info << "while parsing output of "; + print_process (dr, pe, args); + }); + + do + { + // This line should be the start of a field unless it's a + // comment. According to deb822(5), there can be no leading + // whitespaces before `#`. + // + if (l[0] == '#') + { + getline (is, l); + continue; + } + + size_t p (l.find (':')); + + if (p == string::npos) + fail << "expected field name instead of '" << l << "'"; + + // Extract the field name. Note that field names are case- + // insensitive. + // + string n (l, 0, p); + trim (n); + + // Extract the field value. + // + string v (l, p + 1); + trim (v); + + // If we have more lines see if the following line is part of this + // value. + // + while (!eof (getline (is, l)) && (l[0] == ' ' || l[0] == '\t')) + { + // This can either be a "folded" or a "multiline" field and + // which one it is depends on the field semantics. Here we only + // care about Depends and so treat them all as folded (it's + // unclear whether Depends must be a simple field). + // + trim (l); + v += ' '; + v += l; + } + + // See if this is a field of interest. + // + if (icasecmp (n, "Package") == 0) + { + assert (v == name); // Sanity check. + } + else if (icasecmp (n, "Version") == 0) + { + assert (v == ver); // Sanity check. + } + else if (icasecmp (n, "Depends") == 0) + { + r = move (v); + + // Let's not waste time reading any further. + // + break; + } + } + while (!eof (is)); + } + + is.close (); + } + catch (const io_error& e) + { + if (pr.wait ()) + fail << "unable to read " << args[0] << " policy output: " << e; + + // Fall through. + } + + if (!pr.wait () || no_version) + { + diag_record dr (fail); + dr << args[0] << " policy exited with non-zero code"; + + if (verb < 3) + { + dr << "command line: "; + print_process (dr, pe, args); + } + } + } + catch (const process_error& e) + { + error << "unable to execute " << args[0] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + return r; + } + + // Attempt to determine the main package name from its -dev package. Return + // empty string if unable. Save the extracted Depends value to the depends + // argument for diagnostics. + // + static string + main_from_dev (const string& dev_name, + const string& dev_ver, + string& depends) + { + depends = apt_cache_show (dev_name, dev_ver); + + // The format of the Depends value is a comma-seperated list of dependency + // expressions. For example: + // + // Depends: libssl3 (= 3.0.7-1), libc6 (>= 2.34), libfoo | libbar + // + // For the main package we look for a dependency in the form: + // + // * (= ) + // + // Usually it is the first one. + // + string dev_stem (dev_name, 0, dev_name.rfind ("-dev")); + + string r; + for (size_t b (0), e (0); next_word (depends, b, e, ','); ) + { + string d (depends, b, e - b); + trim (d); + + size_t p (d.find (' ')); + if (p != string::npos) + { + if (d.compare (0, dev_stem.size (), dev_stem) == 0) // * + { + size_t q (d.find ('(', p + 1)); + if (q != string::npos && d.back () == ')') // (...) + { + if (d[q + 1] == '=' && d[q + 2] == ' ') // Equal. + { + string v (d, q + 3, d.size () - q - 3 - 1); + trim (v); + + if (v == dev_ver) + { + r.assign (d, 0, p); + break; + } + } + } + } + } + } + + return r; + } + optional system_package_manager_debian:: pkg_status (const package_name& pn, @@ -412,6 +671,13 @@ namespace bpkg bool install, bool fetch) { + // For now we ignore -doc and -dbg package components (but we may want to + // have options controlling this later). Note also that we assume -common + // is pulled automatically by the main package so we ignore it as well. + // + bool need_doc (false); + bool need_dbg (false); + // First check the cache. // { @@ -471,7 +737,8 @@ namespace bpkg // for (const string& n: ns) { - unique_ptr s (parse_debian_name (n)); + unique_ptr s ( + parse_debian_name (n, need_doc, need_dbg)); // Suppress duplicates for good measure based on the main package // name (and falling back to -dev if empty). @@ -493,31 +760,211 @@ namespace bpkg } } - // First look for an already installed package. + // Guess unknown main package given the dev package and its version. + // + auto guess_main = [&pn] (package_status_debian& s, const string& ver) + { + string depends; + s.main = main_from_dev (s.dev, ver, depends); + + if (s.main.empty ()) + { + fail << "unable to guess main Debian package for " << s.dev << ' ' + << ver << + info << s.dev << " Depends value: " << depends << + info << "consider specifying explicit mapping in " << pn + << " package manifest"; + } + }; + + // Calculate the package status from individual package components. + // Return nullopt if there is a component without installed or candidate + // version (which means the package cannot be installed). + // + // @@ Maybe we shouldn't be considering extras for partially_installed + // determination? + // + using status_type = package_status::status_type; + + auto status = [] (const vector& pps) -> optional + { + bool i (false), u (false); + + for (const package_policy& pp: pps) + { + if (pp.installed_version.empty () && pp.candidate_version.empty ()) + return nullopt; + + (pp.installed_version.empty () ? u : i) = true; + } + + return (!u ? package_status::installed : + !i ? package_status::not_installed : + package_status::partially_installed); + }; + + // First look for an already fully installed package. // unique_ptr r; - for (unique_ptr& c: rs) + for (unique_ptr& ps: rs) { - vector pps; + vector& pps (ps->package_policies); - // @@ TODO: rest of packages (and main can be empty). - // - pps.push_back (package_policy {c->main, "", ""}); + if (!ps->main.empty ()) pps.emplace_back (ps->main); + if (!ps->dev.empty ()) pps.emplace_back (ps->dev); + if (!ps->doc.empty () && need_doc) pps.emplace_back (ps->doc); + if (!ps->dbg.empty () && need_dbg) pps.emplace_back (ps->dbg); + if (!ps->common.empty () && false) pps.emplace_back (ps->common); + for (const string& n: ps->extras) pps.emplace_back (n); apt_cache_policy (pps); + + // Handle the unknown main package. + // + if (ps->main.empty ()) + { + const package_policy& dev (pps.front ()); + + // Note that at this stage we can only use the installed dev package + // (since the candidate version may change after fetch). + // + if (dev.installed_version.empty ()) + continue; + + guess_main (*ps, dev.installed_version); + pps.emplace (pps.begin (), ps->main); + apt_cache_policy (pps, 1); + } + + optional s (status (pps)); + + if (!s) + continue; + + if (*s == package_status::installed) + { + const package_policy& main (pps.front ()); + + ps->status = *s; + ps->system_name = main.name; + ps->system_version = main.installed_version; + + if (r != nullptr) + { + fail << "multiple installed Debian packages for " << pn << + info << "first package: " << r->main << " " << r->system_version << + info << "second package: " << ps->main << " " << ps->system_version << + info << "consider specifying the desired version manually"; + } + + r = move (ps); + } } // Next look for available versions if we are allowed to install. // if (r == nullptr && install) { - if (fetch && !fetched_) + // If we weren't instructed to fetch or we already feteched, then we + // don't need to re-run apt_cache_policy(). + // + bool requery; + if ((requery = fetch && !fetched_)) { // @@ TODO: apt-get update fetched_ = true; } + + for (unique_ptr& ps: rs) + { + vector& pps (ps->package_policies); + + if (requery) + apt_cache_policy (pps); + + // Handle the unknown main package. + // + if (ps->main.empty ()) + { + const package_policy& dev (pps.front ()); + + // Note that this time we use the candidate version. + // + if (dev.candidate_version.empty ()) + { + ps = nullptr; // Not installable. + continue; + } + + guess_main (*ps, dev.candidate_version); + pps.emplace (pps.begin (), ps->main); + apt_cache_policy (pps, 1); + + // @@ What if the main version doesn't match dev? Or it must? Or we + // use the candidate_version for main? Fuzzy. + } + + optional s (status (pps)); + + if (!s) + { + ps = nullptr; // Not installable. + continue; + } + + assert (*s != package_status::installed); // Sanity check. + + const package_policy& main (pps.front ()); + + ps->status = *s; + ps->system_name = main.name; + ps->system_version = main.candidate_version; + + // Prefer partially installed to not installed. This makes detecting + // ambiguity a bit trickier so we handle partially installed here and + // not installed in a separate loop below. + // + if (ps->status == package_status::partially_installed) + { + if (r != nullptr) + { + fail << "multiple partially installed Debian packages for " << pn << + info << "first package: " << r->main << " " << r->system_version << + info << "second package: " << ps->main << " " << ps->system_version << + info << "consider specifying the desired version manually"; + } + + r = move (ps); + } + } + + if (r == nullptr) + { + for (unique_ptr& ps: rs) + { + if (ps == nullptr) + continue; + + assert (ps->status != package_status::not_installed); // Sanity check. + + if (r != nullptr) + { + fail << "multiple available Debian packages for " << pn << + info << "first package: " << r->main << " " << r->system_version << + info << "second package: " << ps->main << " " << ps->system_version << + info << "consider installing the desired package manually"; + } + + r = move (ps); + } + } + } + + if (r != nullptr) + { + // @@ TODO: map system version to bpkg version. } // Cache. -- cgit v1.1