More work on Debian implementation

1 files changed, 488 insertions, 41 deletions

diff --git a/bpkg/system-package-manager-debian.cxx b/bpkg/system-package-manager-debian.cxx
index f1e78a1..d294416 100644
--- a/bpkg/system-package-manager-debian.cxx
+++ b/bpkg/system-package-manager-debian.cxx
@@ -41,7 +41,7 @@ namespace bpkg
   // libssl3 libssl-dev libssl-doc
   //
   // libcurl4 libcurl4-doc libcurl4-openssl-dev
-  // libcurl3-gnutls libcurl4-gnutls-dev
+  // libcurl3-gnutls libcurl4-gnutls-dev         (yes, 3 and 4)
   //
   // Based on that, it seems our best bet when trying to automatically map our
   // library package name to Debian package names is to go for the -dev
@@ -56,6 +56,17 @@ namespace bpkg
   // For executable packages there is normally no -dev packages but -dbg,
   // -doc, and -common are plausible.
   //
+  struct package_policy // apt-cache policy output
+  {
+    reference_wrapper<const string> name;
+
+    string installed_version; // Empty if none.
+    string candidate_version; // Empty if none.
+
+    explicit
+    package_policy (const string& n): name (n) {}
+  };
+
   class system_package_status_debian: public system_package_status
   {
   public:
@@ -66,6 +77,8 @@ namespace bpkg
     string common;
     strings extras;
 
+    vector<package_policy> package_policies;
+
     explicit
     system_package_status_debian (string m, string d = {})
         : main (move (m)), dev (move (d))
@@ -117,11 +130,15 @@ namespace bpkg
   // well, you are out of luck, I guess).
   //
   // Note also that for now we treat all the packages from the non-main groups
-  // as extras. But in the future we may decide to sort them out like the main
-  // group.
+  // as extras (but in the future we may decide to sort them out like the main
+  // group). For now we omit the -common package (assuming it's pulled by the
+  // main package) as well as -doc and -dbg unless requested (the
+  // extra_{doc,dbg} arguments).
   //
   static unique_ptr<package_status_debian>
-  parse_debian_name (const string& nv)
+  parse_debian_name (const string& nv,
+                     bool extra_doc,
+                     bool extra_dbg)
   {
     auto split = [] (const string& s, char d) -> strings
     {
@@ -203,14 +220,12 @@ namespace bpkg
       {
         unique_ptr<package_status_debian> g (parse_group (gs[i]));
 
-        // @@ Shouldn't we filter some based on what we are installing?
-
-        if (!g->main.empty ())   r->extras.push_back (move (g->main));
-        if (!g->dev.empty ())    r->extras.push_back (move (g->dev));
-        if (!g->doc.empty ())    r->extras.push_back (move (g->doc));
-        if (!g->dbg.empty ())    r->extras.push_back (move (g->dbg));
-        if (!g->common.empty ()) r->extras.push_back (move (g->common));
-        if (!g->extras.empty ()) r->extras.insert (
+        if (!g->main.empty ())             r->extras.push_back (move (g->main));
+        if (!g->dev.empty ())              r->extras.push_back (move (g->dev));
+        if (!g->doc.empty () && extra_doc) r->extras.push_back (move (g->doc));
+        if (!g->dbg.empty () && extra_dbg) r->extras.push_back (move (g->dbg));
+        if (!g->common.empty () && false)  r->extras.push_back (move (g->common));
+        if (!g->extras.empty ())           r->extras.insert (
           r->extras.end (),
           make_move_iterator (g->extras.begin ()),
           make_move_iterator (g->extras.end ()));
@@ -220,22 +235,20 @@ namespace bpkg
     return r;
   }
 
+  static process_path apt_cache;
+
   // Obtain the installed and candidate versions for the specified list
   // of Debian packages by executing apt-cache policy.
   //
-  struct package_policy
-  {
-    string name;
-    string installed_version; // Empty if none.
-    string candidate_version; // Empty if none.
-  };
-
-  static process_path apt_cache;
-
+  // If the n argument is not 0, then only query the first n packages.
+  //
   static void
-  apt_cache_policy (vector<package_policy>& pps)
+  apt_cache_policy (vector<package_policy>& pps, size_t n = 0)
   {
-    assert (!pps.empty ());
+    if (n == 0)
+      n = pps.size ();
+
+    assert (n != 0 && n <= pps.size ());
 
     // In particular, --quite makes sure we don't get a noice (N) printed to
     // stderr if the package is unknown. It does not appear to affect error
@@ -243,13 +256,17 @@ namespace bpkg
     //
     cstrings args {"apt-cache", "policy", "--quiet"};
 
-    for (const package_policy& pp: pps)
+    for (size_t i (0); i != n; ++i)
     {
-      assert (!pp.name.empty ()             &&
-              pp.installed_version.empty () &&
-              pp.candidate_version.empty ());
+      package_policy& pp (pps[i]);
+
+      const string& n (pp.name);
+      assert (!n.empty ());
+
+      pp.installed_version.clear ();
+      pp.candidate_version.clear ();
 
-      args.push_back (pp.name.c_str ());
+      args.push_back (n.c_str ());
     }
 
     args.push_back (nullptr);
@@ -314,7 +331,7 @@ namespace bpkg
               print_process (dr, pe, args);
             });
 
-          auto i (pps.begin ());
+          size_t i (0);
 
           string l;
           for (getline (is, l); !eof (is); )
@@ -328,11 +345,13 @@ namespace bpkg
 
             // Skip until this package.
             //
-            for (; i != pps.end () && i->name != l; ++i) ;
+            for (; i != n && pps[i].name.get () != l; ++i) ;
 
-            if (i == pps.end ())
+            if (i == n)
               fail << "unexpected package name '" << l << "'";
 
+            package_policy& pp (pps[i]);
+
             auto parse_version = [&l] (const string& n) -> string
             {
               size_t s (n.size ());
@@ -357,14 +376,14 @@ namespace bpkg
             if (eof (getline (is, l)))
               fail << "expected Installed version line after package name";
 
-            i->installed_version = parse_version ("Installed");
+            pp.installed_version = parse_version ("Installed");
 
             // Get the candidate version line.
             //
             if (eof (getline (is, l)))
               fail << "expected Candidate version line after Installed version";
 
-            i->installed_version = parse_version ("Candidate");
+            pp.candidate_version = parse_version ("Candidate");
 
             // Skip the rest of the indented lines (or blanks, just in case).
             //
@@ -405,6 +424,246 @@ namespace bpkg
     }
   }
 
+  // Return the Depends value, if any, for the specified package and version.
+  // Fail if either package or version is unknown.
+  //
+  static string
+  apt_cache_show (const string& name, const string& ver)
+  {
+    assert (!name.empty () && !ver.empty ());
+
+    string spec (name + '=' + ver);
+
+    // In particular, --quite makes sure we don't get noices (N) printed to
+    // stderr. It does not appear to affect error diagnostics (try showing
+    // information for an unknown package).
+    //
+    const char* args[] = {
+      "apt-cache", "show", "--quiet", spec.c_str (), nullptr};
+
+    // Note that for this command there seems to be no need to run with the C
+    // locale since the output is presumably not localizable. But let's do it
+    // for good measure and also seeing that we try to backfit some
+    // diagnostics into apt-cache (see no_version below).
+    //
+    const char* evars[] = {"LC_ALL=C", nullptr};
+
+    string r;
+    try
+    {
+      if (apt_cache.empty ())
+        apt_cache = process::path_search (args[0]);
+
+      process_env pe (apt_cache, evars);
+
+      if (verb >= 3)
+        print_process (pe, args);
+
+      // Redirect stdout to a pipe. For good measure also redirect stdin to
+      // /dev/null to make sure there are no prompts of any kind.
+      //
+      process pr (apt_cache,
+                  args,
+                  -2      /* stdin */,
+                  -1      /* stdout */,
+                  2       /* stderr */,
+                  nullptr /* cwd */,
+                  evars);
+
+      bool no_version (false);
+      try
+      {
+        ifdstream is (move (pr.in_ofd), fdstream_mode::skip, ifdstream::badbit);
+
+        // The output of `apt-cache show <pkg>=<ver>` appears to be a single
+        // Debian control file in the RFC 822 encoding followed by a blank
+        // line. See deb822(5) for details. Here is a representative example:
+        //
+        // Package: libcurl4
+        // Version: 7.85.0-1
+        // Depends: libbrotli1 (>= 0.6.0), libc6 (>= 2.34), ...
+        // Description-en: easy-to-use client-side URL transfer library
+        //  libcurl is an easy-to-use client-side URL transfer library.
+        //
+        // Note that if the package is unknown, then we get an error but if
+        // the version is unknown, we get no output (and a note if running
+        // without --quiet).
+        //
+        string l;
+        if (eof (getline (is, l)))
+        {
+          // The unknown version case. Issue diagnostics consistent with the
+          // unknown package case, at least for the English locale.
+          //
+          text << "E: No package version found";
+          no_version = true;
+        }
+        else
+        {
+          auto df = make_diag_frame (
+            [&pe, &args] (diag_record& dr)
+            {
+              dr << info << "while parsing output of ";
+              print_process (dr, pe, args);
+            });
+
+          do
+          {
+            // This line should be the start of a field unless it's a
+            // comment. According to deb822(5), there can be no leading
+            // whitespaces before `#`.
+            //
+            if (l[0] == '#')
+            {
+              getline (is, l);
+              continue;
+            }
+
+            size_t p (l.find (':'));
+
+            if (p == string::npos)
+              fail << "expected field name instead of '" << l << "'";
+
+            // Extract the field name. Note that field names are case-
+            // insensitive.
+            //
+            string n (l, 0, p);
+            trim (n);
+
+            // Extract the field value.
+            //
+            string v (l, p + 1);
+            trim (v);
+
+            // If we have more lines see if the following line is part of this
+            // value.
+            //
+            while (!eof (getline (is, l)) && (l[0] == ' ' || l[0] == '\t'))
+            {
+              // This can either be a "folded" or a "multiline" field and
+              // which one it is depends on the field semantics. Here we only
+              // care about Depends and so treat them all as folded (it's
+              // unclear whether Depends must be a simple field).
+              //
+              trim (l);
+              v += ' ';
+              v += l;
+            }
+
+            // See if this is a field of interest.
+            //
+            if (icasecmp (n, "Package") == 0)
+            {
+              assert (v == name); // Sanity check.
+            }
+            else if (icasecmp (n, "Version") == 0)
+            {
+              assert (v == ver); // Sanity check.
+            }
+            else if (icasecmp (n, "Depends") == 0)
+            {
+              r = move (v);
+
+              // Let's not waste time reading any further.
+              //
+              break;
+            }
+          }
+          while (!eof (is));
+        }
+
+        is.close ();
+      }
+      catch (const io_error& e)
+      {
+        if (pr.wait ())
+          fail << "unable to read " << args[0] << " policy output: " << e;
+
+        // Fall through.
+      }
+
+      if (!pr.wait () || no_version)
+      {
+        diag_record dr (fail);
+        dr << args[0] << " policy exited with non-zero code";
+
+        if (verb < 3)
+        {
+          dr << "command line: ";
+          print_process (dr, pe, args);
+        }
+      }
+    }
+    catch (const process_error& e)
+    {
+      error << "unable to execute " << args[0] << ": " << e;
+
+      if (e.child)
+        exit (1);
+
+      throw failed ();
+    }
+
+    return r;
+  }
+
+  // Attempt to determine the main package name from its -dev package. Return
+  // empty string if unable. Save the extracted Depends value to the depends
+  // argument for diagnostics.
+  //
+  static string
+  main_from_dev (const string& dev_name,
+                 const string& dev_ver,
+                 string& depends)
+  {
+    depends = apt_cache_show (dev_name, dev_ver);
+
+    // The format of the Depends value is a comma-seperated list of dependency
+    // expressions. For example:
+    //
+    // Depends: libssl3 (= 3.0.7-1), libc6 (>= 2.34), libfoo | libbar
+    //
+    // For the main package we look for a dependency in the form:
+    //
+    // <dev-stem>* (= <dev-ver>)
+    //
+    // Usually it is the first one.
+    //
+    string dev_stem (dev_name, 0, dev_name.rfind ("-dev"));
+
+    string r;
+    for (size_t b (0), e (0); next_word (depends, b, e, ','); )
+    {
+      string d (depends, b, e - b);
+      trim (d);
+
+      size_t p (d.find (' '));
+      if (p != string::npos)
+      {
+        if (d.compare (0, dev_stem.size (), dev_stem) == 0) // <dev-stem>*
+        {
+          size_t q (d.find ('(', p + 1));
+          if (q != string::npos && d.back () == ')') // (...)
+          {
+            if (d[q + 1] == '=' && d[q + 2] == ' ') // Equal.
+            {
+              string v (d, q + 3, d.size () - q - 3 - 1);
+              trim (v);
+
+              if (v == dev_ver)
+              {
+                r.assign (d, 0, p);
+                break;
+              }
+            }
+          }
+        }
+      }
+    }
+
+    return r;
+  }
+
   optional<const system_package_status*>
   system_package_manager_debian::
   pkg_status (const package_name& pn,
@@ -412,6 +671,13 @@ namespace bpkg
               bool install,
               bool fetch)
   {
+    // For now we ignore -doc and -dbg package components (but we may want to
+    // have options controlling this later). Note also that we assume -common
+    // is pulled automatically by the main package so we ignore it as well.
+    //
+    bool need_doc (false);
+    bool need_dbg (false);
+
     // First check the cache.
     //
     {
@@ -471,7 +737,8 @@ namespace bpkg
         //
         for (const string& n: ns)
         {
-          unique_ptr<package_status_debian> s (parse_debian_name (n));
+          unique_ptr<package_status_debian> s (
+            parse_debian_name (n, need_doc, need_dbg));
 
           // Suppress duplicates for good measure based on the main package
           // name (and falling back to -dev if empty).
@@ -493,31 +760,211 @@ namespace bpkg
       }
     }
 
-    // First look for an already installed package.
+    // Guess unknown main package given the dev package and its version.
+    //
+    auto guess_main = [&pn] (package_status_debian& s, const string& ver)
+    {
+      string depends;
+      s.main = main_from_dev (s.dev, ver, depends);
+
+      if (s.main.empty ())
+      {
+        fail << "unable to guess main Debian package for " << s.dev << ' '
+             << ver <<
+          info << s.dev << " Depends value: " << depends <<
+          info << "consider specifying explicit mapping in " << pn
+             << " package manifest";
+      }
+    };
+
+    // Calculate the package status from individual package components.
+    // Return nullopt if there is a component without installed or candidate
+    // version (which means the package cannot be installed).
+    //
+    // @@ Maybe we shouldn't be considering extras for partially_installed
+    //    determination?
+    //
+    using status_type = package_status::status_type;
+
+    auto status = [] (const vector<package_policy>& pps) -> optional<status_type>
+    {
+      bool i (false), u (false);
+
+      for (const package_policy& pp: pps)
+      {
+        if (pp.installed_version.empty () && pp.candidate_version.empty ())
+          return nullopt;
+
+        (pp.installed_version.empty () ? u : i) = true;
+      }
+
+      return (!u ? package_status::installed :
+              !i ? package_status::not_installed :
+              package_status::partially_installed);
+    };
+
+    // First look for an already fully installed package.
     //
     unique_ptr<package_status_debian> r;
 
-    for (unique_ptr<package_status_debian>& c: rs)
+    for (unique_ptr<package_status_debian>& ps: rs)
     {
-      vector<package_policy> pps;
+      vector<package_policy>& pps (ps->package_policies);
 
-      // @@ TODO: rest of packages (and main can be empty).
-      //
-      pps.push_back (package_policy {c->main, "", ""});
+      if (!ps->main.empty ())            pps.emplace_back (ps->main);
+      if (!ps->dev.empty ())             pps.emplace_back (ps->dev);
+      if (!ps->doc.empty () && need_doc) pps.emplace_back (ps->doc);
+      if (!ps->dbg.empty () && need_dbg) pps.emplace_back (ps->dbg);
+      if (!ps->common.empty () && false) pps.emplace_back (ps->common);
+      for (const string& n: ps->extras)  pps.emplace_back (n);
 
       apt_cache_policy (pps);
+
+      // Handle the unknown main package.
+      //
+      if (ps->main.empty ())
+      {
+        const package_policy& dev (pps.front ());
+
+        // Note that at this stage we can only use the installed dev package
+        // (since the candidate version may change after fetch).
+        //
+        if (dev.installed_version.empty ())
+          continue;
+
+        guess_main (*ps, dev.installed_version);
+        pps.emplace (pps.begin (), ps->main);
+        apt_cache_policy (pps, 1);
+      }
+
+      optional<status_type> s (status (pps));
+
+      if (!s)
+        continue;
+
+      if (*s == package_status::installed)
+      {
+        const package_policy& main (pps.front ());
+
+        ps->status = *s;
+        ps->system_name = main.name;
+        ps->system_version = main.installed_version;
+
+        if (r != nullptr)
+        {
+          fail << "multiple installed Debian packages for " << pn <<
+            info << "first package: " << r->main << " " << r->system_version <<
+            info << "second package: " << ps->main << " " << ps->system_version <<
+            info << "consider specifying the desired version manually";
+        }
+
+        r = move (ps);
+      }
     }
 
     // Next look for available versions if we are allowed to install.
     //
     if (r == nullptr && install)
     {
-      if (fetch && !fetched_)
+      // If we weren't instructed to fetch or we already feteched, then we
+      // don't need to re-run apt_cache_policy().
+      //
+      bool requery;
+      if ((requery = fetch && !fetched_))
       {
         // @@ TODO: apt-get update
 
         fetched_ = true;
       }
+
+      for (unique_ptr<package_status_debian>& ps: rs)
+      {
+        vector<package_policy>& pps (ps->package_policies);
+
+        if (requery)
+          apt_cache_policy (pps);
+
+        // Handle the unknown main package.
+        //
+        if (ps->main.empty ())
+        {
+          const package_policy& dev (pps.front ());
+
+          // Note that this time we use the candidate version.
+          //
+          if (dev.candidate_version.empty ())
+          {
+            ps = nullptr; // Not installable.
+            continue;
+          }
+
+          guess_main (*ps, dev.candidate_version);
+          pps.emplace (pps.begin (), ps->main);
+          apt_cache_policy (pps, 1);
+
+          // @@ What if the main version doesn't match dev? Or it must? Or we
+          //    use the candidate_version for main? Fuzzy.
+        }
+
+        optional<status_type> s (status (pps));
+
+        if (!s)
+        {
+          ps = nullptr; // Not installable.
+          continue;
+        }
+
+        assert (*s != package_status::installed); // Sanity check.
+
+        const package_policy& main (pps.front ());
+
+        ps->status = *s;
+        ps->system_name = main.name;
+        ps->system_version = main.candidate_version;
+
+        // Prefer partially installed to not installed. This makes detecting
+        // ambiguity a bit trickier so we handle partially installed here and
+        // not installed in a separate loop below.
+        //
+        if (ps->status == package_status::partially_installed)
+        {
+          if (r != nullptr)
+          {
+            fail << "multiple partially installed Debian packages for " << pn <<
+              info << "first package: " << r->main << " " << r->system_version <<
+              info << "second package: " << ps->main << " " << ps->system_version <<
+              info << "consider specifying the desired version manually";
+          }
+
+          r = move (ps);
+        }
+      }
+
+      if (r == nullptr)
+      {
+        for (unique_ptr<package_status_debian>& ps: rs)
+        {
+          if (ps == nullptr)
+            continue;
+
+          assert (ps->status != package_status::not_installed); // Sanity check.
+
+          if (r != nullptr)
+          {
+            fail << "multiple available Debian packages for " << pn <<
+              info << "first package: " << r->main << " " << r->system_version <<
+              info << "second package: " << ps->main << " " << ps->system_version <<
+              info << "consider installing the desired package manually";
+          }
+
+          r = move (ps);
+        }
+      }
+    }
+
+    if (r != nullptr)
+    {
+      // @@ TODO: map system version to bpkg version.
     }
 
     // Cache.