Add support for loading package version reviews

1 files changed, 386 insertions, 18 deletions

diff --git a/load/load.cxx b/load/load.cxx
index 2b2cd56..f79b606 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -3,6 +3,7 @@
 
 #include <signal.h> // signal()
 
+#include <map>
 #include <cerrno>
 #include <chrono>
 #include <thread>   // this_thread::sleep_for()
@@ -31,6 +32,7 @@
 #include <libbrep/package.hxx>
 #include <libbrep/package-odb.hxx>
 #include <libbrep/database-lock.hxx>
+#include <libbrep/review-manifest.hxx>
 
 #include <load/load-options.hxx>
 #include <load/options-types.hxx>
@@ -56,6 +58,7 @@ static const char* help_info (
 
 static const path packages     ("packages.manifest");
 static const path repositories ("repositories.manifest");
+static const path reviews      ("reviews.manifest");
 
 // Retry executing bpkg on recoverable errors for about 10 seconds.
 //
@@ -362,6 +365,56 @@ repository_info (const options& lo, const string& rl, const cstrings& options)
   }
 }
 
+// Map of package versions to their metadata information in the form it is
+// stored in the database (reviews summary, etc).
+//
+// This map is filled by recursively traversing the metadata directory and
+// parsing the encountered metadata manifest files (reviews.manifest, etc; see
+// --metadata option for background on metadata). Afterwards, this map is used
+// as a data source for the being persisted/updated package objects.
+//
+struct package_version_key
+{
+  package_name  name;
+  brep::version version;
+
+  package_version_key (package_name n, brep::version v)
+      : name (move (n)), version (move (v)) {}
+
+  bool
+  operator< (const package_version_key& k) const
+  {
+    if (int r = name.compare (k.name))
+      return r < 0;
+
+    return version < k.version;
+  }
+};
+
+class package_version_metadata
+{
+public:
+  // Extracted from the package metadata directory. Must match the respective
+  // package manifest information.
+  //
+  package_name project;
+
+  optional<reviews_summary> reviews;
+
+  // The directory the metadata manifest files are located. It has the
+  // <project>/<package>/<version> form and is only used for diagnostics.
+  //
+  dir_path
+  directory () const
+  {
+    assert (reviews); // At least one kind of metadata must be present.
+    return reviews->manifest_file.directory ();
+  }
+};
+
+using package_version_metadata_map = std::map<package_version_key,
+                                              package_version_metadata>;
+
 // Load the repository packages from the packages.manifest file and persist
 // the repository. Should be called once per repository.
 //
@@ -372,7 +425,8 @@ load_packages (const options& lo,
                database& db,
                bool ignore_unknown,
                const manifest_name_values& overrides,
-               const string& overrides_name)
+               const string& overrides_name,
+               optional<package_version_metadata_map>& metadata)
 {
   // packages_timestamp other than timestamp_nonexistent signals the
   // repository packages are already loaded.
@@ -728,6 +782,31 @@ load_packages (const options& lo,
               keys_to_objects (move (pm.build_configs[i].bot_keys));
         }
 
+        optional<reviews_summary> rvs;
+
+        if (metadata)
+        {
+          auto i (metadata->find (package_version_key {pm.name, pm.version}));
+
+          if (i != metadata->end ())
+          {
+            package_version_metadata& md (i->second);
+
+            if (md.project != project)
+            {
+              cerr << "error: project '" << project << "' of package "
+                   << pm.name << ' ' << pm.version << " doesn't match "
+                   << "metadata directory path "
+                   << lo.metadata () / md.directory ();
+
+              throw failed ();
+            }
+
+            if (md.reviews)
+              rvs = move (md.reviews);
+          }
+        }
+
         p = make_shared<package> (
           move (pm.name),
           move (pm.version),
@@ -758,6 +837,7 @@ load_packages (const options& lo,
           move (pm.build_auxiliaries),
           move (bot_keys),
           move (build_configs),
+          move (rvs),
           move (pm.location),
           move (pm.fragment),
           move (pm.sha256sum),
@@ -1153,13 +1233,16 @@ load_repositories (const options& lo,
 
     // We don't apply overrides to the external packages.
     //
+    optional<package_version_metadata_map> metadata;
+
     load_packages (lo,
                    pr,
                    !pr->cache_location.empty () ? pr->cache_location : cl,
                    db,
                    ignore_unknown,
                    manifest_name_values () /* overrides */,
-                   "" /* overrides_name */);
+                   "" /* overrides_name */,
+                   metadata);
 
     load_repositories (lo,
                        pr,
@@ -1778,6 +1861,11 @@ try
     }
   }
 
+  // Note: the interactive tenant implies private.
+  //
+  if (ops.interactive_specified ())
+    ops.private_ (true);
+
   // Parse and validate overrides, if specified.
   //
   // Note that here we make sure that the overrides manifest is valid.
@@ -1818,34 +1906,236 @@ try
     ops.db_port (),
     "options='-c default_transaction_isolation=serializable'");
 
+  // Load the description of all the internal repositories from the
+  // configuration file.
+  //
+  internal_repositories irs (load_repositories (path (argv[1])));
+
   // Prevent several brep utility instances from updating the package database
   // simultaneously.
   //
   database_lock l (db);
 
-  transaction t (db.begin ());
-
-  // Check that the package database schema matches the current one.
+  // Check that the package database schema matches the current one and if the
+  // package information needs to be (re-)loaded.
   //
-  const string ds ("package");
-  if (schema_catalog::current_version (db, ds) != db.schema_version (ds))
+  bool load_pkgs;
   {
-    cerr << "error: package database schema differs from the current one"
-         << endl << "  info: use brep-migrate to migrate the database" << endl;
-    throw failed ();
+    transaction t (db.begin ());
+
+    // Check the database schema match.
+    //
+    const string ds ("package");
+
+    if (schema_catalog::current_version (db, ds) != db.schema_version (ds))
+    {
+      cerr << "error: package database schema differs from the current one"
+           << endl << "  info: use brep-migrate to migrate the database" << endl;
+      throw failed ();
+    }
+
+    load_pkgs = (ops.force () || changed (tnt, irs, db));
+
+    t.commit ();
   }
 
-  // Note: the interactive tenant implies private.
+  // Check if the package versions metadata needs to be (re-)loaded and, if
+  // that's the case, stash it in the memory.
   //
-  if (ops.interactive_specified ())
-    ops.private_ (true);
+  optional<package_version_metadata_map> metadata;
+  if (ops.metadata_specified () && (load_pkgs || ops.metadata_changed ()))
+  {
+    metadata = package_version_metadata_map ();
 
-  // Load the description of all the internal repositories from the
-  // configuration file.
+    const dir_path& d (ops.metadata ());
+
+    // The first level are package projects.
+    //
+    try
+    {
+      for (const dir_entry& e: dir_iterator (d, dir_iterator::no_follow))
+      {
+        const string& n (e.path ().string ());
+
+        if (e.type () != entry_type::directory || n[0] == '.')
+          continue;
+
+        package_name project;
+
+        try
+        {
+          project = package_name (n);
+        }
+        catch (const invalid_argument& e)
+        {
+          cerr << "error: name of subdirectory '" << n << "' in " << d
+               << " is not a project name: " << e << endl;
+          throw failed ();
+        }
+
+        // The second level are package names.
+        //
+        dir_path pd (d / path_cast<dir_path> (e.path ()));
+
+        try
+        {
+          for (const dir_entry& e: dir_iterator (pd, dir_iterator::no_follow))
+          {
+            const string& n (e.path ().string ());
+
+            if (e.type () != entry_type::directory || n[0] == '.')
+              continue;
+
+            package_name name;
+
+            try
+            {
+              name = package_name (n);
+            }
+            catch (const invalid_argument& e)
+            {
+              cerr << "error: name of subdirectory '" << n << "' in " << pd
+                   << " is not a package name: " << e << endl;
+              throw failed ();
+            }
+
+            // The third level are package versions.
+            //
+            dir_path vd (pd / path_cast<dir_path> (e.path ()));
+
+            try
+            {
+              for (const dir_entry& e: dir_iterator (vd,
+                                                     dir_iterator::no_follow))
+              {
+                const string& n (e.path ().string ());
+
+                if (e.type () != entry_type::directory || n[0] == '.')
+                  continue;
+
+                version ver;
+
+                try
+                {
+                  ver = version (n);
+                }
+                catch (const invalid_argument& e)
+                {
+                  cerr << "error: name of subdirectory '" << n << "' in " << vd
+                       << " is not a package version: " << e << endl;
+                  throw failed ();
+                }
+
+                dir_path md (vd / path_cast<dir_path> (e.path ()));
+
+                // Parse the reviews.manifest file, if present.
+                //
+                // Note that semantically, the absent manifest file and the
+                // empty manifest list are equivalent and result in an absent
+                // reviews summary.
+                //
+                optional<reviews_summary> rs;
+                {
+                  path rf (md / reviews);
+
+                  try
+                  {
+                    if (file_exists (rf))
+                    {
+                      ifdstream ifs (rf);
+                      manifest_parser mp (ifs, rf.string ());
+
+                      // Count the passed and failed reviews.
+                      //
+                      size_t ps (0);
+                      size_t fl (0);
+
+                      for (review_manifest& m:
+                             review_manifests (mp, ops.ignore_unknown ()))
+                      {
+                        bool fail (false);
+
+                        for (const review_aspect& r: m.results)
+                        {
+                          switch (r.result)
+                          {
+                          case review_result::fail: fail = true; break;
+
+                          case review_result::unchanged:
+                            {
+                              cerr << "error: unsupported review result "
+                                   << "'unchanged' in " << rf << endl;
+                              throw failed ();
+                            }
+
+                          case review_result::pass: break; // Noop
+                          }
+                        }
+
+                        ++(fail ? fl : ps);
+                      }
+
+                      if (ps + fl != 0)
+                        rs = reviews_summary {ps, fl, rf.relative (d)};
+                    }
+                  }
+                  catch (const manifest_parsing& e)
+                  {
+                    cerr << "error: unable to parse reviews: " << e << endl;
+                    throw failed ();
+                  }
+                  catch (const io_error& e)
+                  {
+                    cerr << "error: unable to read " << rf << ": " << e << endl;
+                    throw failed ();
+                  }
+                  catch (const system_error& e)
+                  {
+                    cerr << "error: unable to stat " << rf << ": " << e << endl;
+                    throw failed ();
+                  }
+                }
+
+                // Add the package version metadata to the map if any kind of
+                // metadata is present.
+                //
+                if (rs)
+                {
+                  (*metadata)[package_version_key {name, move (ver)}] =
+                    package_version_metadata {project, move (rs)};
+                }
+              }
+            }
+            catch (const system_error& e)
+            {
+              cerr << "error: unable to iterate over " << vd << ": " << e
+                   << endl;
+              throw failed ();
+            }
+          }
+        }
+        catch (const system_error& e)
+        {
+          cerr << "error: unable to iterate over " << pd << ": " << e << endl;
+          throw failed ();
+        }
+      }
+    }
+    catch (const system_error& e)
+    {
+      cerr << "error: unable to iterate over " << d << ": " << e << endl;
+      throw failed ();
+    }
+  }
+
+  // Bail out if no package information nor metadata needs to be loaded.
   //
-  internal_repositories irs (load_repositories (path (argv[1])));
+  if (!load_pkgs && !metadata)
+    return 0;
+
+  transaction t (db.begin ());
 
-  if (ops.force () || changed (tnt, irs, db))
+  if (load_pkgs)
   {
     shared_ptr<tenant> t; // Not NULL in the --existing-tenant mode.
 
@@ -2015,7 +2305,8 @@ try
                      db,
                      ops.ignore_unknown (),
                      overrides,
-                     ops.overrides_file ().string ());
+                     ops.overrides_file ().string (),
+                     metadata);
     }
 
     // On the second pass over the internal repositories we load their
@@ -2070,6 +2361,83 @@ try
       }
     }
   }
+  else if (metadata)
+  {
+    // Iterate over the packages which contain metadata and apply the changes,
+    // if present. Erase the metadata map entries which introduce such
+    // changes, so at the end only the newly added metadata is left in the
+    // map.
+    //
+    using query = query<package>;
+
+    for (package& p: db.query<package> (query::reviews.pass.is_not_null ()))
+    {
+      bool u (false);
+      auto i (metadata->find (package_version_key {p.name, p.version}));
+
+      if (i == metadata->end ())
+      {
+        // Mark the section as loaded, so the reviews summary is updated.
+        //
+        p.reviews_section.load ();
+        p.reviews = nullopt;
+        u = true;
+      }
+      else
+      {
+        package_version_metadata& md (i->second);
+
+        if (md.project != p.project)
+        {
+          cerr << "error: project '" << p.project << "' of package "
+               << p.name << ' ' << p.version << " doesn't match metadata "
+               << "directory path " << ops.metadata () / md.directory ();
+
+          throw failed ();
+        }
+
+        db.load (p, p.reviews_section);
+
+        if (p.reviews != md.reviews)
+        {
+          p.reviews = move (md.reviews);
+          u = true;
+        }
+
+        metadata->erase (i);
+      }
+
+      if (u)
+        db.update (p);
+    }
+
+    // Add the newly added metadata to the packages.
+    //
+    for (auto& m: *metadata)
+    {
+      if (shared_ptr<package> p =
+          db.find<package> (package_id (tnt, m.first.name, m.first.version)))
+      {
+        package_version_metadata& md (m.second);
+
+        if (m.second.project != p->project)
+        {
+          cerr << "error: project '" << p->project << "' of package "
+               << p->name << ' ' << p->version << " doesn't match metadata "
+               << "directory path " << ops.metadata () / md.directory ();
+
+          throw failed ();
+        }
+
+        // Mark the section as loaded, so the reviews summary is updated.
+        //
+        p->reviews_section.load ();
+        p->reviews = move (md.reviews);
+
+        db.update (p);
+      }
+    }
+  }
 
   t.commit ();
   return 0;