Add support for builds manifest value

7 files changed, 360 insertions, 141 deletions

diff --git a/mod/build-config.cxx b/mod/build-config.cxx
index e838a59..5d3f46b 100644
--- a/mod/build-config.cxx
+++ b/mod/build-config.cxx
@@ -6,9 +6,10 @@
 
 #include <map>
 #include <sstream>
+#include <algorithm> // find()
 
 #include <libbutl/sha256.mxx>
-#include <libbutl/utility.mxx>    // throw_generic_error()
+#include <libbutl/utility.mxx>    // throw_generic_error(), alpha(), etc.
 #include <libbutl/openssl.mxx>
 #include <libbutl/filesystem.mxx>
 
@@ -150,21 +151,116 @@ namespace brep
   }
 
   bool
-  match (const string& config_pattern,
-         const optional<string>& target_pattern,
-         const build_config& c)
+  exclude (const build_class_exprs& exprs,
+           const build_constraints& constrs,
+           const build_config& cfg,
+           string* reason)
   {
-    return path_match (config_pattern, c.name) &&
-      (!target_pattern || path_match (*target_pattern, c.target.string ()));
-  }
+    // Save the first sentence of the reason, lower-case the first letter if
+    // the beginning looks like a word (the second character is the
+    // lower-case letter or space).
+    //
+    auto sanitize = [] (const string& reason)
+    {
+      string r (reason.substr (0, reason.find ('.')));
 
-  bool
-  exclude (const build_package& p, const build_config& c)
-  {
-    for (const auto& bc: p.constraints)
+      char c;
+      size_t n (r.size ());
+
+      if (n > 0            &&
+          alpha (c = r[0]) &&
+          c == ucase (c)   &&
+          (n == 1 || (alpha (c = r[1]) && c == lcase (c)) || c == ' '))
+        r[0] = lcase (r[0]);
+
+      return r;
+    };
+
+    bool r (false);
+
+    // First, match the configuration against the package underlying build
+    // class set and expressions.
+    //
+    // Determine the underlying class set. Note that in the future we can
+    // potentially extend the underlying set with the special classes.
+    //
+    build_class_expr ucs (
+      !exprs.empty () && !exprs.front ().underlying_classes.empty ()
+      ? exprs.front ()
+      : build_class_expr ("default", "Default"));
+
+    // Transform the combined package build configuration class expression,
+    // making the underlying class set a starting set for the original
+    // expression and a restricting set, simultaneously. For example, for the
+    // expression:
+    //
+    // default legacy : -msvc
+    //
+    // the resulting expression will be:
+    //
+    // +default +legacy -msvc &( +default +legacy )
+    //
+    //
+    build_class_exprs es;
+    es.emplace_back (ucs.underlying_classes, '+', ucs.comment);
+    es.insert       (es.end (), exprs.begin (), exprs.end ());
+    es.emplace_back (ucs.underlying_classes, '&', ucs.comment);
+
+    // We will use a comment of the first encountered excluding expression
+    // (changing the result from true to false) or non-including one (leaving
+    // the false result) as an exclusion reason.
+    //
+    for (const build_class_expr& e: es)
+    {
+      bool pr (r);
+      e.match (cfg.classes, r);
+
+      if (reason != nullptr)
+      {
+        // Reset the reason which, if saved, makes no sense anymore.
+        //
+        if (r)
+        {
+          reason->clear ();
+        }
+        else if (reason->empty () &&
+                 //
+                 // Exclusion.
+                 //
+                 (pr              ||
+                  //
+                  // Non-inclusion. Make sure that the build class expression
+                  // is empty or starts with an addition (+...).
+                  //
+                  e.expr.empty () ||
+                  e.expr.front ().operation == '+'))
+        {
+          *reason = sanitize (e.comment);
+        }
+      }
+    }
+
+    if (!r)
+      return true;
+
+    // Now check if the configuration is excluded/included via the patterns.
+    //
+    const string& cn (cfg.name);
+    string tg (cfg.target.string ());
+
+    for (const build_constraint& c: constrs)
     {
-      if (match (bc.config, bc.target, c))
-        return bc.exclusion;
+      if (path_match (c.config, cn) &&
+          (!c.target || path_match (*c.target, tg)))
+      {
+        if (!c.exclusion)
+          return false;
+
+        if (reason != nullptr)
+          *reason = sanitize (c.comment);
+
+        return true;
+      }
     }
 
     return false;
diff --git a/mod/build-config.hxx b/mod/build-config.hxx
index cdc7efb..e7ee9eb 100644
--- a/mod/build-config.hxx
+++ b/mod/build-config.hxx
@@ -6,6 +6,7 @@
 #define MOD_BUILD_CONFIG_HXX
 
 #include <map>
+#include <algorithm> // find()
 
 #include <libbbot/build-config.hxx>
 
@@ -21,6 +22,8 @@
 //
 namespace brep
 {
+  using bbot::build_config;
+
   // Return pointer to the shared build configurations instance, creating one
   // on the first call. Throw tab_parsing on parsing error, io_error on the
   // underlying OS error. Is not thread-safe.
@@ -52,17 +55,31 @@ namespace brep
   string
   force_rebuild_url (const string& host, const dir_path& root, const build&);
 
-  // Match a build configuration against the name and target patterns.
+  // Check if the configuration belongs to the specified class.
   //
-  bool
-  match (const string& config_pattern,
-         const optional<string>& target_pattern,
-         const bbot::build_config&);
+  inline bool
+  belongs (const build_config& cfg, const char* cls)
+  {
+    const strings& cs (cfg.classes);
+    return find (cs.begin (), cs.end (), cls) != cs.end ();
+  }
 
-  // Return true if a package excludes the specified build configuration.
+  // Return true if the specified build configuration is excluded by a package
+  // based on its underlying build class set, build class expressions, and
+  // build constraints, potentially extending the underlying set with the
+  // special classes. Set the exclusion reason if requested.
   //
   bool
-  exclude (const build_package&, const bbot::build_config&);
+  exclude (const build_class_exprs&,
+           const build_constraints&,
+           const build_config&,
+           string* reason = nullptr);
+
+  inline bool
+  exclude (const build_package& p, const build_config& c, string* r = nullptr)
+  {
+    return exclude (p.builds, p.constraints, c, r);
+  }
 }
 
 #endif // MOD_BUILD_CONFIG_HXX
diff --git a/mod/mod-build-configs.cxx b/mod/mod-build-configs.cxx
index 63bbe0b..d3d5191 100644
--- a/mod/mod-build-configs.cxx
+++ b/mod/mod-build-configs.cxx
@@ -89,9 +89,10 @@ handle (request& rq, response& rs)
 
   for (const build_config& c: *build_conf_)
   {
-    s << TR(CLASS="config")
-      <<   TD << SPAN(CLASS="value") << c.name << ~SPAN << ~TD
-      << ~TR;
+    if (belongs (c, "all"))
+      s << TR(CLASS="config")
+        <<   TD << SPAN(CLASS="value") << c.name << ~SPAN << ~TD
+        << ~TR;
   }
 
   s <<         ~TBODY
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 8b985be..177e67f 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -225,11 +225,17 @@ handle (request& rq, response&)
 
   // Make sure the build configuration still exists.
   //
-  if (build_conf_map_->find (id.configuration.c_str ()) ==
-      build_conf_map_->end ())
+  const bbot::build_config* cfg;
   {
-    warn_expired ("no build configuration");
-    return true;
+    auto i (build_conf_map_->find (id.configuration.c_str ()));
+
+    if (i == build_conf_map_->end ())
+    {
+      warn_expired ("no build configuration");
+      return true;
+    }
+
+    cfg = i->second;
   }
 
   // Load the built package (if present).
@@ -261,7 +267,8 @@ handle (request& rq, response&)
   // Load and update the package build configuration (if present).
   //
   // NULL if the package build doesn't exist or is not updated for any reason
-  // (authentication failed, etc).
+  // (authentication failed, etc) or the configuration is excluded by the
+  // package.
   //
   shared_ptr<build> bld;
 
@@ -390,7 +397,11 @@ handle (request& rq, response&)
 
         build_db_->update (b);
 
-        bld = move (b);
+        shared_ptr<build_package> p (
+          build_db_->load<build_package> (b->id.package));
+
+        if (belongs (*cfg, "all") && !exclude (*p, *cfg))
+          bld = move (b);
       }
     }
 
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index d3efb41..03c053a 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -328,6 +328,8 @@ handle (request& rq, response& rs)
     using pkg_query = query<buildable_package>;
     using prep_pkg_query = prepared_query<buildable_package>;
 
+    // Exclude archived tenants.
+    //
     pkg_query pq (!pkg_query::build_tenant::archived);
 
     // Filter by repositories canonical names (if requested).
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index fbdae4c..30d3696 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -5,6 +5,7 @@
 #include <mod/mod-builds.hxx>
 
 #include <set>
+#include <algorithm> // find_if()
 
 #include <libstudxml/serializer.hxx>
 
@@ -441,37 +442,167 @@ handle (request& rq, response& rs)
       s << DIV_COUNTER (build_count, "Build", "Builds");
   };
 
+  // We will not display hidden configurations, unless the configuration is
+  // specified explicitly.
+  //
+  cstrings conf_names;
+
+  if (params.configuration ().empty () ||
+      params.configuration ().find_first_of ("*?") != string::npos)
+  {
+    for (const auto& c: *build_conf_map_)
+    {
+      if (belongs (*c.second, "all"))
+        conf_names.push_back (c.first);
+    }
+  }
+  else
+    conf_names = *build_conf_names_;
+
   size_t count;
   size_t page (params.page ());
 
-  if (params.result () != "unbuilt")
+  if (params.result () != "unbuilt") // Print package build configurations.
   {
-    transaction t (build_db_->begin ());
+    // It seems impossible to filter out the package-excluded configuration
+    // builds via the database query. Thus, we will traverse through builds
+    // that pass the form filter and match them against expressions and
+    // constraints of a package they are builds of.
+    //
+    // We will calculate the total builds count and cache build objects for
+    // printing on the same pass. Note that we need to print the count before
+    // printing the builds.
+    //
+    count = 0;
+    vector<shared_ptr<build>> builds;
 
-    count = build_db_->query_value<package_build_count> (
-      build_query<package_build_count> (
-        *build_conf_names_, params, tn, nullopt /* archived */));
+    // Prepare the package build prepared query.
+    //
+    using query = query<package_build>;
+    using prep_query = prepared_query<package_build>;
 
-    // Print the filter form.
+    query q (build_query<package_build> (
+               conf_names, params, tn, nullopt /* archived */));
+
+    // Specify the portion. Note that we will be querying builds in chunks,
+    // not to hold locks for too long.
     //
-    print_form (query_toolchains (), count);
+    size_t offset (0);
 
     // Print package build configurations ordered by the timestamp (later goes
     // first).
     //
+    q += "ORDER BY" + query::build::timestamp + "DESC" +
+      "OFFSET" + query::_ref (offset) + "LIMIT 50";
+
+    connection_ptr conn (build_db_->connection ());
+
+    prep_query pq (
+      conn->prepare_query<package_build> ("mod-builds-query", q));
+
+    // Note that we can't skip the proper number of builds in the database
+    // query for a page numbers greater than one. So we will query builds from
+    // the very beginning and skip the appropriate number of them while
+    // iterating through the query result.
+    //
+    size_t skip (page * page_configs);
+    size_t print (page_configs);
+
+    // Note that adjacent builds may well relate to the same package. We will
+    // use this fact for a cheap optimization, loading the build package only
+    // if it differs from the previous one.
+    //
+    shared_ptr<build_package> p;
+
+    for (bool ne (true); ne; )
+    {
+      transaction t (conn->begin ());
+
+      // Query package builds (and cache the result).
+      //
+      auto bs (pq.execute ());
+
+      if ((ne = !bs.empty ()))
+      {
+        offset += bs.size ();
+
+        // Iterate over builds and cache build objects that should be printed.
+        // Skip the appropriate number of them (for page number greater than
+        // one).
+        //
+        for (auto& pb: bs)
+        {
+          shared_ptr<build>& b (pb.build);
+
+          // Prior to loading the package object check if it is already
+          // loaded.
+          //
+          if (p == nullptr || p->id != b->id.package)
+            p = build_db_->load<build_package> (b->id.package);
+
+          auto i (build_conf_map_->find (b->configuration.c_str ()));
+          assert (i != build_conf_map_->end ());
+
+          // Match the configuration against the package build
+          // expressions/constraints.
+          //
+          if (!exclude (*p, *i->second))
+          {
+            if (skip != 0)
+              --skip;
+            else if (print != 0)
+            {
+              // As we query builds in multiple transactions we may see the
+              // same build multiple times. Let's skip the duplicates. Note:
+              // we don't increment the counter in this case.
+              //
+              if (find_if (builds.begin (),
+                           builds.end (),
+                           [&b] (const shared_ptr<build>& pb)
+                           {
+                             return b->id == pb->id;
+                           }) != builds.end ())
+                continue;
+
+              if (b->state == build_state::built)
+              {
+                build_db_->load (*b, b->results_section);
+
+                // Let's clear unneeded result logs for builds being cached.
+                //
+                for (operation_result& r: b->results)
+                  r.log.clear ();
+              }
+
+              builds.push_back (move (b));
+
+              --print;
+            }
+
+            ++count;
+          }
+        }
+      }
+
+      // Print the filter form after the build count is calculated. Note:
+      // query_toolchains() must be called inside the build db transaction.
+      //
+      else
+        print_form (query_toolchains (), count);
+
+      t.commit ();
+    }
+
+    // Finally, print the cached package build configurations.
+    //
     timestamp now (system_clock::now ());
 
     // Enclose the subsequent tables to be able to use nth-child CSS selector.
     //
     s << DIV;
-    for (auto& pb: build_db_->query<package_build> (
-           build_query<package_build> (
-             *build_conf_names_, params, tn, nullopt /* archived */) +
-           "ORDER BY" + query<package_build>::build::timestamp + "DESC" +
-           "OFFSET" + to_string (page * page_configs) +
-           "LIMIT" + to_string (page_configs)))
+    for (const shared_ptr<build>& pb: builds)
     {
-      build& b (*pb.build);
+      const build& b (*pb);
 
       string ts (butl::to_string (b.timestamp,
                                   "%Y-%m-%d %H:%M:%S %Z",
@@ -479,9 +610,6 @@ handle (request& rq, response& rs)
                                   true) +
                  " (" + butl::to_string (now - b.timestamp, false) + " ago)");
 
-      if (b.state == build_state::built)
-        build_db_->load (b, b.results_section);
-
       s << TABLE(CLASS="proplist build")
         <<   TBODY
         <<     TR_NAME (b.package_name, string (), root, b.tenant)
@@ -505,8 +633,6 @@ handle (request& rq, response& rs)
         << ~TABLE;
     }
     s << ~DIV;
-
-    t.commit ();
   }
   else // Print unbuilt package configurations.
   {
@@ -609,12 +735,12 @@ handle (request& rq, response& rs)
       //
       // Note that we also need to deduct the package-excluded configurations
       // count from the maximum possible number of unbuilt configurations. The
-      // only way to achieve this is to traverse through the build-constrained
-      // packages and match their constraints against our configurations.
+      // only way to achieve this is to traverse through the packages and
+      // match their build expressions/constraints against our configurations.
       //
       // Also note that some existing builds can now be excluded by packages
-      // due to the build configuration target change. We should deduct such
-      // builds count from the number of existing package builds.
+      // due to the build configuration target or class set change. We should
+      // deduct such builds count from the number of existing package builds.
       //
       size_t nmax (
         config_toolchains.size () *
@@ -624,7 +750,7 @@ handle (request& rq, response& rs)
 
       size_t ncur = build_db_->query_value<package_build_count> (
         build_query<package_build_count> (
-          *build_conf_names_, bld_params, tn, false /* archived */));
+          conf_names, bld_params, tn, false /* archived */));
 
       // From now we will be using specific package name and version for each
       // build database query.
@@ -665,23 +791,23 @@ handle (request& rq, response& rs)
 
         size_t nt (tc == "*" ? toolchains.size () : 1);
 
-        // The number of build-constrained packages can potentially be large,
-        // and we may implement some caching in the future. However, the
-        // caching will not be easy as the cached values depend on the filter
-        // form parameters.
+        // The number of packages can potentially be large, and we may
+        // implement some caching in the future. However, the caching will not
+        // be easy as the cached values depend on the filter form parameters.
         //
-        query<build_constrained_package> q (
-          package_query<build_constrained_package> (
+        query<buildable_package> q (
+          package_query<buildable_package> (
             params, tn, false /* archived */));
 
-        for (const auto& p: build_db_->query<build_constrained_package> (q))
+        for (auto& bp: build_db_->query<buildable_package> (q))
         {
-          const build_package& bp (*p.package);
-          id = bp.id;
+          id = move (bp.id);
+
+          shared_ptr<build_package> p (build_db_->load<build_package> (id));
 
           for (const auto& c: configs)
           {
-            if (exclude (bp, *c))
+            if (exclude (*p, *c))
             {
               nmax -= nt;
 
@@ -706,9 +832,10 @@ handle (request& rq, response& rs)
     //
     // 1: package name
     // 2: package version (descending)
-    // 3: configuration name
-    // 4: toolchain name
-    // 5: toolchain version (descending)
+    // 3: package tenant
+    // 4: configuration name
+    // 5: toolchain name
+    // 6: toolchain version (descending)
     //
     // Prepare the build package prepared query.
     //
@@ -764,30 +891,12 @@ handle (request& rq, response& rs)
       // Note that the query already constrains the tenant via the build
       // package id.
       //
-      build_query<package_build> (*build_conf_names_,
-                                  bld_params,
-                                  nullopt /* tenant */,
-                                  false /* archived */));
+      build_query<package_build> (
+        conf_names, bld_params, nullopt /* tenant */, false /* archived */));
 
     prep_bld_query bld_prep_query (
       conn->prepare_query<package_build> ("mod-builds-build-query", bq));
 
-    // Prepare the build-constrained package prepared query.
-    //
-    // For each build-constrained package we will exclude the corresponding
-    // configurations from being printed.
-    //
-    using ctr_query = query<build_constrained_package>;
-    using prep_ctr_query = prepared_query<build_constrained_package>;
-
-    ctr_query cq (
-      package_id_eq<build_constrained_package> (
-        ctr_query::build_package::id, id));
-
-    prep_ctr_query ctr_prep_query (
-      conn->prepare_query<build_constrained_package> (
-        "mod-builds-build-constrained-package-query", cq));
-
     size_t skip (page * page_configs);
     size_t print (page_configs);
 
@@ -816,29 +925,20 @@ handle (request& rq, response& rs)
           id = move (p.id);
 
           // Copy configuration/toolchain combinations for this package,
-          // skipping explicitly excluded configurations.
+          // skipping excluded configurations.
           //
           set<config_toolchain> unbuilt_configs;
           {
-            build_constrained_package p;
-            if (ctr_prep_query.execute_one (p))
+            shared_ptr<build_package> p (build_db_->load<build_package> (id));
+
+            for (const auto& ct: config_toolchains)
             {
-              const build_package& bp (*p.package);
-              for (const auto& ct: config_toolchains)
-              {
-                auto i (build_conf_map_->find (ct.configuration.c_str ()));
-                assert (i != build_conf_map_->end ());
+              auto i (build_conf_map_->find (ct.configuration.c_str ()));
+              assert (i != build_conf_map_->end ());
 
-                if (!exclude (bp, *i->second))
-                  unbuilt_configs.insert (ct);
-              }
+              if (!exclude (*p, *i->second))
+                unbuilt_configs.insert (ct);
             }
-            else
-              // For libc++, the set's copy-assignment operator requires the
-              // element type to be copy-assignable, for some reason.
-              //
-              unbuilt_configs.insert (config_toolchains.begin (),
-                                      config_toolchains.end ());
           }
 
           // Iterate through the package configuration builds and erase them
@@ -856,7 +956,7 @@ handle (request& rq, response& rs)
           //
           for (const auto& ct: unbuilt_configs)
           {
-            if (skip > 0)
+            if (skip != 0)
             {
               --skip;
               continue;
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index bafe8f7..9566c8f 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -10,8 +10,6 @@
 #include <odb/database.hxx>
 #include <odb/transaction.hxx>
 
-#include <libbutl/utility.mxx> // alpha(), ucase(), lcase()
-
 #include <web/xhtml.hxx>
 #include <web/module.hxx>
 #include <web/mime-url-encoding.hxx>
@@ -390,24 +388,48 @@ handle (request& rq, response& rs)
 
   if (builds)
   {
+    using bbot::build_config;
+
     s << H3 << "Builds" << ~H3
       << DIV(ID="builds");
 
+    auto exclude = [&pkg] (const build_config& cfg, string* reason = nullptr)
+    {
+      return brep::exclude (pkg->builds, pkg->build_constraints, cfg, reason);
+    };
+
     timestamp now (system_clock::now ());
     transaction t (build_db_->begin ());
 
-    // Print built package configurations.
+    // Print built package configurations, except those that are hidden or
+    // excluded by the package.
     //
+    cstrings conf_names;
+
+    for (const auto& c: *build_conf_map_)
+    {
+      if (belongs (*c.second, "all"))
+        conf_names.push_back (c.first);
+    }
+
     using query = query<build>;
 
     for (auto& b: build_db_->query<build> (
            (query::id.package == pkg->id &&
 
-            query::id.configuration.in_range (build_conf_names_->begin (),
-                                              build_conf_names_->end ())) +
+            query::id.configuration.in_range (conf_names.begin (),
+                                              conf_names.end ())) +
 
            "ORDER BY" + query::timestamp + "DESC"))
     {
+      auto i (build_conf_map_->find (b.configuration.c_str ()));
+      assert (i != build_conf_map_->end ());
+
+      const build_config& cfg (*i->second);
+
+      if (exclude (cfg))
+        continue;
+
       string ts (butl::to_string (b.timestamp,
                                   "%Y-%m-%d %H:%M:%S %Z",
                                   true,
@@ -430,42 +452,12 @@ handle (request& rq, response& rs)
         << ~TABLE;
     }
 
-    // Print configurations that are excluded by the package.
+    // Print the package build exclusions that belong to the 'default' class.
     //
-    auto excluded = [&pkg] (const bbot::build_config& c, string& reason)
-    {
-      for (const auto& bc: pkg->build_constraints)
-      {
-        if (match (bc.config, bc.target, c))
-        {
-          if (!bc.exclusion)
-            return false;
-
-          // Save the first sentence of the exclusion comment, lower-case the
-          // first letter if the beginning looks like a word (the second
-          // character is the lower-case letter or space).
-          //
-          reason = bc.comment.substr (0, bc.comment.find ('.'));
-
-          char c;
-          size_t n (reason.size ());
-          if (n > 0 && alpha (c = reason[0]) && c == ucase (c) &&
-              (n == 1 ||
-               (alpha (c = reason[1]) && c == lcase (c)) ||
-               c == ' '))
-            reason[0] = lcase (reason[0]);
-
-          return true;
-        }
-      }
-
-      return false;
-    };
-
     for (const auto& c: *build_conf_)
     {
       string reason;
-      if (excluded (c, reason))
+      if (belongs (c, "default") && exclude (c, &reason))
       {
         s << TABLE(CLASS="proplist build")
           <<   TBODY