Add support for build include/exclude manifest values

5 files changed, 342 insertions, 118 deletions

diff --git a/mod/build-config.cxx b/mod/build-config.cxx
index 6b59e54..fed2ee9 100644
--- a/mod/build-config.cxx
+++ b/mod/build-config.cxx
@@ -146,4 +146,31 @@ namespace brep
       "&cf=" + mime_url_encode (b.configuration) +
       "&tc=" + b.toolchain_version.string () + "&reason=";
   }
+
+  bool
+  match (const string& config_pattern, const optional<string>& target_pattern,
+         const build_config& c)
+  {
+    return path_match (config_pattern, c.name) &&
+      (!target_pattern ||
+       (c.target && path_match (*target_pattern, c.target->string ())));
+  }
+
+  bool
+  exclude (const build_package& p, const build_config& c)
+  {
+    for (const auto& bc: p.constraints)
+    {
+      if (!bc.exclusion && match (bc.config, bc.target, c))
+        return false;
+    }
+
+    for (const auto& bc: p.constraints)
+    {
+      if (bc.exclusion && match (bc.config, bc.target, c))
+        return true;
+    }
+
+    return false;
+  }
 }
diff --git a/mod/build-config.hxx b/mod/build-config.hxx
index b49819d..3c89ceb 100644
--- a/mod/build-config.hxx
+++ b/mod/build-config.hxx
@@ -13,9 +13,12 @@
 #include <libbrep/utility.hxx>
 
 #include <libbrep/build.hxx>
+#include <libbrep/build-package.hxx>
 
 #include <mod/options.hxx>
 
+// Various build-related state and utilities.
+//
 namespace brep
 {
   // Return pointer to the shared build configurations instance, creating one
@@ -48,6 +51,18 @@ namespace brep
   //
   string
   force_rebuild_url (const string& host, const dir_path& root, const build&);
+
+  // Match a build configuration against the name and target patterns.
+  //
+  bool
+  match (const string& config_pattern,
+         const optional<string>& target_pattern,
+         const bbot::build_config&);
+
+  // Return true if a package excludes the specified build configuration.
+  //
+  bool
+  exclude (const build_package&, const bbot::build_config&);
 }
 
 #endif // MOD_BUILD_CONFIG_HXX
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 1a7b272..771b1ee 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -442,80 +442,94 @@ handle (request& rq, response& rs)
 
         if (!configs.empty ())
         {
-          config_machine& cm (configs.begin ()->second);
-          machine_header_manifest& mh (*cm.machine);
-          build_id bid (move (id), cm.config->name, toolchain_version);
-          shared_ptr<build> b (build_db_->find<build> (bid));
-          optional<string> cl (challenge ());
-
-          // If build configuration doesn't exist then create the new one and
-          // persist. Otherwise put it into the building state, refresh the
-          // timestamp and update.
+          // Find the first build configuration that is not excluded by the
+          // package.
           //
-          if (b == nullptr)
-          {
-            b = make_shared<build> (move (bid.package.name),
-                                    move (bp.version),
-                                    move (bid.configuration),
-                                    move (tqm.toolchain_name),
-                                    move (toolchain_version),
-                                    move (agent_fp),
-                                    move (cl),
-                                    mh.name,
-                                    move (mh.summary),
-                                    cm.config->target);
-
-            build_db_->persist (b);
-          }
-          else
+          shared_ptr<build_package> p (build_db_->load<build_package> (id));
+
+          auto i (configs.begin ());
+          auto e (configs.end ());
+          for (; i != e && exclude (*p, *i->second.config); ++i) ;
+
+          if (i != e)
           {
-            // The package configuration is in the building state, and there
-            // are no results.
+            config_machine& cm (i->second);
+            machine_header_manifest& mh (*cm.machine);
+            build_id bid (move (id), cm.config->name, toolchain_version);
+            shared_ptr<build> b (build_db_->find<build> (bid));
+            optional<string> cl (challenge ());
+
+            // If build configuration doesn't exist then create the new one and
+            // persist. Otherwise put it into the building state, refresh the
+            // timestamp and update.
             //
-            // Note that in both cases we keep the status intact to be able to
-            // compare it with the final one in the result request handling in
-            // order to decide if to send the notification email. The same is
-            // true for the forced flag (in the sense that we don't set the
-            // force state to unforced).
-            //
-            // Load the section to assert the above statement.
-            //
-            build_db_->load (*b, b->results_section);
+            if (b == nullptr)
+            {
+              b = make_shared<build> (move (bid.package.name),
+                                      move (bp.version),
+                                      move (bid.configuration),
+                                      move (tqm.toolchain_name),
+                                      move (toolchain_version),
+                                      move (agent_fp),
+                                      move (cl),
+                                      mh.name,
+                                      move (mh.summary),
+                                      cm.config->target);
+
+              build_db_->persist (b);
+            }
+            else
+            {
+              // The package configuration is in the building state, and there
+              // are no results.
+              //
+              // Note that in both cases we keep the status intact to be able
+              // to compare it with the final one in the result request
+              // handling in order to decide if to send the notification email.
+              // The same is true for the forced flag (in the sense that we
+              // don't set the force state to unforced).
+              //
+              // Load the section to assert the above statement.
+              //
+              build_db_->load (*b, b->results_section);
 
-            assert (b->state == build_state::building && b->results.empty ());
+              assert (b->state == build_state::building &&
+                      b->results.empty ());
 
-            b->state = build_state::building;
+              b->state = build_state::building;
 
-            // Switch the force state not to reissue the task after the forced
-            // rebuild timeout. Note that the result handler will still
-            // recognize that the rebuild was forced.
-            //
-            if (b->force == force_state::forcing)
-              b->force = force_state::forced;
-
-            b->toolchain_name = move (tqm.toolchain_name);
-            b->agent_fingerprint = move (agent_fp);
-            b->agent_challenge = move (cl);
-            b->machine = mh.name;
-            b->machine_summary = move (mh.summary);
-            b->target = cm.config->target;
-            b->timestamp = timestamp::clock::now ();
-
-            build_db_->update (b);
-          }
+              // Switch the force state not to reissue the task after the
+              // forced rebuild timeout. Note that the result handler will
+              // still recognize that the rebuild was forced.
+              //
+              if (b->force == force_state::forcing)
+                b->force = force_state::forced;
 
-          // Finally, prepare the task response manifest.
-          //
-          shared_ptr<build_package> p (
-            build_db_->load<build_package> (b->id.package));
+              b->toolchain_name = move (tqm.toolchain_name);
+              b->agent_fingerprint = move (agent_fp);
+              b->agent_challenge = move (cl);
+              b->machine = mh.name;
+              b->machine_summary = move (mh.summary);
+              b->target = cm.config->target;
+              b->timestamp = timestamp::clock::now ();
 
-          p->internal_repository.load ();
+              build_db_->update (b);
+            }
 
-          tsm = task (move (b), move (p), cm);
+            // Finally, prepare the task response manifest.
+            //
+            // We iterate over buildable packages.
+            //
+            assert (p->internal_repository != nullptr);
+
+            p->internal_repository.load ();
+
+            tsm = task (move (b), move (p), cm);
+          }
         }
 
         // If the task response manifest is prepared, then bail out from the
-        // package loop, commit transactions and respond.
+        // package loop, commit the transaction and respond.
         //
         if (!tsm.session.empty ())
           break;
@@ -556,10 +570,10 @@ handle (request& rq, response& rs)
 
       // Pick the first package configuration from the ordered list.
       //
-      // Note that the configurations may not match the required criteria
-      // anymore (as we have committed the database transactions that were
-      // used to collect this data) so we recheck. If we find one that matches
-      // then put it into the building state, refresh the timestamp and
+      // Note that the configurations and packages may not match the required
+      // criteria anymore (as we have committed the database transactions that
+      // were used to collect this data) so we recheck. If we find one that
+      // matches then put it into the building state, refresh the timestamp and
       // update. Note that we don't amend the status and the force state to
       // have them available in the result request handling (see above).
       //
@@ -582,25 +596,28 @@ handle (request& rq, response& rs)
             //
             assert (i != cfg_machines.end ());
             const config_machine& cm (i->second);
-            const machine_header_manifest& mh (*cm.machine);
 
-            // Load the package (if still present).
+            // Rebuild the package if still present, is buildable and doesn't
+            // exclude the configuration.
             //
             shared_ptr<build_package> p (
               build_db_->find<build_package> (b->id.package));
 
-            if (p != nullptr)
+            if (p != nullptr && p->internal_repository != nullptr &&
+                !exclude (*p, *cm.config))
             {
               assert (b->status);
 
               b->state = build_state::building;
-              b->machine = mh.name;
 
               // Can't move from, as may need them on the next iteration.
               //
               b->agent_fingerprint = agent_fp;
               b->agent_challenge = cl;
               b->toolchain_name = tqm.toolchain_name;
+
+              const machine_header_manifest& mh (*cm.machine);
+              b->machine = mh.name;
               b->machine_summary = mh.summary;
 
               b->target = cm.config->target;
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index 439bd05..c7a308e 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -98,7 +98,9 @@ build_query (const brep::cstrings& configs, const brep::params::builds& params)
   using query = query<T>;
   using qb = typename query::build;
 
-  query q (qb::id.configuration.in_range (configs.begin (), configs.end ()));
+  query q (!configs.empty ()
+           ? qb::id.configuration.in_range (configs.begin (), configs.end ())
+           : query (true));
 
   // Note that there is no error reported if the filter parameters parsing
   // fails. Instead, it is considered that no package builds match such a
@@ -199,13 +201,12 @@ build_query (const brep::cstrings& configs, const brep::params::builds& params)
   return q;
 }
 
-template <typename T>
+template <typename T, typename P = typename query<T>::build_package>
 static inline query<T>
 package_query (const brep::params::builds& params)
 {
   using namespace brep;
   using query = query<T>;
-  using qp = typename query::build_package;
 
   query q (true);
 
@@ -217,12 +218,12 @@ package_query (const brep::params::builds& params)
     // Package name.
     //
     if (!params.name ().empty ())
-      q = q && qp::id.name.like (transform (params.name ()));
+      q = q && P::id.name.like (transform (params.name ()));
 
     // Package version.
     //
     if (!params.version ().empty () && params.version () != "*")
-      q = q && compare_version_eq (qp::id.version,
+      q = q && compare_version_eq (P::id.version,
                                    version (params.version ()), // May throw.
                                    true);
   }
@@ -234,6 +235,20 @@ package_query (const brep::params::builds& params)
   return q;
 }
 
+template <typename T, typename ID>
+static inline query<T>
+package_id_eq (const ID& x, const brep::package_id& y)
+{
+  using query = query<T>;
+  const auto& qv (x.version);
+
+  return x.name == query::_ref (y.name) &&
+    qv.epoch == query::_ref (y.version.epoch) &&
+    qv.canonical_upstream == query::_ref (y.version.canonical_upstream) &&
+    qv.canonical_release == query::_ref (y.version.canonical_release) &&
+    qv.revision == query::_ref (y.version.revision);
+}
+
 static const vector<pair<string, string>> build_results ({
     {"unbuilt", "<unbuilt>"},
     {"*", "*"},
@@ -283,7 +298,7 @@ handle (request& rq, response& rs)
     //
     // This hack is required to avoid the "flash of unstyled content", which
     // happens due to the presence of the autofocus attribute in the input
-    // element of the search form. The problem appears in Firefox and has a
+    // element of the filter form. The problem appears in Firefox and has a
     // (4-year old, at the time of this writing) bug report:
     //
     // https://bugzilla.mozilla.org/show_bug.cgi?id=712130.
@@ -491,17 +506,15 @@ handle (request& rq, response& rs)
     };
 
     // Note that config_toolchains contains shallow references to the toolchain
-    // names and versions, and in particular to the selected ones (tc_name and
-    // tc_version).
+    // names and versions.
     //
-    string tc_name;
-    version tc_version;
     set<config_toolchain> config_toolchains;
-
     {
       transaction t (build_db_->begin ());
       toolchains = query_toolchains ();
 
+      string tc_name;
+      version tc_version;
       const string& tc (params.toolchain ());
 
       if (tc != "*")
@@ -527,6 +540,7 @@ handle (request& rq, response& rs)
 
       const string& pc (params.configuration ());
       const string& tg (params.target ());
+      vector<const build_config*> configs;
 
       for (const auto& c: *build_conf_)
       {
@@ -537,11 +551,13 @@ handle (request& rq, response& rs)
              : tg == "*" ||
                (c.target && path_match (tg, c.target->string ()))))
         {
-          if (tc != "*") // Filter by toolchain.
-            config_toolchains.insert ({c.name, tc_name, tc_version});
-          else           // Add all toolchains.
+          configs.push_back (&c);
+
+          for (const auto& t: toolchains)
           {
-            for (const auto& t: toolchains)
+            // Filter by toolchain.
+            //
+            if (tc == "*" || (t.first == tc_name && t.second == tc_version))
               config_toolchains.insert ({c.name, t.first, t.second});
           }
         }
@@ -551,6 +567,15 @@ handle (request& rq, response& rs)
       // difference between the maximum possible number of unbuilt
       // configurations and the number of existing package builds.
       //
+      // Note that we also need to deduct the package-excluded configurations
+      // count from the maximum possible number of unbuilt configurations. The
+      // only way to achieve this is to traverse through the build-constrained
+      // packages and match their constraints against our configurations.
+      //
+      // Also note that some existing builds can now be excluded by packages
+      // due to the build configuration target change. We should deduct such
+      // builds count from the number of existing package builds.
+      //
       size_t nmax (config_toolchains.size () *
                    build_db_->query_value<buildable_package_count> (
                      package_query<buildable_package_count> (params)));
@@ -558,6 +583,63 @@ handle (request& rq, response& rs)
       size_t ncur = build_db_->query_value<package_build_count> (
         build_query<package_build_count> (*build_conf_names_, bld_params));
 
+      // From now we will be using specific package name and version for each
+      // build database query.
+      //
+      bld_params.name ().clear ();
+      bld_params.version ().clear ();
+
+      if (!config_toolchains.empty ())
+      {
+        // Prepare the build count prepared query.
+        //
+        // For each package-excluded configuration we will query the number of
+        // existing builds.
+        //
+        using bld_query = query<package_build_count>;
+        using prep_bld_query = prepared_query<package_build_count>;
+
+        package_id id;
+        string config;
+
+        bld_query bq (
+          package_id_eq<package_build_count> (
+            bld_query::build::id.package, id) &&
+          bld_query::build::id.configuration == bld_query::_ref (config) &&
+          build_query<package_build_count> (cstrings (), bld_params));
+
+        prep_bld_query bld_prep_query (
+          build_db_->prepare_query<package_build_count> (
+            "mod-builds-build-count-query", bq));
+
+        size_t nt (tc == "*" ? toolchains.size () : 1);
+
+        // The number of build-constrained packages can potentially be large,
+        // and we may implement some caching in the future. However, the
+        // caching will not be easy as the cached values depend on the filter
+        // form parameters.
+        //
+        using query = query<build_constrained_package>;
+        query q (package_query<build_constrained_package, query> (params));
+
+        for (const auto& p: build_db_->query<build_constrained_package> (q))
+        {
+          const build_package& bp (*p.package);
+          id = bp.id;
+
+          for (const auto& c: configs)
+          {
+            if (exclude (bp, *c))
+            {
+              nmax -= nt;
+
+              config = c->name;
+              ncur -= bld_prep_query.execute_value ();
+            }
+          }
+        }
+      }
+
       assert (nmax >= ncur);
       count = nmax - ncur;
 
@@ -620,36 +702,37 @@ handle (request& rq, response& rs)
     using bld_query = query<package_build>;
     using prep_bld_query = prepared_query<package_build>;
 
-    // We will use specific package name and version for each database query.
-    //
-    bld_params.name ().clear ();
-    bld_params.version ().clear ();
-
     package_id id;
-    const auto& qv (bld_query::build::id.package.version);
 
     bld_query bq (
-      bld_query::build::id.package.name == bld_query::_ref (id.name) &&
-
-      qv.epoch == bld_query::_ref (id.version.epoch) &&
-      qv.canonical_upstream ==
-      bld_query::_ref (id.version.canonical_upstream) &&
-      qv.canonical_release ==
-      bld_query::_ref (id.version.canonical_release) &&
-      qv.revision == bld_query::_ref (id.version.revision) &&
-
+      package_id_eq<package_build> (bld_query::build::id.package, id) &&
       build_query<package_build> (*build_conf_names_, bld_params));
 
     prep_bld_query bld_prep_query (
       conn->prepare_query<package_build> ("mod-builds-build-query", bq));
 
+    // Prepare the build-constrained package prepared query.
+    //
+    // For each build-constrained package we will exclude the corresponding
+    // configurations from being printed.
+    //
+    using ctr_query = query<build_constrained_package>;
+    using prep_ctr_query = prepared_query<build_constrained_package>;
+
+    ctr_query cq (
+      package_id_eq<build_constrained_package> (ctr_query::id, id));
+
+    prep_ctr_query ctr_prep_query (
+      conn->prepare_query<build_constrained_package> (
+        "mod-builds-build-constrained-package-query", cq));
+
     size_t skip (page * page_configs);
     size_t print (page_configs);
 
     // Enclose the subsequent tables to be able to use nth-child CSS selector.
     //
     s << DIV;
-    for (bool prn (true); prn; )
+    while (print != 0)
     {
       transaction t (conn->begin ());
 
@@ -657,27 +740,47 @@ handle (request& rq, response& rs)
       //
       auto packages (pkg_prep_query.execute ());
 
-      if ((prn = !packages.empty ()))
+      if (packages.empty ())
+        print = 0;
+      else
       {
         offset += packages.size ();
 
         // Iterate over packages and print unbuilt configurations. Skip the
         // appropriate number of them first (for page number greater than one).
         //
-        for (auto& pv: packages)
+        for (auto& p: packages)
         {
-          id = move (pv.id);
+          id = move (p.id);
 
-          // Make a copy for this package.
+          // Copy configuration/toolchain combinations for this package,
+          // skipping explicitly excluded configurations.
           //
-          auto unbuilt_configs (config_toolchains);
+          set<config_toolchain> unbuilt_configs;
+          {
+            build_constrained_package p;
+            if (ctr_prep_query.execute_one (p))
+            {
+              const build_package& bp (*p.package);
+              for (const auto& ct: config_toolchains)
+              {
+                auto i (build_conf_map_->find (ct.configuration.c_str ()));
+                assert (i != build_conf_map_->end ());
+
+                if (!exclude (bp, *i->second))
+                  unbuilt_configs.insert (ct);
+              }
+            }
+            else
+              unbuilt_configs = config_toolchains;
+          }
 
           // Iterate through the package configuration builds and erase them
           // from the unbuilt configurations set.
           //
           for (const auto& pb: bld_prep_query.execute ())
           {
-            build& b (*pb.build);
+            const build& b (*pb.build);
 
             unbuilt_configs.erase ({
                 b.id.configuration, b.toolchain_name, b.toolchain_version});
@@ -693,12 +796,6 @@ handle (request& rq, response& rs)
               continue;
             }
 
-            if (print-- == 0)
-            {
-              prn = false;
-              break;
-            }
-
             auto i (build_conf_map_->find (ct.configuration.c_str ()));
             assert (i != build_conf_map_->end ());
 
@@ -707,7 +804,7 @@ handle (request& rq, response& rs)
             s << TABLE(CLASS="proplist build")
               <<   TBODY
               <<     TR_NAME (id.name, string (), root)
-              <<     TR_VERSION (id.name, pv.version, root)
+              <<     TR_VERSION (id.name, p.version, root)
               <<     TR_VALUE ("toolchain",
                                string (ct.toolchain_name) + '-' +
                                ct.toolchain_version.string ())
@@ -715,9 +812,12 @@ handle (request& rq, response& rs)
               <<     TR_VALUE ("target", tg ? tg->string () : "<default>")
               <<   ~TBODY
               << ~TABLE;
+
+            if (--print == 0) // Bail out the configuration loop.
+              break;
           }
 
-          if (!prn)
+          if (print == 0) // Bail out the package loop.
             break;
         }
       }
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index fd8d9a1..4da13ab 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -10,6 +10,8 @@
 #include <odb/database.hxx>
 #include <odb/transaction.hxx>
 
+#include <libbutl/utility.hxx> // alpha(), ucase(), lcase()
+
 #include <web/xhtml.hxx>
 #include <web/module.hxx>
 #include <web/mime-url-encoding.hxx>
@@ -23,6 +25,7 @@
 #include <mod/options.hxx>
 
 using namespace std;
+using namespace butl;
 using namespace odb::core;
 using namespace brep::cli;
 
@@ -280,6 +283,14 @@ handle (request& rq, response& rs)
       << ~TABLE;
   }
 
+  // Don't display the page builds section for stub packages.
+  //
+  bool builds (build_db_ != nullptr &&
+               ver.compare (wildcard_version, true) != 0);
+
+  if (builds)
+    package_db_->load (*pkg, pkg->build_section);
+
   t.commit ();
 
   const auto& rm (pkg->requirements);
@@ -325,9 +336,7 @@ handle (request& rq, response& rs)
       << ~TABLE;
   }
 
-  // Don't display the section for stub packages.
-  //
-  if (build_db_ != nullptr && ver.compare (wildcard_version, true) != 0)
+  if (builds)
   {
     s << H3 << "Builds" << ~H3
       << DIV(ID="builds");
@@ -335,6 +344,8 @@ handle (request& rq, response& rs)
     timestamp now (timestamp::clock::now ());
     transaction t (build_db_->begin ());
 
+    // Print built package configurations.
+    //
     using query = query<build>;
 
     for (auto& b: build_db_->query<build> (
@@ -368,6 +379,60 @@ handle (request& rq, response& rs)
         << ~TABLE;
     }
 
+    // Print configurations that are excluded by the package.
+    //
+    auto excluded = [&pkg] (const bbot::build_config& c, string& reason)
+    {
+      for (const auto& bc: pkg->build_constraints)
+      {
+        if (!bc.exclusion && match (bc.config, bc.target, c))
+          return false;
+      }
+
+      for (const auto& bc: pkg->build_constraints)
+      {
+        if (bc.exclusion && match (bc.config, bc.target, c))
+        {
+          // Save the first sentence of the exclusion comment, lower-case the
+          // first letter if the beginning looks like a word (the second
+          // character is the lower-case letter or space).
+          //
+          reason = bc.comment.substr (0, bc.comment.find ('.'));
+
+          char c;
+          size_t n (reason.size ());
+          if (n > 0 && alpha (c = reason[0]) && c == ucase (c) &&
+              (n == 1 ||
+               (alpha (c = reason[1]) && c == lcase (c)) ||
+               c == ' '))
+            reason[0] = lcase (reason[0]);
+
+          return true;
+        }
+      }
+
+      return false;
+    };
+
+    for (const auto& c: *build_conf_)
+    {
+      string reason;
+      if (excluded (c, reason))
+      {
+        s << TABLE(CLASS="proplist build")
+          <<   TBODY
+          <<     TR_VALUE ("config",
+                           c.name + " / " +
+                           (c.target ? c.target->string () : "<default>"))
+          <<     TR_VALUE ("result",
+                           !reason.empty ()
+                           ? "excluded (" + reason + ')'
+                           : "excluded")
+          <<   ~TBODY
+          << ~TABLE;
+      }
+    }
+
     t.commit ();
 
     s << ~DIV;