Add support for interactive CI mode

10 files changed, 256 insertions, 78 deletions

diff --git a/mod/buildfile b/mod/buildfile
index 191d966..ff9cd60 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -50,7 +50,7 @@ if $cli.configured
   cli.options += --std c++11 -I $src_root --include-with-brackets    \
 --include-prefix mod --guard-prefix MOD --generate-specifier         \
 --cxx-prologue "#include <mod/types-parsers.hxx>"                    \
---cli-namespace brep::cli --generate-file-scanner --option-length 41 \
+--cli-namespace brep::cli --generate-file-scanner --option-length 45 \
 --generate-modifier --generate-description --option-prefix ""
 
   # Include the generated cli files into the distribution and don't remove
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 8a684fe..bec362a 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -398,6 +398,10 @@ handle (request& rq, response&)
         b->status = rqm.result.status;
         b->force  = force_state::unforced;
 
+        // Cleanup the interactive build login information.
+        //
+        b->interactive = nullopt;
+
         // Cleanup the authentication data.
         //
         b->agent_fingerprint = nullopt;
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 04b2a36..ebf434a 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -4,12 +4,14 @@
 #include <mod/mod-build-task.hxx>
 
 #include <map>
+#include <regex>
 #include <chrono>
 
 #include <odb/database.hxx>
 #include <odb/transaction.hxx>
 #include <odb/schema-catalog.hxx>
 
+#include <libbutl/regex.mxx>
 #include <libbutl/sha256.mxx>
 #include <libbutl/utility.mxx>             // compare_c_string
 #include <libbutl/openssl.mxx>
@@ -195,11 +197,12 @@ handle (request& rq, response& rs)
   {
     vector<shared_ptr<build>> rebuilds;
 
-    // Create the task response manifest. The package must have the internal
-    // repository loaded.
+    // Create the task response manifest. Must be called inside the build db
+    // transaction.
     //
     auto task = [this] (shared_ptr<build>&& b,
                         shared_ptr<build_package>&& p,
+                        shared_ptr<build_tenant>&& t,
                         const config_machine& cm) -> task_response_manifest
     {
       uint64_t ts (
@@ -218,7 +221,11 @@ handle (request& rq, response& rs)
                          tenant_dir (options_->root (), b->tenant).string () +
                          "?build-result");
 
-      lazy_shared_ptr<build_repository> r (p->internal_repository);
+      assert (transaction::has_current ());
+
+      assert (p->internal ()); // The package is expected to be buildable.
+
+      lazy_shared_ptr<build_repository> r (p->internal_repository.load ());
 
       strings fps;
       if (r->certificate_fingerprint)
@@ -265,7 +272,8 @@ handle (request& rq, response& rs)
                           cm.config->target,
                           cm.config->environment,
                           cm.config->args,
-                          cm.config->warning_regexes);
+                          cm.config->warning_regexes,
+                          move (t->interactive));
 
       return task_response_manifest (move (session),
                                      move (b->agent_challenge),
@@ -430,12 +438,66 @@ handle (request& rq, response& rs)
         pkg_query::build_repository::id.canonical_name.in_range (rp.begin (),
                                                                  rp.end ());
 
+    // Transform (in-place) the interactive login information into the actual
+    // login command, if specified in the manifest and the transformation
+    // regexes are specified in the configuration.
+    //
+    if (tqm.interactive_login &&
+        options_->build_interactive_login_specified ())
+    {
+      optional<string> lc;
+      string l (tqm.agent + ' ' + *tqm.interactive_login);
+
+      // Use the first matching regex for the transformation.
+      //
+      for (const pair<regex, string>& rf: options_->build_interactive_login ())
+      {
+        pair<string, bool> r (regex_replace_match (l, rf.first, rf.second));
+
+        if (r.second)
+        {
+          lc = move (r.first);
+          break;
+        }
+      }
+
+      if (!lc)
+        throw invalid_request (400, "unable to match login info '" + l + "'");
+
+      tqm.interactive_login = move (lc);
+    }
+
+    // If the interactive mode if false or true, then filter out the
+    // respective packages. Otherwise, order them so that packages from the
+    // interactive build tenants appear first.
+    //
+    interactive_mode imode (tqm.effective_interactive_mode ());
+
+    switch (imode)
+    {
+    case interactive_mode::false_:
+      {
+        pq = pq && pkg_query::build_tenant::interactive.is_null ();
+        break;
+      }
+    case interactive_mode::true_:
+      {
+        pq = pq && pkg_query::build_tenant::interactive.is_not_null ();
+        break;
+      }
+    case interactive_mode::both: break; // See below.
+    }
+
     // Specify the portion.
     //
     size_t offset (0);
 
-    pq += "ORDER BY" +
-      pkg_query::build_package::id.tenant + "," +
+    pq += "ORDER BY";
+
+    if (imode == interactive_mode::both)
+      pq += pkg_query::build_tenant::interactive + "NULLS LAST,";
+
+    pq += pkg_query::build_package::id.tenant + "," +
       pkg_query::build_package::id.name +
       order_by_version (pkg_query::build_package::id.version, false) +
       "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50";
@@ -512,8 +574,8 @@ handle (request& rq, response& rs)
         // configurations that remained can be built. We will take the first
         // one, if present.
         //
-        // Also save the built package configurations for which it's time to be
-        // rebuilt.
+        // Also save the built package configurations for which it's time to
+        // be rebuilt.
         //
         config_machines configs (cfg_machines); // Make a copy for this pkg.
         auto pkg_builds (bld_prep_query.execute ());
@@ -567,6 +629,16 @@ handle (request& rq, response& rs)
             shared_ptr<build> b (build_db_->find<build> (bid));
             optional<string> cl (challenge ());
 
+            shared_ptr<build_tenant> t (
+              build_db_->load<build_tenant> (bid.package.tenant));
+
+            // Move the interactive build login information into the build
+            // object, if the package to be built interactively.
+            //
+            optional<string> login (t->interactive
+                                    ? move (tqm.interactive_login)
+                                    : nullopt);
+
             // If build configuration doesn't exist then create the new one
             // and persist. Otherwise put it into the building state, refresh
             // the timestamp and update.
@@ -579,6 +651,7 @@ handle (request& rq, response& rs)
                                       move (bid.configuration),
                                       move (bid.toolchain_name),
                                       move (toolchain_version),
+                                      move (login),
                                       move (agent_fp),
                                       move (cl),
                                       mh.name,
@@ -606,6 +679,7 @@ handle (request& rq, response& rs)
                       b->results.empty ());
 
               b->state = build_state::building;
+              b->interactive = move (login);
 
               // Switch the force state not to reissue the task after the
               // forced rebuild timeout. Note that the result handler will
@@ -626,13 +700,7 @@ handle (request& rq, response& rs)
 
             // Finally, prepare the task response manifest.
             //
-            // We iterate over buildable packages.
-            //
-            assert (p->internal ());
-
-            p->internal_repository.load ();
-
-            tsm = task (move (b), move (p), cm);
+            tsm = task (move (b), move (p), move (t), cm);
           }
         }
 
@@ -704,20 +772,40 @@ handle (request& rq, response& rs)
             assert (i != cfg_machines.end ());
             const config_machine& cm (i->second);
 
-            // Rebuild the package if still present, is buildable and doesn't
-            // exclude the configuration.
+            // Rebuild the package if still present, is buildable, doesn't
+            // exclude the configuration, and matches the request's
+            // interactive mode.
+            //
+            // Note that while change of the latter seems rather far fetched,
+            // let's check it for good measure.
             //
             shared_ptr<build_package> p (
               build_db_->find<build_package> (b->id.package));
 
-            if (p != nullptr   &&
-                p->internal () &&
+            shared_ptr<build_tenant> t (
+              p != nullptr
+              ? build_db_->load<build_tenant> (p->id.tenant)
+              : nullptr);
+
+            if (p != nullptr                          &&
+                p->buildable                          &&
+                (t->interactive.has_value () ==
+                 (imode != interactive_mode::false_)) &&
                 !exclude (p->builds, p->constraints, *cm.config))
             {
               assert (b->status);
 
               b->state = build_state::building;
 
+              // Save the interactive build login information into the build
+              // object, if the package to be built interactively.
+              //
+              // Can't move from, as may need it on the next iteration.
+              //
+              b->interactive = t->interactive
+                               ? tqm.interactive_login
+                               : nullopt;
+
               // Can't move from, as may need them on the next iteration.
               //
               b->agent_fingerprint = agent_fp;
@@ -738,9 +826,7 @@ handle (request& rq, response& rs)
 
               build_db_->update (b);
 
-              p->internal_repository.load ();
-
-              tsm = task (move (b), move (p), cm);
+              tsm = task (move (b), move (p), move (t), cm);
             }
           }
 
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index ab9e93e..5ffe6dd 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -133,6 +133,8 @@ match (const C qc, const string& pattern)
   return qc + "SIMILAR TO" + query<T>::_val (transform (pattern));
 }
 
+// If tenant is absent, then query builds from all the public tenants.
+//
 template <typename T>
 static inline query<T>
 build_query (const brep::cstrings* configs,
@@ -143,18 +145,18 @@ build_query (const brep::cstrings* configs,
   using namespace brep;
   using query = query<T>;
   using qb = typename query::build;
-
-  query q (configs != nullptr
-           ? qb::id.configuration.in_range (configs->begin (), configs->end ())
-           : query (true));
+  using qt = typename query::build_tenant;
 
   const auto& pid (qb::id.package);
 
-  if (tenant)
-    q = q && pid.tenant == *tenant;
+  query q (tenant ? pid.tenant == *tenant : !qt::private_);
 
   if (archived)
-    q = q && query::build_tenant::archived == *archived;
+    q = q && qt::archived == *archived;
+
+  if (configs != nullptr)
+    q = q && qb::id.configuration.in_range (configs->begin (),
+                                            configs->end ());
 
   // Note that there is no error reported if the filter parameters parsing
   // fails. Instead, it is considered that no package builds match such a
@@ -267,6 +269,8 @@ build_query (const brep::cstrings* configs,
   return q;
 }
 
+// If tenant is absent, then query packages from all the public tenants.
+//
 template <typename T>
 static inline query<T>
 package_query (const brep::params::builds& params,
@@ -276,14 +280,12 @@ package_query (const brep::params::builds& params,
   using namespace brep;
   using query = query<T>;
   using qp = typename query::build_package;
+  using qt = typename query::build_tenant;
 
-  query q (true);
-
-  if (tenant)
-    q = q && qp::id.tenant == *tenant;
+  query q (tenant ? qp::id.tenant == *tenant : !qt::private_);
 
   if (archived)
-    q = q && query::build_tenant::archived == *archived;
+    q = q && qt::archived == *archived;
 
   // Note that there is no error reported if the filter parameters parsing
   // fails. Instead, it is considered that no packages match such a query.
@@ -383,7 +385,7 @@ handle (request& rq, response& rs)
     <<     DIV(ID="content");
 
   // If the tenant is empty then we are in the global view and will display
-  // builds from all the tenants.
+  // builds from all the public tenants.
   //
   optional<string> tn;
   if (!tenant.empty ())
@@ -672,14 +674,18 @@ handle (request& rq, response& rs)
         <<     TR_VALUE ("config", b.configuration)
         <<     TR_VALUE ("machine", b.machine)
         <<     TR_VALUE ("target", b.target.string ())
-        <<     TR_VALUE ("timestamp", ts)
-        <<     TR_BUILD_RESULT (b, host, root);
+        <<     TR_VALUE ("timestamp", ts);
+
+      if (b.interactive) // Note: can only be present for the building state.
+        s <<   TR_VALUE ("login", *b.interactive);
+
+      s <<     TR_BUILD_RESULT (b, host, root);
 
       // In the global view mode add the tenant builds link. Note that the
       // global view (and the link) makes sense only in the multi-tenant mode.
       //
       if (!tn && !b.tenant.empty ())
-        s << TR_TENANT (tenant_name, "builds", root, b.tenant);
+        s <<   TR_TENANT (tenant_name, "builds", root, b.tenant);
 
       s <<   ~TBODY
         << ~TABLE;
@@ -809,15 +815,20 @@ handle (request& rq, response& rs)
 
         const auto& bid (bld_query::build::id);
 
-        bld_query bq (equal<package_build_count> (bid.package, id)  &&
-                      bid.configuration == bld_query::_ref (config) &&
+        bld_query bq (
+          equal<package_build_count> (bid.package, id)  &&
+          bid.configuration == bld_query::_ref (config) &&
 
           // Note that the query already constrains configurations via the
-          // configuration name and the tenant via the build package id.
+          // configuration name.
+          //
+          // Also note that while the query already constrains the tenant via
+          // the build package id, we still need to pass the tenant not to
+          // erroneously filter out the private tenants.
           //
           build_query<package_build_count> (nullptr /* configs */,
                                             bld_params,
-                                            nullopt /* tenant */,
+                                            tn,
                                             false /* archived */));
 
         prep_bld_query bld_prep_query (
@@ -925,11 +936,12 @@ handle (request& rq, response& rs)
     bld_query bq (
       equal<package_build> (bld_query::build::id.package, id) &&
 
-      // Note that the query already constrains the tenant via the build
-      // package id.
+      // Note that while the query already constrains the tenant via the build
+      // package id, we still need to pass the tenant not to erroneously
+      // filter out the private tenants.
       //
       build_query<package_build> (
-        &conf_names, bld_params, nullopt /* tenant */, false /* archived */));
+        &conf_names, bld_params, tn, false /* archived */));
 
     prep_bld_query bld_prep_query (
       conn->prepare_query<package_build> ("mod-builds-build-query", bq));
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index d2da93f..4da72b6 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -376,15 +376,18 @@ handle (request& rq, response& rs)
           s.next ("package", p);
       }
 
+      if (params.interactive_specified ())
+        s.next ("interactive", params.interactive ());
+
+      if (!simulate.empty ())
+        s.next ("simulate", simulate);
+
       s.next ("timestamp",
               butl::to_string (ts,
                                "%Y-%m-%dT%H:%M:%SZ",
                                false /* special */,
                                false /* local */));
 
-      if (!simulate.empty ())
-        s.next ("simulate", simulate);
-
       // Serialize the User-Agent HTTP header and the client IP address.
       //
       optional<string> ip;
@@ -412,10 +415,11 @@ handle (request& rq, response& rs)
       {
         const string& n (nv.name);
 
-        if (n != "repository" &&
-            n != "_"          &&
-            n != "package"    &&
-            n != "overrides"  &&
+        if (n != "repository"  &&
+            n != "_"           &&
+            n != "package"     &&
+            n != "overrides"   &&
+            n != "interactive" &&
             n != "simulate")
           s.next (n, nv.value ? *nv.value : "");
       }
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index a7682ec..a23989e 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -472,7 +472,7 @@ handle (request& rq, response& rs)
     }
   }
 
-  bool archived (package_db_->load<brep::tenant> (tenant)->archived);
+  shared_ptr<brep::tenant> tn (package_db_->load<brep::tenant> (tenant));
 
   t.commit ();
 
@@ -504,7 +504,7 @@ handle (request& rq, response& rs)
     //
     vector<pair<string, version>> toolchains;
 
-    if (!archived)
+    if (!tn->archived)
     {
       using query = query<toolchain>;
 
@@ -568,8 +568,12 @@ handle (request& rq, response& rs)
                          b.toolchain_version.string ())
         <<     TR_VALUE ("config",
                          b.configuration + " / " + b.target.string ())
-        <<     TR_VALUE ("timestamp", ts)
-        <<     TR_BUILD_RESULT (b, host, root)
+        <<     TR_VALUE ("timestamp", ts);
+
+      if (b.interactive) // Note: can only be present for the building state.
+        s <<   TR_VALUE ("login", *b.interactive);
+
+      s <<     TR_BUILD_RESULT (b, host, root)
         <<   ~TBODY
         << ~TABLE;
 
@@ -606,22 +610,27 @@ handle (request& rq, response& rs)
         << ~TABLE;
     }
 
-    // Print the package build exclusions that belong to the 'default' class.
+    // Print the package build exclusions that belong to the 'default' class,
+    // unless the package is built interactively (normally for a single
+    // configuration).
     //
-    for (const auto& c: *build_conf_)
+    if (!tn->interactive)
     {
-      string reason;
-      if (belongs (c, "default") && exclude (c, &reason))
+      for (const auto& c: *build_conf_)
       {
-        s << TABLE(CLASS="proplist build")
-          <<   TBODY
-          <<     TR_VALUE ("config", c.name + " / " + c.target.string ())
-          <<     TR_VALUE ("result",
-                           !reason.empty ()
-                           ? "excluded (" + reason + ')'
-                           : "excluded")
-          <<   ~TBODY
-          << ~TABLE;
+        string reason;
+        if (belongs (c, "default") && exclude (c, &reason))
+        {
+          s << TABLE(CLASS="proplist build")
+            <<   TBODY
+            <<     TR_VALUE ("config", c.name + " / " + c.target.string ())
+            <<     TR_VALUE ("result",
+                             !reason.empty ()
+                             ? "excluded (" + reason + ')'
+                             : "excluded")
+            <<   ~TBODY
+            << ~TABLE;
+        }
       }
     }
 
diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx
index 65c7c5b..222b817 100644
--- a/mod/mod-packages.cxx
+++ b/mod/mod-packages.cxx
@@ -49,8 +49,8 @@ init (scanner& s)
     options_->root (dir_path ("/"));
 
   // Check that the database 'package' schema matches the current one. It's
-  // enough to perform the check in just a single module implementation (and we
-  // don't do in the dispatcher because it doesn't use the database).
+  // enough to perform the check in just a single module implementation (and
+  // we don't do in the dispatcher because it doesn't use the database).
   //
   // Note that the failure can be reported by each web server worker process.
   // While it could be tempting to move the check to the
@@ -137,7 +137,7 @@ handle (request& rq, response& rs)
     <<     DIV(ID="content");
 
   // If the tenant is empty then we are in the global view and will display
-  // packages from all the tenants.
+  // packages from all the public tenants.
   //
   optional<string> tn;
   if (!tenant.empty ())
diff --git a/mod/module.cli b/mod/module.cli
index b59158a..3ba6ecd 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -1,6 +1,8 @@
 // file      : mod/options.cli -*- C++ -*-
 // license   : MIT; see accompanying LICENSE file
 
+include <regex>;
+
 include <libbpkg/manifest.hxx>; // repository_location
 
 include <web/xhtml/fragment.hxx>;
@@ -307,7 +309,7 @@ namespace brep
          edge. The value is treated as an XHTML5 fragment."
       }
 
-      vector<page_menu> menu;
+      vector<page_menu> menu
       {
         "<label=link>",
         "Web page menu. Each entry is displayed in the page header in the
@@ -341,7 +343,7 @@ namespace brep
          The default is 500 (~ 80 characters * 6 lines)."
       }
 
-      uint16_t package-changes = 5000;
+      uint16_t package-changes = 5000
       {
         "<len>",
         "Number of package changes characters to display in brief pages. The
@@ -394,6 +396,18 @@ namespace brep
         "Time to wait before considering the expected task result lost. Must be
          specified in seconds. The default is 3 hours."
       }
+
+      vector<pair<std::regex, string>> build-interactive-login
+      {
+        "</regex/replacement/>",
+        "Regular expressions for transforming the interactive build login
+         information, for example, into the actual command that can be used
+         by the user. The regular expressions are matched against the
+         \"<agent>\ <interactive-login>\" string containing the respective
+         task request manifest values. The first matching expression is used
+         for the transformation. If no expression matches, then the task
+         request is considered invalid, unless no expressions are specified."
+      }
     };
 
     class build_result: build, package_db, build_db, handler
@@ -837,6 +851,10 @@ namespace brep
       //
       string overrides;
 
+      // Interactive build execution breakpoint.
+      //
+      string interactive;
+
       // Submission simulation outcome.
       //
       string simulate;
diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx
index dc21e97..422b353 100644
--- a/mod/types-parsers.cxx
+++ b/mod/types-parsers.cxx
@@ -3,11 +3,15 @@
 
 #include <mod/types-parsers.hxx>
 
+#include <sstream>
+
+#include <libbutl/regex.mxx>
 #include <libbutl/timestamp.mxx> // from_string()
 
 #include <mod/module-options.hxx>
 
 using namespace std;
+using namespace butl;
 using namespace bpkg;
 using namespace web::xhtml;
 
@@ -75,9 +79,9 @@ namespace brep
         string t ("1970-01-01 ");
         t += v;
 
-        x = butl::from_string (t.c_str (),
-                               "%Y-%m-%d %H:%M",
-                               false /* local */).time_since_epoch ();
+        x = from_string (t.c_str (),
+                         "%Y-%m-%d %H:%M",
+                         false /* local */).time_since_epoch ();
         return;
       }
       catch (const invalid_argument&) {}
@@ -181,5 +185,37 @@ namespace brep
         throw invalid_value (o, v);
       }
     }
+
+    // Parse the '/regex/replacement/' string into the regex/replacement pair.
+    //
+    void parser<pair<std::regex, string>>::
+    parse (pair<std::regex, string>& x, bool& xs, scanner& s)
+    {
+      xs = true;
+      const char* o (s.next ());
+
+      if (!s.more ())
+        throw missing_value (o);
+
+      const char* v (s.next ());
+
+      try
+      {
+        x = regex_replace_parse (v);
+      }
+      catch (const invalid_argument& e)
+      {
+        throw invalid_value (o, v, e.what ());
+      }
+      catch (const regex_error& e)
+      {
+        // Sanitize the description.
+        //
+        ostringstream os;
+        os << e;
+
+        throw invalid_value (o, v, os.str ());
+      }
+    }
   }
 }
diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx
index 6b851eb..05a7263 100644
--- a/mod/types-parsers.hxx
+++ b/mod/types-parsers.hxx
@@ -7,6 +7,8 @@
 #ifndef MOD_TYPES_PARSERS_HXX
 #define MOD_TYPES_PARSERS_HXX
 
+#include <regex>
+
 #include <libbpkg/manifest.hxx> // repository_location
 
 #include <web/xhtml/fragment.hxx>
@@ -75,6 +77,13 @@ namespace brep
       static void
       parse (web::xhtml::fragment&, bool&, scanner&);
     };
+
+    template <>
+    struct parser<pair<std::regex, string>>
+    {
+      static void
+      parse (pair<std::regex, string>&, bool&, scanner&);
+    };
   }
 }