From d4900d85f7a5d791f89821713d02d3dd19361044 Mon Sep 17 00:00:00 2001
From: Karen Arutyunov <karen@codesynthesis.com>
Date: Thu, 22 Feb 2024 11:17:25 +0300
Subject: Add support for tenant-associated service notifications

---
 mod/buildfile                       |   5 +
 mod/ci-common.cxx                   | 494 ++++++++++++++++++++++++++++++
 mod/ci-common.hxx                   |  96 ++++++
 mod/database-module.cxx             |  56 ++++
 mod/database-module.hxx             |  23 +-
 mod/external-handler.cxx            |   3 +-
 mod/external-handler.hxx            |   2 +-
 mod/mod-build-force.cxx             |  78 ++++-
 mod/mod-build-force.hxx             |   8 +-
 mod/mod-build-log.cxx               |   2 +-
 mod/mod-build-result.cxx            | 159 +++++++++-
 mod/mod-build-result.hxx            |   8 +-
 mod/mod-build-task.cxx              | 395 ++++++++++++++++++++----
 mod/mod-build-task.hxx              |   8 +-
 mod/mod-builds.cxx                  |  10 +
 mod/mod-ci.cxx                      | 585 +++++++++++-------------------------
 mod/mod-ci.hxx                      |  49 ++-
 mod/mod-package-version-details.cxx |   2 +-
 mod/mod-repository-root.cxx         |  35 ++-
 mod/mod-repository-root.hxx         |   3 +
 mod/module.cli                      |  57 +++-
 mod/page.cxx                        |   1 -
 mod/page.hxx                        |   7 +-
 mod/tenant-service.hxx              | 107 +++++++
 24 files changed, 1673 insertions(+), 520 deletions(-)
 create mode 100644 mod/ci-common.cxx
 create mode 100644 mod/ci-common.hxx
 create mode 100644 mod/tenant-service.hxx

(limited to 'mod')

diff --git a/mod/buildfile b/mod/buildfile
index 6693a35..c3895dc 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -35,6 +35,11 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}}               \
                     {hxx ixx txx cxx}{+{$libu_src}  } \
                     $libs
 
+# Add support for tenant-associated service notifications to the CI module for
+# the debugging of the notifications machinery.
+#
+cxx.poptions += -DBREP_CI_TENANT_SERVICE
+
 libus{mod}: ../web/xhtml/libus{xhtml}
 libue{mod}: ../web/xhtml/libue{xhtml}
 
diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx
new file mode 100644
index 0000000..cb61e66
--- /dev/null
+++ b/mod/ci-common.cxx
@@ -0,0 +1,494 @@
+// file      : mod/ci-common.cxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#include <mod/ci-common.hxx>
+
+#include <libbutl/uuid.hxx>
+#include <libbutl/fdstream.hxx>
+#include <libbutl/sendmail.hxx>
+#include <libbutl/timestamp.hxx>
+#include <libbutl/filesystem.hxx>
+#include <libbutl/process-io.hxx>          // operator<<(ostream, process_args)
+#include <libbutl/manifest-serializer.hxx>
+
+#include <mod/external-handler.hxx>
+
+namespace brep
+{
+  using namespace std;
+  using namespace butl;
+
+  void ci_start::
+  init (shared_ptr<options::ci_start> o)
+  {
+    // Verify the data directory satisfies the requirements.
+    //
+    const dir_path& d (o->ci_data ());
+
+    if (d.relative ())
+      throw runtime_error ("ci-data directory path must be absolute");
+
+    if (!dir_exists (d))
+      throw runtime_error ("ci-data directory '" + d.string () +
+                           "' does not exist");
+
+    if (o->ci_handler_specified () && o->ci_handler ().relative ())
+      throw runtime_error ("ci-handler path must be absolute");
+
+    options_ = move (o);
+  }
+
+  optional<ci_start::start_result> ci_start::
+  start (const basic_mark& error,
+         const basic_mark& warn,
+         const basic_mark* trace,
+         optional<tenant_service>&& service,
+         const repository_location& repository,
+         const vector<package>& packages,
+         const optional<string>& client_ip,
+         const optional<string>& user_agent,
+         const optional<string>& interactive,
+         const optional<string>& simulate,
+         const vector<pair<string, string>>& custom_request,
+         const vector<pair<string, string>>& overrides)
+  {
+    using serializer    = manifest_serializer;
+    using serialization = manifest_serialization;
+
+    assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+    // If the tenant service is specified, then its type may not be empty.
+    //
+    assert (!service || !service->type.empty ());
+
+    // Generate the request id.
+    //
+    // Note that it will also be used as a CI result manifest reference,
+    // unless the latter is provided by the external handler.
+    //
+    string request_id;
+
+    try
+    {
+      request_id = uuid::generate ().string ();
+    }
+    catch (const system_error& e)
+    {
+      error << "unable to generate request id: " << e;
+      return nullopt;
+    }
+
+    // Create the submission data directory.
+    //
+    dir_path dd (options_->ci_data () / dir_path (request_id));
+
+    try
+    {
+      // It's highly unlikely but still possible that the directory already
+      // exists. This can only happen if the generated uuid is not unique.
+      //
+      if (try_mkdir (dd) == mkdir_status::already_exists)
+        throw_generic_error (EEXIST);
+    }
+    catch (const system_error& e)
+    {
+      error << "unable to create directory '" << dd << "': " << e;
+      return nullopt;
+    }
+
+    auto_rmdir ddr (dd);
+
+    // Return the start_result object for the client errors (normally the bad
+    // request status code (400) for the client data serialization errors).
+    //
+    auto client_error = [&request_id] (uint16_t status, string message)
+    {
+      return start_result {status,
+                           move (message),
+                           request_id,
+                           vector<pair<string, string>> ()};
+    };
+
+    // Serialize the CI request manifest to a stream. On the serialization
+    // error return false together with the start_result object containing the
+    // bad request (400) code and the error message. On the stream error pass
+    // through the io_error exception. Otherwise return true.
+    //
+    timestamp ts (system_clock::now ());
+
+    auto rqm = [&request_id,
+                &ts,
+                &service,
+                &repository,
+                &packages,
+                &client_ip,
+                &user_agent,
+                &interactive,
+                &simulate,
+                &custom_request,
+                &client_error] (ostream& os, bool long_lines = false)
+      -> pair<bool, optional<start_result>>
+    {
+      try
+      {
+        serializer s (os, "request", long_lines);
+
+        // Serialize the submission manifest header.
+        //
+        s.next ("", "1"); // Start of manifest.
+        s.next ("id", request_id);
+        s.next ("repository", repository.string ());
+
+        for (const package& p: packages)
+        {
+          if (!p.version)
+            s.next ("package", p.name.string ());
+          else
+            s.next ("package",
+                    p.name.string () + '/' + p.version->string ());
+        }
+
+        if (interactive)
+          s.next ("interactive", *interactive);
+
+        if (simulate)
+          s.next ("simulate", *simulate);
+
+        s.next ("timestamp",
+                butl::to_string (ts,
+                                 "%Y-%m-%dT%H:%M:%SZ",
+                                 false /* special */,
+                                 false /* local */));
+
+        if (client_ip)
+          s.next ("client-ip", *client_ip);
+
+        if (user_agent)
+          s.next ("user-agent", *user_agent);
+
+        if (service)
+        {
+          // Note that if the service id is not specified, then the handler
+          // will use the generated reference instead.
+          //
+          if (!service->id.empty ())
+            s.next ("service-id", service->id);
+
+          s.next ("service-type", service->type);
+
+          if (service->data)
+            s.next ("service-data", *service->data);
+        }
+
+        // Serialize the request custom parameters.
+        //
+        // Note that the serializer constraints the custom parameter names
+        // (can't start with '#', can't contain ':' and the whitespaces,
+        // etc).
+        //
+        for (const pair<string, string>& nv: custom_request)
+          s.next (nv.first, nv.second);
+
+        s.next ("", ""); // End of manifest.
+        return make_pair (true, optional<start_result> ());
+      }
+      catch (const serialization& e)
+      {
+        return make_pair (false,
+                          optional<start_result> (
+                            client_error (400,
+                                          string ("invalid parameter: ") +
+                                          e.what ())));
+      }
+    };
+
+    // Serialize the CI request manifest to the submission directory.
+    //
+    path rqf (dd / "request.manifest");
+
+    try
+    {
+      ofdstream os (rqf);
+      pair<bool, optional<start_result>> r (rqm (os));
+      os.close ();
+
+      if (!r.first)
+        return move (*r.second);
+    }
+    catch (const io_error& e)
+    {
+      error << "unable to write to '" << rqf << "': " << e;
+      return nullopt;
+    }
+
+    // Serialize the CI overrides manifest to a stream. On the serialization
+    // error return false together with the start_result object containing the
+    // bad request (400) code and the error message. On the stream error pass
+    // through the io_error exception. Otherwise return true.
+    //
+    auto ovm = [&overrides, &client_error] (ostream& os,
+                                            bool long_lines = false)
+      -> pair<bool, optional<start_result>>
+    {
+      try
+      {
+        serializer s (os, "overrides", long_lines);
+
+        s.next ("", "1"); // Start of manifest.
+
+        for (const pair<string, string>& nv: overrides)
+          s.next (nv.first, nv.second);
+
+        s.next ("", ""); // End of manifest.
+        return make_pair (true, optional<start_result> ());
+      }
+      catch (const serialization& e)
+      {
+        return make_pair (false,
+                          optional<start_result> (
+                            client_error (
+                              400,
+                              string ("invalid manifest override: ") +
+                              e.what ())));
+      }
+    };
+
+    // Serialize the CI overrides manifest to the submission directory.
+    //
+    path ovf (dd / "overrides.manifest");
+
+    if (!overrides.empty ())
+    try
+    {
+      ofdstream os (ovf);
+      pair<bool, optional<start_result>> r (ovm (os));
+      os.close ();
+
+      if (!r.first)
+        return move (*r.second);
+    }
+    catch (const io_error& e)
+    {
+      error << "unable to write to '" << ovf << "': " << e;
+      return nullopt;
+    }
+
+    // Given that the submission data is now successfully persisted we are no
+    // longer in charge of removing it, except for the cases when the
+    // submission handler terminates with an error (see below for details).
+    //
+    ddr.cancel ();
+
+    // If the handler terminates with non-zero exit status or specifies 5XX
+    // (HTTP server error) submission result manifest status value, then we
+    // stash the submission data directory for troubleshooting. Otherwise, if
+    // it's the 4XX (HTTP client error) status value, then we remove the
+    // directory.
+    //
+    auto stash_submit_dir = [&dd, error] ()
+    {
+      if (dir_exists (dd))
+      try
+      {
+        mvdir (dd, dir_path (dd + ".fail"));
+      }
+      catch (const system_error& e)
+      {
+        // Not much we can do here. Let's just log the issue and bail out
+        // leaving the directory in place.
+        //
+        error << "unable to rename directory '" << dd << "': " << e;
+      }
+    };
+
+    // Run the submission handler, if specified, reading the CI result
+    // manifest from its stdout and parse it into the resulting manifest
+    // object. Otherwise, create implied CI result manifest.
+    //
+    start_result sr;
+
+    if (options_->ci_handler_specified ())
+    {
+      using namespace external_handler;
+
+      optional<result_manifest> r (run (options_->ci_handler (),
+                                        options_->ci_handler_argument (),
+                                        dd,
+                                        options_->ci_handler_timeout (),
+                                        error,
+                                        warn,
+                                        trace));
+      if (!r)
+      {
+        stash_submit_dir ();
+        return nullopt; // The diagnostics is already issued.
+      }
+
+      sr.status = r->status;
+
+      for (manifest_name_value& nv: r->values)
+      {
+        string& n (nv.name);
+        string& v (nv.value);
+
+        if (n == "message")
+          sr.message = move (v);
+        else if (n == "reference")
+          sr.reference = move (v);
+        else if (n != "status")
+          sr.custom_result.emplace_back (move (n), move (v));
+      }
+
+      if (sr.reference.empty ())
+        sr.reference = move (request_id);
+    }
+    else // Create the implied CI result manifest.
+    {
+      sr.status = 200;
+      sr.message = "CI request is queued";
+      sr.reference = move (request_id);
+    }
+
+    // Serialize the CI result manifest manifest to a stream. On the
+    // serialization error log the error description and return false, on the
+    // stream error pass through the io_error exception, otherwise return
+    // true.
+    //
+    auto rsm = [&sr, &error] (ostream& os, bool long_lines = false) -> bool
+    {
+      try
+      {
+        serialize_manifest (sr, os, long_lines);
+        return true;
+      }
+      catch (const serialization& e)
+      {
+        error << "ref " << sr.reference << ": unable to serialize handler's "
+              << "output: " << e;
+        return false;
+      }
+    };
+
+    // If the submission data directory still exists then perform an
+    // appropriate action on it, depending on the submission result status.
+    // Note that the handler could move or remove the directory.
+    //
+    if (dir_exists (dd))
+    {
+      // Remove the directory if the client error is detected.
+      //
+      if (sr.status >= 400 && sr.status < 500)
+      {
+        rmdir_r (dd);
+      }
+      //
+      // Otherwise, save the result manifest, into the directory. Also stash
+      // the directory for troubleshooting in case of the server error.
+      //
+      else
+      {
+        path rsf (dd / "result.manifest");
+
+        try
+        {
+          ofdstream os (rsf);
+
+          // Not being able to stash the result manifest is not a reason to
+          // claim the submission failed. The error is logged nevertheless.
+          //
+          rsm (os);
+
+          os.close ();
+        }
+        catch (const io_error& e)
+        {
+          // Not fatal (see above).
+          //
+          error << "unable to write to '" << rsf << "': " << e;
+        }
+
+        if (sr.status >= 500 && sr.status < 600)
+          stash_submit_dir ();
+      }
+    }
+
+    // Send email, if configured, and the CI request submission is not
+    // simulated. Use the long lines manifest serialization mode for the
+    // convenience of copying/clicking URLs they contain.
+    //
+    // Note that we don't consider the email sending failure to be a
+    // submission failure as the submission data is successfully persisted and
+    // the handler is successfully executed, if configured. One can argue that
+    // email can be essential for the submission processing and missing it
+    // would result in the incomplete submission. In this case it's natural to
+    // assume that the web server error log is monitored and the email sending
+    // failure will be noticed.
+    //
+    if (options_->ci_email_specified () && !simulate)
+    try
+    {
+      // Redirect the diagnostics to the web server error log.
+      //
+      sendmail sm ([trace] (const char* args[], size_t n)
+                   {
+                     if (trace != nullptr)
+                       *trace << process_args {args, n};
+                   },
+                   2 /* stderr */,
+                   options_->email (),
+                   "CI request submission (" + sr.reference + ')',
+                   {options_->ci_email ()});
+
+      // Write the CI request manifest.
+      //
+      pair<bool, optional<start_result>> r (
+        rqm (sm.out, true /* long_lines */));
+
+      assert (r.first); // The serialization succeeded once, so can't fail now.
+
+      // Write the CI overrides manifest.
+      //
+      sm.out << "\n\n";
+
+      r = ovm (sm.out, true /* long_lines */);
+      assert (r.first); // The serialization succeeded once, so can't fail now.
+
+      // Write the CI result manifest.
+      //
+      sm.out << "\n\n";
+
+      // We don't care about the result (see above).
+      //
+      rsm (sm.out, true /* long_lines */);
+
+      sm.out.close ();
+
+      if (!sm.wait ())
+        error << "sendmail " << *sm.exit;
+    }
+    // Handle process_error and io_error (both derive from system_error).
+    //
+    catch (const system_error& e)
+    {
+      error << "sendmail error: " << e;
+    }
+
+    return optional<start_result> (move (sr));
+  }
+
+  void ci_start::
+  serialize_manifest (const start_result& r, ostream& os, bool long_lines)
+  {
+    manifest_serializer s (os, "result", long_lines);
+
+    s.next ("", "1");                        // Start of manifest.
+    s.next ("status", to_string (r.status));
+    s.next ("message", r.message);
+    s.next ("reference", r.reference);
+
+    for (const pair<string, string>& nv: r.custom_result)
+      s.next (nv.first, nv.second);
+
+    s.next ("", "");                         // End of manifest.
+  }
+}
diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx
new file mode 100644
index 0000000..6f62c4b
--- /dev/null
+++ b/mod/ci-common.hxx
@@ -0,0 +1,96 @@
+// file      : mod/ci-common.hxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef MOD_CI_COMMON_HXX
+#define MOD_CI_COMMON_HXX
+
+#include <odb/forward.hxx> // database
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+#include <libbrep/common.hxx>
+
+#include <mod/diagnostics.hxx>
+#include <mod/module-options.hxx>
+
+namespace brep
+{
+  class ci_start
+  {
+  public:
+    void
+    init (shared_ptr<options::ci_start>);
+
+    // If the request handling has been performed normally, then return the
+    // information that corresponds to the CI result manifest (see CI Result
+    // Manifest in the manual). Otherwise (some internal has error occured),
+    // log the error and return nullopt.
+    //
+    // The arguments correspond to the CI request and overrides manifest
+    // values (see CI Request and Overrides Manifests in the manual). Note:
+    // request id and timestamp are generated by the implementation.
+    //
+    struct package
+    {
+      package_name name;
+      optional<brep::version> version;
+    };
+    // Note that the inability to generate the reference is an internal
+    // error. Thus, it is not optional.
+    //
+    struct start_result
+    {
+      uint16_t status;
+      string message;
+      string reference;
+      vector<pair<string, string>> custom_result;
+    };
+
+    // In the optional service information, if id is empty, then the generated
+    // reference is used instead.
+    //
+    optional<start_result>
+    start (const basic_mark& error,
+           const basic_mark& warn,
+           const basic_mark* trace,
+           optional<tenant_service>&&,
+           const repository_location& repository,
+           const vector<package>& packages,
+           const optional<string>& client_ip,
+           const optional<string>& user_agent,
+           const optional<string>& interactive = nullopt,
+           const optional<string>& simulate = nullopt,
+           const vector<pair<string, string>>& custom_request = {},
+           const vector<pair<string, string>>& overrides = {});
+
+    // Helpers.
+    //
+
+    // Serialize the start result as a CI result manifest.
+    //
+    static void
+    serialize_manifest (const start_result&, ostream&, bool long_lines = false);
+
+  private:
+    shared_ptr<options::ci_start> options_;
+  };
+
+  class ci_cancel
+  {
+  public:
+    void
+    init (shared_ptr<options::ci_cancel>, shared_ptr<odb::core::database>);
+
+    // @@ TODO Archive the tenant.
+    //
+    void
+    cancel (/*...*/);
+
+  private:
+    shared_ptr<options::ci_cancel> options_;
+    shared_ptr<odb::core::database> build_db_;
+  };
+}
+
+#endif // MOD_CI_COMMON_HXX
diff --git a/mod/database-module.cxx b/mod/database-module.cxx
index f598bfd..07babc6 100644
--- a/mod/database-module.cxx
+++ b/mod/database-module.cxx
@@ -3,13 +3,20 @@
 
 #include <mod/database-module.hxx>
 
+#include <odb/database.hxx>
 #include <odb/exceptions.hxx>
+#include <odb/transaction.hxx>
+
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
 
 #include <mod/database.hxx>
 #include <mod/module-options.hxx>
 
 namespace brep
 {
+  using namespace odb::core;
+
   // While currently the user-defined copy constructor is not required (we
   // don't need to deep copy nullptr's), it is a good idea to keep the
   // placeholder ready for less trivial cases.
@@ -68,4 +75,53 @@ namespace brep
 
     throw;
   }
+
+  void database_module::
+  update_tenant_service_state (
+    const connection_ptr& conn,
+    const string& tid,
+    const function<optional<string> (const tenant_service&)>& f)
+  {
+    assert (f != nullptr); // Shouldn't be called otherwise.
+
+    // Must be initialized via the init(options::build_db) function call.
+    //
+    assert (build_db_ != nullptr);
+
+    for (size_t retry (retry_);; )
+    {
+      try
+      {
+        transaction tr (conn->begin ());
+
+        shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tid));
+
+        if (t != nullptr && t->service)
+        {
+          tenant_service& s (*t->service);
+
+          if (optional<string> data = f (s))
+          {
+            s.data = move (*data);
+            build_db_->update (t);
+          }
+        }
+
+        tr.commit ();
+
+        // Bail out if we have successfully updated the service state.
+        //
+        break;
+      }
+      catch (const odb::recoverable& e)
+      {
+        if (retry-- == 0)
+          throw;
+
+        HANDLER_DIAG;
+        l1 ([&]{trace << e << "; " << retry + 1 << " tenant service "
+                      << "state update retries left";});
+      }
+    }
+  }
 }
diff --git a/mod/database-module.hxx b/mod/database-module.hxx
index f72ba83..910cb35 100644
--- a/mod/database-module.hxx
+++ b/mod/database-module.hxx
@@ -4,7 +4,7 @@
 #ifndef MOD_DATABASE_MODULE_HXX
 #define MOD_DATABASE_MODULE_HXX
 
-#include <odb/forward.hxx> // database
+#include <odb/forward.hxx> // odb::core::database, odb::core::connection_ptr
 
 #include <libbrep/types.hxx>
 #include <libbrep/utility.hxx>
@@ -14,6 +14,8 @@
 
 namespace brep
 {
+  struct tenant_service;
+
   // A handler that utilises the database. Specifically, it will retry the
   // request in the face of recoverable database failures (deadlock, loss of
   // connection, etc) up to a certain number of times.
@@ -50,6 +52,25 @@ namespace brep
     virtual bool
     handle (request&, response&) = 0;
 
+    // Helpers.
+    //
+
+    // Update the tenant-associated service state if the specified
+    // notification callback-returned function (expected to be not NULL)
+    // returns the new state data.
+    //
+    // Specifically, start the database transaction, query the service state,
+    // and call the callback-returned function on this state. If this call
+    // returns the data string (rather than nullopt), then update the service
+    // state with this data and persist the change. Repeat all the above steps
+    // on the recoverable database failures (deadlocks, etc).
+    //
+    void
+    update_tenant_service_state (
+      const odb::core::connection_ptr&,
+      const string& tid,
+      const function<optional<string> (const tenant_service&)>&);
+
   protected:
     size_t retry_ = 0; // Max of all retries.
 
diff --git a/mod/external-handler.cxx b/mod/external-handler.cxx
index dc4c0fd..3a85bd8 100644
--- a/mod/external-handler.cxx
+++ b/mod/external-handler.cxx
@@ -15,7 +15,8 @@
 
 #include <libbutl/process.hxx>
 #include <libbutl/fdstream.hxx>
-#include <libbutl/process-io.hxx> // operator<<(ostream, process_args)
+#include <libbutl/process-io.hxx>      // operator<<(ostream, process_args)
+#include <libbutl/manifest-parser.hxx>
 
 using namespace std;
 using namespace butl;
diff --git a/mod/external-handler.hxx b/mod/external-handler.hxx
index be16e5b..0276a25 100644
--- a/mod/external-handler.hxx
+++ b/mod/external-handler.hxx
@@ -4,7 +4,7 @@
 #ifndef MOD_EXTERNAL_HANDLER_HXX
 #define MOD_EXTERNAL_HANDLER_HXX
 
-#include <libbutl/manifest-parser.hxx>
+#include <libbutl/manifest-types.hxx>
 
 #include <libbrep/types.hxx>
 #include <libbrep/utility.hxx>
diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx
index 04e1883..dea89de 100644
--- a/mod/mod-build-force.cxx
+++ b/mod/mod-build-force.cxx
@@ -12,20 +12,28 @@
 #include <libbrep/build-odb.hxx>
 
 #include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
 
 using namespace std;
 using namespace brep::cli;
 using namespace odb::core;
 
+brep::build_force::
+build_force (const tenant_service_map& tsm)
+    : tenant_service_map_ (tsm)
+{
+}
+
 // While currently the user-defined copy constructor is not required (we don't
 // need to deep copy nullptr's), it is a good idea to keep the placeholder
 // ready for less trivial cases.
 //
 brep::build_force::
-build_force (const build_force& r)
+build_force (const build_force& r, const tenant_service_map& tsm)
     : database_module (r),
       build_config_module (r),
-      options_ (r.initialized_ ? r.options_ : nullptr)
+      options_ (r.initialized_ ? r.options_ : nullptr),
+      tenant_service_map_ (tsm)
 {
 }
 
@@ -173,15 +181,26 @@ handle (request& rq, response& rs)
   // Load the package build configuration (if present), set the force flag and
   // update the object's persistent state.
   //
+  // If the incomplete package build is being forced to rebuild and the
+  // tenant_service_build_queued callback is associated with the package
+  // tenant, then stash the state, the build object, and the callback pointer
+  // for the subsequent service `queued` notification.
+  //
+  const tenant_service_build_queued* tsq (nullptr);
+  optional<pair<tenant_service, shared_ptr<build>>> tss;
+
+  connection_ptr conn (build_db_->connection ());
   {
-    transaction t (build_db_->begin ());
+    transaction t (conn->begin ());
 
     package_build pb;
+    shared_ptr<build> b;
+
     if (!build_db_->query_one<package_build> (
-          query<package_build>::build::id == id, pb))
+          query<package_build>::build::id == id, pb) ||
+        (b = move (pb.build))->state == build_state::queued)
       config_expired ("no package build");
 
-    shared_ptr<build> b (pb.build);
     force_state force (b->state == build_state::built
                        ? force_state::forced
                        : force_state::forcing);
@@ -211,11 +230,60 @@ handle (request& rq, response& rs)
 
       b->force = force;
       build_db_->update (b);
+
+      if (force == force_state::forcing)
+      {
+        shared_ptr<build_tenant> t (build_db_->load<build_tenant> (b->tenant));
+
+        if (t->service)
+        {
+          auto i (tenant_service_map_.find (t->service->type));
+
+          if (i != tenant_service_map_.end ())
+          {
+            tsq = dynamic_cast<const tenant_service_build_queued*> (
+              i->second.get ());
+
+            if (tsq != nullptr)
+            {
+              // If we ought to call the
+              // tenant_service_build_queued::build_queued() callback, then
+              // also set the package tenant's queued timestamp to the current
+              // time to prevent the notifications race (see
+              // tenant::queued_timestamp for details).
+              //
+              t->queued_timestamp = system_clock::now ();
+              build_db_->update (t);
+
+              tss = make_pair (move (*t->service), move (b));
+            }
+          }
+        }
+      }
     }
 
     t.commit ();
   }
 
+  // If the incomplete package build is being forced to rebuild and the
+  // tenant-associated third-party service needs to be notified about the
+  // queued builds, then call the tenant_service_build_queued::build_queued()
+  // callback function and update the service state, if requested.
+  //
+  if (tsq != nullptr)
+  {
+    assert (tss); // Wouldn't be here otherwise.
+
+    const tenant_service& ss (tss->first);
+    build& b (*tss->second);
+
+    vector<build> qbs;
+    qbs.push_back (move (b));
+
+    if (auto f = tsq->build_queued (ss, qbs, build_state::building))
+      update_tenant_service_state (conn, qbs.back ().tenant, f);
+  }
+
   // We have all the data, so don't buffer the response content.
   //
   ostream& os (rs.content (200, "text/plain;charset=utf-8", false));
diff --git a/mod/mod-build-force.hxx b/mod/mod-build-force.hxx
index 22df383..ea9c141 100644
--- a/mod/mod-build-force.hxx
+++ b/mod/mod-build-force.hxx
@@ -8,6 +8,7 @@
 #include <libbrep/utility.hxx>
 
 #include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
 #include <mod/database-module.hxx>
 #include <mod/build-config-module.hxx>
 
@@ -16,13 +17,13 @@ namespace brep
   class build_force: public database_module, private build_config_module
   {
   public:
-    build_force () = default;
+    explicit
+    build_force (const tenant_service_map&);
 
     // Create a shallow copy (handling instance) if initialized and a deep
     // copy (context exemplar) otherwise.
     //
-    explicit
-    build_force (const build_force&);
+    build_force (const build_force&, const tenant_service_map&);
 
     virtual bool
     handle (request&, response&);
@@ -39,6 +40,7 @@ namespace brep
 
   private:
     shared_ptr<options::build_force> options_;
+    const tenant_service_map& tenant_service_map_;
   };
 }
 
diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx
index 3841fad..fae506b 100644
--- a/mod/mod-build-log.cxx
+++ b/mod/mod-build-log.cxx
@@ -227,7 +227,7 @@ handle (request& rq, response& rs)
           query<package_build>::build::id == id, pb))
       config_expired ("no package build");
 
-    b = pb.build;
+    b = move (pb.build);
     if (b->state != build_state::built)
       config_expired ("state is " + to_string (b->state));
     else
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 24b518d..7023e39 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -24,6 +24,7 @@
 
 #include <mod/build.hxx>          // *_url()
 #include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
 
 using namespace std;
 using namespace butl;
@@ -31,14 +32,21 @@ using namespace bbot;
 using namespace brep::cli;
 using namespace odb::core;
 
+brep::build_result::
+build_result (const tenant_service_map& tsm)
+    : tenant_service_map_ (tsm)
+{
+}
+
 // While currently the user-defined copy constructor is not required (we don't
 // need to deep copy nullptr's), it is a good idea to keep the placeholder
 // ready for less trivial cases.
 //
 brep::build_result::
-build_result (const build_result& r)
+build_result (const build_result& r, const tenant_service_map& tsm)
     : build_result_module (r),
-      options_ (r.initialized_ ? r.options_  : nullptr)
+      options_ (r.initialized_ ? r.options_  : nullptr),
+      tenant_service_map_ (tsm)
 {
 }
 
@@ -186,14 +194,33 @@ handle (request& rq, response&)
   bool build_notify (false);
   bool unforced (true);
 
+  // If the package is built (result status differs from interrupt, etc) and
+  // the package tenant has a third-party service state associated with it,
+  // then check if the tenant_service_build_built callback is registered for
+  // the type of the associated service. If it is, then stash the state, the
+  // build object, and the callback pointer for the subsequent service `built`
+  // notification. Note that we send this notification for the skip result as
+  // well, since it is semantically equivalent to the previous build result
+  // with the actual build process being optimized out.
+  //
+  // If the package build is interrupted and the tenant_service_build_queued
+  // callback is associated with the package tenant, then stash the state, the
+  // build object, and the callback pointer for the subsequent service
+  // `queued` notification.
+  //
+  const tenant_service_build_built* tsb (nullptr);
+  const tenant_service_build_queued* tsq (nullptr);
+  optional<pair<tenant_service, shared_ptr<build>>> tss;
+
   // Note that if the session authentication fails (probably due to the
   // authentication settings change), then we log this case with the warning
   // severity and respond with the 200 HTTP code as if the challenge is
   // valid. The thinking is that we shouldn't alarm a law-abaiding agent and
   // shouldn't provide any information to a malicious one.
   //
+  connection_ptr conn (build_db_->connection ());
   {
-    transaction t (build_db_->begin ());
+    transaction t (conn->begin ());
 
     package_build pb;
 
@@ -221,11 +248,38 @@ handle (request& rq, response&)
     }
     else if (authenticate_session (*options_, rqm.challenge, *b, rqm.session))
     {
+      const tenant_service_base* ts (nullptr);
+
+      shared_ptr<build_tenant> t (build_db_->load<build_tenant> (b->tenant));
+
+      if (t->service)
+      {
+        auto i (tenant_service_map_.find (t->service->type));
+
+        if (i != tenant_service_map_.end ())
+          ts = i->second.get ();
+      }
+
       // If the build is interrupted, then revert it to the original built
-      // state if this is a rebuild and delete it from the database otherwise.
+      // state if this is a rebuild. Otherwise (initial build), turn the build
+      // into the queued state if the tenant_service_build_queued callback is
+      // registered for the package tenant and delete it from the database
+      // otherwise.
+      //
+      // Note that if the tenant_service_build_queued callback is registered,
+      // we always send the `queued` notification for the interrupted build,
+      // even when we reverse it to the original built state. We could also
+      // turn the build into the queued state in this case, but it feels that
+      // there is no harm in keeping the previous build information available
+      // for the user.
       //
       if (rs == result_status::interrupt)
       {
+        // Schedule the `queued` notification, if the
+        // tenant_service_build_queued callback is registered for the tenant.
+        //
+        tsq = dynamic_cast<const tenant_service_build_queued*> (ts);
+
         if (b->status) // Is this a rebuild?
         {
           b->state = build_state::built;
@@ -248,14 +302,57 @@ handle (request& rq, response&)
           // Note that we are unable to restore the pre-rebuild timestamp
           // since it has been overwritten when the build task was issued.
           // That, however, feels ok and we just keep it unchanged.
+          //
+          // Moreover, we actually use the fact that the build's timestamp is
+          // greater then its soft_timestamp as an indication that the build
+          // object represents the interrupted rebuild (see the build_task
+          // handler for details).
 
           build_db_->update (b);
         }
-        else
-          build_db_->erase (b);
+        else           // Initial build.
+        {
+          if (tsq != nullptr)
+          {
+            // Since this is not a rebuild, there are no operation results and
+            // thus we don't need to load the results section to erase results
+            // from the database.
+            //
+            assert (b->results.empty ());
+
+            *b = build (move (b->tenant),
+                        move (b->package_name),
+                        move (b->package_version),
+                        move (b->target),
+                        move (b->target_config_name),
+                        move (b->package_config_name),
+                        move (b->toolchain_name),
+                        move (b->toolchain_version));
+
+            build_db_->update (b);
+          }
+          else
+            build_db_->erase (b);
+        }
+
+        // If we ought to call the tenant_service_build_queued::build_queued()
+        // callback, then also set the package tenant's queued timestamp to
+        // the current time to prevent the notifications race (see
+        // tenant::queued_timestamp for details).
+        //
+        if (tsq != nullptr)
+        {
+          t->queued_timestamp = system_clock::now ();
+          build_db_->update (t);
+        }
       }
-      else
+      else // Regular or skip build result.
       {
+        // Schedule the `built` notification, if the
+        // tenant_service_build_built callback is registered for the tenant.
+        //
+        tsb = dynamic_cast<const tenant_service_build_built*> (ts);
+
         // Verify the result status/checksums.
         //
         // Specifically, if the result status is skip, then it can only be in
@@ -334,7 +431,8 @@ handle (request& rq, response&)
         b->soft_timestamp = b->timestamp;
 
         // If the result status is other than skip, then save the status,
-        // results, and checksums and update the hard timestamp.
+        // results, and checksums and update the hard timestamp. Also stash
+        // the service notification information, if present.
         //
         if (rs != result_status::skip)
         {
@@ -372,18 +470,61 @@ handle (request& rq, response&)
             build_db_->load (*pkg, pkg->constraints_section);
 
             if (!exclude (*cfg, pkg->builds, pkg->constraints, *tc))
-              bld = move (b);
+              bld = b;
           }
         }
         else
           warn << "cannot find configuration '" << b->package_config_name
                << "' for package " << pkg->id.name << '/' << pkg->version;
       }
+
+      // If required, stash the service notification information.
+      //
+      if (tsb != nullptr || tsq != nullptr)
+        tss = make_pair (move (*t->service), move (b));
     }
 
     t.commit ();
   }
 
+  // We either notify about the queued build or notify about the built package
+  // or don't notify at all.
+  //
+  assert (tsb == nullptr || tsq == nullptr);
+
+  // If the package build is interrupted and the tenant-associated third-party
+  // service needs to be notified about the queued builds, then call the
+  // tenant_service_build_queued::build_queued() callback function and update
+  // the service state, if requested.
+  //
+  if (tsq != nullptr)
+  {
+    assert (tss); // Wouldn't be here otherwise.
+
+    const tenant_service& ss (tss->first);
+
+    vector<build> qbs;
+    qbs.push_back (move (*tss->second));
+
+    if (auto f = tsq->build_queued (ss, qbs, build_state::building))
+      update_tenant_service_state (conn, qbs.back ().tenant, f);
+  }
+
+  // If a third-party service needs to be notified about the built package,
+  // then call the tenant_service_build_built::build_built() callback function
+  // and update the service state, if requested.
+  //
+  if (tsb != nullptr)
+  {
+    assert (tss); // Wouldn't be here otherwise.
+
+    const tenant_service& ss (tss->first);
+    const build& b (*tss->second);
+
+    if (auto f = tsb->build_built (ss, b))
+      update_tenant_service_state (conn, b.tenant, f);
+  }
+
   if (bld == nullptr)
     return true;
 
diff --git a/mod/mod-build-result.hxx b/mod/mod-build-result.hxx
index 87ef1f2..96449d5 100644
--- a/mod/mod-build-result.hxx
+++ b/mod/mod-build-result.hxx
@@ -8,6 +8,7 @@
 #include <libbrep/utility.hxx>
 
 #include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
 #include <mod/build-result-module.hxx>
 
 namespace brep
@@ -15,13 +16,13 @@ namespace brep
   class build_result: public build_result_module
   {
   public:
-    build_result () = default;
+    explicit
+    build_result (const tenant_service_map&);
 
     // Create a shallow copy (handling instance) if initialized and a deep
     // copy (context exemplar) otherwise.
     //
-    explicit
-    build_result (const build_result&);
+    build_result (const build_result&, const tenant_service_map&);
 
     virtual bool
     handle (request&, response&);
@@ -35,6 +36,7 @@ namespace brep
 
   private:
     shared_ptr<options::build_result> options_;
+    const tenant_service_map& tenant_service_map_;
   };
 }
 
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 01d14cd..1a06ce1 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -58,15 +58,22 @@ rand (size_t min_val, size_t max_val)
       static_cast<unsigned long long> (max_val)) (rand_gen));
 }
 
+brep::build_task::
+build_task (const tenant_service_map& tsm)
+    : tenant_service_map_ (tsm)
+{
+}
+
 // While currently the user-defined copy constructor is not required (we don't
 // need to deep copy nullptr's), it is a good idea to keep the placeholder
 // ready for less trivial cases.
 //
 brep::build_task::
-build_task (const build_task& r)
+build_task (const build_task& r, const tenant_service_map& tsm)
     : database_module (r),
       build_config_module (r),
-      options_ (r.initialized_ ? r.options_ : nullptr)
+      options_ (r.initialized_ ? r.options_ : nullptr),
+      tenant_service_map_ (tsm)
 {
 }
 
@@ -117,9 +124,15 @@ init (scanner& s)
     options_->root (dir_path ("/"));
 }
 
+// Skip tenants with the freshly queued packages from the consideration (see
+// tenant::queued_timestamp for the details on the service notifications race
+// prevention).
+//
 template <typename T>
 static inline query<T>
-package_query (brep::params::build_task& params, interactive_mode imode)
+package_query (brep::params::build_task& params,
+               interactive_mode imode,
+               uint64_t queued_expiration_ns)
 {
   using namespace brep;
   using query = query<T>;
@@ -153,7 +166,9 @@ package_query (brep::params::build_task& params, interactive_mode imode)
   case interactive_mode::both: break;
   }
 
-  return q;
+  return q &&
+         (query::build_tenant::queued_timestamp.is_null () ||
+          query::build_tenant::queued_timestamp < queued_expiration_ns);
 }
 
 bool brep::build_task::
@@ -261,7 +276,7 @@ handle (request& rq, response& rs)
 
   config_machines conf_machines;
 
-  for (const auto& c: *target_conf_)
+  for (const build_target_config& c: *target_conf_)
   {
     for (auto& m: tqm.machines)
     {
@@ -303,34 +318,34 @@ handle (request& rq, response& rs)
     // Create the task response manifest. Must be called inside the build db
     // transaction.
     //
-    auto task = [this] (shared_ptr<build>&& b,
-                        shared_ptr<build_package>&& p,
+    auto task = [this] (const build& b,
+                        build_package&& p,
                         build_package_config&& pc,
                         optional<string>&& interactive,
                         const config_machine& cm) -> task_response_manifest
     {
       uint64_t ts (
         chrono::duration_cast<std::chrono::nanoseconds> (
-          b->timestamp.time_since_epoch ()).count ());
-
-      string session (b->tenant + '/'                      +
-                      b->package_name.string () + '/'      +
-                      b->package_version.string () + '/'   +
-                      b->target.string () + '/'            +
-                      b->target_config_name + '/'          +
-                      b->package_config_name + '/'         +
-                      b->toolchain_name + '/'              +
-                      b->toolchain_version.string () + '/' +
+          b.timestamp.time_since_epoch ()).count ());
+
+      string session (b.tenant + '/'                      +
+                      b.package_name.string () + '/'      +
+                      b.package_version.string () + '/'   +
+                      b.target.string () + '/'            +
+                      b.target_config_name + '/'          +
+                      b.package_config_name + '/'         +
+                      b.toolchain_name + '/'              +
+                      b.toolchain_version.string () + '/' +
                       to_string (ts));
 
-      string tenant (tenant_dir (options_->root (), b->tenant).string ());
+      string tenant (tenant_dir (options_->root (), b.tenant).string ());
       string result_url (options_->host () + tenant + "?build-result");
 
       assert (transaction::has_current ());
 
-      assert (p->internal ()); // The package is expected to be buildable.
+      assert (p.internal ()); // The package is expected to be buildable.
 
-      shared_ptr<build_repository> r (p->internal_repository.load ());
+      shared_ptr<build_repository> r (p.internal_repository.load ());
 
       strings fps;
       if (r->certificate_fingerprint)
@@ -341,9 +356,9 @@ handle (request& rq, response& rs)
       //
       small_vector<bpkg::test_dependency, 1> tests;
 
-      build_db_->load (*p, p->requirements_tests_section);
+      build_db_->load (p, p.requirements_tests_section);
 
-      for (const build_test_dependency& td: p->tests)
+      for (const build_test_dependency& td: p.tests)
       {
         // Don't exclude unresolved external tests.
         //
@@ -390,16 +405,17 @@ handle (request& rq, response& rs)
                             move (td.reflect));
       }
 
-      bool module_pkg (
-        b->package_name.string ().compare (0, 10, "libbuild2-") == 0);
+      package_name& pn (p.id.name);
 
-      task_manifest task (move (b->package_name),
-                          move (b->package_version),
+      bool module_pkg (pn.string ().compare (0, 10, "libbuild2-") == 0);
+
+      task_manifest task (move (pn),
+                          move (p.version),
                           move (r->location),
                           move (fps),
-                          move (p->requirements),
+                          move (p.requirements),
                           move (tests),
-                          move (b->dependency_checksum),
+                          b.dependency_checksum,
                           cm.machine->name,
                           cm.config->target,
                           cm.config->environment,
@@ -408,7 +424,7 @@ handle (request& rq, response& rs)
                           belongs (*cm.config, module_pkg ? "build2" : "host"),
                           cm.config->warning_regexes,
                           move (interactive),
-                          move (b->worker_checksum));
+                          b.worker_checksum);
 
       // Collect the build artifacts upload URLs, skipping those which are
       // excluded with the upload-*-exclude configuration options.
@@ -434,7 +450,7 @@ handle (request& rq, response& rs)
         };
 
         if (!exclude (options_->upload_toolchain_exclude (),
-                      b->toolchain_name) &&
+                      b.toolchain_name) &&
             !exclude (options_->upload_repository_exclude (),
                       r->canonical_name))
         {
@@ -444,15 +460,15 @@ handle (request& rq, response& rs)
       }
 
       return task_response_manifest (move (session),
-                                     move (b->agent_challenge),
+                                     b.agent_challenge,
                                      move (result_url),
                                      move (upload_urls),
-                                     move (b->agent_checksum),
+                                     b.agent_checksum,
                                      move (task));
     };
 
-    // Calculate the build (building state) or rebuild (built state)
-    // expiration time for package configurations.
+    // Calculate the build/rebuild (building/built state) and the `queued`
+    // notifications expiration time for package configurations.
     //
     timestamp now (system_clock::now ());
 
@@ -476,6 +492,9 @@ handle (request& rq, response& rs)
     timestamp forced_rebuild_expiration (
       expiration (options_->build_forced_rebuild_timeout ()));
 
+    uint64_t queued_expiration_ns (
+      expiration_ns (options_->build_queued_timeout ()));
+
     // Calculate the soft/hard rebuild expiration time, based on the
     // respective build-{soft,hard}-rebuild-timeout and
     // build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration
@@ -626,6 +645,7 @@ handle (request& rq, response& rs)
     // Convert butl::standard_version type to brep::version.
     //
     brep::version toolchain_version (tqm.toolchain_version.string ());
+    string&       toolchain_name (tqm.toolchain_name);
 
     // Prepare the buildable package prepared query.
     //
@@ -646,7 +666,9 @@ handle (request& rq, response& rs)
     using pkg_query = query<buildable_package>;
     using prep_pkg_query = prepared_query<buildable_package>;
 
-    pkg_query pq (package_query<buildable_package> (params, imode));
+    pkg_query pq (package_query<buildable_package> (params,
+                                                    imode,
+                                                    queued_expiration_ns));
 
     // Transform (in-place) the interactive login information into the actual
     // login command, if specified in the manifest and the transformation
@@ -800,7 +822,9 @@ handle (request& rq, response& rs)
     {
       using query = query<buildable_package_count>;
 
-      query q (package_query<buildable_package_count> (params, imode));
+      query q (package_query<buildable_package_count> (params,
+                                                       imode,
+                                                       queued_expiration_ns));
 
       transaction t (build_db_->begin ());
 
@@ -896,17 +920,18 @@ handle (request& rq, response& rs)
         equal<build> (bld_query::id.package, id)                          &&
         bld_query::id.package_config_name == bld_query::_ref (pkg_config) &&
         sq                                                                &&
-        bld_query::id.toolchain_name == tqm.toolchain_name                &&
+        bld_query::id.toolchain_name == toolchain_name                    &&
 
         compare_version_eq (bld_query::id.toolchain_version,
                             canonical_version (toolchain_version),
                             true /* revision */)                          &&
 
-        (bld_query::state == "built"                          ||
-         (bld_query::force == "forcing" &&
-          bld_query::timestamp > forced_result_expiration_ns) ||
-         (bld_query::force != "forcing" && // Unforced or forced.
-          bld_query::timestamp > normal_result_expiration_ns)));
+        (bld_query::state == "built" ||
+         (bld_query::state == "building" &&
+          ((bld_query::force == "forcing" &&
+            bld_query::timestamp > forced_result_expiration_ns) ||
+           (bld_query::force != "forcing" && // Unforced or forced.
+            bld_query::timestamp > normal_result_expiration_ns)))));
 
       prep_bld_query bld_prep_query (
         conn->prepare_query<build> ("mod-build-task-build-query", bq));
@@ -971,6 +996,107 @@ handle (request& rq, response& rs)
       //
       optional<string> start_tenant;
 
+      // If the build task is created and the tenant of the being built
+      // package has a third-party service state associated with it, then
+      // check if the tenant_service_build_building and/or
+      // tenant_service_build_queued callbacks are registered for the type of
+      // the associated service. If they are, then stash the state, the build
+      // object, and the callback pointers for the subsequent service
+      // notifications.
+      //
+      // Also, if the tenant_service_build_queued callback is registered, then
+      // create, persist, and stash the queued build objects for all the
+      // unbuilt by the current toolchain and not yet queued configurations of
+      // the package the build task is created for. Note that for the task
+      // build, we need to make sure that the third-party service receives the
+      // `queued` notification prior to the `building` notification (see
+      // mod/tenant-service.hxx for valid transitions). The `queued`
+      // notification is assumed to be already sent for the build if the
+      // respective object exists and any of the following is true for it:
+      //
+      // - It is in the queued state (initial_state is build_state::queued).
+      //
+      // - It is a user-forced rebuild of an incomplete build
+      //   (rebuild_forced_build is true).
+      //
+      // - It is a rebuild of an interrupted rebuild (rebuild_forced_build is
+      //   true).
+      //
+      const tenant_service_build_building* tsb (nullptr);
+      const tenant_service_build_queued* tsq (nullptr);
+      optional<pair<tenant_service, shared_ptr<build>>> tss;
+      vector<build> qbs;
+      optional<build_state> initial_state;
+      bool rebuild_forced_build (false);
+      bool rebuild_interrupted_rebuild (false);
+
+      // Create, persist, and return the queued build objects for all the
+      // unbuilt by the current toolchain and not yet queued configurations of
+      // the specified package.
+      //
+      // Note that the build object argument is only used for the toolchain
+      // information retrieval. Also note that the package constraints section
+      // is expected to be loaded.
+      //
+      auto queue_builds = [this] (const build_package& p, const build& b)
+      {
+        assert (p.constraints_section.loaded ());
+
+        // Query the existing build ids and stash them into the set.
+        //
+        set<build_id> existing_builds;
+
+        using query = query<package_build_id>;
+
+        query q (query::build::id.package == p.id                     &&
+                 query::build::id.toolchain_name == b.toolchain_name  &&
+                 compare_version_eq (query::build::id.toolchain_version,
+                                     b.id.toolchain_version,
+                                     true /* revision */));
+
+        for (build_id& id: build_db_->query<package_build_id> (q))
+          existing_builds.emplace (move (id));
+
+        // Go through all the potential package builds and queue those which
+        // are not in the existing builds set.
+        //
+        vector<build> r;
+
+        for (const build_package_config& pc: p.configs)
+        {
+          for (const build_target_config& tc: *target_conf_)
+          {
+            if (!exclude (pc, p.builds, p.constraints, tc))
+            {
+              build_id id (p.id,
+                           tc.target, tc.name,
+                           pc.name,
+                           b.toolchain_name, b.toolchain_version);
+
+              if (existing_builds.find (id) == existing_builds.end ())
+              {
+                r.emplace_back (move (id.package.tenant),
+                                move (id.package.name),
+                                p.version,
+                                move (id.target),
+                                move (id.target_config_name),
+                                move (id.package_config_name),
+                                move (id.toolchain_name),
+                                b.toolchain_version);
+
+                // @@ TODO Persist the whole vector of builds with a single
+                //         operation if/when bulk operations support is added
+                //         for objects with containers.
+                //
+                build_db_->persist (r.back ());
+              }
+            }
+          }
+        }
+
+        return r;
+      };
+
       for (bool done (false); tsm.session.empty () && !done; )
       {
         transaction t (conn->begin ());
@@ -1163,7 +1289,7 @@ handle (request& rq, response& rs)
                               tc.target,
                               tc.name,
                               pc,
-                              tqm.toolchain_name,
+                              toolchain_name,
                               toolchain_version);
 
                 // Can there be any existing builds for such a tenant? Doesn't
@@ -1279,7 +1405,7 @@ handle (request& rq, response& rs)
                               cm.config->target,
                               cm.config->name,
                               move (pkg_config),
-                              move (tqm.toolchain_name),
+                              move (toolchain_name),
                               toolchain_version);
 
                 shared_ptr<build> b (build_db_->find<build> (bid));
@@ -1318,16 +1444,20 @@ handle (request& rq, response& rs)
                 }
                 else
                 {
-                  // The build configuration is in the building state.
+                  // The build configuration is in the building or queued
+                  // state.
                   //
-                  // Note that in both cases we keep the status intact to be
-                  // able to compare it with the final one in the result
-                  // request handling in order to decide if to send the
-                  // notification email or to revert it to the built state if
-                  // interrupted. The same is true for the forced flag (in
-                  // the sense that we don't set the force state to unforced).
+                  // Note that in both the building and built cases we keep
+                  // the status intact to be able to compare it with the final
+                  // one in the result request handling in order to decide if
+                  // to send the notification email or to revert it to the
+                  // built state if interrupted. The same is true for the
+                  // forced flag (in the sense that we don't set the force
+                  // state to unforced).
                   //
-                  assert (b->state == build_state::building);
+                  assert (b->state != build_state::built);
+
+                  initial_state = b->state;
 
                   b->state = build_state::building;
                   b->interactive = move (login);
@@ -1337,7 +1467,10 @@ handle (request& rq, response& rs)
                   // still recognize that the rebuild was forced.
                   //
                   if (b->force == force_state::forcing)
+                  {
                     b->force = force_state::forced;
+                    rebuild_forced_build = true;
+                  }
 
                   b->agent_fingerprint = move (agent_fp);
                   b->agent_challenge = move (cl);
@@ -1364,21 +1497,58 @@ handle (request& rq, response& rs)
                   build_db_->update (b);
                 }
 
+                shared_ptr<build_tenant> t (
+                  build_db_->load<build_tenant> (b->tenant));
+
                 // Archive an interactive tenant.
                 //
                 if (bp.interactive)
                 {
-                  shared_ptr<build_tenant> t (
-                    build_db_->load<build_tenant> (b->id.package.tenant));
-
                   t->archived = true;
                   build_db_->update (t);
                 }
 
-                // Finally, prepare the task response manifest.
+                // Finally, stash the service notification information, if
+                // present, and prepare the task response manifest.
                 //
+                if (t->service)
+                {
+                  auto i (tenant_service_map_.find (t->service->type));
+
+                  if (i != tenant_service_map_.end ())
+                  {
+                    const tenant_service_base* s (i->second.get ());
+
+                    tsb = dynamic_cast<const tenant_service_build_building*> (s);
+                    tsq = dynamic_cast<const tenant_service_build_queued*> (s);
+
+                    if (tsq != nullptr)
+                    {
+                      qbs = queue_builds (*p, *b);
+
+                      // If we ought to call the
+                      // tenant_service_build_queued::build_queued() callback,
+                      // then also set the package tenant's queued timestamp
+                      // to the current time to prevent the notifications race
+                      // (see tenant::queued_timestamp for details).
+                      //
+                      if (!qbs.empty ()  ||
+                          !initial_state ||
+                          (*initial_state != build_state::queued &&
+                           !rebuild_forced_build))
+                      {
+                        t->queued_timestamp = system_clock::now ();
+                        build_db_->update (t);
+                      }
+                    }
+
+                    if (tsb != nullptr || tsq != nullptr)
+                      tss = make_pair (move (*t->service), b);
+                  }
+                }
+
                 tsm = task (
-                  move (b), move (p), move (pc), move (bp.interactive), cm);
+                  *b, move (*p), move (pc), move (bp.interactive), cm);
 
                 break; // Bail out from the package configurations loop.
               }
@@ -1493,6 +1663,11 @@ handle (request& rq, response& rs)
                 {
                   assert (b->status);
 
+                  initial_state = build_state::built;
+
+                  rebuild_interrupted_rebuild =
+                    (b->timestamp > b->soft_timestamp);
+
                   b->state = build_state::building;
 
                   // Save the interactive build login information into the
@@ -1535,8 +1710,45 @@ handle (request& rq, response& rs)
 
                   build_db_->update (b);
 
+                  // Stash the service notification information, if present,
+                  // and prepare the task response manifest.
+                  //
+                  if (t->service)
+                  {
+                    auto i (tenant_service_map_.find (t->service->type));
+
+                    if (i != tenant_service_map_.end ())
+                    {
+                      const tenant_service_base* s (i->second.get ());
+
+                      tsb = dynamic_cast<const tenant_service_build_building*> (s);
+                      tsq = dynamic_cast<const tenant_service_build_queued*> (s);
+
+                      if (tsq != nullptr)
+                      {
+                        qbs = queue_builds (*p, *b);
+
+                        // If we ought to call the
+                        // tenant_service_build_queued::build_queued()
+                        // callback, then also set the package tenant's queued
+                        // timestamp to the current time to prevent the
+                        // notifications race (see tenant::queued_timestamp
+                        // for details).
+                        //
+                        if (!qbs.empty () || !rebuild_interrupted_rebuild)
+                        {
+                          t->queued_timestamp = system_clock::now ();
+                          build_db_->update (t);
+                        }
+                      }
+
+                      if (tsb != nullptr || tsq != nullptr)
+                        tss = make_pair (move (*t->service), b);
+                    }
+                  }
+
                   tsm = task (
-                    move (b), move (p), move (*pc), move (t->interactive), cm);
+                    *b, move (*p), move (*pc), move (t->interactive), cm);
                 }
               }
             }
@@ -1558,6 +1770,73 @@ handle (request& rq, response& rs)
             break;
         }
       }
+
+      // If the tenant-associated third-party service needs to be notified
+      // about the queued builds, then call the
+      // tenant_service_build_queued::build_queued() callback function and
+      // update the service state, if requested.
+      //
+      if (tsq != nullptr)
+      {
+        assert (tss); // Wouldn't be here otherwise.
+
+        const tenant_service& ss (tss->first);
+
+        // If the task build has no initial state (is just created), then
+        // temporarily move it into the list of the queued builds until the
+        // `queued` notification is delivered. Afterwards, restore it so that
+        // the `building` notification can also be sent.
+        //
+        build& b (*tss->second);
+        bool restore_build (false);
+
+        if (!initial_state)
+        {
+          qbs.push_back (move (b));
+          restore_build = true;
+        }
+
+        if (!qbs.empty ())
+        {
+          if (auto f = tsq->build_queued (ss, qbs, nullopt /* initial_state */))
+            update_tenant_service_state (conn, qbs.back ().tenant, f);
+        }
+
+        // Send the `queued` notification for the task build, unless it is
+        // already sent, and update the service state, if requested.
+        //
+        if (initial_state                         &&
+            *initial_state != build_state::queued &&
+            !rebuild_interrupted_rebuild          &&
+            !rebuild_forced_build)
+        {
+          qbs.clear ();
+          qbs.push_back (move (b));
+          restore_build = true;
+
+          if (auto f = tsq->build_queued (ss, qbs, initial_state))
+            update_tenant_service_state (conn, qbs.back ().tenant, f);
+        }
+
+        if (restore_build)
+          b = move (qbs.back ());
+      }
+
+      // If a third-party service needs to be notified about the package
+      // build, then call the tenant_service_build_built::build_building()
+      // callback function and, if requested, update the tenant-associated
+      // service state.
+      //
+      if (tsb != nullptr)
+      {
+        assert (tss); // Wouldn't be here otherwise.
+
+        const tenant_service& ss (tss->first);
+        const build& b (*tss->second);
+
+        if (auto f = tsb->build_building (ss, b))
+          update_tenant_service_state (conn, b.tenant, f);
+      }
     }
   }
 
diff --git a/mod/mod-build-task.hxx b/mod/mod-build-task.hxx
index 7875db1..d0b3d44 100644
--- a/mod/mod-build-task.hxx
+++ b/mod/mod-build-task.hxx
@@ -8,6 +8,7 @@
 #include <libbrep/utility.hxx>
 
 #include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
 #include <mod/database-module.hxx>
 #include <mod/build-config-module.hxx>
 
@@ -16,13 +17,13 @@ namespace brep
   class build_task: public database_module, private build_config_module
   {
   public:
-    build_task () = default;
+    explicit
+    build_task (const tenant_service_map&);
 
     // Create a shallow copy (handling instance) if initialized and a deep
     // copy (context exemplar) otherwise.
     //
-    explicit
-    build_task (const build_task&);
+    build_task (const build_task&, const tenant_service_map&);
 
     virtual bool
     handle (request&, response&);
@@ -36,6 +37,7 @@ namespace brep
 
   private:
     shared_ptr<options::build_task> options_;
+    const tenant_service_map& tenant_service_map_;
   };
 }
 
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index f260b72..b0de618 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -225,13 +225,19 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids,
     // Build result.
     //
     const string& rs (params.result ());
+    bool add_state (true);
 
     if (rs != "*")
     {
       if (rs == "pending")
+      {
         q = q && qb::force != "unforced";
+      }
       else if (rs == "building")
+      {
         q = q && qb::state == "building";
+        add_state = false;
+      }
       else
       {
         query sq (qb::status == rs);
@@ -259,8 +265,12 @@ build_query (const brep::vector<brep::build_target_config_id>* config_ids,
         // well (rebuild).
         //
         q = q && qb::state == "built" && sq;
+        add_state = false;
       }
     }
+
+    if (add_state)
+      q = q && qb::state != "queued";
   }
   catch (const invalid_argument&)
   {
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index df2365a..fec603e 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -3,18 +3,11 @@
 
 #include <mod/mod-ci.hxx>
 
-#include <ostream>
-
-#include <libbutl/uuid.hxx>
-#include <libbutl/sendmail.hxx>
 #include <libbutl/fdstream.hxx>
-#include <libbutl/timestamp.hxx>
-#include <libbutl/filesystem.hxx>
-#include <libbutl/process-io.hxx>          // operator<<(ostream, process_args)
 #include <libbutl/manifest-parser.hxx>
 #include <libbutl/manifest-serializer.hxx>
 
-#include <libbpkg/manifest.hxx>
+#include <libbpkg/manifest.hxx>     // package_manifest
 #include <libbpkg/package-name.hxx>
 
 #include <web/server/module.hxx>
@@ -23,20 +16,35 @@
 
 #include <mod/page.hxx>
 #include <mod/module-options.hxx>
-#include <mod/external-handler.hxx>
 
 using namespace std;
 using namespace butl;
 using namespace web;
 using namespace brep::cli;
 
+#ifdef BREP_CI_TENANT_SERVICE
+brep::ci::
+ci (tenant_service_map& tsm)
+    : tenant_service_map_ (tsm)
+{
+}
+#endif
+
 brep::ci::
+#ifdef BREP_CI_TENANT_SERVICE
+ci (const ci& r, tenant_service_map& tsm)
+#else
 ci (const ci& r)
+#endif
     : handler (r),
+      ci_start (r),
       options_ (r.initialized_ ? r.options_ : nullptr),
       form_ (r.initialized_ || r.form_ == nullptr
              ? r.form_
              : make_shared<xhtml::fragment> (*r.form_))
+#ifdef BREP_CI_TENANT_SERVICE
+    , tenant_service_map_ (tsm)
+#endif
 {
 }
 
@@ -45,22 +53,25 @@ init (scanner& s)
 {
   HANDLER_DIAG;
 
+#ifdef BREP_CI_TENANT_SERVICE
+  {
+    shared_ptr<tenant_service_base> ts (
+      dynamic_pointer_cast<tenant_service_base> (shared_from_this ()));
+
+    assert (ts != nullptr); // By definition.
+
+    tenant_service_map_["ci"] = move (ts);
+  }
+#endif
+
   options_ = make_shared<options::ci> (
     s, unknown_mode::fail, unknown_mode::fail);
 
-  // Verify that the CI request handling is setup properly, if configured.
+  // Prepare for the CI requests handling, if configured.
   //
   if (options_->ci_data_specified ())
   {
-    // Verify the data directory satisfies the requirements.
-    //
-    const dir_path& d (options_->ci_data ());
-
-    if (d.relative ())
-      fail << "ci-data directory path must be absolute";
-
-    if (!dir_exists (d))
-      fail << "ci-data directory '" << d << "' does not exist";
+    ci_start::init (make_shared<options::ci_start> (*options_));
 
     // Parse XHTML5 form file, if configured.
     //
@@ -87,10 +98,6 @@ init (scanner& s)
         fail << "unable to read ci-form file '" << ci_form << "': " << e;
       }
     }
-
-    if (options_->ci_handler_specified () &&
-        options_->ci_handler ().relative ())
-      fail << "ci-handler path must be absolute";
   }
 
   if (options_->root ().empty ())
@@ -130,9 +137,8 @@ handle (request& rq, response& rs)
   //
   // return respond_error (); // Request is handled with an error.
   //
-  string request_id; // Will be set later.
-  auto respond_manifest = [&rs, &request_id] (status_code status,
-                                              const string& message) -> bool
+  auto respond_manifest = [&rs] (status_code status,
+                                 const string& message) -> bool
   {
     serializer s (rs.content (status, "text/manifest;charset=utf-8"),
                   "response");
@@ -140,10 +146,6 @@ handle (request& rq, response& rs)
     s.next ("", "1");                      // Start of manifest.
     s.next ("status", to_string (status));
     s.next ("message", message);
-
-    if (!request_id.empty ())
-      s.next ("reference", request_id);
-
     s.next ("", "");                       // End of manifest.
     return true;
   };
@@ -234,9 +236,11 @@ handle (request& rq, response& rs)
   if (rl.empty () || rl.local ())
     return respond_manifest (400, "invalid repository location");
 
-  // Verify the package name[/version] arguments.
+  // Parse the package name[/version] arguments.
   //
-  for (const string& s: params.package())
+  vector<package> packages;
+
+  for (const string& s: params.package ())
   {
     //  Let's skip the potentially unfilled package form fields.
     //
@@ -245,18 +249,21 @@ handle (request& rq, response& rs)
 
     try
     {
+      package pkg;
       size_t p (s.find ('/'));
 
       if (p != string::npos)
       {
-        package_name (string (s, 0, p));
+        pkg.name = package_name (string (s, 0, p));
 
         // Not to confuse with module::version.
         //
-        bpkg::version (string (s, p + 1));
+        pkg.version = bpkg::version (string (s, p + 1));
       }
       else
-        package_name p (s); // Not to confuse with the s variable declaration.
+        pkg.name = package_name (s);
+
+      packages.push_back (move (pkg));
     }
     catch (const invalid_argument&)
     {
@@ -265,31 +272,49 @@ handle (request& rq, response& rs)
   }
 
   // Verify that unknown parameter values satisfy the requirements (contain
-  // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n').
+  // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n') and
+  // stash them.
   //
   // Actually, the expected ones must satisfy too, so check them as well.
   //
-  string what;
-  for (const name_value& nv: rps)
+  vector<pair<string, string>> custom_request;
   {
-    if (nv.value &&
-        !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t"))
-      return respond_manifest (400,
-                               "invalid parameter " + nv.name + ": " + what);
+    string what;
+    for (const name_value& nv: rps)
+    {
+      if (nv.value &&
+          !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t"))
+        return respond_manifest (400,
+                                 "invalid parameter " + nv.name + ": " + what);
+
+      const string& n (nv.name);
+
+      if (n != "repository"  &&
+          n != "_"           &&
+          n != "package"     &&
+          n != "overrides"   &&
+          n != "interactive" &&
+          n != "simulate")
+         custom_request.emplace_back (n, nv.value ? *nv.value : "");
+    }
   }
 
   // Parse and validate overrides, if present.
   //
-  vector<manifest_name_value> overrides;
+  vector<pair<string, string>> overrides;
 
   if (params.overrides_specified ())
   try
   {
     istream& is (rq.open_upload ("overrides"));
     parser mp (is, "overrides");
-    overrides = parse_manifest (mp);
+    vector<manifest_name_value> ovrs (parse_manifest (mp));
+
+    package_manifest::validate_overrides (ovrs, mp.name ());
 
-    package_manifest::validate_overrides (overrides, mp.name ());
+    overrides.reserve (ovrs.size ());
+    for (manifest_name_value& nv: ovrs)
+      overrides.emplace_back (move (nv.name), move (nv.value));
   }
   // Note that invalid_argument (thrown by open_upload() function call) can
   // mean both no overrides upload or multiple overrides uploads.
@@ -310,383 +335,127 @@ handle (request& rq, response& rs)
     return respond_error ();
   }
 
-  try
-  {
-    // Note that from now on the result manifest we respond with will contain
-    // the reference value.
-    //
-    request_id = uuid::generate ().string ();
-  }
-  catch (const system_error& e)
-  {
-    error << "unable to generate request id: " << e;
-    return respond_error ();
-  }
-
-  // Create the submission data directory.
+  // Stash the User-Agent HTTP header and the client IP address.
   //
-  dir_path dd (options_->ci_data () / dir_path (request_id));
-
-  try
+  optional<string> client_ip;
+  optional<string> user_agent;
+  for (const name_value& h: rq.headers ())
   {
-    // It's highly unlikely but still possible that the directory already
-    // exists. This can only happen if the generated uuid is not unique.
-    //
-    if (try_mkdir (dd) == mkdir_status::already_exists)
-      throw_generic_error (EEXIST);
+    if (icasecmp (h.name, ":Client-IP") == 0)
+      client_ip = h.value;
+    else if (icasecmp (h.name, "User-Agent") == 0)
+      user_agent = h.value;
   }
-  catch (const system_error& e)
-  {
-    error << "unable to create directory '" << dd << "': " << e;
-    return respond_error ();
-  }
-
-  auto_rmdir ddr (dd);
-
-  // Serialize the CI request manifest to a stream. On the serialization error
-  // respond to the client with the manifest containing the bad request (400)
-  // code and return false, on the stream error pass through the io_error
-  // exception, otherwise return true.
-  //
-  timestamp ts (system_clock::now ());
-
-  auto rqm = [&request_id,
-              &rl,
-              &ts,
-              &simulate,
-              &rq,
-              &rps,
-              &params,
-              &respond_manifest]
-             (ostream& os, bool long_lines = false) -> bool
-  {
-    try
-    {
-      serializer s (os, "request", long_lines);
 
-      // Serialize the submission manifest header.
-      //
-      s.next ("", "1");                // Start of manifest.
-      s.next ("id", request_id);
-      s.next ("repository", rl.string ());
-
-      for (const string& p: params.package ())
-      {
-        if (!p.empty ()) // Skip empty package names (see above for details).
-          s.next ("package", p);
-      }
-
-      if (params.interactive_specified ())
-        s.next ("interactive", params.interactive ());
-
-      if (!simulate.empty ())
-        s.next ("simulate", simulate);
-
-      s.next ("timestamp",
-              butl::to_string (ts,
-                               "%Y-%m-%dT%H:%M:%SZ",
-                               false /* special */,
-                               false /* local */));
-
-      // Serialize the User-Agent HTTP header and the client IP address.
-      //
-      optional<string> ip;
-      optional<string> ua;
-      for (const name_value& h: rq.headers ())
-      {
-        if (icasecmp (h.name, ":Client-IP") == 0)
-          ip = h.value;
-        else if (icasecmp (h.name, "User-Agent") == 0)
-          ua = h.value;
-      }
-
-      if (ip)
-        s.next ("client-ip", *ip);
-
-      if (ua)
-        s.next ("user-agent", *ua);
-
-      // Serialize the request parameters.
-      //
-      // Note that the serializer constraints the parameter names (can't start
-      // with '#', can't contain ':' and the whitespaces, etc.).
-      //
-      for (const name_value& nv: rps)
-      {
-        const string& n (nv.name);
-
-        if (n != "repository"  &&
-            n != "_"           &&
-            n != "package"     &&
-            n != "overrides"   &&
-            n != "interactive" &&
-            n != "simulate")
-          s.next (n, nv.value ? *nv.value : "");
-      }
-
-      s.next ("", ""); // End of manifest.
-      return true;
-    }
-    catch (const serialization& e)
-    {
-      respond_manifest (400, string ("invalid parameter: ") + e.what ());
-      return false;
-    }
-  };
-
-  // Serialize the CI request manifest to the submission directory.
-  //
-  path rqf (dd / "request.manifest");
+  optional<start_result> r (start (error,
+                                   warn,
+                                   verb_ ? &trace : nullptr,
+#ifdef BREP_CI_TENANT_SERVICE
+                                   tenant_service ("", "ci"),
+#else
+                                   nullopt /* service */,
+#endif
+                                   rl,
+                                   packages,
+                                   client_ip,
+                                   user_agent,
+                                   (params.interactive_specified ()
+                                    ? params.interactive ()
+                                    : optional<string> ()),
+                                   (!simulate.empty ()
+                                    ? simulate
+                                    : optional<string> ()),
+                                   custom_request,
+                                   overrides));
+
+  if (!r)
+    return respond_error (); // The diagnostics is already issued.
 
   try
   {
-    ofdstream os (rqf);
-    bool r (rqm (os));
-    os.close ();
-
-    if (!r)
-      return true; // The client is already responded with the manifest.
-  }
-  catch (const io_error& e)
-  {
-    error << "unable to write to '" << rqf << "': " << e;
-    return respond_error ();
+    serialize_manifest (*r,
+                        rs.content (r->status, "text/manifest;charset=utf-8"));
   }
-
-  // Serialize the CI overrides manifest to a stream. On the stream error pass
-  // through the io_error exception.
-  //
-  // Note that it can't throw the serialization exception as the override
-  // manifest is parsed from the stream and so verified.
-  //
-  auto ovm = [&overrides] (ostream& os, bool long_lines = false)
+  catch (const serialization& e)
   {
-    try
-    {
-      serializer s (os, "overrides", long_lines);
-      serialize_manifest (s, overrides);
-    }
-    catch (const serialization&) {assert (false);} // See above.
-  };
+    error << "ref " << r->reference << ": unable to serialize handler's "
+          << "output: " << e;
 
-  // Serialize the CI overrides manifest to the submission directory.
-  //
-  path ovf (dd / "overrides.manifest");
-
-  if (!overrides.empty ())
-  try
-  {
-    ofdstream os (ovf);
-    ovm (os);
-    os.close ();
-  }
-  catch (const io_error& e)
-  {
-    error << "unable to write to '" << ovf << "': " << e;
     return respond_error ();
   }
 
-  // Given that the submission data is now successfully persisted we are no
-  // longer in charge of removing it, except for the cases when the submission
-  // handler terminates with an error (see below for details).
-  //
-  ddr.cancel ();
-
-  // If the handler terminates with non-zero exit status or specifies 5XX
-  // (HTTP server error) submission result manifest status value, then we
-  // stash the submission data directory for troubleshooting. Otherwise, if
-  // it's the 4XX (HTTP client error) status value, then we remove the
-  // directory.
-  //
-  auto stash_submit_dir = [&dd, error] ()
-  {
-    if (dir_exists (dd))
-    try
-    {
-      mvdir (dd, dir_path (dd + ".fail"));
-    }
-    catch (const system_error& e)
-    {
-      // Not much we can do here. Let's just log the issue and bail out
-      // leaving the directory in place.
-      //
-      error << "unable to rename directory '" << dd << "': " << e;
-    }
-  };
-
-  // Run the submission handler, if specified, reading the result manifest
-  // from its stdout and caching it as a name/value pair list for later use
-  // (forwarding to the client, sending via email, etc). Otherwise, create
-  // implied result manifest.
-  //
-  status_code sc;
-  vector<manifest_name_value> rvs;
-
-  if (options_->ci_handler_specified ())
-  {
-    using namespace external_handler;
-
-    optional<result_manifest> r (run (options_->ci_handler (),
-                                      options_->ci_handler_argument (),
-                                      dd,
-                                      options_->ci_handler_timeout (),
-                                      error,
-                                      warn,
-                                      verb_ ? &trace : nullptr));
-    if (!r)
-    {
-      stash_submit_dir ();
-      return respond_error (); // The diagnostics is already issued.
-    }
-
-    sc = r->status;
-    rvs = move (r->values);
-  }
-  else // Create the implied result manifest.
-  {
-    sc = 200;
-
-    auto add = [&rvs] (string n, string v)
-    {
-      manifest_name_value nv {
-        move (n), move (v),
-        0 /* name_line */,  0 /* name_column */,
-        0 /* value_line */, 0 /* value_column */,
-        0 /* start_pos */, 0 /* colon_pos */, 0 /* end_pos */};
-
-      rvs.emplace_back (move (nv));
-    };
-
-    add ("status", "200");
-    add ("message", "CI request is queued");
-    add ("reference", request_id);
-  }
-
-  assert (!rvs.empty ()); // Produced by the handler or is implied.
-
-  // Serialize the submission result manifest to a stream. On the
-  // serialization error log the error description and return false, on the
-  // stream error pass through the io_error exception, otherwise return true.
-  //
-  auto rsm = [&rvs, &error, &request_id] (ostream& os,
-                                          bool long_lines = false) -> bool
-  {
-    try
-    {
-      serializer s (os, "result", long_lines);
-      serialize_manifest (s, rvs);
-      return true;
-    }
-    catch (const serialization& e)
-    {
-      error << "ref " << request_id << ": unable to serialize handler's "
-            << "output: " << e;
-      return false;
-    }
-  };
-
-  // If the submission data directory still exists then perform an appropriate
-  // action on it, depending on the submission result status. Note that the
-  // handler could move or remove the directory.
-  //
-  if (dir_exists (dd))
-  {
-    // Remove the directory if the client error is detected.
-    //
-    if (sc >= 400 && sc < 500)
-    {
-      rmdir_r (dd);
-    }
-    //
-    // Otherwise, save the result manifest, into the directory. Also stash the
-    // directory for troubleshooting in case of the server error.
-    //
-    else
-    {
-      path rsf (dd / "result.manifest");
-
-      try
-      {
-        ofdstream os (rsf);
-
-        // Not being able to stash the result manifest is not a reason to
-        // claim the submission failed. The error is logged nevertheless.
-        //
-        rsm (os);
-
-        os.close ();
-      }
-      catch (const io_error& e)
-      {
-        // Not fatal (see above).
-        //
-        error << "unable to write to '" << rsf << "': " << e;
-      }
-
-      if (sc >= 500 && sc < 600)
-        stash_submit_dir ();
-    }
-  }
-
-  // Send email, if configured, and the CI request submission is not simulated.
-  // Use the long lines manifest serialization mode for the convenience of
-  // copying/clicking URLs they contain.
-  //
-  // Note that we don't consider the email sending failure to be a submission
-  // failure as the submission data is successfully persisted and the handler
-  // is successfully executed, if configured. One can argue that email can be
-  // essential for the submission processing and missing it would result in
-  // the incomplete submission. In this case it's natural to assume that the
-  // web server error log is monitored and the email sending failure will be
-  // noticed.
-  //
-  if (options_->ci_email_specified () && simulate.empty ())
-  try
-  {
-    // Redirect the diagnostics to the web server error log.
-    //
-    sendmail sm ([&trace, this] (const char* args[], size_t n)
-                 {
-                   l2 ([&]{trace << process_args {args, n};});
-                 },
-                 2 /* stderr */,
-                 options_->email (),
-                 "CI request submission (" + request_id + ')',
-                 {options_->ci_email ()});
-
-    // Write the CI request manifest.
-    //
-    bool r (rqm (sm.out, true /* long_lines */));
-    assert (r); // The serialization succeeded once, so can't fail now.
-
-    // Write the CI overrides manifest.
-    //
-    sm.out << "\n\n";
-
-    ovm (sm.out, true /* long_lines */);
-
-    // Write the CI result manifest.
-    //
-    sm.out << "\n\n";
-
-    // We don't care about the result (see above).
-    //
-    rsm (sm.out, true /* long_lines */);
-
-    sm.out.close ();
+  return true;
+}
 
-    if (!sm.wait ())
-      error << "sendmail " << *sm.exit;
-  }
-  // Handle process_error and io_error (both derive from system_error).
-  //
-  catch (const system_error& e)
-  {
-    error << "sendmail error: " << e;
-  }
+#ifdef BREP_CI_TENANT_SERVICE
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_queued (const tenant_service&,
+              const vector<build>& bs,
+              optional<build_state> initial_state) const
+{
+  return [&bs, initial_state] (const tenant_service& ts)
+         {
+           optional<string> r (ts.data);
+
+           for (const build& b: bs)
+           {
+             string s ((!initial_state
+                        ? "queued "
+                        : "queued " + to_string (*initial_state) + ' ') +
+                       b.package_name.string () + '/'                   +
+                       b.package_version.string () + '/'                +
+                       b.target.string () + '/'                         +
+                       b.target_config_name + '/'                       +
+                       b.package_config_name + '/'                      +
+                       b.toolchain_name + '/'                           +
+                       b.toolchain_version.string ());
+
+             if (r)
+             {
+               *r += ", ";
+               *r += s;
+             }
+             else
+               r = move (s);
+           }
+
+           return r;
+         };
+}
 
-  if (!rsm (rs.content (sc, "text/manifest;charset=utf-8")))
-    return respond_error (); // The error description is already logged.
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_building (const tenant_service&, const build& b) const
+{
+  return [&b] (const tenant_service& ts)
+         {
+           string s ("building "                       +
+                     b.package_name.string () + '/'    +
+                     b.package_version.string () + '/' +
+                     b.target.string () + '/'          +
+                     b.target_config_name + '/'        +
+                     b.package_config_name + '/'       +
+                     b.toolchain_name + '/'            +
+                     b.toolchain_version.string ());
+
+           return ts.data ? *ts.data + ", " + s : s;
+         };
+}
 
-  return true;
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_built (const tenant_service&, const build& b) const
+{
+  return [&b] (const tenant_service& ts)
+         {
+           string s ("built "                          +
+                     b.package_name.string () + '/'    +
+                     b.package_version.string () + '/' +
+                     b.target.string () + '/'          +
+                     b.target_config_name + '/'        +
+                     b.package_config_name + '/'       +
+                     b.toolchain_name + '/'            +
+                     b.toolchain_version.string ());
+
+           return ts.data ? *ts.data + ", " + s : s;
+         };
 }
+#endif
diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx
index 431f53b..3b1e1be 100644
--- a/mod/mod-ci.hxx
+++ b/mod/mod-ci.hxx
@@ -9,14 +9,39 @@
 #include <libbrep/types.hxx>
 #include <libbrep/utility.hxx>
 
+#include <libbrep/build.hxx>
+#include <libbrep/common.hxx> // tenant_service
+
 #include <mod/module.hxx>
 #include <mod/module-options.hxx>
 
+#include <mod/ci-common.hxx>
+
+#ifdef BREP_CI_TENANT_SERVICE
+#  include <mod/tenant-service.hxx>
+#endif
+
 namespace brep
 {
-  class ci: public handler
+  class ci: public handler,
+            private ci_start
+#ifdef BREP_CI_TENANT_SERVICE
+          , public tenant_service_build_queued,
+            public tenant_service_build_building,
+            public tenant_service_build_built
+#endif
   {
   public:
+
+#ifdef BREP_CI_TENANT_SERVICE
+    explicit
+    ci (tenant_service_map&);
+
+    // Create a shallow copy (handling instance) if initialized and a deep
+    // copy (context exemplar) otherwise.
+    //
+    ci (const ci&, tenant_service_map&);
+#else
     ci () = default;
 
     // Create a shallow copy (handling instance) if initialized and a deep
@@ -24,12 +49,26 @@ namespace brep
     //
     explicit
     ci (const ci&);
+#endif
 
     virtual bool
-    handle (request&, response&);
+    handle (request&, response&) override;
 
     virtual const cli::options&
-    cli_options () const {return options::ci::description ();}
+    cli_options () const override {return options::ci::description ();}
+
+#ifdef BREP_CI_TENANT_SERVICE
+    virtual function<optional<string> (const tenant_service&)>
+    build_queued (const tenant_service&,
+                  const vector<build>&,
+                  optional<build_state> initial_state) const override;
+
+    virtual function<optional<string> (const tenant_service&)>
+    build_building (const tenant_service&, const build&) const override;
+
+    virtual function<optional<string> (const tenant_service&)>
+    build_built (const tenant_service&, const build&) const override;
+#endif
 
   private:
     virtual void
@@ -38,6 +77,10 @@ namespace brep
   private:
     shared_ptr<options::ci> options_;
     shared_ptr<web::xhtml::fragment> form_;
+
+#ifdef BREP_CI_TENANT_SERVICE
+    tenant_service_map& tenant_service_map_;
+#endif
   };
 }
 
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index 35a1a22..51b21c6 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -802,7 +802,7 @@ handle (request& rq, response& rs)
     // Print the package built configurations in the time-descending order.
     //
     for (auto& b: build_db_->query<build> (
-           (query::id.package == pkg->id && sq) +
+           (query::id.package == pkg->id && query::state != "queued" && sq) +
            "ORDER BY" + query::timestamp + "DESC"))
     {
       string ts (butl::to_string (b.timestamp,
diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx
index 1b18996..34b4007 100644
--- a/mod/mod-repository-root.cxx
+++ b/mod/mod-repository-root.cxx
@@ -108,18 +108,31 @@ namespace brep
   //
   repository_root::
   repository_root ()
-      : packages_ (make_shared<packages> ()),
+      :
+        //
+        // Only create and populate the tenant service map in the examplar
+        // passing a reference to it to all the sub-handler exemplars. Note
+        // that we dispatch the tenant service callbacks to the examplar
+        // without creating a new instance for each callback (thus the
+        // callbacks are const).
+        //
+        tenant_service_map_ (make_shared<tenant_service_map> ()),
+        packages_ (make_shared<packages> ()),
         package_details_ (make_shared<package_details> ()),
         package_version_details_ (make_shared<package_version_details> ()),
         repository_details_ (make_shared<repository_details> ()),
-        build_task_ (make_shared<build_task> ()),
-        build_result_ (make_shared<build_result> ()),
-        build_force_ (make_shared<build_force> ()),
+        build_task_ (make_shared<build_task> (*tenant_service_map_)),
+        build_result_ (make_shared<build_result> (*tenant_service_map_)),
+        build_force_ (make_shared<build_force> (*tenant_service_map_)),
         build_log_ (make_shared<build_log> ()),
         builds_ (make_shared<builds> ()),
         build_configs_ (make_shared<build_configs> ()),
         submit_ (make_shared<submit> ()),
+#ifdef BREP_CI_TENANT_SERVICE
+        ci_ (make_shared<ci> (*tenant_service_map_)),
+#else
         ci_ (make_shared<ci> ()),
+#endif
         upload_ (make_shared<upload> ())
   {
   }
@@ -127,6 +140,10 @@ namespace brep
   repository_root::
   repository_root (const repository_root& r)
       : handler (r),
+        tenant_service_map_ (
+          r.initialized_
+          ? r.tenant_service_map_
+          : make_shared<tenant_service_map> ()),
         //
         // Deep/shallow-copy sub-handlers depending on whether this is an
         // exemplar/handler.
@@ -151,15 +168,15 @@ namespace brep
         build_task_ (
           r.initialized_
           ? r.build_task_
-          : make_shared<build_task> (*r.build_task_)),
+          : make_shared<build_task> (*r.build_task_, *tenant_service_map_)),
         build_result_ (
           r.initialized_
           ? r.build_result_
-          : make_shared<build_result> (*r.build_result_)),
+          : make_shared<build_result> (*r.build_result_, *tenant_service_map_)),
         build_force_ (
           r.initialized_
           ? r.build_force_
-          : make_shared<build_force> (*r.build_force_)),
+          : make_shared<build_force> (*r.build_force_, *tenant_service_map_)),
         build_log_ (
           r.initialized_
           ? r.build_log_
@@ -179,7 +196,11 @@ namespace brep
         ci_ (
           r.initialized_
           ? r.ci_
+#ifdef BREP_CI_TENANT_SERVICE
+          : make_shared<ci> (*r.ci_, *tenant_service_map_)),
+#else
           : make_shared<ci> (*r.ci_)),
+#endif
         upload_ (
           r.initialized_
           ? r.upload_
diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx
index 4f40c94..aa60fda 100644
--- a/mod/mod-repository-root.hxx
+++ b/mod/mod-repository-root.hxx
@@ -9,6 +9,7 @@
 
 #include <mod/module.hxx>
 #include <mod/module-options.hxx>
+#include <mod/tenant-service.hxx>
 
 namespace brep
 {
@@ -59,6 +60,8 @@ namespace brep
     version ();
 
   private:
+    shared_ptr<tenant_service_map> tenant_service_map_;
+
     shared_ptr<packages> packages_;
     shared_ptr<package_details> package_details_;
     shared_ptr<package_version_details> package_version_details_;
diff --git a/mod/module.cli b/mod/module.cli
index 3fdd7e3..3e81b38 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -21,7 +21,7 @@ namespace brep
   {
     // Option groups.
     //
-    class handler
+    class repository_email
     {
       string email
       {
@@ -29,7 +29,10 @@ namespace brep
         "Repository email. This email is used for the \cb{From:} header in
          emails send by \cb{brep} (for example, build failure notifications)."
       }
+    };
 
+    class handler
+    {
       string host
       {
         "<host>",
@@ -279,6 +282,15 @@ namespace brep
          the same as for the \cb{build-alt-soft-rebuild-stop} option but
          for the \cb{build-hard-rebuild-timeout} option."
       }
+
+      size_t build-queued-timeout = 30
+      {
+        "<seconds>",
+        "Time to wait before assuming the \cb{queued} notifications are
+         delivered for package CI requests submitted via third-party services
+         (GitHub, etc). During this time a package is not considered for a
+         build. Must be specified in seconds. Default is 30 seconds."
+      }
     };
 
     class build_db
@@ -563,7 +575,7 @@ namespace brep
       }
     };
 
-    class build_result: build, package_db, build_db, handler
+    class build_result: build, package_db, build_db, repository_email, handler
     {
       size_t build-result-request-max-size = 10485760
       {
@@ -626,7 +638,7 @@ namespace brep
       }
     };
 
-    class submit: page, handler
+    class submit: page, repository_email, handler
     {
       dir_path submit-data
       {
@@ -706,7 +718,7 @@ namespace brep
       }
     };
 
-    class ci: page, handler
+    class ci_start: repository_email
     {
       dir_path ci-data
       {
@@ -721,15 +733,6 @@ namespace brep
          granted to the user that runs the web server."
       }
 
-      path ci-form
-      {
-        "<file>",
-        "The package CI form fragment. If specified, then its contents are
-         treated as an XHTML5 fragment that is inserted into the <body>
-         element of the CI page. If unspecified, then no CI page will be
-         displayed. Note that the file path must be absolute."
-      }
-
       string ci-email
       {
         "<email>",
@@ -766,7 +769,33 @@ namespace brep
       }
     };
 
-    class upload: build, build_db, build_upload, handler
+    class ci_cancel
+    {
+    };
+
+    class ci: ci_start, page, handler
+    {
+      // Classic CI-specific options.
+      //
+
+      path ci-form
+      {
+        "<file>",
+        "The package CI form fragment. If specified, then its contents are
+         treated as an XHTML5 fragment that is inserted into the <body>
+         element of the CI page. If unspecified, then no CI page will be
+         displayed. Note that the file path must be absolute."
+      }
+    };
+
+    class ci_github: ci_start, ci_cancel, build_db, handler
+    {
+      // GitHub CI-specific options (e.g., request timeout when invoking
+      // GitHub APIs).
+      //
+    };
+
+    class upload: build, build_db, build_upload, repository_email, handler
     {
     };
 
diff --git a/mod/page.cxx b/mod/page.cxx
index d844a89..5483183 100644
--- a/mod/page.cxx
+++ b/mod/page.cxx
@@ -761,7 +761,6 @@ namespace brep
     }
     else
     {
-
       // If no unsuccessful operation results available, then print the
       // overall build status. If there are any operation results available,
       // then also print unsuccessful operation statuses with the links to the
diff --git a/mod/page.hxx b/mod/page.hxx
index f3c27d5..cac2b8b 100644
--- a/mod/page.hxx
+++ b/mod/page.hxx
@@ -473,7 +473,12 @@ namespace brep
                      bool a,
                      const string& h,
                      const dir_path& r):
-        build_ (b), archived_ (a), host_ (h), root_ (r) {}
+        build_ (b), archived_ (a), host_ (h), root_ (r)
+    {
+      // We don't expect a queued build to ever be displayed.
+      //
+      assert (build_.state != build_state::queued);
+    }
 
     void
     operator() (xml::serializer&) const;
diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx
new file mode 100644
index 0000000..a7bc941
--- /dev/null
+++ b/mod/tenant-service.hxx
@@ -0,0 +1,107 @@
+// file      : mod/tenant-service.hxx -*- C++ -*-
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef MOD_TENANT_SERVICE_HXX
+#define MOD_TENANT_SERVICE_HXX
+
+#include <map>
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+#include <libbrep/build.hxx>
+
+namespace brep
+{
+  class tenant_service_base
+  {
+  public:
+    virtual ~tenant_service_base () = default;
+  };
+
+  // Possible build notifications:
+  //
+  // queued
+  // building
+  // built
+  //
+  // Possible transitions:
+  //
+  //          -> queued
+  // queued   -> building
+  // building -> queued   (interrupted & re-queued due to higher priority task)
+  // building -> built
+  // built    -> queued   (periodic or user-forced rebuild)
+  //
+  // While the implementation tries to make sure the notifications arrive in
+  // the correct order, this is currently done by imposing delays (some
+  // natural, such as building->built, and some artificial, such as
+  // queued->building). As result, it is unlikely but possible to be notified
+  // about the state transitions in the wrong order, especially if the
+  // notifications take a long time. To minimize the chance of this happening,
+  // the service implementation should strive to batch the queued state
+  // notifications (or which there could be hundreds) in a single request if
+  // at all possible. Also, if supported by the third-party API, it makes
+  // sense for the implementation to protect against overwriting later states
+  // with earlier. For example, if it's possible to place a condition on a
+  // notification, it makes sense to only set the state to queued if none of
+  // the later states (e.g., building) are already in effect.
+  //
+  // Note also that it's possible for the build to get deleted at any stage
+  // without any further notifications. This can happen, for example, due to
+  // data retention timeout or because the build configuration (buildtab
+  // entry) is no longer present. There is no explicit `deleted` transition
+  // notification because such situations (i.e., when a notification sequence
+  // is abandoned half way) are not expected to arise ordinarily in a
+  // properly-configured brep instance. And the third-party service is
+  // expected to deal with them using some overall timeout/expiration
+  // mechanism which it presumably has.
+  //
+  // Each build notification is in its own interface since a service may not
+  // be interested in all of them while computing the information to pass is
+  // expensive.
+  //
+  class tenant_service_build_queued: public virtual tenant_service_base
+  {
+  public:
+    // If the returned function is not NULL, it is called to update the
+    // service data. It should return the new data or nullopt if no update is
+    // necessary. Note: tenant_service::data passed to the callback and to the
+    // returned function may not be the same. Also, the returned function may
+    // be called multiple times (on transaction retries).
+    //
+    // The passed initial_state indicates the logical initial state and is
+    // either absent, `building` (interrupted), or `built` (rebuild). Note
+    // that all the passed build objects have the same initial state.
+    //
+    // The implementation of this and the below functions should normally not
+    // need to make any decisions based on the passed build::state. Rather,
+    // the function name suffix (_queued, _building, _built) signify the
+    // logical end state.
+    //
+    virtual function<optional<string> (const tenant_service&)>
+    build_queued (const tenant_service&,
+                  const vector<build>&,
+                  optional<build_state> initial_state) const = 0;
+  };
+
+  class tenant_service_build_building: public virtual tenant_service_base
+  {
+  public:
+    virtual function<optional<string> (const tenant_service&)>
+    build_building (const tenant_service&, const build&) const = 0;
+  };
+
+  class tenant_service_build_built: public virtual tenant_service_base
+  {
+  public:
+    virtual function<optional<string> (const tenant_service&)>
+    build_built (const tenant_service&, const build&) const = 0;
+  };
+
+  // Map of service type (tenant_service::type) to service.
+  //
+  using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>;
+}
+
+#endif // MOD_TENANT_SERVICE_HXX
-- 
cgit v1.1