Add support for build_unloaded() notification for tenant-associated services

10 files changed, 622 insertions, 69 deletions

diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx
index cb61e66..bd283f4 100644
--- a/mod/ci-common.cxx
+++ b/mod/ci-common.cxx
@@ -3,6 +3,9 @@
 
 #include <mod/ci-common.hxx>
 
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
 #include <libbutl/uuid.hxx>
 #include <libbutl/fdstream.hxx>
 #include <libbutl/sendmail.hxx>
@@ -11,6 +14,9 @@
 #include <libbutl/process-io.hxx>          // operator<<(ostream, process_args)
 #include <libbutl/manifest-serializer.hxx>
 
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
+
 #include <mod/external-handler.hxx>
 
 namespace brep
@@ -38,13 +44,16 @@ namespace brep
     options_ = move (o);
   }
 
-  optional<ci_start::start_result> ci_start::
+  static optional<ci_start::start_result>
   start (const basic_mark& error,
          const basic_mark& warn,
          const basic_mark* trace,
+         const options::ci_start& ops,
+         string&& request_id,
          optional<tenant_service>&& service,
+         bool service_load,
          const repository_location& repository,
-         const vector<package>& packages,
+         const vector<ci_start::package>& packages,
          const optional<string>& client_ip,
          const optional<string>& user_agent,
          const optional<string>& interactive,
@@ -55,32 +64,15 @@ namespace brep
     using serializer    = manifest_serializer;
     using serialization = manifest_serialization;
 
-    assert (options_ != nullptr); // Shouldn't be called otherwise.
+    using result = ci_start::start_result;
 
     // If the tenant service is specified, then its type may not be empty.
     //
     assert (!service || !service->type.empty ());
 
-    // Generate the request id.
-    //
-    // Note that it will also be used as a CI result manifest reference,
-    // unless the latter is provided by the external handler.
-    //
-    string request_id;
-
-    try
-    {
-      request_id = uuid::generate ().string ();
-    }
-    catch (const system_error& e)
-    {
-      error << "unable to generate request id: " << e;
-      return nullopt;
-    }
-
     // Create the submission data directory.
     //
-    dir_path dd (options_->ci_data () / dir_path (request_id));
+    dir_path dd (ops.ci_data () / dir_path (request_id));
 
     try
     {
@@ -103,10 +95,10 @@ namespace brep
     //
     auto client_error = [&request_id] (uint16_t status, string message)
     {
-      return start_result {status,
-                           move (message),
-                           request_id,
-                           vector<pair<string, string>> ()};
+      return result {status,
+                     move (message),
+                     request_id,
+                     vector<pair<string, string>> ()};
     };
 
     // Serialize the CI request manifest to a stream. On the serialization
@@ -119,6 +111,7 @@ namespace brep
     auto rqm = [&request_id,
                 &ts,
                 &service,
+                service_load,
                 &repository,
                 &packages,
                 &client_ip,
@@ -127,7 +120,7 @@ namespace brep
                 &simulate,
                 &custom_request,
                 &client_error] (ostream& os, bool long_lines = false)
-      -> pair<bool, optional<start_result>>
+      -> pair<bool, optional<result>>
     {
       try
       {
@@ -139,7 +132,7 @@ namespace brep
         s.next ("id", request_id);
         s.next ("repository", repository.string ());
 
-        for (const package& p: packages)
+        for (const ci_start::package& p: packages)
         {
           if (!p.version)
             s.next ("package", p.name.string ());
@@ -178,6 +171,8 @@ namespace brep
 
           if (service->data)
             s.next ("service-data", *service->data);
+
+          s.next ("service-action", service_load ? "load" : "start");
         }
 
         // Serialize the request custom parameters.
@@ -190,12 +185,12 @@ namespace brep
           s.next (nv.first, nv.second);
 
         s.next ("", ""); // End of manifest.
-        return make_pair (true, optional<start_result> ());
+        return make_pair (true, optional<result> ());
       }
       catch (const serialization& e)
       {
         return make_pair (false,
-                          optional<start_result> (
+                          optional<result> (
                             client_error (400,
                                           string ("invalid parameter: ") +
                                           e.what ())));
@@ -209,7 +204,7 @@ namespace brep
     try
     {
       ofdstream os (rqf);
-      pair<bool, optional<start_result>> r (rqm (os));
+      pair<bool, optional<result>> r (rqm (os));
       os.close ();
 
       if (!r.first)
@@ -228,7 +223,7 @@ namespace brep
     //
     auto ovm = [&overrides, &client_error] (ostream& os,
                                             bool long_lines = false)
-      -> pair<bool, optional<start_result>>
+      -> pair<bool, optional<result>>
     {
       try
       {
@@ -240,12 +235,12 @@ namespace brep
           s.next (nv.first, nv.second);
 
         s.next ("", ""); // End of manifest.
-        return make_pair (true, optional<start_result> ());
+        return make_pair (true, optional<result> ());
       }
       catch (const serialization& e)
       {
         return make_pair (false,
-                          optional<start_result> (
+                          optional<result> (
                             client_error (
                               400,
                               string ("invalid manifest override: ") +
@@ -261,7 +256,7 @@ namespace brep
     try
     {
       ofdstream os (ovf);
-      pair<bool, optional<start_result>> r (ovm (os));
+      pair<bool, optional<result>> r (ovm (os));
       os.close ();
 
       if (!r.first)
@@ -305,16 +300,16 @@ namespace brep
     // manifest from its stdout and parse it into the resulting manifest
     // object. Otherwise, create implied CI result manifest.
     //
-    start_result sr;
+    result sr;
 
-    if (options_->ci_handler_specified ())
+    if (ops.ci_handler_specified ())
     {
       using namespace external_handler;
 
-      optional<result_manifest> r (run (options_->ci_handler (),
-                                        options_->ci_handler_argument (),
+      optional<result_manifest> r (run (ops.ci_handler (),
+                                        ops.ci_handler_argument (),
                                         dd,
-                                        options_->ci_handler_timeout (),
+                                        ops.ci_handler_timeout (),
                                         error,
                                         warn,
                                         trace));
@@ -358,7 +353,7 @@ namespace brep
     {
       try
       {
-        serialize_manifest (sr, os, long_lines);
+        ci_start::serialize_manifest (sr, os, long_lines);
         return true;
       }
       catch (const serialization& e)
@@ -424,7 +419,7 @@ namespace brep
     // assume that the web server error log is monitored and the email sending
     // failure will be noticed.
     //
-    if (options_->ci_email_specified () && !simulate)
+    if (ops.ci_email_specified () && !simulate)
     try
     {
       // Redirect the diagnostics to the web server error log.
@@ -435,14 +430,13 @@ namespace brep
                        *trace << process_args {args, n};
                    },
                    2 /* stderr */,
-                   options_->email (),
+                   ops.email (),
                    "CI request submission (" + sr.reference + ')',
-                   {options_->ci_email ()});
+                   {ops.ci_email ()});
 
       // Write the CI request manifest.
       //
-      pair<bool, optional<start_result>> r (
-        rqm (sm.out, true /* long_lines */));
+      pair<bool, optional<result>> r (rqm (sm.out, true /* long_lines */));
 
       assert (r.first); // The serialization succeeded once, so can't fail now.
 
@@ -473,7 +467,55 @@ namespace brep
       error << "sendmail error: " << e;
     }
 
-    return optional<start_result> (move (sr));
+    return optional<result> (move (sr));
+  }
+
+  optional<ci_start::start_result> ci_start::
+  start (const basic_mark& error,
+         const basic_mark& warn,
+         const basic_mark* trace,
+         optional<tenant_service>&& service,
+         const repository_location& repository,
+         const vector<package>& packages,
+         const optional<string>& client_ip,
+         const optional<string>& user_agent,
+         const optional<string>& interactive,
+         const optional<string>& simulate,
+         const vector<pair<string, string>>& custom_request,
+         const vector<pair<string, string>>& overrides) const
+  {
+    assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+    // Generate the request id.
+    //
+    // Note that it will also be used as a CI result manifest reference,
+    // unless the latter is provided by the external handler.
+    //
+    string request_id;
+
+    try
+    {
+      request_id = uuid::generate ().string ();
+    }
+    catch (const system_error& e)
+    {
+      error << "unable to generate request id: " << e;
+      return nullopt;
+    }
+
+    return brep::start (error, warn, trace,
+                        *options_,
+                        move (request_id),
+                        move (service),
+                        false /* service_load */,
+                        repository,
+                        packages,
+                        client_ip,
+                        user_agent,
+                        interactive,
+                        simulate,
+                        custom_request,
+                        overrides);
   }
 
   void ci_start::
@@ -491,4 +533,206 @@ namespace brep
 
     s.next ("", "");                         // End of manifest.
   }
+
+  optional<string> ci_start::
+  create (const basic_mark& error,
+          const basic_mark&,
+          const basic_mark* trace,
+          odb::core::database& db,
+          tenant_service&& service) const
+  {
+    using namespace odb::core;
+
+    // Generate the request id.
+    //
+    string request_id;
+
+    try
+    {
+      request_id = uuid::generate ().string ();
+    }
+    catch (const system_error& e)
+    {
+      error << "unable to generate request id: " << e;
+      return nullopt;
+    }
+
+    // Use the generated request id if the tenant service id is not specified.
+    //
+    if (service.id.empty ())
+      service.id = request_id;
+
+    // Delay the first load attempt for 10 seconds (see mod-build-task.cxx for
+    // details).
+    //
+    build_tenant t (move (request_id),
+                    move (service),
+                    system_clock::now () - chrono::seconds (40 - 10));
+
+    {
+      assert (!transaction::has_current ());
+
+      transaction tr (db.begin ());
+
+      // Note that in contrast to brep-load, we know that the tenant id is
+      // unique and thus we don't try to remove a tenant with such an id.
+      // There is also not much reason to assume that we may have switched
+      // from the single-tenant mode here and remove the respective tenant,
+      // unless we are in the tenant-service functionality development mode.
+      //
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+      cstrings ts ({""});
+
+      db.erase_query<build_package> (
+        query<build_package>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+      db.erase_query<build_repository> (
+        query<build_repository>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+      db.erase_query<build_public_key> (
+        query<build_public_key>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+      db.erase_query<build_tenant> (
+        query<build_tenant>::id.in_range (ts.begin (), ts.end ()));
+#endif
+
+      db.persist (t);
+
+      tr.commit ();
+    }
+
+    if (trace != nullptr)
+      *trace << "unloaded CI request " << t.id << " for service "
+             << t.service->id << ' ' << t.service->type << " is created";
+
+    return move (t.id);
+  }
+
+  optional<ci_start::start_result> ci_start::
+  load (const basic_mark& error,
+        const basic_mark& warn,
+        const basic_mark* trace,
+        odb::core::database& db,
+        tenant_service&& service,
+        const repository_location& repository) const
+  {
+    using namespace odb::core;
+
+    string request_id;
+    {
+      assert (!transaction::has_current ());
+
+      transaction tr (db.begin ());
+
+      using query = query<build_tenant>;
+
+      shared_ptr<build_tenant> t (
+        db.query_one<build_tenant> (query::service.id == service.id &&
+                                    query::service.type == service.type));
+
+      if (t == nullptr)
+      {
+        error << "unable to find tenant for service " << service.id << ' '
+              << service.type;
+
+        return nullopt;
+      }
+      else if (t->archived)
+      {
+        error << "tenant " << t->id << " for service " << service.id << ' '
+              << service.type << " is already archived";
+
+        return nullopt;
+      }
+      else if (!t->loaded_timestamp)
+      {
+        error << "tenant " << t->id << " for service " << service.id << ' '
+              << service.type << " is already loaded";
+
+        return nullopt;
+      }
+
+      t->loaded_timestamp = nullopt;
+      db.update (t);
+
+      tr.commit ();
+
+      request_id = move (t->id);
+    }
+
+    assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+    optional<start_result> r (brep::start (error, warn, trace,
+                                           *options_,
+                                           move (request_id),
+                                           move (service),
+                                           true /* service_load */,
+                                           repository,
+                                           {} /* packages */,
+                                           nullopt /* client_ip */,
+                                           nullopt /* user_agent */,
+                                           nullopt /* interactive */,
+                                           nullopt /* simulate */,
+                                           {} /* custom_request */,
+                                           {} /* overrides */));
+
+    // Note: on error (r == nullopt) the diagnostics is already issued.
+    //
+    if (trace != nullptr && r)
+      *trace << "CI request for '" << repository << "' is "
+             << (r->status != 200 ? "not " : "") << "loaded: "
+             << r->message << " (reference: " << r->reference << ')';
+
+    return r;
+  }
+
+  void ci_start::
+  abandon (const basic_mark& error,
+           const basic_mark&,
+           const basic_mark* trace,
+           odb::core::database& db,
+           tenant_service&& service) const
+  {
+    using namespace odb::core;
+
+    assert (!transaction::has_current ());
+
+    transaction tr (db.begin ());
+
+    using query = query<build_tenant>;
+
+    shared_ptr<build_tenant> t (
+      db.query_one<build_tenant> (query::service.id == service.id &&
+                                  query::service.type == service.type));
+
+    if (t == nullptr)
+    {
+      error << "unable to find tenant for service " << service.id << ' '
+            << service.type;
+
+      return;
+    }
+    else if (!t->loaded_timestamp)
+    {
+      error << "tenant " << t->id << " for service " << service.id << ' '
+            << service.type << " is already loaded";
+
+      return;
+    }
+
+    // We could probably remove the tenant from the database, but let's just
+    // archive it and keep for troubleshooting.
+    //
+    if (!t->archived)
+    {
+      t->archived = true;
+      db.update (t);
+    }
+
+    tr.commit ();
+
+    if (trace != nullptr)
+      *trace << "unloaded CI request " << t->id << " for service "
+             << service.id << ' ' << service.type << " is abandoned";
+  }
 }
diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx
index 6f62c4b..45c959a 100644
--- a/mod/ci-common.hxx
+++ b/mod/ci-common.hxx
@@ -36,6 +36,7 @@ namespace brep
       package_name name;
       optional<brep::version> version;
     };
+
     // Note that the inability to generate the reference is an internal
     // error. Thus, it is not optional.
     //
@@ -62,7 +63,47 @@ namespace brep
            const optional<string>& interactive = nullopt,
            const optional<string>& simulate = nullopt,
            const vector<pair<string, string>>& custom_request = {},
-           const vector<pair<string, string>>& overrides = {});
+           const vector<pair<string, string>>& overrides = {}) const;
+
+    // Create an unloaded CI request returning start_result::reference on
+    // success and nullopt on an internal error. Such a request is not started
+    // until loaded with the load() function below. See also the
+    // build_unloaded() tenant services notification.
+    //
+    // Note: should be called out of the database transaction.
+    //
+    optional<string>
+    create (const basic_mark& error,
+            const basic_mark& warn,
+            const basic_mark* trace,
+            odb::core::database&,
+            tenant_service&&) const;
+
+    // Load (and start) previously created (as unloaded) CI request. Similarly
+    // to the start() function, return nullopt on an internal error.
+    //
+    // Note that tenant_service::id is used to identify the CI request tenant.
+    //
+    // Note: should be called out of the database transaction.
+    //
+    optional<start_result>
+    load (const basic_mark& error,
+          const basic_mark& warn,
+          const basic_mark* trace,
+          odb::core::database&,
+          tenant_service&&,
+          const repository_location& repository) const;
+
+    // Abandon previously created (as unloaded) CI request.
+    //
+    // Note: should be called out of the database transaction.
+    //
+    void
+    abandon (const basic_mark& error,
+             const basic_mark& warn,
+             const basic_mark* trace,
+             odb::core::database&,
+             tenant_service&&) const;
 
     // Helpers.
     //
diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx
index bdae356..168a835 100644
--- a/mod/mod-build-force.cxx
+++ b/mod/mod-build-force.cxx
@@ -192,7 +192,14 @@ handle (request& rq, response& rs)
   optional<pair<tenant_service, shared_ptr<build>>> tss;
   tenant_service_build_queued::build_queued_hints qhs;
 
+  // Acquire the database connection for the subsequent transactions.
+  //
+  // Note that we will release it prior to any potentially time-consuming
+  // operations (such as HTTP requests) and re-acquire it again afterwards,
+  // if required.
+  //
   connection_ptr conn (build_db_->connection ());
+
   {
     transaction t (conn->begin ());
 
@@ -297,14 +304,28 @@ handle (request& rq, response& rs)
     vector<build> qbs;
     qbs.push_back (move (b));
 
+    // Release the database connection since the build_queued() notification
+    // can potentially be time-consuming (e.g., it may perform an HTTP
+    // request).
+    //
+    conn.reset ();
+
     if (auto f = tsq->build_queued (ss,
                                     qbs,
                                     build_state::building,
                                     qhs,
                                     log_writer_))
+    {
+      conn = build_db_->connection ();
       update_tenant_service_state (conn, qbs.back ().tenant, f);
+    }
   }
 
+  // Release the database connection prior to writing into the unbuffered
+  // response stream.
+  //
+  conn.reset ();
+
   // We have all the data, so don't buffer the response content.
   //
   ostream& os (rs.content (200, "text/plain;charset=utf-8", false));
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index ccce17f..64503aa 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -207,13 +207,20 @@ handle (request& rq, response&)
   optional<pair<tenant_service, shared_ptr<build>>> tss;
   tenant_service_build_queued::build_queued_hints qhs;
 
+  // Acquire the database connection for the subsequent transactions.
+  //
+  // Note that we will release it prior to any potentially time-consuming
+  // operations (such as HTTP requests) and re-acquire it again afterwards,
+  // if required.
+  //
+  connection_ptr conn (build_db_->connection ());
+
   // Note that if the session authentication fails (probably due to the
   // authentication settings change), then we log this case with the warning
   // severity and respond with the 200 HTTP code as if the challenge is
   // valid. The thinking is that we shouldn't alarm a law-abaiding agent and
   // shouldn't provide any information to a malicious one.
   //
-  connection_ptr conn (build_db_->connection ());
   {
     transaction t (conn->begin ());
 
@@ -518,12 +525,20 @@ handle (request& rq, response&)
     vector<build> qbs;
     qbs.push_back (move (*tss->second));
 
+    // Release the database connection since build_queued() notification can
+    // potentially be time-consuming (e.g., it may perform an HTTP request).
+    //
+    conn.reset ();
+
     if (auto f = tsq->build_queued (ss,
                                     qbs,
                                     build_state::building,
                                     qhs,
                                     log_writer_))
+    {
+      conn = build_db_->connection ();
       update_tenant_service_state (conn, qbs.back ().tenant, f);
+    }
   }
 
   // If a third-party service needs to be notified about the built package,
@@ -537,8 +552,16 @@ handle (request& rq, response&)
     const tenant_service& ss (tss->first);
     const build& b (*tss->second);
 
+    // Release the database connection since build_built() notification can
+    // potentially be time-consuming (e.g., it may perform an HTTP request).
+    //
+    conn.reset ();
+
     if (auto f = tsb->build_built (ss, b, log_writer_))
+    {
+      conn = build_db_->connection ();
       update_tenant_service_state (conn, b.tenant, f);
+    }
   }
 
   if (bld != nullptr)
@@ -549,6 +572,9 @@ handle (request& rq, response&)
     if (!build_notify)
       (cfg->email ? cfg->email : pkg->build_email) = email ();
 
+    if (conn == nullptr)
+      conn = build_db_->connection ();
+
     send_notification_email (*options_,
                              conn,
                              *bld,
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 07aff8d..0752603 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -399,6 +399,92 @@ handle (request& rq, response& rs)
     }
   }
 
+  // Acquire the database connection for the subsequent transactions.
+  //
+  // Note that we will release it prior to any potentially time-consuming
+  // operations (such as HTTP requests) and re-acquire it again afterwards,
+  // if required.
+  //
+  connection_ptr conn (build_db_->connection ());
+
+  timestamp now (system_clock::now ());
+
+  auto expiration = [&now] (size_t timeout) -> timestamp
+  {
+    return now - chrono::seconds (timeout);
+  };
+
+  auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t
+  {
+    return chrono::duration_cast<chrono::nanoseconds> (
+      expiration (timeout).time_since_epoch ()).count ();
+  };
+
+  // Perform some housekeeping first.
+  //
+  // Notify a tenant-associated third-party service about the unloaded CI
+  // request, if present.
+  //
+  {
+    const tenant_service_build_unloaded* tsu (nullptr);
+
+    transaction tr (conn->begin ());
+
+    using query = query<build_tenant>;
+
+    // Pick the unloaded tenant with the earliest loaded timestamp, skipping
+    // those whose timestamp is less than 40 seconds ago.
+    //
+    // NOTE: don't forget to update ci_start::create() if changing the timeout
+    // here.
+    //
+    shared_ptr<build_tenant> t (
+      build_db_->query_one<build_tenant> (
+        (query::loaded_timestamp <= expiration_ns (40) &&
+         !query::archived)                   +
+        "ORDER BY" + query::loaded_timestamp +
+        "LIMIT 1"));
+
+    if (t != nullptr && t->service)
+    {
+      auto i (tenant_service_map_.find (t->service->type));
+
+      if (i != tenant_service_map_.end ())
+      {
+        tsu = dynamic_cast<const tenant_service_build_unloaded*> (
+          i->second.get ());
+
+        if (tsu != nullptr)
+        {
+          // If we ought to call the
+          // tenant_service_build_unloaded::build_unloaded() callback, then
+          // set the package tenant's loaded timestamp to the current time to
+          // prevent the notifications race.
+          //
+          t->loaded_timestamp = system_clock::now ();
+          build_db_->update (t);
+        }
+      }
+    }
+
+    tr.commit ();
+
+    if (tsu != nullptr)
+    {
+      // Release the database connection since the build_unloaded()
+      // notification can potentially be time-consuming (e.g., it may perform
+      // an HTTP request).
+      //
+      conn.reset ();
+
+      if (auto f = tsu->build_unloaded (move (*t->service), log_writer_))
+      {
+        conn = build_db_->connection ();
+        update_tenant_service_state (conn, t->id, f);
+      }
+    }
+  }
+
   // Go through package build configurations until we find one that has no
   // build target configuration present in the database, or is in the building
   // state but expired (collectively called unbuilt). If such a target
@@ -524,19 +610,6 @@ handle (request& rq, response& rs)
     // Calculate the build/rebuild (building/built state) and the `queued`
     // notifications expiration time for package configurations.
     //
-    timestamp now (system_clock::now ());
-
-    auto expiration = [&now] (size_t timeout) -> timestamp
-    {
-      return now - chrono::seconds (timeout);
-    };
-
-    auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t
-    {
-      return chrono::duration_cast<chrono::nanoseconds> (
-        expiration (timeout).time_since_epoch ()).count ();
-    };
-
     uint64_t normal_result_expiration_ns (
       expiration_ns (options_->build_result_timeout ()));
 
@@ -825,7 +898,10 @@ handle (request& rq, response& rs)
                                                        imode,
                                                        queued_expiration_ns));
 
-      transaction t (build_db_->begin ());
+      if (conn == nullptr)
+        conn = build_db_->connection ();
+
+      transaction t (conn->begin ());
 
       // If there are any non-archived interactive build tenants, then the
       // chosen randomization approach doesn't really work since interactive
@@ -886,7 +962,8 @@ handle (request& rq, response& rs)
         "OFFSET" + pkg_query::_ref (offset)                            +
         "LIMIT" + pkg_query::_ref (limit);
 
-      connection_ptr conn (build_db_->connection ());
+      if (conn == nullptr)
+        conn = build_db_->connection ();
 
       prep_pkg_query pkg_prep_query (
         conn->prepare_query<buildable_package> (
@@ -2226,12 +2303,20 @@ handle (request& rq, response& rs)
 
         if (!qbs.empty ())
         {
+          // Release the database connection since the build_queued()
+          // notification can potentially be time-consuming (e.g., it may
+          // perform an HTTP request).
+          //
+          conn.reset ();
+
           if (auto f = tsq->build_queued (ss,
                                           qbs,
                                           nullopt /* initial_state */,
                                           qhs,
                                           log_writer_))
           {
+            conn = build_db_->connection ();
+
             if (optional<string> data =
                 update_tenant_service_state (conn, qbs.back ().tenant, f))
               ss.data = move (data);
@@ -2250,12 +2335,20 @@ handle (request& rq, response& rs)
           qbs.push_back (move (b));
           restore_build = true;
 
+          // Release the database connection since the build_queued()
+          // notification can potentially be time-consuming (e.g., it may
+          // perform an HTTP request).
+          //
+          conn.reset ();
+
           if (auto f = tsq->build_queued (ss,
                                           qbs,
                                           initial_state,
                                           qhs,
                                           log_writer_))
           {
+            conn = build_db_->connection ();
+
             if (optional<string> data =
                 update_tenant_service_state (conn, qbs.back ().tenant, f))
               ss.data = move (data);
@@ -2278,8 +2371,16 @@ handle (request& rq, response& rs)
         tenant_service& ss (tss->first);
         const build& b (*tss->second);
 
+        // Release the database connection since the build_building()
+        // notification can potentially be time-consuming (e.g., it may
+        // perform an HTTP request).
+        //
+        conn.reset ();
+
         if (auto f = tsb->build_building (ss, b, log_writer_))
         {
+          conn = build_db_->connection ();
+
           if (optional<string> data =
               update_tenant_service_state (conn, b.tenant, f))
             ss.data = move (data);
@@ -2306,6 +2407,9 @@ handle (request& rq, response& rs)
         const tenant_service_build_built* tsb (nullptr);
         optional<pair<tenant_service, shared_ptr<build>>> tss;
         {
+          if (conn == nullptr)
+            conn = build_db_->connection ();
+
           transaction t (conn->begin ());
 
           shared_ptr<build> b (build_db_->find<build> (task_build->id));
@@ -2395,8 +2499,16 @@ handle (request& rq, response& rs)
           tenant_service& ss (tss->first);
           const build& b (*tss->second);
 
+          // Release the database connection since the build_built()
+          // notification can potentially be time-consuming (e.g., it may
+          // perform an HTTP request).
+          //
+          conn.reset ();
+
           if (auto f = tsb->build_built (ss, b, log_writer_))
           {
+            conn = build_db_->connection ();
+
             if (optional<string> data =
                 update_tenant_service_state (conn, b.tenant, f))
               ss.data = move (data);
@@ -2407,6 +2519,10 @@ handle (request& rq, response& rs)
       // Send notification emails for all the aborted builds.
       //
       for (const aborted_build& ab: aborted_builds)
+      {
+        if (conn == nullptr)
+          conn = build_db_->connection ();
+
         send_notification_email (*options_,
                                  conn,
                                  *ab.b,
@@ -2415,9 +2531,14 @@ handle (request& rq, response& rs)
                                  ab.what,
                                  error,
                                  verb_ >= 2 ? &trace : nullptr);
+      }
     }
   }
 
+  // Release the database connection as soon as possible.
+  //
+  conn.reset ();
+
   serialize_task_response_manifest ();
   return true;
 }
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index 5974d45..18e38fc 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -36,7 +36,12 @@ ci (const ci& r, tenant_service_map& tsm)
 #else
 ci (const ci& r)
 #endif
-    : handler (r),
+    :
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
+      handler (r),
+  #else
+      database_module (r),
+#endif
       ci_start (r),
       options_ (r.initialized_ ? r.options_ : nullptr),
       form_ (r.initialized_ || r.form_ == nullptr
@@ -100,6 +105,13 @@ init (scanner& s)
     }
   }
 
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+  if (!options_->build_config_specified ())
+    fail << "package building functionality must be enabled";
+
+  database_module::init (*options_, options_->build_db_retry ());
+#endif
+
   if (options_->root ().empty ())
     options_->root (dir_path ("/"));
 }
@@ -347,6 +359,7 @@ handle (request& rq, response& rs)
       user_agent = h.value;
   }
 
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
   optional<start_result> r (start (error,
                                    warn,
                                    verb_ ? &trace : nullptr,
@@ -367,6 +380,23 @@ handle (request& rq, response& rs)
                                     : optional<string> ()),
                                    custom_request,
                                    overrides));
+#else
+  assert (build_db_ != nullptr); // Wouldn't be here otherwise.
+
+  optional<start_result> r;
+
+  if (optional<string> ref = create (error,
+                                     warn,
+                                     verb_ ? &trace : nullptr,
+                                     *build_db_,
+                                     tenant_service ("", "ci", rl.string ())))
+  {
+    string msg ("unloaded CI request is created: " +
+                options_->host () + tenant_dir (root, *ref).string ());
+
+    r = start_result {200, move (msg), move (*ref), {}};
+  }
+#endif
 
   if (!r)
     return respond_error (); // The diagnostics is already issued.
@@ -472,4 +502,35 @@ build_built (const tenant_service&,
            return ts.data ? *ts.data + ", " + s : s;
          };
 }
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_unloaded (tenant_service&& ts,
+                const diag_epilogue& log_writer) const noexcept
+{
+  NOTIFICATION_DIAG (log_writer);
+
+  assert (ts.data); // Repository location.
+
+  try
+  {
+    repository_location rl (*ts.data);
+
+    if (!load (error, warn, verb_ ? &trace : nullptr,
+               *build_db_,
+               move (ts),
+               rl))
+      return nullptr; // The diagnostics is already issued.
+  }
+  catch (const invalid_argument& e)
+  {
+    error << "invalid repository location '" << *ts.data << "' stored for "
+          << "tenant service " << ts.id << ' ' << ts.type;
+
+    return nullptr;
+  }
+
+  return [] (const tenant_service& ts) {return "loaded " + *ts.data;};
+}
+#endif
 #endif
diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx
index 1e2ee15..a83b9d3 100644
--- a/mod/mod-ci.hxx
+++ b/mod/mod-ci.hxx
@@ -17,18 +17,34 @@
 
 #include <mod/ci-common.hxx>
 
+#if defined(BREP_CI_TENANT_SERVICE_UNLOADED) && !defined(BREP_CI_TENANT_SERVICE)
+#  error BREP_CI_TENANT_SERVICE must be defined if BREP_CI_TENANT_SERVICE_UNLOADED is defined
+#endif
+
 #ifdef BREP_CI_TENANT_SERVICE
 #  include <mod/tenant-service.hxx>
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+#  include <mod/database-module.hxx>
+#endif
 #endif
 
 namespace brep
 {
-  class ci: public handler,
+  class ci:
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
+            public handler,
+#else
+            public database_module,
+#endif
             private ci_start
 #ifdef BREP_CI_TENANT_SERVICE
           , public tenant_service_build_queued,
             public tenant_service_build_building,
             public tenant_service_build_built
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+          , tenant_service_build_unloaded
+#endif
 #endif
   {
   public:
@@ -74,6 +90,12 @@ namespace brep
     build_built (const tenant_service&,
                  const build&,
                  const diag_epilogue& log_writer) const noexcept override;
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+    virtual function<optional<string> (const tenant_service&)>
+    build_unloaded (tenant_service&&,
+                    const diag_epilogue& log_writer) const noexcept override;
+#endif
 #endif
 
   private:
diff --git a/mod/module.cli b/mod/module.cli
index a107ffe..5f63930 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -800,7 +800,7 @@ namespace brep
     {
     };
 
-    class ci: ci_start, page, repository_url, handler
+    class ci: ci_start, build, build_db, page, repository_url, handler
     {
       // Classic CI-specific options.
       //
@@ -815,7 +815,7 @@ namespace brep
       }
     };
 
-    class ci_github: ci_start, ci_cancel, build_db, handler
+    class ci_github: ci_start, ci_cancel, build, build_db, handler
     {
       // GitHub CI-specific options (e.g., request timeout when invoking
       // GitHub APIs).
diff --git a/mod/page.cxx b/mod/page.cxx
index bc2e42d..177fb64 100644
--- a/mod/page.cxx
+++ b/mod/page.cxx
@@ -739,7 +739,7 @@ namespace brep
       << ~TR;
   }
 
-  // BUILD_RESULT
+  // TR_BUILD_RESULT
   //
   void TR_BUILD_RESULT::
   operator() (serializer& s) const
diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx
index 9205f76..b7f5c02 100644
--- a/mod/tenant-service.hxx
+++ b/mod/tenant-service.hxx
@@ -21,7 +21,8 @@ namespace brep
     virtual ~tenant_service_base () = default;
   };
 
-  // Possible build notifications:
+  // Possible build notifications (see also the unloaded special notification
+  // below):
   //
   // queued
   // building
@@ -121,6 +122,22 @@ namespace brep
                  const diag_epilogue& log_writer) const noexcept = 0;
   };
 
+  // This notification is only made on unloaded CI requests created with the
+  // ci_start::create() call and until they are loaded with ci_start::load()
+  // or, alternatively, abandoned with ci_start::abandon().
+  //
+  // Note: make sure the implementation of this notification does not take
+  // too long (currently 40 seconds) to avoid nested notifications. Note
+  // also that the first notification is delayed (currently 10 seconds).
+  //
+  class tenant_service_build_unloaded: public virtual tenant_service_base
+  {
+  public:
+    virtual function<optional<string> (const tenant_service&)>
+    build_unloaded (tenant_service&&,
+                    const diag_epilogue& log_writer) const noexcept = 0;
+  };
+
   // Map of service type (tenant_service::type) to service.
   //
   using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>;