From bad54b28bcc59fe5d19ecaf486f52e6359009e68 Mon Sep 17 00:00:00 2001
From: Karen Arutyunov <karen@codesynthesis.com>
Date: Tue, 21 May 2024 13:48:43 +0300
Subject: Add support for build_unloaded() notification for tenant-associated
 services

---
 mod/ci-common.cxx | 336 ++++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 290 insertions(+), 46 deletions(-)

(limited to 'mod/ci-common.cxx')

diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx
index cb61e66..bd283f4 100644
--- a/mod/ci-common.cxx
+++ b/mod/ci-common.cxx
@@ -3,6 +3,9 @@
 
 #include <mod/ci-common.hxx>
 
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
 #include <libbutl/uuid.hxx>
 #include <libbutl/fdstream.hxx>
 #include <libbutl/sendmail.hxx>
@@ -11,6 +14,9 @@
 #include <libbutl/process-io.hxx>          // operator<<(ostream, process_args)
 #include <libbutl/manifest-serializer.hxx>
 
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
+
 #include <mod/external-handler.hxx>
 
 namespace brep
@@ -38,13 +44,16 @@ namespace brep
     options_ = move (o);
   }
 
-  optional<ci_start::start_result> ci_start::
+  static optional<ci_start::start_result>
   start (const basic_mark& error,
          const basic_mark& warn,
          const basic_mark* trace,
+         const options::ci_start& ops,
+         string&& request_id,
          optional<tenant_service>&& service,
+         bool service_load,
          const repository_location& repository,
-         const vector<package>& packages,
+         const vector<ci_start::package>& packages,
          const optional<string>& client_ip,
          const optional<string>& user_agent,
          const optional<string>& interactive,
@@ -55,32 +64,15 @@ namespace brep
     using serializer    = manifest_serializer;
     using serialization = manifest_serialization;
 
-    assert (options_ != nullptr); // Shouldn't be called otherwise.
+    using result = ci_start::start_result;
 
     // If the tenant service is specified, then its type may not be empty.
     //
     assert (!service || !service->type.empty ());
 
-    // Generate the request id.
-    //
-    // Note that it will also be used as a CI result manifest reference,
-    // unless the latter is provided by the external handler.
-    //
-    string request_id;
-
-    try
-    {
-      request_id = uuid::generate ().string ();
-    }
-    catch (const system_error& e)
-    {
-      error << "unable to generate request id: " << e;
-      return nullopt;
-    }
-
     // Create the submission data directory.
     //
-    dir_path dd (options_->ci_data () / dir_path (request_id));
+    dir_path dd (ops.ci_data () / dir_path (request_id));
 
     try
     {
@@ -103,10 +95,10 @@ namespace brep
     //
     auto client_error = [&request_id] (uint16_t status, string message)
     {
-      return start_result {status,
-                           move (message),
-                           request_id,
-                           vector<pair<string, string>> ()};
+      return result {status,
+                     move (message),
+                     request_id,
+                     vector<pair<string, string>> ()};
     };
 
     // Serialize the CI request manifest to a stream. On the serialization
@@ -119,6 +111,7 @@ namespace brep
     auto rqm = [&request_id,
                 &ts,
                 &service,
+                service_load,
                 &repository,
                 &packages,
                 &client_ip,
@@ -127,7 +120,7 @@ namespace brep
                 &simulate,
                 &custom_request,
                 &client_error] (ostream& os, bool long_lines = false)
-      -> pair<bool, optional<start_result>>
+      -> pair<bool, optional<result>>
     {
       try
       {
@@ -139,7 +132,7 @@ namespace brep
         s.next ("id", request_id);
         s.next ("repository", repository.string ());
 
-        for (const package& p: packages)
+        for (const ci_start::package& p: packages)
         {
           if (!p.version)
             s.next ("package", p.name.string ());
@@ -178,6 +171,8 @@ namespace brep
 
           if (service->data)
             s.next ("service-data", *service->data);
+
+          s.next ("service-action", service_load ? "load" : "start");
         }
 
         // Serialize the request custom parameters.
@@ -190,12 +185,12 @@ namespace brep
           s.next (nv.first, nv.second);
 
         s.next ("", ""); // End of manifest.
-        return make_pair (true, optional<start_result> ());
+        return make_pair (true, optional<result> ());
       }
       catch (const serialization& e)
       {
         return make_pair (false,
-                          optional<start_result> (
+                          optional<result> (
                             client_error (400,
                                           string ("invalid parameter: ") +
                                           e.what ())));
@@ -209,7 +204,7 @@ namespace brep
     try
     {
       ofdstream os (rqf);
-      pair<bool, optional<start_result>> r (rqm (os));
+      pair<bool, optional<result>> r (rqm (os));
       os.close ();
 
       if (!r.first)
@@ -228,7 +223,7 @@ namespace brep
     //
     auto ovm = [&overrides, &client_error] (ostream& os,
                                             bool long_lines = false)
-      -> pair<bool, optional<start_result>>
+      -> pair<bool, optional<result>>
     {
       try
       {
@@ -240,12 +235,12 @@ namespace brep
           s.next (nv.first, nv.second);
 
         s.next ("", ""); // End of manifest.
-        return make_pair (true, optional<start_result> ());
+        return make_pair (true, optional<result> ());
       }
       catch (const serialization& e)
       {
         return make_pair (false,
-                          optional<start_result> (
+                          optional<result> (
                             client_error (
                               400,
                               string ("invalid manifest override: ") +
@@ -261,7 +256,7 @@ namespace brep
     try
     {
       ofdstream os (ovf);
-      pair<bool, optional<start_result>> r (ovm (os));
+      pair<bool, optional<result>> r (ovm (os));
       os.close ();
 
       if (!r.first)
@@ -305,16 +300,16 @@ namespace brep
     // manifest from its stdout and parse it into the resulting manifest
     // object. Otherwise, create implied CI result manifest.
     //
-    start_result sr;
+    result sr;
 
-    if (options_->ci_handler_specified ())
+    if (ops.ci_handler_specified ())
     {
       using namespace external_handler;
 
-      optional<result_manifest> r (run (options_->ci_handler (),
-                                        options_->ci_handler_argument (),
+      optional<result_manifest> r (run (ops.ci_handler (),
+                                        ops.ci_handler_argument (),
                                         dd,
-                                        options_->ci_handler_timeout (),
+                                        ops.ci_handler_timeout (),
                                         error,
                                         warn,
                                         trace));
@@ -358,7 +353,7 @@ namespace brep
     {
       try
       {
-        serialize_manifest (sr, os, long_lines);
+        ci_start::serialize_manifest (sr, os, long_lines);
         return true;
       }
       catch (const serialization& e)
@@ -424,7 +419,7 @@ namespace brep
     // assume that the web server error log is monitored and the email sending
     // failure will be noticed.
     //
-    if (options_->ci_email_specified () && !simulate)
+    if (ops.ci_email_specified () && !simulate)
     try
     {
       // Redirect the diagnostics to the web server error log.
@@ -435,14 +430,13 @@ namespace brep
                        *trace << process_args {args, n};
                    },
                    2 /* stderr */,
-                   options_->email (),
+                   ops.email (),
                    "CI request submission (" + sr.reference + ')',
-                   {options_->ci_email ()});
+                   {ops.ci_email ()});
 
       // Write the CI request manifest.
       //
-      pair<bool, optional<start_result>> r (
-        rqm (sm.out, true /* long_lines */));
+      pair<bool, optional<result>> r (rqm (sm.out, true /* long_lines */));
 
       assert (r.first); // The serialization succeeded once, so can't fail now.
 
@@ -473,7 +467,55 @@ namespace brep
       error << "sendmail error: " << e;
     }
 
-    return optional<start_result> (move (sr));
+    return optional<result> (move (sr));
+  }
+
+  optional<ci_start::start_result> ci_start::
+  start (const basic_mark& error,
+         const basic_mark& warn,
+         const basic_mark* trace,
+         optional<tenant_service>&& service,
+         const repository_location& repository,
+         const vector<package>& packages,
+         const optional<string>& client_ip,
+         const optional<string>& user_agent,
+         const optional<string>& interactive,
+         const optional<string>& simulate,
+         const vector<pair<string, string>>& custom_request,
+         const vector<pair<string, string>>& overrides) const
+  {
+    assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+    // Generate the request id.
+    //
+    // Note that it will also be used as a CI result manifest reference,
+    // unless the latter is provided by the external handler.
+    //
+    string request_id;
+
+    try
+    {
+      request_id = uuid::generate ().string ();
+    }
+    catch (const system_error& e)
+    {
+      error << "unable to generate request id: " << e;
+      return nullopt;
+    }
+
+    return brep::start (error, warn, trace,
+                        *options_,
+                        move (request_id),
+                        move (service),
+                        false /* service_load */,
+                        repository,
+                        packages,
+                        client_ip,
+                        user_agent,
+                        interactive,
+                        simulate,
+                        custom_request,
+                        overrides);
   }
 
   void ci_start::
@@ -491,4 +533,206 @@ namespace brep
 
     s.next ("", "");                         // End of manifest.
   }
+
+  optional<string> ci_start::
+  create (const basic_mark& error,
+          const basic_mark&,
+          const basic_mark* trace,
+          odb::core::database& db,
+          tenant_service&& service) const
+  {
+    using namespace odb::core;
+
+    // Generate the request id.
+    //
+    string request_id;
+
+    try
+    {
+      request_id = uuid::generate ().string ();
+    }
+    catch (const system_error& e)
+    {
+      error << "unable to generate request id: " << e;
+      return nullopt;
+    }
+
+    // Use the generated request id if the tenant service id is not specified.
+    //
+    if (service.id.empty ())
+      service.id = request_id;
+
+    // Delay the first load attempt for 10 seconds (see mod-build-task.cxx for
+    // details).
+    //
+    build_tenant t (move (request_id),
+                    move (service),
+                    system_clock::now () - chrono::seconds (40 - 10));
+
+    {
+      assert (!transaction::has_current ());
+
+      transaction tr (db.begin ());
+
+      // Note that in contrast to brep-load, we know that the tenant id is
+      // unique and thus we don't try to remove a tenant with such an id.
+      // There is also not much reason to assume that we may have switched
+      // from the single-tenant mode here and remove the respective tenant,
+      // unless we are in the tenant-service functionality development mode.
+      //
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+      cstrings ts ({""});
+
+      db.erase_query<build_package> (
+        query<build_package>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+      db.erase_query<build_repository> (
+        query<build_repository>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+      db.erase_query<build_public_key> (
+        query<build_public_key>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+      db.erase_query<build_tenant> (
+        query<build_tenant>::id.in_range (ts.begin (), ts.end ()));
+#endif
+
+      db.persist (t);
+
+      tr.commit ();
+    }
+
+    if (trace != nullptr)
+      *trace << "unloaded CI request " << t.id << " for service "
+             << t.service->id << ' ' << t.service->type << " is created";
+
+    return move (t.id);
+  }
+
+  optional<ci_start::start_result> ci_start::
+  load (const basic_mark& error,
+        const basic_mark& warn,
+        const basic_mark* trace,
+        odb::core::database& db,
+        tenant_service&& service,
+        const repository_location& repository) const
+  {
+    using namespace odb::core;
+
+    string request_id;
+    {
+      assert (!transaction::has_current ());
+
+      transaction tr (db.begin ());
+
+      using query = query<build_tenant>;
+
+      shared_ptr<build_tenant> t (
+        db.query_one<build_tenant> (query::service.id == service.id &&
+                                    query::service.type == service.type));
+
+      if (t == nullptr)
+      {
+        error << "unable to find tenant for service " << service.id << ' '
+              << service.type;
+
+        return nullopt;
+      }
+      else if (t->archived)
+      {
+        error << "tenant " << t->id << " for service " << service.id << ' '
+              << service.type << " is already archived";
+
+        return nullopt;
+      }
+      else if (!t->loaded_timestamp)
+      {
+        error << "tenant " << t->id << " for service " << service.id << ' '
+              << service.type << " is already loaded";
+
+        return nullopt;
+      }
+
+      t->loaded_timestamp = nullopt;
+      db.update (t);
+
+      tr.commit ();
+
+      request_id = move (t->id);
+    }
+
+    assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+    optional<start_result> r (brep::start (error, warn, trace,
+                                           *options_,
+                                           move (request_id),
+                                           move (service),
+                                           true /* service_load */,
+                                           repository,
+                                           {} /* packages */,
+                                           nullopt /* client_ip */,
+                                           nullopt /* user_agent */,
+                                           nullopt /* interactive */,
+                                           nullopt /* simulate */,
+                                           {} /* custom_request */,
+                                           {} /* overrides */));
+
+    // Note: on error (r == nullopt) the diagnostics is already issued.
+    //
+    if (trace != nullptr && r)
+      *trace << "CI request for '" << repository << "' is "
+             << (r->status != 200 ? "not " : "") << "loaded: "
+             << r->message << " (reference: " << r->reference << ')';
+
+    return r;
+  }
+
+  void ci_start::
+  abandon (const basic_mark& error,
+           const basic_mark&,
+           const basic_mark* trace,
+           odb::core::database& db,
+           tenant_service&& service) const
+  {
+    using namespace odb::core;
+
+    assert (!transaction::has_current ());
+
+    transaction tr (db.begin ());
+
+    using query = query<build_tenant>;
+
+    shared_ptr<build_tenant> t (
+      db.query_one<build_tenant> (query::service.id == service.id &&
+                                  query::service.type == service.type));
+
+    if (t == nullptr)
+    {
+      error << "unable to find tenant for service " << service.id << ' '
+            << service.type;
+
+      return;
+    }
+    else if (!t->loaded_timestamp)
+    {
+      error << "tenant " << t->id << " for service " << service.id << ' '
+            << service.type << " is already loaded";
+
+      return;
+    }
+
+    // We could probably remove the tenant from the database, but let's just
+    // archive it and keep for troubleshooting.
+    //
+    if (!t->archived)
+    {
+      t->archived = true;
+      db.update (t);
+    }
+
+    tr.commit ();
+
+    if (trace != nullptr)
+      *trace << "unloaded CI request " << t->id << " for service "
+             << service.id << ' ' << service.type << " is abandoned";
+  }
 }
-- 
cgit v1.1