diff options
Diffstat (limited to 'mod')
| -rw-r--r-- | mod/build-config-module.cxx | 33 | ||||
| -rw-r--r-- | mod/build-config-module.hxx | 12 | ||||
| -rw-r--r-- | mod/build-result-module.cxx | 145 | ||||
| -rw-r--r-- | mod/build-target-config.cxx | 8 | ||||
| -rw-r--r-- | mod/build-target-config.hxx | 23 | ||||
| -rw-r--r-- | mod/buildfile | 1 | ||||
| -rw-r--r-- | mod/ci-common.cxx | 357 | ||||
| -rw-r--r-- | mod/ci-common.hxx | 79 | ||||
| -rw-r--r-- | mod/database-module.cxx | 10 | ||||
| -rw-r--r-- | mod/database-module.hxx | 5 | ||||
| -rw-r--r-- | mod/mod-build-configs.cxx | 14 | ||||
| -rw-r--r-- | mod/mod-build-force.cxx | 23 | ||||
| -rw-r--r-- | mod/mod-build-log.cxx | 2 | ||||
| -rw-r--r-- | mod/mod-build-result.cxx | 30 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 591 | ||||
| -rw-r--r-- | mod/mod-builds.cxx | 31 | ||||
| -rw-r--r-- | mod/mod-ci.cxx | 129 | ||||
| -rw-r--r-- | mod/mod-ci.hxx | 47 | ||||
| -rw-r--r-- | mod/mod-package-details.cxx | 4 | ||||
| -rw-r--r-- | mod/mod-package-version-details.cxx | 8 | ||||
| -rw-r--r-- | mod/mod-packages.cxx | 2 | ||||
| -rw-r--r-- | mod/mod-repository-details.cxx | 2 | ||||
| -rw-r--r-- | mod/mod-repository-root.cxx | 14 | ||||
| -rw-r--r-- | mod/mod-repository-root.hxx | 2 | ||||
| -rw-r--r-- | mod/module.cli | 28 | ||||
| -rw-r--r-- | mod/page.cxx | 16 | ||||
| -rw-r--r-- | mod/page.hxx | 13 | ||||
| -rw-r--r-- | mod/tenant-service.hxx | 19 |
28 files changed, 1301 insertions, 347 deletions
diff --git a/mod/build-config-module.cxx b/mod/build-config-module.cxx index 97c9f9e..1f4ad42 100644 --- a/mod/build-config-module.cxx +++ b/mod/build-config-module.cxx @@ -148,26 +148,35 @@ namespace brep } bool build_config_module:: - belongs (const build_target_config& cfg, const char* cls) const + derived (const string& c, const char* bc) const { + if (c == bc) + return true; + + // Go through base classes. + // const map<string, string>& im (target_conf_->class_inheritance_map); - for (const string& c: cfg.classes) + for (auto i (im.find (c)); i != im.end (); ) { - if (c == cls) + const string& base (i->second); + + if (base == bc) return true; - // Go through base classes. - // - for (auto i (im.find (c)); i != im.end (); ) - { - const string& base (i->second); + i = im.find (base); + } - if (base == cls) - return true; + return false; + } - i = im.find (base); - } + bool build_config_module:: + belongs (const build_target_config& cfg, const char* cls) const + { + for (const string& c: cfg.classes) + { + if (derived (c, cls)) + return true; } return false; diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index 78661c3..bbbe952 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -36,8 +36,9 @@ namespace brep void init (const options::build&); + template <typename K> bool - exclude (const build_package_config& pc, + exclude (const build_package_config_template<K>& pc, const build_class_exprs& common_builds, const build_constraints& common_constraints, const build_target_config& tc, @@ -53,15 +54,20 @@ namespace brep default_all_ucs); } + // Return true if a class is derived from the base class, recursively. + // + bool + derived (const string&, const char* base_class) const; + // Check if the configuration belongs to the specified class. // bool belongs (const build_target_config&, const char*) const; bool - belongs (const build_target_config& cfg, const string& cls) const + belongs (const build_target_config& cfg, const string& classes) const { - return belongs (cfg, cls.c_str ()); + return belongs (cfg, classes.c_str ()); } // Target/configuration/toolchain combination that, in particular, can be diff --git a/mod/build-result-module.cxx b/mod/build-result-module.cxx index 68fbe4c..9ac1390 100644 --- a/mod/build-result-module.cxx +++ b/mod/build-result-module.cxx @@ -3,11 +3,16 @@ #include <mod/build-result-module.hxx> +#include <odb/database.hxx> + #include <libbutl/openssl.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/process-io.hxx> #include <libbutl/semantic-version.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + namespace brep { using namespace std; @@ -230,54 +235,112 @@ namespace brep else { assert (b.agent_fingerprint && challenge); - auto i (bot_agent_key_map_->find (*b.agent_fingerprint)); - // The agent's key is recently replaced. + auto auth = [&challenge, + &b, + &o, + &fail, &trace, + &warn_auth, + this] (const path& key) + { + bool r (false); + + try + { + openssl os ([&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }, + path ("-"), fdstream_mode::text, 2, + process_env (o.openssl (), o.openssl_envvar ()), + use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", + o.openssl_option (), + use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", + "-pubin", + "-inkey", key); + + for (const auto& c: *challenge) + os.out.put (c); // Sets badbit on failure. + + os.out.close (); + + string s; + getline (os.in, s); + + bool v (os.in.eof ()); + os.in.close (); + + if (os.wait () && v) + { + r = (s == *b.agent_challenge); + + if (!r) + warn_auth ("challenge mismatched"); + } + else // The signature is presumably meaningless. + warn_auth ("unable to verify challenge"); + } + catch (const system_error& e) + { + fail << "unable to verify challenge: " << e; + } + + return r; + }; + + const string& fp (*b.agent_fingerprint); + auto i (bot_agent_key_map_->find (fp)); + + // Note that it is possible that the default vs custom bot + // classification has changed since the task request time. It feels that + // there is nothing wrong with that and we will handle that + // automatically. // - if (i == bot_agent_key_map_->end ()) + if (i != bot_agent_key_map_->end ()) // Default bot? { - warn_auth ("agent's public key not found"); + r = auth (i->second); } - else - try + else // Custom bot. { - openssl os ([&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }, - path ("-"), fdstream_mode::text, 2, - process_env (o.openssl (), o.openssl_envvar ()), - use_openssl_pkeyutl_ ? "pkeyutl" : "rsautl", - o.openssl_option (), - use_openssl_pkeyutl_ ? "-verifyrecover" : "-verify", - "-pubin", - "-inkey", - i->second); - - for (const auto& c: *challenge) - os.out.put (c); // Sets badbit on failure. - - os.out.close (); - - string s; - getline (os.in, s); - - bool v (os.in.eof ()); - os.in.close (); - - if (os.wait () && v) - { - r = (s == *b.agent_challenge); + shared_ptr<build_public_key> k ( + build_db_->find<build_public_key> (public_key_id (b.tenant, fp))); - if (!r) - warn_auth ("challenge mismatched"); + if (k != nullptr) + { + // Temporarily save the key data to disk (note that it's the + // challenge which is passed via stdin to openssl). Hopefully /tmp + // is using tmpfs. + // + auto_rmfile arm; + + try + { + arm = auto_rmfile (path::temp_path ("brep-custom-bot-key")); + } + catch (const system_error& e) + { + fail << "unable to obtain temporary file: " << e; + } + + try + { + ofdstream os (arm.path); + os << *k; + os.close (); + } + catch (const io_error& e) + { + fail << "unable to write to '" << arm.path << "': " << e; + } + + r = auth (arm.path); + } + else + { + // The agent's key is recently replaced. + // + warn_auth ("agent's public key not found"); } - else // The signature is presumably meaningless. - warn_auth ("unable to verify challenge"); - } - catch (const system_error& e) - { - fail << "unable to verify challenge: " << e; } } diff --git a/mod/build-target-config.cxx b/mod/build-target-config.cxx index a30cf07..a30e281 100644 --- a/mod/build-target-config.cxx +++ b/mod/build-target-config.cxx @@ -21,17 +21,13 @@ namespace brep {"all"}, '+', "All."); bool - exclude (const build_package_config& pc, - const build_class_exprs& cbs, - const build_constraints& ccs, + exclude (const build_class_exprs& exprs, + const build_constraints& constrs, const build_target_config& tc, const map<string, string>& class_inheritance_map, string* reason, bool default_all_ucs) { - const build_class_exprs& exprs (pc.effective_builds (cbs)); - const build_constraints& constrs (pc.effective_constraints (ccs)); - // Save the first sentence of the reason, lower-case the first letter if // the beginning looks like a word (all subsequent characters until a // whitespace are lower-case letters). diff --git a/mod/build-target-config.hxx b/mod/build-target-config.hxx index 180ca80..60d159c 100644 --- a/mod/build-target-config.hxx +++ b/mod/build-target-config.hxx @@ -32,14 +32,31 @@ namespace brep // configuration set needlessly). // bool - exclude (const build_package_config&, - const build_class_exprs& common_builds, - const build_constraints& common_constraints, + exclude (const build_class_exprs& builds, + const build_constraints& constraints, const build_target_config&, const std::map<string, string>& class_inheritance_map, string* reason = nullptr, bool default_all_ucs = false); + template <typename K> + inline bool + exclude (const build_package_config_template<K>& pc, + const build_class_exprs& common_builds, + const build_constraints& common_constraints, + const build_target_config& tc, + const std::map<string, string>& class_inheritance_map, + string* reason = nullptr, + bool default_all_ucs = false) + { + return exclude (pc.effective_builds (common_builds), + pc.effective_constraints (common_constraints), + tc, + class_inheritance_map, + reason, + default_all_ucs); + } + // Convert dash-separated components (target, build target configuration // name, machine name) or a pattern thereof into a path, replacing dashes // with slashes (directory separators), `**` with `*/**/*`, and appending diff --git a/mod/buildfile b/mod/buildfile index c3895dc..2d6ef39 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -39,6 +39,7 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \ # the debugging of the notifications machinery. # cxx.poptions += -DBREP_CI_TENANT_SERVICE +#cxx.poptions += -DBREP_CI_TENANT_SERVICE_UNLOADED libus{mod}: ../web/xhtml/libus{xhtml} libue{mod}: ../web/xhtml/libue{xhtml} diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx index cb61e66..c0ef89f 100644 --- a/mod/ci-common.cxx +++ b/mod/ci-common.cxx @@ -3,6 +3,9 @@ #include <mod/ci-common.hxx> +#include <odb/database.hxx> +#include <odb/transaction.hxx> + #include <libbutl/uuid.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/sendmail.hxx> @@ -11,6 +14,9 @@ #include <libbutl/process-io.hxx> // operator<<(ostream, process_args) #include <libbutl/manifest-serializer.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + #include <mod/external-handler.hxx> namespace brep @@ -38,13 +44,16 @@ namespace brep options_ = move (o); } - optional<ci_start::start_result> ci_start:: + static optional<ci_start::start_result> start (const basic_mark& error, const basic_mark& warn, const basic_mark* trace, + const options::ci_start& ops, + string&& request_id, optional<tenant_service>&& service, + bool service_load, const repository_location& repository, - const vector<package>& packages, + const vector<ci_start::package>& packages, const optional<string>& client_ip, const optional<string>& user_agent, const optional<string>& interactive, @@ -55,32 +64,15 @@ namespace brep using serializer = manifest_serializer; using serialization = manifest_serialization; - assert (options_ != nullptr); // Shouldn't be called otherwise. + using result = ci_start::start_result; // If the tenant service is specified, then its type may not be empty. // assert (!service || !service->type.empty ()); - // Generate the request id. - // - // Note that it will also be used as a CI result manifest reference, - // unless the latter is provided by the external handler. - // - string request_id; - - try - { - request_id = uuid::generate ().string (); - } - catch (const system_error& e) - { - error << "unable to generate request id: " << e; - return nullopt; - } - // Create the submission data directory. // - dir_path dd (options_->ci_data () / dir_path (request_id)); + dir_path dd (ops.ci_data () / dir_path (request_id)); try { @@ -103,10 +95,10 @@ namespace brep // auto client_error = [&request_id] (uint16_t status, string message) { - return start_result {status, - move (message), - request_id, - vector<pair<string, string>> ()}; + return result {status, + move (message), + request_id, + vector<pair<string, string>> ()}; }; // Serialize the CI request manifest to a stream. On the serialization @@ -119,6 +111,7 @@ namespace brep auto rqm = [&request_id, &ts, &service, + service_load, &repository, &packages, &client_ip, @@ -127,7 +120,7 @@ namespace brep &simulate, &custom_request, &client_error] (ostream& os, bool long_lines = false) - -> pair<bool, optional<start_result>> + -> pair<bool, optional<result>> { try { @@ -139,7 +132,7 @@ namespace brep s.next ("id", request_id); s.next ("repository", repository.string ()); - for (const package& p: packages) + for (const ci_start::package& p: packages) { if (!p.version) s.next ("package", p.name.string ()); @@ -178,6 +171,8 @@ namespace brep if (service->data) s.next ("service-data", *service->data); + + s.next ("service-action", service_load ? "load" : "start"); } // Serialize the request custom parameters. @@ -190,12 +185,12 @@ namespace brep s.next (nv.first, nv.second); s.next ("", ""); // End of manifest. - return make_pair (true, optional<start_result> ()); + return make_pair (true, optional<result> ()); } catch (const serialization& e) { return make_pair (false, - optional<start_result> ( + optional<result> ( client_error (400, string ("invalid parameter: ") + e.what ()))); @@ -209,7 +204,7 @@ namespace brep try { ofdstream os (rqf); - pair<bool, optional<start_result>> r (rqm (os)); + pair<bool, optional<result>> r (rqm (os)); os.close (); if (!r.first) @@ -228,7 +223,7 @@ namespace brep // auto ovm = [&overrides, &client_error] (ostream& os, bool long_lines = false) - -> pair<bool, optional<start_result>> + -> pair<bool, optional<result>> { try { @@ -240,12 +235,12 @@ namespace brep s.next (nv.first, nv.second); s.next ("", ""); // End of manifest. - return make_pair (true, optional<start_result> ()); + return make_pair (true, optional<result> ()); } catch (const serialization& e) { return make_pair (false, - optional<start_result> ( + optional<result> ( client_error ( 400, string ("invalid manifest override: ") + @@ -261,7 +256,7 @@ namespace brep try { ofdstream os (ovf); - pair<bool, optional<start_result>> r (ovm (os)); + pair<bool, optional<result>> r (ovm (os)); os.close (); if (!r.first) @@ -305,16 +300,16 @@ namespace brep // manifest from its stdout and parse it into the resulting manifest // object. Otherwise, create implied CI result manifest. // - start_result sr; + result sr; - if (options_->ci_handler_specified ()) + if (ops.ci_handler_specified ()) { using namespace external_handler; - optional<result_manifest> r (run (options_->ci_handler (), - options_->ci_handler_argument (), + optional<result_manifest> r (run (ops.ci_handler (), + ops.ci_handler_argument (), dd, - options_->ci_handler_timeout (), + ops.ci_handler_timeout (), error, warn, trace)); @@ -358,7 +353,7 @@ namespace brep { try { - serialize_manifest (sr, os, long_lines); + ci_start::serialize_manifest (sr, os, long_lines); return true; } catch (const serialization& e) @@ -424,7 +419,7 @@ namespace brep // assume that the web server error log is monitored and the email sending // failure will be noticed. // - if (options_->ci_email_specified () && !simulate) + if (ops.ci_email_specified () && !simulate) try { // Redirect the diagnostics to the web server error log. @@ -435,14 +430,13 @@ namespace brep *trace << process_args {args, n}; }, 2 /* stderr */, - options_->email (), + ops.email (), "CI request submission (" + sr.reference + ')', - {options_->ci_email ()}); + {ops.ci_email ()}); // Write the CI request manifest. // - pair<bool, optional<start_result>> r ( - rqm (sm.out, true /* long_lines */)); + pair<bool, optional<result>> r (rqm (sm.out, true /* long_lines */)); assert (r.first); // The serialization succeeded once, so can't fail now. @@ -473,7 +467,55 @@ namespace brep error << "sendmail error: " << e; } - return optional<start_result> (move (sr)); + return optional<result> (move (sr)); + } + + optional<ci_start::start_result> ci_start:: + start (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + optional<tenant_service>&& service, + const repository_location& repository, + const vector<package>& packages, + const optional<string>& client_ip, + const optional<string>& user_agent, + const optional<string>& interactive, + const optional<string>& simulate, + const vector<pair<string, string>>& custom_request, + const vector<pair<string, string>>& overrides) const + { + assert (options_ != nullptr); // Shouldn't be called otherwise. + + // Generate the request id. + // + // Note that it will also be used as a CI result manifest reference, + // unless the latter is provided by the external handler. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + return brep::start (error, warn, trace, + *options_, + move (request_id), + move (service), + false /* service_load */, + repository, + packages, + client_ip, + user_agent, + interactive, + simulate, + custom_request, + overrides); } void ci_start:: @@ -491,4 +533,227 @@ namespace brep s.next ("", ""); // End of manifest. } + + optional<string> ci_start:: + create (const basic_mark& error, + const basic_mark&, + const basic_mark* trace, + odb::core::database& db, + tenant_service&& service, + duration notify_interval, + duration notify_delay) const + { + using namespace odb::core; + + // Generate the request id. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + // Use the generated request id if the tenant service id is not specified. + // + if (service.id.empty ()) + service.id = request_id; + + build_tenant t (move (request_id), + move (service), + system_clock::now () - notify_interval + notify_delay, + notify_interval); + { + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + // Note that in contrast to brep-load, we know that the tenant id is + // unique and thus we don't try to remove a tenant with such an id. + // There is also not much reason to assume that we may have switched + // from the single-tenant mode here and remove the respective tenant, + // unless we are in the tenant-service functionality development mode. + // +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + cstrings ts ({""}); + + db.erase_query<build_package> ( + query<build_package>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_repository> ( + query<build_repository>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_public_key> ( + query<build_public_key>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_tenant> ( + query<build_tenant>::id.in_range (ts.begin (), ts.end ())); +#endif + + db.persist (t); + + tr.commit (); + } + + if (trace != nullptr) + *trace << "unloaded CI request " << t.id << " for service " + << t.service->id << ' ' << t.service->type << " is created"; + + return move (t.id); + } + + optional<ci_start::start_result> ci_start:: + load (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database& db, + tenant_service&& service, + const repository_location& repository) const + { + using namespace odb::core; + + string request_id; + { + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + using query = query<build_tenant>; + + shared_ptr<build_tenant> t ( + db.query_one<build_tenant> (query::service.id == service.id && + query::service.type == service.type)); + + if (t == nullptr) + { + error << "unable to find tenant for service " << service.id << ' ' + << service.type; + + return nullopt; + } + else if (t->archived) + { + error << "tenant " << t->id << " for service " << service.id << ' ' + << service.type << " is already archived"; + + return nullopt; + } + else if (!t->unloaded_timestamp) + { + error << "tenant " << t->id << " for service " << service.id << ' ' + << service.type << " is already loaded"; + + return nullopt; + } + + t->unloaded_timestamp = nullopt; + db.update (t); + + tr.commit (); + + request_id = move (t->id); + } + + assert (options_ != nullptr); // Shouldn't be called otherwise. + + optional<start_result> r (brep::start (error, warn, trace, + *options_, + move (request_id), + move (service), + true /* service_load */, + repository, + {} /* packages */, + nullopt /* client_ip */, + nullopt /* user_agent */, + nullopt /* interactive */, + nullopt /* simulate */, + {} /* custom_request */, + {} /* overrides */)); + + // Note: on error (r == nullopt) the diagnostics is already issued. + // + if (trace != nullptr && r) + *trace << "CI request for '" << repository << "' is " + << (r->status != 200 ? "not " : "") << "loaded: " + << r->message << " (reference: " << r->reference << ')'; + + return r; + } + + optional<tenant_service> ci_start:: + cancel (const basic_mark&, + const basic_mark&, + const basic_mark* trace, + odb::core::database& db, + const string& type, + const string& id) const + { + using namespace odb::core; + + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + using query = query<build_tenant>; + + shared_ptr<build_tenant> t ( + db.query_one<build_tenant> (query::service.id == id && + query::service.type == type)); + if (t == nullptr) + return nullopt; + + optional<tenant_service> r (move (t->service)); + t->service = nullopt; + t->archived = true; + db.update (t); + + tr.commit (); + + if (trace != nullptr) + *trace << "CI request " << t->id << " for service " << id << ' ' << type + << " is canceled"; + + return r; + } + + bool ci_start:: + cancel (const basic_mark&, + const basic_mark&, + const basic_mark* trace, + const string& reason, + odb::core::database& db, + const string& tid) const + { + using namespace odb::core; + + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + shared_ptr<build_tenant> t (db.find<build_tenant> (tid)); + + if (t == nullptr) + return false; + + if (!t->archived) + { + t->archived = true; + db.update (t); + } + + tr.commit (); + + if (trace != nullptr) + *trace << "CI request " << tid << " is canceled: " + << (reason.size () < 50 + ? reason + : string (reason, 0, 50) + "..."); + + return true; + } } diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx index 6f62c4b..848bca1 100644 --- a/mod/ci-common.hxx +++ b/mod/ci-common.hxx @@ -36,6 +36,7 @@ namespace brep package_name name; optional<brep::version> version; }; + // Note that the inability to generate the reference is an internal // error. Thus, it is not optional. // @@ -62,7 +63,67 @@ namespace brep const optional<string>& interactive = nullopt, const optional<string>& simulate = nullopt, const vector<pair<string, string>>& custom_request = {}, - const vector<pair<string, string>>& overrides = {}); + const vector<pair<string, string>>& overrides = {}) const; + + // Create an unloaded CI request returning start_result::reference on + // success and nullopt on an internal error. Such a request is not started + // until loaded with the load() function below. Configure the time + // interval between the build_unloaded() notifications for the being + // created tenant and set the initial delay for the first notification. + // See also the build_unloaded() tenant services notification. + // + // Note: should be called out of the database transaction. + // + optional<string> + create (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + tenant_service&&, + duration notify_interval, + duration notify_delay) const; + + // Load (and start) previously created (as unloaded) CI request. Similarly + // to the start() function, return nullopt on an internal error. + // + // Note that tenant_service::id is used to identify the CI request tenant. + // + // Note: should be called out of the database transaction. + // + optional<start_result> + load (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + tenant_service&&, + const repository_location& repository) const; + + // Cancel previously created or started CI request. Return the service + // state or nullopt if there is no tenant for such a type/id pair. + // + // Note: should be called out of the database transaction. + // + optional<tenant_service> + cancel (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + const string& type, + const string& id) const; + + // Cancel previously created or started CI request. Return false if there + // is no tenant for the specified tenant id. Note that the reason argument + // is only used for tracing. + // + // Note: should be called out of the database transaction. + // + bool + cancel (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + const string& reason, + odb::core::database&, + const string& tenant_id) const; // Helpers. // @@ -75,22 +136,6 @@ namespace brep private: shared_ptr<options::ci_start> options_; }; - - class ci_cancel - { - public: - void - init (shared_ptr<options::ci_cancel>, shared_ptr<odb::core::database>); - - // @@ TODO Archive the tenant. - // - void - cancel (/*...*/); - - private: - shared_ptr<options::ci_cancel> options_; - shared_ptr<odb::core::database> build_db_; - }; } #endif // MOD_CI_COMMON_HXX diff --git a/mod/database-module.cxx b/mod/database-module.cxx index 07babc6..bbb3e59 100644 --- a/mod/database-module.cxx +++ b/mod/database-module.cxx @@ -76,7 +76,7 @@ namespace brep throw; } - void database_module:: + optional<string> database_module:: update_tenant_service_state ( const connection_ptr& conn, const string& tid, @@ -88,6 +88,8 @@ namespace brep // assert (build_db_ != nullptr); + optional<string> r; + for (size_t retry (retry_);; ) { try @@ -104,6 +106,8 @@ namespace brep { s.data = move (*data); build_db_->update (t); + + r = move (s.data); } } @@ -121,7 +125,11 @@ namespace brep HANDLER_DIAG; l1 ([&]{trace << e << "; " << retry + 1 << " tenant service " << "state update retries left";}); + + r = nullopt; // Prepare for the next iteration. } } + + return r; } } diff --git a/mod/database-module.hxx b/mod/database-module.hxx index 910cb35..298afbf 100644 --- a/mod/database-module.hxx +++ b/mod/database-module.hxx @@ -57,7 +57,8 @@ namespace brep // Update the tenant-associated service state if the specified // notification callback-returned function (expected to be not NULL) - // returns the new state data. + // returns the new state data. Return the service state data, if updated, + // and nullopt otherwise. // // Specifically, start the database transaction, query the service state, // and call the callback-returned function on this state. If this call @@ -65,7 +66,7 @@ namespace brep // state with this data and persist the change. Repeat all the above steps // on the recoverable database failures (deadlocks, etc). // - void + optional<string> update_tenant_service_state ( const odb::core::connection_ptr&, const string& tid, diff --git a/mod/mod-build-configs.cxx b/mod/mod-build-configs.cxx index 9282544..ce79edb 100644 --- a/mod/mod-build-configs.cxx +++ b/mod/mod-build-configs.cxx @@ -30,8 +30,6 @@ build_configs (const build_configs& r) void brep::build_configs:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_configs> ( s, unknown_mode::fail, unknown_mode::fail); @@ -127,19 +125,19 @@ handle (request& rq, response& rs) s << DIV(ID="filter-heading") << "Build Configuration Classes" << ~DIV << P(ID="filter"); + bool printed (false); for (auto b (cls.begin ()), i (b), e (cls.end ()); i != e; ++i) { - // Skip the 'hidden' class. + // Skip the hidden classes. // const string& c (*i); - if (c != "hidden") + if (!derived (c, "hidden")) { - // Note that here we rely on the fact that the first class in the list - // can never be 'hidden' (is always 'all'). - // - if (i != b) + if (printed) s << ' '; + else + printed = true; print_class_name (c, c == selected_class); diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx index bdae356..ea921e9 100644 --- a/mod/mod-build-force.cxx +++ b/mod/mod-build-force.cxx @@ -42,8 +42,6 @@ build_force (const build_force& r, const tenant_service_map& tsm) void brep::build_force:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_force> ( s, unknown_mode::fail, unknown_mode::fail); @@ -192,7 +190,14 @@ handle (request& rq, response& rs) optional<pair<tenant_service, shared_ptr<build>>> tss; tenant_service_build_queued::build_queued_hints qhs; + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // connection_ptr conn (build_db_->connection ()); + { transaction t (conn->begin ()); @@ -297,14 +302,28 @@ handle (request& rq, response& rs) vector<build> qbs; qbs.push_back (move (b)); + // Release the database connection since the build_queued() notification + // can potentially be time-consuming (e.g., it may perform an HTTP + // request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, build_state::building, qhs, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, qbs.back ().tenant, f); + } } + // Release the database connection prior to writing into the unbuffered + // response stream. + // + conn.reset (); + // We have all the data, so don't buffer the response content. // ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx index c8e803b..5487f6e 100644 --- a/mod/mod-build-log.cxx +++ b/mod/mod-build-log.cxx @@ -34,8 +34,6 @@ build_log (const build_log& r) void brep::build_log:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_log> ( s, unknown_mode::fail, unknown_mode::fail); diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index ccce17f..3ba18e1 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -49,8 +49,6 @@ build_result (const build_result& r, const tenant_service_map& tsm) void brep::build_result:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_result> ( s, unknown_mode::fail, unknown_mode::fail); @@ -207,13 +205,20 @@ handle (request& rq, response&) optional<pair<tenant_service, shared_ptr<build>>> tss; tenant_service_build_queued::build_queued_hints qhs; + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // + connection_ptr conn (build_db_->connection ()); + // Note that if the session authentication fails (probably due to the // authentication settings change), then we log this case with the warning // severity and respond with the 200 HTTP code as if the challenge is // valid. The thinking is that we shouldn't alarm a law-abaiding agent and // shouldn't provide any information to a malicious one. // - connection_ptr conn (build_db_->connection ()); { transaction t (conn->begin ()); @@ -518,12 +523,20 @@ handle (request& rq, response&) vector<build> qbs; qbs.push_back (move (*tss->second)); + // Release the database connection since build_queued() notification can + // potentially be time-consuming (e.g., it may perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, build_state::building, qhs, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, qbs.back ().tenant, f); + } } // If a third-party service needs to be notified about the built package, @@ -537,8 +550,16 @@ handle (request& rq, response&) const tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since build_built() notification can + // potentially be time-consuming (e.g., it may perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_built (ss, b, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, b.tenant, f); + } } if (bld != nullptr) @@ -549,6 +570,9 @@ handle (request& rq, response&) if (!build_notify) (cfg->email ? cfg->email : pkg->build_email) = email (); + if (conn == nullptr) + conn = build_db_->connection (); + send_notification_email (*options_, conn, *bld, diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 4dd1e3d..6be77f6 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -44,6 +44,28 @@ using namespace odb::core; static thread_local mt19937 rand_gen (random_device {} ()); +// The challenge (nonce) is randomly generated for every build task if brep is +// configured to authenticate bbot agents. +// +// Nonce generator must guarantee a probabilistically insignificant chance +// of repeating a previously generated value. The common approach is to use +// counters or random number generators (alone or in combination), that +// produce values of the sufficient length. 64-bit non-repeating and +// 512-bit random numbers are considered to be more than sufficient for +// most practical purposes. +// +// We will produce the challenge as the sha256sum of the 512-bit random +// number and the 64-bit current timestamp combination. The latter is +// not really a non-repeating counter and can't be used alone. However +// adding it is a good and cheap uniqueness improvement. +// +// Note that since generating a challenge is not exactly cheap/fast, we will +// generate it in advance for every task request, out of the database +// transaction, and will cache it if it turns out that it wasn't used (no +// package configuration to (re-)build, etc). +// +static thread_local optional<string> challenge; + // Generate a random number in the specified range (max value is included). // static inline size_t @@ -132,7 +154,8 @@ init (scanner& s) // template <typename T> static inline query<T> -package_query (brep::params::build_task& params, +package_query (bool custom_bot, + brep::params::build_task& params, interactive_mode imode, uint64_t queued_expiration_ns) { @@ -141,9 +164,39 @@ package_query (brep::params::build_task& params, query q (!query::build_tenant::archived); + if (custom_bot) + { + // Note that we could potentially only query the packages which refer to + // this custom bot key in one of their build configurations. For that we + // would need to additionally join the current query tables with the bot + // fingerprint-containing build_package_bot_keys and + // build_package_config_bot_keys tables and use the SELECT DISTINCT + // clause. The problem is that we also use the ORDER BY clause and in this + // case PostgreSQL requires all the ORDER BY clause expressions to also be + // present in the SELECT DISTINCT clause and fails with the 'for SELECT + // DISTINCT, ORDER BY expressions must appear in select list' error if + // that's not the case. Also note that in the ODB-generated code the + // 'build_package.project::TEXT' expression in the SELECT DISTINCT clause + // (see the CITEXT type mapping for details in libbrep/common.hxx) would + // not match the 'build_package.name' expression in the ORDER BY clause + // and so we will end up with the mentioned error. One (hackish) way to + // fix that would be to add a dummy member of the string type for the + // build_package.name column. This all sounds quite hairy at the moment + // and it also feels that this can potentially pessimize querying the + // packages built with the default bots only. Thus let's keep it simple + // for now and filter packages by the bot fingerprint at the program + // level. + // + q = q && (query::build_package::custom_bot.is_null () || + query::build_package::custom_bot); + } + else + q = q && (query::build_package::custom_bot.is_null () || + !query::build_package::custom_bot); + // Filter by repositories canonical names (if requested). // - const vector<string>& rp (params.repository ()); + const strings& rp (params.repository ()); if (!rp.empty ()) q = q && @@ -213,25 +266,33 @@ handle (request& rq, response& rs) throw invalid_request (400, e.what ()); } - // Obtain the agent's public key fingerprint if requested. If the fingerprint - // is requested but is not present in the request or is unknown, then respond - // with 401 HTTP code (unauthorized). + // Obtain the agent's public key fingerprint if requested. If the + // fingerprint is requested but is not present in the request, then respond + // with 401 HTTP code (unauthorized). If a key with the specified + // fingerprint is not present in the build bot agent keys directory, then + // assume that this is a custom build bot. + // + // Note that if the agent authentication is not configured (the agent keys + // directory is not specified), then the bot can never be custom and its + // fingerprint is ignored, if present. // optional<string> agent_fp; + bool custom_bot (false); if (bot_agent_key_map_ != nullptr) { - if (!tqm.fingerprint || - bot_agent_key_map_->find (*tqm.fingerprint) == - bot_agent_key_map_->end ()) + if (!tqm.fingerprint) throw invalid_request (401, "unauthorized"); agent_fp = move (tqm.fingerprint); + + custom_bot = (bot_agent_key_map_->find (*agent_fp) == + bot_agent_key_map_->end ()); } // The resulting task manifest and the related build, package, and // configuration objects. Note that the latter 3 are only meaningful if the - // session in the task manifest is not empty. + // the task manifest is present. // task_response_manifest task_response; shared_ptr<build> task_build; @@ -338,6 +399,79 @@ handle (request& rq, response& rs) } } + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // + connection_ptr conn (build_db_->connection ()); + + // Perform some housekeeping first. + // + // Notify a tenant-associated third-party service about the unloaded CI + // request, if present. + // + { + const tenant_service_build_unloaded* tsu (nullptr); + + transaction tr (conn->begin ()); + + using query = query<build_tenant>; + + // Pick the unloaded tenant with the earliest loaded timestamp, skipping + // those which were already picked recently. + // + shared_ptr<build_tenant> t ( + build_db_->query_one<build_tenant> ( + (!query::archived && + query::unloaded_timestamp.is_not_null () && + (query::unloaded_timestamp + + "<= EXTRACT (EPOCH FROM NOW()) * 1000000000 - " + + query::unloaded_notify_interval)) + + "ORDER BY" + query::unloaded_timestamp + + "LIMIT 1")); + + if (t != nullptr && t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + tsu = dynamic_cast<const tenant_service_build_unloaded*> ( + i->second.get ()); + + if (tsu != nullptr) + { + // If we ought to call the + // tenant_service_build_unloaded::build_unloaded() callback, then + // set the package tenant's loaded timestamp to the current time to + // prevent the notifications race. + // + t->unloaded_timestamp = system_clock::now (); + build_db_->update (t); + } + } + } + + tr.commit (); + + if (tsu != nullptr) + { + // Release the database connection since the build_unloaded() + // notification can potentially be time-consuming (e.g., it may perform + // an HTTP request). + // + conn.reset (); + + if (auto f = tsu->build_unloaded (move (*t->service), log_writer_)) + { + conn = build_db_->connection (); + update_tenant_service_state (conn, t->id, f); + } + } + } + // Go through package build configurations until we find one that has no // build target configuration present in the database, or is in the building // state but expired (collectively called unbuilt). If such a target @@ -578,63 +712,6 @@ handle (request& rq, response& rs) : optional<size_t> ()), options_->build_hard_rebuild_timeout ())); - // Return the challenge (nonce) if brep is configured to authenticate bbot - // agents. Return nullopt otherwise. - // - // Nonce generator must guarantee a probabilistically insignificant chance - // of repeating a previously generated value. The common approach is to use - // counters or random number generators (alone or in combination), that - // produce values of the sufficient length. 64-bit non-repeating and - // 512-bit random numbers are considered to be more than sufficient for - // most practical purposes. - // - // We will produce the challenge as the sha256sum of the 512-bit random - // number and the 64-bit current timestamp combination. The latter is - // not really a non-repeating counter and can't be used alone. However - // adding it is a good and cheap uniqueness improvement. - // - auto challenge = [&agent_fp, &now, &fail, &trace, this] () - { - optional<string> r; - - if (agent_fp) - { - try - { - auto print_args = [&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }; - - openssl os (print_args, - nullfd, path ("-"), 2, - process_env (options_->openssl (), - options_->openssl_envvar ()), - "rand", - options_->openssl_option (), 64); - - vector<char> nonce (os.in.read_binary ()); - os.in.close (); - - if (!os.wait () || nonce.size () != 64) - fail << "unable to generate nonce"; - - uint64_t t (chrono::duration_cast<chrono::nanoseconds> ( - now.time_since_epoch ()).count ()); - - sha256 cs (nonce.data (), nonce.size ()); - cs.append (&t, sizeof (t)); - r = cs.string (); - } - catch (const system_error& e) - { - fail << "unable to generate nonce: " << e; - } - } - - return r; - }; - // Convert butl::standard_version type to brep::version. // brep::version toolchain_version (tqm.toolchain_version.string ()); @@ -659,7 +736,8 @@ handle (request& rq, response& rs) using pkg_query = query<buildable_package>; using prep_pkg_query = prepared_query<buildable_package>; - pkg_query pq (package_query<buildable_package> (params, + pkg_query pq (package_query<buildable_package> (custom_bot, + params, imode, queued_expiration_ns)); @@ -815,11 +893,15 @@ handle (request& rq, response& rs) { using query = query<buildable_package_count>; - query q (package_query<buildable_package_count> (params, + query q (package_query<buildable_package_count> (custom_bot, + params, imode, queued_expiration_ns)); - transaction t (build_db_->begin ()); + if (conn == nullptr) + conn = build_db_->connection (); + + transaction t (conn->begin ()); // If there are any non-archived interactive build tenants, then the // chosen randomization approach doesn't really work since interactive @@ -880,7 +962,8 @@ handle (request& rq, response& rs) "OFFSET" + pkg_query::_ref (offset) + "LIMIT" + pkg_query::_ref (limit); - connection_ptr conn (build_db_->connection ()); + if (conn == nullptr) + conn = build_db_->connection (); prep_pkg_query pkg_prep_query ( conn->prepare_query<buildable_package> ( @@ -1241,7 +1324,8 @@ handle (request& rq, response& rs) // small_vector<bpkg::test_dependency, 1> tests; - build_db_->load (*p, p->requirements_tests_section); + if (!p->requirements_tests_section.loaded ()) + build_db_->load (*p, p->requirements_tests_section); for (const build_test_dependency& td: p->tests) { @@ -1257,13 +1341,12 @@ handle (request& rq, response& rs) // Try to use the test package configuration named the same as the // current configuration of the main package. If there is no such - // a configuration or it excludes the current target - // configuration, then fallback to using the default configuration - // (which must exist). If in this case the default configuration - // excludes the current target configuration, then exclude this - // external test package from the build task. + // a configuration, then fallback to using the default + // configuration (which must exist). If the selected test package + // configuration excludes the current target configuration, then + // exclude this external test package from the build task. // - // Note that potentially the test package default configuration + // Note that potentially the selected test package configuration // may contain some (bpkg) arguments associated, but we currently // don't provide build bot worker with such information. This, // however, is probably too far fetched so let's keep it simple @@ -1271,6 +1354,13 @@ handle (request& rq, response& rs) // const build_package_config* tpc (find (pc.name, tp->configs)); + if (tpc == nullptr) + { + tpc = find ("default", tp->configs); + + assert (tpc != nullptr); // Must always be present. + } + // Use the `all` class as a least restrictive default underlying // build class set. Note that we should only apply the explicit // build restrictions to the external test packages (think about @@ -1279,32 +1369,15 @@ handle (request& rq, response& rs) // build_db_->load (*tp, tp->constraints_section); - if (tpc == nullptr || - exclude (*tpc, + if (exclude (*tpc, tp->builds, tp->constraints, tc, nullptr /* reason */, true /* default_all_ucs */)) - { - // If the current package configuration is "default", then we - // cannot fallback and just exclude the test package outright. - // - if (pc.name == "default") - continue; - - tpc = find ("default", tp->configs); + continue; - assert (tpc != nullptr); // Must always be present. - - if (exclude (*tpc, - tp->builds, - tp->constraints, - tc, - nullptr /* reason */, - true /* default_all_ucs */)) - continue; - } + build_db_->load (*tp, tp->auxiliaries_section); for (const build_auxiliary& ba: tpc->effective_auxiliaries (tp->auxiliaries)) @@ -1325,20 +1398,56 @@ handle (request& rq, response& rs) vector<auxiliary_machine> tms; vector<build_machine> bms; - tms.reserve (picked_machines.size ()); - bms.reserve (picked_machines.size ()); - - for (pair<auxiliary_config_machine, string>& pm: picked_machines) + if (size_t n = picked_machines.size ()) { - const machine_header_manifest& m (*pm.first.machine); - tms.push_back (auxiliary_machine {m.name, move (pm.second)}); - bms.push_back (build_machine {m.name, m.summary}); + tms.reserve (n); + bms.reserve (n); + + for (pair<auxiliary_config_machine, string>& pm: picked_machines) + { + const machine_header_manifest& m (*pm.first.machine); + tms.push_back (auxiliary_machine {m.name, move (pm.second)}); + bms.push_back (build_machine {m.name, m.summary}); + } } return collect_auxiliaries_result { move (tms), move (bms), move (tests)}; }; + if (agent_fp && !challenge) + try + { + auto print_args = [&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }; + + openssl os (print_args, + nullfd, path ("-"), 2, + process_env (options_->openssl (), + options_->openssl_envvar ()), + "rand", + options_->openssl_option (), 64); + + vector<char> nonce (os.in.read_binary ()); + os.in.close (); + + if (!os.wait () || nonce.size () != 64) + fail << "unable to generate nonce"; + + uint64_t t (chrono::duration_cast<chrono::nanoseconds> ( + now.time_since_epoch ()).count ()); + + sha256 cs (nonce.data (), nonce.size ()); + cs.append (&t, sizeof (t)); + challenge = cs.string (); + } + catch (const system_error& e) + { + fail << "unable to generate nonce: " << e; + } + // While at it, collect the aborted for various reasons builds // (interactive builds in multiple configurations, builds with too many // auxiliary machines, etc) to send the notification emails at the end @@ -1357,9 +1466,9 @@ handle (request& rq, response& rs) // bool unforced (true); - for (bool done (false); task_response.session.empty () && !done; ) + for (bool done (false); !task_response.task && !done; ) { - transaction t (conn->begin ()); + transaction tr (conn->begin ()); // We need to be careful in the random package ordering mode not to // miss the end after having wrapped around. @@ -1394,7 +1503,7 @@ handle (request& rq, response& rs) // if (chunk_size == 0) { - t.commit (); + tr.commit (); if (start_offset != 0 && offset >= start_offset) offset = 0; @@ -1411,12 +1520,24 @@ handle (request& rq, response& rs) // to bail out in the random package ordering mode for some reason (no // more untried positions, need to restart, etc). // + // Note that it is not uncommon for the sequentially examined packages + // to belong to the same tenant (single tenant mode, etc). Thus, we + // will cache the loaded tenant objects. + // + shared_ptr<build_tenant> t; + for (auto& bp: packages) { shared_ptr<build_package>& p (bp.package); id = p->id; + // Reset the tenant cache if the current package belongs to a + // different tenant. + // + if (t != nullptr && t->id != id.tenant) + t = nullptr; + // If we are in the random package ordering mode, then cache the // tenant the start offset refers to, if not cached yet, and check // if we are still iterating over packages from this tenant @@ -1476,8 +1597,6 @@ handle (request& rq, response& rs) // if (bp.interactive) { - shared_ptr<build_tenant> t; - // Note that the tenant can be archived via some other package on // some previous iteration. Skip the package if that's the case. // @@ -1486,7 +1605,9 @@ handle (request& rq, response& rs) // if (!bp.archived) { - t = build_db_->load<build_tenant> (id.tenant); + if (t == nullptr) + t = build_db_->load<build_tenant> (id.tenant); + bp.archived = t->archived; } @@ -1596,8 +1717,48 @@ handle (request& rq, response& rs) } } + // If true, then the package is (being) built for some + // configurations. + // + // Note that since we only query the built and forced rebuild + // objects there can be false negatives. + // + bool package_built (false); + + build_db_->load (*p, p->bot_keys_section); + for (const build_package_config& pc: p->configs) { + // If this is a custom bot, then skip this configuration if it + // doesn't contain this bot's public key in its custom bot keys + // list. Otherwise (this is a default bot), skip this + // configuration if its custom bot keys list is not empty. + // + { + const build_package_bot_keys& bks ( + pc.effective_bot_keys (p->bot_keys)); + + if (custom_bot) + { + assert (agent_fp); // Wouldn't be here otherwise. + + if (find_if ( + bks.begin (), bks.end (), + [&agent_fp] (const lazy_shared_ptr<build_public_key>& k) + { + return k.object_id ().fingerprint == *agent_fp; + }) == bks.end ()) + { + continue; + } + } + else + { + if (!bks.empty ()) + continue; + } + } + pkg_config = pc.name; // Iterate through the built configurations and erase them from the @@ -1610,6 +1771,9 @@ handle (request& rq, response& rs) config_machines configs (conf_machines); // Make copy for this pkg. auto pkg_builds (bld_prep_query.execute ()); + if (!package_built && !pkg_builds.empty ()) + package_built = true; + for (auto i (pkg_builds.begin ()); i != pkg_builds.end (); ++i) { auto j ( @@ -1637,35 +1801,38 @@ handle (request& rq, response& rs) // the package configuration and for which all the requested // auxiliary machines can be provided. // - auto i (configs.begin ()); - auto e (configs.end ()); + const config_machine* cm (nullptr); + optional<collect_auxiliaries_result> aux; build_db_->load (*p, p->constraints_section); - optional<collect_auxiliaries_result> aux; - for (; i != e; ++i) + for (auto i (configs.begin ()), e (configs.end ()); i != e; ++i) { - const build_target_config& tc (*i->second.config); + cm = &i->second; + const build_target_config& tc (*cm->config); + + if (!exclude (pc, p->builds, p->constraints, tc)) + { + if (!p->auxiliaries_section.loaded ()) + build_db_->load (*p, p->auxiliaries_section); - if (!exclude (pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, pc, tc))) - break; + if ((aux = collect_auxiliaries (p, pc, tc))) + break; + } } - if (i != e) + if (aux) { - config_machine& cm (i->second); - machine_header_manifest& mh (*cm.machine); + machine_header_manifest& mh (*cm->machine); build_id bid (move (id), - cm.config->target, - cm.config->name, + cm->config->target, + cm->config->name, move (pkg_config), move (toolchain_name), toolchain_version); shared_ptr<build> b (build_db_->find<build> (bid)); - optional<string> cl (challenge ()); // Move the interactive build login information into the build // object, if the package to be built interactively. @@ -1690,12 +1857,14 @@ handle (request& rq, response& rs) move (toolchain_version), move (login), move (agent_fp), - move (cl), + move (challenge), build_machine { mh.name, move (mh.summary)}, move (aux->build_auxiliary_machines), - controller_checksum (*cm.config), - machine_checksum (*cm.machine)); + controller_checksum (*cm->config), + machine_checksum (*cm->machine)); + + challenge = nullopt; build_db_->persist (b); } @@ -1732,7 +1901,10 @@ handle (request& rq, response& rs) } b->agent_fingerprint = move (agent_fp); - b->agent_challenge = move (cl); + + b->agent_challenge = move (challenge); + challenge = nullopt; + b->machine = build_machine {mh.name, move (mh.summary)}; // Mark the section as loaded, so auxiliary_machines are @@ -1743,8 +1915,8 @@ handle (request& rq, response& rs) b->auxiliary_machines = move (aux->build_auxiliary_machines); - string ccs (controller_checksum (*cm.config)); - string mcs (machine_checksum (*cm.machine)); + string ccs (controller_checksum (*cm->config)); + string mcs (machine_checksum (*cm->machine)); // Issue the hard rebuild if it is forced or the // configuration or machine has changed. @@ -1763,8 +1935,8 @@ handle (request& rq, response& rs) build_db_->update (b); } - shared_ptr<build_tenant> t ( - build_db_->load<build_tenant> (b->tenant)); + if (t == nullptr) + t = build_db_->load<build_tenant> (b->tenant); // Archive an interactive tenant. // @@ -1811,7 +1983,7 @@ handle (request& rq, response& rs) } if (tsb != nullptr || tsq != nullptr) - tss = make_pair (move (*t->service), b); + tss = make_pair (*t->service, b); } } @@ -1821,31 +1993,57 @@ handle (request& rq, response& rs) move (aux->tests), move (aux->task_auxiliary_machines), move (bp.interactive), - cm); + *cm); task_build = move (b); task_package = move (p); task_config = &pc; + package_built = true; + break; // Bail out from the package configurations loop. } } } - // If the task response manifest is prepared, then bail out from the - // package loop, commit the transaction and respond. + // If the task manifest is prepared, then bail out from the package + // loop, commit the transaction and respond. Otherwise, stash the + // build toolchain into the tenant, unless it is already stashed or + // the current package already has some configurations (being) + // built. // - if (!task_response.session.empty ()) + if (!task_response.task) + { + // Note that since there can be false negatives for the + // package_built flag (see above), there can be redundant tenant + // queries which, however, seems harmless (query uses the primary + // key and the object memory footprint is small). + // + if (!package_built) + { + if (t == nullptr) + t = build_db_->load<build_tenant> (p->id.tenant); + + if (!t->toolchain) + { + t->toolchain = build_toolchain {toolchain_name, + toolchain_version}; + + build_db_->update (t); + } + } + } + else break; } - t.commit (); + tr.commit (); } // If we don't have an unbuilt package, then let's see if we have a // build configuration to rebuild. // - if (task_response.session.empty () && !rebuilds.empty ()) + if (!task_response.task && !rebuilds.empty ()) { // Sort the configuration rebuild list with the following sort // priority: @@ -1875,8 +2073,6 @@ handle (request& rq, response& rs) sort (rebuilds.begin (), rebuilds.end (), cmp); - optional<string> cl (challenge ()); - // Pick the first build configuration from the ordered list. // // Note that the configurations and packages may not match the @@ -1935,13 +2131,17 @@ handle (request& rq, response& rs) (t->interactive.has_value () == (imode == interactive_mode::true_)))) { + const build_target_config& tc (*cm.config); + build_db_->load (*p, p->constraints_section); - const build_target_config& tc (*cm.config); + if (exclude (*pc, p->builds, p->constraints, tc)) + continue; + + build_db_->load (*p, p->auxiliaries_section); - optional<collect_auxiliaries_result> aux; - if (!exclude (*pc, p->builds, p->constraints, tc) && - (aux = collect_auxiliaries (p, *pc, tc))) + if (optional<collect_auxiliaries_result> aux = + collect_auxiliaries (p, *pc, tc)) { assert (b->status); @@ -1963,10 +2163,10 @@ handle (request& rq, response& rs) unforced = (b->force == force_state::unforced); - // Can't move from, as may need them on the next iteration. - // - b->agent_fingerprint = agent_fp; - b->agent_challenge = cl; + b->agent_fingerprint = move (agent_fp); + + b->agent_challenge = move (challenge); + challenge = nullopt; const machine_header_manifest& mh (*cm.machine); b->machine = build_machine {mh.name, mh.summary}; @@ -2059,16 +2259,19 @@ handle (request& rq, response& rs) } catch (const odb::deadlock&) { - // Just try with the next rebuild. But first, reset the task - // response manifest that we may have prepared. + // Just try with the next rebuild. But first, restore the agent's + // fingerprint and challenge and reset the task manifest and the + // session that we may have prepared. // + agent_fp = move (b->agent_fingerprint); + challenge = move (b->agent_challenge); task_response = task_response_manifest (); } - // If the task response manifest is prepared, then bail out from the - // package configuration rebuilds loop and respond. + // If the task manifest is prepared, then bail out from the package + // configuration rebuilds loop and respond. // - if (!task_response.session.empty ()) + if (task_response.task) break; } } @@ -2082,7 +2285,7 @@ handle (request& rq, response& rs) { assert (tss); // Wouldn't be here otherwise. - const tenant_service& ss (tss->first); + tenant_service& ss (tss->first); // If the task build has no initial state (is just created), then // temporarily move it into the list of the queued builds until the @@ -2100,12 +2303,24 @@ handle (request& rq, response& rs) if (!qbs.empty ()) { + // Release the database connection since the build_queued() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, nullopt /* initial_state */, qhs, log_writer_)) - update_tenant_service_state (conn, qbs.back ().tenant, f); + { + conn = build_db_->connection (); + + if (optional<string> data = + update_tenant_service_state (conn, qbs.back ().tenant, f)) + ss.data = move (data); + } } // Send the `queued` notification for the task build, unless it is @@ -2120,12 +2335,24 @@ handle (request& rq, response& rs) qbs.push_back (move (b)); restore_build = true; + // Release the database connection since the build_queued() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, initial_state, qhs, log_writer_)) - update_tenant_service_state (conn, qbs.back ().tenant, f); + { + conn = build_db_->connection (); + + if (optional<string> data = + update_tenant_service_state (conn, qbs.back ().tenant, f)) + ss.data = move (data); + } } if (restore_build) @@ -2141,16 +2368,28 @@ handle (request& rq, response& rs) { assert (tss); // Wouldn't be here otherwise. - const tenant_service& ss (tss->first); + tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since the build_building() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_building (ss, b, log_writer_)) - update_tenant_service_state (conn, b.tenant, f); + { + conn = build_db_->connection (); + + if (optional<string> data = + update_tenant_service_state (conn, b.tenant, f)) + ss.data = move (data); + } } - // If the task response manifest is prepared, then check that the number - // of the build auxiliary machines is less than 10. If that's not the - // case, then turn the build into the built state with the abort status. + // If the task manifest is prepared, then check that the number of the + // build auxiliary machines is less than 10. If that's not the case, + // then turn the build into the built state with the abort status. // if (task_response.task && task_response.task->auxiliary_machines.size () > 9) @@ -2168,6 +2407,9 @@ handle (request& rq, response& rs) const tenant_service_build_built* tsb (nullptr); optional<pair<tenant_service, shared_ptr<build>>> tss; { + if (conn == nullptr) + conn = build_db_->connection (); + transaction t (conn->begin ()); shared_ptr<build> b (build_db_->find<build> (task_build->id)); @@ -2254,17 +2496,33 @@ handle (request& rq, response& rs) { assert (tss); // Wouldn't be here otherwise. - const tenant_service& ss (tss->first); + tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since the build_built() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_built (ss, b, log_writer_)) - update_tenant_service_state (conn, b.tenant, f); + { + conn = build_db_->connection (); + + if (optional<string> data = + update_tenant_service_state (conn, b.tenant, f)) + ss.data = move (data); + } } } // Send notification emails for all the aborted builds. // for (const aborted_build& ab: aborted_builds) + { + if (conn == nullptr) + conn = build_db_->connection (); + send_notification_email (*options_, conn, *ab.b, @@ -2273,9 +2531,14 @@ handle (request& rq, response& rs) ab.what, error, verb_ >= 2 ? &trace : nullptr); + } } } + // Release the database connection as soon as possible. + // + conn.reset (); + serialize_task_response_manifest (); return true; } diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index b0de618..81d4649 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -50,8 +50,6 @@ builds (const builds& r) void brep::builds:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::builds> ( s, unknown_mode::fail, unknown_mode::fail); @@ -393,8 +391,11 @@ handle (request& rq, response& rs) if (!tenant.empty ()) tn = tenant; - // Return the list of distinct toolchain name/version pairs. The build db - // transaction must be started. + // Return the list of distinct toolchain name/version pairs. If no builds + // are present for the tenant, then fallback to the toolchain recorded in + // the tenant object, if present. + // + // Note: the build db transaction must be started. // using toolchains = vector<pair<string, version>>; @@ -410,11 +411,19 @@ handle (request& rq, response& rs) false /* first */))) r.emplace_back (move (t.name), move (t.version)); + if (r.empty ()) + { + shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tenant)); + + if (t != nullptr && t->toolchain) + r.emplace_back (t->toolchain->name, t->toolchain->version); + } + return r; }; auto print_form = [&s, ¶ms, this] (const toolchains& toolchains, - size_t build_count) + optional<size_t> build_count) { // Print the package builds filter form on the first page only. // @@ -525,7 +534,7 @@ handle (request& rq, response& rs) conf_ids.push_back (c.first); } - size_t count; + optional<size_t> count; size_t page (params.page ()); if (params.result () != "unbuilt") // Print package build configurations. @@ -656,7 +665,7 @@ handle (request& rq, response& rs) --print; } - ++count; + ++(*count); } } @@ -937,7 +946,7 @@ handle (request& rq, response& rs) count = npos - ncur; } else - count = 0; + count = nullopt; // Unknown count. } // Print the filter form. @@ -1148,7 +1157,11 @@ handle (request& rq, response& rs) add_filter ("pc", pkg_cfg); add_filter ("rs", params.result (), "*"); - s << DIV_PAGER (page, count, page_configs, options_->build_pages (), u) + s << DIV_PAGER (page, + count ? *count : 0, + page_configs, + options_->build_pages (), + u) << ~DIV << ~BODY << ~HTML; diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index 5974d45..8c47bc4 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -22,6 +22,8 @@ using namespace butl; using namespace web; using namespace brep::cli; +// ci +// #ifdef BREP_CI_TENANT_SERVICE brep::ci:: ci (tenant_service_map& tsm) @@ -36,7 +38,12 @@ ci (const ci& r, tenant_service_map& tsm) #else ci (const ci& r) #endif - : handler (r), + : +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED + handler (r), + #else + database_module (r), +#endif ci_start (r), options_ (r.initialized_ ? r.options_ : nullptr), form_ (r.initialized_ || r.form_ == nullptr @@ -100,6 +107,13 @@ init (scanner& s) } } +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + if (!options_->build_config_specified ()) + fail << "package building functionality must be enabled"; + + database_module::init (*options_, options_->build_db_retry ()); +#endif + if (options_->root ().empty ()) options_->root (dir_path ("/")); } @@ -347,6 +361,7 @@ handle (request& rq, response& rs) user_agent = h.value; } +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED optional<start_result> r (start (error, warn, verb_ ? &trace : nullptr, @@ -367,6 +382,25 @@ handle (request& rq, response& rs) : optional<string> ()), custom_request, overrides)); +#else + assert (build_db_ != nullptr); // Wouldn't be here otherwise. + + optional<start_result> r; + + if (optional<string> ref = create (error, + warn, + verb_ ? &trace : nullptr, + *build_db_, + tenant_service ("", "ci", rl.string ()), + chrono::seconds (40), + chrono::seconds (10))) + { + string msg ("unloaded CI request is created: " + + options_->host () + tenant_dir (root, *ref).string ()); + + r = start_result {200, move (msg), move (*ref), {}}; + } +#endif if (!r) return respond_error (); // The diagnostics is already issued. @@ -472,4 +506,97 @@ build_built (const tenant_service&, return ts.data ? *ts.data + ", " + s : s; }; } + +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED +function<optional<string> (const brep::tenant_service&)> brep::ci:: +build_unloaded (tenant_service&& ts, + const diag_epilogue& log_writer) const noexcept +{ + NOTIFICATION_DIAG (log_writer); + + assert (ts.data); // Repository location. + + try + { + repository_location rl (*ts.data); + + if (!load (error, warn, verb_ ? &trace : nullptr, + *build_db_, + move (ts), + rl)) + return nullptr; // The diagnostics is already issued. + } + catch (const invalid_argument& e) + { + error << "invalid repository location '" << *ts.data << "' stored for " + << "tenant service " << ts.id << ' ' << ts.type; + + return nullptr; + } + + return [] (const tenant_service& ts) {return "loaded " + *ts.data;}; +} +#endif #endif + +// ci_cancel +// +brep::ci_cancel:: +ci_cancel (const ci_cancel& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::ci_cancel:: +init (scanner& s) +{ + options_ = make_shared<options::ci_cancel> ( + s, unknown_mode::fail, unknown_mode::fail); + + if (options_->build_config_specified ()) + database_module::init (*options_, options_->build_db_retry ()); +} + +bool brep::ci_cancel:: +handle (request& rq, response& rs) +{ + HANDLER_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + params::ci_cancel params; + + try + { + name_value_scanner s (rq.parameters (1024)); + params = params::ci_cancel (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + const string& reason (params.reason ()); + + if (reason.empty ()) + throw invalid_request (400, "missing CI request cancellation reason"); + + // Verify the tenant id. + // + const string tid (params.id ()); + + if (tid.empty ()) + throw invalid_request (400, "invalid CI request id"); + + if (!cancel (error, warn, verb_ ? &trace : nullptr, reason, *build_db_, tid)) + throw invalid_request (400, "unknown CI request id"); + + // We have all the data, so don't buffer the response content. + // + ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); + os << "CI request " << tid << " has been canceled"; + + return true; +} diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx index 1e2ee15..bd91e99 100644 --- a/mod/mod-ci.hxx +++ b/mod/mod-ci.hxx @@ -16,6 +16,11 @@ #include <mod/module-options.hxx> #include <mod/ci-common.hxx> +#include <mod/database-module.hxx> + +#if defined(BREP_CI_TENANT_SERVICE_UNLOADED) && !defined(BREP_CI_TENANT_SERVICE) +# error BREP_CI_TENANT_SERVICE must be defined if BREP_CI_TENANT_SERVICE_UNLOADED is defined +#endif #ifdef BREP_CI_TENANT_SERVICE # include <mod/tenant-service.hxx> @@ -23,12 +28,20 @@ namespace brep { - class ci: public handler, + class ci: +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED + public handler, +#else + public database_module, +#endif private ci_start #ifdef BREP_CI_TENANT_SERVICE , public tenant_service_build_queued, public tenant_service_build_building, public tenant_service_build_built +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + , tenant_service_build_unloaded +#endif #endif { public: @@ -74,6 +87,12 @@ namespace brep build_built (const tenant_service&, const build&, const diag_epilogue& log_writer) const noexcept override; + +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + virtual function<optional<string> (const tenant_service&)> + build_unloaded (tenant_service&&, + const diag_epilogue& log_writer) const noexcept override; +#endif #endif private: @@ -88,6 +107,32 @@ namespace brep tenant_service_map& tenant_service_map_; #endif }; + + class ci_cancel: public database_module, + private ci_start + { + public: + ci_cancel () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + ci_cancel (const ci_cancel&); + + virtual bool + handle (request&, response&) override; + + virtual const cli::options& + cli_options () const override {return options::ci_cancel::description ();} + + private: + virtual void + init (cli::scanner&) override; + + private: + shared_ptr<options::ci_cancel> options_; + }; } #endif // MOD_MOD_CI_HXX diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx index 5cf0759..1fb51da 100644 --- a/mod/mod-package-details.cxx +++ b/mod/mod-package-details.cxx @@ -37,8 +37,6 @@ package_details (const package_details& r) void brep::package_details:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::package_details> ( s, unknown_mode::fail, unknown_mode::fail); @@ -227,7 +225,7 @@ handle (request& rq, response& rs) << ~TABLE; } - auto pkg_count ( + size_t pkg_count ( package_db_->query_value<package_count> ( search_params<package_count> (squery, tenant, name))); diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 51b21c6..91923e5 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -541,7 +541,7 @@ handle (request& rq, response& rs) // builds = false; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { const build_class_exprs& exprs (pc.effective_builds (pkg->builds)); @@ -726,7 +726,7 @@ handle (request& rq, response& rs) s << H3 << "Builds" << ~H3 << DIV(ID="builds"); - auto exclude = [&pkg, this] (const build_package_config& pc, + auto exclude = [&pkg, this] (const package_build_config& pc, const build_target_config& tc, string* rs = nullptr) { @@ -767,7 +767,7 @@ handle (request& rq, response& rs) query sq (false); set<config_toolchain> unbuilt_configs; - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& bc: *target_conf_map_) { @@ -886,7 +886,7 @@ handle (request& rq, response& rs) // if (!tn->interactive) { - for (const build_package_config& pc: pkg->build_configs) + for (const package_build_config& pc: pkg->build_configs) { for (const auto& tc: *target_conf_) { diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx index 914a841..6026024 100644 --- a/mod/mod-packages.cxx +++ b/mod/mod-packages.cxx @@ -156,7 +156,7 @@ handle (request& rq, response& rs) session sn; transaction t (package_db_->begin ()); - auto pkg_count ( + size_t pkg_count ( package_db_->query_value<latest_package_count> ( search_param<latest_package_count> (squery, tn))); diff --git a/mod/mod-repository-details.cxx b/mod/mod-repository-details.cxx index 082903b..93b6c9e 100644 --- a/mod/mod-repository-details.cxx +++ b/mod/mod-repository-details.cxx @@ -39,8 +39,6 @@ repository_details (const repository_details& r) void brep::repository_details:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::repository_details> ( s, unknown_mode::fail, unknown_mode::fail); diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index 34b4007..bc861a8 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -133,6 +133,7 @@ namespace brep #else ci_ (make_shared<ci> ()), #endif + ci_cancel_ (make_shared<ci_cancel> ()), upload_ (make_shared<upload> ()) { } @@ -201,6 +202,10 @@ namespace brep #else : make_shared<ci> (*r.ci_)), #endif + ci_cancel_ ( + r.initialized_ + ? r.ci_cancel_ + : make_shared<ci_cancel> (*r.ci_cancel_)), upload_ ( r.initialized_ ? r.upload_ @@ -231,6 +236,7 @@ namespace brep append (r, build_configs_->options ()); append (r, submit_->options ()); append (r, ci_->options ()); + append (r, ci_cancel_->options ()); append (r, upload_->options ()); return r; } @@ -277,6 +283,7 @@ namespace brep sub_init (*build_configs_, "build_configs"); sub_init (*submit_, "submit"); sub_init (*ci_, "ci"); + sub_init (*ci_cancel_, "ci-cancel"); sub_init (*upload_, "upload"); // Parse own configuration options. @@ -473,6 +480,13 @@ namespace brep return handle ("ci", param); } + else if (func == "ci-cancel") + { + if (handler_ == nullptr) + handler_.reset (new ci_cancel (*ci_cancel_)); + + return handle ("ci-cancel", param); + } else if (func == "upload") { if (handler_ == nullptr) diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx index aa60fda..990587e 100644 --- a/mod/mod-repository-root.hxx +++ b/mod/mod-repository-root.hxx @@ -25,6 +25,7 @@ namespace brep class build_configs; class submit; class ci; + class ci_cancel; class upload; class repository_root: public handler @@ -74,6 +75,7 @@ namespace brep shared_ptr<build_configs> build_configs_; shared_ptr<submit> submit_; shared_ptr<ci> ci_; + shared_ptr<ci_cancel> ci_cancel_; shared_ptr<upload> upload_; shared_ptr<options::repository_root> options_; diff --git a/mod/module.cli b/mod/module.cli index a107ffe..5133935 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -796,11 +796,7 @@ namespace brep } }; - class ci_cancel - { - }; - - class ci: ci_start, page, repository_url, handler + class ci: ci_start, build, build_db, page, repository_url, handler { // Classic CI-specific options. // @@ -815,7 +811,11 @@ namespace brep } }; - class ci_github: ci_start, ci_cancel, build_db, handler + class ci_cancel: build, build_db, handler + { + }; + + class ci_github: ci_start, build, build_db, handler { // GitHub CI-specific options (e.g., request timeout when invoking // GitHub APIs). @@ -1099,6 +1099,22 @@ namespace brep string simulate; }; + // All parameters are non-optional. + // + class ci_cancel + { + // CI task tenant id. + // + // Note that the ci-cancel parameter is renamed to '_' by the root + // handler (see the request_proxy class for details). + // + string id | _; + + // CI task canceling reason. Must not be empty. + // + string reason; + }; + // Parameters other than challenge must be all present. // // Note also that besides these parameters there can be others. We don't diff --git a/mod/page.cxx b/mod/page.cxx index 5483183..177fb64 100644 --- a/mod/page.cxx +++ b/mod/page.cxx @@ -137,9 +137,17 @@ namespace brep void DIV_COUNTER:: operator() (serializer& s) const { - s << DIV(ID="count") - << count_ << " " - << (count_ % 10 == 1 && count_ % 100 != 11 ? singular_ : plural_) + s << DIV(ID="count"); + + if (count_) + s << *count_; + else + s << '?'; + + s << ' ' + << (count_ && *count_ % 10 == 1 && *count_ % 100 != 11 + ? singular_ + : plural_) << ~DIV; } @@ -731,7 +739,7 @@ namespace brep << ~TR; } - // BUILD_RESULT + // TR_BUILD_RESULT // void TR_BUILD_RESULT:: operator() (serializer& s) const diff --git a/mod/page.hxx b/mod/page.hxx index cac2b8b..7329e2d 100644 --- a/mod/page.hxx +++ b/mod/page.hxx @@ -82,21 +82,24 @@ namespace brep // Generate counter element. // - // It could be redunant to distinguish between singular and plural word forms - // if it wouldn't be so cheap in English, and phrase '1 Packages' wouldn't - // look that ugly. + // If the count argument is nullopt, then it is assumed that the count is + // unknown and the '?' character is printed instead of the number. + // + // Note that it could be redunant to distinguish between singular and plural + // word forms if it wouldn't be so cheap in English, and phrase '1 Packages' + // wouldn't look that ugly. // class DIV_COUNTER { public: - DIV_COUNTER (size_t c, const char* s, const char* p) + DIV_COUNTER (optional<size_t> c, const char* s, const char* p) : count_ (c), singular_ (s), plural_ (p) {} void operator() (xml::serializer&) const; private: - size_t count_; + optional<size_t> count_; const char* singular_; const char* plural_; }; diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx index 9205f76..b7f5c02 100644 --- a/mod/tenant-service.hxx +++ b/mod/tenant-service.hxx @@ -21,7 +21,8 @@ namespace brep virtual ~tenant_service_base () = default; }; - // Possible build notifications: + // Possible build notifications (see also the unloaded special notification + // below): // // queued // building @@ -121,6 +122,22 @@ namespace brep const diag_epilogue& log_writer) const noexcept = 0; }; + // This notification is only made on unloaded CI requests created with the + // ci_start::create() call and until they are loaded with ci_start::load() + // or, alternatively, abandoned with ci_start::abandon(). + // + // Note: make sure the implementation of this notification does not take + // too long (currently 40 seconds) to avoid nested notifications. Note + // also that the first notification is delayed (currently 10 seconds). + // + class tenant_service_build_unloaded: public virtual tenant_service_base + { + public: + virtual function<optional<string> (const tenant_service&)> + build_unloaded (tenant_service&&, + const diag_epilogue& log_writer) const noexcept = 0; + }; + // Map of service type (tenant_service::type) to service. // using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>; |