From d4900d85f7a5d791f89821713d02d3dd19361044 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Thu, 22 Feb 2024 11:17:25 +0300 Subject: Add support for tenant-associated service notifications --- brep/handler/ci/ci-load.in | 27 ++ doc/manual.cli | 11 + etc/brep-module.conf | 8 + etc/private/install/brep-module.conf | 8 + libbrep/build-extra.sql | 6 +- libbrep/build-package.hxx | 2 + libbrep/build.cxx | 94 +++++- libbrep/build.hxx | 43 ++- libbrep/build.xml | 2 + libbrep/common.hxx | 30 ++ libbrep/package.cxx | 8 +- libbrep/package.hxx | 43 ++- libbrep/package.xml | 16 + load/load.cli | 22 ++ load/load.cxx | 55 +++- mod/buildfile | 5 + mod/ci-common.cxx | 494 +++++++++++++++++++++++++++++ mod/ci-common.hxx | 96 ++++++ mod/database-module.cxx | 56 ++++ mod/database-module.hxx | 23 +- mod/external-handler.cxx | 3 +- mod/external-handler.hxx | 2 +- mod/mod-build-force.cxx | 78 ++++- mod/mod-build-force.hxx | 8 +- mod/mod-build-log.cxx | 2 +- mod/mod-build-result.cxx | 159 +++++++++- mod/mod-build-result.hxx | 8 +- mod/mod-build-task.cxx | 395 +++++++++++++++++++---- mod/mod-build-task.hxx | 8 +- mod/mod-builds.cxx | 10 + mod/mod-ci.cxx | 585 +++++++++++------------------------ mod/mod-ci.hxx | 49 ++- mod/mod-package-version-details.cxx | 2 +- mod/mod-repository-root.cxx | 35 ++- mod/mod-repository-root.hxx | 3 + mod/module.cli | 57 +++- mod/page.cxx | 1 - mod/page.hxx | 7 +- mod/tenant-service.hxx | 107 +++++++ monitor/monitor.cxx | 8 +- web/server/module.hxx | 3 +- 41 files changed, 2041 insertions(+), 538 deletions(-) create mode 100644 mod/ci-common.cxx create mode 100644 mod/ci-common.hxx create mode 100644 mod/tenant-service.hxx diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in index dbdc450..3f04ea8 100644 --- a/brep/handler/ci/ci-load.in +++ b/brep/handler/ci/ci-load.in @@ -108,6 +108,13 @@ declare -A packages # spec= +# Third party service information which, if specified, needs to be associated +# with the being created tenant. +# +service_id= +service_type= +service_data= + while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in repository) repository="$v" ;; @@ -122,6 +129,10 @@ while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do fi spec="$spec$v" ;; + + service-id) service_id="$v" ;; + service-type) service_type="$v" ;; + service-data) service_data="$v" ;; esac done @@ -141,6 +152,12 @@ if [[ -n "$simulate" && "$simulate" != "success" ]]; then exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" fi +# Use the generated reference if the tenant service id is not specified. +# +if [[ -n "$service_type" && -z "$service_id" ]]; then + service_id="$reference" +fi + message_suffix= if [[ -n "$result_url" ]]; then message_suffix=": $result_url/@$reference" # Append the tenant id. @@ -306,6 +323,16 @@ if [[ -n "$interactive" ]]; then loader_options+=(--interactive "$interactive") fi +# Pass the tenant service information, if specified, to the loader. +# +if [[ -n "$service_id" ]]; then + loader_options+=(--service-id "$service_id" --service-type "$service_type") + + if [[ -n "$service_data" ]]; then + loader_options+=(--service-data "$service_data") + fi +fi + run "$loader" "${loader_options[@]}" "$loadtab" # Remove the no longer needed CI request data directory. diff --git a/doc/manual.cli b/doc/manual.cli index 6dab404..2b96393 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -341,12 +341,23 @@ repository: timestamp: [client-ip]: [user-agent]: +[service-id]: +[service-type]: +[service-data]: \ The \c{package} value can be repeated multiple times. The \c{timestamp} value is in the ISO-8601 \c{--
T::Z} form (always UTC). Note also that \c{client-ip} can be IPv4 or IPv6. +Note that some CI service implementations may serve as backends for +third-party services. The latter may initiate CI tasks, providing all the +required information via some custom protocol, and expect the CI service to +notify it about the progress. In this case the third-party service type as +well as optionally the third-party id and custom state data can be +communicated to the underlying CI handler program via the respective +\c{service-*} manifest values. + \h#ci-overrides-manifest|CI Overrides Manifest| diff --git a/etc/brep-module.conf b/etc/brep-module.conf index 606c2d5..093d1f8 100644 --- a/etc/brep-module.conf +++ b/etc/brep-module.conf @@ -201,6 +201,14 @@ menu About=?about # build-alt-hard-rebuild-stop +# Time to wait before assuming the 'queued' notifications are delivered for +# package CI requests submitted via third-party services (GitHub, etc). During +# this time a package is not considered for a build. Must be specified in +# seconds. Default is 30 seconds. +# +# build-queued-timeout 30 + + # The maximum size of the build task request manifest accepted. Note that the # HTTP POST request body is cached to retry database transactions in the face # of recoverable failures (deadlock, loss of connection, etc). Default is diff --git a/etc/private/install/brep-module.conf b/etc/private/install/brep-module.conf index df439f4..4e5eb87 100644 --- a/etc/private/install/brep-module.conf +++ b/etc/private/install/brep-module.conf @@ -201,6 +201,14 @@ menu About=?about # build-alt-hard-rebuild-stop +# Time to wait before assuming the 'queued' notifications are delivered for +# package CI requests submitted via third-party services (GitHub, etc). During +# this time a package is not considered for a build. Must be specified in +# seconds. Default is 30 seconds. +# +# build-queued-timeout 30 + + # The maximum size of the build task request manifest accepted. Note that the # HTTP POST request body is cached to retry database transactions in the face # of recoverable failures (deadlock, loss of connection, etc). Default is diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index 23015f4..c15ddc1 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -36,7 +36,11 @@ CREATE FOREIGN TABLE build_tenant ( id TEXT NOT NULL, private BOOLEAN NOT NULL, interactive TEXT NULL, - archived BOOLEAN NOT NULL) + archived BOOLEAN NOT NULL, + service_id TEXT NULL, + service_type TEXT NULL, + service_data TEXT NULL, + queued_timestamp BIGINT NULL) SERVER package_server OPTIONS (table_name 'tenant'); -- The foreign table for build_repository object. diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 6d7fb14..a0e1082 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -35,6 +35,8 @@ namespace brep bool private_; optional interactive; bool archived; + optional service; + optional queued_timestamp; // Database mapping. // diff --git a/libbrep/build.cxx b/libbrep/build.cxx index c8a2cd1..8fadfa3 100644 --- a/libbrep/build.cxx +++ b/libbrep/build.cxx @@ -12,6 +12,7 @@ namespace brep { switch (s) { + case build_state::queued: return "queued"; case build_state::building: return "building"; case build_state::built: return "built"; } @@ -22,7 +23,8 @@ namespace brep build_state to_build_state (const string& s) { - if (s == "building") return build_state::building; + if (s == "queued") return build_state::queued; + else if (s == "building") return build_state::building; else if (s == "built") return build_state::built; else throw invalid_argument ("invalid build state '" + s + '\''); } @@ -91,6 +93,96 @@ namespace brep { } + build:: + build (string tnt, + package_name_type pnm, + version pvr, + target_triplet trg, + string tcf, + string pcf, + string tnm, version tvr) + : id (package_id (move (tnt), move (pnm), pvr), + move (trg), + move (tcf), + move (pcf), + move (tnm), tvr), + tenant (id.package.tenant), + package_name (id.package.name), + package_version (move (pvr)), + target (id.target), + target_config_name (id.target_config_name), + package_config_name (id.package_config_name), + toolchain_name (id.toolchain_name), + toolchain_version (move (tvr)), + state (build_state::queued), + timestamp (timestamp_type::clock::now ()), + force (force_state::unforced) + { + } + + build:: + build (build&& b) + : id (move (b.id)), + tenant (id.package.tenant), + package_name (id.package.name), + package_version (move (b.package_version)), + target (id.target), + target_config_name (id.target_config_name), + package_config_name (id.package_config_name), + toolchain_name (id.toolchain_name), + toolchain_version (move (b.toolchain_version)), + state (b.state), + interactive (move (b.interactive)), + timestamp (b.timestamp), + force (b.force), + status (b.status), + soft_timestamp (b.soft_timestamp), + hard_timestamp (b.hard_timestamp), + agent_fingerprint (move (b.agent_fingerprint)), + agent_challenge (move (b.agent_challenge)), + machine (move (b.machine)), + machine_summary (move (b.machine_summary)), + results (move (b.results)), + results_section (move (b.results_section)), + controller_checksum (move (b.controller_checksum)), + machine_checksum (move (b.machine_checksum)), + agent_checksum (move (b.agent_checksum)), + worker_checksum (move (b.worker_checksum)), + dependency_checksum (move (b.dependency_checksum)) + { + } + + build& build:: + operator= (build&& b) + { + if (this != &b) + { + id = move (b.id); + package_version = move (b.package_version); + toolchain_version = move (b.toolchain_version); + state = b.state; + interactive = move (b.interactive); + timestamp = b.timestamp; + force = b.force; + status = b.status; + soft_timestamp = b.soft_timestamp; + hard_timestamp = b.hard_timestamp; + agent_fingerprint = move (b.agent_fingerprint); + agent_challenge = move (b.agent_challenge); + machine = move (b.machine); + machine_summary = move (b.machine_summary); + results = move (b.results); + results_section = move (b.results_section); + controller_checksum = move (b.controller_checksum); + machine_checksum = move (b.machine_checksum); + agent_checksum = move (b.agent_checksum); + worker_checksum = move (b.worker_checksum); + dependency_checksum = move (b.dependency_checksum); + } + + return *this; + } + // build_delay // build_delay:: diff --git a/libbrep/build.hxx b/libbrep/build.hxx index adad535..236f73c 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -28,7 +28,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 23, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 24, closed) // We have to keep these mappings at the global scope instead of inside the // brep namespace because they need to be also effective in the bbot namespace @@ -129,8 +129,14 @@ namespace brep // build_state // + // The queued build state is semantically equivalent to a non-existent + // build. It is only used for those tenants, which have a third-party + // service associated that requires the `queued` notifications (see + // mod/tenant-service.hxx for background). + // enum class build_state: std::uint8_t { + queued, building, built }; @@ -212,6 +218,23 @@ namespace brep string controller_checksum, string machine_checksum); + // Create the build object with the queued state. + // + build (string tenant, + package_name_type, version, + target_triplet, + string target_config_name, + string package_config_name, + string toolchain_name, version toolchain_version); + + // Move-only type. + // + build (build&&); + build& operator= (build&&); + + build (const build&) = delete; + build& operator= (const build&) = delete; + build_id id; string& tenant; // Tracks id.package.tenant. @@ -315,9 +338,6 @@ namespace brep #pragma db member(results_section) load(lazy) update(always) - build (const build&) = delete; - build& operator= (const build&) = delete; - private: friend class odb::access; @@ -377,7 +397,7 @@ namespace brep canonical_version version_; }; - // Build of an existing buildable package. + // Builds of existing buildable packages. // #pragma db view \ object(build) \ @@ -408,6 +428,19 @@ namespace brep #pragma db member(result) column("count(" + build::id.package.name + ")") }; + // Ids of existing buildable package builds. + // + #pragma db view object(build) \ + object(build_package inner: \ + brep::operator== (build::id.package, build_package::id) && \ + build_package::buildable) + struct package_build_id + { + build_id id; + + operator build_id& () {return id;} + }; + // Used to track the package build delays since the last build or, if not // present, since the first opportunity to build the package. // diff --git a/libbrep/build.xml b/libbrep/build.xml index 0a25488..e757aba 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,6 @@ + + diff --git a/libbrep/common.hxx b/libbrep/common.hxx index fec22e8..11aae67 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -127,6 +127,20 @@ namespace brep std::chrono::duration_cast ( \ std::chrono::nanoseconds (?)))) + using optional_timestamp = optional; + using optional_uint64 = optional; + + #pragma db map type(optional_timestamp) as(brep::optional_uint64) \ + to((?) \ + ? std::chrono::duration_cast ( \ + (?)->time_since_epoch ()).count () \ + : brep::optional_uint64 ()) \ + from((?) \ + ? brep::timestamp ( \ + std::chrono::duration_cast ( \ + std::chrono::nanoseconds (*(?)))) \ + : brep::optional_timestamp ()) + // version // using bpkg::version; @@ -517,6 +531,22 @@ namespace brep #pragma db member(requirement_key::middle) column("alternative_index") #pragma db member(requirement_key::inner) column("index") + // Third-party service state which may optionally be associated with a + // tenant (see also mod/tenant-service.hxx for background). + // + #pragma db value + struct tenant_service + { + string id; + string type; + optional data; + + tenant_service () = default; + + tenant_service (string i, string t, optional d = nullopt) + : id (move (i)), type (move (t)), data (move (d)) {} + }; + // Version comparison operators. // // They allow comparing objects that have epoch, canonical_upstream, diff --git a/libbrep/package.cxx b/libbrep/package.cxx index e5e7767..2320547 100644 --- a/libbrep/package.cxx +++ b/libbrep/package.cxx @@ -40,11 +40,15 @@ namespace brep // tenant // tenant:: - tenant (string i, bool p, optional r) + tenant (string i, + bool p, + optional r, + optional s) : id (move (i)), private_ (p), interactive (move (r)), - creation_timestamp (timestamp::clock::now ()) + creation_timestamp (timestamp::clock::now ()), + service (move (s)) { } diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 06e6335..9bb9af9 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 29, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 30, closed) namespace brep { @@ -241,31 +241,62 @@ namespace brep // Create the tenant object with the timestamp set to now and the archived // flag set to false. // - explicit - tenant (string id, bool private_, optional interactive); + tenant (string id, + bool private_, + optional interactive, + optional); string id; // If true, display the packages in the web interface only in the tenant // view mode. // - bool private_; // Note: foreign-mapped in build. + bool private_; // Note: foreign-mapped in build. // Interactive package build breakpoint. // // If present, then packages from this tenant will only be built // interactively and only non-interactively otherwise. // - optional interactive; // Note: foreign-mapped in build. + optional interactive; // Note: foreign-mapped in build. timestamp creation_timestamp; - bool archived = false; // Note: foreign-mapped in build. + bool archived = false; // Note: foreign-mapped in build. + + optional service; // Note: foreign-mapped in build. + + // Note that due to the implementation complexity and performance + // considerations, the service notifications are not synchronized. This + // leads to a potential race, so that before we have sent the `queued` + // notification for a package build, some other thread (potentially in a + // different process) could have already sent the `building` notification + // for it. It feels like there is no easy way to reliably fix that. + // Instead, we just decrease the probability of such a notifications + // sequence failure by delaying builds of the freshly queued packages for + // some time. Specifically, whenever the `queued` notification is ought + // to be sent (normally out of the database transaction, since it likely + // sends an HTTP request, etc) the tenant's queued_timestamp member is set + // to the current time. During the configured time interval since that + // time point the build tasks may not be issued for the tenant's packages. + // + // Also note that while there are similar potential races for other + // notification sequences, their probability is rather low due to the + // natural reasons (non-zero build task execution time, etc) and thus we + // just ignore them. + // + optional queued_timestamp; // Note: foreign-mapped in build. // Database mapping. // #pragma db member(id) id #pragma db member(private_) + #pragma db index("tenant_service_i") \ + unique \ + members(service.id, service.type) + + #pragma db index member(service.id) + private: friend class odb::access; tenant () = default; diff --git a/libbrep/package.xml b/libbrep/package.xml index bf7915e..6852f75 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,20 @@ + + + + + + + + + + + + + + + + diff --git a/load/load.cli b/load/load.cli index 16b5f9f..1b7c4c5 100644 --- a/load/load.cli +++ b/load/load.cli @@ -77,6 +77,28 @@ class options breakpoint. Implies \cb{--private}." }; + std::string --service-id + { + "", + "Third party service information to associate with the being created + tenant. Requires the \cb{--tenant} and \cb{--service-type} options to be + specified." + }; + + std::string --service-type + { + "", + "Type of the service to associate with the being created tenant. Requires + the \cb{--service-id} option to be specified." + }; + + std::string --service-data + { + "", + "Service data to associate with the being created tenant. Requires the + \cb{--service-id} option to be specified." + }; + brep::path --overrides-file { "", diff --git a/load/load.cxx b/load/load.cxx index b644a3a..56e4e19 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -1492,6 +1492,40 @@ try throw failed (); } + // Verify the --service-* options. + // + if (ops.service_id_specified ()) + { + if (!ops.tenant_specified ()) + { + cerr << "error: --service-id requires --tenant" << endl; + throw failed (); + } + + if (ops.service_type ().empty ()) + { + cerr << "error: --service-id requires --service-type" + << endl; + throw failed (); + } + } + else + { + if (ops.service_type_specified ()) + { + cerr << "error: --service-type requires --service-id" + << endl; + throw failed (); + } + + if (ops.service_data_specified ()) + { + cerr << "error: --service-data requires --service-id" + << endl; + throw failed (); + } + } + // Parse and validate overrides, if specified. // // Note that here we make sure that the overrides manifest is valid. @@ -1591,11 +1625,30 @@ try // Persist the tenant. // + // Note that if the tenant service is specified and some tenant with the + // same service id and type is already persisted, then we will end up with + // the `object already persistent` error and terminate with the exit code + // 1 (fatal error). We could potentially dedicate a special exit code for + // such a case, so that the caller may recognize it and behave accordingly + // (CI request handler can treat it as a client error rather than an + // internal error, etc). However, let's first see if it ever becomes a + // problem. + // + optional service; + + if (ops.service_id_specified ()) + service = tenant_service (ops.service_id (), + ops.service_type (), + (ops.service_data_specified () + ? ops.service_data () + : optional ())); + db.persist (tenant (tnt, ops.private_ (), (ops.interactive_specified () ? ops.interactive () - : optional ()))); + : optional ()), + move (service))); // On the first pass over the internal repositories we load their // certificate information and packages. diff --git a/mod/buildfile b/mod/buildfile index 6693a35..c3895dc 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -35,6 +35,11 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \ {hxx ixx txx cxx}{+{$libu_src} } \ $libs +# Add support for tenant-associated service notifications to the CI module for +# the debugging of the notifications machinery. +# +cxx.poptions += -DBREP_CI_TENANT_SERVICE + libus{mod}: ../web/xhtml/libus{xhtml} libue{mod}: ../web/xhtml/libue{xhtml} diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx new file mode 100644 index 0000000..cb61e66 --- /dev/null +++ b/mod/ci-common.cxx @@ -0,0 +1,494 @@ +// file : mod/ci-common.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include +#include +#include +#include +#include +#include // operator<<(ostream, process_args) +#include + +#include + +namespace brep +{ + using namespace std; + using namespace butl; + + void ci_start:: + init (shared_ptr o) + { + // Verify the data directory satisfies the requirements. + // + const dir_path& d (o->ci_data ()); + + if (d.relative ()) + throw runtime_error ("ci-data directory path must be absolute"); + + if (!dir_exists (d)) + throw runtime_error ("ci-data directory '" + d.string () + + "' does not exist"); + + if (o->ci_handler_specified () && o->ci_handler ().relative ()) + throw runtime_error ("ci-handler path must be absolute"); + + options_ = move (o); + } + + optional ci_start:: + start (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + optional&& service, + const repository_location& repository, + const vector& packages, + const optional& client_ip, + const optional& user_agent, + const optional& interactive, + const optional& simulate, + const vector>& custom_request, + const vector>& overrides) + { + using serializer = manifest_serializer; + using serialization = manifest_serialization; + + assert (options_ != nullptr); // Shouldn't be called otherwise. + + // If the tenant service is specified, then its type may not be empty. + // + assert (!service || !service->type.empty ()); + + // Generate the request id. + // + // Note that it will also be used as a CI result manifest reference, + // unless the latter is provided by the external handler. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + // Create the submission data directory. + // + dir_path dd (options_->ci_data () / dir_path (request_id)); + + try + { + // It's highly unlikely but still possible that the directory already + // exists. This can only happen if the generated uuid is not unique. + // + if (try_mkdir (dd) == mkdir_status::already_exists) + throw_generic_error (EEXIST); + } + catch (const system_error& e) + { + error << "unable to create directory '" << dd << "': " << e; + return nullopt; + } + + auto_rmdir ddr (dd); + + // Return the start_result object for the client errors (normally the bad + // request status code (400) for the client data serialization errors). + // + auto client_error = [&request_id] (uint16_t status, string message) + { + return start_result {status, + move (message), + request_id, + vector> ()}; + }; + + // Serialize the CI request manifest to a stream. On the serialization + // error return false together with the start_result object containing the + // bad request (400) code and the error message. On the stream error pass + // through the io_error exception. Otherwise return true. + // + timestamp ts (system_clock::now ()); + + auto rqm = [&request_id, + &ts, + &service, + &repository, + &packages, + &client_ip, + &user_agent, + &interactive, + &simulate, + &custom_request, + &client_error] (ostream& os, bool long_lines = false) + -> pair> + { + try + { + serializer s (os, "request", long_lines); + + // Serialize the submission manifest header. + // + s.next ("", "1"); // Start of manifest. + s.next ("id", request_id); + s.next ("repository", repository.string ()); + + for (const package& p: packages) + { + if (!p.version) + s.next ("package", p.name.string ()); + else + s.next ("package", + p.name.string () + '/' + p.version->string ()); + } + + if (interactive) + s.next ("interactive", *interactive); + + if (simulate) + s.next ("simulate", *simulate); + + s.next ("timestamp", + butl::to_string (ts, + "%Y-%m-%dT%H:%M:%SZ", + false /* special */, + false /* local */)); + + if (client_ip) + s.next ("client-ip", *client_ip); + + if (user_agent) + s.next ("user-agent", *user_agent); + + if (service) + { + // Note that if the service id is not specified, then the handler + // will use the generated reference instead. + // + if (!service->id.empty ()) + s.next ("service-id", service->id); + + s.next ("service-type", service->type); + + if (service->data) + s.next ("service-data", *service->data); + } + + // Serialize the request custom parameters. + // + // Note that the serializer constraints the custom parameter names + // (can't start with '#', can't contain ':' and the whitespaces, + // etc). + // + for (const pair& nv: custom_request) + s.next (nv.first, nv.second); + + s.next ("", ""); // End of manifest. + return make_pair (true, optional ()); + } + catch (const serialization& e) + { + return make_pair (false, + optional ( + client_error (400, + string ("invalid parameter: ") + + e.what ()))); + } + }; + + // Serialize the CI request manifest to the submission directory. + // + path rqf (dd / "request.manifest"); + + try + { + ofdstream os (rqf); + pair> r (rqm (os)); + os.close (); + + if (!r.first) + return move (*r.second); + } + catch (const io_error& e) + { + error << "unable to write to '" << rqf << "': " << e; + return nullopt; + } + + // Serialize the CI overrides manifest to a stream. On the serialization + // error return false together with the start_result object containing the + // bad request (400) code and the error message. On the stream error pass + // through the io_error exception. Otherwise return true. + // + auto ovm = [&overrides, &client_error] (ostream& os, + bool long_lines = false) + -> pair> + { + try + { + serializer s (os, "overrides", long_lines); + + s.next ("", "1"); // Start of manifest. + + for (const pair& nv: overrides) + s.next (nv.first, nv.second); + + s.next ("", ""); // End of manifest. + return make_pair (true, optional ()); + } + catch (const serialization& e) + { + return make_pair (false, + optional ( + client_error ( + 400, + string ("invalid manifest override: ") + + e.what ()))); + } + }; + + // Serialize the CI overrides manifest to the submission directory. + // + path ovf (dd / "overrides.manifest"); + + if (!overrides.empty ()) + try + { + ofdstream os (ovf); + pair> r (ovm (os)); + os.close (); + + if (!r.first) + return move (*r.second); + } + catch (const io_error& e) + { + error << "unable to write to '" << ovf << "': " << e; + return nullopt; + } + + // Given that the submission data is now successfully persisted we are no + // longer in charge of removing it, except for the cases when the + // submission handler terminates with an error (see below for details). + // + ddr.cancel (); + + // If the handler terminates with non-zero exit status or specifies 5XX + // (HTTP server error) submission result manifest status value, then we + // stash the submission data directory for troubleshooting. Otherwise, if + // it's the 4XX (HTTP client error) status value, then we remove the + // directory. + // + auto stash_submit_dir = [&dd, error] () + { + if (dir_exists (dd)) + try + { + mvdir (dd, dir_path (dd + ".fail")); + } + catch (const system_error& e) + { + // Not much we can do here. Let's just log the issue and bail out + // leaving the directory in place. + // + error << "unable to rename directory '" << dd << "': " << e; + } + }; + + // Run the submission handler, if specified, reading the CI result + // manifest from its stdout and parse it into the resulting manifest + // object. Otherwise, create implied CI result manifest. + // + start_result sr; + + if (options_->ci_handler_specified ()) + { + using namespace external_handler; + + optional r (run (options_->ci_handler (), + options_->ci_handler_argument (), + dd, + options_->ci_handler_timeout (), + error, + warn, + trace)); + if (!r) + { + stash_submit_dir (); + return nullopt; // The diagnostics is already issued. + } + + sr.status = r->status; + + for (manifest_name_value& nv: r->values) + { + string& n (nv.name); + string& v (nv.value); + + if (n == "message") + sr.message = move (v); + else if (n == "reference") + sr.reference = move (v); + else if (n != "status") + sr.custom_result.emplace_back (move (n), move (v)); + } + + if (sr.reference.empty ()) + sr.reference = move (request_id); + } + else // Create the implied CI result manifest. + { + sr.status = 200; + sr.message = "CI request is queued"; + sr.reference = move (request_id); + } + + // Serialize the CI result manifest manifest to a stream. On the + // serialization error log the error description and return false, on the + // stream error pass through the io_error exception, otherwise return + // true. + // + auto rsm = [&sr, &error] (ostream& os, bool long_lines = false) -> bool + { + try + { + serialize_manifest (sr, os, long_lines); + return true; + } + catch (const serialization& e) + { + error << "ref " << sr.reference << ": unable to serialize handler's " + << "output: " << e; + return false; + } + }; + + // If the submission data directory still exists then perform an + // appropriate action on it, depending on the submission result status. + // Note that the handler could move or remove the directory. + // + if (dir_exists (dd)) + { + // Remove the directory if the client error is detected. + // + if (sr.status >= 400 && sr.status < 500) + { + rmdir_r (dd); + } + // + // Otherwise, save the result manifest, into the directory. Also stash + // the directory for troubleshooting in case of the server error. + // + else + { + path rsf (dd / "result.manifest"); + + try + { + ofdstream os (rsf); + + // Not being able to stash the result manifest is not a reason to + // claim the submission failed. The error is logged nevertheless. + // + rsm (os); + + os.close (); + } + catch (const io_error& e) + { + // Not fatal (see above). + // + error << "unable to write to '" << rsf << "': " << e; + } + + if (sr.status >= 500 && sr.status < 600) + stash_submit_dir (); + } + } + + // Send email, if configured, and the CI request submission is not + // simulated. Use the long lines manifest serialization mode for the + // convenience of copying/clicking URLs they contain. + // + // Note that we don't consider the email sending failure to be a + // submission failure as the submission data is successfully persisted and + // the handler is successfully executed, if configured. One can argue that + // email can be essential for the submission processing and missing it + // would result in the incomplete submission. In this case it's natural to + // assume that the web server error log is monitored and the email sending + // failure will be noticed. + // + if (options_->ci_email_specified () && !simulate) + try + { + // Redirect the diagnostics to the web server error log. + // + sendmail sm ([trace] (const char* args[], size_t n) + { + if (trace != nullptr) + *trace << process_args {args, n}; + }, + 2 /* stderr */, + options_->email (), + "CI request submission (" + sr.reference + ')', + {options_->ci_email ()}); + + // Write the CI request manifest. + // + pair> r ( + rqm (sm.out, true /* long_lines */)); + + assert (r.first); // The serialization succeeded once, so can't fail now. + + // Write the CI overrides manifest. + // + sm.out << "\n\n"; + + r = ovm (sm.out, true /* long_lines */); + assert (r.first); // The serialization succeeded once, so can't fail now. + + // Write the CI result manifest. + // + sm.out << "\n\n"; + + // We don't care about the result (see above). + // + rsm (sm.out, true /* long_lines */); + + sm.out.close (); + + if (!sm.wait ()) + error << "sendmail " << *sm.exit; + } + // Handle process_error and io_error (both derive from system_error). + // + catch (const system_error& e) + { + error << "sendmail error: " << e; + } + + return optional (move (sr)); + } + + void ci_start:: + serialize_manifest (const start_result& r, ostream& os, bool long_lines) + { + manifest_serializer s (os, "result", long_lines); + + s.next ("", "1"); // Start of manifest. + s.next ("status", to_string (r.status)); + s.next ("message", r.message); + s.next ("reference", r.reference); + + for (const pair& nv: r.custom_result) + s.next (nv.first, nv.second); + + s.next ("", ""); // End of manifest. + } +} diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx new file mode 100644 index 0000000..6f62c4b --- /dev/null +++ b/mod/ci-common.hxx @@ -0,0 +1,96 @@ +// file : mod/ci-common.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_CI_COMMON_HXX +#define MOD_CI_COMMON_HXX + +#include // database + +#include +#include + +#include + +#include +#include + +namespace brep +{ + class ci_start + { + public: + void + init (shared_ptr); + + // If the request handling has been performed normally, then return the + // information that corresponds to the CI result manifest (see CI Result + // Manifest in the manual). Otherwise (some internal has error occured), + // log the error and return nullopt. + // + // The arguments correspond to the CI request and overrides manifest + // values (see CI Request and Overrides Manifests in the manual). Note: + // request id and timestamp are generated by the implementation. + // + struct package + { + package_name name; + optional version; + }; + // Note that the inability to generate the reference is an internal + // error. Thus, it is not optional. + // + struct start_result + { + uint16_t status; + string message; + string reference; + vector> custom_result; + }; + + // In the optional service information, if id is empty, then the generated + // reference is used instead. + // + optional + start (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + optional&&, + const repository_location& repository, + const vector& packages, + const optional& client_ip, + const optional& user_agent, + const optional& interactive = nullopt, + const optional& simulate = nullopt, + const vector>& custom_request = {}, + const vector>& overrides = {}); + + // Helpers. + // + + // Serialize the start result as a CI result manifest. + // + static void + serialize_manifest (const start_result&, ostream&, bool long_lines = false); + + private: + shared_ptr options_; + }; + + class ci_cancel + { + public: + void + init (shared_ptr, shared_ptr); + + // @@ TODO Archive the tenant. + // + void + cancel (/*...*/); + + private: + shared_ptr options_; + shared_ptr build_db_; + }; +} + +#endif // MOD_CI_COMMON_HXX diff --git a/mod/database-module.cxx b/mod/database-module.cxx index f598bfd..07babc6 100644 --- a/mod/database-module.cxx +++ b/mod/database-module.cxx @@ -3,13 +3,20 @@ #include +#include #include +#include + +#include +#include #include #include namespace brep { + using namespace odb::core; + // While currently the user-defined copy constructor is not required (we // don't need to deep copy nullptr's), it is a good idea to keep the // placeholder ready for less trivial cases. @@ -68,4 +75,53 @@ namespace brep throw; } + + void database_module:: + update_tenant_service_state ( + const connection_ptr& conn, + const string& tid, + const function (const tenant_service&)>& f) + { + assert (f != nullptr); // Shouldn't be called otherwise. + + // Must be initialized via the init(options::build_db) function call. + // + assert (build_db_ != nullptr); + + for (size_t retry (retry_);; ) + { + try + { + transaction tr (conn->begin ()); + + shared_ptr t (build_db_->find (tid)); + + if (t != nullptr && t->service) + { + tenant_service& s (*t->service); + + if (optional data = f (s)) + { + s.data = move (*data); + build_db_->update (t); + } + } + + tr.commit (); + + // Bail out if we have successfully updated the service state. + // + break; + } + catch (const odb::recoverable& e) + { + if (retry-- == 0) + throw; + + HANDLER_DIAG; + l1 ([&]{trace << e << "; " << retry + 1 << " tenant service " + << "state update retries left";}); + } + } + } } diff --git a/mod/database-module.hxx b/mod/database-module.hxx index f72ba83..910cb35 100644 --- a/mod/database-module.hxx +++ b/mod/database-module.hxx @@ -4,7 +4,7 @@ #ifndef MOD_DATABASE_MODULE_HXX #define MOD_DATABASE_MODULE_HXX -#include // database +#include // odb::core::database, odb::core::connection_ptr #include #include @@ -14,6 +14,8 @@ namespace brep { + struct tenant_service; + // A handler that utilises the database. Specifically, it will retry the // request in the face of recoverable database failures (deadlock, loss of // connection, etc) up to a certain number of times. @@ -50,6 +52,25 @@ namespace brep virtual bool handle (request&, response&) = 0; + // Helpers. + // + + // Update the tenant-associated service state if the specified + // notification callback-returned function (expected to be not NULL) + // returns the new state data. + // + // Specifically, start the database transaction, query the service state, + // and call the callback-returned function on this state. If this call + // returns the data string (rather than nullopt), then update the service + // state with this data and persist the change. Repeat all the above steps + // on the recoverable database failures (deadlocks, etc). + // + void + update_tenant_service_state ( + const odb::core::connection_ptr&, + const string& tid, + const function (const tenant_service&)>&); + protected: size_t retry_ = 0; // Max of all retries. diff --git a/mod/external-handler.cxx b/mod/external-handler.cxx index dc4c0fd..3a85bd8 100644 --- a/mod/external-handler.cxx +++ b/mod/external-handler.cxx @@ -15,7 +15,8 @@ #include #include -#include // operator<<(ostream, process_args) +#include // operator<<(ostream, process_args) +#include using namespace std; using namespace butl; diff --git a/mod/external-handler.hxx b/mod/external-handler.hxx index be16e5b..0276a25 100644 --- a/mod/external-handler.hxx +++ b/mod/external-handler.hxx @@ -4,7 +4,7 @@ #ifndef MOD_EXTERNAL_HANDLER_HXX #define MOD_EXTERNAL_HANDLER_HXX -#include +#include #include #include diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx index 04e1883..dea89de 100644 --- a/mod/mod-build-force.cxx +++ b/mod/mod-build-force.cxx @@ -12,20 +12,28 @@ #include #include +#include using namespace std; using namespace brep::cli; using namespace odb::core; +brep::build_force:: +build_force (const tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} + // While currently the user-defined copy constructor is not required (we don't // need to deep copy nullptr's), it is a good idea to keep the placeholder // ready for less trivial cases. // brep::build_force:: -build_force (const build_force& r) +build_force (const build_force& r, const tenant_service_map& tsm) : database_module (r), build_config_module (r), - options_ (r.initialized_ ? r.options_ : nullptr) + options_ (r.initialized_ ? r.options_ : nullptr), + tenant_service_map_ (tsm) { } @@ -173,15 +181,26 @@ handle (request& rq, response& rs) // Load the package build configuration (if present), set the force flag and // update the object's persistent state. // + // If the incomplete package build is being forced to rebuild and the + // tenant_service_build_queued callback is associated with the package + // tenant, then stash the state, the build object, and the callback pointer + // for the subsequent service `queued` notification. + // + const tenant_service_build_queued* tsq (nullptr); + optional>> tss; + + connection_ptr conn (build_db_->connection ()); { - transaction t (build_db_->begin ()); + transaction t (conn->begin ()); package_build pb; + shared_ptr b; + if (!build_db_->query_one ( - query::build::id == id, pb)) + query::build::id == id, pb) || + (b = move (pb.build))->state == build_state::queued) config_expired ("no package build"); - shared_ptr b (pb.build); force_state force (b->state == build_state::built ? force_state::forced : force_state::forcing); @@ -211,11 +230,60 @@ handle (request& rq, response& rs) b->force = force; build_db_->update (b); + + if (force == force_state::forcing) + { + shared_ptr t (build_db_->load (b->tenant)); + + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + tsq = dynamic_cast ( + i->second.get ()); + + if (tsq != nullptr) + { + // If we ought to call the + // tenant_service_build_queued::build_queued() callback, then + // also set the package tenant's queued timestamp to the current + // time to prevent the notifications race (see + // tenant::queued_timestamp for details). + // + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + + tss = make_pair (move (*t->service), move (b)); + } + } + } + } } t.commit (); } + // If the incomplete package build is being forced to rebuild and the + // tenant-associated third-party service needs to be notified about the + // queued builds, then call the tenant_service_build_queued::build_queued() + // callback function and update the service state, if requested. + // + if (tsq != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + const tenant_service& ss (tss->first); + build& b (*tss->second); + + vector qbs; + qbs.push_back (move (b)); + + if (auto f = tsq->build_queued (ss, qbs, build_state::building)) + update_tenant_service_state (conn, qbs.back ().tenant, f); + } + // We have all the data, so don't buffer the response content. // ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); diff --git a/mod/mod-build-force.hxx b/mod/mod-build-force.hxx index 22df383..ea9c141 100644 --- a/mod/mod-build-force.hxx +++ b/mod/mod-build-force.hxx @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -16,13 +17,13 @@ namespace brep class build_force: public database_module, private build_config_module { public: - build_force () = default; + explicit + build_force (const tenant_service_map&); // Create a shallow copy (handling instance) if initialized and a deep // copy (context exemplar) otherwise. // - explicit - build_force (const build_force&); + build_force (const build_force&, const tenant_service_map&); virtual bool handle (request&, response&); @@ -39,6 +40,7 @@ namespace brep private: shared_ptr options_; + const tenant_service_map& tenant_service_map_; }; } diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx index 3841fad..fae506b 100644 --- a/mod/mod-build-log.cxx +++ b/mod/mod-build-log.cxx @@ -227,7 +227,7 @@ handle (request& rq, response& rs) query::build::id == id, pb)) config_expired ("no package build"); - b = pb.build; + b = move (pb.build); if (b->state != build_state::built) config_expired ("state is " + to_string (b->state)); else diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index 24b518d..7023e39 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -24,6 +24,7 @@ #include // *_url() #include +#include using namespace std; using namespace butl; @@ -31,14 +32,21 @@ using namespace bbot; using namespace brep::cli; using namespace odb::core; +brep::build_result:: +build_result (const tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} + // While currently the user-defined copy constructor is not required (we don't // need to deep copy nullptr's), it is a good idea to keep the placeholder // ready for less trivial cases. // brep::build_result:: -build_result (const build_result& r) +build_result (const build_result& r, const tenant_service_map& tsm) : build_result_module (r), - options_ (r.initialized_ ? r.options_ : nullptr) + options_ (r.initialized_ ? r.options_ : nullptr), + tenant_service_map_ (tsm) { } @@ -186,14 +194,33 @@ handle (request& rq, response&) bool build_notify (false); bool unforced (true); + // If the package is built (result status differs from interrupt, etc) and + // the package tenant has a third-party service state associated with it, + // then check if the tenant_service_build_built callback is registered for + // the type of the associated service. If it is, then stash the state, the + // build object, and the callback pointer for the subsequent service `built` + // notification. Note that we send this notification for the skip result as + // well, since it is semantically equivalent to the previous build result + // with the actual build process being optimized out. + // + // If the package build is interrupted and the tenant_service_build_queued + // callback is associated with the package tenant, then stash the state, the + // build object, and the callback pointer for the subsequent service + // `queued` notification. + // + const tenant_service_build_built* tsb (nullptr); + const tenant_service_build_queued* tsq (nullptr); + optional>> tss; + // Note that if the session authentication fails (probably due to the // authentication settings change), then we log this case with the warning // severity and respond with the 200 HTTP code as if the challenge is // valid. The thinking is that we shouldn't alarm a law-abaiding agent and // shouldn't provide any information to a malicious one. // + connection_ptr conn (build_db_->connection ()); { - transaction t (build_db_->begin ()); + transaction t (conn->begin ()); package_build pb; @@ -221,11 +248,38 @@ handle (request& rq, response&) } else if (authenticate_session (*options_, rqm.challenge, *b, rqm.session)) { + const tenant_service_base* ts (nullptr); + + shared_ptr t (build_db_->load (b->tenant)); + + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + ts = i->second.get (); + } + // If the build is interrupted, then revert it to the original built - // state if this is a rebuild and delete it from the database otherwise. + // state if this is a rebuild. Otherwise (initial build), turn the build + // into the queued state if the tenant_service_build_queued callback is + // registered for the package tenant and delete it from the database + // otherwise. + // + // Note that if the tenant_service_build_queued callback is registered, + // we always send the `queued` notification for the interrupted build, + // even when we reverse it to the original built state. We could also + // turn the build into the queued state in this case, but it feels that + // there is no harm in keeping the previous build information available + // for the user. // if (rs == result_status::interrupt) { + // Schedule the `queued` notification, if the + // tenant_service_build_queued callback is registered for the tenant. + // + tsq = dynamic_cast (ts); + if (b->status) // Is this a rebuild? { b->state = build_state::built; @@ -248,14 +302,57 @@ handle (request& rq, response&) // Note that we are unable to restore the pre-rebuild timestamp // since it has been overwritten when the build task was issued. // That, however, feels ok and we just keep it unchanged. + // + // Moreover, we actually use the fact that the build's timestamp is + // greater then its soft_timestamp as an indication that the build + // object represents the interrupted rebuild (see the build_task + // handler for details). build_db_->update (b); } - else - build_db_->erase (b); + else // Initial build. + { + if (tsq != nullptr) + { + // Since this is not a rebuild, there are no operation results and + // thus we don't need to load the results section to erase results + // from the database. + // + assert (b->results.empty ()); + + *b = build (move (b->tenant), + move (b->package_name), + move (b->package_version), + move (b->target), + move (b->target_config_name), + move (b->package_config_name), + move (b->toolchain_name), + move (b->toolchain_version)); + + build_db_->update (b); + } + else + build_db_->erase (b); + } + + // If we ought to call the tenant_service_build_queued::build_queued() + // callback, then also set the package tenant's queued timestamp to + // the current time to prevent the notifications race (see + // tenant::queued_timestamp for details). + // + if (tsq != nullptr) + { + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + } } - else + else // Regular or skip build result. { + // Schedule the `built` notification, if the + // tenant_service_build_built callback is registered for the tenant. + // + tsb = dynamic_cast (ts); + // Verify the result status/checksums. // // Specifically, if the result status is skip, then it can only be in @@ -334,7 +431,8 @@ handle (request& rq, response&) b->soft_timestamp = b->timestamp; // If the result status is other than skip, then save the status, - // results, and checksums and update the hard timestamp. + // results, and checksums and update the hard timestamp. Also stash + // the service notification information, if present. // if (rs != result_status::skip) { @@ -372,18 +470,61 @@ handle (request& rq, response&) build_db_->load (*pkg, pkg->constraints_section); if (!exclude (*cfg, pkg->builds, pkg->constraints, *tc)) - bld = move (b); + bld = b; } } else warn << "cannot find configuration '" << b->package_config_name << "' for package " << pkg->id.name << '/' << pkg->version; } + + // If required, stash the service notification information. + // + if (tsb != nullptr || tsq != nullptr) + tss = make_pair (move (*t->service), move (b)); } t.commit (); } + // We either notify about the queued build or notify about the built package + // or don't notify at all. + // + assert (tsb == nullptr || tsq == nullptr); + + // If the package build is interrupted and the tenant-associated third-party + // service needs to be notified about the queued builds, then call the + // tenant_service_build_queued::build_queued() callback function and update + // the service state, if requested. + // + if (tsq != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + const tenant_service& ss (tss->first); + + vector qbs; + qbs.push_back (move (*tss->second)); + + if (auto f = tsq->build_queued (ss, qbs, build_state::building)) + update_tenant_service_state (conn, qbs.back ().tenant, f); + } + + // If a third-party service needs to be notified about the built package, + // then call the tenant_service_build_built::build_built() callback function + // and update the service state, if requested. + // + if (tsb != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + const tenant_service& ss (tss->first); + const build& b (*tss->second); + + if (auto f = tsb->build_built (ss, b)) + update_tenant_service_state (conn, b.tenant, f); + } + if (bld == nullptr) return true; diff --git a/mod/mod-build-result.hxx b/mod/mod-build-result.hxx index 87ef1f2..96449d5 100644 --- a/mod/mod-build-result.hxx +++ b/mod/mod-build-result.hxx @@ -8,6 +8,7 @@ #include #include +#include #include namespace brep @@ -15,13 +16,13 @@ namespace brep class build_result: public build_result_module { public: - build_result () = default; + explicit + build_result (const tenant_service_map&); // Create a shallow copy (handling instance) if initialized and a deep // copy (context exemplar) otherwise. // - explicit - build_result (const build_result&); + build_result (const build_result&, const tenant_service_map&); virtual bool handle (request&, response&); @@ -35,6 +36,7 @@ namespace brep private: shared_ptr options_; + const tenant_service_map& tenant_service_map_; }; } diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 01d14cd..1a06ce1 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -58,15 +58,22 @@ rand (size_t min_val, size_t max_val) static_cast (max_val)) (rand_gen)); } +brep::build_task:: +build_task (const tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} + // While currently the user-defined copy constructor is not required (we don't // need to deep copy nullptr's), it is a good idea to keep the placeholder // ready for less trivial cases. // brep::build_task:: -build_task (const build_task& r) +build_task (const build_task& r, const tenant_service_map& tsm) : database_module (r), build_config_module (r), - options_ (r.initialized_ ? r.options_ : nullptr) + options_ (r.initialized_ ? r.options_ : nullptr), + tenant_service_map_ (tsm) { } @@ -117,9 +124,15 @@ init (scanner& s) options_->root (dir_path ("/")); } +// Skip tenants with the freshly queued packages from the consideration (see +// tenant::queued_timestamp for the details on the service notifications race +// prevention). +// template static inline query -package_query (brep::params::build_task& params, interactive_mode imode) +package_query (brep::params::build_task& params, + interactive_mode imode, + uint64_t queued_expiration_ns) { using namespace brep; using query = query; @@ -153,7 +166,9 @@ package_query (brep::params::build_task& params, interactive_mode imode) case interactive_mode::both: break; } - return q; + return q && + (query::build_tenant::queued_timestamp.is_null () || + query::build_tenant::queued_timestamp < queued_expiration_ns); } bool brep::build_task:: @@ -261,7 +276,7 @@ handle (request& rq, response& rs) config_machines conf_machines; - for (const auto& c: *target_conf_) + for (const build_target_config& c: *target_conf_) { for (auto& m: tqm.machines) { @@ -303,34 +318,34 @@ handle (request& rq, response& rs) // Create the task response manifest. Must be called inside the build db // transaction. // - auto task = [this] (shared_ptr&& b, - shared_ptr&& p, + auto task = [this] (const build& b, + build_package&& p, build_package_config&& pc, optional&& interactive, const config_machine& cm) -> task_response_manifest { uint64_t ts ( chrono::duration_cast ( - b->timestamp.time_since_epoch ()).count ()); - - string session (b->tenant + '/' + - b->package_name.string () + '/' + - b->package_version.string () + '/' + - b->target.string () + '/' + - b->target_config_name + '/' + - b->package_config_name + '/' + - b->toolchain_name + '/' + - b->toolchain_version.string () + '/' + + b.timestamp.time_since_epoch ()).count ()); + + string session (b.tenant + '/' + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string () + '/' + to_string (ts)); - string tenant (tenant_dir (options_->root (), b->tenant).string ()); + string tenant (tenant_dir (options_->root (), b.tenant).string ()); string result_url (options_->host () + tenant + "?build-result"); assert (transaction::has_current ()); - assert (p->internal ()); // The package is expected to be buildable. + assert (p.internal ()); // The package is expected to be buildable. - shared_ptr r (p->internal_repository.load ()); + shared_ptr r (p.internal_repository.load ()); strings fps; if (r->certificate_fingerprint) @@ -341,9 +356,9 @@ handle (request& rq, response& rs) // small_vector tests; - build_db_->load (*p, p->requirements_tests_section); + build_db_->load (p, p.requirements_tests_section); - for (const build_test_dependency& td: p->tests) + for (const build_test_dependency& td: p.tests) { // Don't exclude unresolved external tests. // @@ -390,16 +405,17 @@ handle (request& rq, response& rs) move (td.reflect)); } - bool module_pkg ( - b->package_name.string ().compare (0, 10, "libbuild2-") == 0); + package_name& pn (p.id.name); - task_manifest task (move (b->package_name), - move (b->package_version), + bool module_pkg (pn.string ().compare (0, 10, "libbuild2-") == 0); + + task_manifest task (move (pn), + move (p.version), move (r->location), move (fps), - move (p->requirements), + move (p.requirements), move (tests), - move (b->dependency_checksum), + b.dependency_checksum, cm.machine->name, cm.config->target, cm.config->environment, @@ -408,7 +424,7 @@ handle (request& rq, response& rs) belongs (*cm.config, module_pkg ? "build2" : "host"), cm.config->warning_regexes, move (interactive), - move (b->worker_checksum)); + b.worker_checksum); // Collect the build artifacts upload URLs, skipping those which are // excluded with the upload-*-exclude configuration options. @@ -434,7 +450,7 @@ handle (request& rq, response& rs) }; if (!exclude (options_->upload_toolchain_exclude (), - b->toolchain_name) && + b.toolchain_name) && !exclude (options_->upload_repository_exclude (), r->canonical_name)) { @@ -444,15 +460,15 @@ handle (request& rq, response& rs) } return task_response_manifest (move (session), - move (b->agent_challenge), + b.agent_challenge, move (result_url), move (upload_urls), - move (b->agent_checksum), + b.agent_checksum, move (task)); }; - // Calculate the build (building state) or rebuild (built state) - // expiration time for package configurations. + // Calculate the build/rebuild (building/built state) and the `queued` + // notifications expiration time for package configurations. // timestamp now (system_clock::now ()); @@ -476,6 +492,9 @@ handle (request& rq, response& rs) timestamp forced_rebuild_expiration ( expiration (options_->build_forced_rebuild_timeout ())); + uint64_t queued_expiration_ns ( + expiration_ns (options_->build_queued_timeout ())); + // Calculate the soft/hard rebuild expiration time, based on the // respective build-{soft,hard}-rebuild-timeout and // build-alt-{soft,hard}-rebuild-{start,stop,timeout} configuration @@ -626,6 +645,7 @@ handle (request& rq, response& rs) // Convert butl::standard_version type to brep::version. // brep::version toolchain_version (tqm.toolchain_version.string ()); + string& toolchain_name (tqm.toolchain_name); // Prepare the buildable package prepared query. // @@ -646,7 +666,9 @@ handle (request& rq, response& rs) using pkg_query = query; using prep_pkg_query = prepared_query; - pkg_query pq (package_query (params, imode)); + pkg_query pq (package_query (params, + imode, + queued_expiration_ns)); // Transform (in-place) the interactive login information into the actual // login command, if specified in the manifest and the transformation @@ -800,7 +822,9 @@ handle (request& rq, response& rs) { using query = query; - query q (package_query (params, imode)); + query q (package_query (params, + imode, + queued_expiration_ns)); transaction t (build_db_->begin ()); @@ -896,17 +920,18 @@ handle (request& rq, response& rs) equal (bld_query::id.package, id) && bld_query::id.package_config_name == bld_query::_ref (pkg_config) && sq && - bld_query::id.toolchain_name == tqm.toolchain_name && + bld_query::id.toolchain_name == toolchain_name && compare_version_eq (bld_query::id.toolchain_version, canonical_version (toolchain_version), true /* revision */) && - (bld_query::state == "built" || - (bld_query::force == "forcing" && - bld_query::timestamp > forced_result_expiration_ns) || - (bld_query::force != "forcing" && // Unforced or forced. - bld_query::timestamp > normal_result_expiration_ns))); + (bld_query::state == "built" || + (bld_query::state == "building" && + ((bld_query::force == "forcing" && + bld_query::timestamp > forced_result_expiration_ns) || + (bld_query::force != "forcing" && // Unforced or forced. + bld_query::timestamp > normal_result_expiration_ns))))); prep_bld_query bld_prep_query ( conn->prepare_query ("mod-build-task-build-query", bq)); @@ -971,6 +996,107 @@ handle (request& rq, response& rs) // optional start_tenant; + // If the build task is created and the tenant of the being built + // package has a third-party service state associated with it, then + // check if the tenant_service_build_building and/or + // tenant_service_build_queued callbacks are registered for the type of + // the associated service. If they are, then stash the state, the build + // object, and the callback pointers for the subsequent service + // notifications. + // + // Also, if the tenant_service_build_queued callback is registered, then + // create, persist, and stash the queued build objects for all the + // unbuilt by the current toolchain and not yet queued configurations of + // the package the build task is created for. Note that for the task + // build, we need to make sure that the third-party service receives the + // `queued` notification prior to the `building` notification (see + // mod/tenant-service.hxx for valid transitions). The `queued` + // notification is assumed to be already sent for the build if the + // respective object exists and any of the following is true for it: + // + // - It is in the queued state (initial_state is build_state::queued). + // + // - It is a user-forced rebuild of an incomplete build + // (rebuild_forced_build is true). + // + // - It is a rebuild of an interrupted rebuild (rebuild_forced_build is + // true). + // + const tenant_service_build_building* tsb (nullptr); + const tenant_service_build_queued* tsq (nullptr); + optional>> tss; + vector qbs; + optional initial_state; + bool rebuild_forced_build (false); + bool rebuild_interrupted_rebuild (false); + + // Create, persist, and return the queued build objects for all the + // unbuilt by the current toolchain and not yet queued configurations of + // the specified package. + // + // Note that the build object argument is only used for the toolchain + // information retrieval. Also note that the package constraints section + // is expected to be loaded. + // + auto queue_builds = [this] (const build_package& p, const build& b) + { + assert (p.constraints_section.loaded ()); + + // Query the existing build ids and stash them into the set. + // + set existing_builds; + + using query = query; + + query q (query::build::id.package == p.id && + query::build::id.toolchain_name == b.toolchain_name && + compare_version_eq (query::build::id.toolchain_version, + b.id.toolchain_version, + true /* revision */)); + + for (build_id& id: build_db_->query (q)) + existing_builds.emplace (move (id)); + + // Go through all the potential package builds and queue those which + // are not in the existing builds set. + // + vector r; + + for (const build_package_config& pc: p.configs) + { + for (const build_target_config& tc: *target_conf_) + { + if (!exclude (pc, p.builds, p.constraints, tc)) + { + build_id id (p.id, + tc.target, tc.name, + pc.name, + b.toolchain_name, b.toolchain_version); + + if (existing_builds.find (id) == existing_builds.end ()) + { + r.emplace_back (move (id.package.tenant), + move (id.package.name), + p.version, + move (id.target), + move (id.target_config_name), + move (id.package_config_name), + move (id.toolchain_name), + b.toolchain_version); + + // @@ TODO Persist the whole vector of builds with a single + // operation if/when bulk operations support is added + // for objects with containers. + // + build_db_->persist (r.back ()); + } + } + } + } + + return r; + }; + for (bool done (false); tsm.session.empty () && !done; ) { transaction t (conn->begin ()); @@ -1163,7 +1289,7 @@ handle (request& rq, response& rs) tc.target, tc.name, pc, - tqm.toolchain_name, + toolchain_name, toolchain_version); // Can there be any existing builds for such a tenant? Doesn't @@ -1279,7 +1405,7 @@ handle (request& rq, response& rs) cm.config->target, cm.config->name, move (pkg_config), - move (tqm.toolchain_name), + move (toolchain_name), toolchain_version); shared_ptr b (build_db_->find (bid)); @@ -1318,16 +1444,20 @@ handle (request& rq, response& rs) } else { - // The build configuration is in the building state. + // The build configuration is in the building or queued + // state. // - // Note that in both cases we keep the status intact to be - // able to compare it with the final one in the result - // request handling in order to decide if to send the - // notification email or to revert it to the built state if - // interrupted. The same is true for the forced flag (in - // the sense that we don't set the force state to unforced). + // Note that in both the building and built cases we keep + // the status intact to be able to compare it with the final + // one in the result request handling in order to decide if + // to send the notification email or to revert it to the + // built state if interrupted. The same is true for the + // forced flag (in the sense that we don't set the force + // state to unforced). // - assert (b->state == build_state::building); + assert (b->state != build_state::built); + + initial_state = b->state; b->state = build_state::building; b->interactive = move (login); @@ -1337,7 +1467,10 @@ handle (request& rq, response& rs) // still recognize that the rebuild was forced. // if (b->force == force_state::forcing) + { b->force = force_state::forced; + rebuild_forced_build = true; + } b->agent_fingerprint = move (agent_fp); b->agent_challenge = move (cl); @@ -1364,21 +1497,58 @@ handle (request& rq, response& rs) build_db_->update (b); } + shared_ptr t ( + build_db_->load (b->tenant)); + // Archive an interactive tenant. // if (bp.interactive) { - shared_ptr t ( - build_db_->load (b->id.package.tenant)); - t->archived = true; build_db_->update (t); } - // Finally, prepare the task response manifest. + // Finally, stash the service notification information, if + // present, and prepare the task response manifest. // + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + const tenant_service_base* s (i->second.get ()); + + tsb = dynamic_cast (s); + tsq = dynamic_cast (s); + + if (tsq != nullptr) + { + qbs = queue_builds (*p, *b); + + // If we ought to call the + // tenant_service_build_queued::build_queued() callback, + // then also set the package tenant's queued timestamp + // to the current time to prevent the notifications race + // (see tenant::queued_timestamp for details). + // + if (!qbs.empty () || + !initial_state || + (*initial_state != build_state::queued && + !rebuild_forced_build)) + { + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + } + } + + if (tsb != nullptr || tsq != nullptr) + tss = make_pair (move (*t->service), b); + } + } + tsm = task ( - move (b), move (p), move (pc), move (bp.interactive), cm); + *b, move (*p), move (pc), move (bp.interactive), cm); break; // Bail out from the package configurations loop. } @@ -1493,6 +1663,11 @@ handle (request& rq, response& rs) { assert (b->status); + initial_state = build_state::built; + + rebuild_interrupted_rebuild = + (b->timestamp > b->soft_timestamp); + b->state = build_state::building; // Save the interactive build login information into the @@ -1535,8 +1710,45 @@ handle (request& rq, response& rs) build_db_->update (b); + // Stash the service notification information, if present, + // and prepare the task response manifest. + // + if (t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + const tenant_service_base* s (i->second.get ()); + + tsb = dynamic_cast (s); + tsq = dynamic_cast (s); + + if (tsq != nullptr) + { + qbs = queue_builds (*p, *b); + + // If we ought to call the + // tenant_service_build_queued::build_queued() + // callback, then also set the package tenant's queued + // timestamp to the current time to prevent the + // notifications race (see tenant::queued_timestamp + // for details). + // + if (!qbs.empty () || !rebuild_interrupted_rebuild) + { + t->queued_timestamp = system_clock::now (); + build_db_->update (t); + } + } + + if (tsb != nullptr || tsq != nullptr) + tss = make_pair (move (*t->service), b); + } + } + tsm = task ( - move (b), move (p), move (*pc), move (t->interactive), cm); + *b, move (*p), move (*pc), move (t->interactive), cm); } } } @@ -1558,6 +1770,73 @@ handle (request& rq, response& rs) break; } } + + // If the tenant-associated third-party service needs to be notified + // about the queued builds, then call the + // tenant_service_build_queued::build_queued() callback function and + // update the service state, if requested. + // + if (tsq != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + const tenant_service& ss (tss->first); + + // If the task build has no initial state (is just created), then + // temporarily move it into the list of the queued builds until the + // `queued` notification is delivered. Afterwards, restore it so that + // the `building` notification can also be sent. + // + build& b (*tss->second); + bool restore_build (false); + + if (!initial_state) + { + qbs.push_back (move (b)); + restore_build = true; + } + + if (!qbs.empty ()) + { + if (auto f = tsq->build_queued (ss, qbs, nullopt /* initial_state */)) + update_tenant_service_state (conn, qbs.back ().tenant, f); + } + + // Send the `queued` notification for the task build, unless it is + // already sent, and update the service state, if requested. + // + if (initial_state && + *initial_state != build_state::queued && + !rebuild_interrupted_rebuild && + !rebuild_forced_build) + { + qbs.clear (); + qbs.push_back (move (b)); + restore_build = true; + + if (auto f = tsq->build_queued (ss, qbs, initial_state)) + update_tenant_service_state (conn, qbs.back ().tenant, f); + } + + if (restore_build) + b = move (qbs.back ()); + } + + // If a third-party service needs to be notified about the package + // build, then call the tenant_service_build_built::build_building() + // callback function and, if requested, update the tenant-associated + // service state. + // + if (tsb != nullptr) + { + assert (tss); // Wouldn't be here otherwise. + + const tenant_service& ss (tss->first); + const build& b (*tss->second); + + if (auto f = tsb->build_building (ss, b)) + update_tenant_service_state (conn, b.tenant, f); + } } } diff --git a/mod/mod-build-task.hxx b/mod/mod-build-task.hxx index 7875db1..d0b3d44 100644 --- a/mod/mod-build-task.hxx +++ b/mod/mod-build-task.hxx @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -16,13 +17,13 @@ namespace brep class build_task: public database_module, private build_config_module { public: - build_task () = default; + explicit + build_task (const tenant_service_map&); // Create a shallow copy (handling instance) if initialized and a deep // copy (context exemplar) otherwise. // - explicit - build_task (const build_task&); + build_task (const build_task&, const tenant_service_map&); virtual bool handle (request&, response&); @@ -36,6 +37,7 @@ namespace brep private: shared_ptr options_; + const tenant_service_map& tenant_service_map_; }; } diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index f260b72..b0de618 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -225,13 +225,19 @@ build_query (const brep::vector* config_ids, // Build result. // const string& rs (params.result ()); + bool add_state (true); if (rs != "*") { if (rs == "pending") + { q = q && qb::force != "unforced"; + } else if (rs == "building") + { q = q && qb::state == "building"; + add_state = false; + } else { query sq (qb::status == rs); @@ -259,8 +265,12 @@ build_query (const brep::vector* config_ids, // well (rebuild). // q = q && qb::state == "built" && sq; + add_state = false; } } + + if (add_state) + q = q && qb::state != "queued"; } catch (const invalid_argument&) { diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index df2365a..fec603e 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -3,18 +3,11 @@ #include -#include - -#include -#include #include -#include -#include -#include // operator<<(ostream, process_args) #include #include -#include +#include // package_manifest #include #include @@ -23,20 +16,35 @@ #include #include -#include using namespace std; using namespace butl; using namespace web; using namespace brep::cli; +#ifdef BREP_CI_TENANT_SERVICE +brep::ci:: +ci (tenant_service_map& tsm) + : tenant_service_map_ (tsm) +{ +} +#endif + brep::ci:: +#ifdef BREP_CI_TENANT_SERVICE +ci (const ci& r, tenant_service_map& tsm) +#else ci (const ci& r) +#endif : handler (r), + ci_start (r), options_ (r.initialized_ ? r.options_ : nullptr), form_ (r.initialized_ || r.form_ == nullptr ? r.form_ : make_shared (*r.form_)) +#ifdef BREP_CI_TENANT_SERVICE + , tenant_service_map_ (tsm) +#endif { } @@ -45,22 +53,25 @@ init (scanner& s) { HANDLER_DIAG; +#ifdef BREP_CI_TENANT_SERVICE + { + shared_ptr ts ( + dynamic_pointer_cast (shared_from_this ())); + + assert (ts != nullptr); // By definition. + + tenant_service_map_["ci"] = move (ts); + } +#endif + options_ = make_shared ( s, unknown_mode::fail, unknown_mode::fail); - // Verify that the CI request handling is setup properly, if configured. + // Prepare for the CI requests handling, if configured. // if (options_->ci_data_specified ()) { - // Verify the data directory satisfies the requirements. - // - const dir_path& d (options_->ci_data ()); - - if (d.relative ()) - fail << "ci-data directory path must be absolute"; - - if (!dir_exists (d)) - fail << "ci-data directory '" << d << "' does not exist"; + ci_start::init (make_shared (*options_)); // Parse XHTML5 form file, if configured. // @@ -87,10 +98,6 @@ init (scanner& s) fail << "unable to read ci-form file '" << ci_form << "': " << e; } } - - if (options_->ci_handler_specified () && - options_->ci_handler ().relative ()) - fail << "ci-handler path must be absolute"; } if (options_->root ().empty ()) @@ -130,9 +137,8 @@ handle (request& rq, response& rs) // // return respond_error (); // Request is handled with an error. // - string request_id; // Will be set later. - auto respond_manifest = [&rs, &request_id] (status_code status, - const string& message) -> bool + auto respond_manifest = [&rs] (status_code status, + const string& message) -> bool { serializer s (rs.content (status, "text/manifest;charset=utf-8"), "response"); @@ -140,10 +146,6 @@ handle (request& rq, response& rs) s.next ("", "1"); // Start of manifest. s.next ("status", to_string (status)); s.next ("message", message); - - if (!request_id.empty ()) - s.next ("reference", request_id); - s.next ("", ""); // End of manifest. return true; }; @@ -234,9 +236,11 @@ handle (request& rq, response& rs) if (rl.empty () || rl.local ()) return respond_manifest (400, "invalid repository location"); - // Verify the package name[/version] arguments. + // Parse the package name[/version] arguments. // - for (const string& s: params.package()) + vector packages; + + for (const string& s: params.package ()) { // Let's skip the potentially unfilled package form fields. // @@ -245,18 +249,21 @@ handle (request& rq, response& rs) try { + package pkg; size_t p (s.find ('/')); if (p != string::npos) { - package_name (string (s, 0, p)); + pkg.name = package_name (string (s, 0, p)); // Not to confuse with module::version. // - bpkg::version (string (s, p + 1)); + pkg.version = bpkg::version (string (s, p + 1)); } else - package_name p (s); // Not to confuse with the s variable declaration. + pkg.name = package_name (s); + + packages.push_back (move (pkg)); } catch (const invalid_argument&) { @@ -265,31 +272,49 @@ handle (request& rq, response& rs) } // Verify that unknown parameter values satisfy the requirements (contain - // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n'). + // only UTF-8 encoded graphic characters plus '\t', '\r', and '\n') and + // stash them. // // Actually, the expected ones must satisfy too, so check them as well. // - string what; - for (const name_value& nv: rps) + vector> custom_request; { - if (nv.value && - !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t")) - return respond_manifest (400, - "invalid parameter " + nv.name + ": " + what); + string what; + for (const name_value& nv: rps) + { + if (nv.value && + !utf8 (*nv.value, what, codepoint_types::graphic, U"\n\r\t")) + return respond_manifest (400, + "invalid parameter " + nv.name + ": " + what); + + const string& n (nv.name); + + if (n != "repository" && + n != "_" && + n != "package" && + n != "overrides" && + n != "interactive" && + n != "simulate") + custom_request.emplace_back (n, nv.value ? *nv.value : ""); + } } // Parse and validate overrides, if present. // - vector overrides; + vector> overrides; if (params.overrides_specified ()) try { istream& is (rq.open_upload ("overrides")); parser mp (is, "overrides"); - overrides = parse_manifest (mp); + vector ovrs (parse_manifest (mp)); + + package_manifest::validate_overrides (ovrs, mp.name ()); - package_manifest::validate_overrides (overrides, mp.name ()); + overrides.reserve (ovrs.size ()); + for (manifest_name_value& nv: ovrs) + overrides.emplace_back (move (nv.name), move (nv.value)); } // Note that invalid_argument (thrown by open_upload() function call) can // mean both no overrides upload or multiple overrides uploads. @@ -310,383 +335,127 @@ handle (request& rq, response& rs) return respond_error (); } - try - { - // Note that from now on the result manifest we respond with will contain - // the reference value. - // - request_id = uuid::generate ().string (); - } - catch (const system_error& e) - { - error << "unable to generate request id: " << e; - return respond_error (); - } - - // Create the submission data directory. + // Stash the User-Agent HTTP header and the client IP address. // - dir_path dd (options_->ci_data () / dir_path (request_id)); - - try + optional client_ip; + optional user_agent; + for (const name_value& h: rq.headers ()) { - // It's highly unlikely but still possible that the directory already - // exists. This can only happen if the generated uuid is not unique. - // - if (try_mkdir (dd) == mkdir_status::already_exists) - throw_generic_error (EEXIST); + if (icasecmp (h.name, ":Client-IP") == 0) + client_ip = h.value; + else if (icasecmp (h.name, "User-Agent") == 0) + user_agent = h.value; } - catch (const system_error& e) - { - error << "unable to create directory '" << dd << "': " << e; - return respond_error (); - } - - auto_rmdir ddr (dd); - - // Serialize the CI request manifest to a stream. On the serialization error - // respond to the client with the manifest containing the bad request (400) - // code and return false, on the stream error pass through the io_error - // exception, otherwise return true. - // - timestamp ts (system_clock::now ()); - - auto rqm = [&request_id, - &rl, - &ts, - &simulate, - &rq, - &rps, - ¶ms, - &respond_manifest] - (ostream& os, bool long_lines = false) -> bool - { - try - { - serializer s (os, "request", long_lines); - // Serialize the submission manifest header. - // - s.next ("", "1"); // Start of manifest. - s.next ("id", request_id); - s.next ("repository", rl.string ()); - - for (const string& p: params.package ()) - { - if (!p.empty ()) // Skip empty package names (see above for details). - s.next ("package", p); - } - - if (params.interactive_specified ()) - s.next ("interactive", params.interactive ()); - - if (!simulate.empty ()) - s.next ("simulate", simulate); - - s.next ("timestamp", - butl::to_string (ts, - "%Y-%m-%dT%H:%M:%SZ", - false /* special */, - false /* local */)); - - // Serialize the User-Agent HTTP header and the client IP address. - // - optional ip; - optional ua; - for (const name_value& h: rq.headers ()) - { - if (icasecmp (h.name, ":Client-IP") == 0) - ip = h.value; - else if (icasecmp (h.name, "User-Agent") == 0) - ua = h.value; - } - - if (ip) - s.next ("client-ip", *ip); - - if (ua) - s.next ("user-agent", *ua); - - // Serialize the request parameters. - // - // Note that the serializer constraints the parameter names (can't start - // with '#', can't contain ':' and the whitespaces, etc.). - // - for (const name_value& nv: rps) - { - const string& n (nv.name); - - if (n != "repository" && - n != "_" && - n != "package" && - n != "overrides" && - n != "interactive" && - n != "simulate") - s.next (n, nv.value ? *nv.value : ""); - } - - s.next ("", ""); // End of manifest. - return true; - } - catch (const serialization& e) - { - respond_manifest (400, string ("invalid parameter: ") + e.what ()); - return false; - } - }; - - // Serialize the CI request manifest to the submission directory. - // - path rqf (dd / "request.manifest"); + optional r (start (error, + warn, + verb_ ? &trace : nullptr, +#ifdef BREP_CI_TENANT_SERVICE + tenant_service ("", "ci"), +#else + nullopt /* service */, +#endif + rl, + packages, + client_ip, + user_agent, + (params.interactive_specified () + ? params.interactive () + : optional ()), + (!simulate.empty () + ? simulate + : optional ()), + custom_request, + overrides)); + + if (!r) + return respond_error (); // The diagnostics is already issued. try { - ofdstream os (rqf); - bool r (rqm (os)); - os.close (); - - if (!r) - return true; // The client is already responded with the manifest. - } - catch (const io_error& e) - { - error << "unable to write to '" << rqf << "': " << e; - return respond_error (); + serialize_manifest (*r, + rs.content (r->status, "text/manifest;charset=utf-8")); } - - // Serialize the CI overrides manifest to a stream. On the stream error pass - // through the io_error exception. - // - // Note that it can't throw the serialization exception as the override - // manifest is parsed from the stream and so verified. - // - auto ovm = [&overrides] (ostream& os, bool long_lines = false) + catch (const serialization& e) { - try - { - serializer s (os, "overrides", long_lines); - serialize_manifest (s, overrides); - } - catch (const serialization&) {assert (false);} // See above. - }; + error << "ref " << r->reference << ": unable to serialize handler's " + << "output: " << e; - // Serialize the CI overrides manifest to the submission directory. - // - path ovf (dd / "overrides.manifest"); - - if (!overrides.empty ()) - try - { - ofdstream os (ovf); - ovm (os); - os.close (); - } - catch (const io_error& e) - { - error << "unable to write to '" << ovf << "': " << e; return respond_error (); } - // Given that the submission data is now successfully persisted we are no - // longer in charge of removing it, except for the cases when the submission - // handler terminates with an error (see below for details). - // - ddr.cancel (); - - // If the handler terminates with non-zero exit status or specifies 5XX - // (HTTP server error) submission result manifest status value, then we - // stash the submission data directory for troubleshooting. Otherwise, if - // it's the 4XX (HTTP client error) status value, then we remove the - // directory. - // - auto stash_submit_dir = [&dd, error] () - { - if (dir_exists (dd)) - try - { - mvdir (dd, dir_path (dd + ".fail")); - } - catch (const system_error& e) - { - // Not much we can do here. Let's just log the issue and bail out - // leaving the directory in place. - // - error << "unable to rename directory '" << dd << "': " << e; - } - }; - - // Run the submission handler, if specified, reading the result manifest - // from its stdout and caching it as a name/value pair list for later use - // (forwarding to the client, sending via email, etc). Otherwise, create - // implied result manifest. - // - status_code sc; - vector rvs; - - if (options_->ci_handler_specified ()) - { - using namespace external_handler; - - optional r (run (options_->ci_handler (), - options_->ci_handler_argument (), - dd, - options_->ci_handler_timeout (), - error, - warn, - verb_ ? &trace : nullptr)); - if (!r) - { - stash_submit_dir (); - return respond_error (); // The diagnostics is already issued. - } - - sc = r->status; - rvs = move (r->values); - } - else // Create the implied result manifest. - { - sc = 200; - - auto add = [&rvs] (string n, string v) - { - manifest_name_value nv { - move (n), move (v), - 0 /* name_line */, 0 /* name_column */, - 0 /* value_line */, 0 /* value_column */, - 0 /* start_pos */, 0 /* colon_pos */, 0 /* end_pos */}; - - rvs.emplace_back (move (nv)); - }; - - add ("status", "200"); - add ("message", "CI request is queued"); - add ("reference", request_id); - } - - assert (!rvs.empty ()); // Produced by the handler or is implied. - - // Serialize the submission result manifest to a stream. On the - // serialization error log the error description and return false, on the - // stream error pass through the io_error exception, otherwise return true. - // - auto rsm = [&rvs, &error, &request_id] (ostream& os, - bool long_lines = false) -> bool - { - try - { - serializer s (os, "result", long_lines); - serialize_manifest (s, rvs); - return true; - } - catch (const serialization& e) - { - error << "ref " << request_id << ": unable to serialize handler's " - << "output: " << e; - return false; - } - }; - - // If the submission data directory still exists then perform an appropriate - // action on it, depending on the submission result status. Note that the - // handler could move or remove the directory. - // - if (dir_exists (dd)) - { - // Remove the directory if the client error is detected. - // - if (sc >= 400 && sc < 500) - { - rmdir_r (dd); - } - // - // Otherwise, save the result manifest, into the directory. Also stash the - // directory for troubleshooting in case of the server error. - // - else - { - path rsf (dd / "result.manifest"); - - try - { - ofdstream os (rsf); - - // Not being able to stash the result manifest is not a reason to - // claim the submission failed. The error is logged nevertheless. - // - rsm (os); - - os.close (); - } - catch (const io_error& e) - { - // Not fatal (see above). - // - error << "unable to write to '" << rsf << "': " << e; - } - - if (sc >= 500 && sc < 600) - stash_submit_dir (); - } - } - - // Send email, if configured, and the CI request submission is not simulated. - // Use the long lines manifest serialization mode for the convenience of - // copying/clicking URLs they contain. - // - // Note that we don't consider the email sending failure to be a submission - // failure as the submission data is successfully persisted and the handler - // is successfully executed, if configured. One can argue that email can be - // essential for the submission processing and missing it would result in - // the incomplete submission. In this case it's natural to assume that the - // web server error log is monitored and the email sending failure will be - // noticed. - // - if (options_->ci_email_specified () && simulate.empty ()) - try - { - // Redirect the diagnostics to the web server error log. - // - sendmail sm ([&trace, this] (const char* args[], size_t n) - { - l2 ([&]{trace << process_args {args, n};}); - }, - 2 /* stderr */, - options_->email (), - "CI request submission (" + request_id + ')', - {options_->ci_email ()}); - - // Write the CI request manifest. - // - bool r (rqm (sm.out, true /* long_lines */)); - assert (r); // The serialization succeeded once, so can't fail now. - - // Write the CI overrides manifest. - // - sm.out << "\n\n"; - - ovm (sm.out, true /* long_lines */); - - // Write the CI result manifest. - // - sm.out << "\n\n"; - - // We don't care about the result (see above). - // - rsm (sm.out, true /* long_lines */); - - sm.out.close (); + return true; +} - if (!sm.wait ()) - error << "sendmail " << *sm.exit; - } - // Handle process_error and io_error (both derive from system_error). - // - catch (const system_error& e) - { - error << "sendmail error: " << e; - } +#ifdef BREP_CI_TENANT_SERVICE +function (const brep::tenant_service&)> brep::ci:: +build_queued (const tenant_service&, + const vector& bs, + optional initial_state) const +{ + return [&bs, initial_state] (const tenant_service& ts) + { + optional r (ts.data); + + for (const build& b: bs) + { + string s ((!initial_state + ? "queued " + : "queued " + to_string (*initial_state) + ' ') + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string ()); + + if (r) + { + *r += ", "; + *r += s; + } + else + r = move (s); + } + + return r; + }; +} - if (!rsm (rs.content (sc, "text/manifest;charset=utf-8"))) - return respond_error (); // The error description is already logged. +function (const brep::tenant_service&)> brep::ci:: +build_building (const tenant_service&, const build& b) const +{ + return [&b] (const tenant_service& ts) + { + string s ("building " + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string ()); + + return ts.data ? *ts.data + ", " + s : s; + }; +} - return true; +function (const brep::tenant_service&)> brep::ci:: +build_built (const tenant_service&, const build& b) const +{ + return [&b] (const tenant_service& ts) + { + string s ("built " + + b.package_name.string () + '/' + + b.package_version.string () + '/' + + b.target.string () + '/' + + b.target_config_name + '/' + + b.package_config_name + '/' + + b.toolchain_name + '/' + + b.toolchain_version.string ()); + + return ts.data ? *ts.data + ", " + s : s; + }; } +#endif diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx index 431f53b..3b1e1be 100644 --- a/mod/mod-ci.hxx +++ b/mod/mod-ci.hxx @@ -9,14 +9,39 @@ #include #include +#include +#include // tenant_service + #include #include +#include + +#ifdef BREP_CI_TENANT_SERVICE +# include +#endif + namespace brep { - class ci: public handler + class ci: public handler, + private ci_start +#ifdef BREP_CI_TENANT_SERVICE + , public tenant_service_build_queued, + public tenant_service_build_building, + public tenant_service_build_built +#endif { public: + +#ifdef BREP_CI_TENANT_SERVICE + explicit + ci (tenant_service_map&); + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + ci (const ci&, tenant_service_map&); +#else ci () = default; // Create a shallow copy (handling instance) if initialized and a deep @@ -24,12 +49,26 @@ namespace brep // explicit ci (const ci&); +#endif virtual bool - handle (request&, response&); + handle (request&, response&) override; virtual const cli::options& - cli_options () const {return options::ci::description ();} + cli_options () const override {return options::ci::description ();} + +#ifdef BREP_CI_TENANT_SERVICE + virtual function (const tenant_service&)> + build_queued (const tenant_service&, + const vector&, + optional initial_state) const override; + + virtual function (const tenant_service&)> + build_building (const tenant_service&, const build&) const override; + + virtual function (const tenant_service&)> + build_built (const tenant_service&, const build&) const override; +#endif private: virtual void @@ -38,6 +77,10 @@ namespace brep private: shared_ptr options_; shared_ptr form_; + +#ifdef BREP_CI_TENANT_SERVICE + tenant_service_map& tenant_service_map_; +#endif }; } diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 35a1a22..51b21c6 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -802,7 +802,7 @@ handle (request& rq, response& rs) // Print the package built configurations in the time-descending order. // for (auto& b: build_db_->query ( - (query::id.package == pkg->id && sq) + + (query::id.package == pkg->id && query::state != "queued" && sq) + "ORDER BY" + query::timestamp + "DESC")) { string ts (butl::to_string (b.timestamp, diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index 1b18996..34b4007 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -108,18 +108,31 @@ namespace brep // repository_root:: repository_root () - : packages_ (make_shared ()), + : + // + // Only create and populate the tenant service map in the examplar + // passing a reference to it to all the sub-handler exemplars. Note + // that we dispatch the tenant service callbacks to the examplar + // without creating a new instance for each callback (thus the + // callbacks are const). + // + tenant_service_map_ (make_shared ()), + packages_ (make_shared ()), package_details_ (make_shared ()), package_version_details_ (make_shared ()), repository_details_ (make_shared ()), - build_task_ (make_shared ()), - build_result_ (make_shared ()), - build_force_ (make_shared ()), + build_task_ (make_shared (*tenant_service_map_)), + build_result_ (make_shared (*tenant_service_map_)), + build_force_ (make_shared (*tenant_service_map_)), build_log_ (make_shared ()), builds_ (make_shared ()), build_configs_ (make_shared ()), submit_ (make_shared ()), +#ifdef BREP_CI_TENANT_SERVICE + ci_ (make_shared (*tenant_service_map_)), +#else ci_ (make_shared ()), +#endif upload_ (make_shared ()) { } @@ -127,6 +140,10 @@ namespace brep repository_root:: repository_root (const repository_root& r) : handler (r), + tenant_service_map_ ( + r.initialized_ + ? r.tenant_service_map_ + : make_shared ()), // // Deep/shallow-copy sub-handlers depending on whether this is an // exemplar/handler. @@ -151,15 +168,15 @@ namespace brep build_task_ ( r.initialized_ ? r.build_task_ - : make_shared (*r.build_task_)), + : make_shared (*r.build_task_, *tenant_service_map_)), build_result_ ( r.initialized_ ? r.build_result_ - : make_shared (*r.build_result_)), + : make_shared (*r.build_result_, *tenant_service_map_)), build_force_ ( r.initialized_ ? r.build_force_ - : make_shared (*r.build_force_)), + : make_shared (*r.build_force_, *tenant_service_map_)), build_log_ ( r.initialized_ ? r.build_log_ @@ -179,7 +196,11 @@ namespace brep ci_ ( r.initialized_ ? r.ci_ +#ifdef BREP_CI_TENANT_SERVICE + : make_shared (*r.ci_, *tenant_service_map_)), +#else : make_shared (*r.ci_)), +#endif upload_ ( r.initialized_ ? r.upload_ diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx index 4f40c94..aa60fda 100644 --- a/mod/mod-repository-root.hxx +++ b/mod/mod-repository-root.hxx @@ -9,6 +9,7 @@ #include #include +#include namespace brep { @@ -59,6 +60,8 @@ namespace brep version (); private: + shared_ptr tenant_service_map_; + shared_ptr packages_; shared_ptr package_details_; shared_ptr package_version_details_; diff --git a/mod/module.cli b/mod/module.cli index 3fdd7e3..3e81b38 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -21,7 +21,7 @@ namespace brep { // Option groups. // - class handler + class repository_email { string email { @@ -29,7 +29,10 @@ namespace brep "Repository email. This email is used for the \cb{From:} header in emails send by \cb{brep} (for example, build failure notifications)." } + }; + class handler + { string host { "", @@ -279,6 +282,15 @@ namespace brep the same as for the \cb{build-alt-soft-rebuild-stop} option but for the \cb{build-hard-rebuild-timeout} option." } + + size_t build-queued-timeout = 30 + { + "", + "Time to wait before assuming the \cb{queued} notifications are + delivered for package CI requests submitted via third-party services + (GitHub, etc). During this time a package is not considered for a + build. Must be specified in seconds. Default is 30 seconds." + } }; class build_db @@ -563,7 +575,7 @@ namespace brep } }; - class build_result: build, package_db, build_db, handler + class build_result: build, package_db, build_db, repository_email, handler { size_t build-result-request-max-size = 10485760 { @@ -626,7 +638,7 @@ namespace brep } }; - class submit: page, handler + class submit: page, repository_email, handler { dir_path submit-data { @@ -706,7 +718,7 @@ namespace brep } }; - class ci: page, handler + class ci_start: repository_email { dir_path ci-data { @@ -721,15 +733,6 @@ namespace brep granted to the user that runs the web server." } - path ci-form - { - "", - "The package CI form fragment. If specified, then its contents are - treated as an XHTML5 fragment that is inserted into the - element of the CI page. If unspecified, then no CI page will be - displayed. Note that the file path must be absolute." - } - string ci-email { "", @@ -766,7 +769,33 @@ namespace brep } }; - class upload: build, build_db, build_upload, handler + class ci_cancel + { + }; + + class ci: ci_start, page, handler + { + // Classic CI-specific options. + // + + path ci-form + { + "", + "The package CI form fragment. If specified, then its contents are + treated as an XHTML5 fragment that is inserted into the + element of the CI page. If unspecified, then no CI page will be + displayed. Note that the file path must be absolute." + } + }; + + class ci_github: ci_start, ci_cancel, build_db, handler + { + // GitHub CI-specific options (e.g., request timeout when invoking + // GitHub APIs). + // + }; + + class upload: build, build_db, build_upload, repository_email, handler { }; diff --git a/mod/page.cxx b/mod/page.cxx index d844a89..5483183 100644 --- a/mod/page.cxx +++ b/mod/page.cxx @@ -761,7 +761,6 @@ namespace brep } else { - // If no unsuccessful operation results available, then print the // overall build status. If there are any operation results available, // then also print unsuccessful operation statuses with the links to the diff --git a/mod/page.hxx b/mod/page.hxx index f3c27d5..cac2b8b 100644 --- a/mod/page.hxx +++ b/mod/page.hxx @@ -473,7 +473,12 @@ namespace brep bool a, const string& h, const dir_path& r): - build_ (b), archived_ (a), host_ (h), root_ (r) {} + build_ (b), archived_ (a), host_ (h), root_ (r) + { + // We don't expect a queued build to ever be displayed. + // + assert (build_.state != build_state::queued); + } void operator() (xml::serializer&) const; diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx new file mode 100644 index 0000000..a7bc941 --- /dev/null +++ b/mod/tenant-service.hxx @@ -0,0 +1,107 @@ +// file : mod/tenant-service.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_TENANT_SERVICE_HXX +#define MOD_TENANT_SERVICE_HXX + +#include + +#include +#include + +#include + +namespace brep +{ + class tenant_service_base + { + public: + virtual ~tenant_service_base () = default; + }; + + // Possible build notifications: + // + // queued + // building + // built + // + // Possible transitions: + // + // -> queued + // queued -> building + // building -> queued (interrupted & re-queued due to higher priority task) + // building -> built + // built -> queued (periodic or user-forced rebuild) + // + // While the implementation tries to make sure the notifications arrive in + // the correct order, this is currently done by imposing delays (some + // natural, such as building->built, and some artificial, such as + // queued->building). As result, it is unlikely but possible to be notified + // about the state transitions in the wrong order, especially if the + // notifications take a long time. To minimize the chance of this happening, + // the service implementation should strive to batch the queued state + // notifications (or which there could be hundreds) in a single request if + // at all possible. Also, if supported by the third-party API, it makes + // sense for the implementation to protect against overwriting later states + // with earlier. For example, if it's possible to place a condition on a + // notification, it makes sense to only set the state to queued if none of + // the later states (e.g., building) are already in effect. + // + // Note also that it's possible for the build to get deleted at any stage + // without any further notifications. This can happen, for example, due to + // data retention timeout or because the build configuration (buildtab + // entry) is no longer present. There is no explicit `deleted` transition + // notification because such situations (i.e., when a notification sequence + // is abandoned half way) are not expected to arise ordinarily in a + // properly-configured brep instance. And the third-party service is + // expected to deal with them using some overall timeout/expiration + // mechanism which it presumably has. + // + // Each build notification is in its own interface since a service may not + // be interested in all of them while computing the information to pass is + // expensive. + // + class tenant_service_build_queued: public virtual tenant_service_base + { + public: + // If the returned function is not NULL, it is called to update the + // service data. It should return the new data or nullopt if no update is + // necessary. Note: tenant_service::data passed to the callback and to the + // returned function may not be the same. Also, the returned function may + // be called multiple times (on transaction retries). + // + // The passed initial_state indicates the logical initial state and is + // either absent, `building` (interrupted), or `built` (rebuild). Note + // that all the passed build objects have the same initial state. + // + // The implementation of this and the below functions should normally not + // need to make any decisions based on the passed build::state. Rather, + // the function name suffix (_queued, _building, _built) signify the + // logical end state. + // + virtual function (const tenant_service&)> + build_queued (const tenant_service&, + const vector&, + optional initial_state) const = 0; + }; + + class tenant_service_build_building: public virtual tenant_service_base + { + public: + virtual function (const tenant_service&)> + build_building (const tenant_service&, const build&) const = 0; + }; + + class tenant_service_build_built: public virtual tenant_service_base + { + public: + virtual function (const tenant_service&)> + build_built (const tenant_service&, const build&) const = 0; + }; + + // Map of service type (tenant_service::type) to service. + // + using tenant_service_map = std::map>; +} + +#endif // MOD_TENANT_SERVICE_HXX diff --git a/monitor/monitor.cxx b/monitor/monitor.cxx index 6d6bb99..77f387b 100644 --- a/monitor/monitor.cxx +++ b/monitor/monitor.cxx @@ -675,7 +675,8 @@ namespace brep const auto& bid (bquery::build::id); - bquery bq ((equal (bid.package, id.package) && + bquery bq ((bquery::build::state != "queued" && + equal (bid.package, id.package) && bid.target == bquery::_ref (id.target) && bid.target_config_name == bquery::_ref (id.target_config_name) && @@ -864,8 +865,13 @@ namespace brep b = move (pbs.begin ()->build); } else + { b = db.find (id); + if (b->state == build_state::queued) + b = nullptr; + } + // Note that we consider a build as delayed if it is not // completed in the expected timeframe. So even if the build // task have been issued recently we may still consider the diff --git a/web/server/module.hxx b/web/server/module.hxx index f870163..20f6217 100644 --- a/web/server/module.hxx +++ b/web/server/module.hxx @@ -9,6 +9,7 @@ #include #include #include +#include // enable_shared_from_this #include // uint16_t #include // size_t #include // move() @@ -236,7 +237,7 @@ namespace web // directories (e.g., apache/) if you need to see the code that // does this. // - class handler + class handler: public std::enable_shared_from_this { public: virtual -- cgit v1.1