From 3d7f248ee7aa7e5b470cb9bc2fcf76852e798db5 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Wed, 22 May 2024 19:12:55 +0300 Subject: Make poll interval for unloaded tenant configurable and rename loaded_timestamp tenant member to unloaded_timestamp --- libbrep/build-extra.sql | 3 ++- libbrep/build-package.hxx | 8 +++++--- libbrep/common.hxx | 14 ++++++++++++++ libbrep/package.hxx | 22 +++++++++++++--------- libbrep/package.xml | 7 ++++--- load/load.cxx | 2 +- mod/buildfile | 1 + mod/ci-common.cxx | 17 ++++++++--------- mod/ci-common.hxx | 10 +++++++--- mod/mod-build-task.cxx | 42 +++++++++++++++++++++--------------------- mod/mod-ci.cxx | 4 +++- 11 files changed, 79 insertions(+), 51 deletions(-) diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index c31cc31..0c0f010 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -51,7 +51,8 @@ CREATE FOREIGN TABLE build_tenant ( service_id TEXT NULL, service_type TEXT NULL, service_data TEXT NULL, - loaded_timestamp BIGINT NULL, + unloaded_timestamp BIGINT NULL, + unloaded_notify_interval BIGINT NULL, queued_timestamp BIGINT NULL, toolchain_name TEXT OPTIONS (column_name 'build_toolchain_name') NULL, toolchain_version_epoch INTEGER OPTIONS (column_name 'build_toolchain_version_epoch') NULL, diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 2611678..13645eb 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -35,11 +35,12 @@ namespace brep // Create tenant for an unloaded CI request (see the build_unloaded() // tenant services notification for details). // - build_tenant (string i, tenant_service s, timestamp l) + build_tenant (string i, tenant_service s, timestamp t, duration n) : id (move (i)), creation_timestamp (timestamp::clock::now ()), service (move (s)), - loaded_timestamp (l) {} + unloaded_timestamp (t), + unloaded_notify_interval (n) {} string id; @@ -48,7 +49,8 @@ namespace brep timestamp creation_timestamp; bool archived = false; optional service; - optional loaded_timestamp; + optional unloaded_timestamp; + optional unloaded_notify_interval; optional queued_timestamp; optional toolchain; diff --git a/libbrep/common.hxx b/libbrep/common.hxx index 1433c8c..4be9ce9 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -141,6 +141,20 @@ namespace brep std::chrono::nanoseconds (*(?)))) \ : brep::optional_timestamp ()) + #pragma db map type(duration) as(uint64_t) \ + to(std::chrono::duration_cast (?).count ()) \ + from(brep::duration (std::chrono::nanoseconds (?))) + + using optional_duration = optional; + + #pragma db map type(optional_duration) as(brep::optional_uint64) \ + to((?) \ + ? std::chrono::duration_cast (*(?)).count () \ + : brep::optional_uint64 ()) \ + from((?) \ + ? brep::duration (std::chrono::nanoseconds (*(?))) \ + : brep::optional_duration ()) + // version // using bpkg::version; diff --git a/libbrep/package.hxx b/libbrep/package.hxx index affa2ac..61477a0 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -251,25 +251,29 @@ namespace brep // If this flag is true, then display the packages in the web interface // only in the tenant view mode. // - bool private_; // Note: foreign-mapped in build. + bool private_; // Note: foreign-mapped in build. // Interactive package build breakpoint. // // If present, then packages from this tenant will only be built // interactively and only non-interactively otherwise. // - optional interactive; // Note: foreign-mapped in build. + optional interactive; // Note: foreign-mapped in build. - timestamp creation_timestamp; // Note: foreign-mapped in build. - bool archived = false; // Note: foreign-mapped in build. + timestamp creation_timestamp; // Note: foreign-mapped in build. + bool archived = false; // Note: foreign-mapped in build. - optional service; // Note: foreign-mapped in build. + optional service; // Note: foreign-mapped in build. // If the tenant is loaded, this value is absent. Otherwise it is the time // of the last attempt to load the tenant (see the build_unloaded() tenant // services notification for details). // - optional loaded_timestamp; // Note: foreign-mapped in build. + optional unloaded_timestamp; // Note: foreign-mapped in build. + + // The time interval between attempts to load the tenant, if unloaded. + // + optional unloaded_notify_interval; // Note: foreign-mapped in build. // Note that due to the implementation complexity and performance // considerations, the service notifications are not synchronized. This @@ -290,7 +294,7 @@ namespace brep // natural reasons (non-zero build task execution time, etc) and thus we // just ignore them. // - optional queued_timestamp; // Note: foreign-mapped in build. + optional queued_timestamp; // Note: foreign-mapped in build. // Note that after the package tenant is created but before the first // build object is created, there is no easy way to produce a list of @@ -324,9 +328,9 @@ namespace brep #pragma db index member(service.id) - // Speed-up queries with ordering the result by loaded_timestamp. + // Speed-up queries with ordering the result by unloaded_timestamp. // - #pragma db member(loaded_timestamp) index + #pragma db member(unloaded_timestamp) index private: friend class odb::access; diff --git a/libbrep/package.xml b/libbrep/package.xml index fe010ad..f33119e 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,9 +1,10 @@ - - - + + + + diff --git a/load/load.cxx b/load/load.cxx index 765cf43..ba2da1c 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -1811,7 +1811,7 @@ try throw failed (); } - if (t->loaded_timestamp) + if (t->unloaded_timestamp) { cerr << "error: tenant " << tnt << " is marked as unloaded" << endl; throw failed (); diff --git a/mod/buildfile b/mod/buildfile index c3895dc..ff04ae5 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -39,6 +39,7 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \ # the debugging of the notifications machinery. # cxx.poptions += -DBREP_CI_TENANT_SERVICE +cxx.poptions += -DBREP_CI_TENANT_SERVICE_UNLOADED libus{mod}: ../web/xhtml/libus{xhtml} libue{mod}: ../web/xhtml/libue{xhtml} diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx index bd283f4..6ce47f2 100644 --- a/mod/ci-common.cxx +++ b/mod/ci-common.cxx @@ -539,7 +539,9 @@ namespace brep const basic_mark&, const basic_mark* trace, odb::core::database& db, - tenant_service&& service) const + tenant_service&& service, + duration notify_interval, + duration notify_delay) const { using namespace odb::core; @@ -562,13 +564,10 @@ namespace brep if (service.id.empty ()) service.id = request_id; - // Delay the first load attempt for 10 seconds (see mod-build-task.cxx for - // details). - // build_tenant t (move (request_id), move (service), - system_clock::now () - chrono::seconds (40 - 10)); - + system_clock::now () - notify_interval + notify_delay, + notify_interval); { assert (!transaction::has_current ()); @@ -644,7 +643,7 @@ namespace brep return nullopt; } - else if (!t->loaded_timestamp) + else if (!t->unloaded_timestamp) { error << "tenant " << t->id << " for service " << service.id << ' ' << service.type << " is already loaded"; @@ -652,7 +651,7 @@ namespace brep return nullopt; } - t->loaded_timestamp = nullopt; + t->unloaded_timestamp = nullopt; db.update (t); tr.commit (); @@ -712,7 +711,7 @@ namespace brep return; } - else if (!t->loaded_timestamp) + else if (!t->unloaded_timestamp) { error << "tenant " << t->id << " for service " << service.id << ' ' << service.type << " is already loaded"; diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx index 45c959a..f332dc2 100644 --- a/mod/ci-common.hxx +++ b/mod/ci-common.hxx @@ -67,8 +67,10 @@ namespace brep // Create an unloaded CI request returning start_result::reference on // success and nullopt on an internal error. Such a request is not started - // until loaded with the load() function below. See also the - // build_unloaded() tenant services notification. + // until loaded with the load() function below. Configure the time + // interval between the build_unloaded() notifications for the being + // created tenant and set the initial delay for the first notification. + // See also the build_unloaded() tenant services notification. // // Note: should be called out of the database transaction. // @@ -77,7 +79,9 @@ namespace brep const basic_mark& warn, const basic_mark* trace, odb::core::database&, - tenant_service&&) const; + tenant_service&&, + duration notify_interval, + duration notify_delay) const; // Load (and start) previously created (as unloaded) CI request. Similarly // to the start() function, return nullopt on an internal error. diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 0752603..6be77f6 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -407,19 +407,6 @@ handle (request& rq, response& rs) // connection_ptr conn (build_db_->connection ()); - timestamp now (system_clock::now ()); - - auto expiration = [&now] (size_t timeout) -> timestamp - { - return now - chrono::seconds (timeout); - }; - - auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t - { - return chrono::duration_cast ( - expiration (timeout).time_since_epoch ()).count (); - }; - // Perform some housekeeping first. // // Notify a tenant-associated third-party service about the unloaded CI @@ -433,16 +420,16 @@ handle (request& rq, response& rs) using query = query; // Pick the unloaded tenant with the earliest loaded timestamp, skipping - // those whose timestamp is less than 40 seconds ago. - // - // NOTE: don't forget to update ci_start::create() if changing the timeout - // here. + // those which were already picked recently. // shared_ptr t ( build_db_->query_one ( - (query::loaded_timestamp <= expiration_ns (40) && - !query::archived) + - "ORDER BY" + query::loaded_timestamp + + (!query::archived && + query::unloaded_timestamp.is_not_null () && + (query::unloaded_timestamp + + "<= EXTRACT (EPOCH FROM NOW()) * 1000000000 - " + + query::unloaded_notify_interval)) + + "ORDER BY" + query::unloaded_timestamp + "LIMIT 1")); if (t != nullptr && t->service) @@ -461,7 +448,7 @@ handle (request& rq, response& rs) // set the package tenant's loaded timestamp to the current time to // prevent the notifications race. // - t->loaded_timestamp = system_clock::now (); + t->unloaded_timestamp = system_clock::now (); build_db_->update (t); } } @@ -610,6 +597,19 @@ handle (request& rq, response& rs) // Calculate the build/rebuild (building/built state) and the `queued` // notifications expiration time for package configurations. // + timestamp now (system_clock::now ()); + + auto expiration = [&now] (size_t timeout) -> timestamp + { + return now - chrono::seconds (timeout); + }; + + auto expiration_ns = [&expiration] (size_t timeout) -> uint64_t + { + return chrono::duration_cast ( + expiration (timeout).time_since_epoch ()).count (); + }; + uint64_t normal_result_expiration_ns ( expiration_ns (options_->build_result_timeout ())); diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index 18e38fc..0045002 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -389,7 +389,9 @@ handle (request& rq, response& rs) warn, verb_ ? &trace : nullptr, *build_db_, - tenant_service ("", "ci", rl.string ()))) + tenant_service ("", "ci", rl.string ()), + chrono::seconds (40), + chrono::seconds (10))) { string msg ("unloaded CI request is created: " + options_->host () + tenant_dir (root, *ref).string ()); -- cgit v1.1