diff options
38 files changed, 1281 insertions, 228 deletions
diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in index 3f04ea8..6029b7b 100644 --- a/brep/handler/ci/ci-load.in +++ b/brep/handler/ci/ci-load.in @@ -10,6 +10,11 @@ # brep tenant id to this value and include the resulting URL in the response # message. # +# --cancel-url <url> +# CI task canceling URL base for the response. If specified, the handler will +# append the brep tenant id to this value and include the resulting URL in +# the response message. +# # <loader-path> # Loader program (normally brep-load(1)). # @@ -36,6 +41,7 @@ shopt -s nullglob # Expand no-match globs to nothing rather than themselves. # The handler's own options. # result_url= +cancel_url= while [[ "$#" -gt 0 ]]; do case $1 in --result-url) @@ -43,6 +49,11 @@ while [[ "$#" -gt 0 ]]; do result_url="${1%/}" shift ;; + --cancel-url) + shift + cancel_url="${1%/}" + shift + ;; *) break ;; @@ -114,6 +125,7 @@ spec= service_id= service_type= service_data= +service_load= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in @@ -133,6 +145,14 @@ while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do service-id) service_id="$v" ;; service-type) service_type="$v" ;; service-data) service_data="$v" ;; + + service-action) + if [[ "$v" == "load" ]]; then + service_load=true + elif [[ "$v" != "start" ]]; then + error "unrecognized service action '$v'" + fi + ;; esac done @@ -331,6 +351,12 @@ if [[ -n "$service_id" ]]; then if [[ -n "$service_data" ]]; then loader_options+=(--service-data "$service_data") fi + + # Load the pre-created tenant rather than create a new one. + # + if [[ "$service_load" ]]; then + loader_options+=(--existing-tenant) + fi fi run "$loader" "${loader_options[@]}" "$loadtab" @@ -340,4 +366,11 @@ run "$loader" "${loader_options[@]}" "$loadtab" run rm -r "$data_dir" trace "CI request for '$spec' is queued$message_suffix" -exit_with_manifest 200 "CI request is queued$message_suffix" + +msg="CI request is queued$message_suffix" + +if [[ -n "$cancel_url" ]]; then + msg="$msg"$'\n'"To cancel CI request: $cancel_url=$reference&reason=" +fi + +exit_with_manifest 200 "$msg" diff --git a/doc/manual.cli b/doc/manual.cli index 2b96393..9b85ae6 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -344,6 +344,7 @@ timestamp: <date-time> [service-id]: <string> [service-type]: <string> [service-data]: <string> +[service-action]: <action> \ The \c{package} value can be repeated multiple times. The \c{timestamp} value @@ -356,7 +357,14 @@ required information via some custom protocol, and expect the CI service to notify it about the progress. In this case the third-party service type as well as optionally the third-party id and custom state data can be communicated to the underlying CI handler program via the respective -\c{service-*} manifest values. +\c{service-*} manifest values. Also note that normally a third-party service +has all the required information (repository URL, etc) available at the time +of the CI task initiation, in which case the \c{start} value is specified for +the \c{service-action} manifest value. If that's not the case, the CI task is +only created at the time of the initiation without calling the CI handler +program. In this case the CI handler is called later, when all the required +information is asynchronously gathered by the service. In this case the +\c{load} value is specified for the \c{service-action} manifest value. \h#ci-overrides-manifest|CI Overrides Manifest| diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql index 9e51a51..0c0f010 100644 --- a/libbrep/build-extra.sql +++ b/libbrep/build-extra.sql @@ -46,10 +46,13 @@ CREATE FOREIGN TABLE build_tenant ( id TEXT NOT NULL, private BOOLEAN NOT NULL, interactive TEXT NULL, + creation_timestamp BIGINT NOT NULL, archived BOOLEAN NOT NULL, service_id TEXT NULL, service_type TEXT NULL, service_data TEXT NULL, + unloaded_timestamp BIGINT NULL, + unloaded_notify_interval BIGINT NULL, queued_timestamp BIGINT NULL, toolchain_name TEXT OPTIONS (column_name 'build_toolchain_name') NULL, toolchain_version_epoch INTEGER OPTIONS (column_name 'build_toolchain_version_epoch') NULL, diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx index 9a9c277..13645eb 100644 --- a/libbrep/build-package.hxx +++ b/libbrep/build-package.hxx @@ -32,12 +32,25 @@ namespace brep class build_tenant { public: + // Create tenant for an unloaded CI request (see the build_unloaded() + // tenant services notification for details). + // + build_tenant (string i, tenant_service s, timestamp t, duration n) + : id (move (i)), + creation_timestamp (timestamp::clock::now ()), + service (move (s)), + unloaded_timestamp (t), + unloaded_notify_interval (n) {} + string id; - bool private_; + bool private_ = false; optional<string> interactive; - bool archived; + timestamp creation_timestamp; + bool archived = false; optional<tenant_service> service; + optional<timestamp> unloaded_timestamp; + optional<duration> unloaded_notify_interval; optional<timestamp> queued_timestamp; optional<build_toolchain> toolchain; diff --git a/libbrep/build.hxx b/libbrep/build.hxx index af49c03..55fd42b 100644 --- a/libbrep/build.hxx +++ b/libbrep/build.hxx @@ -28,7 +28,7 @@ // #define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20 -#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 27, closed) +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 28, closed) // We have to keep these mappings at the global scope instead of inside the // brep namespace because they need to be also effective in the bbot namespace diff --git a/libbrep/build.xml b/libbrep/build.xml index 1eba85a..90b4b4f 100644 --- a/libbrep/build.xml +++ b/libbrep/build.xml @@ -1,4 +1,6 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1"> + <changeset version="28"/> + <changeset version="27"/> <changeset version="26"/> diff --git a/libbrep/common.hxx b/libbrep/common.hxx index 1433c8c..4be9ce9 100644 --- a/libbrep/common.hxx +++ b/libbrep/common.hxx @@ -141,6 +141,20 @@ namespace brep std::chrono::nanoseconds (*(?)))) \ : brep::optional_timestamp ()) + #pragma db map type(duration) as(uint64_t) \ + to(std::chrono::duration_cast<std::chrono::nanoseconds> (?).count ()) \ + from(brep::duration (std::chrono::nanoseconds (?))) + + using optional_duration = optional<duration>; + + #pragma db map type(optional_duration) as(brep::optional_uint64) \ + to((?) \ + ? std::chrono::duration_cast<std::chrono::nanoseconds> (*(?)).count () \ + : brep::optional_uint64 ()) \ + from((?) \ + ? brep::duration (std::chrono::nanoseconds (*(?))) \ + : brep::optional_duration ()) + // version // using bpkg::version; diff --git a/libbrep/odb.sh b/libbrep/odb.sh index 608ca41..7c62acb 100755 --- a/libbrep/odb.sh +++ b/libbrep/odb.sh @@ -16,6 +16,8 @@ if test -d ../.bdep; then sed -r -ne 's#^(@[^ ]+ )?([^ ]+)/ .*default.*$#\2#p')" fi + # Note: here we use libodb*, not libbutl-odb. + # inc+=("-I$(echo "$cfg"/libodb-[1-9]*/)") inc+=("-I$(echo "$cfg"/libodb-pgsql-[1-9]*/)") @@ -33,11 +35,8 @@ sed -r -ne 's#^(@[^ ]+ )?([^ ]+)/ .*default.*$#\2#p')" else - inc+=("-I$HOME/work/odb/builds/default/libodb-pgsql-default") - inc+=("-I$HOME/work/odb/libodb-pgsql") - - inc+=("-I$HOME/work/odb/builds/default/libodb-default") - inc+=("-I$HOME/work/odb/libodb") + inc+=("-I$HOME/work/odb/odb/libodb-pgsql") + inc+=("-I$HOME/work/odb/odb/libodb") inc+=(-I.. -I../../libbbot -I../../libbpkg -I../../libbutl) diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 45008d4..76c5836 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -20,7 +20,7 @@ // #define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27 -#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 33, closed) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 34, closed) namespace brep { @@ -133,7 +133,7 @@ namespace brep optional<version_constraint> constraint; // Resolved dependency package. Can be NULL if the repository load was - // shallow and the package dependency could not be resolved. + // shallow or the package dependency could not be resolved. // lazy_shared_ptr<package_type> package; @@ -251,19 +251,29 @@ namespace brep // If this flag is true, then display the packages in the web interface // only in the tenant view mode. // - bool private_; // Note: foreign-mapped in build. + bool private_; // Note: foreign-mapped in build. // Interactive package build breakpoint. // // If present, then packages from this tenant will only be built // interactively and only non-interactively otherwise. // - optional<string> interactive; // Note: foreign-mapped in build. + optional<string> interactive; // Note: foreign-mapped in build. - timestamp creation_timestamp; - bool archived = false; // Note: foreign-mapped in build. + timestamp creation_timestamp; // Note: foreign-mapped in build. + bool archived = false; // Note: foreign-mapped in build. - optional<tenant_service> service; // Note: foreign-mapped in build. + optional<tenant_service> service; // Note: foreign-mapped in build. + + // If the tenant is loaded, this value is absent. Otherwise it is the time + // of the last attempt to load the tenant (see the build_unloaded() tenant + // services notification for details). + // + optional<timestamp> unloaded_timestamp; // Note: foreign-mapped in build. + + // The time interval between attempts to load the tenant, if unloaded. + // + optional<duration> unloaded_notify_interval; // Note: foreign-mapped in build. // Note that due to the implementation complexity and performance // considerations, the service notifications are not synchronized. This @@ -284,7 +294,7 @@ namespace brep // natural reasons (non-zero build task execution time, etc) and thus we // just ignore them. // - optional<timestamp> queued_timestamp; // Note: foreign-mapped in build. + optional<timestamp> queued_timestamp; // Note: foreign-mapped in build. // Note that after the package tenant is created but before the first // build object is created, there is no easy way to produce a list of @@ -318,6 +328,10 @@ namespace brep #pragma db index member(service.id) + // Speed-up queries with ordering the result by unloaded_timestamp. + // + #pragma db member(unloaded_timestamp) index + private: friend class odb::access; tenant () = default; @@ -427,6 +441,20 @@ namespace brep repository (): tenant (id.tenant), canonical_name (id.canonical_name) {} }; + // Repositories count. + // + #pragma db view object(repository) + struct repository_count + { + size_t result; + + operator size_t () const {return result;} + + // Database mapping. + // + #pragma db member(result) column("count(" + repository::id.tenant + ")") + }; + // The 'to' expression calls the PostgreSQL to_tsvector(weighted_text) // function overload (package-extra.sql). Since we are only interested // in "write-only" members of this type, make the 'from' expression diff --git a/libbrep/package.xml b/libbrep/package.xml index 96e93a7..f33119e 100644 --- a/libbrep/package.xml +++ b/libbrep/package.xml @@ -1,4 +1,14 @@ <changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1"> + <changeset version="34"> + <alter-table name="tenant"> + <add-column name="unloaded_timestamp" type="BIGINT" null="true"/> + <add-column name="unloaded_notify_interval" type="BIGINT" null="true"/> + <add-index name="tenant_unloaded_timestamp_i"> + <column name="unloaded_timestamp"/> + </add-index> + </alter-table> + </changeset> + <changeset version="33"> <add-table name="public_key" kind="object"> <column name="tenant" type="TEXT" null="false"/> diff --git a/libbrep/version.hxx.in b/libbrep/version.hxx.in index 3ac3752..9adb5ab 100644 --- a/libbrep/version.hxx.in +++ b/libbrep/version.hxx.in @@ -49,11 +49,11 @@ $libbbot.check(LIBBBOT_VERSION, LIBBBOT_SNAPSHOT)$ #include <odb/version.hxx> -$libodb.check(LIBODB_VERSION, LIBODB_SNAPSHOT)$ +$libodb.check(LIBODB_VERSION_FULL, LIBODB_SNAPSHOT)$ #include <odb/pgsql/version.hxx> -$libodb_pgsql.check(LIBODB_PGSQL_VERSION, LIBODB_PGSQL_SNAPSHOT)$ +$libodb_pgsql.check(LIBODB_PGSQL_VERSION_FULL, LIBODB_PGSQL_SNAPSHOT)$ // For now these are the same. // diff --git a/load/load.cli b/load/load.cli index 99d76f6..bda186a 100644 --- a/load/load.cli +++ b/load/load.cli @@ -7,6 +7,8 @@ include <cstdint>; // uint16_t include <libbrep/types.hxx>; +include <load/options-types.hxx>; + "\section=1" "\name=brep-load" "\summary=load repositories into brep package database" @@ -57,7 +59,7 @@ class options don't detect package dependency cycles." }; - bool --ignore-unresolved-tests + bool --ignore-unresolv-tests { "Ignore tests, examples, and benchmarks package manifest entries which cannot be resolved from the main package's complement repositories, @@ -65,6 +67,16 @@ class options be removed from the main package manifests outright." } + brep::ignore_unresolved_conditional_dependencies --ignore-unresolv-cond + { + "<pkg>", + "Ignore conditional package dependencies which cannot be resolved. The + valid <pkg> values are \cb{all} and \cb{tests}. If \cb{all} is specified, + then unresolved conditional dependencies are ignored in all packages. If + \cb{tests} is specified, then unresolved conditional dependencies are + only ignored in external tests, examples, and benchmarks packages." + } + std::string --tenant { "<id>", @@ -72,6 +84,13 @@ class options specified, then the single-tenant mode is assumed." }; + bool --existing-tenant + { + "Load the repository and package information into the already created empty + tenant rather than into the newly created one. Requires the \cb{--tenant} + option to be specified." + }; + bool --private { "Display the tenant packages in the web interface only in the tenant view diff --git a/load/load.cxx b/load/load.cxx index 474b443..2b2cd56 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -33,6 +33,7 @@ #include <libbrep/database-lock.hxx> #include <load/load-options.hxx> +#include <load/options-types.hxx> using std::cout; using std::cerr; @@ -804,8 +805,8 @@ load_packages (const options& lo, } // A non-stub package is buildable if belongs to at least one - // buildable repository (see libbrep/package.hxx for details). - // Note that if this is an external test package it will be marked as + // buildable repository (see libbrep/package.hxx for details). Note + // that if this is an external test package it will be marked as // unbuildable later (see resolve_dependencies() for details). // if (rp->buildable && !p->buildable && !p->stub ()) @@ -1206,35 +1207,25 @@ find (const lazy_shared_ptr<repository>& r, return false; } -// Resolve package regular dependencies and external tests. Make sure that the -// best matching dependency belongs to the package repositories, their -// complements, recursively, or their immediate prerequisite repositories -// (only for regular dependencies). Set the buildable flag to false for the -// resolved external tests packages. Fail if unable to resolve a regular -// dependency, unless ignore_unresolved is true in which case leave this -// dependency NULL. Fail if unable to resolve an external test, unless -// ignore_unresolved or ignore_unresolved_tests is true in which case leave -// this dependency NULL, if ignore_unresolved_tests is false, and remove the -// respective tests manifest entry otherwise. Should be called once per -// internal package. +// Try to resolve package regular dependencies and external tests. Make sure +// that the best matching dependency belongs to the package repositories, +// their complements, recursively, or their immediate prerequisite +// repositories (only for regular dependencies). Set the buildable flag to +// false for the resolved external tests packages. Leave the package member +// NULL for unresolved dependencies. // static void -resolve_dependencies (package& p, - database& db, - bool ignore_unresolved, - bool ignore_unresolved_tests) +resolve_dependencies (package& p, database& db) { using brep::dependency; using brep::dependency_alternative; using brep::dependency_alternatives; + using brep::test_dependency; // Resolve dependencies for internal packages only. // assert (p.internal ()); - if (p.dependencies.empty () && p.tests.empty ()) - return; - auto resolve = [&p, &db] (dependency& d, bool test) { // Dependency should not be resolved yet. @@ -1324,6 +1315,60 @@ resolve_dependencies (package& p, return false; }; + // Update the package state if any dependency is resolved. + // + bool update (false); + + for (dependency_alternatives& das: p.dependencies) + { + for (dependency_alternative& da: das) + { + for (dependency& d: da) + { + if (resolve (d, false /* test */)) + update = true; + } + } + } + + for (test_dependency& td: p.tests) + { + if (resolve (td, true /* test */)) + update = true; + } + + if (update) + db.update (p); +} + +// Verify that the unresolved dependencies can be ignored. +// +// Specifically, fail for an unresolved regular dependency, unless +// ignore_unresolved is true or this is a conditional dependency and either +// ignore_unresolved_cond argument is 'all' or it is 'tests' and the specified +// package is a tests, examples, or benchmarks package. Fail for an unresolved +// external test, unless ignore_unresolved or ignore_unresolved_tests is +// true. If ignore_unresolved_tests is true, then remove the unresolved tests +// entry from the package manifest. Should be called once per internal package +// after resolve_dependencies() is called for all of them. +// +static void +verify_dependencies ( + package& p, + database& db, + bool ignore_unresolved, + bool ignore_unresolved_tests, + optional<ignore_unresolved_conditional_dependencies> ignore_unresolved_cond) +{ + using brep::dependency; + using brep::dependency_alternative; + using brep::dependency_alternatives; + using brep::test_dependency; + + // Verify dependencies for internal packages only. + // + assert (p.internal ()); + auto bail = [&p] (const dependency& d, const string& what) { cerr << "error: can't resolve " << what << ' ' << d << " for the package " @@ -1334,43 +1379,74 @@ resolve_dependencies (package& p, throw failed (); }; - for (dependency_alternatives& das: p.dependencies) + if (!ignore_unresolved) { - // Practically it is enough to resolve at least one dependency alternative - // to build a package. Meanwhile here we consider an error specifying in - // the manifest file an alternative which can't be resolved, unless - // unresolved dependencies are allowed. + // There must always be a reason why a package is not buildable. // - for (dependency_alternative& da: das) + assert (p.buildable || p.unbuildable_reason); + + bool test (!p.buildable && + *p.unbuildable_reason == unbuildable_reason::test); + + for (dependency_alternatives& das: p.dependencies) { - for (dependency& d: da) + for (dependency_alternative& da: das) { - if (!resolve (d, false /* test */) && !ignore_unresolved) - bail (d, "dependency"); + for (dependency& d: da) + { + if (d.package == nullptr) + { + if (da.enable && ignore_unresolved_cond) + { + switch (*ignore_unresolved_cond) + { + case ignore_unresolved_conditional_dependencies::all: continue; + case ignore_unresolved_conditional_dependencies::tests: + { + if (test) + continue; + + break; + } + } + } + + bail (d, "dependency"); + } + } } } } - for (auto i (p.tests.begin ()); i != p.tests.end (); ) + if (!ignore_unresolved || ignore_unresolved_tests) { - brep::test_dependency& td (*i); + // Update the package state if any test dependency is erased. + // + bool update (false); - if (!resolve (td, true /* test */)) + for (auto i (p.tests.begin ()); i != p.tests.end (); ) { - if (!ignore_unresolved && !ignore_unresolved_tests) - bail (td, to_string (td.type)); + test_dependency& td (*i); - if (ignore_unresolved_tests) + if (td.package == nullptr) { - i = p.tests.erase (i); - continue; + if (!ignore_unresolved && !ignore_unresolved_tests) + bail (td, to_string (td.type)); + + if (ignore_unresolved_tests) + { + i = p.tests.erase (i); + update = true; + continue; + } } + + ++i; } - ++i; + if (update) + db.update (p); } - - db.update (p); // Update the package state. } using package_ids = vector<package_id>; @@ -1429,7 +1505,12 @@ detect_dependency_cycle (const package_id& id, for (const auto& da: das) { for (const auto& d: da) - detect_dependency_cycle (d.package.object_id (), chain, db); + { + // Skip unresolved dependencies. + // + if (d.package != nullptr) + detect_dependency_cycle (d.package.object_id (), chain, db); + } } } @@ -1643,11 +1724,23 @@ try // const string& tnt (ops.tenant ()); - if (ops.tenant_specified () && tnt.empty ()) + if (ops.tenant_specified ()) { - cerr << "error: empty tenant" << endl - << help_info << endl; - throw failed (); + if (tnt.empty ()) + { + cerr << "error: empty tenant" << endl + << help_info << endl; + throw failed (); + } + } + else + { + if (ops.existing_tenant ()) + { + cerr << "error: --existing-tenant requires --tenant" << endl + << help_info << endl; + throw failed (); + } } // Verify the --service-* options. @@ -1656,14 +1749,15 @@ try { if (!ops.tenant_specified ()) { - cerr << "error: --service-id requires --tenant" << endl; + cerr << "error: --service-id requires --tenant" << endl + << help_info << endl; throw failed (); } if (ops.service_type ().empty ()) { - cerr << "error: --service-id requires --service-type" - << endl; + cerr << "error: --service-id requires --service-type" << endl + << help_info << endl; throw failed (); } } @@ -1671,15 +1765,15 @@ try { if (ops.service_type_specified ()) { - cerr << "error: --service-type requires --service-id" - << endl; + cerr << "error: --service-type requires --service-id" << endl + << help_info << endl; throw failed (); } if (ops.service_data_specified ()) { - cerr << "error: --service-data requires --service-id" - << endl; + cerr << "error: --service-data requires --service-id" << endl + << help_info << endl; throw failed (); } } @@ -1753,13 +1847,15 @@ try if (ops.force () || changed (tnt, irs, db)) { + shared_ptr<tenant> t; // Not NULL in the --existing-tenant mode. + // Rebuild repositories persistent state from scratch. // // Note that in the single-tenant mode the tenant must be empty. In the - // multi-tenant mode all tenants must be non-empty. So in the - // single-tenant mode we erase all database objects (possibly from - // multiple tenants). Otherwise, cleanup the specified and the empty - // tenants only. + // multi-tenant mode all tenants, excluding the pre-created ones, must be + // non-empty. So in the single-tenant mode we erase all database objects + // (possibly from multiple tenants). Otherwise, cleanup the empty tenant + // and, unless in the --existing-tenant mode, the specified one. // if (tnt.empty ()) // Single-tenant mode. { @@ -1770,7 +1866,49 @@ try } else // Multi-tenant mode. { - cstrings ts ({tnt.c_str (), ""}); + // NOTE: don't forget to update ci_start::create() if changing anything + // here. + // + cstrings ts ({""}); + + // In the --existing-tenant mode make sure that the specified tenant + // exists, is not archived, not marked as unloaded, and is + // empty. Otherwise (not in the --existing-tenant mode), remove this + // tenant. + // + if (ops.existing_tenant ()) + { + t = db.find<tenant> (tnt); + + if (t == nullptr) + { + cerr << "error: unable to find tenant " << tnt << endl; + throw failed (); + } + + if (t->archived) + { + cerr << "error: tenant " << tnt << " is archived" << endl; + throw failed (); + } + + if (t->unloaded_timestamp) + { + cerr << "error: tenant " << tnt << " is marked as unloaded" << endl; + throw failed (); + } + + size_t n (db.query_value<repository_count> ( + query<repository_count>::id.tenant == tnt)); + + if (n != 0) + { + cerr << "error: tenant " << tnt << " is not empty" << endl; + throw failed (); + } + } + else + ts.push_back (tnt.c_str ()); db.erase_query<package> ( query<package>::id.tenant.in_range (ts.begin (), ts.end ())); @@ -1785,32 +1923,68 @@ try query<tenant>::id.in_range (ts.begin (), ts.end ())); } - // Persist the tenant. + // Craft the tenant service object from the --service-* options. // - // Note that if the tenant service is specified and some tenant with the - // same service id and type is already persisted, then we will end up with - // the `object already persistent` error and terminate with the exit code - // 1 (fatal error). We could potentially dedicate a special exit code for - // such a case, so that the caller may recognize it and behave accordingly - // (CI request handler can treat it as a client error rather than an - // internal error, etc). However, let's first see if it ever becomes a - // problem. + // In the --existing-tenant mode make sure that the specified service + // matches the service associated with the pre-created tenant and update + // the service data, if specified. // optional<tenant_service> service; if (ops.service_id_specified ()) + { service = tenant_service (ops.service_id (), ops.service_type (), (ops.service_data_specified () ? ops.service_data () : optional<string> ())); - db.persist (tenant (tnt, - ops.private_ (), - (ops.interactive_specified () - ? ops.interactive () - : optional<string> ()), - move (service))); + if (ops.existing_tenant ()) + { + assert (t != nullptr); + + if (!t->service) + { + cerr << "error: no service associated with tenant " << tnt << endl; + throw failed (); + } + + if (t->service->id != service->id || t->service->type != service->type) + { + cerr << "error: associated service mismatch for tenant " << tnt << endl << + " info: specified service: " << service->id << ' ' + << service->type << endl << + " info: associated service: " << t->service->id << ' ' + << t->service->type << endl; + throw failed (); + } + + if (service->data) + { + t->service->data = move (service->data); + db.update (t); + } + } + } + + // Persist the tenant. + // + // Note that if the tenant service is specified and some tenant with the + // same service id and type is already persisted, then we will end up with + // the `object already persistent` error and terminate with the exit code + // 1 (fatal error). We could potentially dedicate a special exit code for + // such a case, so that the caller may recognize it and behave accordingly + // (CI request handler can treat it as a client error rather than an + // internal error, etc). However, let's first see if it ever becomes a + // problem. + // + if (!ops.existing_tenant ()) + db.persist (tenant (tnt, + ops.private_ (), + (ops.interactive_specified () + ? ops.interactive () + : optional<string> ()), + move (service))); // On the first pass over the internal repositories we load their // certificate information and packages. @@ -1862,29 +2036,36 @@ try ops.shallow ()); } - // Resolve internal packages dependencies and, unless this is a shallow - // load, make sure there are no package dependency cycles. + // Try to resolve the internal packages dependencies and verify that the + // unresolved ones can be ignored. Unless this is a shallow load, make + // sure there are no package dependency cycles. // { session s; using query = query<package>; - for (auto& p: - db.query<package> ( - query::id.tenant == tnt && - query::internal_repository.canonical_name.is_not_null ())) - resolve_dependencies (p, - db, - ops.shallow (), - ops.ignore_unresolved_tests ()); + query q (query::id.tenant == tnt && + query::internal_repository.canonical_name.is_not_null ()); + + for (auto& p: db.query<package> (q)) + resolve_dependencies (p, db); + + for (auto& p: db.query<package> (q)) + { + verify_dependencies ( + p, + db, + ops.shallow (), + ops.ignore_unresolv_tests (), + (ops.ignore_unresolv_cond_specified () + ? ops.ignore_unresolv_cond () + : optional<ignore_unresolved_conditional_dependencies> ())); + } if (!ops.shallow ()) { package_ids chain; - for (const auto& p: - db.query<package> ( - query::id.tenant == tnt && - query::internal_repository.canonical_name.is_not_null ())) + for (const auto& p: db.query<package> (q)) detect_dependency_cycle (p.id, chain, db); } } diff --git a/load/options-types.hxx b/load/options-types.hxx new file mode 100644 index 0000000..25858f0 --- /dev/null +++ b/load/options-types.hxx @@ -0,0 +1,20 @@ +// file : load/options-types.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef LOAD_OPTIONS_TYPES_HXX +#define LOAD_OPTIONS_TYPES_HXX + +#include <libbrep/types.hxx> + +namespace brep +{ + // Ignore unresolved conditional dependencies. + // + enum class ignore_unresolved_conditional_dependencies + { + all, // For all packages. + tests // Only for external tests, examples, and benchmarks packages. + }; +} + +#endif // LOAD_OPTIONS_TYPES_HXX diff --git a/load/types-parsers.cxx b/load/types-parsers.cxx index 4c4ea9d..a18330d 100644 --- a/load/types-parsers.cxx +++ b/load/types-parsers.cxx @@ -39,4 +39,22 @@ namespace cli xs = true; parse_path (x, s); } + + void parser<ignore_unresolved_conditional_dependencies>:: + parse (ignore_unresolved_conditional_dependencies& x, bool& xs, scanner& s) + { + xs = true; + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + const string v (s.next ()); + if (v == "all") + x = ignore_unresolved_conditional_dependencies::all; + else if (v == "tests") + x = ignore_unresolved_conditional_dependencies::tests; + else + throw invalid_value (o, v); + } } diff --git a/load/types-parsers.hxx b/load/types-parsers.hxx index 1d2a6c9..fcf5113 100644 --- a/load/types-parsers.hxx +++ b/load/types-parsers.hxx @@ -9,6 +9,8 @@ #include <libbrep/types.hxx> +#include <load/options-types.hxx> + namespace cli { class scanner; @@ -22,6 +24,13 @@ namespace cli static void parse (brep::path&, bool&, scanner&); }; + + template <> + struct parser<brep::ignore_unresolved_conditional_dependencies> + { + static void + parse (brep::ignore_unresolved_conditional_dependencies&, bool&, scanner&); + }; } #endif // LOAD_TYPES_PARSERS_HXX @@ -37,7 +37,7 @@ depends: bpkg-util [0.17.0-a.0.1 0.17.0-a.1) # are (currently) not packaged and need to come from the system package # manager. It also requires rsync for tests. # -builds: none +builds: none ; Requires unpackaged software. debian-builds: sys debian-build-exclude: linux_debian_12-** ; libapreq2 not available diff --git a/mod/build-config-module.cxx b/mod/build-config-module.cxx index 97c9f9e..1f4ad42 100644 --- a/mod/build-config-module.cxx +++ b/mod/build-config-module.cxx @@ -148,26 +148,35 @@ namespace brep } bool build_config_module:: - belongs (const build_target_config& cfg, const char* cls) const + derived (const string& c, const char* bc) const { + if (c == bc) + return true; + + // Go through base classes. + // const map<string, string>& im (target_conf_->class_inheritance_map); - for (const string& c: cfg.classes) + for (auto i (im.find (c)); i != im.end (); ) { - if (c == cls) + const string& base (i->second); + + if (base == bc) return true; - // Go through base classes. - // - for (auto i (im.find (c)); i != im.end (); ) - { - const string& base (i->second); + i = im.find (base); + } - if (base == cls) - return true; + return false; + } - i = im.find (base); - } + bool build_config_module:: + belongs (const build_target_config& cfg, const char* cls) const + { + for (const string& c: cfg.classes) + { + if (derived (c, cls)) + return true; } return false; diff --git a/mod/build-config-module.hxx b/mod/build-config-module.hxx index c1630b0..bbbe952 100644 --- a/mod/build-config-module.hxx +++ b/mod/build-config-module.hxx @@ -54,15 +54,20 @@ namespace brep default_all_ucs); } + // Return true if a class is derived from the base class, recursively. + // + bool + derived (const string&, const char* base_class) const; + // Check if the configuration belongs to the specified class. // bool belongs (const build_target_config&, const char*) const; bool - belongs (const build_target_config& cfg, const string& cls) const + belongs (const build_target_config& cfg, const string& classes) const { - return belongs (cfg, cls.c_str ()); + return belongs (cfg, classes.c_str ()); } // Target/configuration/toolchain combination that, in particular, can be diff --git a/mod/buildfile b/mod/buildfile index c3895dc..2d6ef39 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -39,6 +39,7 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \ # the debugging of the notifications machinery. # cxx.poptions += -DBREP_CI_TENANT_SERVICE +#cxx.poptions += -DBREP_CI_TENANT_SERVICE_UNLOADED libus{mod}: ../web/xhtml/libus{xhtml} libue{mod}: ../web/xhtml/libue{xhtml} diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx index cb61e66..c0ef89f 100644 --- a/mod/ci-common.cxx +++ b/mod/ci-common.cxx @@ -3,6 +3,9 @@ #include <mod/ci-common.hxx> +#include <odb/database.hxx> +#include <odb/transaction.hxx> + #include <libbutl/uuid.hxx> #include <libbutl/fdstream.hxx> #include <libbutl/sendmail.hxx> @@ -11,6 +14,9 @@ #include <libbutl/process-io.hxx> // operator<<(ostream, process_args) #include <libbutl/manifest-serializer.hxx> +#include <libbrep/build-package.hxx> +#include <libbrep/build-package-odb.hxx> + #include <mod/external-handler.hxx> namespace brep @@ -38,13 +44,16 @@ namespace brep options_ = move (o); } - optional<ci_start::start_result> ci_start:: + static optional<ci_start::start_result> start (const basic_mark& error, const basic_mark& warn, const basic_mark* trace, + const options::ci_start& ops, + string&& request_id, optional<tenant_service>&& service, + bool service_load, const repository_location& repository, - const vector<package>& packages, + const vector<ci_start::package>& packages, const optional<string>& client_ip, const optional<string>& user_agent, const optional<string>& interactive, @@ -55,32 +64,15 @@ namespace brep using serializer = manifest_serializer; using serialization = manifest_serialization; - assert (options_ != nullptr); // Shouldn't be called otherwise. + using result = ci_start::start_result; // If the tenant service is specified, then its type may not be empty. // assert (!service || !service->type.empty ()); - // Generate the request id. - // - // Note that it will also be used as a CI result manifest reference, - // unless the latter is provided by the external handler. - // - string request_id; - - try - { - request_id = uuid::generate ().string (); - } - catch (const system_error& e) - { - error << "unable to generate request id: " << e; - return nullopt; - } - // Create the submission data directory. // - dir_path dd (options_->ci_data () / dir_path (request_id)); + dir_path dd (ops.ci_data () / dir_path (request_id)); try { @@ -103,10 +95,10 @@ namespace brep // auto client_error = [&request_id] (uint16_t status, string message) { - return start_result {status, - move (message), - request_id, - vector<pair<string, string>> ()}; + return result {status, + move (message), + request_id, + vector<pair<string, string>> ()}; }; // Serialize the CI request manifest to a stream. On the serialization @@ -119,6 +111,7 @@ namespace brep auto rqm = [&request_id, &ts, &service, + service_load, &repository, &packages, &client_ip, @@ -127,7 +120,7 @@ namespace brep &simulate, &custom_request, &client_error] (ostream& os, bool long_lines = false) - -> pair<bool, optional<start_result>> + -> pair<bool, optional<result>> { try { @@ -139,7 +132,7 @@ namespace brep s.next ("id", request_id); s.next ("repository", repository.string ()); - for (const package& p: packages) + for (const ci_start::package& p: packages) { if (!p.version) s.next ("package", p.name.string ()); @@ -178,6 +171,8 @@ namespace brep if (service->data) s.next ("service-data", *service->data); + + s.next ("service-action", service_load ? "load" : "start"); } // Serialize the request custom parameters. @@ -190,12 +185,12 @@ namespace brep s.next (nv.first, nv.second); s.next ("", ""); // End of manifest. - return make_pair (true, optional<start_result> ()); + return make_pair (true, optional<result> ()); } catch (const serialization& e) { return make_pair (false, - optional<start_result> ( + optional<result> ( client_error (400, string ("invalid parameter: ") + e.what ()))); @@ -209,7 +204,7 @@ namespace brep try { ofdstream os (rqf); - pair<bool, optional<start_result>> r (rqm (os)); + pair<bool, optional<result>> r (rqm (os)); os.close (); if (!r.first) @@ -228,7 +223,7 @@ namespace brep // auto ovm = [&overrides, &client_error] (ostream& os, bool long_lines = false) - -> pair<bool, optional<start_result>> + -> pair<bool, optional<result>> { try { @@ -240,12 +235,12 @@ namespace brep s.next (nv.first, nv.second); s.next ("", ""); // End of manifest. - return make_pair (true, optional<start_result> ()); + return make_pair (true, optional<result> ()); } catch (const serialization& e) { return make_pair (false, - optional<start_result> ( + optional<result> ( client_error ( 400, string ("invalid manifest override: ") + @@ -261,7 +256,7 @@ namespace brep try { ofdstream os (ovf); - pair<bool, optional<start_result>> r (ovm (os)); + pair<bool, optional<result>> r (ovm (os)); os.close (); if (!r.first) @@ -305,16 +300,16 @@ namespace brep // manifest from its stdout and parse it into the resulting manifest // object. Otherwise, create implied CI result manifest. // - start_result sr; + result sr; - if (options_->ci_handler_specified ()) + if (ops.ci_handler_specified ()) { using namespace external_handler; - optional<result_manifest> r (run (options_->ci_handler (), - options_->ci_handler_argument (), + optional<result_manifest> r (run (ops.ci_handler (), + ops.ci_handler_argument (), dd, - options_->ci_handler_timeout (), + ops.ci_handler_timeout (), error, warn, trace)); @@ -358,7 +353,7 @@ namespace brep { try { - serialize_manifest (sr, os, long_lines); + ci_start::serialize_manifest (sr, os, long_lines); return true; } catch (const serialization& e) @@ -424,7 +419,7 @@ namespace brep // assume that the web server error log is monitored and the email sending // failure will be noticed. // - if (options_->ci_email_specified () && !simulate) + if (ops.ci_email_specified () && !simulate) try { // Redirect the diagnostics to the web server error log. @@ -435,14 +430,13 @@ namespace brep *trace << process_args {args, n}; }, 2 /* stderr */, - options_->email (), + ops.email (), "CI request submission (" + sr.reference + ')', - {options_->ci_email ()}); + {ops.ci_email ()}); // Write the CI request manifest. // - pair<bool, optional<start_result>> r ( - rqm (sm.out, true /* long_lines */)); + pair<bool, optional<result>> r (rqm (sm.out, true /* long_lines */)); assert (r.first); // The serialization succeeded once, so can't fail now. @@ -473,7 +467,55 @@ namespace brep error << "sendmail error: " << e; } - return optional<start_result> (move (sr)); + return optional<result> (move (sr)); + } + + optional<ci_start::start_result> ci_start:: + start (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + optional<tenant_service>&& service, + const repository_location& repository, + const vector<package>& packages, + const optional<string>& client_ip, + const optional<string>& user_agent, + const optional<string>& interactive, + const optional<string>& simulate, + const vector<pair<string, string>>& custom_request, + const vector<pair<string, string>>& overrides) const + { + assert (options_ != nullptr); // Shouldn't be called otherwise. + + // Generate the request id. + // + // Note that it will also be used as a CI result manifest reference, + // unless the latter is provided by the external handler. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + return brep::start (error, warn, trace, + *options_, + move (request_id), + move (service), + false /* service_load */, + repository, + packages, + client_ip, + user_agent, + interactive, + simulate, + custom_request, + overrides); } void ci_start:: @@ -491,4 +533,227 @@ namespace brep s.next ("", ""); // End of manifest. } + + optional<string> ci_start:: + create (const basic_mark& error, + const basic_mark&, + const basic_mark* trace, + odb::core::database& db, + tenant_service&& service, + duration notify_interval, + duration notify_delay) const + { + using namespace odb::core; + + // Generate the request id. + // + string request_id; + + try + { + request_id = uuid::generate ().string (); + } + catch (const system_error& e) + { + error << "unable to generate request id: " << e; + return nullopt; + } + + // Use the generated request id if the tenant service id is not specified. + // + if (service.id.empty ()) + service.id = request_id; + + build_tenant t (move (request_id), + move (service), + system_clock::now () - notify_interval + notify_delay, + notify_interval); + { + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + // Note that in contrast to brep-load, we know that the tenant id is + // unique and thus we don't try to remove a tenant with such an id. + // There is also not much reason to assume that we may have switched + // from the single-tenant mode here and remove the respective tenant, + // unless we are in the tenant-service functionality development mode. + // +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + cstrings ts ({""}); + + db.erase_query<build_package> ( + query<build_package>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_repository> ( + query<build_repository>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_public_key> ( + query<build_public_key>::id.tenant.in_range (ts.begin (), ts.end ())); + + db.erase_query<build_tenant> ( + query<build_tenant>::id.in_range (ts.begin (), ts.end ())); +#endif + + db.persist (t); + + tr.commit (); + } + + if (trace != nullptr) + *trace << "unloaded CI request " << t.id << " for service " + << t.service->id << ' ' << t.service->type << " is created"; + + return move (t.id); + } + + optional<ci_start::start_result> ci_start:: + load (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database& db, + tenant_service&& service, + const repository_location& repository) const + { + using namespace odb::core; + + string request_id; + { + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + using query = query<build_tenant>; + + shared_ptr<build_tenant> t ( + db.query_one<build_tenant> (query::service.id == service.id && + query::service.type == service.type)); + + if (t == nullptr) + { + error << "unable to find tenant for service " << service.id << ' ' + << service.type; + + return nullopt; + } + else if (t->archived) + { + error << "tenant " << t->id << " for service " << service.id << ' ' + << service.type << " is already archived"; + + return nullopt; + } + else if (!t->unloaded_timestamp) + { + error << "tenant " << t->id << " for service " << service.id << ' ' + << service.type << " is already loaded"; + + return nullopt; + } + + t->unloaded_timestamp = nullopt; + db.update (t); + + tr.commit (); + + request_id = move (t->id); + } + + assert (options_ != nullptr); // Shouldn't be called otherwise. + + optional<start_result> r (brep::start (error, warn, trace, + *options_, + move (request_id), + move (service), + true /* service_load */, + repository, + {} /* packages */, + nullopt /* client_ip */, + nullopt /* user_agent */, + nullopt /* interactive */, + nullopt /* simulate */, + {} /* custom_request */, + {} /* overrides */)); + + // Note: on error (r == nullopt) the diagnostics is already issued. + // + if (trace != nullptr && r) + *trace << "CI request for '" << repository << "' is " + << (r->status != 200 ? "not " : "") << "loaded: " + << r->message << " (reference: " << r->reference << ')'; + + return r; + } + + optional<tenant_service> ci_start:: + cancel (const basic_mark&, + const basic_mark&, + const basic_mark* trace, + odb::core::database& db, + const string& type, + const string& id) const + { + using namespace odb::core; + + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + using query = query<build_tenant>; + + shared_ptr<build_tenant> t ( + db.query_one<build_tenant> (query::service.id == id && + query::service.type == type)); + if (t == nullptr) + return nullopt; + + optional<tenant_service> r (move (t->service)); + t->service = nullopt; + t->archived = true; + db.update (t); + + tr.commit (); + + if (trace != nullptr) + *trace << "CI request " << t->id << " for service " << id << ' ' << type + << " is canceled"; + + return r; + } + + bool ci_start:: + cancel (const basic_mark&, + const basic_mark&, + const basic_mark* trace, + const string& reason, + odb::core::database& db, + const string& tid) const + { + using namespace odb::core; + + assert (!transaction::has_current ()); + + transaction tr (db.begin ()); + + shared_ptr<build_tenant> t (db.find<build_tenant> (tid)); + + if (t == nullptr) + return false; + + if (!t->archived) + { + t->archived = true; + db.update (t); + } + + tr.commit (); + + if (trace != nullptr) + *trace << "CI request " << tid << " is canceled: " + << (reason.size () < 50 + ? reason + : string (reason, 0, 50) + "..."); + + return true; + } } diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx index 6f62c4b..848bca1 100644 --- a/mod/ci-common.hxx +++ b/mod/ci-common.hxx @@ -36,6 +36,7 @@ namespace brep package_name name; optional<brep::version> version; }; + // Note that the inability to generate the reference is an internal // error. Thus, it is not optional. // @@ -62,7 +63,67 @@ namespace brep const optional<string>& interactive = nullopt, const optional<string>& simulate = nullopt, const vector<pair<string, string>>& custom_request = {}, - const vector<pair<string, string>>& overrides = {}); + const vector<pair<string, string>>& overrides = {}) const; + + // Create an unloaded CI request returning start_result::reference on + // success and nullopt on an internal error. Such a request is not started + // until loaded with the load() function below. Configure the time + // interval between the build_unloaded() notifications for the being + // created tenant and set the initial delay for the first notification. + // See also the build_unloaded() tenant services notification. + // + // Note: should be called out of the database transaction. + // + optional<string> + create (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + tenant_service&&, + duration notify_interval, + duration notify_delay) const; + + // Load (and start) previously created (as unloaded) CI request. Similarly + // to the start() function, return nullopt on an internal error. + // + // Note that tenant_service::id is used to identify the CI request tenant. + // + // Note: should be called out of the database transaction. + // + optional<start_result> + load (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + tenant_service&&, + const repository_location& repository) const; + + // Cancel previously created or started CI request. Return the service + // state or nullopt if there is no tenant for such a type/id pair. + // + // Note: should be called out of the database transaction. + // + optional<tenant_service> + cancel (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + odb::core::database&, + const string& type, + const string& id) const; + + // Cancel previously created or started CI request. Return false if there + // is no tenant for the specified tenant id. Note that the reason argument + // is only used for tracing. + // + // Note: should be called out of the database transaction. + // + bool + cancel (const basic_mark& error, + const basic_mark& warn, + const basic_mark* trace, + const string& reason, + odb::core::database&, + const string& tenant_id) const; // Helpers. // @@ -75,22 +136,6 @@ namespace brep private: shared_ptr<options::ci_start> options_; }; - - class ci_cancel - { - public: - void - init (shared_ptr<options::ci_cancel>, shared_ptr<odb::core::database>); - - // @@ TODO Archive the tenant. - // - void - cancel (/*...*/); - - private: - shared_ptr<options::ci_cancel> options_; - shared_ptr<odb::core::database> build_db_; - }; } #endif // MOD_CI_COMMON_HXX diff --git a/mod/mod-build-configs.cxx b/mod/mod-build-configs.cxx index 9282544..ce79edb 100644 --- a/mod/mod-build-configs.cxx +++ b/mod/mod-build-configs.cxx @@ -30,8 +30,6 @@ build_configs (const build_configs& r) void brep::build_configs:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_configs> ( s, unknown_mode::fail, unknown_mode::fail); @@ -127,19 +125,19 @@ handle (request& rq, response& rs) s << DIV(ID="filter-heading") << "Build Configuration Classes" << ~DIV << P(ID="filter"); + bool printed (false); for (auto b (cls.begin ()), i (b), e (cls.end ()); i != e; ++i) { - // Skip the 'hidden' class. + // Skip the hidden classes. // const string& c (*i); - if (c != "hidden") + if (!derived (c, "hidden")) { - // Note that here we rely on the fact that the first class in the list - // can never be 'hidden' (is always 'all'). - // - if (i != b) + if (printed) s << ' '; + else + printed = true; print_class_name (c, c == selected_class); diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx index bdae356..ea921e9 100644 --- a/mod/mod-build-force.cxx +++ b/mod/mod-build-force.cxx @@ -42,8 +42,6 @@ build_force (const build_force& r, const tenant_service_map& tsm) void brep::build_force:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_force> ( s, unknown_mode::fail, unknown_mode::fail); @@ -192,7 +190,14 @@ handle (request& rq, response& rs) optional<pair<tenant_service, shared_ptr<build>>> tss; tenant_service_build_queued::build_queued_hints qhs; + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // connection_ptr conn (build_db_->connection ()); + { transaction t (conn->begin ()); @@ -297,14 +302,28 @@ handle (request& rq, response& rs) vector<build> qbs; qbs.push_back (move (b)); + // Release the database connection since the build_queued() notification + // can potentially be time-consuming (e.g., it may perform an HTTP + // request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, build_state::building, qhs, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, qbs.back ().tenant, f); + } } + // Release the database connection prior to writing into the unbuffered + // response stream. + // + conn.reset (); + // We have all the data, so don't buffer the response content. // ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx index c8e803b..5487f6e 100644 --- a/mod/mod-build-log.cxx +++ b/mod/mod-build-log.cxx @@ -34,8 +34,6 @@ build_log (const build_log& r) void brep::build_log:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_log> ( s, unknown_mode::fail, unknown_mode::fail); diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index ccce17f..3ba18e1 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -49,8 +49,6 @@ build_result (const build_result& r, const tenant_service_map& tsm) void brep::build_result:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::build_result> ( s, unknown_mode::fail, unknown_mode::fail); @@ -207,13 +205,20 @@ handle (request& rq, response&) optional<pair<tenant_service, shared_ptr<build>>> tss; tenant_service_build_queued::build_queued_hints qhs; + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // + connection_ptr conn (build_db_->connection ()); + // Note that if the session authentication fails (probably due to the // authentication settings change), then we log this case with the warning // severity and respond with the 200 HTTP code as if the challenge is // valid. The thinking is that we shouldn't alarm a law-abaiding agent and // shouldn't provide any information to a malicious one. // - connection_ptr conn (build_db_->connection ()); { transaction t (conn->begin ()); @@ -518,12 +523,20 @@ handle (request& rq, response&) vector<build> qbs; qbs.push_back (move (*tss->second)); + // Release the database connection since build_queued() notification can + // potentially be time-consuming (e.g., it may perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, build_state::building, qhs, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, qbs.back ().tenant, f); + } } // If a third-party service needs to be notified about the built package, @@ -537,8 +550,16 @@ handle (request& rq, response&) const tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since build_built() notification can + // potentially be time-consuming (e.g., it may perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_built (ss, b, log_writer_)) + { + conn = build_db_->connection (); update_tenant_service_state (conn, b.tenant, f); + } } if (bld != nullptr) @@ -549,6 +570,9 @@ handle (request& rq, response&) if (!build_notify) (cfg->email ? cfg->email : pkg->build_email) = email (); + if (conn == nullptr) + conn = build_db_->connection (); + send_notification_email (*options_, conn, *bld, diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 07aff8d..6be77f6 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -399,6 +399,79 @@ handle (request& rq, response& rs) } } + // Acquire the database connection for the subsequent transactions. + // + // Note that we will release it prior to any potentially time-consuming + // operations (such as HTTP requests) and re-acquire it again afterwards, + // if required. + // + connection_ptr conn (build_db_->connection ()); + + // Perform some housekeeping first. + // + // Notify a tenant-associated third-party service about the unloaded CI + // request, if present. + // + { + const tenant_service_build_unloaded* tsu (nullptr); + + transaction tr (conn->begin ()); + + using query = query<build_tenant>; + + // Pick the unloaded tenant with the earliest loaded timestamp, skipping + // those which were already picked recently. + // + shared_ptr<build_tenant> t ( + build_db_->query_one<build_tenant> ( + (!query::archived && + query::unloaded_timestamp.is_not_null () && + (query::unloaded_timestamp + + "<= EXTRACT (EPOCH FROM NOW()) * 1000000000 - " + + query::unloaded_notify_interval)) + + "ORDER BY" + query::unloaded_timestamp + + "LIMIT 1")); + + if (t != nullptr && t->service) + { + auto i (tenant_service_map_.find (t->service->type)); + + if (i != tenant_service_map_.end ()) + { + tsu = dynamic_cast<const tenant_service_build_unloaded*> ( + i->second.get ()); + + if (tsu != nullptr) + { + // If we ought to call the + // tenant_service_build_unloaded::build_unloaded() callback, then + // set the package tenant's loaded timestamp to the current time to + // prevent the notifications race. + // + t->unloaded_timestamp = system_clock::now (); + build_db_->update (t); + } + } + } + + tr.commit (); + + if (tsu != nullptr) + { + // Release the database connection since the build_unloaded() + // notification can potentially be time-consuming (e.g., it may perform + // an HTTP request). + // + conn.reset (); + + if (auto f = tsu->build_unloaded (move (*t->service), log_writer_)) + { + conn = build_db_->connection (); + update_tenant_service_state (conn, t->id, f); + } + } + } + // Go through package build configurations until we find one that has no // build target configuration present in the database, or is in the building // state but expired (collectively called unbuilt). If such a target @@ -825,7 +898,10 @@ handle (request& rq, response& rs) imode, queued_expiration_ns)); - transaction t (build_db_->begin ()); + if (conn == nullptr) + conn = build_db_->connection (); + + transaction t (conn->begin ()); // If there are any non-archived interactive build tenants, then the // chosen randomization approach doesn't really work since interactive @@ -886,7 +962,8 @@ handle (request& rq, response& rs) "OFFSET" + pkg_query::_ref (offset) + "LIMIT" + pkg_query::_ref (limit); - connection_ptr conn (build_db_->connection ()); + if (conn == nullptr) + conn = build_db_->connection (); prep_pkg_query pkg_prep_query ( conn->prepare_query<buildable_package> ( @@ -2226,12 +2303,20 @@ handle (request& rq, response& rs) if (!qbs.empty ()) { + // Release the database connection since the build_queued() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, nullopt /* initial_state */, qhs, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, qbs.back ().tenant, f)) ss.data = move (data); @@ -2250,12 +2335,20 @@ handle (request& rq, response& rs) qbs.push_back (move (b)); restore_build = true; + // Release the database connection since the build_queued() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsq->build_queued (ss, qbs, initial_state, qhs, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, qbs.back ().tenant, f)) ss.data = move (data); @@ -2278,8 +2371,16 @@ handle (request& rq, response& rs) tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since the build_building() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_building (ss, b, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, b.tenant, f)) ss.data = move (data); @@ -2306,6 +2407,9 @@ handle (request& rq, response& rs) const tenant_service_build_built* tsb (nullptr); optional<pair<tenant_service, shared_ptr<build>>> tss; { + if (conn == nullptr) + conn = build_db_->connection (); + transaction t (conn->begin ()); shared_ptr<build> b (build_db_->find<build> (task_build->id)); @@ -2395,8 +2499,16 @@ handle (request& rq, response& rs) tenant_service& ss (tss->first); const build& b (*tss->second); + // Release the database connection since the build_built() + // notification can potentially be time-consuming (e.g., it may + // perform an HTTP request). + // + conn.reset (); + if (auto f = tsb->build_built (ss, b, log_writer_)) { + conn = build_db_->connection (); + if (optional<string> data = update_tenant_service_state (conn, b.tenant, f)) ss.data = move (data); @@ -2407,6 +2519,10 @@ handle (request& rq, response& rs) // Send notification emails for all the aborted builds. // for (const aborted_build& ab: aborted_builds) + { + if (conn == nullptr) + conn = build_db_->connection (); + send_notification_email (*options_, conn, *ab.b, @@ -2415,9 +2531,14 @@ handle (request& rq, response& rs) ab.what, error, verb_ >= 2 ? &trace : nullptr); + } } } + // Release the database connection as soon as possible. + // + conn.reset (); + serialize_task_response_manifest (); return true; } diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index 30562f3..81d4649 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -50,8 +50,6 @@ builds (const builds& r) void brep::builds:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::builds> ( s, unknown_mode::fail, unknown_mode::fail); diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index 5974d45..8c47bc4 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -22,6 +22,8 @@ using namespace butl; using namespace web; using namespace brep::cli; +// ci +// #ifdef BREP_CI_TENANT_SERVICE brep::ci:: ci (tenant_service_map& tsm) @@ -36,7 +38,12 @@ ci (const ci& r, tenant_service_map& tsm) #else ci (const ci& r) #endif - : handler (r), + : +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED + handler (r), + #else + database_module (r), +#endif ci_start (r), options_ (r.initialized_ ? r.options_ : nullptr), form_ (r.initialized_ || r.form_ == nullptr @@ -100,6 +107,13 @@ init (scanner& s) } } +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + if (!options_->build_config_specified ()) + fail << "package building functionality must be enabled"; + + database_module::init (*options_, options_->build_db_retry ()); +#endif + if (options_->root ().empty ()) options_->root (dir_path ("/")); } @@ -347,6 +361,7 @@ handle (request& rq, response& rs) user_agent = h.value; } +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED optional<start_result> r (start (error, warn, verb_ ? &trace : nullptr, @@ -367,6 +382,25 @@ handle (request& rq, response& rs) : optional<string> ()), custom_request, overrides)); +#else + assert (build_db_ != nullptr); // Wouldn't be here otherwise. + + optional<start_result> r; + + if (optional<string> ref = create (error, + warn, + verb_ ? &trace : nullptr, + *build_db_, + tenant_service ("", "ci", rl.string ()), + chrono::seconds (40), + chrono::seconds (10))) + { + string msg ("unloaded CI request is created: " + + options_->host () + tenant_dir (root, *ref).string ()); + + r = start_result {200, move (msg), move (*ref), {}}; + } +#endif if (!r) return respond_error (); // The diagnostics is already issued. @@ -472,4 +506,97 @@ build_built (const tenant_service&, return ts.data ? *ts.data + ", " + s : s; }; } + +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED +function<optional<string> (const brep::tenant_service&)> brep::ci:: +build_unloaded (tenant_service&& ts, + const diag_epilogue& log_writer) const noexcept +{ + NOTIFICATION_DIAG (log_writer); + + assert (ts.data); // Repository location. + + try + { + repository_location rl (*ts.data); + + if (!load (error, warn, verb_ ? &trace : nullptr, + *build_db_, + move (ts), + rl)) + return nullptr; // The diagnostics is already issued. + } + catch (const invalid_argument& e) + { + error << "invalid repository location '" << *ts.data << "' stored for " + << "tenant service " << ts.id << ' ' << ts.type; + + return nullptr; + } + + return [] (const tenant_service& ts) {return "loaded " + *ts.data;}; +} +#endif #endif + +// ci_cancel +// +brep::ci_cancel:: +ci_cancel (const ci_cancel& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::ci_cancel:: +init (scanner& s) +{ + options_ = make_shared<options::ci_cancel> ( + s, unknown_mode::fail, unknown_mode::fail); + + if (options_->build_config_specified ()) + database_module::init (*options_, options_->build_db_retry ()); +} + +bool brep::ci_cancel:: +handle (request& rq, response& rs) +{ + HANDLER_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + params::ci_cancel params; + + try + { + name_value_scanner s (rq.parameters (1024)); + params = params::ci_cancel (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + const string& reason (params.reason ()); + + if (reason.empty ()) + throw invalid_request (400, "missing CI request cancellation reason"); + + // Verify the tenant id. + // + const string tid (params.id ()); + + if (tid.empty ()) + throw invalid_request (400, "invalid CI request id"); + + if (!cancel (error, warn, verb_ ? &trace : nullptr, reason, *build_db_, tid)) + throw invalid_request (400, "unknown CI request id"); + + // We have all the data, so don't buffer the response content. + // + ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); + os << "CI request " << tid << " has been canceled"; + + return true; +} diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx index 1e2ee15..bd91e99 100644 --- a/mod/mod-ci.hxx +++ b/mod/mod-ci.hxx @@ -16,6 +16,11 @@ #include <mod/module-options.hxx> #include <mod/ci-common.hxx> +#include <mod/database-module.hxx> + +#if defined(BREP_CI_TENANT_SERVICE_UNLOADED) && !defined(BREP_CI_TENANT_SERVICE) +# error BREP_CI_TENANT_SERVICE must be defined if BREP_CI_TENANT_SERVICE_UNLOADED is defined +#endif #ifdef BREP_CI_TENANT_SERVICE # include <mod/tenant-service.hxx> @@ -23,12 +28,20 @@ namespace brep { - class ci: public handler, + class ci: +#ifndef BREP_CI_TENANT_SERVICE_UNLOADED + public handler, +#else + public database_module, +#endif private ci_start #ifdef BREP_CI_TENANT_SERVICE , public tenant_service_build_queued, public tenant_service_build_building, public tenant_service_build_built +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + , tenant_service_build_unloaded +#endif #endif { public: @@ -74,6 +87,12 @@ namespace brep build_built (const tenant_service&, const build&, const diag_epilogue& log_writer) const noexcept override; + +#ifdef BREP_CI_TENANT_SERVICE_UNLOADED + virtual function<optional<string> (const tenant_service&)> + build_unloaded (tenant_service&&, + const diag_epilogue& log_writer) const noexcept override; +#endif #endif private: @@ -88,6 +107,32 @@ namespace brep tenant_service_map& tenant_service_map_; #endif }; + + class ci_cancel: public database_module, + private ci_start + { + public: + ci_cancel () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + ci_cancel (const ci_cancel&); + + virtual bool + handle (request&, response&) override; + + virtual const cli::options& + cli_options () const override {return options::ci_cancel::description ();} + + private: + virtual void + init (cli::scanner&) override; + + private: + shared_ptr<options::ci_cancel> options_; + }; } #endif // MOD_MOD_CI_HXX diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx index fcd50da..1fb51da 100644 --- a/mod/mod-package-details.cxx +++ b/mod/mod-package-details.cxx @@ -37,8 +37,6 @@ package_details (const package_details& r) void brep::package_details:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::package_details> ( s, unknown_mode::fail, unknown_mode::fail); diff --git a/mod/mod-repository-details.cxx b/mod/mod-repository-details.cxx index 082903b..93b6c9e 100644 --- a/mod/mod-repository-details.cxx +++ b/mod/mod-repository-details.cxx @@ -39,8 +39,6 @@ repository_details (const repository_details& r) void brep::repository_details:: init (scanner& s) { - HANDLER_DIAG; - options_ = make_shared<options::repository_details> ( s, unknown_mode::fail, unknown_mode::fail); diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index 34b4007..bc861a8 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -133,6 +133,7 @@ namespace brep #else ci_ (make_shared<ci> ()), #endif + ci_cancel_ (make_shared<ci_cancel> ()), upload_ (make_shared<upload> ()) { } @@ -201,6 +202,10 @@ namespace brep #else : make_shared<ci> (*r.ci_)), #endif + ci_cancel_ ( + r.initialized_ + ? r.ci_cancel_ + : make_shared<ci_cancel> (*r.ci_cancel_)), upload_ ( r.initialized_ ? r.upload_ @@ -231,6 +236,7 @@ namespace brep append (r, build_configs_->options ()); append (r, submit_->options ()); append (r, ci_->options ()); + append (r, ci_cancel_->options ()); append (r, upload_->options ()); return r; } @@ -277,6 +283,7 @@ namespace brep sub_init (*build_configs_, "build_configs"); sub_init (*submit_, "submit"); sub_init (*ci_, "ci"); + sub_init (*ci_cancel_, "ci-cancel"); sub_init (*upload_, "upload"); // Parse own configuration options. @@ -473,6 +480,13 @@ namespace brep return handle ("ci", param); } + else if (func == "ci-cancel") + { + if (handler_ == nullptr) + handler_.reset (new ci_cancel (*ci_cancel_)); + + return handle ("ci-cancel", param); + } else if (func == "upload") { if (handler_ == nullptr) diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx index aa60fda..990587e 100644 --- a/mod/mod-repository-root.hxx +++ b/mod/mod-repository-root.hxx @@ -25,6 +25,7 @@ namespace brep class build_configs; class submit; class ci; + class ci_cancel; class upload; class repository_root: public handler @@ -74,6 +75,7 @@ namespace brep shared_ptr<build_configs> build_configs_; shared_ptr<submit> submit_; shared_ptr<ci> ci_; + shared_ptr<ci_cancel> ci_cancel_; shared_ptr<upload> upload_; shared_ptr<options::repository_root> options_; diff --git a/mod/module.cli b/mod/module.cli index a107ffe..5133935 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -796,11 +796,7 @@ namespace brep } }; - class ci_cancel - { - }; - - class ci: ci_start, page, repository_url, handler + class ci: ci_start, build, build_db, page, repository_url, handler { // Classic CI-specific options. // @@ -815,7 +811,11 @@ namespace brep } }; - class ci_github: ci_start, ci_cancel, build_db, handler + class ci_cancel: build, build_db, handler + { + }; + + class ci_github: ci_start, build, build_db, handler { // GitHub CI-specific options (e.g., request timeout when invoking // GitHub APIs). @@ -1099,6 +1099,22 @@ namespace brep string simulate; }; + // All parameters are non-optional. + // + class ci_cancel + { + // CI task tenant id. + // + // Note that the ci-cancel parameter is renamed to '_' by the root + // handler (see the request_proxy class for details). + // + string id | _; + + // CI task canceling reason. Must not be empty. + // + string reason; + }; + // Parameters other than challenge must be all present. // // Note also that besides these parameters there can be others. We don't diff --git a/mod/page.cxx b/mod/page.cxx index bc2e42d..177fb64 100644 --- a/mod/page.cxx +++ b/mod/page.cxx @@ -739,7 +739,7 @@ namespace brep << ~TR; } - // BUILD_RESULT + // TR_BUILD_RESULT // void TR_BUILD_RESULT:: operator() (serializer& s) const diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx index 9205f76..b7f5c02 100644 --- a/mod/tenant-service.hxx +++ b/mod/tenant-service.hxx @@ -21,7 +21,8 @@ namespace brep virtual ~tenant_service_base () = default; }; - // Possible build notifications: + // Possible build notifications (see also the unloaded special notification + // below): // // queued // building @@ -121,6 +122,22 @@ namespace brep const diag_epilogue& log_writer) const noexcept = 0; }; + // This notification is only made on unloaded CI requests created with the + // ci_start::create() call and until they are loaded with ci_start::load() + // or, alternatively, abandoned with ci_start::abandon(). + // + // Note: make sure the implementation of this notification does not take + // too long (currently 40 seconds) to avoid nested notifications. Note + // also that the first notification is delayed (currently 10 seconds). + // + class tenant_service_build_unloaded: public virtual tenant_service_base + { + public: + virtual function<optional<string> (const tenant_service&)> + build_unloaded (tenant_service&&, + const diag_epilogue& log_writer) const noexcept = 0; + }; + // Map of service type (tenant_service::type) to service. // using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>; diff --git a/repositories.manifest b/repositories.manifest index da9ee2b..e760afd 100644 --- a/repositories.manifest +++ b/repositories.manifest @@ -3,23 +3,23 @@ summary: build2 package repository web interface repository : role: prerequisite -location: ../libbutl.git##HEAD +location: ../libbutl.git#HEAD : role: prerequisite -location: ../libbpkg.git##HEAD +location: ../libbpkg.git#HEAD : role: prerequisite -location: ../libbbot.git##HEAD +location: ../libbbot.git#HEAD : role: prerequisite -location: ../libbutl.bash.git##HEAD +location: ../libbutl.bash.git#HEAD : role: prerequisite -location: ../bpkg-util.git##HEAD +location: ../bpkg-util.git#HEAD : role: prerequisite @@ -35,11 +35,7 @@ location: https://git.build2.org/packaging/cmark-gfm/cmark-gfm.git##HEAD : role: prerequisite -location: https://git.codesynthesis.com/odb/libodb.git##HEAD - -: -role: prerequisite -location: https://git.codesynthesis.com/odb/libodb-pgsql.git##HEAD +location: https://git.codesynthesis.com/odb/odb.git##HEAD : role: prerequisite |