diff options
Diffstat (limited to 'mod')
| -rw-r--r-- | mod/buildfile | 2 | ||||
| -rw-r--r-- | mod/mod-build-result.cxx | 4 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 130 | ||||
| -rw-r--r-- | mod/mod-builds.cxx | 58 | ||||
| -rw-r--r-- | mod/mod-ci.cxx | 18 | ||||
| -rw-r--r-- | mod/mod-package-version-details.cxx | 43 | ||||
| -rw-r--r-- | mod/mod-packages.cxx | 6 | ||||
| -rw-r--r-- | mod/module.cli | 22 | ||||
| -rw-r--r-- | mod/types-parsers.cxx | 42 | ||||
| -rw-r--r-- | mod/types-parsers.hxx | 9 |
10 files changed, 256 insertions, 78 deletions
diff --git a/mod/buildfile b/mod/buildfile index 191d966..ff9cd60 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -50,7 +50,7 @@ if $cli.configured cli.options += --std c++11 -I $src_root --include-with-brackets \ --include-prefix mod --guard-prefix MOD --generate-specifier \ --cxx-prologue "#include <mod/types-parsers.hxx>" \ ---cli-namespace brep::cli --generate-file-scanner --option-length 41 \ +--cli-namespace brep::cli --generate-file-scanner --option-length 45 \ --generate-modifier --generate-description --option-prefix "" # Include the generated cli files into the distribution and don't remove diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx index 8a684fe..bec362a 100644 --- a/mod/mod-build-result.cxx +++ b/mod/mod-build-result.cxx @@ -398,6 +398,10 @@ handle (request& rq, response&) b->status = rqm.result.status; b->force = force_state::unforced; + // Cleanup the interactive build login information. + // + b->interactive = nullopt; + // Cleanup the authentication data. // b->agent_fingerprint = nullopt; diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx index 04b2a36..ebf434a 100644 --- a/mod/mod-build-task.cxx +++ b/mod/mod-build-task.cxx @@ -4,12 +4,14 @@ #include <mod/mod-build-task.hxx> #include <map> +#include <regex> #include <chrono> #include <odb/database.hxx> #include <odb/transaction.hxx> #include <odb/schema-catalog.hxx> +#include <libbutl/regex.mxx> #include <libbutl/sha256.mxx> #include <libbutl/utility.mxx> // compare_c_string #include <libbutl/openssl.mxx> @@ -195,11 +197,12 @@ handle (request& rq, response& rs) { vector<shared_ptr<build>> rebuilds; - // Create the task response manifest. The package must have the internal - // repository loaded. + // Create the task response manifest. Must be called inside the build db + // transaction. // auto task = [this] (shared_ptr<build>&& b, shared_ptr<build_package>&& p, + shared_ptr<build_tenant>&& t, const config_machine& cm) -> task_response_manifest { uint64_t ts ( @@ -218,7 +221,11 @@ handle (request& rq, response& rs) tenant_dir (options_->root (), b->tenant).string () + "?build-result"); - lazy_shared_ptr<build_repository> r (p->internal_repository); + assert (transaction::has_current ()); + + assert (p->internal ()); // The package is expected to be buildable. + + lazy_shared_ptr<build_repository> r (p->internal_repository.load ()); strings fps; if (r->certificate_fingerprint) @@ -265,7 +272,8 @@ handle (request& rq, response& rs) cm.config->target, cm.config->environment, cm.config->args, - cm.config->warning_regexes); + cm.config->warning_regexes, + move (t->interactive)); return task_response_manifest (move (session), move (b->agent_challenge), @@ -430,12 +438,66 @@ handle (request& rq, response& rs) pkg_query::build_repository::id.canonical_name.in_range (rp.begin (), rp.end ()); + // Transform (in-place) the interactive login information into the actual + // login command, if specified in the manifest and the transformation + // regexes are specified in the configuration. + // + if (tqm.interactive_login && + options_->build_interactive_login_specified ()) + { + optional<string> lc; + string l (tqm.agent + ' ' + *tqm.interactive_login); + + // Use the first matching regex for the transformation. + // + for (const pair<regex, string>& rf: options_->build_interactive_login ()) + { + pair<string, bool> r (regex_replace_match (l, rf.first, rf.second)); + + if (r.second) + { + lc = move (r.first); + break; + } + } + + if (!lc) + throw invalid_request (400, "unable to match login info '" + l + "'"); + + tqm.interactive_login = move (lc); + } + + // If the interactive mode if false or true, then filter out the + // respective packages. Otherwise, order them so that packages from the + // interactive build tenants appear first. + // + interactive_mode imode (tqm.effective_interactive_mode ()); + + switch (imode) + { + case interactive_mode::false_: + { + pq = pq && pkg_query::build_tenant::interactive.is_null (); + break; + } + case interactive_mode::true_: + { + pq = pq && pkg_query::build_tenant::interactive.is_not_null (); + break; + } + case interactive_mode::both: break; // See below. + } + // Specify the portion. // size_t offset (0); - pq += "ORDER BY" + - pkg_query::build_package::id.tenant + "," + + pq += "ORDER BY"; + + if (imode == interactive_mode::both) + pq += pkg_query::build_tenant::interactive + "NULLS LAST,"; + + pq += pkg_query::build_package::id.tenant + "," + pkg_query::build_package::id.name + order_by_version (pkg_query::build_package::id.version, false) + "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"; @@ -512,8 +574,8 @@ handle (request& rq, response& rs) // configurations that remained can be built. We will take the first // one, if present. // - // Also save the built package configurations for which it's time to be - // rebuilt. + // Also save the built package configurations for which it's time to + // be rebuilt. // config_machines configs (cfg_machines); // Make a copy for this pkg. auto pkg_builds (bld_prep_query.execute ()); @@ -567,6 +629,16 @@ handle (request& rq, response& rs) shared_ptr<build> b (build_db_->find<build> (bid)); optional<string> cl (challenge ()); + shared_ptr<build_tenant> t ( + build_db_->load<build_tenant> (bid.package.tenant)); + + // Move the interactive build login information into the build + // object, if the package to be built interactively. + // + optional<string> login (t->interactive + ? move (tqm.interactive_login) + : nullopt); + // If build configuration doesn't exist then create the new one // and persist. Otherwise put it into the building state, refresh // the timestamp and update. @@ -579,6 +651,7 @@ handle (request& rq, response& rs) move (bid.configuration), move (bid.toolchain_name), move (toolchain_version), + move (login), move (agent_fp), move (cl), mh.name, @@ -606,6 +679,7 @@ handle (request& rq, response& rs) b->results.empty ()); b->state = build_state::building; + b->interactive = move (login); // Switch the force state not to reissue the task after the // forced rebuild timeout. Note that the result handler will @@ -626,13 +700,7 @@ handle (request& rq, response& rs) // Finally, prepare the task response manifest. // - // We iterate over buildable packages. - // - assert (p->internal ()); - - p->internal_repository.load (); - - tsm = task (move (b), move (p), cm); + tsm = task (move (b), move (p), move (t), cm); } } @@ -704,20 +772,40 @@ handle (request& rq, response& rs) assert (i != cfg_machines.end ()); const config_machine& cm (i->second); - // Rebuild the package if still present, is buildable and doesn't - // exclude the configuration. + // Rebuild the package if still present, is buildable, doesn't + // exclude the configuration, and matches the request's + // interactive mode. + // + // Note that while change of the latter seems rather far fetched, + // let's check it for good measure. // shared_ptr<build_package> p ( build_db_->find<build_package> (b->id.package)); - if (p != nullptr && - p->internal () && + shared_ptr<build_tenant> t ( + p != nullptr + ? build_db_->load<build_tenant> (p->id.tenant) + : nullptr); + + if (p != nullptr && + p->buildable && + (t->interactive.has_value () == + (imode != interactive_mode::false_)) && !exclude (p->builds, p->constraints, *cm.config)) { assert (b->status); b->state = build_state::building; + // Save the interactive build login information into the build + // object, if the package to be built interactively. + // + // Can't move from, as may need it on the next iteration. + // + b->interactive = t->interactive + ? tqm.interactive_login + : nullopt; + // Can't move from, as may need them on the next iteration. // b->agent_fingerprint = agent_fp; @@ -738,9 +826,7 @@ handle (request& rq, response& rs) build_db_->update (b); - p->internal_repository.load (); - - tsm = task (move (b), move (p), cm); + tsm = task (move (b), move (p), move (t), cm); } } diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index ab9e93e..5ffe6dd 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -133,6 +133,8 @@ match (const C qc, const string& pattern) return qc + "SIMILAR TO" + query<T>::_val (transform (pattern)); } +// If tenant is absent, then query builds from all the public tenants. +// template <typename T> static inline query<T> build_query (const brep::cstrings* configs, @@ -143,18 +145,18 @@ build_query (const brep::cstrings* configs, using namespace brep; using query = query<T>; using qb = typename query::build; - - query q (configs != nullptr - ? qb::id.configuration.in_range (configs->begin (), configs->end ()) - : query (true)); + using qt = typename query::build_tenant; const auto& pid (qb::id.package); - if (tenant) - q = q && pid.tenant == *tenant; + query q (tenant ? pid.tenant == *tenant : !qt::private_); if (archived) - q = q && query::build_tenant::archived == *archived; + q = q && qt::archived == *archived; + + if (configs != nullptr) + q = q && qb::id.configuration.in_range (configs->begin (), + configs->end ()); // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no package builds match such a @@ -267,6 +269,8 @@ build_query (const brep::cstrings* configs, return q; } +// If tenant is absent, then query packages from all the public tenants. +// template <typename T> static inline query<T> package_query (const brep::params::builds& params, @@ -276,14 +280,12 @@ package_query (const brep::params::builds& params, using namespace brep; using query = query<T>; using qp = typename query::build_package; + using qt = typename query::build_tenant; - query q (true); - - if (tenant) - q = q && qp::id.tenant == *tenant; + query q (tenant ? qp::id.tenant == *tenant : !qt::private_); if (archived) - q = q && query::build_tenant::archived == *archived; + q = q && qt::archived == *archived; // Note that there is no error reported if the filter parameters parsing // fails. Instead, it is considered that no packages match such a query. @@ -383,7 +385,7 @@ handle (request& rq, response& rs) << DIV(ID="content"); // If the tenant is empty then we are in the global view and will display - // builds from all the tenants. + // builds from all the public tenants. // optional<string> tn; if (!tenant.empty ()) @@ -672,14 +674,18 @@ handle (request& rq, response& rs) << TR_VALUE ("config", b.configuration) << TR_VALUE ("machine", b.machine) << TR_VALUE ("target", b.target.string ()) - << TR_VALUE ("timestamp", ts) - << TR_BUILD_RESULT (b, host, root); + << TR_VALUE ("timestamp", ts); + + if (b.interactive) // Note: can only be present for the building state. + s << TR_VALUE ("login", *b.interactive); + + s << TR_BUILD_RESULT (b, host, root); // In the global view mode add the tenant builds link. Note that the // global view (and the link) makes sense only in the multi-tenant mode. // if (!tn && !b.tenant.empty ()) - s << TR_TENANT (tenant_name, "builds", root, b.tenant); + s << TR_TENANT (tenant_name, "builds", root, b.tenant); s << ~TBODY << ~TABLE; @@ -809,15 +815,20 @@ handle (request& rq, response& rs) const auto& bid (bld_query::build::id); - bld_query bq (equal<package_build_count> (bid.package, id) && - bid.configuration == bld_query::_ref (config) && + bld_query bq ( + equal<package_build_count> (bid.package, id) && + bid.configuration == bld_query::_ref (config) && // Note that the query already constrains configurations via the - // configuration name and the tenant via the build package id. + // configuration name. + // + // Also note that while the query already constrains the tenant via + // the build package id, we still need to pass the tenant not to + // erroneously filter out the private tenants. // build_query<package_build_count> (nullptr /* configs */, bld_params, - nullopt /* tenant */, + tn, false /* archived */)); prep_bld_query bld_prep_query ( @@ -925,11 +936,12 @@ handle (request& rq, response& rs) bld_query bq ( equal<package_build> (bld_query::build::id.package, id) && - // Note that the query already constrains the tenant via the build - // package id. + // Note that while the query already constrains the tenant via the build + // package id, we still need to pass the tenant not to erroneously + // filter out the private tenants. // build_query<package_build> ( - &conf_names, bld_params, nullopt /* tenant */, false /* archived */)); + &conf_names, bld_params, tn, false /* archived */)); prep_bld_query bld_prep_query ( conn->prepare_query<package_build> ("mod-builds-build-query", bq)); diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx index d2da93f..4da72b6 100644 --- a/mod/mod-ci.cxx +++ b/mod/mod-ci.cxx @@ -376,15 +376,18 @@ handle (request& rq, response& rs) s.next ("package", p); } + if (params.interactive_specified ()) + s.next ("interactive", params.interactive ()); + + if (!simulate.empty ()) + s.next ("simulate", simulate); + s.next ("timestamp", butl::to_string (ts, "%Y-%m-%dT%H:%M:%SZ", false /* special */, false /* local */)); - if (!simulate.empty ()) - s.next ("simulate", simulate); - // Serialize the User-Agent HTTP header and the client IP address. // optional<string> ip; @@ -412,10 +415,11 @@ handle (request& rq, response& rs) { const string& n (nv.name); - if (n != "repository" && - n != "_" && - n != "package" && - n != "overrides" && + if (n != "repository" && + n != "_" && + n != "package" && + n != "overrides" && + n != "interactive" && n != "simulate") s.next (n, nv.value ? *nv.value : ""); } diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index a7682ec..a23989e 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -472,7 +472,7 @@ handle (request& rq, response& rs) } } - bool archived (package_db_->load<brep::tenant> (tenant)->archived); + shared_ptr<brep::tenant> tn (package_db_->load<brep::tenant> (tenant)); t.commit (); @@ -504,7 +504,7 @@ handle (request& rq, response& rs) // vector<pair<string, version>> toolchains; - if (!archived) + if (!tn->archived) { using query = query<toolchain>; @@ -568,8 +568,12 @@ handle (request& rq, response& rs) b.toolchain_version.string ()) << TR_VALUE ("config", b.configuration + " / " + b.target.string ()) - << TR_VALUE ("timestamp", ts) - << TR_BUILD_RESULT (b, host, root) + << TR_VALUE ("timestamp", ts); + + if (b.interactive) // Note: can only be present for the building state. + s << TR_VALUE ("login", *b.interactive); + + s << TR_BUILD_RESULT (b, host, root) << ~TBODY << ~TABLE; @@ -606,22 +610,27 @@ handle (request& rq, response& rs) << ~TABLE; } - // Print the package build exclusions that belong to the 'default' class. + // Print the package build exclusions that belong to the 'default' class, + // unless the package is built interactively (normally for a single + // configuration). // - for (const auto& c: *build_conf_) + if (!tn->interactive) { - string reason; - if (belongs (c, "default") && exclude (c, &reason)) + for (const auto& c: *build_conf_) { - s << TABLE(CLASS="proplist build") - << TBODY - << TR_VALUE ("config", c.name + " / " + c.target.string ()) - << TR_VALUE ("result", - !reason.empty () - ? "excluded (" + reason + ')' - : "excluded") - << ~TBODY - << ~TABLE; + string reason; + if (belongs (c, "default") && exclude (c, &reason)) + { + s << TABLE(CLASS="proplist build") + << TBODY + << TR_VALUE ("config", c.name + " / " + c.target.string ()) + << TR_VALUE ("result", + !reason.empty () + ? "excluded (" + reason + ')' + : "excluded") + << ~TBODY + << ~TABLE; + } } } diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx index 65c7c5b..222b817 100644 --- a/mod/mod-packages.cxx +++ b/mod/mod-packages.cxx @@ -49,8 +49,8 @@ init (scanner& s) options_->root (dir_path ("/")); // Check that the database 'package' schema matches the current one. It's - // enough to perform the check in just a single module implementation (and we - // don't do in the dispatcher because it doesn't use the database). + // enough to perform the check in just a single module implementation (and + // we don't do in the dispatcher because it doesn't use the database). // // Note that the failure can be reported by each web server worker process. // While it could be tempting to move the check to the @@ -137,7 +137,7 @@ handle (request& rq, response& rs) << DIV(ID="content"); // If the tenant is empty then we are in the global view and will display - // packages from all the tenants. + // packages from all the public tenants. // optional<string> tn; if (!tenant.empty ()) diff --git a/mod/module.cli b/mod/module.cli index b59158a..3ba6ecd 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -1,6 +1,8 @@ // file : mod/options.cli -*- C++ -*- // license : MIT; see accompanying LICENSE file +include <regex>; + include <libbpkg/manifest.hxx>; // repository_location include <web/xhtml/fragment.hxx>; @@ -307,7 +309,7 @@ namespace brep edge. The value is treated as an XHTML5 fragment." } - vector<page_menu> menu; + vector<page_menu> menu { "<label=link>", "Web page menu. Each entry is displayed in the page header in the @@ -341,7 +343,7 @@ namespace brep The default is 500 (~ 80 characters * 6 lines)." } - uint16_t package-changes = 5000; + uint16_t package-changes = 5000 { "<len>", "Number of package changes characters to display in brief pages. The @@ -394,6 +396,18 @@ namespace brep "Time to wait before considering the expected task result lost. Must be specified in seconds. The default is 3 hours." } + + vector<pair<std::regex, string>> build-interactive-login + { + "</regex/replacement/>", + "Regular expressions for transforming the interactive build login + information, for example, into the actual command that can be used + by the user. The regular expressions are matched against the + \"<agent>\ <interactive-login>\" string containing the respective + task request manifest values. The first matching expression is used + for the transformation. If no expression matches, then the task + request is considered invalid, unless no expressions are specified." + } }; class build_result: build, package_db, build_db, handler @@ -837,6 +851,10 @@ namespace brep // string overrides; + // Interactive build execution breakpoint. + // + string interactive; + // Submission simulation outcome. // string simulate; diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx index dc21e97..422b353 100644 --- a/mod/types-parsers.cxx +++ b/mod/types-parsers.cxx @@ -3,11 +3,15 @@ #include <mod/types-parsers.hxx> +#include <sstream> + +#include <libbutl/regex.mxx> #include <libbutl/timestamp.mxx> // from_string() #include <mod/module-options.hxx> using namespace std; +using namespace butl; using namespace bpkg; using namespace web::xhtml; @@ -75,9 +79,9 @@ namespace brep string t ("1970-01-01 "); t += v; - x = butl::from_string (t.c_str (), - "%Y-%m-%d %H:%M", - false /* local */).time_since_epoch (); + x = from_string (t.c_str (), + "%Y-%m-%d %H:%M", + false /* local */).time_since_epoch (); return; } catch (const invalid_argument&) {} @@ -181,5 +185,37 @@ namespace brep throw invalid_value (o, v); } } + + // Parse the '/regex/replacement/' string into the regex/replacement pair. + // + void parser<pair<std::regex, string>>:: + parse (pair<std::regex, string>& x, bool& xs, scanner& s) + { + xs = true; + const char* o (s.next ()); + + if (!s.more ()) + throw missing_value (o); + + const char* v (s.next ()); + + try + { + x = regex_replace_parse (v); + } + catch (const invalid_argument& e) + { + throw invalid_value (o, v, e.what ()); + } + catch (const regex_error& e) + { + // Sanitize the description. + // + ostringstream os; + os << e; + + throw invalid_value (o, v, os.str ()); + } + } } } diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx index 6b851eb..05a7263 100644 --- a/mod/types-parsers.hxx +++ b/mod/types-parsers.hxx @@ -7,6 +7,8 @@ #ifndef MOD_TYPES_PARSERS_HXX #define MOD_TYPES_PARSERS_HXX +#include <regex> + #include <libbpkg/manifest.hxx> // repository_location #include <web/xhtml/fragment.hxx> @@ -75,6 +77,13 @@ namespace brep static void parse (web::xhtml::fragment&, bool&, scanner&); }; + + template <> + struct parser<pair<std::regex, string>> + { + static void + parse (pair<std::regex, string>&, bool&, scanner&); + }; } } |