aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2021-03-13 16:09:48 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2021-03-26 20:41:21 +0300
commit6ccee38f43493f8f6e87bab549e9ef952244f39a (patch)
tree3c75f102175fc6a566234e904a818dcd74029755
parentf7327a0b3cd6723cb289819bad1d664cfd5d6618 (diff)
Add support for interactive CI mode
-rw-r--r--brep/handler/ci/ci-load.in57
-rw-r--r--doc/manual.cli12
-rw-r--r--libbrep/build-extra.sql2
-rw-r--r--libbrep/build-package.hxx2
-rw-r--r--libbrep/build.cxx2
-rw-r--r--libbrep/build.hxx10
-rw-r--r--libbrep/build.xml6
-rw-r--r--libbrep/package-extra.sql39
-rw-r--r--libbrep/package.cxx4
-rw-r--r--libbrep/package.hxx17
-rw-r--r--libbrep/package.xml7
-rw-r--r--load/buildfile4
-rw-r--r--load/load.cli13
-rw-r--r--load/load.cxx11
-rw-r--r--mod/buildfile2
-rw-r--r--mod/mod-build-result.cxx4
-rw-r--r--mod/mod-build-task.cxx130
-rw-r--r--mod/mod-builds.cxx58
-rw-r--r--mod/mod-ci.cxx18
-rw-r--r--mod/mod-package-version-details.cxx43
-rw-r--r--mod/mod-packages.cxx6
-rw-r--r--mod/module.cli22
-rw-r--r--mod/types-parsers.cxx42
-rw-r--r--mod/types-parsers.hxx9
-rw-r--r--www/ci.xhtml4
25 files changed, 399 insertions, 125 deletions
diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in
index cd2b879..d568341 100644
--- a/brep/handler/ci/ci-load.in
+++ b/brep/handler/ci/ci-load.in
@@ -25,7 +25,10 @@ verbose= #true
fetch_timeout=60
trap "{ exit 1; }" ERR
-set -o errtrace # Trap ERR in functions.
+set -o errtrace # Trap ERR in functions.
+set -o pipefail # Fail if any pipeline command fails.
+shopt -s lastpipe # Execute last pipeline command in the current shell.
+shopt -s nullglob # Expand no-match globs to nothing rather than themselves.
@import brep/handler/handler@
@import brep/handler/ci/ci@
@@ -33,7 +36,7 @@ set -o errtrace # Trap ERR in functions.
# The handler's own options.
#
result_url=
-while [ $# -gt 0 ]; do
+while [[ "$#" -gt 0 ]]; do
case $1 in
--result-url)
shift
@@ -50,7 +53,7 @@ done
#
loader="$1"
-if [ -z "$loader" ]; then
+if [[ -z "$loader" ]]; then
error "$usage"
fi
@@ -60,7 +63,7 @@ shift
# options.
#
loader_options=()
-while [ $# -gt 1 ]; do
+while [[ "$#" -gt 1 ]]; do
loader_options+=("$1")
shift
done
@@ -69,11 +72,11 @@ done
#
data_dir="${1%/}"
-if [ -z "$data_dir" ]; then
+if [[ -z "$data_dir" ]]; then
error "$usage"
fi
-if [ ! -d "$data_dir" ]; then
+if [[ ! -d "$data_dir" ]]; then
error "'$data_dir' does not exist or is not a directory"
fi
@@ -84,8 +87,9 @@ reference="$(basename "$data_dir")"
#
manifest_parser_start "$data_dir/request.manifest"
-simulate=
repository=
+interactive=
+simulate=
# Package map. We first enter packages from the request manifest as keys and
# setting the values to true. Then we go through the repository package list
@@ -106,13 +110,14 @@ spec=
while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
case "$n" in
- simulate) simulate="$v" ;;
- repository) repository="$v" ;;
+ repository) repository="$v" ;;
+ interactive) interactive="$v" ;;
+ simulate) simulate="$v" ;;
package)
packages["$v"]=true
- if [ -n "$spec" ]; then
+ if [[ -n "$spec" ]]; then
spec="$spec,"
fi
spec="$spec$v"
@@ -122,22 +127,22 @@ done
manifest_parser_finish
-if [ -n "$spec" ]; then
+if [[ -n "$spec" ]]; then
spec="$spec@"
fi
spec="$spec$repository"
-if [ -z "$repository" ]; then
+if [[ -z "$repository" ]]; then
error "repository manifest value expected"
fi
-if [ -n "$simulate" -a "$simulate" != "success" ]; then
+if [[ -n "$simulate" && "$simulate" != "success" ]]; then
exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
fi
message_suffix=
-if [ -n "$result_url" ]; then
+if [[ -n "$result_url" ]]; then
message_suffix=": $result_url/@$reference" # Append the tenant id.
fi
@@ -146,7 +151,7 @@ fi
# Note that we can't assume a real repository URL is specified if simulating
# so trying to query the repository info is not a good idea.
#
-if [ -n "$simulate" ]; then
+if [[ -n "$simulate" ]]; then
run rm -r "$data_dir"
trace "CI request for '$spec' is simulated$message_suffix"
@@ -188,9 +193,9 @@ manifest_values=()
manifest_version=
more=true
-while [ "$more" ]; do
+while [[ "$more" ]]; do
- if [ -n "$manifest_version" ]; then
+ if [[ -n "$manifest_version" ]]; then
manifest_names=("")
manifest_values=("$manifest_version")
fi
@@ -237,8 +242,8 @@ while [ "$more" ]; do
packages_manifest_names+=("${manifest_names[@]}")
packages_manifest_values+=("${manifest_values[@]}")
- if [ -z "$display_name" ]; then
- if [ -n "$project" ]; then
+ if [[ -z "$display_name" ]]; then
+ if [[ -n "$project" ]]; then
display_name="$project"
else
display_name="$name"
@@ -252,7 +257,7 @@ manifest_parser_finish
# the repository.
#
for p in "${!packages[@]}"; do
- if [ "${packages[$p]}" ]; then
+ if [[ "${packages[$p]}" ]]; then
exit_with_manifest 422 "unknown package $p"
fi
done
@@ -260,7 +265,7 @@ done
# Verify that the repository is not empty. Failed that, the repository display
# name wouldn't be set.
#
-if [ -z "$display_name" ]; then
+if [[ -z "$display_name" ]]; then
exit_with_manifest 422 "no packages in repository"
fi
@@ -272,7 +277,7 @@ run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig"
#
manifest_serializer_start "$cache_dir/packages.manifest"
-for ((i=0; i <= ${#packages_manifest_names[@]}; ++i)); do
+for ((i=0; i <= "${#packages_manifest_names[@]}"; ++i)); do
manifest_serialize "${packages_manifest_names[$i]}" \
"${packages_manifest_values[$i]}"
done
@@ -286,7 +291,7 @@ run echo "$repository $display_name cache:cache" >"$loadtab"
# Apply overrides, if uploaded.
#
-if [ -f "$data_dir/overrides.manifest" ]; then
+if [[ -f "$data_dir/overrides.manifest" ]]; then
loader_options+=(--overrides-file "$data_dir/overrides.manifest")
fi
@@ -295,6 +300,12 @@ fi
#
loader_options+=(--force --shallow --tenant "$reference")
+# Build the packages interactively, if requested.
+#
+if [[ -n "$interactive" ]]; then
+ loader_options+=(--interactive "$interactive")
+fi
+
run "$loader" "${loader_options[@]}" "$loadtab"
# Remove the no longer needed CI request data directory.
diff --git a/doc/manual.cli b/doc/manual.cli
index 71a25a5..d0a1923 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -306,6 +306,15 @@ Check violations that are explicitly mentioned above are always reported with
the CI result manifest. Other errors (for example, internal server errors)
might be reported with unformatted text, including HTML.
+If the CI request contains the \c{interactive} parameter, then the CI service
+provides the execution environment login information for each test and stops
+them at the specified breakpoint.
+
+Pre-defined breakpoint ids are \c{error} and \c{warning}. The breakpoint id is
+included into the CI request manifest and the CI service must at least handle
+\c{error} but may recognize additional ids (build phase/command identifiers,
+etc).
+
If the CI request contains the \c{simulate} parameter, then the CI service
simulates the specified outcome of the CI process without actually performing
any externally visible actions (e.g., testing the package, publishing the
@@ -328,8 +337,9 @@ corresponding to the custom request parameters.
id: <request-id>
repository: <url>
[package]: <name>[/<version>]
-timestamp: <date-time>
+[interactive]: <breakpoint>
[simulate]: <outcome>
+timestamp: <date-time>
[client-ip]: <string>
[user-agent]: <string>
\
diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql
index ddc5961..a0cea97 100644
--- a/libbrep/build-extra.sql
+++ b/libbrep/build-extra.sql
@@ -22,6 +22,8 @@ DROP FOREIGN TABLE IF EXISTS build_tenant;
--
CREATE FOREIGN TABLE build_tenant (
id TEXT NOT NULL,
+ private BOOLEAN NOT NULL,
+ interactive TEXT NULL,
archived BOOLEAN NOT NULL)
SERVER package_server OPTIONS (table_name 'tenant');
diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx
index 09ec41d..0cfd609 100644
--- a/libbrep/build-package.hxx
+++ b/libbrep/build-package.hxx
@@ -30,6 +30,8 @@ namespace brep
public:
string id;
+ bool private_;
+ optional<string> interactive;
bool archived;
// Database mapping.
diff --git a/libbrep/build.cxx b/libbrep/build.cxx
index db5bda2..5f8cd71 100644
--- a/libbrep/build.cxx
+++ b/libbrep/build.cxx
@@ -59,6 +59,7 @@ namespace brep
version pvr,
string cfg,
string tnm, version tvr,
+ optional<string> inr,
optional<string> afp, optional<string> ach,
string mnm, string msm,
butl::target_triplet trg)
@@ -72,6 +73,7 @@ namespace brep
toolchain_name (id.toolchain_name),
toolchain_version (move (tvr)),
state (build_state::building),
+ interactive (move (inr)),
timestamp (timestamp_type::clock::now ()),
force (force_state::unforced),
agent_fingerprint (move (afp)), agent_challenge (move (ach)),
diff --git a/libbrep/build.hxx b/libbrep/build.hxx
index 380b17b..49105d1 100644
--- a/libbrep/build.hxx
+++ b/libbrep/build.hxx
@@ -25,7 +25,7 @@
//
#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 12
-#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 12, closed)
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 13, closed)
// We have to keep these mappings at the global scope instead of inside
// the brep namespace because they need to be also effective in the
@@ -178,13 +178,14 @@ namespace brep
using package_name_type = brep::package_name;
// Create the build object with the building state, non-existent status,
- // the timestamp set to now and the force state set to unforced.
+ // the timestamp set to now, and the force state set to unforced.
//
build (string tenant,
package_name_type,
version,
string configuration,
string toolchain_name, version toolchain_version,
+ optional<string> interactive,
optional<string> agent_fingerprint,
optional<string> agent_challenge,
string machine, string machine_summary,
@@ -201,6 +202,11 @@ namespace brep
build_state state;
+ // If present, the login information for the interactive build. May be
+ // present only in the building state.
+ //
+ optional<string> interactive;
+
// Time of the last state change (the creation time initially).
//
timestamp_type timestamp;
diff --git a/libbrep/build.xml b/libbrep/build.xml
index 3af7640..31bb4de 100644
--- a/libbrep/build.xml
+++ b/libbrep/build.xml
@@ -1,4 +1,10 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+ <changeset version="13">
+ <alter-table name="build">
+ <add-column name="interactive" type="TEXT" null="true"/>
+ </alter-table>
+ </changeset>
+
<model version="12">
<table name="build" kind="object">
<column name="package_tenant" type="TEXT" null="false"/>
diff --git a/libbrep/package-extra.sql b/libbrep/package-extra.sql
index fe936ff..5c04147 100644
--- a/libbrep/package-extra.sql
+++ b/libbrep/package-extra.sql
@@ -38,16 +38,17 @@ DROP TYPE IF EXISTS weighted_text CASCADE;
CREATE TYPE weighted_text AS (a TEXT, b TEXT, c TEXT, d TEXT);
-- Return the latest versions of matching a tenant internal packages as a set
--- of package rows. If tenant is NULL, then match all tenants.
+-- of package rows. If tenant is NULL, then match all public tenants.
--
CREATE FUNCTION
latest_packages(IN tenant TEXT)
RETURNS SETOF package AS $$
SELECT p1.*
- FROM package p1 LEFT JOIN package p2 ON (
+ FROM package p1
+ LEFT JOIN package p2 ON (
p1.internal_repository_canonical_name IS NOT NULL AND
- p1.tenant = p2.tenant AND
- p1.name = p2.name AND
+ p1.tenant = p2.tenant AND
+ p1.name = p2.name AND
p2.internal_repository_canonical_name IS NOT NULL AND
(p1.version_epoch < p2.version_epoch OR
p1.version_epoch = p2.version_epoch AND
@@ -56,8 +57,12 @@ RETURNS SETOF package AS $$
(p1.version_canonical_release < p2.version_canonical_release OR
p1.version_canonical_release = p2.version_canonical_release AND
p1.version_revision < p2.version_revision))))
+ JOIN tenant t ON (p1.tenant = t.id)
WHERE
- (latest_packages.tenant IS NULL OR p1.tenant = latest_packages.tenant) AND
+ CASE
+ WHEN latest_packages.tenant IS NULL THEN NOT t.private
+ ELSE p1.tenant = latest_packages.tenant
+ END AND
p1.internal_repository_canonical_name IS NOT NULL AND
p2.name IS NULL;
$$ LANGUAGE SQL STABLE;
@@ -83,7 +88,8 @@ $$ LANGUAGE SQL STABLE;
-- Search for the latest version of an internal packages matching the
-- specified search query and tenant. Return a set of rows containing the
-- package id and search rank. If query is NULL, then match all packages and
--- return 0 rank for all rows. If tenant is NULL, then match all tenants.
+-- return 0 rank for all rows. If tenant is NULL, then match all public
+-- tenants.
--
CREATE FUNCTION
search_latest_packages(IN query tsquery,
@@ -107,9 +113,9 @@ RETURNS SETOF record AS $$
$$ LANGUAGE SQL STABLE;
-- Search for packages matching the search query and tenant and having the
--- specified name. Return a set of rows containing the package id and search
+-- specified name. Return a set of rows containing the package id and search
-- rank. If query is NULL, then match all packages and return 0 rank for all
--- rows. If tenant is NULL, then match all tenants.
+-- rows. If tenant is NULL, then match all public tenants.
--
CREATE FUNCTION
search_packages(IN query tsquery,
@@ -121,19 +127,22 @@ search_packages(IN query tsquery,
OUT version_revision INTEGER,
OUT rank real)
RETURNS SETOF record AS $$
- SELECT tenant, name, version_epoch, version_canonical_upstream,
- version_canonical_release, version_revision,
+ SELECT p.tenant, p.name, p.version_epoch, p.version_canonical_upstream,
+ p.version_canonical_release, p.version_revision,
CASE
WHEN query IS NULL THEN 0
-- Weight mapping: D C B A
ELSE ts_rank_cd('{0.05, 0.2, 0.9, 1.0}', search_index, query)
END AS rank
- FROM package
+ FROM package p JOIN tenant t ON (p.tenant = t.id)
WHERE
- (search_packages.tenant IS NULL OR tenant = search_packages.tenant) AND
- name = search_packages.name AND
- internal_repository_canonical_name IS NOT NULL AND
- (query IS NULL OR search_index @@ query);
+ CASE
+ WHEN search_packages.tenant IS NULL THEN NOT t.private
+ ELSE p.tenant = search_packages.tenant
+ END AND
+ name = search_packages.name AND
+ internal_repository_canonical_name IS NOT NULL AND
+ (query IS NULL OR search_index @@ query);
$$ LANGUAGE SQL STABLE;
-- Parse weighted_text to tsvector.
diff --git a/libbrep/package.cxx b/libbrep/package.cxx
index 564fec7..65fa1ba 100644
--- a/libbrep/package.cxx
+++ b/libbrep/package.cxx
@@ -40,8 +40,10 @@ namespace brep
// tenant
//
tenant::
- tenant (string i)
+ tenant (string i, bool p, optional<string> r)
: id (move (i)),
+ private_ (p),
+ interactive (move (r)),
creation_timestamp (timestamp::clock::now ())
{
}
diff --git a/libbrep/package.hxx b/libbrep/package.hxx
index 33444a9..1619185 100644
--- a/libbrep/package.hxx
+++ b/libbrep/package.hxx
@@ -20,7 +20,7 @@
//
#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 19
-#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 19, closed)
+#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 20, closed)
namespace brep
{
@@ -226,16 +226,29 @@ namespace brep
// flag set to false.
//
explicit
- tenant (string id);
+ tenant (string id, bool private_, optional<string> interactive);
string id;
+ // If true, display the packages in the web interface only in the tenant
+ // view mode.
+ //
+ bool private_; // Note: foreign-mapped in build.
+
+ // Interactive package build breakpoint.
+ //
+ // If present, then packages from this tenant will only be built
+ // interactively and only non-interactively otherwise.
+ //
+ optional<string> interactive; // Note: foreign-mapped in build.
+
timestamp creation_timestamp;
bool archived = false; // Note: foreign-mapped in build.
// Database mapping.
//
#pragma db member(id) id
+ #pragma db member(private_) default(false) // @@ TMP
private:
friend class odb::access;
diff --git a/libbrep/package.xml b/libbrep/package.xml
index 454cdbc..01597c2 100644
--- a/libbrep/package.xml
+++ b/libbrep/package.xml
@@ -1,4 +1,11 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1">
+ <changeset version="20">
+ <alter-table name="tenant">
+ <add-column name="private" type="BOOLEAN" null="false" default="FALSE"/>
+ <add-column name="interactive" type="TEXT" null="true"/>
+ </alter-table>
+ </changeset>
+
<model version="19">
<table name="tenant" kind="object">
<column name="id" type="TEXT" null="false"/>
diff --git a/load/buildfile b/load/buildfile
index b55489f..4278f20 100644
--- a/load/buildfile
+++ b/load/buildfile
@@ -23,8 +23,8 @@ if $cli.configured
cli.options += --std c++11 -I $src_root --include-with-brackets \
--include-prefix load --guard-prefix LOAD --generate-specifier \
---cxx-prologue "#include <load/types-parsers.hxx>" --page-usage print_ \
---ansi-color --long-usage
+--generate-modifier --cxx-prologue "#include <load/types-parsers.hxx>" \
+--page-usage print_ --ansi-color --long-usage
# Include the generated cli files into the distribution and don't remove
# them when cleaning in src (so that clean results in a state identical to
diff --git a/load/load.cli b/load/load.cli
index 05bbb11..16b5f9f 100644
--- a/load/load.cli
+++ b/load/load.cli
@@ -64,6 +64,19 @@ class options
specified, then the single-tenant mode is assumed."
};
+ bool --private
+ {
+ "Display the tenant packages in the web interface only in the tenant view
+ mode."
+ };
+
+ std::string --interactive
+ {
+ "<bkp>",
+ "Build the tenant packages interactively, stopping builds at the specified
+ breakpoint. Implies \cb{--private}."
+ };
+
brep::path --overrides-file
{
"<file>",
diff --git a/load/load.cxx b/load/load.cxx
index 31230a7..0d53a0d 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -1474,6 +1474,11 @@ try
throw failed ();
}
+ // Note: the interactive tenant implies private.
+ //
+ if (ops.interactive_specified ())
+ ops.private_ (true);
+
// Load the description of all the internal repositories from the
// configuration file.
//
@@ -1511,7 +1516,11 @@ try
// Persist the tenant.
//
- db.persist (tenant (tnt));
+ db.persist (tenant (tnt,
+ ops.private_ (),
+ (ops.interactive_specified ()
+ ? ops.interactive ()
+ : optional<string> ())));
// On the first pass over the internal repositories we load their
// certificate information and packages.
diff --git a/mod/buildfile b/mod/buildfile
index 191d966..ff9cd60 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -50,7 +50,7 @@ if $cli.configured
cli.options += --std c++11 -I $src_root --include-with-brackets \
--include-prefix mod --guard-prefix MOD --generate-specifier \
--cxx-prologue "#include <mod/types-parsers.hxx>" \
---cli-namespace brep::cli --generate-file-scanner --option-length 41 \
+--cli-namespace brep::cli --generate-file-scanner --option-length 45 \
--generate-modifier --generate-description --option-prefix ""
# Include the generated cli files into the distribution and don't remove
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index 8a684fe..bec362a 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -398,6 +398,10 @@ handle (request& rq, response&)
b->status = rqm.result.status;
b->force = force_state::unforced;
+ // Cleanup the interactive build login information.
+ //
+ b->interactive = nullopt;
+
// Cleanup the authentication data.
//
b->agent_fingerprint = nullopt;
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index 04b2a36..ebf434a 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -4,12 +4,14 @@
#include <mod/mod-build-task.hxx>
#include <map>
+#include <regex>
#include <chrono>
#include <odb/database.hxx>
#include <odb/transaction.hxx>
#include <odb/schema-catalog.hxx>
+#include <libbutl/regex.mxx>
#include <libbutl/sha256.mxx>
#include <libbutl/utility.mxx> // compare_c_string
#include <libbutl/openssl.mxx>
@@ -195,11 +197,12 @@ handle (request& rq, response& rs)
{
vector<shared_ptr<build>> rebuilds;
- // Create the task response manifest. The package must have the internal
- // repository loaded.
+ // Create the task response manifest. Must be called inside the build db
+ // transaction.
//
auto task = [this] (shared_ptr<build>&& b,
shared_ptr<build_package>&& p,
+ shared_ptr<build_tenant>&& t,
const config_machine& cm) -> task_response_manifest
{
uint64_t ts (
@@ -218,7 +221,11 @@ handle (request& rq, response& rs)
tenant_dir (options_->root (), b->tenant).string () +
"?build-result");
- lazy_shared_ptr<build_repository> r (p->internal_repository);
+ assert (transaction::has_current ());
+
+ assert (p->internal ()); // The package is expected to be buildable.
+
+ lazy_shared_ptr<build_repository> r (p->internal_repository.load ());
strings fps;
if (r->certificate_fingerprint)
@@ -265,7 +272,8 @@ handle (request& rq, response& rs)
cm.config->target,
cm.config->environment,
cm.config->args,
- cm.config->warning_regexes);
+ cm.config->warning_regexes,
+ move (t->interactive));
return task_response_manifest (move (session),
move (b->agent_challenge),
@@ -430,12 +438,66 @@ handle (request& rq, response& rs)
pkg_query::build_repository::id.canonical_name.in_range (rp.begin (),
rp.end ());
+ // Transform (in-place) the interactive login information into the actual
+ // login command, if specified in the manifest and the transformation
+ // regexes are specified in the configuration.
+ //
+ if (tqm.interactive_login &&
+ options_->build_interactive_login_specified ())
+ {
+ optional<string> lc;
+ string l (tqm.agent + ' ' + *tqm.interactive_login);
+
+ // Use the first matching regex for the transformation.
+ //
+ for (const pair<regex, string>& rf: options_->build_interactive_login ())
+ {
+ pair<string, bool> r (regex_replace_match (l, rf.first, rf.second));
+
+ if (r.second)
+ {
+ lc = move (r.first);
+ break;
+ }
+ }
+
+ if (!lc)
+ throw invalid_request (400, "unable to match login info '" + l + "'");
+
+ tqm.interactive_login = move (lc);
+ }
+
+ // If the interactive mode if false or true, then filter out the
+ // respective packages. Otherwise, order them so that packages from the
+ // interactive build tenants appear first.
+ //
+ interactive_mode imode (tqm.effective_interactive_mode ());
+
+ switch (imode)
+ {
+ case interactive_mode::false_:
+ {
+ pq = pq && pkg_query::build_tenant::interactive.is_null ();
+ break;
+ }
+ case interactive_mode::true_:
+ {
+ pq = pq && pkg_query::build_tenant::interactive.is_not_null ();
+ break;
+ }
+ case interactive_mode::both: break; // See below.
+ }
+
// Specify the portion.
//
size_t offset (0);
- pq += "ORDER BY" +
- pkg_query::build_package::id.tenant + "," +
+ pq += "ORDER BY";
+
+ if (imode == interactive_mode::both)
+ pq += pkg_query::build_tenant::interactive + "NULLS LAST,";
+
+ pq += pkg_query::build_package::id.tenant + "," +
pkg_query::build_package::id.name +
order_by_version (pkg_query::build_package::id.version, false) +
"OFFSET" + pkg_query::_ref (offset) + "LIMIT 50";
@@ -512,8 +574,8 @@ handle (request& rq, response& rs)
// configurations that remained can be built. We will take the first
// one, if present.
//
- // Also save the built package configurations for which it's time to be
- // rebuilt.
+ // Also save the built package configurations for which it's time to
+ // be rebuilt.
//
config_machines configs (cfg_machines); // Make a copy for this pkg.
auto pkg_builds (bld_prep_query.execute ());
@@ -567,6 +629,16 @@ handle (request& rq, response& rs)
shared_ptr<build> b (build_db_->find<build> (bid));
optional<string> cl (challenge ());
+ shared_ptr<build_tenant> t (
+ build_db_->load<build_tenant> (bid.package.tenant));
+
+ // Move the interactive build login information into the build
+ // object, if the package to be built interactively.
+ //
+ optional<string> login (t->interactive
+ ? move (tqm.interactive_login)
+ : nullopt);
+
// If build configuration doesn't exist then create the new one
// and persist. Otherwise put it into the building state, refresh
// the timestamp and update.
@@ -579,6 +651,7 @@ handle (request& rq, response& rs)
move (bid.configuration),
move (bid.toolchain_name),
move (toolchain_version),
+ move (login),
move (agent_fp),
move (cl),
mh.name,
@@ -606,6 +679,7 @@ handle (request& rq, response& rs)
b->results.empty ());
b->state = build_state::building;
+ b->interactive = move (login);
// Switch the force state not to reissue the task after the
// forced rebuild timeout. Note that the result handler will
@@ -626,13 +700,7 @@ handle (request& rq, response& rs)
// Finally, prepare the task response manifest.
//
- // We iterate over buildable packages.
- //
- assert (p->internal ());
-
- p->internal_repository.load ();
-
- tsm = task (move (b), move (p), cm);
+ tsm = task (move (b), move (p), move (t), cm);
}
}
@@ -704,20 +772,40 @@ handle (request& rq, response& rs)
assert (i != cfg_machines.end ());
const config_machine& cm (i->second);
- // Rebuild the package if still present, is buildable and doesn't
- // exclude the configuration.
+ // Rebuild the package if still present, is buildable, doesn't
+ // exclude the configuration, and matches the request's
+ // interactive mode.
+ //
+ // Note that while change of the latter seems rather far fetched,
+ // let's check it for good measure.
//
shared_ptr<build_package> p (
build_db_->find<build_package> (b->id.package));
- if (p != nullptr &&
- p->internal () &&
+ shared_ptr<build_tenant> t (
+ p != nullptr
+ ? build_db_->load<build_tenant> (p->id.tenant)
+ : nullptr);
+
+ if (p != nullptr &&
+ p->buildable &&
+ (t->interactive.has_value () ==
+ (imode != interactive_mode::false_)) &&
!exclude (p->builds, p->constraints, *cm.config))
{
assert (b->status);
b->state = build_state::building;
+ // Save the interactive build login information into the build
+ // object, if the package to be built interactively.
+ //
+ // Can't move from, as may need it on the next iteration.
+ //
+ b->interactive = t->interactive
+ ? tqm.interactive_login
+ : nullopt;
+
// Can't move from, as may need them on the next iteration.
//
b->agent_fingerprint = agent_fp;
@@ -738,9 +826,7 @@ handle (request& rq, response& rs)
build_db_->update (b);
- p->internal_repository.load ();
-
- tsm = task (move (b), move (p), cm);
+ tsm = task (move (b), move (p), move (t), cm);
}
}
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index ab9e93e..5ffe6dd 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -133,6 +133,8 @@ match (const C qc, const string& pattern)
return qc + "SIMILAR TO" + query<T>::_val (transform (pattern));
}
+// If tenant is absent, then query builds from all the public tenants.
+//
template <typename T>
static inline query<T>
build_query (const brep::cstrings* configs,
@@ -143,18 +145,18 @@ build_query (const brep::cstrings* configs,
using namespace brep;
using query = query<T>;
using qb = typename query::build;
-
- query q (configs != nullptr
- ? qb::id.configuration.in_range (configs->begin (), configs->end ())
- : query (true));
+ using qt = typename query::build_tenant;
const auto& pid (qb::id.package);
- if (tenant)
- q = q && pid.tenant == *tenant;
+ query q (tenant ? pid.tenant == *tenant : !qt::private_);
if (archived)
- q = q && query::build_tenant::archived == *archived;
+ q = q && qt::archived == *archived;
+
+ if (configs != nullptr)
+ q = q && qb::id.configuration.in_range (configs->begin (),
+ configs->end ());
// Note that there is no error reported if the filter parameters parsing
// fails. Instead, it is considered that no package builds match such a
@@ -267,6 +269,8 @@ build_query (const brep::cstrings* configs,
return q;
}
+// If tenant is absent, then query packages from all the public tenants.
+//
template <typename T>
static inline query<T>
package_query (const brep::params::builds& params,
@@ -276,14 +280,12 @@ package_query (const brep::params::builds& params,
using namespace brep;
using query = query<T>;
using qp = typename query::build_package;
+ using qt = typename query::build_tenant;
- query q (true);
-
- if (tenant)
- q = q && qp::id.tenant == *tenant;
+ query q (tenant ? qp::id.tenant == *tenant : !qt::private_);
if (archived)
- q = q && query::build_tenant::archived == *archived;
+ q = q && qt::archived == *archived;
// Note that there is no error reported if the filter parameters parsing
// fails. Instead, it is considered that no packages match such a query.
@@ -383,7 +385,7 @@ handle (request& rq, response& rs)
<< DIV(ID="content");
// If the tenant is empty then we are in the global view and will display
- // builds from all the tenants.
+ // builds from all the public tenants.
//
optional<string> tn;
if (!tenant.empty ())
@@ -672,14 +674,18 @@ handle (request& rq, response& rs)
<< TR_VALUE ("config", b.configuration)
<< TR_VALUE ("machine", b.machine)
<< TR_VALUE ("target", b.target.string ())
- << TR_VALUE ("timestamp", ts)
- << TR_BUILD_RESULT (b, host, root);
+ << TR_VALUE ("timestamp", ts);
+
+ if (b.interactive) // Note: can only be present for the building state.
+ s << TR_VALUE ("login", *b.interactive);
+
+ s << TR_BUILD_RESULT (b, host, root);
// In the global view mode add the tenant builds link. Note that the
// global view (and the link) makes sense only in the multi-tenant mode.
//
if (!tn && !b.tenant.empty ())
- s << TR_TENANT (tenant_name, "builds", root, b.tenant);
+ s << TR_TENANT (tenant_name, "builds", root, b.tenant);
s << ~TBODY
<< ~TABLE;
@@ -809,15 +815,20 @@ handle (request& rq, response& rs)
const auto& bid (bld_query::build::id);
- bld_query bq (equal<package_build_count> (bid.package, id) &&
- bid.configuration == bld_query::_ref (config) &&
+ bld_query bq (
+ equal<package_build_count> (bid.package, id) &&
+ bid.configuration == bld_query::_ref (config) &&
// Note that the query already constrains configurations via the
- // configuration name and the tenant via the build package id.
+ // configuration name.
+ //
+ // Also note that while the query already constrains the tenant via
+ // the build package id, we still need to pass the tenant not to
+ // erroneously filter out the private tenants.
//
build_query<package_build_count> (nullptr /* configs */,
bld_params,
- nullopt /* tenant */,
+ tn,
false /* archived */));
prep_bld_query bld_prep_query (
@@ -925,11 +936,12 @@ handle (request& rq, response& rs)
bld_query bq (
equal<package_build> (bld_query::build::id.package, id) &&
- // Note that the query already constrains the tenant via the build
- // package id.
+ // Note that while the query already constrains the tenant via the build
+ // package id, we still need to pass the tenant not to erroneously
+ // filter out the private tenants.
//
build_query<package_build> (
- &conf_names, bld_params, nullopt /* tenant */, false /* archived */));
+ &conf_names, bld_params, tn, false /* archived */));
prep_bld_query bld_prep_query (
conn->prepare_query<package_build> ("mod-builds-build-query", bq));
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index d2da93f..4da72b6 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -376,15 +376,18 @@ handle (request& rq, response& rs)
s.next ("package", p);
}
+ if (params.interactive_specified ())
+ s.next ("interactive", params.interactive ());
+
+ if (!simulate.empty ())
+ s.next ("simulate", simulate);
+
s.next ("timestamp",
butl::to_string (ts,
"%Y-%m-%dT%H:%M:%SZ",
false /* special */,
false /* local */));
- if (!simulate.empty ())
- s.next ("simulate", simulate);
-
// Serialize the User-Agent HTTP header and the client IP address.
//
optional<string> ip;
@@ -412,10 +415,11 @@ handle (request& rq, response& rs)
{
const string& n (nv.name);
- if (n != "repository" &&
- n != "_" &&
- n != "package" &&
- n != "overrides" &&
+ if (n != "repository" &&
+ n != "_" &&
+ n != "package" &&
+ n != "overrides" &&
+ n != "interactive" &&
n != "simulate")
s.next (n, nv.value ? *nv.value : "");
}
diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx
index a7682ec..a23989e 100644
--- a/mod/mod-package-version-details.cxx
+++ b/mod/mod-package-version-details.cxx
@@ -472,7 +472,7 @@ handle (request& rq, response& rs)
}
}
- bool archived (package_db_->load<brep::tenant> (tenant)->archived);
+ shared_ptr<brep::tenant> tn (package_db_->load<brep::tenant> (tenant));
t.commit ();
@@ -504,7 +504,7 @@ handle (request& rq, response& rs)
//
vector<pair<string, version>> toolchains;
- if (!archived)
+ if (!tn->archived)
{
using query = query<toolchain>;
@@ -568,8 +568,12 @@ handle (request& rq, response& rs)
b.toolchain_version.string ())
<< TR_VALUE ("config",
b.configuration + " / " + b.target.string ())
- << TR_VALUE ("timestamp", ts)
- << TR_BUILD_RESULT (b, host, root)
+ << TR_VALUE ("timestamp", ts);
+
+ if (b.interactive) // Note: can only be present for the building state.
+ s << TR_VALUE ("login", *b.interactive);
+
+ s << TR_BUILD_RESULT (b, host, root)
<< ~TBODY
<< ~TABLE;
@@ -606,22 +610,27 @@ handle (request& rq, response& rs)
<< ~TABLE;
}
- // Print the package build exclusions that belong to the 'default' class.
+ // Print the package build exclusions that belong to the 'default' class,
+ // unless the package is built interactively (normally for a single
+ // configuration).
//
- for (const auto& c: *build_conf_)
+ if (!tn->interactive)
{
- string reason;
- if (belongs (c, "default") && exclude (c, &reason))
+ for (const auto& c: *build_conf_)
{
- s << TABLE(CLASS="proplist build")
- << TBODY
- << TR_VALUE ("config", c.name + " / " + c.target.string ())
- << TR_VALUE ("result",
- !reason.empty ()
- ? "excluded (" + reason + ')'
- : "excluded")
- << ~TBODY
- << ~TABLE;
+ string reason;
+ if (belongs (c, "default") && exclude (c, &reason))
+ {
+ s << TABLE(CLASS="proplist build")
+ << TBODY
+ << TR_VALUE ("config", c.name + " / " + c.target.string ())
+ << TR_VALUE ("result",
+ !reason.empty ()
+ ? "excluded (" + reason + ')'
+ : "excluded")
+ << ~TBODY
+ << ~TABLE;
+ }
}
}
diff --git a/mod/mod-packages.cxx b/mod/mod-packages.cxx
index 65c7c5b..222b817 100644
--- a/mod/mod-packages.cxx
+++ b/mod/mod-packages.cxx
@@ -49,8 +49,8 @@ init (scanner& s)
options_->root (dir_path ("/"));
// Check that the database 'package' schema matches the current one. It's
- // enough to perform the check in just a single module implementation (and we
- // don't do in the dispatcher because it doesn't use the database).
+ // enough to perform the check in just a single module implementation (and
+ // we don't do in the dispatcher because it doesn't use the database).
//
// Note that the failure can be reported by each web server worker process.
// While it could be tempting to move the check to the
@@ -137,7 +137,7 @@ handle (request& rq, response& rs)
<< DIV(ID="content");
// If the tenant is empty then we are in the global view and will display
- // packages from all the tenants.
+ // packages from all the public tenants.
//
optional<string> tn;
if (!tenant.empty ())
diff --git a/mod/module.cli b/mod/module.cli
index b59158a..3ba6ecd 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -1,6 +1,8 @@
// file : mod/options.cli -*- C++ -*-
// license : MIT; see accompanying LICENSE file
+include <regex>;
+
include <libbpkg/manifest.hxx>; // repository_location
include <web/xhtml/fragment.hxx>;
@@ -307,7 +309,7 @@ namespace brep
edge. The value is treated as an XHTML5 fragment."
}
- vector<page_menu> menu;
+ vector<page_menu> menu
{
"<label=link>",
"Web page menu. Each entry is displayed in the page header in the
@@ -341,7 +343,7 @@ namespace brep
The default is 500 (~ 80 characters * 6 lines)."
}
- uint16_t package-changes = 5000;
+ uint16_t package-changes = 5000
{
"<len>",
"Number of package changes characters to display in brief pages. The
@@ -394,6 +396,18 @@ namespace brep
"Time to wait before considering the expected task result lost. Must be
specified in seconds. The default is 3 hours."
}
+
+ vector<pair<std::regex, string>> build-interactive-login
+ {
+ "</regex/replacement/>",
+ "Regular expressions for transforming the interactive build login
+ information, for example, into the actual command that can be used
+ by the user. The regular expressions are matched against the
+ \"<agent>\ <interactive-login>\" string containing the respective
+ task request manifest values. The first matching expression is used
+ for the transformation. If no expression matches, then the task
+ request is considered invalid, unless no expressions are specified."
+ }
};
class build_result: build, package_db, build_db, handler
@@ -837,6 +851,10 @@ namespace brep
//
string overrides;
+ // Interactive build execution breakpoint.
+ //
+ string interactive;
+
// Submission simulation outcome.
//
string simulate;
diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx
index dc21e97..422b353 100644
--- a/mod/types-parsers.cxx
+++ b/mod/types-parsers.cxx
@@ -3,11 +3,15 @@
#include <mod/types-parsers.hxx>
+#include <sstream>
+
+#include <libbutl/regex.mxx>
#include <libbutl/timestamp.mxx> // from_string()
#include <mod/module-options.hxx>
using namespace std;
+using namespace butl;
using namespace bpkg;
using namespace web::xhtml;
@@ -75,9 +79,9 @@ namespace brep
string t ("1970-01-01 ");
t += v;
- x = butl::from_string (t.c_str (),
- "%Y-%m-%d %H:%M",
- false /* local */).time_since_epoch ();
+ x = from_string (t.c_str (),
+ "%Y-%m-%d %H:%M",
+ false /* local */).time_since_epoch ();
return;
}
catch (const invalid_argument&) {}
@@ -181,5 +185,37 @@ namespace brep
throw invalid_value (o, v);
}
}
+
+ // Parse the '/regex/replacement/' string into the regex/replacement pair.
+ //
+ void parser<pair<std::regex, string>>::
+ parse (pair<std::regex, string>& x, bool& xs, scanner& s)
+ {
+ xs = true;
+ const char* o (s.next ());
+
+ if (!s.more ())
+ throw missing_value (o);
+
+ const char* v (s.next ());
+
+ try
+ {
+ x = regex_replace_parse (v);
+ }
+ catch (const invalid_argument& e)
+ {
+ throw invalid_value (o, v, e.what ());
+ }
+ catch (const regex_error& e)
+ {
+ // Sanitize the description.
+ //
+ ostringstream os;
+ os << e;
+
+ throw invalid_value (o, v, os.str ());
+ }
+ }
}
}
diff --git a/mod/types-parsers.hxx b/mod/types-parsers.hxx
index 6b851eb..05a7263 100644
--- a/mod/types-parsers.hxx
+++ b/mod/types-parsers.hxx
@@ -7,6 +7,8 @@
#ifndef MOD_TYPES_PARSERS_HXX
#define MOD_TYPES_PARSERS_HXX
+#include <regex>
+
#include <libbpkg/manifest.hxx> // repository_location
#include <web/xhtml/fragment.hxx>
@@ -75,6 +77,13 @@ namespace brep
static void
parse (web::xhtml::fragment&, bool&, scanner&);
};
+
+ template <>
+ struct parser<pair<std::regex, string>>
+ {
+ static void
+ parse (pair<std::regex, string>&, bool&, scanner&);
+ };
}
}
diff --git a/www/ci.xhtml b/www/ci.xhtml
index 185f08b..573cca7 100644
--- a/www/ci.xhtml
+++ b/www/ci.xhtml
@@ -13,6 +13,10 @@
<th>package</th>
<td><input type="text" name="package"/></td>
</tr>
+ <tr>
+ <th>interactive</th>
+ <td><input type="text" name="interactive"/></td>
+ </tr>
</tbody>
</table>
<table class="form-table">