aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--brep/handler/ci/ci-load.in15
-rw-r--r--doc/manual.cli10
-rw-r--r--libbrep/build-extra.sql3
-rw-r--r--libbrep/build-package.hxx17
-rw-r--r--libbrep/build.hxx2
-rw-r--r--libbrep/build.xml2
-rw-r--r--libbrep/common.hxx14
-rwxr-xr-xlibbrep/odb.sh9
-rw-r--r--libbrep/package.hxx42
-rw-r--r--libbrep/package.xml10
-rw-r--r--libbrep/version.hxx.in4
-rw-r--r--load/load.cli7
-rw-r--r--load/load.cxx183
-rw-r--r--manifest2
-rw-r--r--mod/buildfile1
-rw-r--r--mod/ci-common.cxx321
-rw-r--r--mod/ci-common.hxx49
-rw-r--r--mod/database-module.cxx10
-rw-r--r--mod/database-module.hxx5
-rw-r--r--mod/mod-build-force.cxx21
-rw-r--r--mod/mod-build-result.cxx28
-rw-r--r--mod/mod-build-task.cxx294
-rw-r--r--mod/mod-ci.cxx65
-rw-r--r--mod/mod-ci.hxx24
-rw-r--r--mod/module.cli4
-rw-r--r--mod/page.cxx2
-rw-r--r--mod/tenant-service.hxx19
-rw-r--r--repositories.manifest16
28 files changed, 972 insertions, 207 deletions
diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in
index 3f04ea8..b3c05f0 100644
--- a/brep/handler/ci/ci-load.in
+++ b/brep/handler/ci/ci-load.in
@@ -114,6 +114,7 @@ spec=
service_id=
service_type=
service_data=
+service_load=
while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
case "$n" in
@@ -133,6 +134,14 @@ while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
service-id) service_id="$v" ;;
service-type) service_type="$v" ;;
service-data) service_data="$v" ;;
+
+ service-action)
+ if [[ "$v" == "load" ]]; then
+ service_load=true
+ elif [[ "$v" != "start" ]]; then
+ error "unrecognized service action '$v'"
+ fi
+ ;;
esac
done
@@ -331,6 +340,12 @@ if [[ -n "$service_id" ]]; then
if [[ -n "$service_data" ]]; then
loader_options+=(--service-data "$service_data")
fi
+
+ # Load the pre-created tenant rather than create a new one.
+ #
+ if [[ "$service_load" ]]; then
+ loader_options+=(--existing-tenant)
+ fi
fi
run "$loader" "${loader_options[@]}" "$loadtab"
diff --git a/doc/manual.cli b/doc/manual.cli
index 2b96393..9b85ae6 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -344,6 +344,7 @@ timestamp: <date-time>
[service-id]: <string>
[service-type]: <string>
[service-data]: <string>
+[service-action]: <action>
\
The \c{package} value can be repeated multiple times. The \c{timestamp} value
@@ -356,7 +357,14 @@ required information via some custom protocol, and expect the CI service to
notify it about the progress. In this case the third-party service type as
well as optionally the third-party id and custom state data can be
communicated to the underlying CI handler program via the respective
-\c{service-*} manifest values.
+\c{service-*} manifest values. Also note that normally a third-party service
+has all the required information (repository URL, etc) available at the time
+of the CI task initiation, in which case the \c{start} value is specified for
+the \c{service-action} manifest value. If that's not the case, the CI task is
+only created at the time of the initiation without calling the CI handler
+program. In this case the CI handler is called later, when all the required
+information is asynchronously gathered by the service. In this case the
+\c{load} value is specified for the \c{service-action} manifest value.
\h#ci-overrides-manifest|CI Overrides Manifest|
diff --git a/libbrep/build-extra.sql b/libbrep/build-extra.sql
index 9e51a51..0c0f010 100644
--- a/libbrep/build-extra.sql
+++ b/libbrep/build-extra.sql
@@ -46,10 +46,13 @@ CREATE FOREIGN TABLE build_tenant (
id TEXT NOT NULL,
private BOOLEAN NOT NULL,
interactive TEXT NULL,
+ creation_timestamp BIGINT NOT NULL,
archived BOOLEAN NOT NULL,
service_id TEXT NULL,
service_type TEXT NULL,
service_data TEXT NULL,
+ unloaded_timestamp BIGINT NULL,
+ unloaded_notify_interval BIGINT NULL,
queued_timestamp BIGINT NULL,
toolchain_name TEXT OPTIONS (column_name 'build_toolchain_name') NULL,
toolchain_version_epoch INTEGER OPTIONS (column_name 'build_toolchain_version_epoch') NULL,
diff --git a/libbrep/build-package.hxx b/libbrep/build-package.hxx
index 9a9c277..13645eb 100644
--- a/libbrep/build-package.hxx
+++ b/libbrep/build-package.hxx
@@ -32,12 +32,25 @@ namespace brep
class build_tenant
{
public:
+ // Create tenant for an unloaded CI request (see the build_unloaded()
+ // tenant services notification for details).
+ //
+ build_tenant (string i, tenant_service s, timestamp t, duration n)
+ : id (move (i)),
+ creation_timestamp (timestamp::clock::now ()),
+ service (move (s)),
+ unloaded_timestamp (t),
+ unloaded_notify_interval (n) {}
+
string id;
- bool private_;
+ bool private_ = false;
optional<string> interactive;
- bool archived;
+ timestamp creation_timestamp;
+ bool archived = false;
optional<tenant_service> service;
+ optional<timestamp> unloaded_timestamp;
+ optional<duration> unloaded_notify_interval;
optional<timestamp> queued_timestamp;
optional<build_toolchain> toolchain;
diff --git a/libbrep/build.hxx b/libbrep/build.hxx
index af49c03..55fd42b 100644
--- a/libbrep/build.hxx
+++ b/libbrep/build.hxx
@@ -28,7 +28,7 @@
//
#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 20
-#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 27, closed)
+#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 28, closed)
// We have to keep these mappings at the global scope instead of inside the
// brep namespace because they need to be also effective in the bbot namespace
diff --git a/libbrep/build.xml b/libbrep/build.xml
index 1eba85a..90b4b4f 100644
--- a/libbrep/build.xml
+++ b/libbrep/build.xml
@@ -1,4 +1,6 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1">
+ <changeset version="28"/>
+
<changeset version="27"/>
<changeset version="26"/>
diff --git a/libbrep/common.hxx b/libbrep/common.hxx
index 1433c8c..4be9ce9 100644
--- a/libbrep/common.hxx
+++ b/libbrep/common.hxx
@@ -141,6 +141,20 @@ namespace brep
std::chrono::nanoseconds (*(?)))) \
: brep::optional_timestamp ())
+ #pragma db map type(duration) as(uint64_t) \
+ to(std::chrono::duration_cast<std::chrono::nanoseconds> (?).count ()) \
+ from(brep::duration (std::chrono::nanoseconds (?)))
+
+ using optional_duration = optional<duration>;
+
+ #pragma db map type(optional_duration) as(brep::optional_uint64) \
+ to((?) \
+ ? std::chrono::duration_cast<std::chrono::nanoseconds> (*(?)).count () \
+ : brep::optional_uint64 ()) \
+ from((?) \
+ ? brep::duration (std::chrono::nanoseconds (*(?))) \
+ : brep::optional_duration ())
+
// version
//
using bpkg::version;
diff --git a/libbrep/odb.sh b/libbrep/odb.sh
index 608ca41..7c62acb 100755
--- a/libbrep/odb.sh
+++ b/libbrep/odb.sh
@@ -16,6 +16,8 @@ if test -d ../.bdep; then
sed -r -ne 's#^(@[^ ]+ )?([^ ]+)/ .*default.*$#\2#p')"
fi
+ # Note: here we use libodb*, not libbutl-odb.
+ #
inc+=("-I$(echo "$cfg"/libodb-[1-9]*/)")
inc+=("-I$(echo "$cfg"/libodb-pgsql-[1-9]*/)")
@@ -33,11 +35,8 @@ sed -r -ne 's#^(@[^ ]+ )?([^ ]+)/ .*default.*$#\2#p')"
else
- inc+=("-I$HOME/work/odb/builds/default/libodb-pgsql-default")
- inc+=("-I$HOME/work/odb/libodb-pgsql")
-
- inc+=("-I$HOME/work/odb/builds/default/libodb-default")
- inc+=("-I$HOME/work/odb/libodb")
+ inc+=("-I$HOME/work/odb/odb/libodb-pgsql")
+ inc+=("-I$HOME/work/odb/odb/libodb")
inc+=(-I.. -I../../libbbot -I../../libbpkg -I../../libbutl)
diff --git a/libbrep/package.hxx b/libbrep/package.hxx
index 45008d4..61477a0 100644
--- a/libbrep/package.hxx
+++ b/libbrep/package.hxx
@@ -20,7 +20,7 @@
//
#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 27
-#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 33, closed)
+#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 34, closed)
namespace brep
{
@@ -251,19 +251,29 @@ namespace brep
// If this flag is true, then display the packages in the web interface
// only in the tenant view mode.
//
- bool private_; // Note: foreign-mapped in build.
+ bool private_; // Note: foreign-mapped in build.
// Interactive package build breakpoint.
//
// If present, then packages from this tenant will only be built
// interactively and only non-interactively otherwise.
//
- optional<string> interactive; // Note: foreign-mapped in build.
+ optional<string> interactive; // Note: foreign-mapped in build.
- timestamp creation_timestamp;
- bool archived = false; // Note: foreign-mapped in build.
+ timestamp creation_timestamp; // Note: foreign-mapped in build.
+ bool archived = false; // Note: foreign-mapped in build.
- optional<tenant_service> service; // Note: foreign-mapped in build.
+ optional<tenant_service> service; // Note: foreign-mapped in build.
+
+ // If the tenant is loaded, this value is absent. Otherwise it is the time
+ // of the last attempt to load the tenant (see the build_unloaded() tenant
+ // services notification for details).
+ //
+ optional<timestamp> unloaded_timestamp; // Note: foreign-mapped in build.
+
+ // The time interval between attempts to load the tenant, if unloaded.
+ //
+ optional<duration> unloaded_notify_interval; // Note: foreign-mapped in build.
// Note that due to the implementation complexity and performance
// considerations, the service notifications are not synchronized. This
@@ -284,7 +294,7 @@ namespace brep
// natural reasons (non-zero build task execution time, etc) and thus we
// just ignore them.
//
- optional<timestamp> queued_timestamp; // Note: foreign-mapped in build.
+ optional<timestamp> queued_timestamp; // Note: foreign-mapped in build.
// Note that after the package tenant is created but before the first
// build object is created, there is no easy way to produce a list of
@@ -318,6 +328,10 @@ namespace brep
#pragma db index member(service.id)
+ // Speed-up queries with ordering the result by unloaded_timestamp.
+ //
+ #pragma db member(unloaded_timestamp) index
+
private:
friend class odb::access;
tenant () = default;
@@ -427,6 +441,20 @@ namespace brep
repository (): tenant (id.tenant), canonical_name (id.canonical_name) {}
};
+ // Repositories count.
+ //
+ #pragma db view object(repository)
+ struct repository_count
+ {
+ size_t result;
+
+ operator size_t () const {return result;}
+
+ // Database mapping.
+ //
+ #pragma db member(result) column("count(" + repository::id.tenant + ")")
+ };
+
// The 'to' expression calls the PostgreSQL to_tsvector(weighted_text)
// function overload (package-extra.sql). Since we are only interested
// in "write-only" members of this type, make the 'from' expression
diff --git a/libbrep/package.xml b/libbrep/package.xml
index 96e93a7..f33119e 100644
--- a/libbrep/package.xml
+++ b/libbrep/package.xml
@@ -1,4 +1,14 @@
<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1">
+ <changeset version="34">
+ <alter-table name="tenant">
+ <add-column name="unloaded_timestamp" type="BIGINT" null="true"/>
+ <add-column name="unloaded_notify_interval" type="BIGINT" null="true"/>
+ <add-index name="tenant_unloaded_timestamp_i">
+ <column name="unloaded_timestamp"/>
+ </add-index>
+ </alter-table>
+ </changeset>
+
<changeset version="33">
<add-table name="public_key" kind="object">
<column name="tenant" type="TEXT" null="false"/>
diff --git a/libbrep/version.hxx.in b/libbrep/version.hxx.in
index 3ac3752..9adb5ab 100644
--- a/libbrep/version.hxx.in
+++ b/libbrep/version.hxx.in
@@ -49,11 +49,11 @@ $libbbot.check(LIBBBOT_VERSION, LIBBBOT_SNAPSHOT)$
#include <odb/version.hxx>
-$libodb.check(LIBODB_VERSION, LIBODB_SNAPSHOT)$
+$libodb.check(LIBODB_VERSION_FULL, LIBODB_SNAPSHOT)$
#include <odb/pgsql/version.hxx>
-$libodb_pgsql.check(LIBODB_PGSQL_VERSION, LIBODB_PGSQL_SNAPSHOT)$
+$libodb_pgsql.check(LIBODB_PGSQL_VERSION_FULL, LIBODB_PGSQL_SNAPSHOT)$
// For now these are the same.
//
diff --git a/load/load.cli b/load/load.cli
index 99d76f6..2061c26 100644
--- a/load/load.cli
+++ b/load/load.cli
@@ -72,6 +72,13 @@ class options
specified, then the single-tenant mode is assumed."
};
+ bool --existing-tenant
+ {
+ "Load the repository and package information into the already created empty
+ tenant rather than into the newly created one. Requires the \cb{--tenant}
+ option to be specified."
+ };
+
bool --private
{
"Display the tenant packages in the web interface only in the tenant view
diff --git a/load/load.cxx b/load/load.cxx
index 5b4692c..ba2da1c 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -437,6 +437,21 @@ load_packages (const options& lo,
if (p == nullptr)
{
+ // Apply the package manifest overrides.
+ //
+ if (!overrides.empty ())
+ try
+ {
+ pm.override (overrides, overrides_name);
+ }
+ catch (const manifest_parsing& e)
+ {
+ cerr << "error: unable to override " << pm.name << ' ' << pm.version
+ << " manifest: " << e << endl;
+
+ throw failed ();
+ }
+
// Convert the package manifest build configurations (contain public
// keys data) into the brep's build package configurations (contain
// public key object lazy pointers). Keep the bot key lists empty if
@@ -465,19 +480,6 @@ load_packages (const options& lo,
if (rp->internal)
{
- if (!overrides.empty ())
- try
- {
- pm.override (overrides, overrides_name);
- }
- catch (const manifest_parsing& e)
- {
- cerr << "error: unable to override " << p << " manifest: " << e
- << endl;
-
- throw failed ();
- }
-
// Create internal package object.
//
// Return nullopt if the text is in a file (can happen if the
@@ -1641,11 +1643,23 @@ try
//
const string& tnt (ops.tenant ());
- if (ops.tenant_specified () && tnt.empty ())
+ if (ops.tenant_specified ())
{
- cerr << "error: empty tenant" << endl
- << help_info << endl;
- throw failed ();
+ if (tnt.empty ())
+ {
+ cerr << "error: empty tenant" << endl
+ << help_info << endl;
+ throw failed ();
+ }
+ }
+ else
+ {
+ if (ops.existing_tenant ())
+ {
+ cerr << "error: --existing-tenant requires --tenant" << endl
+ << help_info << endl;
+ throw failed ();
+ }
}
// Verify the --service-* options.
@@ -1654,14 +1668,15 @@ try
{
if (!ops.tenant_specified ())
{
- cerr << "error: --service-id requires --tenant" << endl;
+ cerr << "error: --service-id requires --tenant" << endl
+ << help_info << endl;
throw failed ();
}
if (ops.service_type ().empty ())
{
- cerr << "error: --service-id requires --service-type"
- << endl;
+ cerr << "error: --service-id requires --service-type" << endl
+ << help_info << endl;
throw failed ();
}
}
@@ -1669,15 +1684,15 @@ try
{
if (ops.service_type_specified ())
{
- cerr << "error: --service-type requires --service-id"
- << endl;
+ cerr << "error: --service-type requires --service-id" << endl
+ << help_info << endl;
throw failed ();
}
if (ops.service_data_specified ())
{
- cerr << "error: --service-data requires --service-id"
- << endl;
+ cerr << "error: --service-data requires --service-id" << endl
+ << help_info << endl;
throw failed ();
}
}
@@ -1751,13 +1766,15 @@ try
if (ops.force () || changed (tnt, irs, db))
{
+ shared_ptr<tenant> t; // Not NULL in the --existing-tenant mode.
+
// Rebuild repositories persistent state from scratch.
//
// Note that in the single-tenant mode the tenant must be empty. In the
- // multi-tenant mode all tenants must be non-empty. So in the
- // single-tenant mode we erase all database objects (possibly from
- // multiple tenants). Otherwise, cleanup the specified and the empty
- // tenants only.
+ // multi-tenant mode all tenants, excluding the pre-created ones, must be
+ // non-empty. So in the single-tenant mode we erase all database objects
+ // (possibly from multiple tenants). Otherwise, cleanup the empty tenant
+ // and, unless in the --existing-tenant mode, the specified one.
//
if (tnt.empty ()) // Single-tenant mode.
{
@@ -1768,7 +1785,49 @@ try
}
else // Multi-tenant mode.
{
- cstrings ts ({tnt.c_str (), ""});
+ // NOTE: don't forget to update ci_start::create() if changing anything
+ // here.
+ //
+ cstrings ts ({""});
+
+ // In the --existing-tenant mode make sure that the specified tenant
+ // exists, is not archived, not marked as unloaded, and is
+ // empty. Otherwise (not in the --existing-tenant mode), remove this
+ // tenant.
+ //
+ if (ops.existing_tenant ())
+ {
+ t = db.find<tenant> (tnt);
+
+ if (t == nullptr)
+ {
+ cerr << "error: unable to find tenant " << tnt << endl;
+ throw failed ();
+ }
+
+ if (t->archived)
+ {
+ cerr << "error: tenant " << tnt << " is archived" << endl;
+ throw failed ();
+ }
+
+ if (t->unloaded_timestamp)
+ {
+ cerr << "error: tenant " << tnt << " is marked as unloaded" << endl;
+ throw failed ();
+ }
+
+ size_t n (db.query_value<repository_count> (
+ query<repository_count>::id.tenant == tnt));
+
+ if (n != 0)
+ {
+ cerr << "error: tenant " << tnt << " is not empty" << endl;
+ throw failed ();
+ }
+ }
+ else
+ ts.push_back (tnt.c_str ());
db.erase_query<package> (
query<package>::id.tenant.in_range (ts.begin (), ts.end ()));
@@ -1783,32 +1842,68 @@ try
query<tenant>::id.in_range (ts.begin (), ts.end ()));
}
- // Persist the tenant.
+ // Craft the tenant service object from the --service-* options.
//
- // Note that if the tenant service is specified and some tenant with the
- // same service id and type is already persisted, then we will end up with
- // the `object already persistent` error and terminate with the exit code
- // 1 (fatal error). We could potentially dedicate a special exit code for
- // such a case, so that the caller may recognize it and behave accordingly
- // (CI request handler can treat it as a client error rather than an
- // internal error, etc). However, let's first see if it ever becomes a
- // problem.
+ // In the --existing-tenant mode make sure that the specified service
+ // matches the service associated with the pre-created tenant and update
+ // the service data, if specified.
//
optional<tenant_service> service;
if (ops.service_id_specified ())
+ {
service = tenant_service (ops.service_id (),
ops.service_type (),
(ops.service_data_specified ()
? ops.service_data ()
: optional<string> ()));
- db.persist (tenant (tnt,
- ops.private_ (),
- (ops.interactive_specified ()
- ? ops.interactive ()
- : optional<string> ()),
- move (service)));
+ if (ops.existing_tenant ())
+ {
+ assert (t != nullptr);
+
+ if (!t->service)
+ {
+ cerr << "error: no service associated with tenant " << tnt << endl;
+ throw failed ();
+ }
+
+ if (t->service->id != service->id || t->service->type != service->type)
+ {
+ cerr << "error: associated service mismatch for tenant " << tnt << endl <<
+ " info: specified service: " << service->id << ' '
+ << service->type << endl <<
+ " info: associated service: " << t->service->id << ' '
+ << t->service->type << endl;
+ throw failed ();
+ }
+
+ if (service->data)
+ {
+ t->service->data = move (service->data);
+ db.update (t);
+ }
+ }
+ }
+
+ // Persist the tenant.
+ //
+ // Note that if the tenant service is specified and some tenant with the
+ // same service id and type is already persisted, then we will end up with
+ // the `object already persistent` error and terminate with the exit code
+ // 1 (fatal error). We could potentially dedicate a special exit code for
+ // such a case, so that the caller may recognize it and behave accordingly
+ // (CI request handler can treat it as a client error rather than an
+ // internal error, etc). However, let's first see if it ever becomes a
+ // problem.
+ //
+ if (!ops.existing_tenant ())
+ db.persist (tenant (tnt,
+ ops.private_ (),
+ (ops.interactive_specified ()
+ ? ops.interactive ()
+ : optional<string> ()),
+ move (service)));
// On the first pass over the internal repositories we load their
// certificate information and packages.
diff --git a/manifest b/manifest
index 63c866e..0dea967 100644
--- a/manifest
+++ b/manifest
@@ -37,7 +37,7 @@ depends: bpkg-util [0.17.0-a.0.1 0.17.0-a.1)
# are (currently) not packaged and need to come from the system package
# manager. It also requires rsync for tests.
#
-builds: none
+builds: none ; Requires unpackaged software.
debian-builds: sys
debian-build-exclude: linux_debian_12-** ; libapreq2 not available
diff --git a/mod/buildfile b/mod/buildfile
index c3895dc..2d6ef39 100644
--- a/mod/buildfile
+++ b/mod/buildfile
@@ -39,6 +39,7 @@ mod{brep}: {hxx ixx txx cxx}{* -module-options -{$libu_src}} \
# the debugging of the notifications machinery.
#
cxx.poptions += -DBREP_CI_TENANT_SERVICE
+#cxx.poptions += -DBREP_CI_TENANT_SERVICE_UNLOADED
libus{mod}: ../web/xhtml/libus{xhtml}
libue{mod}: ../web/xhtml/libue{xhtml}
diff --git a/mod/ci-common.cxx b/mod/ci-common.cxx
index cb61e66..7c41a7b 100644
--- a/mod/ci-common.cxx
+++ b/mod/ci-common.cxx
@@ -3,6 +3,9 @@
#include <mod/ci-common.hxx>
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+
#include <libbutl/uuid.hxx>
#include <libbutl/fdstream.hxx>
#include <libbutl/sendmail.hxx>
@@ -11,6 +14,9 @@
#include <libbutl/process-io.hxx> // operator<<(ostream, process_args)
#include <libbutl/manifest-serializer.hxx>
+#include <libbrep/build-package.hxx>
+#include <libbrep/build-package-odb.hxx>
+
#include <mod/external-handler.hxx>
namespace brep
@@ -38,13 +44,16 @@ namespace brep
options_ = move (o);
}
- optional<ci_start::start_result> ci_start::
+ static optional<ci_start::start_result>
start (const basic_mark& error,
const basic_mark& warn,
const basic_mark* trace,
+ const options::ci_start& ops,
+ string&& request_id,
optional<tenant_service>&& service,
+ bool service_load,
const repository_location& repository,
- const vector<package>& packages,
+ const vector<ci_start::package>& packages,
const optional<string>& client_ip,
const optional<string>& user_agent,
const optional<string>& interactive,
@@ -55,32 +64,15 @@ namespace brep
using serializer = manifest_serializer;
using serialization = manifest_serialization;
- assert (options_ != nullptr); // Shouldn't be called otherwise.
+ using result = ci_start::start_result;
// If the tenant service is specified, then its type may not be empty.
//
assert (!service || !service->type.empty ());
- // Generate the request id.
- //
- // Note that it will also be used as a CI result manifest reference,
- // unless the latter is provided by the external handler.
- //
- string request_id;
-
- try
- {
- request_id = uuid::generate ().string ();
- }
- catch (const system_error& e)
- {
- error << "unable to generate request id: " << e;
- return nullopt;
- }
-
// Create the submission data directory.
//
- dir_path dd (options_->ci_data () / dir_path (request_id));
+ dir_path dd (ops.ci_data () / dir_path (request_id));
try
{
@@ -103,10 +95,10 @@ namespace brep
//
auto client_error = [&request_id] (uint16_t status, string message)
{
- return start_result {status,
- move (message),
- request_id,
- vector<pair<string, string>> ()};
+ return result {status,
+ move (message),
+ request_id,
+ vector<pair<string, string>> ()};
};
// Serialize the CI request manifest to a stream. On the serialization
@@ -119,6 +111,7 @@ namespace brep
auto rqm = [&request_id,
&ts,
&service,
+ service_load,
&repository,
&packages,
&client_ip,
@@ -127,7 +120,7 @@ namespace brep
&simulate,
&custom_request,
&client_error] (ostream& os, bool long_lines = false)
- -> pair<bool, optional<start_result>>
+ -> pair<bool, optional<result>>
{
try
{
@@ -139,7 +132,7 @@ namespace brep
s.next ("id", request_id);
s.next ("repository", repository.string ());
- for (const package& p: packages)
+ for (const ci_start::package& p: packages)
{
if (!p.version)
s.next ("package", p.name.string ());
@@ -178,6 +171,8 @@ namespace brep
if (service->data)
s.next ("service-data", *service->data);
+
+ s.next ("service-action", service_load ? "load" : "start");
}
// Serialize the request custom parameters.
@@ -190,12 +185,12 @@ namespace brep
s.next (nv.first, nv.second);
s.next ("", ""); // End of manifest.
- return make_pair (true, optional<start_result> ());
+ return make_pair (true, optional<result> ());
}
catch (const serialization& e)
{
return make_pair (false,
- optional<start_result> (
+ optional<result> (
client_error (400,
string ("invalid parameter: ") +
e.what ())));
@@ -209,7 +204,7 @@ namespace brep
try
{
ofdstream os (rqf);
- pair<bool, optional<start_result>> r (rqm (os));
+ pair<bool, optional<result>> r (rqm (os));
os.close ();
if (!r.first)
@@ -228,7 +223,7 @@ namespace brep
//
auto ovm = [&overrides, &client_error] (ostream& os,
bool long_lines = false)
- -> pair<bool, optional<start_result>>
+ -> pair<bool, optional<result>>
{
try
{
@@ -240,12 +235,12 @@ namespace brep
s.next (nv.first, nv.second);
s.next ("", ""); // End of manifest.
- return make_pair (true, optional<start_result> ());
+ return make_pair (true, optional<result> ());
}
catch (const serialization& e)
{
return make_pair (false,
- optional<start_result> (
+ optional<result> (
client_error (
400,
string ("invalid manifest override: ") +
@@ -261,7 +256,7 @@ namespace brep
try
{
ofdstream os (ovf);
- pair<bool, optional<start_result>> r (ovm (os));
+ pair<bool, optional<result>> r (ovm (os));
os.close ();
if (!r.first)
@@ -305,16 +300,16 @@ namespace brep
// manifest from its stdout and parse it into the resulting manifest
// object. Otherwise, create implied CI result manifest.
//
- start_result sr;
+ result sr;
- if (options_->ci_handler_specified ())
+ if (ops.ci_handler_specified ())
{
using namespace external_handler;
- optional<result_manifest> r (run (options_->ci_handler (),
- options_->ci_handler_argument (),
+ optional<result_manifest> r (run (ops.ci_handler (),
+ ops.ci_handler_argument (),
dd,
- options_->ci_handler_timeout (),
+ ops.ci_handler_timeout (),
error,
warn,
trace));
@@ -358,7 +353,7 @@ namespace brep
{
try
{
- serialize_manifest (sr, os, long_lines);
+ ci_start::serialize_manifest (sr, os, long_lines);
return true;
}
catch (const serialization& e)
@@ -424,7 +419,7 @@ namespace brep
// assume that the web server error log is monitored and the email sending
// failure will be noticed.
//
- if (options_->ci_email_specified () && !simulate)
+ if (ops.ci_email_specified () && !simulate)
try
{
// Redirect the diagnostics to the web server error log.
@@ -435,14 +430,13 @@ namespace brep
*trace << process_args {args, n};
},
2 /* stderr */,
- options_->email (),
+ ops.email (),
"CI request submission (" + sr.reference + ')',
- {options_->ci_email ()});
+ {ops.ci_email ()});
// Write the CI request manifest.
//
- pair<bool, optional<start_result>> r (
- rqm (sm.out, true /* long_lines */));
+ pair<bool, optional<result>> r (rqm (sm.out, true /* long_lines */));
assert (r.first); // The serialization succeeded once, so can't fail now.
@@ -473,7 +467,55 @@ namespace brep
error << "sendmail error: " << e;
}
- return optional<start_result> (move (sr));
+ return optional<result> (move (sr));
+ }
+
+ optional<ci_start::start_result> ci_start::
+ start (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ optional<tenant_service>&& service,
+ const repository_location& repository,
+ const vector<package>& packages,
+ const optional<string>& client_ip,
+ const optional<string>& user_agent,
+ const optional<string>& interactive,
+ const optional<string>& simulate,
+ const vector<pair<string, string>>& custom_request,
+ const vector<pair<string, string>>& overrides) const
+ {
+ assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+ // Generate the request id.
+ //
+ // Note that it will also be used as a CI result manifest reference,
+ // unless the latter is provided by the external handler.
+ //
+ string request_id;
+
+ try
+ {
+ request_id = uuid::generate ().string ();
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to generate request id: " << e;
+ return nullopt;
+ }
+
+ return brep::start (error, warn, trace,
+ *options_,
+ move (request_id),
+ move (service),
+ false /* service_load */,
+ repository,
+ packages,
+ client_ip,
+ user_agent,
+ interactive,
+ simulate,
+ custom_request,
+ overrides);
}
void ci_start::
@@ -491,4 +533,191 @@ namespace brep
s.next ("", ""); // End of manifest.
}
+
+ optional<string> ci_start::
+ create (const basic_mark& error,
+ const basic_mark&,
+ const basic_mark* trace,
+ odb::core::database& db,
+ tenant_service&& service,
+ duration notify_interval,
+ duration notify_delay) const
+ {
+ using namespace odb::core;
+
+ // Generate the request id.
+ //
+ string request_id;
+
+ try
+ {
+ request_id = uuid::generate ().string ();
+ }
+ catch (const system_error& e)
+ {
+ error << "unable to generate request id: " << e;
+ return nullopt;
+ }
+
+ // Use the generated request id if the tenant service id is not specified.
+ //
+ if (service.id.empty ())
+ service.id = request_id;
+
+ build_tenant t (move (request_id),
+ move (service),
+ system_clock::now () - notify_interval + notify_delay,
+ notify_interval);
+ {
+ assert (!transaction::has_current ());
+
+ transaction tr (db.begin ());
+
+ // Note that in contrast to brep-load, we know that the tenant id is
+ // unique and thus we don't try to remove a tenant with such an id.
+ // There is also not much reason to assume that we may have switched
+ // from the single-tenant mode here and remove the respective tenant,
+ // unless we are in the tenant-service functionality development mode.
+ //
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ cstrings ts ({""});
+
+ db.erase_query<build_package> (
+ query<build_package>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+ db.erase_query<build_repository> (
+ query<build_repository>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+ db.erase_query<build_public_key> (
+ query<build_public_key>::id.tenant.in_range (ts.begin (), ts.end ()));
+
+ db.erase_query<build_tenant> (
+ query<build_tenant>::id.in_range (ts.begin (), ts.end ()));
+#endif
+
+ db.persist (t);
+
+ tr.commit ();
+ }
+
+ if (trace != nullptr)
+ *trace << "unloaded CI request " << t.id << " for service "
+ << t.service->id << ' ' << t.service->type << " is created";
+
+ return move (t.id);
+ }
+
+ optional<ci_start::start_result> ci_start::
+ load (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database& db,
+ tenant_service&& service,
+ const repository_location& repository) const
+ {
+ using namespace odb::core;
+
+ string request_id;
+ {
+ assert (!transaction::has_current ());
+
+ transaction tr (db.begin ());
+
+ using query = query<build_tenant>;
+
+ shared_ptr<build_tenant> t (
+ db.query_one<build_tenant> (query::service.id == service.id &&
+ query::service.type == service.type));
+
+ if (t == nullptr)
+ {
+ error << "unable to find tenant for service " << service.id << ' '
+ << service.type;
+
+ return nullopt;
+ }
+ else if (t->archived)
+ {
+ error << "tenant " << t->id << " for service " << service.id << ' '
+ << service.type << " is already archived";
+
+ return nullopt;
+ }
+ else if (!t->unloaded_timestamp)
+ {
+ error << "tenant " << t->id << " for service " << service.id << ' '
+ << service.type << " is already loaded";
+
+ return nullopt;
+ }
+
+ t->unloaded_timestamp = nullopt;
+ db.update (t);
+
+ tr.commit ();
+
+ request_id = move (t->id);
+ }
+
+ assert (options_ != nullptr); // Shouldn't be called otherwise.
+
+ optional<start_result> r (brep::start (error, warn, trace,
+ *options_,
+ move (request_id),
+ move (service),
+ true /* service_load */,
+ repository,
+ {} /* packages */,
+ nullopt /* client_ip */,
+ nullopt /* user_agent */,
+ nullopt /* interactive */,
+ nullopt /* simulate */,
+ {} /* custom_request */,
+ {} /* overrides */));
+
+ // Note: on error (r == nullopt) the diagnostics is already issued.
+ //
+ if (trace != nullptr && r)
+ *trace << "CI request for '" << repository << "' is "
+ << (r->status != 200 ? "not " : "") << "loaded: "
+ << r->message << " (reference: " << r->reference << ')';
+
+ return r;
+ }
+
+ optional<tenant_service> ci_start::
+ cancel (const basic_mark&,
+ const basic_mark&,
+ const basic_mark* trace,
+ odb::core::database& db,
+ const string& type,
+ const string& id) const
+ {
+ using namespace odb::core;
+
+ assert (!transaction::has_current ());
+
+ transaction tr (db.begin ());
+
+ using query = query<build_tenant>;
+
+ shared_ptr<build_tenant> t (
+ db.query_one<build_tenant> (query::service.id == id &&
+ query::service.type == type));
+ if (t == nullptr)
+ return nullopt;
+
+ optional<tenant_service> r (move (t->service));
+ t->service = nullopt;
+ t->archived = true;
+ db.update (t);
+
+ tr.commit ();
+
+ if (trace != nullptr)
+ *trace << "CI request " << t->id << " for service " << id << ' ' << type
+ << " is canceled";
+
+ return r;
+ }
}
diff --git a/mod/ci-common.hxx b/mod/ci-common.hxx
index 6f62c4b..8efeb26 100644
--- a/mod/ci-common.hxx
+++ b/mod/ci-common.hxx
@@ -36,6 +36,7 @@ namespace brep
package_name name;
optional<brep::version> version;
};
+
// Note that the inability to generate the reference is an internal
// error. Thus, it is not optional.
//
@@ -62,7 +63,53 @@ namespace brep
const optional<string>& interactive = nullopt,
const optional<string>& simulate = nullopt,
const vector<pair<string, string>>& custom_request = {},
- const vector<pair<string, string>>& overrides = {});
+ const vector<pair<string, string>>& overrides = {}) const;
+
+ // Create an unloaded CI request returning start_result::reference on
+ // success and nullopt on an internal error. Such a request is not started
+ // until loaded with the load() function below. Configure the time
+ // interval between the build_unloaded() notifications for the being
+ // created tenant and set the initial delay for the first notification.
+ // See also the build_unloaded() tenant services notification.
+ //
+ // Note: should be called out of the database transaction.
+ //
+ optional<string>
+ create (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database&,
+ tenant_service&&,
+ duration notify_interval,
+ duration notify_delay) const;
+
+ // Load (and start) previously created (as unloaded) CI request. Similarly
+ // to the start() function, return nullopt on an internal error.
+ //
+ // Note that tenant_service::id is used to identify the CI request tenant.
+ //
+ // Note: should be called out of the database transaction.
+ //
+ optional<start_result>
+ load (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database&,
+ tenant_service&&,
+ const repository_location& repository) const;
+
+ // Cancel previously created or started CI request. Return the service
+ // state or nullopt if there is no tenant for such a type/id pair.
+ //
+ // Note: should be called out of the database transaction.
+ //
+ optional<tenant_service>
+ cancel (const basic_mark& error,
+ const basic_mark& warn,
+ const basic_mark* trace,
+ odb::core::database&,
+ const string& type,
+ const string& id) const;
// Helpers.
//
diff --git a/mod/database-module.cxx b/mod/database-module.cxx
index 07babc6..bbb3e59 100644
--- a/mod/database-module.cxx
+++ b/mod/database-module.cxx
@@ -76,7 +76,7 @@ namespace brep
throw;
}
- void database_module::
+ optional<string> database_module::
update_tenant_service_state (
const connection_ptr& conn,
const string& tid,
@@ -88,6 +88,8 @@ namespace brep
//
assert (build_db_ != nullptr);
+ optional<string> r;
+
for (size_t retry (retry_);; )
{
try
@@ -104,6 +106,8 @@ namespace brep
{
s.data = move (*data);
build_db_->update (t);
+
+ r = move (s.data);
}
}
@@ -121,7 +125,11 @@ namespace brep
HANDLER_DIAG;
l1 ([&]{trace << e << "; " << retry + 1 << " tenant service "
<< "state update retries left";});
+
+ r = nullopt; // Prepare for the next iteration.
}
}
+
+ return r;
}
}
diff --git a/mod/database-module.hxx b/mod/database-module.hxx
index 910cb35..298afbf 100644
--- a/mod/database-module.hxx
+++ b/mod/database-module.hxx
@@ -57,7 +57,8 @@ namespace brep
// Update the tenant-associated service state if the specified
// notification callback-returned function (expected to be not NULL)
- // returns the new state data.
+ // returns the new state data. Return the service state data, if updated,
+ // and nullopt otherwise.
//
// Specifically, start the database transaction, query the service state,
// and call the callback-returned function on this state. If this call
@@ -65,7 +66,7 @@ namespace brep
// state with this data and persist the change. Repeat all the above steps
// on the recoverable database failures (deadlocks, etc).
//
- void
+ optional<string>
update_tenant_service_state (
const odb::core::connection_ptr&,
const string& tid,
diff --git a/mod/mod-build-force.cxx b/mod/mod-build-force.cxx
index bdae356..168a835 100644
--- a/mod/mod-build-force.cxx
+++ b/mod/mod-build-force.cxx
@@ -192,7 +192,14 @@ handle (request& rq, response& rs)
optional<pair<tenant_service, shared_ptr<build>>> tss;
tenant_service_build_queued::build_queued_hints qhs;
+ // Acquire the database connection for the subsequent transactions.
+ //
+ // Note that we will release it prior to any potentially time-consuming
+ // operations (such as HTTP requests) and re-acquire it again afterwards,
+ // if required.
+ //
connection_ptr conn (build_db_->connection ());
+
{
transaction t (conn->begin ());
@@ -297,14 +304,28 @@ handle (request& rq, response& rs)
vector<build> qbs;
qbs.push_back (move (b));
+ // Release the database connection since the build_queued() notification
+ // can potentially be time-consuming (e.g., it may perform an HTTP
+ // request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
build_state::building,
qhs,
log_writer_))
+ {
+ conn = build_db_->connection ();
update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
}
+ // Release the database connection prior to writing into the unbuffered
+ // response stream.
+ //
+ conn.reset ();
+
// We have all the data, so don't buffer the response content.
//
ostream& os (rs.content (200, "text/plain;charset=utf-8", false));
diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx
index ccce17f..64503aa 100644
--- a/mod/mod-build-result.cxx
+++ b/mod/mod-build-result.cxx
@@ -207,13 +207,20 @@ handle (request& rq, response&)
optional<pair<tenant_service, shared_ptr<build>>> tss;
tenant_service_build_queued::build_queued_hints qhs;
+ // Acquire the database connection for the subsequent transactions.
+ //
+ // Note that we will release it prior to any potentially time-consuming
+ // operations (such as HTTP requests) and re-acquire it again afterwards,
+ // if required.
+ //
+ connection_ptr conn (build_db_->connection ());
+
// Note that if the session authentication fails (probably due to the
// authentication settings change), then we log this case with the warning
// severity and respond with the 200 HTTP code as if the challenge is
// valid. The thinking is that we shouldn't alarm a law-abaiding agent and
// shouldn't provide any information to a malicious one.
//
- connection_ptr conn (build_db_->connection ());
{
transaction t (conn->begin ());
@@ -518,12 +525,20 @@ handle (request& rq, response&)
vector<build> qbs;
qbs.push_back (move (*tss->second));
+ // Release the database connection since build_queued() notification can
+ // potentially be time-consuming (e.g., it may perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
build_state::building,
qhs,
log_writer_))
+ {
+ conn = build_db_->connection ();
update_tenant_service_state (conn, qbs.back ().tenant, f);
+ }
}
// If a third-party service needs to be notified about the built package,
@@ -537,8 +552,16 @@ handle (request& rq, response&)
const tenant_service& ss (tss->first);
const build& b (*tss->second);
+ // Release the database connection since build_built() notification can
+ // potentially be time-consuming (e.g., it may perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsb->build_built (ss, b, log_writer_))
+ {
+ conn = build_db_->connection ();
update_tenant_service_state (conn, b.tenant, f);
+ }
}
if (bld != nullptr)
@@ -549,6 +572,9 @@ handle (request& rq, response&)
if (!build_notify)
(cfg->email ? cfg->email : pkg->build_email) = email ();
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
send_notification_email (*options_,
conn,
*bld,
diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx
index e0aad4b..6be77f6 100644
--- a/mod/mod-build-task.cxx
+++ b/mod/mod-build-task.cxx
@@ -44,6 +44,28 @@ using namespace odb::core;
static thread_local mt19937 rand_gen (random_device {} ());
+// The challenge (nonce) is randomly generated for every build task if brep is
+// configured to authenticate bbot agents.
+//
+// Nonce generator must guarantee a probabilistically insignificant chance
+// of repeating a previously generated value. The common approach is to use
+// counters or random number generators (alone or in combination), that
+// produce values of the sufficient length. 64-bit non-repeating and
+// 512-bit random numbers are considered to be more than sufficient for
+// most practical purposes.
+//
+// We will produce the challenge as the sha256sum of the 512-bit random
+// number and the 64-bit current timestamp combination. The latter is
+// not really a non-repeating counter and can't be used alone. However
+// adding it is a good and cheap uniqueness improvement.
+//
+// Note that since generating a challenge is not exactly cheap/fast, we will
+// generate it in advance for every task request, out of the database
+// transaction, and will cache it if it turns out that it wasn't used (no
+// package configuration to (re-)build, etc).
+//
+static thread_local optional<string> challenge;
+
// Generate a random number in the specified range (max value is included).
//
static inline size_t
@@ -377,6 +399,79 @@ handle (request& rq, response& rs)
}
}
+ // Acquire the database connection for the subsequent transactions.
+ //
+ // Note that we will release it prior to any potentially time-consuming
+ // operations (such as HTTP requests) and re-acquire it again afterwards,
+ // if required.
+ //
+ connection_ptr conn (build_db_->connection ());
+
+ // Perform some housekeeping first.
+ //
+ // Notify a tenant-associated third-party service about the unloaded CI
+ // request, if present.
+ //
+ {
+ const tenant_service_build_unloaded* tsu (nullptr);
+
+ transaction tr (conn->begin ());
+
+ using query = query<build_tenant>;
+
+ // Pick the unloaded tenant with the earliest loaded timestamp, skipping
+ // those which were already picked recently.
+ //
+ shared_ptr<build_tenant> t (
+ build_db_->query_one<build_tenant> (
+ (!query::archived &&
+ query::unloaded_timestamp.is_not_null () &&
+ (query::unloaded_timestamp +
+ "<= EXTRACT (EPOCH FROM NOW()) * 1000000000 - " +
+ query::unloaded_notify_interval)) +
+ "ORDER BY" + query::unloaded_timestamp +
+ "LIMIT 1"));
+
+ if (t != nullptr && t->service)
+ {
+ auto i (tenant_service_map_.find (t->service->type));
+
+ if (i != tenant_service_map_.end ())
+ {
+ tsu = dynamic_cast<const tenant_service_build_unloaded*> (
+ i->second.get ());
+
+ if (tsu != nullptr)
+ {
+ // If we ought to call the
+ // tenant_service_build_unloaded::build_unloaded() callback, then
+ // set the package tenant's loaded timestamp to the current time to
+ // prevent the notifications race.
+ //
+ t->unloaded_timestamp = system_clock::now ();
+ build_db_->update (t);
+ }
+ }
+ }
+
+ tr.commit ();
+
+ if (tsu != nullptr)
+ {
+ // Release the database connection since the build_unloaded()
+ // notification can potentially be time-consuming (e.g., it may perform
+ // an HTTP request).
+ //
+ conn.reset ();
+
+ if (auto f = tsu->build_unloaded (move (*t->service), log_writer_))
+ {
+ conn = build_db_->connection ();
+ update_tenant_service_state (conn, t->id, f);
+ }
+ }
+ }
+
// Go through package build configurations until we find one that has no
// build target configuration present in the database, or is in the building
// state but expired (collectively called unbuilt). If such a target
@@ -617,63 +712,6 @@ handle (request& rq, response& rs)
: optional<size_t> ()),
options_->build_hard_rebuild_timeout ()));
- // Return the challenge (nonce) if brep is configured to authenticate bbot
- // agents. Return nullopt otherwise.
- //
- // Nonce generator must guarantee a probabilistically insignificant chance
- // of repeating a previously generated value. The common approach is to use
- // counters or random number generators (alone or in combination), that
- // produce values of the sufficient length. 64-bit non-repeating and
- // 512-bit random numbers are considered to be more than sufficient for
- // most practical purposes.
- //
- // We will produce the challenge as the sha256sum of the 512-bit random
- // number and the 64-bit current timestamp combination. The latter is
- // not really a non-repeating counter and can't be used alone. However
- // adding it is a good and cheap uniqueness improvement.
- //
- auto challenge = [&agent_fp, &now, &fail, &trace, this] ()
- {
- optional<string> r;
-
- if (agent_fp)
- {
- try
- {
- auto print_args = [&trace, this] (const char* args[], size_t n)
- {
- l2 ([&]{trace << process_args {args, n};});
- };
-
- openssl os (print_args,
- nullfd, path ("-"), 2,
- process_env (options_->openssl (),
- options_->openssl_envvar ()),
- "rand",
- options_->openssl_option (), 64);
-
- vector<char> nonce (os.in.read_binary ());
- os.in.close ();
-
- if (!os.wait () || nonce.size () != 64)
- fail << "unable to generate nonce";
-
- uint64_t t (chrono::duration_cast<chrono::nanoseconds> (
- now.time_since_epoch ()).count ());
-
- sha256 cs (nonce.data (), nonce.size ());
- cs.append (&t, sizeof (t));
- r = cs.string ();
- }
- catch (const system_error& e)
- {
- fail << "unable to generate nonce: " << e;
- }
- }
-
- return r;
- };
-
// Convert butl::standard_version type to brep::version.
//
brep::version toolchain_version (tqm.toolchain_version.string ());
@@ -860,7 +898,10 @@ handle (request& rq, response& rs)
imode,
queued_expiration_ns));
- transaction t (build_db_->begin ());
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
+ transaction t (conn->begin ());
// If there are any non-archived interactive build tenants, then the
// chosen randomization approach doesn't really work since interactive
@@ -921,7 +962,8 @@ handle (request& rq, response& rs)
"OFFSET" + pkg_query::_ref (offset) +
"LIMIT" + pkg_query::_ref (limit);
- connection_ptr conn (build_db_->connection ());
+ if (conn == nullptr)
+ conn = build_db_->connection ();
prep_pkg_query pkg_prep_query (
conn->prepare_query<buildable_package> (
@@ -1373,6 +1415,39 @@ handle (request& rq, response& rs)
move (tms), move (bms), move (tests)};
};
+ if (agent_fp && !challenge)
+ try
+ {
+ auto print_args = [&trace, this] (const char* args[], size_t n)
+ {
+ l2 ([&]{trace << process_args {args, n};});
+ };
+
+ openssl os (print_args,
+ nullfd, path ("-"), 2,
+ process_env (options_->openssl (),
+ options_->openssl_envvar ()),
+ "rand",
+ options_->openssl_option (), 64);
+
+ vector<char> nonce (os.in.read_binary ());
+ os.in.close ();
+
+ if (!os.wait () || nonce.size () != 64)
+ fail << "unable to generate nonce";
+
+ uint64_t t (chrono::duration_cast<chrono::nanoseconds> (
+ now.time_since_epoch ()).count ());
+
+ sha256 cs (nonce.data (), nonce.size ());
+ cs.append (&t, sizeof (t));
+ challenge = cs.string ();
+ }
+ catch (const system_error& e)
+ {
+ fail << "unable to generate nonce: " << e;
+ }
+
// While at it, collect the aborted for various reasons builds
// (interactive builds in multiple configurations, builds with too many
// auxiliary machines, etc) to send the notification emails at the end
@@ -1758,7 +1833,6 @@ handle (request& rq, response& rs)
toolchain_version);
shared_ptr<build> b (build_db_->find<build> (bid));
- optional<string> cl (challenge ());
// Move the interactive build login information into the build
// object, if the package to be built interactively.
@@ -1783,13 +1857,15 @@ handle (request& rq, response& rs)
move (toolchain_version),
move (login),
move (agent_fp),
- move (cl),
+ move (challenge),
build_machine {
mh.name, move (mh.summary)},
move (aux->build_auxiliary_machines),
controller_checksum (*cm->config),
machine_checksum (*cm->machine));
+ challenge = nullopt;
+
build_db_->persist (b);
}
else
@@ -1825,7 +1901,10 @@ handle (request& rq, response& rs)
}
b->agent_fingerprint = move (agent_fp);
- b->agent_challenge = move (cl);
+
+ b->agent_challenge = move (challenge);
+ challenge = nullopt;
+
b->machine = build_machine {mh.name, move (mh.summary)};
// Mark the section as loaded, so auxiliary_machines are
@@ -1994,8 +2073,6 @@ handle (request& rq, response& rs)
sort (rebuilds.begin (), rebuilds.end (), cmp);
- optional<string> cl (challenge ());
-
// Pick the first build configuration from the ordered list.
//
// Note that the configurations and packages may not match the
@@ -2086,10 +2163,10 @@ handle (request& rq, response& rs)
unforced = (b->force == force_state::unforced);
- // Can't move from, as may need them on the next iteration.
- //
- b->agent_fingerprint = agent_fp;
- b->agent_challenge = cl;
+ b->agent_fingerprint = move (agent_fp);
+
+ b->agent_challenge = move (challenge);
+ challenge = nullopt;
const machine_header_manifest& mh (*cm.machine);
b->machine = build_machine {mh.name, mh.summary};
@@ -2182,9 +2259,12 @@ handle (request& rq, response& rs)
}
catch (const odb::deadlock&)
{
- // Just try with the next rebuild. But first, reset the task
- // manifest and the session that we may have prepared.
+ // Just try with the next rebuild. But first, restore the agent's
+ // fingerprint and challenge and reset the task manifest and the
+ // session that we may have prepared.
//
+ agent_fp = move (b->agent_fingerprint);
+ challenge = move (b->agent_challenge);
task_response = task_response_manifest ();
}
@@ -2205,7 +2285,7 @@ handle (request& rq, response& rs)
{
assert (tss); // Wouldn't be here otherwise.
- const tenant_service& ss (tss->first);
+ tenant_service& ss (tss->first);
// If the task build has no initial state (is just created), then
// temporarily move it into the list of the queued builds until the
@@ -2223,12 +2303,24 @@ handle (request& rq, response& rs)
if (!qbs.empty ())
{
+ // Release the database connection since the build_queued()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
nullopt /* initial_state */,
qhs,
log_writer_))
- update_tenant_service_state (conn, qbs.back ().tenant, f);
+ {
+ conn = build_db_->connection ();
+
+ if (optional<string> data =
+ update_tenant_service_state (conn, qbs.back ().tenant, f))
+ ss.data = move (data);
+ }
}
// Send the `queued` notification for the task build, unless it is
@@ -2243,12 +2335,24 @@ handle (request& rq, response& rs)
qbs.push_back (move (b));
restore_build = true;
+ // Release the database connection since the build_queued()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsq->build_queued (ss,
qbs,
initial_state,
qhs,
log_writer_))
- update_tenant_service_state (conn, qbs.back ().tenant, f);
+ {
+ conn = build_db_->connection ();
+
+ if (optional<string> data =
+ update_tenant_service_state (conn, qbs.back ().tenant, f))
+ ss.data = move (data);
+ }
}
if (restore_build)
@@ -2264,11 +2368,23 @@ handle (request& rq, response& rs)
{
assert (tss); // Wouldn't be here otherwise.
- const tenant_service& ss (tss->first);
+ tenant_service& ss (tss->first);
const build& b (*tss->second);
+ // Release the database connection since the build_building()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsb->build_building (ss, b, log_writer_))
- update_tenant_service_state (conn, b.tenant, f);
+ {
+ conn = build_db_->connection ();
+
+ if (optional<string> data =
+ update_tenant_service_state (conn, b.tenant, f))
+ ss.data = move (data);
+ }
}
// If the task manifest is prepared, then check that the number of the
@@ -2291,6 +2407,9 @@ handle (request& rq, response& rs)
const tenant_service_build_built* tsb (nullptr);
optional<pair<tenant_service, shared_ptr<build>>> tss;
{
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
transaction t (conn->begin ());
shared_ptr<build> b (build_db_->find<build> (task_build->id));
@@ -2377,17 +2496,33 @@ handle (request& rq, response& rs)
{
assert (tss); // Wouldn't be here otherwise.
- const tenant_service& ss (tss->first);
+ tenant_service& ss (tss->first);
const build& b (*tss->second);
+ // Release the database connection since the build_built()
+ // notification can potentially be time-consuming (e.g., it may
+ // perform an HTTP request).
+ //
+ conn.reset ();
+
if (auto f = tsb->build_built (ss, b, log_writer_))
- update_tenant_service_state (conn, b.tenant, f);
+ {
+ conn = build_db_->connection ();
+
+ if (optional<string> data =
+ update_tenant_service_state (conn, b.tenant, f))
+ ss.data = move (data);
+ }
}
}
// Send notification emails for all the aborted builds.
//
for (const aborted_build& ab: aborted_builds)
+ {
+ if (conn == nullptr)
+ conn = build_db_->connection ();
+
send_notification_email (*options_,
conn,
*ab.b,
@@ -2396,9 +2531,14 @@ handle (request& rq, response& rs)
ab.what,
error,
verb_ >= 2 ? &trace : nullptr);
+ }
}
}
+ // Release the database connection as soon as possible.
+ //
+ conn.reset ();
+
serialize_task_response_manifest ();
return true;
}
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
index 5974d45..0045002 100644
--- a/mod/mod-ci.cxx
+++ b/mod/mod-ci.cxx
@@ -36,7 +36,12 @@ ci (const ci& r, tenant_service_map& tsm)
#else
ci (const ci& r)
#endif
- : handler (r),
+ :
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
+ handler (r),
+ #else
+ database_module (r),
+#endif
ci_start (r),
options_ (r.initialized_ ? r.options_ : nullptr),
form_ (r.initialized_ || r.form_ == nullptr
@@ -100,6 +105,13 @@ init (scanner& s)
}
}
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ if (!options_->build_config_specified ())
+ fail << "package building functionality must be enabled";
+
+ database_module::init (*options_, options_->build_db_retry ());
+#endif
+
if (options_->root ().empty ())
options_->root (dir_path ("/"));
}
@@ -347,6 +359,7 @@ handle (request& rq, response& rs)
user_agent = h.value;
}
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
optional<start_result> r (start (error,
warn,
verb_ ? &trace : nullptr,
@@ -367,6 +380,25 @@ handle (request& rq, response& rs)
: optional<string> ()),
custom_request,
overrides));
+#else
+ assert (build_db_ != nullptr); // Wouldn't be here otherwise.
+
+ optional<start_result> r;
+
+ if (optional<string> ref = create (error,
+ warn,
+ verb_ ? &trace : nullptr,
+ *build_db_,
+ tenant_service ("", "ci", rl.string ()),
+ chrono::seconds (40),
+ chrono::seconds (10)))
+ {
+ string msg ("unloaded CI request is created: " +
+ options_->host () + tenant_dir (root, *ref).string ());
+
+ r = start_result {200, move (msg), move (*ref), {}};
+ }
+#endif
if (!r)
return respond_error (); // The diagnostics is already issued.
@@ -472,4 +504,35 @@ build_built (const tenant_service&,
return ts.data ? *ts.data + ", " + s : s;
};
}
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+function<optional<string> (const brep::tenant_service&)> brep::ci::
+build_unloaded (tenant_service&& ts,
+ const diag_epilogue& log_writer) const noexcept
+{
+ NOTIFICATION_DIAG (log_writer);
+
+ assert (ts.data); // Repository location.
+
+ try
+ {
+ repository_location rl (*ts.data);
+
+ if (!load (error, warn, verb_ ? &trace : nullptr,
+ *build_db_,
+ move (ts),
+ rl))
+ return nullptr; // The diagnostics is already issued.
+ }
+ catch (const invalid_argument& e)
+ {
+ error << "invalid repository location '" << *ts.data << "' stored for "
+ << "tenant service " << ts.id << ' ' << ts.type;
+
+ return nullptr;
+ }
+
+ return [] (const tenant_service& ts) {return "loaded " + *ts.data;};
+}
+#endif
#endif
diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx
index 1e2ee15..a83b9d3 100644
--- a/mod/mod-ci.hxx
+++ b/mod/mod-ci.hxx
@@ -17,18 +17,34 @@
#include <mod/ci-common.hxx>
+#if defined(BREP_CI_TENANT_SERVICE_UNLOADED) && !defined(BREP_CI_TENANT_SERVICE)
+# error BREP_CI_TENANT_SERVICE must be defined if BREP_CI_TENANT_SERVICE_UNLOADED is defined
+#endif
+
#ifdef BREP_CI_TENANT_SERVICE
# include <mod/tenant-service.hxx>
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+# include <mod/database-module.hxx>
+#endif
#endif
namespace brep
{
- class ci: public handler,
+ class ci:
+#ifndef BREP_CI_TENANT_SERVICE_UNLOADED
+ public handler,
+#else
+ public database_module,
+#endif
private ci_start
#ifdef BREP_CI_TENANT_SERVICE
, public tenant_service_build_queued,
public tenant_service_build_building,
public tenant_service_build_built
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ , tenant_service_build_unloaded
+#endif
#endif
{
public:
@@ -74,6 +90,12 @@ namespace brep
build_built (const tenant_service&,
const build&,
const diag_epilogue& log_writer) const noexcept override;
+
+#ifdef BREP_CI_TENANT_SERVICE_UNLOADED
+ virtual function<optional<string> (const tenant_service&)>
+ build_unloaded (tenant_service&&,
+ const diag_epilogue& log_writer) const noexcept override;
+#endif
#endif
private:
diff --git a/mod/module.cli b/mod/module.cli
index a107ffe..5f63930 100644
--- a/mod/module.cli
+++ b/mod/module.cli
@@ -800,7 +800,7 @@ namespace brep
{
};
- class ci: ci_start, page, repository_url, handler
+ class ci: ci_start, build, build_db, page, repository_url, handler
{
// Classic CI-specific options.
//
@@ -815,7 +815,7 @@ namespace brep
}
};
- class ci_github: ci_start, ci_cancel, build_db, handler
+ class ci_github: ci_start, ci_cancel, build, build_db, handler
{
// GitHub CI-specific options (e.g., request timeout when invoking
// GitHub APIs).
diff --git a/mod/page.cxx b/mod/page.cxx
index bc2e42d..177fb64 100644
--- a/mod/page.cxx
+++ b/mod/page.cxx
@@ -739,7 +739,7 @@ namespace brep
<< ~TR;
}
- // BUILD_RESULT
+ // TR_BUILD_RESULT
//
void TR_BUILD_RESULT::
operator() (serializer& s) const
diff --git a/mod/tenant-service.hxx b/mod/tenant-service.hxx
index 9205f76..b7f5c02 100644
--- a/mod/tenant-service.hxx
+++ b/mod/tenant-service.hxx
@@ -21,7 +21,8 @@ namespace brep
virtual ~tenant_service_base () = default;
};
- // Possible build notifications:
+ // Possible build notifications (see also the unloaded special notification
+ // below):
//
// queued
// building
@@ -121,6 +122,22 @@ namespace brep
const diag_epilogue& log_writer) const noexcept = 0;
};
+ // This notification is only made on unloaded CI requests created with the
+ // ci_start::create() call and until they are loaded with ci_start::load()
+ // or, alternatively, abandoned with ci_start::abandon().
+ //
+ // Note: make sure the implementation of this notification does not take
+ // too long (currently 40 seconds) to avoid nested notifications. Note
+ // also that the first notification is delayed (currently 10 seconds).
+ //
+ class tenant_service_build_unloaded: public virtual tenant_service_base
+ {
+ public:
+ virtual function<optional<string> (const tenant_service&)>
+ build_unloaded (tenant_service&&,
+ const diag_epilogue& log_writer) const noexcept = 0;
+ };
+
// Map of service type (tenant_service::type) to service.
//
using tenant_service_map = std::map<string, shared_ptr<tenant_service_base>>;
diff --git a/repositories.manifest b/repositories.manifest
index da9ee2b..e760afd 100644
--- a/repositories.manifest
+++ b/repositories.manifest
@@ -3,23 +3,23 @@ summary: build2 package repository web interface repository
:
role: prerequisite
-location: ../libbutl.git##HEAD
+location: ../libbutl.git#HEAD
:
role: prerequisite
-location: ../libbpkg.git##HEAD
+location: ../libbpkg.git#HEAD
:
role: prerequisite
-location: ../libbbot.git##HEAD
+location: ../libbbot.git#HEAD
:
role: prerequisite
-location: ../libbutl.bash.git##HEAD
+location: ../libbutl.bash.git#HEAD
:
role: prerequisite
-location: ../bpkg-util.git##HEAD
+location: ../bpkg-util.git#HEAD
:
role: prerequisite
@@ -35,11 +35,7 @@ location: https://git.build2.org/packaging/cmark-gfm/cmark-gfm.git##HEAD
:
role: prerequisite
-location: https://git.codesynthesis.com/odb/libodb.git##HEAD
-
-:
-role: prerequisite
-location: https://git.codesynthesis.com/odb/libodb-pgsql.git##HEAD
+location: https://git.codesynthesis.com/odb/odb.git##HEAD
:
role: prerequisite
generated by cgit v1.1 at 2024-05-27 18:57:21 +0000