diff options
47 files changed, 2388 insertions, 525 deletions
@@ -91,37 +91,46 @@ bpkg install brep $ cd .. # Back to brep home. -4. Create PostgreSQL User and Database +4. Create PostgreSQL User and Databases $ sudo sudo -u postgres psql # Note: double sudo is not a mistake. -CREATE DATABASE brep TEMPLATE template0 ENCODING 'UTF8' +CREATE DATABASE brep_package TEMPLATE template0 ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8'; +CREATE DATABASE brep_build TEMPLATE template0 ENCODING 'UTF8' LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8'; CREATE USER brep; -GRANT ALL PRIVILEGES ON DATABASE brep TO brep; +GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO brep; CREATE USER "www-data" INHERIT IN ROLE brep; Exit psql (^D), then make sure the logins work: -$ psql +$ psql -d brep_package +^D +$ psql -d brep_build ^D -$ sudo sudo -u www-data psql -d brep +$ sudo sudo -u www-data psql -d brep_package +^D +$ sudo sudo -u www-data psql -d brep_build ^D To troubleshoot, see PostgreSQL logs. -5. Create Database Schema and Load Repositories +5. Create Database Schemes and Load Repositories $ mkdir config $ edit config/loadtab # Loader configuration, see brep-load(1). -$ install/bin/brep-migrate +$ install/bin/brep-migrate package $ install/bin/brep-load config/loadtab +$ install/bin/brep-migrate build + To verify: -$ psql -c 'SELECT name, summary FROM repository' +$ psql -d brep_package -c 'SELECT name, summary FROM repository' +$ psql -d brep_build -c 'SELECT package_name FROM build' # Empty row set. 6. Setup Apache2 Module @@ -143,6 +152,18 @@ can also find this fragment in install/share/brep/etc/brep-apache2.conf): LoadModule brep_module /home/brep/install/libexec/brep/mod_brep.so </IfModule> + # Repository email. This email is used for the From: header in emails + # send by brep (for example, build failure notifications). + # + brep-email admin@example.org + + # Repository host. It specifies the scheme and the host address (but + # not the root path; see brep-root below) that will be used whenever + # brep needs to construct an absolute URL to one of its locations (for + # example, a link to a build log that is being send via email). + # + brep-host https://example.org + # Repository root. This is the part of the URL between the host name # and the start of the repository. For example, root value /pkg means # the repository URL is http://example.org/pkg/. Specify / to use the @@ -192,13 +213,15 @@ can also find this fragment in install/share/brep/etc/brep-apache2.conf): # Require all granted #</Directory> -The output content type of the brep module is application/xhtml+xml and if you -would like to make sure it gets compressed (along with linked CSS), also add -the following lines: +The output content types of the brep module are application/xhtml+xml, +text/manifest and text/plain. If you would like to make sure they get +compressed (along with linked CSS), also add the following lines: # Compress brep output (xhtml+xml) and CSS. # AddOutputFilterByType DEFLATE application/xhtml+xml + AddOutputFilterByType DEFLATE text/manifest + AddOutputFilterByType DEFLATE text/plain AddOutputFilterByType DEFLATE text/css Restart Apache2: @@ -338,14 +361,16 @@ Review brep-module.conf changes that may need to be merged: $ diff -u install/share/brep/etc/brep-module.conf config/brep-module.conf -Migrate database schema: +Migrate database schemes: -$ install/bin/brep-migrate +$ install/bin/brep-migrate package +$ install/bin/brep-migrate build -Note that if instead you need to recreate the whole database (e.g., migration +Note that if instead you need to recreate the whole databases (e.g., migration is not possible), then one way to do it would be: -$ psql -c 'DROP OWNED BY brep' +$ psql -d brep_package -c 'DROP OWNED BY brep' +$ psql -d brep_build -c 'DROP OWNED BY brep' If using systemd, then start and enable the loader: diff --git a/INSTALL-DEV b/INSTALL-DEV index 8ad9ae7..8f5ba6f 100644 --- a/INSTALL-DEV +++ b/INSTALL-DEV @@ -24,39 +24,50 @@ setfacl -m g:www-data:rx ~/ ~/projects group, not user. However, most installations use the same name for both.] -1. Create PostgreSQL User and Database +1. Create PostgreSQL User and Databases $ sudo sudo -u postgres psql # Note: double sudo is not a mistake. -CREATE DATABASE brep TEMPLATE template0 ENCODING 'UTF8' +CREATE DATABASE brep_package TEMPLATE template0 ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8'; +CREATE DATABASE brep_build TEMPLATE template0 ENCODING 'UTF8' LC_COLLATE 'en_US.UTF8' LC_CTYPE 'en_US.UTF8'; CREATE USER <user>; -GRANT ALL PRIVILEGES ON DATABASE brep TO <user>; +GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO <user>; CREATE USER "www-data" INHERIT IN ROLE <user>; Exit psql (^D), then make sure the logins work: -$ psql -d brep -$ sudo sudo -u www-data psql -d brep +$ psql -d brep_package +^D +$ psql -d brep_build +^D +$ sudo sudo -u www-data psql -d brep_package +^D +$ sudo sudo -u www-data psql -d brep_build +^D To troubleshoot, see PostgreSQL logs, for example: $ sudo tail -f /var/log/postgresql/*.log -2. Create Database Schema and Load the Repository +2. Create Database Schemes and Load the Repository All the commands are executed from brep project root. -$ migrate/brep-migrate +$ migrate/brep-migrate package # Or use some other loader config. # $ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab +$ migrate/brep-migrate build + To verify: -$ psql -d brep -c 'SELECT name, summary FROM repository' +$ psql -d brep_package -c 'SELECT name, summary FROM repository' +$ psql -d brep_build -c 'SELECT package_name FROM build' # Empty row set. 3. Setup Apache2 Module @@ -73,6 +84,18 @@ replacing <BREP-OUT-ROOT> and <BREP-SRC-ROOT> with the actual absolute paths LoadModule brep_module <BREP-OUT-ROOT>/mod/mod_brep.so </IfModule> + # Repository email. This email is used for the From: header in emails + # send by brep (for example, build failure notifications). + # + brep-email admin@example.org + + # Repository host. It specifies the scheme and the host address (but + # not the root path; see brep-root below) that will be used whenever + # brep needs to construct an absolute URL to one of its locations (for + # example, a link to a build log that is being send via email). + # + brep-host https://example.org + # Repository root. Use / for web server root. And don't forget to also # update the Location and Alias directives below. # @@ -117,15 +140,17 @@ $ sudo tail -f /var/log/apache2/error.log 4. Reloading During Development -To do a "complete reload" (i.e., recreate database schema, load the repository +To do a "complete reload" (i.e., recreate database schemes, load the repository data, and reload the Apache2 plugin), execute the following from brep/: -migrate/brep-migrate --recreate +migrate/brep-migrate --recreate package +migrate/brep-migrate --recreate build load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab sudo /etc/init.d/apache2 restart sudo systemctl restart apache2 -Note that if instead you need to recreate the whole database (e.g., migration +Note that if instead you need to recreate the whole databases (e.g., migration is not possible), then one way to do it would be: -$ psql -d brep -c 'DROP OWNED BY <user>' +$ psql -d brep_package -c 'DROP OWNED BY <user>' +$ psql -d brep_build -c 'DROP OWNED BY <user>' diff --git a/brep/.gitignore b/brep/.gitignore index 687b168..9852519 100644 --- a/brep/.gitignore +++ b/brep/.gitignore @@ -1,3 +1,8 @@ +common-odb* + package-odb* package.sql package-extra + +build-odb* +build.sql diff --git a/brep/build b/brep/build new file mode 100644 index 0000000..3d969db --- /dev/null +++ b/brep/build @@ -0,0 +1,155 @@ +// file : brep/build -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BREP_BUILD +#define BREP_BUILD + +#include <chrono> + +#include <odb/core.hxx> +#include <odb/section.hxx> + +#include <bbot/manifest> + +#include <brep/types> +#include <brep/utility> + +#include <brep/common> // Must be included last (see assert). + +// Used by the data migration entries. +// +#define LIBBREP_BUILD_SCHEMA_VERSION_BASE 1 + +#pragma db model version(LIBBREP_BUILD_SCHEMA_VERSION_BASE, 1, open) + +// We have to keep these mappings at the global scope instead of inside +// the brep namespace because they need to be also effective in the +// bbot namespace from which we "borrow" types (and some of them use the mapped +// types). +// +#pragma db map type(bbot::result_status) as(std::string) \ + to(to_string (?)) \ + from(bbot::to_result_status (?)) + +namespace brep +{ + #pragma db value + struct build_id + { + package_id package; + string configuration; + + build_id () = default; + build_id (package_id p, string c) + : package (move (p)), configuration (move (c)) {} + }; + + inline bool + operator< (const build_id& x, const build_id& y) + { + return + x.package < y.package ? true : + y.package < x.package ? false : + x.configuration < y.configuration; + } + + // build_state + // + enum class build_state: std::uint8_t + { + untested, + testing, + tested + }; + + string + to_string (build_state); + + build_state + to_build_state (const string&); // May throw invalid_argument. + + inline ostream& + operator<< (ostream& os, build_state s) {return os << to_string (s);} + + #pragma db map type(build_state) as(string) \ + to(to_string (?)) \ + from(brep::to_build_state (?)) + + // result_status + // + using bbot::result_status; + + using optional_result_status = optional<result_status>; + + #pragma db map type(optional_result_status) as(optional_string) \ + to((?) ? bbot::to_string (*(?)) : brep::optional_string ()) \ + from((?) \ + ? bbot::to_result_status (*(?)) \ + : brep::optional_result_status ()) + + // operation_results + // + using bbot::operation_result; + #pragma db value(operation_result) definition + + using bbot::operation_results; + + #pragma db object pointer(shared_ptr) session + class build + { + public: + using timestamp_type = brep::timestamp; + + // Create the build object with the testing state, non-existent status and + // the timestamp set to now. + // + build (string name, version, string configuration); + + build_id id; + + string& package_name; // Tracks id.package.name. + upstream_version package_version; // Original of id.package.version. + string& configuration; // Tracks id.configuration. + + build_state state; + + // Time of the last state change (the creation time initially). + // + timestamp_type timestamp; + + // Present only if the state is 'tested'. + // + optional<result_status> status; + + // Note that the logs are stored as std::string/TEXT which is Ok since + // they are UTF-8 and our database is UTF-8. + // + #pragma db section(results_section) + operation_results results; + + #pragma db load(lazy) update(always) + odb::section results_section; + + // Database mapping. + // + #pragma db member(id) id column("") + + #pragma db member(package_name) transient + #pragma db member(package_version) \ + set(this.package_version.init (this.id.package.version, (?))) + #pragma db member(configuration) transient + + #pragma db member(results) id_column("") value_column("") + + build (const build&) = delete; + build& operator= (const build&) = delete; + + private: + friend class odb::access; + build () + : package_name (id.package.name), configuration (id.configuration) {} + }; +} + +#endif // BREP_BUILD diff --git a/brep/build.cxx b/brep/build.cxx new file mode 100644 index 0000000..c93c062 --- /dev/null +++ b/brep/build.cxx @@ -0,0 +1,45 @@ +// file : brep/build.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <brep/build> + +namespace brep +{ + // build_state + // + string + to_string (build_state s) + { + switch (s) + { + case build_state::untested: return "untested"; + case build_state::testing: return "testing"; + case build_state::tested: return "tested"; + } + + return string (); // Should never reach. + } + + build_state + to_build_state (const string& s) + { + if (s == "untested") return build_state::untested; + else if (s == "testing") return build_state::testing; + else if (s == "tested") return build_state::tested; + else throw invalid_argument ("invalid build state '" + s + "'"); + } + + // build + // + build:: + build (string pnm, version pvr, string cfg) + : id (package_id (move (pnm), pvr), move (cfg)), + package_name (id.package.name), + package_version (move (pvr)), + configuration (id.configuration), + state (build_state::testing), + timestamp (timestamp_type::clock::now ()) + { + } +} diff --git a/brep/build.xml b/brep/build.xml new file mode 100644 index 0000000..5f75928 --- /dev/null +++ b/brep/build.xml @@ -0,0 +1,64 @@ +<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="build" version="1"> + <model version="1"> + <table name="build" kind="object"> + <column name="package_name" type="TEXT" null="false"/> + <column name="package_version_epoch" type="INTEGER" null="false"/> + <column name="package_version_canonical_upstream" type="TEXT" null="false"/> + <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="package_version_revision" type="INTEGER" null="false"/> + <column name="configuration" type="TEXT" null="false"/> + <column name="package_version_upstream" type="TEXT" null="false"/> + <column name="package_version_release" type="TEXT" null="true"/> + <column name="state" type="TEXT" null="false"/> + <column name="timestamp" type="BIGINT" null="false"/> + <column name="status" type="TEXT" null="true"/> + <primary-key> + <column name="package_name"/> + <column name="package_version_epoch"/> + <column name="package_version_canonical_upstream"/> + <column name="package_version_canonical_release"/> + <column name="package_version_revision"/> + <column name="configuration"/> + </primary-key> + </table> + <table name="build_results" kind="container"> + <column name="package_name" type="TEXT" null="false"/> + <column name="package_version_epoch" type="INTEGER" null="false"/> + <column name="package_version_canonical_upstream" type="TEXT" null="false"/> + <column name="package_version_canonical_release" type="TEXT" null="false" options="COLLATE "C""/> + <column name="package_version_revision" type="INTEGER" null="false"/> + <column name="configuration" type="TEXT" null="false"/> + <column name="index" type="BIGINT" null="false"/> + <column name="operation" type="TEXT" null="false"/> + <column name="status" type="TEXT" null="false"/> + <column name="log" type="TEXT" null="false"/> + <foreign-key name="object_id_fk" on-delete="CASCADE"> + <column name="package_name"/> + <column name="package_version_epoch"/> + <column name="package_version_canonical_upstream"/> + <column name="package_version_canonical_release"/> + <column name="package_version_revision"/> + <column name="configuration"/> + <references table="build"> + <column name="package_name"/> + <column name="package_version_epoch"/> + <column name="package_version_canonical_upstream"/> + <column name="package_version_canonical_release"/> + <column name="package_version_revision"/> + <column name="configuration"/> + </references> + </foreign-key> + <index name="build_results_object_id_i"> + <column name="package_name"/> + <column name="package_version_epoch"/> + <column name="package_version_canonical_upstream"/> + <column name="package_version_canonical_release"/> + <column name="package_version_revision"/> + <column name="configuration"/> + </index> + <index name="build_results_index_i"> + <column name="index"/> + </index> + </table> + </model> +</changelog> diff --git a/brep/buildfile b/brep/buildfile index a7debc1..aeb1848 100644 --- a/brep/buildfile +++ b/brep/buildfile @@ -10,8 +10,14 @@ import int_libs = libodb%lib{odb} import int_libs += libodb-pgsql%lib{odb-pgsql} import int_libs += libbutl%lib{butl} import int_libs += libbpkg%lib{bpkg} +import int_libs += libbbot%lib{bbot} lib{brep}: \ +{hxx cxx}{ build } \ +{file }{ build.xml } \ +{hxx ixx cxx}{ build-odb } \ +{hxx cxx}{ common } \ +{hxx ixx cxx}{ common-odb } \ {hxx cxx}{ package } \ {file }{ package.xml } \ {hxx ixx cxx}{ package-odb } \ @@ -23,7 +29,7 @@ lib{brep}: \ {hxx }{ version } \ {hxx }{ wrapper-traits } \ $int_libs \ -sql{package package-extra} +sql{build package package-extra} # For pre-releases use the complete version to make sure they cannot be used # in place of another pre-release or the final version. diff --git a/brep/common b/brep/common new file mode 100644 index 0000000..52f225c --- /dev/null +++ b/brep/common @@ -0,0 +1,341 @@ +// file : brep/common -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef BREP_COMMON +#define BREP_COMMON + +#include <ratio> +#include <chrono> +#include <type_traits> // static_assert + +#include <brep/types> +#include <brep/utility> + +// The uint16_t value range is not fully covered by SMALLINT PostgreSQL type +// to which uint16_t is mapped by default. +// +#pragma db value(uint16_t) type("INTEGER") + +namespace brep +{ + // Use an image type to map bpkg::version to the database since there + // is no way to modify individual components directly. + // + #pragma db value + struct _version + { + uint16_t epoch; + string canonical_upstream; + string canonical_release; + uint16_t revision; + string upstream; + optional<string> release; + }; +} + +#include <bpkg/manifest> + +namespace brep +{ + using optional_version = optional<bpkg::version>; + using _optional_version = optional<_version>; +} + +// Prevent assert() macro expansion in get/set expressions. This should +// appear after all #include directives since the assert() macro is +// redefined in each <assert.h> inclusion. +// +#ifdef ODB_COMPILER +# undef assert +# define assert assert +void assert (int); +#endif + +// We have to keep these mappings at the global scope instead of inside +// the brep namespace because they need to be also effective in the +// bpkg namespace from which we "borrow" types (and some of them use version). +// +#pragma db map type(bpkg::version) as(brep::_version) \ + to(brep::_version{(?).epoch, \ + (?).canonical_upstream, \ + (?).canonical_release, \ + (?).revision, \ + (?).upstream, \ + (?).release}) \ + from(bpkg::version ((?).epoch, \ + std::move ((?).upstream), \ + std::move ((?).release), \ + (?).revision)) + +#pragma db map type(brep::optional_version) as(brep::_optional_version) \ + to((?) \ + ? brep::_version{(?)->epoch, \ + (?)->canonical_upstream, \ + (?)->canonical_release, \ + (?)->revision, \ + (?)->upstream, \ + (?)->release} \ + : brep::_optional_version ()) \ + from((?) \ + ? bpkg::version ((?)->epoch, \ + std::move ((?)->upstream), \ + std::move ((?)->release), \ + (?)->revision) \ + : brep::optional_version ()) + +namespace brep +{ + // path + // + #pragma db map type(path) as(string) to((?).string ()) from(brep::path (?)) + + using optional_path = optional<path>; + using optional_string = optional<string>; + + #pragma db map type(optional_path) as(brep::optional_string) \ + to((?) ? (?)->string () : brep::optional_string ()) \ + from((?) ? brep::path (*(?)) : brep::optional_path ()) + + #pragma db map type(dir_path) as(string) \ + to((?).string ()) from(brep::dir_path (?)) + + // Ensure that timestamp can be represented in nonoseconds without loss of + // accuracy, so the following ODB mapping is adequate. + // + static_assert( + std::ratio_greater_equal<timestamp::period, + std::chrono::nanoseconds::period>::value, + "The following timestamp ODB mapping is invalid"); + + // As it pointed out in butl/timestamp we will overflow in year 2262, but + // by that time some larger basic type will be available for mapping. + // + #pragma db map type(timestamp) as(uint64_t) \ + to(std::chrono::duration_cast<std::chrono::nanoseconds> ( \ + (?).time_since_epoch ()).count ()) \ + from(brep::timestamp ( \ + std::chrono::duration_cast<brep::timestamp::duration> ( \ + std::chrono::nanoseconds (?)))) + + // version + // + using bpkg::version; + + #pragma db value + struct canonical_version + { + uint16_t epoch; + string canonical_upstream; + string canonical_release; + uint16_t revision; + + bool + empty () const noexcept + { + // Note that an empty canonical_upstream doesn't denote an empty + // canonical_version. Remeber, that canonical_upstream doesn't include + // rightmost digit-only zero components? So non-empty version("0") has + // an empty canonical_upstream. + // + return epoch == 0 && canonical_upstream.empty () && + canonical_release.empty () && revision == 0; + } + + // Change collation to ensure the proper comparison of the "absent" release + // with a specified one. + // + // The default collation for UTF8-encoded TEXT columns in PostgreSQL is + // UCA-compliant. This makes the statement 'a' < '~' to be false, which + // in turn makes the statement 2.1.alpha < 2.1 to be false as well. + // + // Unicode Collation Algorithm (UCA): http://unicode.org/reports/tr10/ + // + #pragma db member(canonical_release) options("COLLATE \"C\"") + }; + + #pragma db value transient + struct upstream_version: version + { + #pragma db member(upstream_) virtual(string) \ + get(this.upstream) \ + set(this = brep::version (0, std::move (?), std::string (), 0)) + + #pragma db member(release_) virtual(optional_string) \ + get(this.release) \ + set(this = brep::version ( \ + 0, std::move (this.upstream), std::move (?), 0)) + + upstream_version () = default; + upstream_version (version v): version (move (v)) {} + upstream_version& + operator= (version v) {version& b (*this); b = v; return *this;} + + void + init (const canonical_version& cv, const upstream_version& uv) + { + *this = version (cv.epoch, uv.upstream, uv.release, cv.revision); + assert (cv.canonical_upstream == canonical_upstream && + cv.canonical_release == canonical_release); + } + }; + + // Wildcard version. Satisfies any dependency constraint and is represented + // as 0+0 (which is also the "stub version"; since a real version is always + // greater than the stub version, we reuse it to signify a special case). + // + extern const version wildcard_version; + + #pragma db value + struct package_id + { + string name; + canonical_version version; + + package_id () = default; + package_id (string n, const brep::version& v) + : name (move (n)), + version { + v.epoch, v.canonical_upstream, v.canonical_release, v.revision} + { + } + }; + + // Version comparison operators. + // + // They allow comparing objects that have epoch, canonical_upstream, + // canonical_release, and revision data members. The idea is that this + // works for both query members of types version and canonical_version + // as well as for comparing canonical_version to version. + // + template <typename T1, typename T2> + inline auto + compare_version_eq (const T1& x, const T2& y, bool revision) + -> decltype (x.epoch == y.epoch) + { + // Since we don't quite know what T1 and T2 are (and where the resulting + // expression will run), let's not push our luck with something like + // (!revision || x.revision == y.revision). + // + auto r (x.epoch == y.epoch && + x.canonical_upstream == y.canonical_upstream && + x.canonical_release == y.canonical_release); + + return revision + ? r && x.revision == y.revision + : r; + } + + template <typename T1, typename T2> + inline auto + compare_version_ne (const T1& x, const T2& y, bool revision) + -> decltype (x.epoch == y.epoch) + { + auto r (x.epoch != y.epoch || + x.canonical_upstream != y.canonical_upstream || + x.canonical_release != y.canonical_release); + + return revision + ? r || x.revision != y.revision + : r; + } + + template <typename T1, typename T2> + inline auto + compare_version_lt (const T1& x, const T2& y, bool revision) + -> decltype (x.epoch == y.epoch) + { + auto r ( + x.epoch < y.epoch || + (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream) || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release < y.canonical_release)); + + return revision + ? r || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release == y.canonical_release && x.revision < y.revision) + : r; + } + + template <typename T1, typename T2> + inline auto + compare_version_le (const T1& x, const T2& y, bool revision) + -> decltype (x.epoch == y.epoch) + { + auto r ( + x.epoch < y.epoch || + (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream)); + + return revision + ? r || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release < y.canonical_release) || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release == y.canonical_release && x.revision <= y.revision) + : r || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release <= y.canonical_release); + } + + template <typename T1, typename T2> + inline auto + compare_version_gt (const T1& x, const T2& y, bool revision) + -> decltype (x.epoch == y.epoch) + { + auto r ( + x.epoch > y.epoch || + (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream) || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release > y.canonical_release)); + + return revision + ? r || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release == y.canonical_release && x.revision > y.revision) + : r; + } + + template <typename T1, typename T2> + inline auto + compare_version_ge (const T1& x, const T2& y, bool revision) + -> decltype (x.epoch == y.epoch) + { + auto r ( + x.epoch > y.epoch || + (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream)); + + return revision + ? r || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release > y.canonical_release) || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release == y.canonical_release && x.revision >= y.revision) + : r || + (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && + x.canonical_release >= y.canonical_release); + } + + template <typename T> + inline auto + order_by_version_desc (const T& x) -> //decltype ("ORDER BY" + x.epoch) + decltype (x.epoch == 0) + { + return "ORDER BY" + + x.epoch + "DESC," + + x.canonical_upstream + "DESC," + + x.canonical_release + "DESC," + + x.revision + "DESC"; + } + + inline bool + operator< (const package_id& x, const package_id& y) + { + if (int r = x.name.compare (y.name)) + return r < 0; + + return compare_version_lt (x.version, y.version, true); + } +} + +#endif // BREP_COMMON diff --git a/brep/common.cxx b/brep/common.cxx new file mode 100644 index 0000000..4847977 --- /dev/null +++ b/brep/common.cxx @@ -0,0 +1,10 @@ +// file : brep/common.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <brep/common> + +namespace brep +{ + const version wildcard_version (0, "0", nullopt, 0); +} diff --git a/brep/odb.sh b/brep/odb.sh index 3f8cef2..5e16f45 100755 --- a/brep/odb.sh +++ b/brep/odb.sh @@ -9,10 +9,17 @@ lib="\ -I$HOME/work/odb/libodb-default \ -I$HOME/work/odb/libodb" -$odb $lib -d pgsql --std c++11 --generate-query --generate-schema \ - --schema-format sql --schema-format embedded \ +$odb $lib -d pgsql --std c++11 --generate-query \ --odb-epilogue '#include <brep/wrapper-traits>' \ --hxx-prologue '#include <brep/wrapper-traits>' \ + -I .. -I ../../libbpkg -I ../../libbutl \ + --hxx-suffix "" --include-with-brackets \ + --include-prefix brep --guard-prefix BREP \ + common + +$odb $lib -d pgsql --std c++11 --generate-query --generate-schema \ + --schema-format sql --schema-format embedded --schema-name package \ + --generate-prepared --odb-epilogue '#include <brep/wrapper-traits>' \ --hxx-prologue '#include <brep/package-traits>' \ -I .. -I ../../libbpkg -I ../../libbutl \ --hxx-suffix "" --include-with-brackets \ @@ -20,3 +27,11 @@ $odb $lib -d pgsql --std c++11 --generate-query --generate-schema \ package xxd -i <package-extra.sql >package-extra + +$odb $lib -d pgsql --std c++11 --generate-query --generate-schema \ + --schema-format sql --schema-format embedded --schema-name build \ + --generate-prepared --odb-epilogue '#include <brep/wrapper-traits>' \ + -I .. -I ../../libbbot -I ../../libbpkg -I ../../libbutl \ + --hxx-suffix "" --include-with-brackets \ + --include-prefix brep --guard-prefix BREP \ + build diff --git a/brep/package b/brep/package index 13429d1..d50aa9f 100644 --- a/brep/package +++ b/brep/package @@ -6,94 +6,21 @@ #define BREP_PACKAGE #include <map> -#include <ratio> #include <chrono> -#include <type_traits> // static_assert #include <odb/core.hxx> -#include <odb/forward.hxx> // database #include <odb/nested-container.hxx> #include <brep/types> #include <brep/utility> -// Used by the data migration entries. -// -#define LIBBREP_SCHEMA_VERSION_BASE 3 - -#pragma db model version(LIBBREP_SCHEMA_VERSION_BASE, 3, closed) +#include <brep/common> // Must be included last (see assert). -// The uint16_t value range is not fully covered by SMALLINT PostgreSQL type -// to which uint16_t is mapped by default. +// Used by the data migration entries. // -#pragma db value(uint16_t) type("INTEGER") - -namespace brep -{ - // Use an image type to map bpkg::version to the database since there - // is no way to modify individual components directly. - // - #pragma db value - struct _version - { - uint16_t epoch; - string canonical_upstream; - string canonical_release; - uint16_t revision; - string upstream; - optional<string> release; - }; -} - -#include <bpkg/manifest> - -namespace brep -{ - using optional_version = optional<bpkg::version>; - using _optional_version = optional<_version>; -} +#define LIBBREP_PACKAGE_SCHEMA_VERSION_BASE 3 -// Prevent assert() macro expansion in get/set expressions. This should -// appear after all #include directives since the assert() macro is -// redefined in each <assert.h> inclusion. -// -#ifdef ODB_COMPILER -# undef assert -# define assert assert -void assert (int); -#endif - -// We have to keep these mappings at the global scope instead of inside -// the brep namespace because they need to be also effective in the -// bpkg namespace from which we "borrow" types (and some of them use version). -// -#pragma db map type(bpkg::version) as(brep::_version) \ - to(brep::_version{(?).epoch, \ - (?).canonical_upstream, \ - (?).canonical_release, \ - (?).revision, \ - (?).upstream, \ - (?).release}) \ - from(bpkg::version ((?).epoch, \ - std::move ((?).upstream), \ - std::move ((?).release), \ - (?).revision)) - -#pragma db map type(brep::optional_version) as(brep::_optional_version) \ - to((?) \ - ? brep::_version{(?)->epoch, \ - (?)->canonical_upstream, \ - (?)->canonical_release, \ - (?)->revision, \ - (?)->upstream, \ - (?)->release} \ - : brep::_optional_version ()) \ - from((?) \ - ? bpkg::version ((?)->epoch, \ - std::move ((?)->upstream), \ - std::move ((?)->release), \ - (?)->revision) \ - : brep::optional_version ()) +#pragma db model version(LIBBREP_PACKAGE_SCHEMA_VERSION_BASE, 3, closed) namespace brep { @@ -110,106 +37,6 @@ namespace brep class repository; class package; - // path - // - #pragma db map type(path) as(string) to((?).string ()) from(brep::path (?)) - - using optional_path = optional<path>; - using optional_string = optional<string>; - - #pragma db map type(optional_path) as(brep::optional_string) \ - to((?) ? (?)->string () : brep::optional_string ()) \ - from((?) ? brep::path (*(?)) : brep::optional_path ()) - - #pragma db map type(dir_path) as(string) \ - to((?).string ()) from(brep::dir_path (?)) - - // Ensure that timestamp can be represented in nonoseconds without loss of - // accuracy, so the following ODB mapping is adequate. - // - static_assert( - std::ratio_greater_equal<timestamp::period, - std::chrono::nanoseconds::period>::value, - "The following timestamp ODB mapping is invalid"); - - // As it pointed out in butl/timestamp we will overflow in year 2262, but - // by that time some larger basic type will be available for mapping. - // - #pragma db map type(timestamp) as(uint64_t) \ - to(std::chrono::duration_cast<std::chrono::nanoseconds> ( \ - (?).time_since_epoch ()).count ()) \ - from(brep::timestamp ( \ - std::chrono::duration_cast<brep::timestamp::duration> ( \ - std::chrono::nanoseconds (?)))) - - // version - // - using bpkg::version; - - #pragma db value - struct canonical_version - { - uint16_t epoch; - string canonical_upstream; - string canonical_release; - uint16_t revision; - - bool - empty () const noexcept - { - // Note that an empty canonical_upstream doesn't denote an empty - // canonical_version. Remeber, that canonical_upstream doesn't include - // rightmost digit-only zero components? So non-empty version("0") has - // an empty canonical_upstream. - // - return epoch == 0 && canonical_upstream.empty () && - canonical_release.empty () && revision == 0; - } - - // Change collation to ensure the proper comparison of the "absent" release - // with a specified one. - // - // The default collation for UTF8-encoded TEXT columns in PostgreSQL is - // UCA-compliant. This makes the statement 'a' < '~' to be false, which - // in turn makes the statement 2.1.alpha < 2.1 to be false as well. - // - // Unicode Collation Algorithm (UCA): http://unicode.org/reports/tr10/ - // - #pragma db member(canonical_release) options("COLLATE \"C\"") - }; - - #pragma db value transient - struct upstream_version: version - { - #pragma db member(upstream_) virtual(string) \ - get(this.upstream) \ - set(this = brep::version (0, std::move (?), std::string (), 0)) - - #pragma db member(release_) virtual(optional_string) \ - get(this.release) \ - set(this = brep::version ( \ - 0, std::move (this.upstream), std::move (?), 0)) - - upstream_version () = default; - upstream_version (version v): version (move (v)) {} - upstream_version& - operator= (version v) {version& b (*this); b = v; return *this;} - - void - init (const canonical_version& cv, const upstream_version& uv) - { - *this = version (cv.epoch, uv.upstream, uv.release, cv.revision); - assert (cv.canonical_upstream == canonical_upstream && - cv.canonical_release == canonical_release); - } - }; - - // Wildcard version. Satisfies any dependency constraint and is represented - // as 0+0 (which is also the "stub version"; since a real version is always - // greater than the stub version, we reuse it to signify a special case). - // - extern const version wildcard_version; - // priority // using bpkg::priority; @@ -244,21 +71,6 @@ namespace brep #pragma db value(dependency_constraint) definition - #pragma db value - struct package_id - { - string name; - canonical_version version; - - package_id () = default; - package_id (string n, const brep::version& v) - : name (move (n)), - version { - v.epoch, v.canonical_upstream, v.canonical_release, v.revision} - { - } - }; - // Notes: // // 1. Will the package be always resolvable? What if it is in @@ -670,141 +482,17 @@ namespace brep package_id id; }; - // Version comparison operators. - // - // They allow comparing objects that have epoch, canonical_upstream, - // canonical_release, and revision data members. The idea is that this - // works for both query members of types version and canonical_version - // as well as for comparing canonical_version to version. - // - template <typename T1, typename T2> - inline auto - compare_version_eq (const T1& x, const T2& y, bool revision) - -> decltype (x.epoch == y.epoch) - { - // Since we don't quite know what T1 and T2 are (and where the resulting - // expression will run), let's not push our luck with something like - // (!revision || x.revision == y.revision). - // - auto r (x.epoch == y.epoch && - x.canonical_upstream == y.canonical_upstream && - x.canonical_release == y.canonical_release); - - return revision - ? r && x.revision == y.revision - : r; - } - - template <typename T1, typename T2> - inline auto - compare_version_ne (const T1& x, const T2& y, bool revision) - -> decltype (x.epoch == y.epoch) - { - auto r (x.epoch != y.epoch || - x.canonical_upstream != y.canonical_upstream || - x.canonical_release != y.canonical_release); - - return revision - ? r || x.revision != y.revision - : r; - } - - template <typename T1, typename T2> - inline auto - compare_version_lt (const T1& x, const T2& y, bool revision) - -> decltype (x.epoch == y.epoch) - { - auto r ( - x.epoch < y.epoch || - (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream) || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release < y.canonical_release)); - - return revision - ? r || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release == y.canonical_release && x.revision < y.revision) - : r; - } - - template <typename T1, typename T2> - inline auto - compare_version_le (const T1& x, const T2& y, bool revision) - -> decltype (x.epoch == y.epoch) - { - auto r ( - x.epoch < y.epoch || - (x.epoch == y.epoch && x.canonical_upstream < y.canonical_upstream)); - - return revision - ? r || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release < y.canonical_release) || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release == y.canonical_release && x.revision <= y.revision) - : r || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release <= y.canonical_release); - } - - template <typename T1, typename T2> - inline auto - compare_version_gt (const T1& x, const T2& y, bool revision) - -> decltype (x.epoch == y.epoch) - { - auto r ( - x.epoch > y.epoch || - (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream) || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release > y.canonical_release)); - - return revision - ? r || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release == y.canonical_release && x.revision > y.revision) - : r; - } - - template <typename T1, typename T2> - inline auto - compare_version_ge (const T1& x, const T2& y, bool revision) - -> decltype (x.epoch == y.epoch) + #pragma db view object(package) + struct package_version { - auto r ( - x.epoch > y.epoch || - (x.epoch == y.epoch && x.canonical_upstream > y.canonical_upstream)); - - return revision - ? r || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release > y.canonical_release) || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release == y.canonical_release && x.revision >= y.revision) - : r || - (x.epoch == y.epoch && x.canonical_upstream == y.canonical_upstream && - x.canonical_release >= y.canonical_release); - } - - template <typename T> - inline auto - order_by_version_desc (const T& x) -> //decltype ("ORDER BY" + x.epoch) - decltype (x.epoch == 0) - { - return "ORDER BY" - + x.epoch + "DESC," - + x.canonical_upstream + "DESC," - + x.canonical_release + "DESC," - + x.revision + "DESC"; - } - - inline bool - operator< (const package_id& x, const package_id& y) - { - if (int r = x.name.compare (y.name)) - return r < 0; + package_id id; + upstream_version version; - return compare_version_lt (x.version, y.version, true); - } + // Database mapping. + // + #pragma db member(id) column("") + #pragma db member(version) set(this.version.init (this.id.version, (?))) + }; } #endif // BREP_PACKAGE diff --git a/brep/package.cxx b/brep/package.cxx index eacfabe..eaf1029 100644 --- a/brep/package.cxx +++ b/brep/package.cxx @@ -13,8 +13,6 @@ using namespace odb::core; namespace brep { - const version wildcard_version (0, "0", nullopt, 0); - // dependency // string dependency:: diff --git a/brep/package.xml b/brep/package.xml index 69f1a6a..39494ce 100644 --- a/brep/package.xml +++ b/brep/package.xml @@ -1,4 +1,4 @@ -<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" version="1"> +<changelog xmlns="http://www.codesynthesis.com/xmlns/odb/changelog" database="pgsql" schema-name="package" version="1"> <model version="3"> <table name="repository" kind="object"> <column name="name" type="TEXT" null="false"/> diff --git a/brep/utility b/brep/utility index 631af7f..b359fc6 100644 --- a/brep/utility +++ b/brep/utility @@ -27,6 +27,6 @@ namespace brep // <butl/utility> // using butl::reverse_iterate; -}; +} #endif // BREP_UTILITY diff --git a/brep/version b/brep/version index f648b5e..e1db981 100644 --- a/brep/version +++ b/brep/version @@ -6,6 +6,7 @@ #include <butl/version> // LIBBUTL_VERSION #include <bpkg/version> // LIBBPKG_VERSION +#include <bbot/version> // LIBBBOT_VERSION // Version format is AABBCCDD where // @@ -44,4 +45,8 @@ # error incompatible libbpkg version #endif +#if LIBBBOT_VERSION != 49901 +# error incompatible libbbot version +#endif + #endif // BREP_VERSION diff --git a/etc/brep-apache2.conf b/etc/brep-apache2.conf index b7a481e..5114623 100644 --- a/etc/brep-apache2.conf +++ b/etc/brep-apache2.conf @@ -8,6 +8,18 @@ LoadModule brep_module /home/brep/install/libexec/brep/mod_brep.so </IfModule> + # Repository email. This email is used for the From: header in emails + # send by brep (for example, build failure notifications). + # + brep-email admin@example.org + + # Repository host. It specifies the scheme and the host address (but + # not the root path; see brep-root below) that will be used whenever + # brep needs to construct an absolute URL to one of its locations (for + # example, a link to a build log that is being send via email). + # + brep-host https://example.org + # Repository root. This is the part of the URL between the host name # and the start of the repository. For example, root value /pkg means # the repository URL is http://example.org/pkg/. Specify / to use the diff --git a/etc/brep-module.conf b/etc/brep-module.conf index 760d4ce..1d9d56d 100644 --- a/etc/brep-module.conf +++ b/etc/brep-module.conf @@ -26,7 +26,7 @@ menu About=?about # Number of pages in navigation (pager). # -# search-pages 5 +# search-pages 5 # Number of package description characters to display in brief pages. @@ -36,31 +36,82 @@ menu About=?about # Number of package changes characters to display in brief pages. # -# package-changes 5000 +# package-changes 5000 -# Database connection configuration. By default, brep will try to connect to -# the local instance of PostgreSQL with the operating system-default mechanism -# (Unix-domain socket, etc) and use operating system (login) user name and the -# database called 'brep'. See brep(1) for details. +# The package database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system-default +# mechanism (Unix-domain socket, etc) and use operating system (login) user +# name and the database called 'brep_package'. See brep(1) for details. # -# db-user -# db-password -# db-name brep -# db-host -# db-port +# package-db-user +# package-db-password +# package-db-name brep_package +# package-db-host +# package-db-port -# The maximum number of concurrent database connections per web server +# The maximum number of concurrent package database connections per web server # process. If 0, then no limitation is applied. # -# db-max-connections 5 +# package-db-max-connections 5 -# The maximum number of times to retry database transactions in the +# The maximum number of times to retry package database transactions in the # face of recoverable failures (deadlock, loss of connection, etc). # -# db-retry 10 +# package-db-retry 10 + + +# Build configuration file. If not specified (default), then the package +# building functionality will be disabled. If specified, then the build +# database must be configured (see next). Note: must be an absolute path. +# +# build-config + +# The maximum size of the build task request manifest accepted. Note that the +# HTTP POST request body is cached to retry database transactions in the face +# of recoverable failures (deadlock, loss of connection, etc). Default is +# 100K. +# +# build-task-request-max-size 102400 + + +# Time to wait before considering the expected task result lost. Must be +# specified in seconds. Default is 3 hours. +# +# build-result-timeout 10800 + + +# The maximum size of the build result manifest accepted. Note that the HTTP +# POST request body is cached to retry database transactions in the face of +# recoverable failures (deadlock, loss of connection, etc). Default is 10M +# +# build-result-request-max-size 10485760 + + +# The build database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system-default +# mechanism (Unix-domain socket, etc) and use operating system (login) user +# name and the database called 'brep_build'. See brep(1) for details. +# +# build-db-user +# build-db-password +# build-db-name brep_build +# build-db-host +# build-db-port + + +# The maximum number of concurrent build database connections per web server +# process. If 0, then no limitation is applied. +# +# build-db-max-connections 5 + + +# The maximum number of times to retry build database transactions in the +# face of recoverable failures (deadlock, loss of connection, etc). +# +# build-db-retry 10 # Trace verbosity. Disabled by default. diff --git a/load/load.cli b/load/load.cli index 7850d86..fe05ad2 100644 --- a/load/load.cli +++ b/load/load.cli @@ -28,8 +28,8 @@ include <brep/types>; repository and package information into the database, suitable for consumption by the \cb{brep} web module. - Note that \cb{brep-load} expects the database schema to have already been - created using \l{brep-migrate(1)}. + Note that \cb{brep-load} expects the database \cb{package} schema to have + already been created using \l{brep-migrate(1)}. Also note that \cb{brep-load} requires \l{bpkg(1)} to fetch repository information. See \cb{--bpkg} for more information on the package manager @@ -54,10 +54,11 @@ class options expected to work." } - std::string --db-name|-n = "brep" + std::string --db-name|-n = "brep_package" { "<name>", - "Database name. If not specified, then '\cb{brep}' is used by default." + "Database name. If not specified, then \cb{brep_package} is used by + default." } std::string --db-host|-h diff --git a/load/load.cxx b/load/load.cxx index 8856e38..e611df2 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -50,12 +50,6 @@ struct failed {}; static const char* help_info ( " info: run 'brep-load --help' for more information"); -static inline bool -space (char c) noexcept -{ - return c == ' ' || c == '\t'; -} - struct internal_repository { repository_location location; @@ -297,7 +291,7 @@ repository_info (const options& lo, const string& rl, const cstrings& options) { cerr << "error: unable to execute " << args[0] << ": " << e << endl; - if (e.child ()) + if (e.child) exit (1); throw failed (); @@ -965,6 +959,7 @@ try { cout << "brep-load " << BREP_VERSION_STR << endl << "libbrep " << LIBBREP_VERSION_STR << endl + << "libbbot " << LIBBBOT_VERSION_STR << endl << "libbpkg " << LIBBPKG_VERSION_STR << endl << "libbutl " << LIBBUTL_VERSION_STR << endl << "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl @@ -1018,12 +1013,13 @@ try transaction t (db.begin ()); - // Check that the database schema matches the current one. + // Check that the database 'package' schema matches the current one. // - if (schema_catalog::current_version (db) != db.schema_version ()) + const string ds ("package"); + if (schema_catalog::current_version (db, ds) != db.schema_version (ds)) { - cerr << "error: database schema differs from the current one" << endl - << " info: use brep-migrate to migrate the database" << endl; + cerr << "error: database 'package' schema differs from the current one" + << endl << " info: use brep-migrate to migrate the database" << endl; return 1; } @@ -21,3 +21,4 @@ depends: libodb >= 2.5.0-a10 depends: libodb-pgsql >= 2.5.0-a10 depends: libbutl == 0.5.0-a1 depends: libbpkg == 0.5.0-a1 +depends: libbbot == 0.5.0-a1 diff --git a/migrate/migrate.cli b/migrate/migrate.cli index 42a1c2e..54edd70 100644 --- a/migrate/migrate.cli +++ b/migrate/migrate.cli @@ -17,13 +17,14 @@ include <cstdint>; // uint16_t \cb{brep-migrate --help}\n \cb{brep-migrate --version}\n - \c{\b{brep-migrate} [<options>]} + \c{\b{brep-migrate} [<options>] <schema>} \h|DESCRIPTION| In its default mode \cb{brep-migrate} creates the database schema if it doesn't already exist. Otherwise, it migrates the existing schema and data - to the current version, if needed. + to the current version, if needed. The valid schema names are \cb{package} + and \cb{build}. If the \cb{--recreate} option is specified, then \cb{brep-migrate} instead recreates the database schema. That is, it drops all the existing tables @@ -66,10 +67,11 @@ class options expected to work." } - std::string --db-name|-n = "brep" + std::string --db-name|-n { "<name>", - "Database name. If not specified, then '\cb{brep}' is used by default." + "Database name. If not specified, then it is implicitly derived by + prefixing the schema name with \cb{brep_}." } std::string --db-host|-h diff --git a/migrate/migrate.cxx b/migrate/migrate.cxx index 98bdfbd..c700efe 100644 --- a/migrate/migrate.cxx +++ b/migrate/migrate.cxx @@ -41,7 +41,7 @@ class schema { public: explicit - schema (const char* extra); + schema (const char* extra, string name); void create (database&) const; @@ -50,12 +50,14 @@ public: drop (database&) const; private: + string name_; strings drop_statements_; strings create_statements_; }; schema:: -schema (const char* s) +schema (const char* s, string name) + : name_ (move (name)) { // Remove comments, saving the cleaned SQL code into statements. // @@ -176,7 +178,7 @@ drop (database& db) const // db.execute (s); - schema_catalog::drop_schema (db); + schema_catalog::drop_schema (db, name_); } void schema:: @@ -184,7 +186,7 @@ create (database& db) const { drop (db); - schema_catalog::create_schema (db); + schema_catalog::create_schema (db, name_); for (const auto& s: create_statements_) db.execute (s); @@ -205,6 +207,7 @@ try { cout << "brep-migrate " << BREP_VERSION_STR << endl << "libbrep " << LIBBREP_VERSION_STR << endl + << "libbbot " << LIBBBOT_VERSION_STR << endl << "libbpkg " << LIBBPKG_VERSION_STR << endl << "libbutl " << LIBBUTL_VERSION_STR << endl << "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl @@ -229,7 +232,19 @@ try return p.wait () ? 0 : 1; } - if (argc > 1) + if (!scan.more ()) + { + cerr << "error: no database schema specified" << endl + << help_info << endl; + return 1; + } + + const string db_schema (scan.next ()); + + if (db_schema != "package" && db_schema != "build") + throw cli::unknown_argument (db_schema); + + if (scan.more ()) { cerr << "error: unexpected argument encountered" << endl << help_info << endl; @@ -246,7 +261,9 @@ try odb::pgsql::database db ( ops.db_user (), ops.db_password (), - ops.db_name (), + !ops.db_name ().empty () + ? ops.db_name () + : "brep_" + db_schema, ops.db_host (), ops.db_port (), "options='-c default_transaction_isolation=serializable'"); @@ -261,7 +278,7 @@ try // transaction useless as all consequitive queries in that transaction will // be ignored by PostgreSQL. // - schema_version schema_version (db.schema_version ()); + schema_version schema_version (db.schema_version (db_schema)); // It is impossible to operate with the database which is out of the // [base_version, current_version] range due to the lack of the knowlege @@ -269,13 +286,13 @@ try // if (schema_version > 0) { - if (schema_version < schema_catalog::base_version (db)) + if (schema_version < schema_catalog::base_version (db, db_schema)) { cerr << "error: database schema is too old" << endl; throw failed (); } - if (schema_version > schema_catalog::current_version (db)) + if (schema_version > schema_catalog::current_version (db, db_schema)) { cerr << "error: database schema is too new" << endl; throw failed (); @@ -292,7 +309,7 @@ try // database (followed with the database creation for the --recreate option). // if ((create || drop) && schema_version != 0 && - schema_version != schema_catalog::current_version (db)) + schema_version != schema_catalog::current_version (db, db_schema)) { cerr << "error: database schema requires migration" << endl << " info: either migrate the database first or drop the entire " @@ -304,11 +321,14 @@ try if (create || drop) { - static const char extras[] = { + static const char package_extras[] = { #include <brep/package-extra> , '\0'}; - schema s (extras); + schema s (db_schema == "package" + ? package_extras + : "", + db_schema); if (create) s.create (db); @@ -319,10 +339,10 @@ try { // Register the data migration functions. // - // static const data_migration_entry<2, LIBBREP_SCHEMA_VERSION_BASE> + // static const data_migration_entry<2, LIBBREP_XXX_SCHEMA_VERSION_BASE> // migrate_v2_entry (&migrate_v2); // - schema_catalog::migrate (db); + schema_catalog::migrate (db, 0, db_schema); } t.commit (); diff --git a/mod/build-config b/mod/build-config new file mode 100644 index 0000000..a5713d7 --- /dev/null +++ b/mod/build-config @@ -0,0 +1,23 @@ +// file : mod/build-config -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_BUILD_CONFIG +#define MOD_BUILD_CONFIG + +#include <bbot/build-config> + +#include <brep/types> +#include <brep/utility> + +namespace brep +{ + // Return pointer to the shared build configurations instance, creating one + // on the first call. Throw tab_parsing on parsing error, io_error on the + // underlying OS error. Is not thread-safe. + // + shared_ptr<const bbot::build_configs> + shared_build_config (const path&); +} + +#endif // MOD_BUILD_CONFIG diff --git a/mod/build-config.cxx b/mod/build-config.cxx new file mode 100644 index 0000000..11be1b9 --- /dev/null +++ b/mod/build-config.cxx @@ -0,0 +1,31 @@ +// file : mod/build-config.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/build-config> + +#include <map> + +namespace brep +{ + using namespace bbot; + + shared_ptr<const build_configs> + shared_build_config (const path& p) + { + static std::map<path, weak_ptr<build_configs>> configs; + + auto i (configs.find (p)); + if (i != configs.end ()) + { + if (shared_ptr<build_configs> c = i->second.lock ()) + return c; + } + + shared_ptr<build_configs> c ( + make_shared<build_configs> (parse_buildtab (p))); + + configs[p] = c; + return c; + } +} diff --git a/mod/buildfile b/mod/buildfile index 228c149..2926a1b 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -14,13 +14,18 @@ import libs += libodb%lib{odb} import libs += libodb-pgsql%lib{odb-pgsql} import libs += libbutl%lib{butl} import libs += libbpkg%lib{bpkg} +import libs += libbbot%lib{bbot} include ../brep/ mod{brep}: \ + {hxx cxx}{ build-config } \ {hxx cxx}{ database } \ {hxx cxx}{ database-module } \ {hxx cxx}{ diagnostics } \ + {hxx cxx}{ mod-build-log } \ + {hxx cxx}{ mod-build-result } \ + {hxx cxx}{ mod-build-task } \ {hxx cxx}{ mod-package-details } \ {hxx cxx}{ mod-package-search } \ {hxx cxx}{ mod-package-version-details } \ @@ -57,7 +62,7 @@ if $cli.configured # parameters uniformly with a single catch block. # cli.options += --std c++11 -I $src_root --include-with-brackets \ ---include-prefix mod --guard-prefix MOD \ +--include-prefix mod --guard-prefix MOD --generate-specifier \ --cxx-prologue "#include <mod/types-parsers>" \ --cli-namespace brep::cli --generate-file-scanner --suppress-usage \ --generate-modifier --generate-description --option-prefix "" diff --git a/mod/database b/mod/database index 8e9fdd1..9a83752 100644 --- a/mod/database +++ b/mod/database @@ -10,17 +10,18 @@ #include <brep/types> #include <brep/utility> -#include <mod/options> - namespace brep { - // Returns pointer to the shared database instance, creating one on the - // first call. On subsequent calls ensures passed host and port equals - // to ones of the existing database instance throwing runtime_error - // otherwise. Is not thread-safe. + // Return pointer to the shared database instance, creating one on the first + // call. Throw odb::exception on failure. Is not thread-safe. // shared_ptr<odb::core::database> - shared_database (const options::db&); + shared_database (string user, + string password, + string name, + string host, + uint16_t port, + size_t max_connections); } #endif // MOD_DATABASE diff --git a/mod/database-module b/mod/database-module index 034324b..3799e7b 100644 --- a/mod/database-module +++ b/mod/database-module @@ -10,6 +10,8 @@ #include <brep/types> #include <brep/utility> +#include <bbot/build-config> + #include <mod/module> #include <mod/options> @@ -36,15 +38,31 @@ namespace brep // using module::init; + // Initialize the package database instance. Throw odb::exception on + // failure. + // + void + init (const options::package_db&, size_t retry); + + // Initialize the build database instance and parse build configuration + // file. Throw odb::exception on database failure, tab_parsing on parsing + // error, system_error on the underlying OS error. + // void - init (const options::db&); + init (const options::build&, const options::build_db&, size_t retry); virtual bool handle (request&, response&) = 0; protected: - size_t retry_; - shared_ptr<odb::core::database> db_; + size_t retry_ = 0; // Max of all retries. + + shared_ptr<odb::core::database> package_db_; + + // These are NULL if not building. + // + shared_ptr<odb::core::database> build_db_; + shared_ptr<const bbot::build_configs> build_conf_; private: virtual bool diff --git a/mod/database-module.cxx b/mod/database-module.cxx index 6fd5025..e7a6883 100644 --- a/mod/database-module.cxx +++ b/mod/database-module.cxx @@ -4,13 +4,23 @@ #include <mod/database-module> +#include <errno.h> // EIO + +#include <sstream> + #include <odb/exceptions.hxx> +#include <butl/utility> // throw_generic_error() + #include <mod/options> #include <mod/database> +#include <mod/build-config> namespace brep { + using namespace std; + using namespace butl; + // While currently the user-defined copy constructor is not required (we // don't need to deep copy nullptr's), it is a good idea to keep the // placeholder ready for less trivial cases. @@ -19,15 +29,49 @@ namespace brep database_module (const database_module& r) : module (r), retry_ (r.retry_), - db_ (r.initialized_ ? r.db_ : nullptr) + package_db_ (r.initialized_ ? r.package_db_ : nullptr), + build_db_ (r.initialized_ ? r.build_db_ : nullptr), + build_conf_ (r.initialized_ ? r.build_conf_ : nullptr) + { + } + + void database_module:: + init (const options::package_db& o, size_t retry) { + package_db_ = shared_database (o.package_db_user (), + o.package_db_password (), + o.package_db_name (), + o.package_db_host (), + o.package_db_port (), + o.package_db_max_connections ()); + + retry_ = retry_ < retry ? retry : retry_; } void database_module:: - init (const options::db& o) + init (const options::build& bo, const options::build_db& dbo, size_t retry) { - retry_ = o.db_retry (); - db_ = shared_database (o); + try + { + build_conf_ = shared_build_config (bo.build_config ()); + } + catch (const io_error& e) + { + ostringstream os; + os << "unable to read build configuration '" << bo.build_config () + << "': " << e; + + throw_generic_error (EIO, os.str ().c_str ()); + } + + build_db_ = shared_database (dbo.build_db_user (), + dbo.build_db_password (), + dbo.build_db_name (), + dbo.build_db_host (), + dbo.build_db_port (), + dbo.build_db_max_connections ()); + + retry_ = retry_ < retry ? retry : retry_; } bool database_module:: diff --git a/mod/database.cxx b/mod/database.cxx index f8fa1e5..22a0563 100644 --- a/mod/database.cxx +++ b/mod/database.cxx @@ -11,30 +11,43 @@ namespace brep { - namespace options + struct db_key { - bool - operator< (const db& x, const db& y) - { - int r; - if ((r = x.db_user ().compare (y.db_user ())) != 0 || - (r = x.db_password ().compare (y.db_password ())) != 0 || - (r = x.db_name ().compare (y.db_name ())) != 0 || - (r = x.db_host ().compare (y.db_host ()))) - return r < 0; - - return x.db_port () < y.db_port (); - } + string user; + string password; + string name; + string host; + uint16_t port; + }; + + static bool + operator< (const db_key& x, const db_key& y) + { + int r; + if ((r = x.user.compare (y.user)) != 0 || + (r = x.password.compare (y.password)) != 0 || + (r = x.name.compare (y.name)) != 0 || + (r = x.host.compare (y.host))) + return r < 0; + + return x.port < y.port; } using namespace odb; shared_ptr<database> - shared_database (const options::db& o) + shared_database (string user, + string password, + string name, + string host, + uint16_t port, + size_t max_connections) { - static std::map<options::db, weak_ptr<database>> databases; + static std::map<db_key, weak_ptr<database>> databases; + + db_key k ({move (user), move (password), move (name), host, port}); - auto i (databases.find (o)); + auto i (databases.find (k)); if (i != databases.end ()) { if (shared_ptr<database> d = i->second.lock ()) @@ -42,19 +55,19 @@ namespace brep } unique_ptr<pgsql::connection_factory> - f (new pgsql::connection_pool_factory (o.db_max_connections ())); + f (new pgsql::connection_pool_factory (max_connections)); shared_ptr<database> d ( make_shared<pgsql::database> ( - o.db_user (), - o.db_password (), - o.db_name (), - o.db_host (), - o.db_port (), + k.user, + k.password, + k.name, + k.host, + k.port, "options='-c default_transaction_isolation=serializable'", move (f))); - databases[o] = d; + databases[move (k)] = d; return d; } } diff --git a/mod/mod-build-log b/mod/mod-build-log new file mode 100644 index 0000000..8395546 --- /dev/null +++ b/mod/mod-build-log @@ -0,0 +1,45 @@ +// file : mod/mod-build-log -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_BUILD_LOG +#define MOD_MOD_BUILD_LOG + +#include <brep/types> +#include <brep/utility> + +#include <mod/options> +#include <mod/database-module> + +namespace brep +{ + class build_log: public database_module + { + public: + build_log () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + build_log (const build_log&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const + { + return options::build_log::description (); + } + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::build_log> options_; + }; +} + +#endif // MOD_MOD_BUILD_LOG diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx new file mode 100644 index 0000000..5342e3e --- /dev/null +++ b/mod/mod-build-log.cxx @@ -0,0 +1,220 @@ +// file : mod/mod-build-log.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-build-log> + +#include <algorithm> // find_if() + +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <web/module> + +#include <brep/build> +#include <brep/build-odb> +#include <brep/package> +#include <brep/package-odb> + +#include <mod/options> + +using namespace std; +using namespace bbot; +using namespace brep::cli; +using namespace odb::core; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::build_log:: +build_log (const build_log& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::build_log:: +init (scanner& s) +{ + MODULE_DIAG; + + options_ = make_shared<options::build_log> ( + s, unknown_mode::fail, unknown_mode::fail); + + database_module::init (static_cast<options::package_db> (*options_), + options_->package_db_retry ()); + + if (options_->build_config_specified ()) + database_module::init (static_cast<options::build> (*options_), + static_cast<options::build_db> (*options_), + options_->build_db_retry ()); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); +} + +bool brep::build_log:: +handle (request& rq, response& rs) +{ + using brep::version; // Not to confuse with module::version. + + MODULE_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + // Parse the HTTP request URL path (without the root directory) to obtain + // the build package name/version, the configuration name and the optional + // operation name. If the operation is not specified then print logs for all + // the operations. + // + // Note that the URL path must be in the following form: + // + // <package-name>/<package-version>/log/<config-name>[/<operation>] + // + // Also note that the presence of the first 3 components is guaranteed by + // the repository_root module. + // + build_id id; + string op; + + path lpath (rq.path ().leaf (options_->root ())); + + try + { + auto i (lpath.begin ()); + + assert (i != lpath.end ()); + string name (*i++); + + if (name.empty ()) + throw invalid_argument ("empty package name"); + + assert (i != lpath.end ()); + + version version; + + // Intercept exception handling to add the parsing error attribution. + // + try + { + version = brep::version (*i++); + } + catch (const invalid_argument& e) + { + throw invalid_argument ( + string ("invalid package version: ") + e.what ()); + } + + assert (i != lpath.end () && *i == "log"); + + if (++i == lpath.end ()) + throw invalid_argument ("no configuration name"); + + id = build_id (package_id (move (name), version), *i++); + + if (id.configuration.empty ()) + throw invalid_argument ("empty configuration name"); + + if (i != lpath.end ()) + op = *i++; + + if (i != lpath.end ()) + throw invalid_argument ("unexpected path component"); + } + catch (const invalid_argument& e) + { + throw invalid_request (400, e.what ()); + } + + // Make sure no parameters passed. + // + try + { + name_value_scanner s (rq.parameters ()); + params::build_log (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + // If the package build configuration expired (no such configuration, + // package, etc), then we log this case with the trace severity and respond + // with the 404 HTTP code (not found but may be available in the future). + // The thinking is that this may be or may not be a problem with the + // controller's setup (expires too fast or the link from some ancient email + // is opened). + // + auto config_expired = [&trace, &lpath, this] (const string& d) + { + l2 ([&]{trace << "package build configuration for " << lpath + << " expired: " << d;}); + + throw invalid_request (404, "package build configuration expired: " + d); + }; + + // Make sure the build configuration still exists. + // + auto i ( + find_if ( + build_conf_->begin (), build_conf_->end (), + [&id] (const build_config& c) {return c.name == id.configuration;})); + + if (i == build_conf_->end ()) + config_expired ("no configuration"); + + // Make sure the package still exists. + // + { + transaction t (package_db_->begin ()); + shared_ptr<package> p (package_db_->find<package> (id.package)); + t.commit (); + + if (p == nullptr) + config_expired ("no package"); + } + + // Load the package build configuration (if present). + // + shared_ptr<build> b; + { + transaction t (build_db_->begin ()); + b = build_db_->find<build> (id); + + if (b == nullptr) + config_expired ("no package configuration"); + else if (b->state != build_state::tested) + config_expired ("state is " + to_string (b->state)); + else + build_db_->load (*b, b->results_section); + + t.commit (); + } + + // We have all the data so don't buffer the response content. + // + ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); + + if (op.empty ()) + { + for (const auto& r: b->results) + os << r.log; + } + else + { + const operation_results& r (b->results); + + auto i ( + find_if (r.begin (), r.end (), + [&op] (const operation_result& v) {return v.operation == op;})); + + if (i == r.end ()) + config_expired ("no operation"); + + os << i->log; + } + + return true; +} diff --git a/mod/mod-build-result b/mod/mod-build-result new file mode 100644 index 0000000..8afa697 --- /dev/null +++ b/mod/mod-build-result @@ -0,0 +1,42 @@ +// file : mod/mod-build-result -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_BUILD_RESULT +#define MOD_MOD_BUILD_RESULT + +#include <brep/types> +#include <brep/utility> + +#include <mod/options> +#include <mod/database-module> + +namespace brep +{ + class build_result: public database_module + { + public: + build_result () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + build_result (const build_result&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const {return options::build_result::description ();} + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::build_result> options_; + }; +} + +#endif // MOD_MOD_BUILD_RESULT diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx new file mode 100644 index 0000000..fb6edb5 --- /dev/null +++ b/mod/mod-build-result.cxx @@ -0,0 +1,303 @@ +// file : mod/mod-build-result.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-build-result> + +#include <algorithm> // find_if() + +#include <butl/sendmail> +#include <butl/process-io> +#include <butl/manifest-parser> +#include <butl/manifest-serializer> + +#include <bbot/manifest> + +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <web/module> + +#include <brep/build> +#include <brep/build-odb> +#include <brep/package> +#include <brep/package-odb> + +#include <mod/options> + +using namespace std; +using namespace butl; +using namespace bbot; +using namespace brep::cli; +using namespace odb::core; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::build_result:: +build_result (const build_result& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::build_result:: +init (scanner& s) +{ + MODULE_DIAG; + + options_ = make_shared<options::build_result> ( + s, unknown_mode::fail, unknown_mode::fail); + + database_module::init (static_cast<options::package_db> (*options_), + options_->package_db_retry ()); + + if (options_->build_config_specified ()) + database_module::init (static_cast<options::build> (*options_), + static_cast<options::build_db> (*options_), + options_->build_db_retry ()); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); +} + +bool brep::build_result:: +handle (request& rq, response&) +{ + using brep::version; // Not to confuse with module::version. + + MODULE_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + // Make sure no parameters passed. + // + try + { + name_value_scanner s (rq.parameters ()); + params::build_result (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + result_request_manifest rqm; + + try + { + size_t limit (options_->build_result_request_max_size ()); + manifest_parser p (rq.content (limit, limit), "result_request_manifest"); + rqm = result_request_manifest (p); + } + catch (const manifest_parsing& e) + { + throw invalid_request (400, e.what ()); + } + + // Parse the task response session to obtain the build configuration name, + // and to make sure the session matches the result manifest's package name + // and version. + // + build_id id; + + try + { + const string& s (rqm.session); + + size_t p (s.find ('/')); // End of package name. + + if (p == 0) + throw invalid_argument ("empty package name"); + + if (p == string::npos) + throw invalid_argument ("no package version"); + + string& name (rqm.result.name); + if (name.compare (0, name.size (), s, 0, p) != 0) + throw invalid_argument ("package name mismatch"); + + size_t b (p + 1); // Start of version. + p = s.find ('/', b); // End of version. + + if (p == string::npos) + throw invalid_argument ("no configuration name"); + + version version; + + // Intercept exception handling to add the parsing error attribution. + // + try + { + version = brep::version (string (s, b, p - b)); + } + catch (const invalid_argument& e) + { + throw invalid_argument ( + string ("invalid package version: ") + e.what ()); + } + + if (version != rqm.result.version) + throw invalid_argument ("package version mismatch"); + + id = build_id (package_id (move (name), version), string (s, p + 1)); + + if (id.configuration.empty ()) + throw invalid_argument ("empty configuration name"); + } + catch (const invalid_argument& e) + { + throw invalid_request (400, string ("invalid session: ") + e.what ()); + } + + // If the session expired (no such configuration, package, etc), then we log + // this case with the warning severity and respond with the 200 HTTP code as + // if the session is valid. The thinking is that this is a problem with the + // controller's setup (expires too fast), not with the agent's. + // + auto warn_expired = [&rqm, &warn] (const string& d) + { + warn << "session '" << rqm.session << "' expired: " << d; + }; + + // Make sure the build configuration still exists. + // + auto i ( + find_if ( + build_conf_->begin (), build_conf_->end (), + [&id] (const build_config& c) {return c.name == id.configuration;})); + + if (i == build_conf_->end ()) + { + warn_expired ("no build configuration"); + return true; + } + + // Load the built package (if present). + // + shared_ptr<package> p; + { + transaction t (package_db_->begin ()); + p = package_db_->find<package> (id.package); + t.commit (); + } + + if (p == nullptr) + { + warn_expired ("no package"); + return true; + } + + // Load and update the package build configuration (if present). + // + shared_ptr<build> b; + { + transaction t (build_db_->begin ()); + b = build_db_->find<build> (id); + + if (b == nullptr) + warn_expired ("no package configuration"); + else if (b->state != build_state::testing) + { + warn_expired ("package configuration state is " + to_string (b->state)); + b = nullptr; + } + else + { + b->state = build_state::tested; + b->timestamp = timestamp::clock::now (); + + assert (!b->status); + b->status = rqm.result.status; + + // Need to manually load the lazy-load section prior to changing its + // members and updating the object's persistent state. + // + // @@ TODO: ODB now allows marking a section as loaded so can optimize + // this. + // + build_db_->load (*b, b->results_section); + b->results = move (rqm.result.results); + + build_db_->update (b); + } + + t.commit (); + } + + if (b == nullptr) + return true; // Warning is already logged. + + // Send email to the package owner. + // + try + { + string subj (b->package_name + '/' + b->package_version.string () + ' ' + + b->configuration + " build: " + to_string (*b->status)); + + // If the package email address is not specified, then it is assumed to be + // the same as the project email address. + // + const string& to (p->package_email + ? *p->package_email + : p->email); + + auto print_args = [&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }; + + // Redirect the diagnostics to webserver error log. + // + // Note: if using this somewhere else, then need to factor out all this + // exit status handling code. + // + sendmail sm (print_args, + 2, + options_->email (), + subj, + {to}); + + if (b->results.empty ()) + sm.out << "No operations results available." << endl; + else + { + for (const auto& r: b->results) + sm.out << r.operation << ": " << r.status << ", " + << options_->host () << options_->root ().representation () + << b->package_name << '/' << b->package_version << "/log/" + << b->configuration << '/' << r.operation << endl; + } + + sm.out.close (); + + if (!sm.wait ()) + { + diag_record dr (error); + dr << "sendmail "; + + assert (sm.exit); + const process_exit& e (*sm.exit); + + if (e.normal ()) + dr << "exited with code " << static_cast<uint16_t> (e.code ()); + else + { + dr << "terminated abnormally: " << e.description (); + + if (e.core ()) + dr << " (core dumped)"; + } + } + } + // Handle process_error and io_error (both derive from system_error). + // + catch (const system_error& e) + { + error << "sendmail error: " << e; + } + + return true; +} diff --git a/mod/mod-build-task b/mod/mod-build-task new file mode 100644 index 0000000..051357e --- /dev/null +++ b/mod/mod-build-task @@ -0,0 +1,42 @@ +// file : mod/mod-build-task -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_BUILD_TASK +#define MOD_MOD_BUILD_TASK + +#include <brep/types> +#include <brep/utility> + +#include <mod/options> +#include <mod/database-module> + +namespace brep +{ + class build_task: public database_module + { + public: + build_task () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + build_task (const build_task&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const {return options::build_task::description ();} + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::build_task> options_; + }; +} + +#endif // MOD_MOD_BUILD_TASK diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx new file mode 100644 index 0000000..1cb5386 --- /dev/null +++ b/mod/mod-build-task.cxx @@ -0,0 +1,389 @@ +// file : mod/mod-build-task.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-build-task> + +#include <map> +#include <chrono> + +#include <butl/utility> // compare_c_string +#include <butl/filesystem> // path_match() +#include <butl/manifest-parser> +#include <butl/manifest-serializer> + +#include <bbot/manifest> +#include <bbot/build-config> + +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <web/module> + +#include <brep/build> +#include <brep/build-odb> +#include <brep/package> +#include <brep/package-odb> + +#include <mod/options> + +using namespace std; +using namespace butl; +using namespace bbot; +using namespace brep::cli; +using namespace odb::core; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::build_task:: +build_task (const build_task& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::build_task:: +init (scanner& s) +{ + MODULE_DIAG; + + options_ = make_shared<options::build_task> ( + s, unknown_mode::fail, unknown_mode::fail); + + database_module::init (static_cast<options::package_db> (*options_), + options_->package_db_retry ()); + + if (options_->build_config_specified ()) + database_module::init (static_cast<options::build> (*options_), + static_cast<options::build_db> (*options_), + options_->build_db_retry ()); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); +} + +bool brep::build_task:: +handle (request& rq, response& rs) +{ + MODULE_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + // Make sure no parameters passed. + // + try + { + name_value_scanner s (rq.parameters ()); + params::build_task (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + task_request_manifest tqm; + + try + { + size_t limit (options_->build_task_request_max_size ()); + manifest_parser p (rq.content (limit, limit), "task_request_manifest"); + tqm = task_request_manifest (p); + } + catch (const manifest_parsing& e) + { + throw invalid_request (400, e.what ()); + } + + task_response_manifest tsm; + + // Map build configurations to machines that are capable of building them. + // The first matching machine is selected for each configuration. Also + // create the configuration name list for use in database queries. + // + struct config_machine + { + const build_config* config; + const machine_header_manifest* machine; + }; + + using config_machines = map<const char*, config_machine, compare_c_string>; + + cstrings cfg_names; + config_machines cfg_machines; + + for (const auto& c: *build_conf_) + { + for (auto& m: tqm.machines) + { + if (path_match (c.machine_pattern, m.name) && + cfg_machines.insert ( + make_pair (c.name.c_str (), config_machine ({&c, &m}))).second) + cfg_names.push_back (c.name.c_str ()); + } + } + + // Go through packages until we find one that has no build configuration + // present in the database, or has the untested one, or in the testing state + // but expired, or the one, which build failed abnormally and expired. If + // such a package configuration is found then put it into the testing state, + // set the current timestamp and respond with the task for building this + // package configuration. + // + if (!cfg_machines.empty ()) + { + // Calculate the expiration time for package configurations being in the + // testing state or those, which build failed abnormally. + // + timestamp expiration (timestamp::clock::now () - + chrono::seconds (options_->build_result_timeout ())); + + uint64_t expiration_ns ( + std::chrono::duration_cast<std::chrono::nanoseconds> ( + expiration.time_since_epoch ()).count ()); + + // Prepare the package version prepared query. + // + // Note that the number of packages can be large and so, in order not to + // hold locks for too long, we will restrict the number of packages being + // queried in a single transaction. To achieve this we will iterate through + // packages using the OFFSET/LIMIT pair and sort the query result. + // + // Note that this approach can result in missing some packages or + // iterating multiple times over some of them. However there is nothing + // harmful in that: updates are infrequent and missed packages will be + // picked up on the next request. + // + using pkg_query = query<package_version>; + using prep_pkg_query = prepared_query<package_version>; + + size_t offset (0); // See the package version query. + + // Skip external and stub packages. + // + pkg_query pq ((pkg_query::internal_repository.is_not_null () && + compare_version_ne (pkg_query::id.version, + wildcard_version, + true)) + + "ORDER BY" + + pkg_query::id.name + "," + + pkg_query::id.version.epoch + "," + + pkg_query::id.version.canonical_upstream + "," + + pkg_query::id.version.canonical_release + "," + + pkg_query::id.version.revision + + "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"); + + connection_ptr pkg_conn (package_db_->connection ()); + + prep_pkg_query pkg_prep_query ( + pkg_conn->prepare_query<package_version> ( + "mod-build-task-package-version-query", pq)); + + // Prepare the build prepared query. + // + // Note that we can not query the database for configurations that a + // package was not built with, as the database contains only those package + // configurations that have already been acted upon (initially empty). + // + // This is why we query the database for package configurations that + // should not be built (in the tested state with the build terminated + // normally or not expired, or in the testing state and not expired). + // Having such a list we will select the first build configuration that is + // not in the list (if available) for the response. + // + using bld_query = query<build>; + using prep_bld_query = prepared_query<build>; + + package_id id; // See the build query. + + const auto& qv (bld_query::id.package.version); + + bld_query bq ( + bld_query::id.package.name == bld_query::_ref (id.name) && + + qv.epoch == bld_query::_ref (id.version.epoch) && + qv.canonical_upstream == + bld_query::_ref (id.version.canonical_upstream) && + qv.canonical_release == bld_query::_ref (id.version.canonical_release) && + qv.revision == bld_query::_ref (id.version.revision) && + + bld_query::id.configuration.in_range (cfg_names.begin (), + cfg_names.end ()) && + + ((bld_query::state == "tested" && + ((bld_query::status != "abort" && bld_query::status != "abnormal") || + bld_query::timestamp > expiration_ns)) || + + (bld_query::state == "testing" && + bld_query::timestamp > expiration_ns))); + + connection_ptr bld_conn (build_db_->connection ()); + + prep_bld_query bld_prep_query ( + bld_conn->prepare_query<build> ( + "mod-build-task-package-build-query", bq)); + + while (tsm.session.empty ()) + { + // Start the package database transaction. + // + transaction pt (pkg_conn->begin ()); + + // Query package versions. + // + auto package_versions (pkg_prep_query.execute ()); + + // Bail out if there is nothing left. + // + if (package_versions.empty ()) + { + pt.commit (); + break; + } + + offset += package_versions.size (); + + // Start the build database transaction. + // + { + transaction bt (bld_conn->begin (), false); + transaction::current (bt); + + // Iterate over packages until we find one that needs building. + // + for (auto& pv: package_versions) + { + id = move (pv.id); + + // Iterate through the package configurations and erase those that + // don't need building from the build configuration map. All those + // configurations that remained can be built. We will take the first + // one, if present. + // + config_machines configs (cfg_machines); // Make a copy for this pkg. + + for (const auto& pc: bld_prep_query.execute ()) + { + auto i (configs.find (pc.id.configuration.c_str ())); + + // Outdated configurations are already excluded with the database + // query. + // + assert (i != configs.end ()); + configs.erase (i); + } + + if (!configs.empty ()) + { + config_machine& cm (configs.begin ()->second); + const build_config& cfg (*cm.config); + + build_id bid (move (id), cfg.name); + shared_ptr<build> b (build_db_->find<build> (bid)); + + // If build configuration doesn't exist then create the new one + // and persist. Otherwise put it into the testing state, refresh + // the timestamp and update. + // + if (b == nullptr) + { + b = make_shared<build> (move (bid.package.name), + move (pv.version), + move (bid.configuration)); + + build_db_->persist (b); + } + else + { + // If the package configuration is in the tested state, then we + // need to cleanup the status and results prior to the update. + // Otherwise the status is already absent and there are no + // results. + // + // Load the section to make sure results are updated for the + // tested state, otherwise assert there are no results. + // + build_db_->load (*b, b->results_section); + + if (b->state == build_state::tested) + { + assert (b->status); + b->status = nullopt; + + b->results.clear (); + } + else + { + assert (!b->status && b->results.empty ()); + } + + b->state = build_state::testing; + b->timestamp = timestamp::clock::now (); + + build_db_->update (b); + } + + // Finally, prepare the task response manifest. + // + tsm.session = b->package_name + '/' + + b->package_version.string () + '/' + b->configuration; + + // @@ We don't support challenge at the moment, so leave it absent. + // + + tsm.result_url = options_->host () + options_->root ().string () + + "?build-result"; + + // Switch to the package database transaction to load the package. + // + transaction::current (pt); + + shared_ptr<package> p (package_db_->load<package> (b->id.package)); + shared_ptr<repository> r (p->internal_repository.load ()); + + // Switch back to the build database transaction. + // + transaction::current (bt); + + strings fp; + if (r->certificate) + fp.emplace_back (move (r->certificate->fingerprint)); + + tsm.task = task_manifest ( + move (b->package_name), + move (b->package_version), + move (r->location), + move (fp), + move (cm.machine->name), + cfg.target, + cfg.vars); + } + + // If task response manifest is filled, then can bail out from the + // package loop, commit transactions and respond. + // + if (!tsm.session.empty ()) + break; + } + + bt.commit (); // Commit the build database transaction. + } + + transaction::current (pt); // Switch to the package database transaction. + pt.commit (); + } + } + + // @@ Probably it would be a good idea to also send some cache control + // headers to avoid caching by HTTP proxies. That would require extension + // of the web::response interface. + // + + manifest_serializer s (rs.content (200, "text/manifest;charset=utf-8"), + "task_response_manifest"); + tsm.serialize (s); + + return true; +} diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx index 95bf087..fd3fd6d 100644 --- a/mod/mod-package-details.cxx +++ b/mod/mod-package-details.cxx @@ -42,7 +42,7 @@ init (scanner& s) options_ = make_shared<options::package_details> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); @@ -148,17 +148,17 @@ handle (request& rq, response& rs) << ~DIV; session sn; - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); shared_ptr<package> pkg; { latest_package lp; - if (!db_->query_one<latest_package> ( + if (!package_db_->query_one<latest_package> ( query<latest_package>( "(" + query<latest_package>::_val (name) + ")"), lp)) throw invalid_request (404, "Package '" + name + "' not found"); - pkg = db_->load<package> (lp.id); + pkg = package_db_->load<package> (lp.id); } const auto& licenses (pkg->license_alternatives); @@ -187,7 +187,7 @@ handle (request& rq, response& rs) } auto pkg_count ( - db_->query_value<package_count> ( + package_db_->query_value<package_count> ( search_params<package_count> (name, squery))); s << FORM_SEARCH (squery) @@ -197,7 +197,7 @@ handle (request& rq, response& rs) // s << DIV; for (const auto& pr: - db_->query<package_search_rank> ( + package_db_->query<package_search_rank> ( search_params<package_search_rank> (name, squery) + "ORDER BY rank DESC, version_epoch DESC, " "version_canonical_upstream DESC, version_canonical_release DESC, " @@ -205,7 +205,7 @@ handle (request& rq, response& rs) "OFFSET" + to_string (page * res_page) + "LIMIT" + to_string (res_page))) { - shared_ptr<package> p (db_->load<package> (pr.id)); + shared_ptr<package> p (package_db_->load<package> (pr.id)); s << TABLE(CLASS="proplist version") << TBODY diff --git a/mod/mod-package-search.cxx b/mod/mod-package-search.cxx index 02b10d6..4988d42 100644 --- a/mod/mod-package-search.cxx +++ b/mod/mod-package-search.cxx @@ -44,14 +44,14 @@ init (scanner& s) options_ = make_shared<options::package_search> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); - // Check that the database schema matches the current one. It's enough to - // perform the check in just a single module implementation (and we don't - // do in the dispatcher because it doesn't use the database). + // Check that the database 'package' schema matches the current one. It's + // enough to perform the check in just a single module implementation (and we + // don't do in the dispatcher because it doesn't use the database). // // Note that the failure can be reported by each web server worker process. // While it could be tempting to move the check to the @@ -59,8 +59,10 @@ init (scanner& s) // be called by a different process (usually the web server root one) not // having the proper permissions to access the database. // - if (schema_catalog::current_version (*db_) != db_->schema_version ()) - fail << "database schema differs from the current one (module " + const string ds ("package"); + if (schema_catalog::current_version (*package_db_, ds) != + package_db_->schema_version (ds)) + fail << "database 'package' schema differs from the current one (module " << BREP_VERSION_STR << ")"; } @@ -133,10 +135,10 @@ handle (request& rq, response& rs) << DIV(ID="content"); session sn; - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); auto pkg_count ( - db_->query_value<latest_package_count> ( + package_db_->query_value<latest_package_count> ( search_param<latest_package_count> (squery))); s << FORM_SEARCH (squery) @@ -146,13 +148,13 @@ handle (request& rq, response& rs) // s << DIV; for (const auto& pr: - db_->query<latest_package_search_rank> ( + package_db_->query<latest_package_search_rank> ( search_param<latest_package_search_rank> (squery) + "ORDER BY rank DESC, name" + "OFFSET" + to_string (page * res_page) + "LIMIT" + to_string (res_page))) { - shared_ptr<package> p (db_->load<package> (pr.id)); + shared_ptr<package> p (package_db_->load<package> (pr.id)); s << TABLE(CLASS="proplist package") << TBODY diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 89acf89..b0e6ead 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -43,7 +43,7 @@ init (scanner& s) options_ = make_shared<options::package_version_details> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); @@ -130,11 +130,11 @@ handle (request& rq, response& rs) shared_ptr<package> pkg; session sn; - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); try { - pkg = db_->load<package> (package_id (name, ver)); + pkg = package_db_->load<package> (package_id (name, ver)); // If the requested package turned up to be an "external" one just // respond that no "internal" package is present. diff --git a/mod/mod-repository-details.cxx b/mod/mod-repository-details.cxx index ab87cb2..3a73955 100644 --- a/mod/mod-repository-details.cxx +++ b/mod/mod-repository-details.cxx @@ -49,7 +49,7 @@ init (scanner& s) options_ = make_shared<options::repository_details> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); @@ -90,12 +90,12 @@ handle (request& rq, response& rs) << DIV_HEADER (root, options_->logo (), options_->menu ()) << DIV(ID="content"); - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); using query = query<repository>; for (const auto& r: - db_->query<repository> ( + package_db_->query<repository> ( query::internal + "ORDER BY" + query::priority)) { //@@ Feels like a lot of trouble (e.g., id_attribute()) for very diff --git a/mod/mod-repository-root b/mod/mod-repository-root index 57db93a..6a40e10 100644 --- a/mod/mod-repository-root +++ b/mod/mod-repository-root @@ -17,6 +17,9 @@ namespace brep class package_details; class package_version_details; class repository_details; + class build_task; + class build_result; + class build_log; class repository_root: public module { @@ -55,6 +58,9 @@ namespace brep shared_ptr<package_details> package_details_; shared_ptr<package_version_details> package_version_details_; shared_ptr<repository_details> repository_details_; + shared_ptr<build_task> build_task_; + shared_ptr<build_result> build_result_; + shared_ptr<build_log> build_log_; shared_ptr<options::repository_root> options_; // Sub-module the request is dispatched to. Initially is NULL. It is set diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index 1bc10cb..e156308 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -12,6 +12,9 @@ #include <mod/module> #include <mod/options> +#include <mod/mod-build-log> +#include <mod/mod-build-task> +#include <mod/mod-build-result> #include <mod/mod-package-search> #include <mod/mod-package-details> #include <mod/mod-repository-details> @@ -55,7 +58,10 @@ namespace brep : package_search_ (make_shared<package_search> ()), package_details_ (make_shared<package_details> ()), package_version_details_ (make_shared<package_version_details> ()), - repository_details_ (make_shared<repository_details> ()) + repository_details_ (make_shared<repository_details> ()), + build_task_ (make_shared<build_task> ()), + build_result_ (make_shared<build_result> ()), + build_log_ (make_shared<build_log> ()) { } @@ -83,6 +89,18 @@ namespace brep r.initialized_ ? r.repository_details_ : make_shared<repository_details> (*r.repository_details_)), + build_task_ ( + r.initialized_ + ? r.build_task_ + : make_shared<build_task> (*r.build_task_)), + build_result_ ( + r.initialized_ + ? r.build_result_ + : make_shared<build_result> (*r.build_result_)), + build_log_ ( + r.initialized_ + ? r.build_log_ + : make_shared<build_log> (*r.build_log_)), options_ ( r.initialized_ ? r.options_ @@ -101,6 +119,9 @@ namespace brep append (r, package_details_->options ()); append (r, package_version_details_->options ()); append (r, repository_details_->options ()); + append (r, build_task_->options ()); + append (r, build_result_->options ()); + append (r, build_log_->options ()); return r; } @@ -109,17 +130,38 @@ namespace brep void repository_root:: init (const name_values& v) { - auto sub_init ([this, &v](module& m) + auto sub_init = [this, &v] (module& m, const char* name) + { + // Initialize sub-module. Intercept exception handling to add sub-module + // attribution. + // + try { m.init (filter (v, m.options ()), *log_); - }); + } + catch (const std::exception& e) + { + // Any exception thrown by this function terminates the web server. All + // exception types inherited from std::exception are handled by the web + // server as std::exception. The only sensible way to handle them is to + // log the error prior terminating. By that reason it is valid to + // reduce all these types to a single one. + // + ostringstream os; + os << name << ": " << e; + throw runtime_error (os.str ()); + } + }; // Initialize sub-modules. // - sub_init (*package_search_); - sub_init (*package_details_); - sub_init (*package_version_details_); - sub_init (*repository_details_); + sub_init (*package_search_, "package_search"); + sub_init (*package_details_, "package_details"); + sub_init (*package_version_details_, "package_version_details"); + sub_init (*repository_details_, "repository_details"); + sub_init (*build_task_, "build_task"); + sub_init (*build_result_, "build_result"); + sub_init (*build_log_, "build_log"); // Parse own configuration options. // @@ -155,8 +197,7 @@ namespace brep // Delegate the request handling to the selected sub-module. Intercept // exception handling to add sub-module attribution. // - auto handle = - [&rs, this] (request& rq, const char* name) -> bool + auto handle = [&rs, this] (request& rq, const char* name) -> bool { try { @@ -194,15 +235,24 @@ namespace brep // if (lpath.empty ()) { - // Dispatch request handling to the repository_details or the - // package_search module depending on the function name passed as a - // first HTTP request parameter. The parameter should have no value - // specified. Example: cppget.org/?about + // Dispatch request handling to the repository_details, the build_task, + // the build_result or the package_search module depending on the + // function name passed as a first HTTP request parameter. The parameter + // should have no value specified. Example: cppget.org/?about // const name_values& params (rq.parameters ()); if (!params.empty () && !params.front ().value) { - if (params.front ().name == "about") + // Cleanup not to confuse the selected module with the unknown + // parameter. + // + name_values p (params); + p.erase (p.begin ()); + + request_proxy rp (rq, p); + const string& fn (params.front ().name); + + if (fn == "about") { if (handler_ == nullptr) handler_.reset (new repository_details (*repository_details_)); @@ -236,8 +286,9 @@ namespace brep } else { - // Dispatch request handling to the package_details or the - // package_version_details module depending on the HTTP request URL path. + // Dispatch request handling to the package_details, the + // package_version_details or the build_log module depending on the HTTP + // request URL path. // auto i (lpath.begin ()); assert (i != lpath.end ()); @@ -279,6 +330,13 @@ namespace brep return handle (rq, "package_version_details"); } + else if (*i == "log") + { + if (handler_ == nullptr) + handler_.reset (new build_log (*build_log_)); + + return handle (rq, "build_log"); + } } } @@ -295,6 +353,7 @@ namespace brep info << "module " << BREP_VERSION_STR << ", libbrep " << LIBBREP_VERSION_STR + << ", libbbot " << LIBBBOT_VERSION_STR << ", libbpkg " << LIBBPKG_VERSION_STR << ", libbutl " << LIBBUTL_VERSION_STR; } diff --git a/mod/options-types b/mod/options-types index 6c2e42f..d9eaf9e 100644 --- a/mod/options-types +++ b/mod/options-types @@ -26,7 +26,6 @@ namespace brep page_menu () = default; page_menu (string b, string l): label (move (b)), link (move (l)) {} }; - } #endif // MOD_OPTIONS_TYPES diff --git a/mod/options.cli b/mod/options.cli index fb86fcd..4dbcd89 100644 --- a/mod/options.cli +++ b/mod/options.cli @@ -18,13 +18,30 @@ namespace brep // class module { + string email + { + "<email>", + "Repository email. This email is used for the \cb{From:} header in + emails send by \cb{brep} (for example, build failure notifications)." + } + + string host + { + "<host>", + "Repository host. It specifies the scheme and the host address (but + not the root path; see \cb{root} below) that will be used whenever + \cb{brep} needs to construct an absolute URL to one of its locations + (for example, a link to a build log that is being send via email)." + } + dir_path root = "/" { "<path>" "Repository root. That is, this is the part of the URL between the host name and the start of the repository. For example, root value - '\cb{/pkg}' means the repository URL is http://example.org/pkg/. - Specify '\cb{/}' to use the web server root (http://example.org/)." + '\cb{/pkg}' means the repository URL is \cb{http://example.org/pkg/}. + Specify '\cb{/}' to use the web server root + (\cb{http://example.org/})." } uint16_t verbosity = 0 @@ -35,56 +52,124 @@ namespace brep } }; - class db + class package_db { - string db-user + string package-db-user { "<user>", - "Database user name. If not specified, then operating system (login) - name is used." + "Package database user name. If not specified, then operating system + (login) name is used." } - string db-password + string package-db-password { "<pass>", - "Database password. If not specified, then login without password is - expected to work." + "Package database password. If not specified, then login without + password is expected to work." } - string db-name = "brep" + string package-db-name = "brep_package" { "<name>", - "Database name. If not specified, then '\cb{brep}' is used by - default." + "Package database name. If not specified, then \cb{brep_package} is + used by default." } - string db-host + string package-db-host { "<host>", - "Database host name, address, or socket. If not specified, then + "Package database host name, address, or socket. If not specified, then connect to \cb{localhost} using the operating system-default mechanism (Unix-domain socket, etc)." } - uint16_t db-port = 0 + uint16_t package-db-port = 0 { "<port>", - "Database port number. If not specified, the default port is used." + "Package database port number. If not specified, the default port is + used." } - size_t db-max-connections = 5 + size_t package-db-max-connections = 5 { "<num>", - "The maximum number of concurrent database connections per web server - process. If 0, then no limitation is applied. The default is 5." + "The maximum number of concurrent package database connections per web + server process. If 0, then no limitation is applied. The default is + 5." } - size_t db-retry = 10 + size_t package-db-retry = 10 { "<num>", - "The maximum number of times to retry database transactions in the - face of recoverable failures (deadlock, loss of connection, etc). The - default is 10." + "The maximum number of times to retry package database transactions in + the face of recoverable failures (deadlock, loss of connection, etc). + The default is 10." + } + }; + + class build + { + path build-config + { + "<buildtab>", + "Build configuration file. If not specified, then the package building + functionality will be disabled. If specified, then the build database + must be configured (see \cb{build-db-*})." + } + }; + + class build_db + { + string build-db-user + { + "<user>", + "Build database user name. If not specified, then operating system + (login) name is used." + } + + string build-db-password + { + "<pass>", + "Build database password. If not specified, then login without + password is expected to work." + } + + string build-db-name = "brep_build" + { + "<name>", + "Build database name. If not specified, then \cb{brep_build} is used + by default." + } + + string build-db-host + { + "<host>", + "Build database host name, address, or socket. If not specified, then + connect to \cb{localhost} using the operating system-default + mechanism (Unix-domain socket, etc)." + } + + uint16_t build-db-port = 0 + { + "<port>", + "Build database port number. If not specified, the default port is + used." + } + + size_t build-db-max-connections = 5 + { + "<num>", + "The maximum number of concurrent build database connections per web + server process. If 0, then no limitation is applied. The default is + 5." + } + + size_t build-db-retry = 10 + { + "<num>", + "The maximum number of times to retry build database transactions in + the face of recoverable failures (deadlock, loss of connection, etc). + The default is 10." } }; @@ -141,7 +226,7 @@ namespace brep // Module options. // - class package_search: search, db, page, module + class package_search: search, package_db, page, module { string search-title = "Packages" { @@ -151,21 +236,56 @@ namespace brep } }; - class package_details: package, search, db, page, module + class package_details: package, search, package_db, page, module { }; - class package_version_details: package, db, page, module + class package_version_details: package, package_db, page, module { }; - class repository_details: db, page, module + class repository_details: package_db, page, module { }; class repository_root: module { }; + + class build_task: build, package_db, build_db, module + { + size_t build-task-request-max-size = 102400 + { + "<bytes>", + "The maximum size of the build task request manifest accepted. Note + that the HTTP POST request body is cached to retry database + transactions in the face of recoverable failures (deadlock, loss of + connection, etc). The default is 100K." + } + + size_t build-result-timeout = 10800 + { + "<minutes>", + "Time to wait before considering the expected task result lost. Must be + specified in seconds. The default is 3 hours." + } + }; + + class build_result: build, package_db, build_db, module + { + size_t build-result-request-max-size = 10240000 + { + "<bytes>", + "The maximum size of the build result manifest accepted. Note that the + HTTP POST request body is cached to retry database transactions in the + face of recoverable failures (deadlock, loss of connection, etc). The + default is 10M." + } + }; + + class build_log: build, package_db, build_db, module + { + }; } // Web module HTTP request parameters. @@ -213,5 +333,23 @@ namespace brep // No parameters so far. // }; + + class build_task + { + // No parameters so far. + // + }; + + class build_result + { + // No parameters so far. + // + }; + + class build_log + { + // No parameters so far. + // + }; } } diff --git a/mod/types-parsers b/mod/types-parsers index 1082292..eb206e4 100644 --- a/mod/types-parsers +++ b/mod/types-parsers @@ -25,31 +25,38 @@ namespace brep struct parser; template <> + struct parser<path> + { + static void + parse (path&, bool&, scanner&); + }; + + template <> struct parser<dir_path> { static void - parse (dir_path&, scanner&); + parse (dir_path&, bool&, scanner&); }; template <> struct parser<page_form> { static void - parse (page_form&, scanner&); + parse (page_form&, bool&, scanner&); }; template <> struct parser<page_menu> { static void - parse (page_menu&, scanner&); + parse (page_menu&, bool&, scanner&); }; template <> struct parser<web::xhtml::fragment> { static void - parse (web::xhtml::fragment&, scanner&); + parse (web::xhtml::fragment&, bool&, scanner&); }; } } diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx index 3c51da9..fb293a3 100644 --- a/mod/types-parsers.cxx +++ b/mod/types-parsers.cxx @@ -36,17 +36,26 @@ namespace brep } } + void parser<path>:: + parse (path& x, bool& xs, scanner& s) + { + xs = true; + parse_path (x, s); + } + void parser<dir_path>:: - parse (dir_path& x, scanner& s) + parse (dir_path& x, bool& xs, scanner& s) { + xs = true; parse_path (x, s); } // Parse page_form. // void parser<page_form>:: - parse (page_form& x, scanner& s) + parse (page_form& x, bool& xs, scanner& s) { + xs = true; const char* o (s.next ()); if (!s.more ()) @@ -64,8 +73,9 @@ namespace brep // Parse page_menu. // void parser<page_menu>:: - parse (page_menu& x, scanner& s) + parse (page_menu& x, bool& xs, scanner& s) { + xs = true; const char* o (s.next ()); if (!s.more ()) @@ -92,8 +102,9 @@ namespace brep // Parse web::xhtml::fragment. // void parser<fragment>:: - parse (fragment& x, scanner& s) + parse (fragment& x, bool& xs, scanner& s) { + xs = true; const char* o (s.next ()); if (!s.more ()) diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx index a0ab10f..f2c97b5 100644 --- a/tests/load/driver.cxx +++ b/tests/load/driver.cxx @@ -63,7 +63,7 @@ main (int argc, char* argv[]) string user; string password; - string name ("brep"); + string name ("brep_package"); string host; unsigned int port (0); int i (2); diff --git a/web/apache/request.cxx b/web/apache/request.cxx index f69fedc..b4fb080 100644 --- a/web/apache/request.cxx +++ b/web/apache/request.cxx @@ -29,10 +29,12 @@ #include <algorithm> // min() #include <butl/optional> +#include <butl/timestamp> #include <web/mime-url-encoding> using namespace std; +using namespace butl; namespace web { @@ -519,10 +521,8 @@ namespace web if (max_age) { - chrono::system_clock::time_point tp ( - chrono::system_clock::now () + *max_age); - - time_t t (chrono::system_clock::to_time_t (tp)); + timestamp tp (timestamp::clock::now () + *max_age); + time_t t (timestamp::clock::to_time_t (tp)); // Assume global locale is not changed and still "C". // |