diff options
Diffstat (limited to 'mod')
| -rw-r--r-- | mod/build-config | 23 | ||||
| -rw-r--r-- | mod/build-config.cxx | 31 | ||||
| -rw-r--r-- | mod/buildfile | 7 | ||||
| -rw-r--r-- | mod/database | 15 | ||||
| -rw-r--r-- | mod/database-module | 24 | ||||
| -rw-r--r-- | mod/database-module.cxx | 52 | ||||
| -rw-r--r-- | mod/database.cxx | 59 | ||||
| -rw-r--r-- | mod/mod-build-log | 45 | ||||
| -rw-r--r-- | mod/mod-build-log.cxx | 220 | ||||
| -rw-r--r-- | mod/mod-build-result | 42 | ||||
| -rw-r--r-- | mod/mod-build-result.cxx | 303 | ||||
| -rw-r--r-- | mod/mod-build-task | 42 | ||||
| -rw-r--r-- | mod/mod-build-task.cxx | 389 | ||||
| -rw-r--r-- | mod/mod-package-details.cxx | 14 | ||||
| -rw-r--r-- | mod/mod-package-search.cxx | 22 | ||||
| -rw-r--r-- | mod/mod-package-version-details.cxx | 6 | ||||
| -rw-r--r-- | mod/mod-repository-details.cxx | 6 | ||||
| -rw-r--r-- | mod/mod-repository-root | 6 | ||||
| -rw-r--r-- | mod/mod-repository-root.cxx | 91 | ||||
| -rw-r--r-- | mod/options-types | 1 | ||||
| -rw-r--r-- | mod/options.cli | 192 | ||||
| -rw-r--r-- | mod/types-parsers | 15 | ||||
| -rw-r--r-- | mod/types-parsers.cxx | 19 |
23 files changed, 1511 insertions, 113 deletions
diff --git a/mod/build-config b/mod/build-config new file mode 100644 index 0000000..a5713d7 --- /dev/null +++ b/mod/build-config @@ -0,0 +1,23 @@ +// file : mod/build-config -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_BUILD_CONFIG +#define MOD_BUILD_CONFIG + +#include <bbot/build-config> + +#include <brep/types> +#include <brep/utility> + +namespace brep +{ + // Return pointer to the shared build configurations instance, creating one + // on the first call. Throw tab_parsing on parsing error, io_error on the + // underlying OS error. Is not thread-safe. + // + shared_ptr<const bbot::build_configs> + shared_build_config (const path&); +} + +#endif // MOD_BUILD_CONFIG diff --git a/mod/build-config.cxx b/mod/build-config.cxx new file mode 100644 index 0000000..11be1b9 --- /dev/null +++ b/mod/build-config.cxx @@ -0,0 +1,31 @@ +// file : mod/build-config.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/build-config> + +#include <map> + +namespace brep +{ + using namespace bbot; + + shared_ptr<const build_configs> + shared_build_config (const path& p) + { + static std::map<path, weak_ptr<build_configs>> configs; + + auto i (configs.find (p)); + if (i != configs.end ()) + { + if (shared_ptr<build_configs> c = i->second.lock ()) + return c; + } + + shared_ptr<build_configs> c ( + make_shared<build_configs> (parse_buildtab (p))); + + configs[p] = c; + return c; + } +} diff --git a/mod/buildfile b/mod/buildfile index 228c149..2926a1b 100644 --- a/mod/buildfile +++ b/mod/buildfile @@ -14,13 +14,18 @@ import libs += libodb%lib{odb} import libs += libodb-pgsql%lib{odb-pgsql} import libs += libbutl%lib{butl} import libs += libbpkg%lib{bpkg} +import libs += libbbot%lib{bbot} include ../brep/ mod{brep}: \ + {hxx cxx}{ build-config } \ {hxx cxx}{ database } \ {hxx cxx}{ database-module } \ {hxx cxx}{ diagnostics } \ + {hxx cxx}{ mod-build-log } \ + {hxx cxx}{ mod-build-result } \ + {hxx cxx}{ mod-build-task } \ {hxx cxx}{ mod-package-details } \ {hxx cxx}{ mod-package-search } \ {hxx cxx}{ mod-package-version-details } \ @@ -57,7 +62,7 @@ if $cli.configured # parameters uniformly with a single catch block. # cli.options += --std c++11 -I $src_root --include-with-brackets \ ---include-prefix mod --guard-prefix MOD \ +--include-prefix mod --guard-prefix MOD --generate-specifier \ --cxx-prologue "#include <mod/types-parsers>" \ --cli-namespace brep::cli --generate-file-scanner --suppress-usage \ --generate-modifier --generate-description --option-prefix "" diff --git a/mod/database b/mod/database index 8e9fdd1..9a83752 100644 --- a/mod/database +++ b/mod/database @@ -10,17 +10,18 @@ #include <brep/types> #include <brep/utility> -#include <mod/options> - namespace brep { - // Returns pointer to the shared database instance, creating one on the - // first call. On subsequent calls ensures passed host and port equals - // to ones of the existing database instance throwing runtime_error - // otherwise. Is not thread-safe. + // Return pointer to the shared database instance, creating one on the first + // call. Throw odb::exception on failure. Is not thread-safe. // shared_ptr<odb::core::database> - shared_database (const options::db&); + shared_database (string user, + string password, + string name, + string host, + uint16_t port, + size_t max_connections); } #endif // MOD_DATABASE diff --git a/mod/database-module b/mod/database-module index 034324b..3799e7b 100644 --- a/mod/database-module +++ b/mod/database-module @@ -10,6 +10,8 @@ #include <brep/types> #include <brep/utility> +#include <bbot/build-config> + #include <mod/module> #include <mod/options> @@ -36,15 +38,31 @@ namespace brep // using module::init; + // Initialize the package database instance. Throw odb::exception on + // failure. + // + void + init (const options::package_db&, size_t retry); + + // Initialize the build database instance and parse build configuration + // file. Throw odb::exception on database failure, tab_parsing on parsing + // error, system_error on the underlying OS error. + // void - init (const options::db&); + init (const options::build&, const options::build_db&, size_t retry); virtual bool handle (request&, response&) = 0; protected: - size_t retry_; - shared_ptr<odb::core::database> db_; + size_t retry_ = 0; // Max of all retries. + + shared_ptr<odb::core::database> package_db_; + + // These are NULL if not building. + // + shared_ptr<odb::core::database> build_db_; + shared_ptr<const bbot::build_configs> build_conf_; private: virtual bool diff --git a/mod/database-module.cxx b/mod/database-module.cxx index 6fd5025..e7a6883 100644 --- a/mod/database-module.cxx +++ b/mod/database-module.cxx @@ -4,13 +4,23 @@ #include <mod/database-module> +#include <errno.h> // EIO + +#include <sstream> + #include <odb/exceptions.hxx> +#include <butl/utility> // throw_generic_error() + #include <mod/options> #include <mod/database> +#include <mod/build-config> namespace brep { + using namespace std; + using namespace butl; + // While currently the user-defined copy constructor is not required (we // don't need to deep copy nullptr's), it is a good idea to keep the // placeholder ready for less trivial cases. @@ -19,15 +29,49 @@ namespace brep database_module (const database_module& r) : module (r), retry_ (r.retry_), - db_ (r.initialized_ ? r.db_ : nullptr) + package_db_ (r.initialized_ ? r.package_db_ : nullptr), + build_db_ (r.initialized_ ? r.build_db_ : nullptr), + build_conf_ (r.initialized_ ? r.build_conf_ : nullptr) + { + } + + void database_module:: + init (const options::package_db& o, size_t retry) { + package_db_ = shared_database (o.package_db_user (), + o.package_db_password (), + o.package_db_name (), + o.package_db_host (), + o.package_db_port (), + o.package_db_max_connections ()); + + retry_ = retry_ < retry ? retry : retry_; } void database_module:: - init (const options::db& o) + init (const options::build& bo, const options::build_db& dbo, size_t retry) { - retry_ = o.db_retry (); - db_ = shared_database (o); + try + { + build_conf_ = shared_build_config (bo.build_config ()); + } + catch (const io_error& e) + { + ostringstream os; + os << "unable to read build configuration '" << bo.build_config () + << "': " << e; + + throw_generic_error (EIO, os.str ().c_str ()); + } + + build_db_ = shared_database (dbo.build_db_user (), + dbo.build_db_password (), + dbo.build_db_name (), + dbo.build_db_host (), + dbo.build_db_port (), + dbo.build_db_max_connections ()); + + retry_ = retry_ < retry ? retry : retry_; } bool database_module:: diff --git a/mod/database.cxx b/mod/database.cxx index f8fa1e5..22a0563 100644 --- a/mod/database.cxx +++ b/mod/database.cxx @@ -11,30 +11,43 @@ namespace brep { - namespace options + struct db_key { - bool - operator< (const db& x, const db& y) - { - int r; - if ((r = x.db_user ().compare (y.db_user ())) != 0 || - (r = x.db_password ().compare (y.db_password ())) != 0 || - (r = x.db_name ().compare (y.db_name ())) != 0 || - (r = x.db_host ().compare (y.db_host ()))) - return r < 0; - - return x.db_port () < y.db_port (); - } + string user; + string password; + string name; + string host; + uint16_t port; + }; + + static bool + operator< (const db_key& x, const db_key& y) + { + int r; + if ((r = x.user.compare (y.user)) != 0 || + (r = x.password.compare (y.password)) != 0 || + (r = x.name.compare (y.name)) != 0 || + (r = x.host.compare (y.host))) + return r < 0; + + return x.port < y.port; } using namespace odb; shared_ptr<database> - shared_database (const options::db& o) + shared_database (string user, + string password, + string name, + string host, + uint16_t port, + size_t max_connections) { - static std::map<options::db, weak_ptr<database>> databases; + static std::map<db_key, weak_ptr<database>> databases; + + db_key k ({move (user), move (password), move (name), host, port}); - auto i (databases.find (o)); + auto i (databases.find (k)); if (i != databases.end ()) { if (shared_ptr<database> d = i->second.lock ()) @@ -42,19 +55,19 @@ namespace brep } unique_ptr<pgsql::connection_factory> - f (new pgsql::connection_pool_factory (o.db_max_connections ())); + f (new pgsql::connection_pool_factory (max_connections)); shared_ptr<database> d ( make_shared<pgsql::database> ( - o.db_user (), - o.db_password (), - o.db_name (), - o.db_host (), - o.db_port (), + k.user, + k.password, + k.name, + k.host, + k.port, "options='-c default_transaction_isolation=serializable'", move (f))); - databases[o] = d; + databases[move (k)] = d; return d; } } diff --git a/mod/mod-build-log b/mod/mod-build-log new file mode 100644 index 0000000..8395546 --- /dev/null +++ b/mod/mod-build-log @@ -0,0 +1,45 @@ +// file : mod/mod-build-log -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_BUILD_LOG +#define MOD_MOD_BUILD_LOG + +#include <brep/types> +#include <brep/utility> + +#include <mod/options> +#include <mod/database-module> + +namespace brep +{ + class build_log: public database_module + { + public: + build_log () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + build_log (const build_log&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const + { + return options::build_log::description (); + } + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::build_log> options_; + }; +} + +#endif // MOD_MOD_BUILD_LOG diff --git a/mod/mod-build-log.cxx b/mod/mod-build-log.cxx new file mode 100644 index 0000000..5342e3e --- /dev/null +++ b/mod/mod-build-log.cxx @@ -0,0 +1,220 @@ +// file : mod/mod-build-log.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-build-log> + +#include <algorithm> // find_if() + +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <web/module> + +#include <brep/build> +#include <brep/build-odb> +#include <brep/package> +#include <brep/package-odb> + +#include <mod/options> + +using namespace std; +using namespace bbot; +using namespace brep::cli; +using namespace odb::core; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::build_log:: +build_log (const build_log& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::build_log:: +init (scanner& s) +{ + MODULE_DIAG; + + options_ = make_shared<options::build_log> ( + s, unknown_mode::fail, unknown_mode::fail); + + database_module::init (static_cast<options::package_db> (*options_), + options_->package_db_retry ()); + + if (options_->build_config_specified ()) + database_module::init (static_cast<options::build> (*options_), + static_cast<options::build_db> (*options_), + options_->build_db_retry ()); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); +} + +bool brep::build_log:: +handle (request& rq, response& rs) +{ + using brep::version; // Not to confuse with module::version. + + MODULE_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + // Parse the HTTP request URL path (without the root directory) to obtain + // the build package name/version, the configuration name and the optional + // operation name. If the operation is not specified then print logs for all + // the operations. + // + // Note that the URL path must be in the following form: + // + // <package-name>/<package-version>/log/<config-name>[/<operation>] + // + // Also note that the presence of the first 3 components is guaranteed by + // the repository_root module. + // + build_id id; + string op; + + path lpath (rq.path ().leaf (options_->root ())); + + try + { + auto i (lpath.begin ()); + + assert (i != lpath.end ()); + string name (*i++); + + if (name.empty ()) + throw invalid_argument ("empty package name"); + + assert (i != lpath.end ()); + + version version; + + // Intercept exception handling to add the parsing error attribution. + // + try + { + version = brep::version (*i++); + } + catch (const invalid_argument& e) + { + throw invalid_argument ( + string ("invalid package version: ") + e.what ()); + } + + assert (i != lpath.end () && *i == "log"); + + if (++i == lpath.end ()) + throw invalid_argument ("no configuration name"); + + id = build_id (package_id (move (name), version), *i++); + + if (id.configuration.empty ()) + throw invalid_argument ("empty configuration name"); + + if (i != lpath.end ()) + op = *i++; + + if (i != lpath.end ()) + throw invalid_argument ("unexpected path component"); + } + catch (const invalid_argument& e) + { + throw invalid_request (400, e.what ()); + } + + // Make sure no parameters passed. + // + try + { + name_value_scanner s (rq.parameters ()); + params::build_log (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + // If the package build configuration expired (no such configuration, + // package, etc), then we log this case with the trace severity and respond + // with the 404 HTTP code (not found but may be available in the future). + // The thinking is that this may be or may not be a problem with the + // controller's setup (expires too fast or the link from some ancient email + // is opened). + // + auto config_expired = [&trace, &lpath, this] (const string& d) + { + l2 ([&]{trace << "package build configuration for " << lpath + << " expired: " << d;}); + + throw invalid_request (404, "package build configuration expired: " + d); + }; + + // Make sure the build configuration still exists. + // + auto i ( + find_if ( + build_conf_->begin (), build_conf_->end (), + [&id] (const build_config& c) {return c.name == id.configuration;})); + + if (i == build_conf_->end ()) + config_expired ("no configuration"); + + // Make sure the package still exists. + // + { + transaction t (package_db_->begin ()); + shared_ptr<package> p (package_db_->find<package> (id.package)); + t.commit (); + + if (p == nullptr) + config_expired ("no package"); + } + + // Load the package build configuration (if present). + // + shared_ptr<build> b; + { + transaction t (build_db_->begin ()); + b = build_db_->find<build> (id); + + if (b == nullptr) + config_expired ("no package configuration"); + else if (b->state != build_state::tested) + config_expired ("state is " + to_string (b->state)); + else + build_db_->load (*b, b->results_section); + + t.commit (); + } + + // We have all the data so don't buffer the response content. + // + ostream& os (rs.content (200, "text/plain;charset=utf-8", false)); + + if (op.empty ()) + { + for (const auto& r: b->results) + os << r.log; + } + else + { + const operation_results& r (b->results); + + auto i ( + find_if (r.begin (), r.end (), + [&op] (const operation_result& v) {return v.operation == op;})); + + if (i == r.end ()) + config_expired ("no operation"); + + os << i->log; + } + + return true; +} diff --git a/mod/mod-build-result b/mod/mod-build-result new file mode 100644 index 0000000..8afa697 --- /dev/null +++ b/mod/mod-build-result @@ -0,0 +1,42 @@ +// file : mod/mod-build-result -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_BUILD_RESULT +#define MOD_MOD_BUILD_RESULT + +#include <brep/types> +#include <brep/utility> + +#include <mod/options> +#include <mod/database-module> + +namespace brep +{ + class build_result: public database_module + { + public: + build_result () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + build_result (const build_result&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const {return options::build_result::description ();} + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::build_result> options_; + }; +} + +#endif // MOD_MOD_BUILD_RESULT diff --git a/mod/mod-build-result.cxx b/mod/mod-build-result.cxx new file mode 100644 index 0000000..fb6edb5 --- /dev/null +++ b/mod/mod-build-result.cxx @@ -0,0 +1,303 @@ +// file : mod/mod-build-result.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-build-result> + +#include <algorithm> // find_if() + +#include <butl/sendmail> +#include <butl/process-io> +#include <butl/manifest-parser> +#include <butl/manifest-serializer> + +#include <bbot/manifest> + +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <web/module> + +#include <brep/build> +#include <brep/build-odb> +#include <brep/package> +#include <brep/package-odb> + +#include <mod/options> + +using namespace std; +using namespace butl; +using namespace bbot; +using namespace brep::cli; +using namespace odb::core; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::build_result:: +build_result (const build_result& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::build_result:: +init (scanner& s) +{ + MODULE_DIAG; + + options_ = make_shared<options::build_result> ( + s, unknown_mode::fail, unknown_mode::fail); + + database_module::init (static_cast<options::package_db> (*options_), + options_->package_db_retry ()); + + if (options_->build_config_specified ()) + database_module::init (static_cast<options::build> (*options_), + static_cast<options::build_db> (*options_), + options_->build_db_retry ()); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); +} + +bool brep::build_result:: +handle (request& rq, response&) +{ + using brep::version; // Not to confuse with module::version. + + MODULE_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + // Make sure no parameters passed. + // + try + { + name_value_scanner s (rq.parameters ()); + params::build_result (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + result_request_manifest rqm; + + try + { + size_t limit (options_->build_result_request_max_size ()); + manifest_parser p (rq.content (limit, limit), "result_request_manifest"); + rqm = result_request_manifest (p); + } + catch (const manifest_parsing& e) + { + throw invalid_request (400, e.what ()); + } + + // Parse the task response session to obtain the build configuration name, + // and to make sure the session matches the result manifest's package name + // and version. + // + build_id id; + + try + { + const string& s (rqm.session); + + size_t p (s.find ('/')); // End of package name. + + if (p == 0) + throw invalid_argument ("empty package name"); + + if (p == string::npos) + throw invalid_argument ("no package version"); + + string& name (rqm.result.name); + if (name.compare (0, name.size (), s, 0, p) != 0) + throw invalid_argument ("package name mismatch"); + + size_t b (p + 1); // Start of version. + p = s.find ('/', b); // End of version. + + if (p == string::npos) + throw invalid_argument ("no configuration name"); + + version version; + + // Intercept exception handling to add the parsing error attribution. + // + try + { + version = brep::version (string (s, b, p - b)); + } + catch (const invalid_argument& e) + { + throw invalid_argument ( + string ("invalid package version: ") + e.what ()); + } + + if (version != rqm.result.version) + throw invalid_argument ("package version mismatch"); + + id = build_id (package_id (move (name), version), string (s, p + 1)); + + if (id.configuration.empty ()) + throw invalid_argument ("empty configuration name"); + } + catch (const invalid_argument& e) + { + throw invalid_request (400, string ("invalid session: ") + e.what ()); + } + + // If the session expired (no such configuration, package, etc), then we log + // this case with the warning severity and respond with the 200 HTTP code as + // if the session is valid. The thinking is that this is a problem with the + // controller's setup (expires too fast), not with the agent's. + // + auto warn_expired = [&rqm, &warn] (const string& d) + { + warn << "session '" << rqm.session << "' expired: " << d; + }; + + // Make sure the build configuration still exists. + // + auto i ( + find_if ( + build_conf_->begin (), build_conf_->end (), + [&id] (const build_config& c) {return c.name == id.configuration;})); + + if (i == build_conf_->end ()) + { + warn_expired ("no build configuration"); + return true; + } + + // Load the built package (if present). + // + shared_ptr<package> p; + { + transaction t (package_db_->begin ()); + p = package_db_->find<package> (id.package); + t.commit (); + } + + if (p == nullptr) + { + warn_expired ("no package"); + return true; + } + + // Load and update the package build configuration (if present). + // + shared_ptr<build> b; + { + transaction t (build_db_->begin ()); + b = build_db_->find<build> (id); + + if (b == nullptr) + warn_expired ("no package configuration"); + else if (b->state != build_state::testing) + { + warn_expired ("package configuration state is " + to_string (b->state)); + b = nullptr; + } + else + { + b->state = build_state::tested; + b->timestamp = timestamp::clock::now (); + + assert (!b->status); + b->status = rqm.result.status; + + // Need to manually load the lazy-load section prior to changing its + // members and updating the object's persistent state. + // + // @@ TODO: ODB now allows marking a section as loaded so can optimize + // this. + // + build_db_->load (*b, b->results_section); + b->results = move (rqm.result.results); + + build_db_->update (b); + } + + t.commit (); + } + + if (b == nullptr) + return true; // Warning is already logged. + + // Send email to the package owner. + // + try + { + string subj (b->package_name + '/' + b->package_version.string () + ' ' + + b->configuration + " build: " + to_string (*b->status)); + + // If the package email address is not specified, then it is assumed to be + // the same as the project email address. + // + const string& to (p->package_email + ? *p->package_email + : p->email); + + auto print_args = [&trace, this] (const char* args[], size_t n) + { + l2 ([&]{trace << process_args {args, n};}); + }; + + // Redirect the diagnostics to webserver error log. + // + // Note: if using this somewhere else, then need to factor out all this + // exit status handling code. + // + sendmail sm (print_args, + 2, + options_->email (), + subj, + {to}); + + if (b->results.empty ()) + sm.out << "No operations results available." << endl; + else + { + for (const auto& r: b->results) + sm.out << r.operation << ": " << r.status << ", " + << options_->host () << options_->root ().representation () + << b->package_name << '/' << b->package_version << "/log/" + << b->configuration << '/' << r.operation << endl; + } + + sm.out.close (); + + if (!sm.wait ()) + { + diag_record dr (error); + dr << "sendmail "; + + assert (sm.exit); + const process_exit& e (*sm.exit); + + if (e.normal ()) + dr << "exited with code " << static_cast<uint16_t> (e.code ()); + else + { + dr << "terminated abnormally: " << e.description (); + + if (e.core ()) + dr << " (core dumped)"; + } + } + } + // Handle process_error and io_error (both derive from system_error). + // + catch (const system_error& e) + { + error << "sendmail error: " << e; + } + + return true; +} diff --git a/mod/mod-build-task b/mod/mod-build-task new file mode 100644 index 0000000..051357e --- /dev/null +++ b/mod/mod-build-task @@ -0,0 +1,42 @@ +// file : mod/mod-build-task -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_BUILD_TASK +#define MOD_MOD_BUILD_TASK + +#include <brep/types> +#include <brep/utility> + +#include <mod/options> +#include <mod/database-module> + +namespace brep +{ + class build_task: public database_module + { + public: + build_task () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + build_task (const build_task&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const {return options::build_task::description ();} + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr<options::build_task> options_; + }; +} + +#endif // MOD_MOD_BUILD_TASK diff --git a/mod/mod-build-task.cxx b/mod/mod-build-task.cxx new file mode 100644 index 0000000..1cb5386 --- /dev/null +++ b/mod/mod-build-task.cxx @@ -0,0 +1,389 @@ +// file : mod/mod-build-task.cxx -*- C++ -*- +// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd +// license : MIT; see accompanying LICENSE file + +#include <mod/mod-build-task> + +#include <map> +#include <chrono> + +#include <butl/utility> // compare_c_string +#include <butl/filesystem> // path_match() +#include <butl/manifest-parser> +#include <butl/manifest-serializer> + +#include <bbot/manifest> +#include <bbot/build-config> + +#include <odb/database.hxx> +#include <odb/transaction.hxx> + +#include <web/module> + +#include <brep/build> +#include <brep/build-odb> +#include <brep/package> +#include <brep/package-odb> + +#include <mod/options> + +using namespace std; +using namespace butl; +using namespace bbot; +using namespace brep::cli; +using namespace odb::core; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::build_task:: +build_task (const build_task& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::build_task:: +init (scanner& s) +{ + MODULE_DIAG; + + options_ = make_shared<options::build_task> ( + s, unknown_mode::fail, unknown_mode::fail); + + database_module::init (static_cast<options::package_db> (*options_), + options_->package_db_retry ()); + + if (options_->build_config_specified ()) + database_module::init (static_cast<options::build> (*options_), + static_cast<options::build_db> (*options_), + options_->build_db_retry ()); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); +} + +bool brep::build_task:: +handle (request& rq, response& rs) +{ + MODULE_DIAG; + + if (build_db_ == nullptr) + throw invalid_request (501, "not implemented"); + + // Make sure no parameters passed. + // + try + { + name_value_scanner s (rq.parameters ()); + params::build_task (s, unknown_mode::fail, unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + task_request_manifest tqm; + + try + { + size_t limit (options_->build_task_request_max_size ()); + manifest_parser p (rq.content (limit, limit), "task_request_manifest"); + tqm = task_request_manifest (p); + } + catch (const manifest_parsing& e) + { + throw invalid_request (400, e.what ()); + } + + task_response_manifest tsm; + + // Map build configurations to machines that are capable of building them. + // The first matching machine is selected for each configuration. Also + // create the configuration name list for use in database queries. + // + struct config_machine + { + const build_config* config; + const machine_header_manifest* machine; + }; + + using config_machines = map<const char*, config_machine, compare_c_string>; + + cstrings cfg_names; + config_machines cfg_machines; + + for (const auto& c: *build_conf_) + { + for (auto& m: tqm.machines) + { + if (path_match (c.machine_pattern, m.name) && + cfg_machines.insert ( + make_pair (c.name.c_str (), config_machine ({&c, &m}))).second) + cfg_names.push_back (c.name.c_str ()); + } + } + + // Go through packages until we find one that has no build configuration + // present in the database, or has the untested one, or in the testing state + // but expired, or the one, which build failed abnormally and expired. If + // such a package configuration is found then put it into the testing state, + // set the current timestamp and respond with the task for building this + // package configuration. + // + if (!cfg_machines.empty ()) + { + // Calculate the expiration time for package configurations being in the + // testing state or those, which build failed abnormally. + // + timestamp expiration (timestamp::clock::now () - + chrono::seconds (options_->build_result_timeout ())); + + uint64_t expiration_ns ( + std::chrono::duration_cast<std::chrono::nanoseconds> ( + expiration.time_since_epoch ()).count ()); + + // Prepare the package version prepared query. + // + // Note that the number of packages can be large and so, in order not to + // hold locks for too long, we will restrict the number of packages being + // queried in a single transaction. To achieve this we will iterate through + // packages using the OFFSET/LIMIT pair and sort the query result. + // + // Note that this approach can result in missing some packages or + // iterating multiple times over some of them. However there is nothing + // harmful in that: updates are infrequent and missed packages will be + // picked up on the next request. + // + using pkg_query = query<package_version>; + using prep_pkg_query = prepared_query<package_version>; + + size_t offset (0); // See the package version query. + + // Skip external and stub packages. + // + pkg_query pq ((pkg_query::internal_repository.is_not_null () && + compare_version_ne (pkg_query::id.version, + wildcard_version, + true)) + + "ORDER BY" + + pkg_query::id.name + "," + + pkg_query::id.version.epoch + "," + + pkg_query::id.version.canonical_upstream + "," + + pkg_query::id.version.canonical_release + "," + + pkg_query::id.version.revision + + "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50"); + + connection_ptr pkg_conn (package_db_->connection ()); + + prep_pkg_query pkg_prep_query ( + pkg_conn->prepare_query<package_version> ( + "mod-build-task-package-version-query", pq)); + + // Prepare the build prepared query. + // + // Note that we can not query the database for configurations that a + // package was not built with, as the database contains only those package + // configurations that have already been acted upon (initially empty). + // + // This is why we query the database for package configurations that + // should not be built (in the tested state with the build terminated + // normally or not expired, or in the testing state and not expired). + // Having such a list we will select the first build configuration that is + // not in the list (if available) for the response. + // + using bld_query = query<build>; + using prep_bld_query = prepared_query<build>; + + package_id id; // See the build query. + + const auto& qv (bld_query::id.package.version); + + bld_query bq ( + bld_query::id.package.name == bld_query::_ref (id.name) && + + qv.epoch == bld_query::_ref (id.version.epoch) && + qv.canonical_upstream == + bld_query::_ref (id.version.canonical_upstream) && + qv.canonical_release == bld_query::_ref (id.version.canonical_release) && + qv.revision == bld_query::_ref (id.version.revision) && + + bld_query::id.configuration.in_range (cfg_names.begin (), + cfg_names.end ()) && + + ((bld_query::state == "tested" && + ((bld_query::status != "abort" && bld_query::status != "abnormal") || + bld_query::timestamp > expiration_ns)) || + + (bld_query::state == "testing" && + bld_query::timestamp > expiration_ns))); + + connection_ptr bld_conn (build_db_->connection ()); + + prep_bld_query bld_prep_query ( + bld_conn->prepare_query<build> ( + "mod-build-task-package-build-query", bq)); + + while (tsm.session.empty ()) + { + // Start the package database transaction. + // + transaction pt (pkg_conn->begin ()); + + // Query package versions. + // + auto package_versions (pkg_prep_query.execute ()); + + // Bail out if there is nothing left. + // + if (package_versions.empty ()) + { + pt.commit (); + break; + } + + offset += package_versions.size (); + + // Start the build database transaction. + // + { + transaction bt (bld_conn->begin (), false); + transaction::current (bt); + + // Iterate over packages until we find one that needs building. + // + for (auto& pv: package_versions) + { + id = move (pv.id); + + // Iterate through the package configurations and erase those that + // don't need building from the build configuration map. All those + // configurations that remained can be built. We will take the first + // one, if present. + // + config_machines configs (cfg_machines); // Make a copy for this pkg. + + for (const auto& pc: bld_prep_query.execute ()) + { + auto i (configs.find (pc.id.configuration.c_str ())); + + // Outdated configurations are already excluded with the database + // query. + // + assert (i != configs.end ()); + configs.erase (i); + } + + if (!configs.empty ()) + { + config_machine& cm (configs.begin ()->second); + const build_config& cfg (*cm.config); + + build_id bid (move (id), cfg.name); + shared_ptr<build> b (build_db_->find<build> (bid)); + + // If build configuration doesn't exist then create the new one + // and persist. Otherwise put it into the testing state, refresh + // the timestamp and update. + // + if (b == nullptr) + { + b = make_shared<build> (move (bid.package.name), + move (pv.version), + move (bid.configuration)); + + build_db_->persist (b); + } + else + { + // If the package configuration is in the tested state, then we + // need to cleanup the status and results prior to the update. + // Otherwise the status is already absent and there are no + // results. + // + // Load the section to make sure results are updated for the + // tested state, otherwise assert there are no results. + // + build_db_->load (*b, b->results_section); + + if (b->state == build_state::tested) + { + assert (b->status); + b->status = nullopt; + + b->results.clear (); + } + else + { + assert (!b->status && b->results.empty ()); + } + + b->state = build_state::testing; + b->timestamp = timestamp::clock::now (); + + build_db_->update (b); + } + + // Finally, prepare the task response manifest. + // + tsm.session = b->package_name + '/' + + b->package_version.string () + '/' + b->configuration; + + // @@ We don't support challenge at the moment, so leave it absent. + // + + tsm.result_url = options_->host () + options_->root ().string () + + "?build-result"; + + // Switch to the package database transaction to load the package. + // + transaction::current (pt); + + shared_ptr<package> p (package_db_->load<package> (b->id.package)); + shared_ptr<repository> r (p->internal_repository.load ()); + + // Switch back to the build database transaction. + // + transaction::current (bt); + + strings fp; + if (r->certificate) + fp.emplace_back (move (r->certificate->fingerprint)); + + tsm.task = task_manifest ( + move (b->package_name), + move (b->package_version), + move (r->location), + move (fp), + move (cm.machine->name), + cfg.target, + cfg.vars); + } + + // If task response manifest is filled, then can bail out from the + // package loop, commit transactions and respond. + // + if (!tsm.session.empty ()) + break; + } + + bt.commit (); // Commit the build database transaction. + } + + transaction::current (pt); // Switch to the package database transaction. + pt.commit (); + } + } + + // @@ Probably it would be a good idea to also send some cache control + // headers to avoid caching by HTTP proxies. That would require extension + // of the web::response interface. + // + + manifest_serializer s (rs.content (200, "text/manifest;charset=utf-8"), + "task_response_manifest"); + tsm.serialize (s); + + return true; +} diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx index 95bf087..fd3fd6d 100644 --- a/mod/mod-package-details.cxx +++ b/mod/mod-package-details.cxx @@ -42,7 +42,7 @@ init (scanner& s) options_ = make_shared<options::package_details> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); @@ -148,17 +148,17 @@ handle (request& rq, response& rs) << ~DIV; session sn; - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); shared_ptr<package> pkg; { latest_package lp; - if (!db_->query_one<latest_package> ( + if (!package_db_->query_one<latest_package> ( query<latest_package>( "(" + query<latest_package>::_val (name) + ")"), lp)) throw invalid_request (404, "Package '" + name + "' not found"); - pkg = db_->load<package> (lp.id); + pkg = package_db_->load<package> (lp.id); } const auto& licenses (pkg->license_alternatives); @@ -187,7 +187,7 @@ handle (request& rq, response& rs) } auto pkg_count ( - db_->query_value<package_count> ( + package_db_->query_value<package_count> ( search_params<package_count> (name, squery))); s << FORM_SEARCH (squery) @@ -197,7 +197,7 @@ handle (request& rq, response& rs) // s << DIV; for (const auto& pr: - db_->query<package_search_rank> ( + package_db_->query<package_search_rank> ( search_params<package_search_rank> (name, squery) + "ORDER BY rank DESC, version_epoch DESC, " "version_canonical_upstream DESC, version_canonical_release DESC, " @@ -205,7 +205,7 @@ handle (request& rq, response& rs) "OFFSET" + to_string (page * res_page) + "LIMIT" + to_string (res_page))) { - shared_ptr<package> p (db_->load<package> (pr.id)); + shared_ptr<package> p (package_db_->load<package> (pr.id)); s << TABLE(CLASS="proplist version") << TBODY diff --git a/mod/mod-package-search.cxx b/mod/mod-package-search.cxx index 02b10d6..4988d42 100644 --- a/mod/mod-package-search.cxx +++ b/mod/mod-package-search.cxx @@ -44,14 +44,14 @@ init (scanner& s) options_ = make_shared<options::package_search> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); - // Check that the database schema matches the current one. It's enough to - // perform the check in just a single module implementation (and we don't - // do in the dispatcher because it doesn't use the database). + // Check that the database 'package' schema matches the current one. It's + // enough to perform the check in just a single module implementation (and we + // don't do in the dispatcher because it doesn't use the database). // // Note that the failure can be reported by each web server worker process. // While it could be tempting to move the check to the @@ -59,8 +59,10 @@ init (scanner& s) // be called by a different process (usually the web server root one) not // having the proper permissions to access the database. // - if (schema_catalog::current_version (*db_) != db_->schema_version ()) - fail << "database schema differs from the current one (module " + const string ds ("package"); + if (schema_catalog::current_version (*package_db_, ds) != + package_db_->schema_version (ds)) + fail << "database 'package' schema differs from the current one (module " << BREP_VERSION_STR << ")"; } @@ -133,10 +135,10 @@ handle (request& rq, response& rs) << DIV(ID="content"); session sn; - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); auto pkg_count ( - db_->query_value<latest_package_count> ( + package_db_->query_value<latest_package_count> ( search_param<latest_package_count> (squery))); s << FORM_SEARCH (squery) @@ -146,13 +148,13 @@ handle (request& rq, response& rs) // s << DIV; for (const auto& pr: - db_->query<latest_package_search_rank> ( + package_db_->query<latest_package_search_rank> ( search_param<latest_package_search_rank> (squery) + "ORDER BY rank DESC, name" + "OFFSET" + to_string (page * res_page) + "LIMIT" + to_string (res_page))) { - shared_ptr<package> p (db_->load<package> (pr.id)); + shared_ptr<package> p (package_db_->load<package> (pr.id)); s << TABLE(CLASS="proplist package") << TBODY diff --git a/mod/mod-package-version-details.cxx b/mod/mod-package-version-details.cxx index 89acf89..b0e6ead 100644 --- a/mod/mod-package-version-details.cxx +++ b/mod/mod-package-version-details.cxx @@ -43,7 +43,7 @@ init (scanner& s) options_ = make_shared<options::package_version_details> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); @@ -130,11 +130,11 @@ handle (request& rq, response& rs) shared_ptr<package> pkg; session sn; - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); try { - pkg = db_->load<package> (package_id (name, ver)); + pkg = package_db_->load<package> (package_id (name, ver)); // If the requested package turned up to be an "external" one just // respond that no "internal" package is present. diff --git a/mod/mod-repository-details.cxx b/mod/mod-repository-details.cxx index ab87cb2..3a73955 100644 --- a/mod/mod-repository-details.cxx +++ b/mod/mod-repository-details.cxx @@ -49,7 +49,7 @@ init (scanner& s) options_ = make_shared<options::repository_details> ( s, unknown_mode::fail, unknown_mode::fail); - database_module::init (*options_); + database_module::init (*options_, options_->package_db_retry ()); if (options_->root ().empty ()) options_->root (dir_path ("/")); @@ -90,12 +90,12 @@ handle (request& rq, response& rs) << DIV_HEADER (root, options_->logo (), options_->menu ()) << DIV(ID="content"); - transaction t (db_->begin ()); + transaction t (package_db_->begin ()); using query = query<repository>; for (const auto& r: - db_->query<repository> ( + package_db_->query<repository> ( query::internal + "ORDER BY" + query::priority)) { //@@ Feels like a lot of trouble (e.g., id_attribute()) for very diff --git a/mod/mod-repository-root b/mod/mod-repository-root index 57db93a..6a40e10 100644 --- a/mod/mod-repository-root +++ b/mod/mod-repository-root @@ -17,6 +17,9 @@ namespace brep class package_details; class package_version_details; class repository_details; + class build_task; + class build_result; + class build_log; class repository_root: public module { @@ -55,6 +58,9 @@ namespace brep shared_ptr<package_details> package_details_; shared_ptr<package_version_details> package_version_details_; shared_ptr<repository_details> repository_details_; + shared_ptr<build_task> build_task_; + shared_ptr<build_result> build_result_; + shared_ptr<build_log> build_log_; shared_ptr<options::repository_root> options_; // Sub-module the request is dispatched to. Initially is NULL. It is set diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index 1bc10cb..e156308 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -12,6 +12,9 @@ #include <mod/module> #include <mod/options> +#include <mod/mod-build-log> +#include <mod/mod-build-task> +#include <mod/mod-build-result> #include <mod/mod-package-search> #include <mod/mod-package-details> #include <mod/mod-repository-details> @@ -55,7 +58,10 @@ namespace brep : package_search_ (make_shared<package_search> ()), package_details_ (make_shared<package_details> ()), package_version_details_ (make_shared<package_version_details> ()), - repository_details_ (make_shared<repository_details> ()) + repository_details_ (make_shared<repository_details> ()), + build_task_ (make_shared<build_task> ()), + build_result_ (make_shared<build_result> ()), + build_log_ (make_shared<build_log> ()) { } @@ -83,6 +89,18 @@ namespace brep r.initialized_ ? r.repository_details_ : make_shared<repository_details> (*r.repository_details_)), + build_task_ ( + r.initialized_ + ? r.build_task_ + : make_shared<build_task> (*r.build_task_)), + build_result_ ( + r.initialized_ + ? r.build_result_ + : make_shared<build_result> (*r.build_result_)), + build_log_ ( + r.initialized_ + ? r.build_log_ + : make_shared<build_log> (*r.build_log_)), options_ ( r.initialized_ ? r.options_ @@ -101,6 +119,9 @@ namespace brep append (r, package_details_->options ()); append (r, package_version_details_->options ()); append (r, repository_details_->options ()); + append (r, build_task_->options ()); + append (r, build_result_->options ()); + append (r, build_log_->options ()); return r; } @@ -109,17 +130,38 @@ namespace brep void repository_root:: init (const name_values& v) { - auto sub_init ([this, &v](module& m) + auto sub_init = [this, &v] (module& m, const char* name) + { + // Initialize sub-module. Intercept exception handling to add sub-module + // attribution. + // + try { m.init (filter (v, m.options ()), *log_); - }); + } + catch (const std::exception& e) + { + // Any exception thrown by this function terminates the web server. All + // exception types inherited from std::exception are handled by the web + // server as std::exception. The only sensible way to handle them is to + // log the error prior terminating. By that reason it is valid to + // reduce all these types to a single one. + // + ostringstream os; + os << name << ": " << e; + throw runtime_error (os.str ()); + } + }; // Initialize sub-modules. // - sub_init (*package_search_); - sub_init (*package_details_); - sub_init (*package_version_details_); - sub_init (*repository_details_); + sub_init (*package_search_, "package_search"); + sub_init (*package_details_, "package_details"); + sub_init (*package_version_details_, "package_version_details"); + sub_init (*repository_details_, "repository_details"); + sub_init (*build_task_, "build_task"); + sub_init (*build_result_, "build_result"); + sub_init (*build_log_, "build_log"); // Parse own configuration options. // @@ -155,8 +197,7 @@ namespace brep // Delegate the request handling to the selected sub-module. Intercept // exception handling to add sub-module attribution. // - auto handle = - [&rs, this] (request& rq, const char* name) -> bool + auto handle = [&rs, this] (request& rq, const char* name) -> bool { try { @@ -194,15 +235,24 @@ namespace brep // if (lpath.empty ()) { - // Dispatch request handling to the repository_details or the - // package_search module depending on the function name passed as a - // first HTTP request parameter. The parameter should have no value - // specified. Example: cppget.org/?about + // Dispatch request handling to the repository_details, the build_task, + // the build_result or the package_search module depending on the + // function name passed as a first HTTP request parameter. The parameter + // should have no value specified. Example: cppget.org/?about // const name_values& params (rq.parameters ()); if (!params.empty () && !params.front ().value) { - if (params.front ().name == "about") + // Cleanup not to confuse the selected module with the unknown + // parameter. + // + name_values p (params); + p.erase (p.begin ()); + + request_proxy rp (rq, p); + const string& fn (params.front ().name); + + if (fn == "about") { if (handler_ == nullptr) handler_.reset (new repository_details (*repository_details_)); @@ -236,8 +286,9 @@ namespace brep } else { - // Dispatch request handling to the package_details or the - // package_version_details module depending on the HTTP request URL path. + // Dispatch request handling to the package_details, the + // package_version_details or the build_log module depending on the HTTP + // request URL path. // auto i (lpath.begin ()); assert (i != lpath.end ()); @@ -279,6 +330,13 @@ namespace brep return handle (rq, "package_version_details"); } + else if (*i == "log") + { + if (handler_ == nullptr) + handler_.reset (new build_log (*build_log_)); + + return handle (rq, "build_log"); + } } } @@ -295,6 +353,7 @@ namespace brep info << "module " << BREP_VERSION_STR << ", libbrep " << LIBBREP_VERSION_STR + << ", libbbot " << LIBBBOT_VERSION_STR << ", libbpkg " << LIBBPKG_VERSION_STR << ", libbutl " << LIBBUTL_VERSION_STR; } diff --git a/mod/options-types b/mod/options-types index 6c2e42f..d9eaf9e 100644 --- a/mod/options-types +++ b/mod/options-types @@ -26,7 +26,6 @@ namespace brep page_menu () = default; page_menu (string b, string l): label (move (b)), link (move (l)) {} }; - } #endif // MOD_OPTIONS_TYPES diff --git a/mod/options.cli b/mod/options.cli index fb86fcd..4dbcd89 100644 --- a/mod/options.cli +++ b/mod/options.cli @@ -18,13 +18,30 @@ namespace brep // class module { + string email + { + "<email>", + "Repository email. This email is used for the \cb{From:} header in + emails send by \cb{brep} (for example, build failure notifications)." + } + + string host + { + "<host>", + "Repository host. It specifies the scheme and the host address (but + not the root path; see \cb{root} below) that will be used whenever + \cb{brep} needs to construct an absolute URL to one of its locations + (for example, a link to a build log that is being send via email)." + } + dir_path root = "/" { "<path>" "Repository root. That is, this is the part of the URL between the host name and the start of the repository. For example, root value - '\cb{/pkg}' means the repository URL is http://example.org/pkg/. - Specify '\cb{/}' to use the web server root (http://example.org/)." + '\cb{/pkg}' means the repository URL is \cb{http://example.org/pkg/}. + Specify '\cb{/}' to use the web server root + (\cb{http://example.org/})." } uint16_t verbosity = 0 @@ -35,56 +52,124 @@ namespace brep } }; - class db + class package_db { - string db-user + string package-db-user { "<user>", - "Database user name. If not specified, then operating system (login) - name is used." + "Package database user name. If not specified, then operating system + (login) name is used." } - string db-password + string package-db-password { "<pass>", - "Database password. If not specified, then login without password is - expected to work." + "Package database password. If not specified, then login without + password is expected to work." } - string db-name = "brep" + string package-db-name = "brep_package" { "<name>", - "Database name. If not specified, then '\cb{brep}' is used by - default." + "Package database name. If not specified, then \cb{brep_package} is + used by default." } - string db-host + string package-db-host { "<host>", - "Database host name, address, or socket. If not specified, then + "Package database host name, address, or socket. If not specified, then connect to \cb{localhost} using the operating system-default mechanism (Unix-domain socket, etc)." } - uint16_t db-port = 0 + uint16_t package-db-port = 0 { "<port>", - "Database port number. If not specified, the default port is used." + "Package database port number. If not specified, the default port is + used." } - size_t db-max-connections = 5 + size_t package-db-max-connections = 5 { "<num>", - "The maximum number of concurrent database connections per web server - process. If 0, then no limitation is applied. The default is 5." + "The maximum number of concurrent package database connections per web + server process. If 0, then no limitation is applied. The default is + 5." } - size_t db-retry = 10 + size_t package-db-retry = 10 { "<num>", - "The maximum number of times to retry database transactions in the - face of recoverable failures (deadlock, loss of connection, etc). The - default is 10." + "The maximum number of times to retry package database transactions in + the face of recoverable failures (deadlock, loss of connection, etc). + The default is 10." + } + }; + + class build + { + path build-config + { + "<buildtab>", + "Build configuration file. If not specified, then the package building + functionality will be disabled. If specified, then the build database + must be configured (see \cb{build-db-*})." + } + }; + + class build_db + { + string build-db-user + { + "<user>", + "Build database user name. If not specified, then operating system + (login) name is used." + } + + string build-db-password + { + "<pass>", + "Build database password. If not specified, then login without + password is expected to work." + } + + string build-db-name = "brep_build" + { + "<name>", + "Build database name. If not specified, then \cb{brep_build} is used + by default." + } + + string build-db-host + { + "<host>", + "Build database host name, address, or socket. If not specified, then + connect to \cb{localhost} using the operating system-default + mechanism (Unix-domain socket, etc)." + } + + uint16_t build-db-port = 0 + { + "<port>", + "Build database port number. If not specified, the default port is + used." + } + + size_t build-db-max-connections = 5 + { + "<num>", + "The maximum number of concurrent build database connections per web + server process. If 0, then no limitation is applied. The default is + 5." + } + + size_t build-db-retry = 10 + { + "<num>", + "The maximum number of times to retry build database transactions in + the face of recoverable failures (deadlock, loss of connection, etc). + The default is 10." } }; @@ -141,7 +226,7 @@ namespace brep // Module options. // - class package_search: search, db, page, module + class package_search: search, package_db, page, module { string search-title = "Packages" { @@ -151,21 +236,56 @@ namespace brep } }; - class package_details: package, search, db, page, module + class package_details: package, search, package_db, page, module { }; - class package_version_details: package, db, page, module + class package_version_details: package, package_db, page, module { }; - class repository_details: db, page, module + class repository_details: package_db, page, module { }; class repository_root: module { }; + + class build_task: build, package_db, build_db, module + { + size_t build-task-request-max-size = 102400 + { + "<bytes>", + "The maximum size of the build task request manifest accepted. Note + that the HTTP POST request body is cached to retry database + transactions in the face of recoverable failures (deadlock, loss of + connection, etc). The default is 100K." + } + + size_t build-result-timeout = 10800 + { + "<minutes>", + "Time to wait before considering the expected task result lost. Must be + specified in seconds. The default is 3 hours." + } + }; + + class build_result: build, package_db, build_db, module + { + size_t build-result-request-max-size = 10240000 + { + "<bytes>", + "The maximum size of the build result manifest accepted. Note that the + HTTP POST request body is cached to retry database transactions in the + face of recoverable failures (deadlock, loss of connection, etc). The + default is 10M." + } + }; + + class build_log: build, package_db, build_db, module + { + }; } // Web module HTTP request parameters. @@ -213,5 +333,23 @@ namespace brep // No parameters so far. // }; + + class build_task + { + // No parameters so far. + // + }; + + class build_result + { + // No parameters so far. + // + }; + + class build_log + { + // No parameters so far. + // + }; } } diff --git a/mod/types-parsers b/mod/types-parsers index 1082292..eb206e4 100644 --- a/mod/types-parsers +++ b/mod/types-parsers @@ -25,31 +25,38 @@ namespace brep struct parser; template <> + struct parser<path> + { + static void + parse (path&, bool&, scanner&); + }; + + template <> struct parser<dir_path> { static void - parse (dir_path&, scanner&); + parse (dir_path&, bool&, scanner&); }; template <> struct parser<page_form> { static void - parse (page_form&, scanner&); + parse (page_form&, bool&, scanner&); }; template <> struct parser<page_menu> { static void - parse (page_menu&, scanner&); + parse (page_menu&, bool&, scanner&); }; template <> struct parser<web::xhtml::fragment> { static void - parse (web::xhtml::fragment&, scanner&); + parse (web::xhtml::fragment&, bool&, scanner&); }; } } diff --git a/mod/types-parsers.cxx b/mod/types-parsers.cxx index 3c51da9..fb293a3 100644 --- a/mod/types-parsers.cxx +++ b/mod/types-parsers.cxx @@ -36,17 +36,26 @@ namespace brep } } + void parser<path>:: + parse (path& x, bool& xs, scanner& s) + { + xs = true; + parse_path (x, s); + } + void parser<dir_path>:: - parse (dir_path& x, scanner& s) + parse (dir_path& x, bool& xs, scanner& s) { + xs = true; parse_path (x, s); } // Parse page_form. // void parser<page_form>:: - parse (page_form& x, scanner& s) + parse (page_form& x, bool& xs, scanner& s) { + xs = true; const char* o (s.next ()); if (!s.more ()) @@ -64,8 +73,9 @@ namespace brep // Parse page_menu. // void parser<page_menu>:: - parse (page_menu& x, scanner& s) + parse (page_menu& x, bool& xs, scanner& s) { + xs = true; const char* o (s.next ()); if (!s.more ()) @@ -92,8 +102,9 @@ namespace brep // Parse web::xhtml::fragment. // void parser<fragment>:: - parse (fragment& x, scanner& s) + parse (fragment& x, bool& xs, scanner& s) { + xs = true; const char* o (s.next ()); if (!s.more ()) |