diff options
Diffstat (limited to 'clean/clean.cxx')
| -rw-r--r-- | clean/clean.cxx | 573 |
1 files changed, 405 insertions, 168 deletions
diff --git a/clean/clean.cxx b/clean/clean.cxx index 102b165..deba1db 100644 --- a/clean/clean.cxx +++ b/clean/clean.cxx @@ -2,7 +2,9 @@ // copyright : Copyright (c) 2014-2018 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file +#include <map> #include <set> +#include <chrono> #include <iostream> #include <odb/database.hxx> @@ -17,6 +19,8 @@ #include <libbrep/build.hxx> #include <libbrep/build-odb.hxx> +#include <libbrep/package.hxx> +#include <libbrep/package-odb.hxx> #include <libbrep/build-package.hxx> #include <libbrep/build-package-odb.hxx> #include <libbrep/database-lock.hxx> @@ -25,226 +29,459 @@ using namespace std; using namespace bbot; -using namespace brep; using namespace odb::core; -// Operation failed, diagnostics has already been issued. -// -struct failed {}; +namespace brep +{ + // Operation failed, diagnostics has already been issued. + // + struct failed {}; -static const char* help_info ( - " info: run 'brep-clean --help' for more information"); + static const char* help_info ( + " info: run 'brep-clean --help' for more information"); -int -main (int argc, char* argv[]) -try -{ - cli::argv_scanner scan (argc, argv, true); - options ops (scan); + static int + clean_builds (const options&, cli::argv_scanner&, odb::pgsql::database&); - // Version. - // - if (ops.version ()) + static int + clean_tenants (const options&, cli::argv_scanner&, odb::pgsql::database&); + + static int + main (int argc, char* argv[]) + try { - cout << "brep-clean " << BREP_VERSION_ID << endl - << "libbrep " << LIBBREP_VERSION_ID << endl - << "libbbot " << LIBBBOT_VERSION_ID << endl - << "libbpkg " << LIBBPKG_VERSION_ID << endl - << "libbutl " << LIBBUTL_VERSION_ID << endl - << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl - << "This is free software released under the MIT license." << endl; + cli::argv_scanner scan (argc, argv, true); + options ops (scan); - return 0; - } + // Version. + // + if (ops.version ()) + { + cout << "brep-clean " << BREP_VERSION_ID << endl + << "libbrep " << LIBBREP_VERSION_ID << endl + << "libbbot " << LIBBBOT_VERSION_ID << endl + << "libbpkg " << LIBBPKG_VERSION_ID << endl + << "libbutl " << LIBBUTL_VERSION_ID << endl + << "Copyright (c) 2014-2018 Code Synthesis Ltd" << endl + << "This is free software released under the MIT license." << endl; + + return 0; + } - // Help. - // - if (ops.help ()) - { - butl::pager p ("brep-clean help", - false, - ops.pager_specified () ? &ops.pager () : nullptr, - &ops.pager_option ()); + // Help. + // + if (ops.help ()) + { + butl::pager p ("brep-clean help", + false, + ops.pager_specified () ? &ops.pager () : nullptr, + &ops.pager_option ()); + + print_usage (p.stream ()); - print_usage (p.stream ()); + // If the pager failed, assume it has issued some diagnostics. + // + return p.wait () ? 0 : 1; + } - // If the pager failed, assume it has issued some diagnostics. + // Detect the mode. // - return p.wait () ? 0 : 1; - } + if (!scan.more ()) + { + cerr << "error: 'builds' or 'tenants' is expected" << endl + << help_info << endl; + return 1; + } - const toolchain_timeouts& timeouts (ops.stale_timeout ()); + const string mode (scan.next ()); - auto i (timeouts.find (string ())); - timestamp default_timeout (i != timeouts.end () - ? i->second - : timestamp_nonexistent); + if (mode != "builds" && mode != "tenants") + throw cli::unknown_argument (mode); - // Load configurations names. - // - if (!scan.more ()) - { - cerr << "error: configuration file expected" << endl - << help_info << endl; - return 1; - } + const string db_schema (mode == "builds" ? "build" : "package"); - set<string> configs; - for (auto& c: parse_buildtab (path (scan.next ()))) - configs.emplace (move (c.name)); + const string& db_name (!ops.db_name ().empty () + ? ops.db_name () + : "brep_" + db_schema); - if (scan.more ()) - { - cerr << "error: unexpected argument encountered" << endl - << help_info << endl; - return 1; - } + odb::pgsql::database db ( + ops.db_user (), + ops.db_password (), + db_name, + ops.db_host (), + ops.db_port (), + "options='-c default_transaction_isolation=serializable'"); - odb::pgsql::database build_db ( - ops.db_user (), - ops.db_password (), - ops.db_name (), - ops.db_host (), - ops.db_port (), - "options='-c default_transaction_isolation=serializable'"); + // Prevent several brep-clean/migrate instances from updating build + // database simultaneously. + // + database_lock l (db); - // Prevent several brep-clean/migrate instances from updating build database - // simultaneously. - // - database_lock l (build_db); + // Check that the database schema matches the current one. + // + if (schema_catalog::current_version (db, db_schema) != + db.schema_version (db_schema)) + { + cerr << "error: " << db_schema << " database schema differs from the " + << "current one" << endl + << " info: use brep-migrate to migrate the database" << endl; + return 1; + } - // Check that the build database schema matches the current one. + return mode == "builds" + ? clean_builds (ops, scan, db) + : clean_tenants (ops, scan, db); + } + catch (const database_locked&) + { + cerr << "brep-clean or brep-migrate is running" << endl; + return 2; + } + catch (const recoverable& e) + { + cerr << "recoverable database error: " << e << endl; + return 3; + } + catch (const cli::exception& e) + { + cerr << "error: " << e << endl << help_info << endl; + return 1; + } + catch (const failed&) + { + return 1; // Diagnostics has already been issued. + } + // Fully qualified to avoid ambiguity with odb exception. // - const string bs ("build"); - if (schema_catalog::current_version (build_db, bs) != - build_db.schema_version (bs)) + catch (const std::exception& e) { - cerr << "error: build database schema differs from the current one" - << endl << " info: use brep-migrate to migrate the database" << endl; + cerr << "error: " << e << endl; return 1; } - // Prepare the build prepared query. + // Convert timeout duration into the time point. Return + // timestamp_nonexistent (never expire) for zero argument. Return nullopt if + // the argument is invalid. // - // Query package builds in chunks in order not to hold locks for too long. - // Sort the result by package version to minimize number of queries to the - // package database. - // - using bld_query = query<build>; - using prep_bld_query = prepared_query<build>; + static optional<timestamp> + timeout (const string& tm) + { + char* e (nullptr); + uint64_t t (strtoull (tm.c_str (), &e, 10)); - size_t offset (0); - bld_query bq ("ORDER BY" + - bld_query::id.package.tenant + "," + - bld_query::id.package.name + - order_by_version_desc (bld_query::id.package.version, false) + - "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + if (*e != '\0' || tm.empty ()) + return nullopt; - connection_ptr conn (build_db.connection ()); + if (t == 0) + return timestamp_nonexistent; - prep_bld_query bld_prep_query ( - conn->prepare_query<build> ("build-query", bq)); + return system_clock::now () - chrono::hours (t); + } - // Prepare the package version query. - // - // Query buildable packages every time the new package name is encountered - // during iterating over the package builds. Such a query will be made once - // per package name due to the builds query sorting criteria (see above). - // - using pkg_query = query<buildable_package>; - using prep_pkg_query = prepared_query<buildable_package>; + static int + clean_builds (const options& ops, + cli::argv_scanner& scan, + odb::pgsql::database& db) + { + // Load configurations names. + // + if (!scan.more ()) + { + cerr << "error: configuration file expected" << endl + << help_info << endl; + return 1; + } - package_name pkg_name; - set<version> package_versions; + path cp; - pkg_query pq ( - pkg_query::build_package::id.name == pkg_query::_ref (pkg_name)); + try + { + cp = path (scan.next ()); + } + catch (const invalid_path& e) + { + cerr << "error: configuration file expected instead of '" << e.path + << "'" << endl + << help_info << endl; + return 1; + } - prep_pkg_query pkg_prep_query ( - conn->prepare_query<buildable_package> ("package-query", pq)); + set<string> configs; - for (bool ne (true); ne; ) - { - transaction t (conn->begin ()); + try + { + for (auto& c: parse_buildtab (cp)) + configs.emplace (move (c.name)); + } + catch (const io_error& e) + { + cerr << "error: unable to read '" << cp << "': " << e << endl; + return 1; + } - // Query builds. + // Parse timestamps. // - auto builds (bld_prep_query.execute ()); + map<string, timestamp> timeouts; // Toolchain timeouts. + timestamp default_timeout; // timestamp_nonexistent - if ((ne = !builds.empty ())) + while (scan.more ()) { - for (const auto& b: builds) + string a (scan.next ()); + + string tc; + optional<timestamp> to; + + size_t p (a.find ('=')); + + if (p == string::npos) + to = timeout (a); + else if (p > 0) // Note: toolchain name can't be empty. { - auto i (timeouts.find (b.toolchain_name)); + tc = string (a, 0, p); + to = timeout (string (a, p + 1)); + } - timestamp et (i != timeouts.end () - ? i->second - : default_timeout); + // Note that the default timeout can't be zero. + // + if (!to || (*to == timestamp_nonexistent && tc.empty ())) + { + cerr << "error: timeout expected instead of '" << a << "'" << endl + << help_info << endl; + return 1; + } - bool cleanup ( - // Check that the build is not stale. - // - b.timestamp <= et || + if (tc.empty ()) + default_timeout = *to; - // Check that the build configuration is still present. - // - // Note that we unable to detect configuration changes and rely on - // periodic rebuilds to take care of that. - // - configs.find (b.configuration) == configs.end ()); + timeouts[move (tc)] = move (*to); + } - // Check that the build package still exists. - // - if (!cleanup) + // Prepare the build prepared query. + // + // Query package builds in chunks in order not to hold locks for too long. + // Sort the result by package version to minimize number of queries to the + // package database. + // + using bld_query = query<build>; + using prep_bld_query = prepared_query<build>; + + size_t offset (0); + bld_query bq ("ORDER BY" + + bld_query::id.package.tenant + "," + + bld_query::id.package.name + + order_by_version_desc (bld_query::id.package.version, + false) + + "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + + connection_ptr conn (db.connection ()); + + prep_bld_query bld_prep_query ( + conn->prepare_query<build> ("build-query", bq)); + + // Prepare the package version query. + // + // Query buildable packages every time the new tenant or package name is + // encountered during iterating over the package builds. Such a query will + // be made once per tenant package name due to the builds query sorting + // criteria (see above). + // + using pkg_query = query<buildable_package>; + using prep_pkg_query = prepared_query<buildable_package>; + + string tnt; + package_name pkg_name; + set<version> package_versions; + + pkg_query pq ( + pkg_query::build_package::id.tenant == pkg_query::_ref (tnt) && + pkg_query::build_package::id.name == pkg_query::_ref (pkg_name)); + + prep_pkg_query pkg_prep_query ( + conn->prepare_query<buildable_package> ("package-query", pq)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + // Query builds. + // + auto builds (bld_prep_query.execute ()); + + if ((ne = !builds.empty ())) + { + for (const auto& b: builds) { - if (pkg_name != b.package_name) - { - pkg_name = b.package_name; - package_versions.clear (); + auto i (timeouts.find (b.toolchain_name)); - for (auto& p: pkg_prep_query.execute ()) - package_versions.emplace (move (p.version)); + timestamp et (i != timeouts.end () + ? i->second + : default_timeout); + + bool cleanup ( + // Check that the build is not stale. + // + b.timestamp <= et || + + // Check that the build configuration is still present. + // + // Note that we unable to detect configuration changes and rely on + // periodic rebuilds to take care of that. + // + configs.find (b.configuration) == configs.end ()); + + // Check that the build package still exists. + // + if (!cleanup) + { + if (tnt != b.tenant || pkg_name != b.package_name) + { + tnt = b.tenant; + pkg_name = b.package_name; + package_versions.clear (); + + for (auto& p: pkg_prep_query.execute ()) + package_versions.emplace (move (p.version)); + } + + cleanup = package_versions.find (b.package_version) == + package_versions.end (); } - cleanup = package_versions.find (b.package_version) == - package_versions.end (); + if (cleanup) + db.erase (b); + else + ++offset; } - - if (cleanup) - build_db.erase (b); - else - ++offset; } + + t.commit (); } - t.commit (); + return 0; } - return 0; -} -catch (const database_locked&) -{ - cerr << "brep-clean or brep-migrate is running" << endl; - return 2; -} -catch (const recoverable& e) -{ - cerr << "recoverable database error: " << e << endl; - return 3; -} -catch (const cli::exception& e) -{ - cerr << "error: " << e << endl << help_info << endl; - return 1; -} -catch (const failed&) -{ - return 1; // Diagnostics has already been issued. + static int + clean_tenants (const options& ops, + cli::argv_scanner& scan, + odb::pgsql::database& db) + { + if (!scan.more ()) + { + cerr << "error: timeout expected" << endl + << help_info << endl; + return 1; + } + + string a (scan.next ()); + optional<timestamp> to (timeout (a)); + + // Note that the timeout can't be zero. + // + if (!to || *to == timestamp_nonexistent) + { + cerr << "error: timeout expected instead of '" << a << "'" << endl + << help_info << endl; + return 1; + } + + if (scan.more ()) + { + cerr << "error: unexpected argument encountered" << endl + << help_info << endl; + return 1; + } + + uint64_t ns ( + chrono::duration_cast<chrono::nanoseconds> ( + to->time_since_epoch ()).count ()); + + // Query tenants in chunks in order not to hold locks for too long. + // + connection_ptr conn (db.connection ()); + + // Archive (rather then delete) old tenants, if requested. + // + if (ops.archive ()) + { + using query = query<tenant>; + using pquery = prepared_query<tenant>; + + query q ((query::creation_timestamp < ns && !query::archived) + + "LIMIT 100"); + + pquery pq (conn->prepare_query<tenant> ("tenant-query", q)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + auto tenants (pq.execute ()); + if ((ne = !tenants.empty ())) + { + for (auto& t: tenants) + { + t.archived = true; + db.update (t); + } + } + + t.commit (); + } + + return 0; + } + + // Delete old tenants. + // + // Note that we don't delete dangling builds for the deleted packages. + // Doing so would require to operate on two databases, complicating the + // code and the utility interface. Note that dangling builds are never + // considered in the web interface and are always deleted with the + // 'brep-clean builds' command. + // + using query = query<tenant_id>; + using pquery = prepared_query<tenant_id>; + + query q ((query::creation_timestamp < ns) + "LIMIT 100"); + pquery pq (conn->prepare_query<tenant_id> ("tenant-id-query", q)); + + for (bool ne (true); ne; ) + { + transaction t (conn->begin ()); + + auto tenant_ids (pq.execute ()); + if ((ne = !tenant_ids.empty ())) + { + // Cache tenant ids and erase packages, repositories, and tenants at + // once. + // + strings tids; + tids.reserve (tenant_ids.size ()); + + for (auto& tid: tenant_ids) + tids.push_back (move (tid.value)); + + using odb::query; + + db.erase_query<package> ( + query<package>::id.tenant.in_range (tids.begin (), tids.end ())); + + db.erase_query<repository> ( + query<repository>::id.tenant.in_range (tids.begin (), tids.end ())); + + db.erase_query<tenant> ( + query<tenant>::id.in_range (tids.begin (), tids.end ())); + } + + t.commit (); + } + + return 0; + } } -// Fully qualified to avoid ambiguity with odb exception. -// -catch (const std::exception& e) + +int +main (int argc, char* argv[]) { - cerr << "error: " << e << endl; - return 1; + return brep::main (argc, argv); } |