diff options
Diffstat (limited to 'clean')
| -rw-r--r-- | clean/buildfile | 8 | ||||
| -rw-r--r-- | clean/clean.cli | 5 | ||||
| -rw-r--r-- | clean/clean.cxx | 210 |
3 files changed, 155 insertions, 68 deletions
diff --git a/clean/buildfile b/clean/buildfile index a183ff5..b91b1a0 100644 --- a/clean/buildfile +++ b/clean/buildfile @@ -1,5 +1,4 @@ # file : clean/buildfile -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file import libs = libodb%lib{odb} @@ -8,9 +7,14 @@ import libs += libbutl%lib{butl} import libs += libbbot%lib{bbot} include ../libbrep/ +include ../mod/ exe{brep-clean}: {hxx ixx cxx}{* -clean-options} {hxx ixx cxx}{clean-options} \ - ../libbrep/lib{brep} $libs + ../mod/libue{mod} ../libbrep/lib{brep} $libs + +# Build options. +# +obj{clean}: cxx.poptions += -DBREP_COPYRIGHT=\"$copyright\" # Generated options parser. # diff --git a/clean/clean.cli b/clean/clean.cli index 434b32f..d3be4d6 100644 --- a/clean/clean.cli +++ b/clean/clean.cli @@ -1,5 +1,4 @@ // file : clean/clean.cli -// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file include <vector>; @@ -128,8 +127,8 @@ Fatal error.| \li|\cb{2} -An instance of \cb{brep-clean} or \l{brep-migrate(1)} is already running. Try -again.| +An instance of \cb{brep-clean} or some other \cb{brep} utility is already +running. Try again.| \li|\cb{3} diff --git a/clean/clean.cxx b/clean/clean.cxx index a48308b..80c688b 100644 --- a/clean/clean.cxx +++ b/clean/clean.cxx @@ -1,5 +1,4 @@ // file : clean/clean.cxx -*- C++ -*- -// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file #include <map> @@ -13,9 +12,7 @@ #include <odb/pgsql/database.hxx> -#include <libbutl/pager.mxx> - -#include <libbbot/build-config.hxx> +#include <libbutl/pager.hxx> #include <libbrep/build.hxx> #include <libbrep/build-odb.hxx> @@ -25,10 +22,11 @@ #include <libbrep/build-package-odb.hxx> #include <libbrep/database-lock.hxx> +#include <mod/build-target-config.hxx> + #include <clean/clean-options.hxx> using namespace std; -using namespace bbot; using namespace odb::core; namespace brep @@ -62,7 +60,7 @@ namespace brep << "libbbot " << LIBBBOT_VERSION_ID << endl << "libbpkg " << LIBBPKG_VERSION_ID << endl << "libbutl " << LIBBUTL_VERSION_ID << endl - << "Copyright (c) 2014-2019 Code Synthesis Ltd" << endl + << "Copyright (c) " << BREP_COPYRIGHT << "." << endl << "This is free software released under the MIT license." << endl; return 0; @@ -112,8 +110,8 @@ namespace brep ops.db_port (), "options='-c default_transaction_isolation=serializable'"); - // Prevent several brep-clean/migrate instances from updating build - // database simultaneously. + // Prevent several brep utility instances from updating the database + // simultaneously. // database_lock l (db); @@ -206,12 +204,13 @@ namespace brep return 1; } - set<string> configs; + // Load build target configurations. + // + build_target_configs configs; try { - for (auto& c: parse_buildtab (cp)) - configs.emplace (move (c.name)); + configs = bbot::parse_buildtab (cp); } catch (const io_error& e) { @@ -219,6 +218,13 @@ namespace brep return 1; } + // Note: contains shallow references to the configuration targets/names. + // + set<build_target_config_id> configs_set; + + for (const build_target_config& c: configs) + configs_set.insert (build_target_config_id {c.target, c.name}); + // Parse timestamps. // map<string, timestamp> timeouts; // Toolchain timeouts. @@ -260,18 +266,26 @@ namespace brep // // Query package builds in chunks in order not to hold locks for too long. // Sort the result by package version to minimize number of queries to the - // package database. + // package database. Note that we still need to sort by configuration and + // toolchain to make sure that builds are sorted consistently across + // queries and we don't miss any of them. // using bld_query = query<build>; using prep_bld_query = prepared_query<build>; size_t offset (0); bld_query bq ("ORDER BY" + - bld_query::id.package.tenant + "," + - bld_query::id.package.name + + bld_query::id.package.tenant + "," + + bld_query::id.package.name + order_by_version_desc (bld_query::id.package.version, - false) + - "OFFSET" + bld_query::_ref (offset) + "LIMIT 100"); + false) + "," + + bld_query::id.target + "," + + bld_query::id.target_config_name + "," + + bld_query::id.package_config_name + "," + + bld_query::id.toolchain_name + + order_by_version (bld_query::id.toolchain_version, + false /* first */) + + "OFFSET" + bld_query::_ref (offset) + "LIMIT 2000"); connection_ptr conn (db.connection ()); @@ -285,76 +299,143 @@ namespace brep // be made once per tenant package name due to the builds query sorting // criteria (see above). // - using pkg_query = query<buildable_package>; - using prep_pkg_query = prepared_query<buildable_package>; + using pkg_query = query<build_package_version>; + using prep_pkg_query = prepared_query<build_package_version>; string tnt; package_name pkg_name; set<version> package_versions; - pkg_query pq ( - pkg_query::build_package::id.tenant == pkg_query::_ref (tnt) && - pkg_query::build_package::id.name == pkg_query::_ref (pkg_name)); + pkg_query pq (pkg_query::buildable && + pkg_query::id.tenant == pkg_query::_ref (tnt) && + pkg_query::id.name == pkg_query::_ref (pkg_name)); prep_pkg_query pkg_prep_query ( - conn->prepare_query<buildable_package> ("package-query", pq)); + conn->prepare_query<build_package_version> ("package-query", pq)); + + // On the recoverable database error we will retry querying/traversing + // builds in the current chunk, up to 10 times. If we still end up with + // the recoverable database error, then just skip this builds chunk. + // + const size_t max_retries (10); + size_t retry (max_retries); + + // If we fail to erase some builds due to the recoverable database error + // and no builds are erased during this run, then we terminate with the + // exit code 3 (recoverable database error). + // + bool erased (false); + optional<string> re; for (bool ne (true); ne; ) { - transaction t (conn->begin ()); - - // Query builds. - // - auto builds (bld_prep_query.execute ()); + size_t n (0); - if ((ne = !builds.empty ())) + try { - for (const auto& b: builds) - { - auto i (timeouts.find (b.toolchain_name)); + transaction t (conn->begin ()); - timestamp et (i != timeouts.end () - ? i->second - : default_timeout); + // Query builds. + // + auto builds (bld_prep_query.execute ()); - bool cleanup ( - // Check that the build is not stale. - // - b.timestamp <= et || + n = builds.size (); - // Check that the build configuration is still present. - // - // Note that we unable to detect configuration changes and rely on - // periodic rebuilds to take care of that. - // - configs.find (b.configuration) == configs.end ()); + size_t not_erased (0); - // Check that the build package still exists. - // - if (!cleanup) + if ((ne = (n != 0))) + { + for (const auto& b: builds) { - if (tnt != b.tenant || pkg_name != b.package_name) + auto i (timeouts.find (b.toolchain_name)); + + timestamp et (i != timeouts.end () + ? i->second + : default_timeout); + + // Note that we don't consider the case when both the + // configuration and the package still exist but the package now + // excludes the configuration (configuration is now of the legacy + // class instead of the default class, etc). Should we handle this + // case and re-implement in a way brep-monitor does it? Probably + // not since the described situation is not very common and + // storing some extra builds which sooner or later will be wiped + // out due to the timeout is harmless. The current implementation, + // however, is simpler and consumes less resources in runtime + // (doesn't load build package objects, etc). + // + bool cleanup ( + // Check that the build is not stale. + // + b.timestamp <= et || + + // Check that the build configuration is still present. + // + // Note that we unable to detect configuration changes and rely + // on periodic rebuilds to take care of that. + // + configs_set.find ( + build_target_config_id { + b.target, b.target_config_name}) == configs_set.end ()); + + // Check that the build package still exists. + // + if (!cleanup) { - tnt = b.tenant; - pkg_name = b.package_name; - package_versions.clear (); - - for (auto& p: pkg_prep_query.execute ()) - package_versions.emplace (move (p.version)); + if (tnt != b.tenant || pkg_name != b.package_name) + { + tnt = b.tenant; + pkg_name = b.package_name; + package_versions.clear (); + + for (auto& p: pkg_prep_query.execute ()) + package_versions.emplace (move (p.version)); + } + + cleanup = package_versions.find (b.package_version) == + package_versions.end (); } - cleanup = package_versions.find (b.package_version) == - package_versions.end (); + if (cleanup) + db.erase (b); + else + ++not_erased; } + } + + t.commit (); + + if (!erased) + erased = (not_erased != n); + + offset += not_erased; + retry = max_retries; + } + catch (const recoverable& e) + { + // Re-iterate over the current builds chunk, unless there are no more + // attempts left. In the later case stash the error message, if not + // stashed yet, and skip the current builds chunk. + // + if (retry-- == 0) + { + offset += n; + retry = max_retries; - if (cleanup) - db.erase (b); - else - ++offset; + if (!re) + re = e.what (); } + + tnt = ""; + pkg_name = package_name (); + package_versions.clear (); } + } - t.commit (); + if (re && !erased) + { + cerr << "recoverable database error: " << *re << endl; + return 3; } return 0; @@ -452,8 +533,8 @@ namespace brep auto tenant_ids (pq.execute ()); if ((ne = !tenant_ids.empty ())) { - // Cache tenant ids and erase packages, repositories, and tenants at - // once. + // Cache tenant ids and erase packages, repositories, public keys, and + // tenants at once. // strings tids; tids.reserve (tenant_ids.size ()); @@ -469,6 +550,9 @@ namespace brep db.erase_query<repository> ( query<repository>::id.tenant.in_range (tids.begin (), tids.end ())); + db.erase_query<public_key> ( + query<public_key>::id.tenant.in_range (tids.begin (), tids.end ())); + db.erase_query<tenant> ( query<tenant>::id.in_range (tids.begin (), tids.end ())); } |