aboutsummaryrefslogtreecommitdiff
path: root/mod/mod-builds.cxx
diff options
context:
space:
mode:
Diffstat (limited to 'mod/mod-builds.cxx')
-rw-r--r--mod/mod-builds.cxx823
1 files changed, 456 insertions, 367 deletions
diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx
index e749461..30562f3 100644
--- a/mod/mod-builds.cxx
+++ b/mod/mod-builds.cxx
@@ -1,25 +1,25 @@
// file : mod/mod-builds.cxx -*- C++ -*-
-// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
// license : MIT; see accompanying LICENSE file
#include <mod/mod-builds.hxx>
#include <set>
-#include <algorithm> // find_if()
#include <libstudxml/serializer.hxx>
#include <odb/database.hxx>
#include <odb/transaction.hxx>
-#include <libbutl/timestamp.mxx> // to_string()
-#include <libbutl/path-pattern.mxx>
+#include <libbutl/utility.hxx> // compare_c_string
+#include <libbutl/timestamp.hxx> // to_string()
+#include <libbutl/path-pattern.hxx>
#include <libbbot/manifest.hxx> // to_result_status(), to_string(result_status)
-#include <web/xhtml.hxx>
-#include <web/module.hxx>
-#include <web/mime-url-encoding.hxx>
+#include <web/server/module.hxx>
+#include <web/server/mime-url-encoding.hxx>
+
+#include <web/xhtml/serialization.hxx>
#include <libbrep/build.hxx>
#include <libbrep/build-odb.hxx>
@@ -27,11 +27,10 @@
#include <libbrep/build-package-odb.hxx>
#include <mod/page.hxx>
-#include <mod/options.hxx>
+#include <mod/module-options.hxx>
using namespace std;
using namespace butl;
-using namespace bbot;
using namespace web;
using namespace odb::core;
using namespace brep::cli;
@@ -110,7 +109,11 @@ transform (const string& pattern)
case '|':
case '+':
case '{':
- case '(': r += '\\'; break;
+ case '}':
+ case '(':
+ case ')':
+ case '[':
+ case ']': r += '\\'; break;
}
r += c;
@@ -129,28 +132,32 @@ match (const C qc, const string& pattern)
return qc + "SIMILAR TO" + query<T>::_val (transform (pattern));
}
+// If tenant is absent, then query builds from all the public tenants.
+//
template <typename T>
static inline query<T>
-build_query (const brep::cstrings* configs,
+build_query (const brep::vector<brep::build_target_config_id>* config_ids,
const brep::params::builds& params,
- const brep::optional<brep::string>& tenant,
- const brep::optional<bool>& archived)
+ const brep::optional<brep::string>& tenant)
{
using namespace brep;
using query = query<T>;
using qb = typename query::build;
-
- query q (configs != nullptr
- ? qb::id.configuration.in_range (configs->begin (), configs->end ())
- : query (true));
+ using qt = typename query::build_tenant;
const auto& pid (qb::id.package);
- if (tenant)
- q = q && pid.tenant == *tenant;
+ query q (tenant ? pid.tenant == *tenant : !qt::private_);
+
+ if (config_ids != nullptr)
+ {
+ query sq (false);
+ for (const auto& id: *config_ids)
+ sq = sq || (qb::id.target == id.target &&
+ qb::id.target_config_name == id.config);
- if (archived)
- q = q && query::build_tenant::archived == *archived;
+ q = q && sq;
+ }
// Note that there is no error reported if the filter parameters parsing
// fails. Instead, it is considered that no package builds match such a
@@ -169,7 +176,7 @@ build_query (const brep::cstrings* configs,
{
// May throw invalid_argument.
//
- version v (params.version (), false /* fold_zero_revision */);
+ version v (params.version (), version::none);
q = q && compare_version_eq (pid.version,
canonical_version (v),
@@ -178,11 +185,11 @@ build_query (const brep::cstrings* configs,
// Build toolchain name/version.
//
- const string& tc (params.toolchain ());
+ const string& th (params.toolchain ());
- if (tc != "*")
+ if (th != "*")
{
- size_t p (tc.find ('-'));
+ size_t p (th.find ('-'));
if (p == string::npos) // Invalid format.
throw invalid_argument ("");
@@ -190,8 +197,8 @@ build_query (const brep::cstrings* configs,
// the exact version revision, so an absent and zero revisions have the
// same semantics and the zero revision is folded.
//
- string tn (tc, 0, p);
- version tv (string (tc, p + 1)); // May throw invalid_argument.
+ string tn (th, 0, p);
+ version tv (string (th, p + 1)); // May throw invalid_argument.
q = q &&
qb::id.toolchain_name == tn &&
@@ -200,35 +207,44 @@ build_query (const brep::cstrings* configs,
true /* revision */);
}
- // Build configuration name.
+ // Build target.
//
- if (!params.configuration ().empty ())
- q = q && match<T> (qb::id.configuration, params.configuration ());
+ if (!params.target ().empty ())
+ q = q && match<T> (qb::id.target, params.target ());
- // Build machine name.
+ // Build target configuration name.
//
- if (!params.machine ().empty ())
- q = q && match<T> (qb::machine, params.machine ());
+ if (!params.target_config ().empty ())
+ q = q && match<T> (qb::id.target_config_name, params.target_config ());
- // Build target.
+ // Build package configuration name.
//
- if (!params.target ().empty ())
- q = q && match<T> (qb::target, params.target ());
+ if (!params.package_config ().empty ())
+ q = q && match<T> (qb::id.package_config_name, params.package_config ());
// Build result.
//
const string& rs (params.result ());
+ bool add_state (true);
if (rs != "*")
{
if (rs == "pending")
+ {
q = q && qb::force != "unforced";
+ }
else if (rs == "building")
+ {
q = q && qb::state == "building";
+ add_state = false;
+ }
else
{
query sq (qb::status == rs);
- result_status st (to_result_status(rs)); // May throw invalid_argument.
+
+ // May throw invalid_argument.
+ //
+ result_status st (bbot::to_result_status (rs));
if (st != result_status::success)
{
@@ -249,8 +265,12 @@ build_query (const brep::cstrings* configs,
// well (rebuild).
//
q = q && qb::state == "built" && sq;
+ add_state = false;
}
}
+
+ if (add_state)
+ q = q && qb::state != "queued";
}
catch (const invalid_argument&)
{
@@ -260,23 +280,19 @@ build_query (const brep::cstrings* configs,
return q;
}
+// If tenant is absent, then query packages from all the public tenants.
+//
template <typename T>
static inline query<T>
package_query (const brep::params::builds& params,
- const brep::optional<brep::string>& tenant,
- const brep::optional<bool>& archived)
+ const brep::optional<brep::string>& tenant)
{
using namespace brep;
using query = query<T>;
using qp = typename query::build_package;
+ using qt = typename query::build_tenant;
- query q (true);
-
- if (tenant)
- q = q && qp::id.tenant == *tenant;
-
- if (archived)
- q = q && query::build_tenant::archived == *archived;
+ query q (tenant ? qp::id.tenant == *tenant : !qt::private_);
// Note that there is no error reported if the filter parameters parsing
// fails. Instead, it is considered that no packages match such a query.
@@ -294,7 +310,7 @@ package_query (const brep::params::builds& params,
{
// May throw invalid_argument.
//
- version v (params.version (), false /* fold_zero_revision */);
+ version v (params.version (), version::none);
q = q && compare_version_eq (qp::id.version,
canonical_version (v),
@@ -309,22 +325,6 @@ package_query (const brep::params::builds& params,
return q;
}
-template <typename T, typename ID>
-static inline query<T>
-package_id_eq (const ID& x, const brep::package_id& y)
-{
- using query = query<T>;
- const auto& qv (x.version);
-
- return
- x.tenant == query::_ref (y.tenant) &&
- x.name == query::_ref (y.name) &&
- qv.epoch == query::_ref (y.version.epoch) &&
- qv.canonical_upstream == query::_ref (y.version.canonical_upstream) &&
- qv.canonical_release == query::_ref (y.version.canonical_release) &&
- qv.revision == query::_ref (y.version.revision);
-}
-
static const vector<pair<string, string>> build_results ({
{"unbuilt", "<unbuilt>"},
{"*", "*"},
@@ -364,11 +364,6 @@ handle (request& rq, response& rs)
throw invalid_request (400, e.what ());
}
- // Override the name parameter for the old URL (see options.cli for details).
- //
- if (params.name_legacy_specified ())
- params.name (params.name_legacy ());
-
const char* title ("Builds");
xml::serializer s (rs.content (), title);
@@ -392,14 +387,17 @@ handle (request& rq, response& rs)
<< DIV(ID="content");
// If the tenant is empty then we are in the global view and will display
- // builds from all the tenants.
+ // builds from all the public tenants.
//
optional<string> tn;
if (!tenant.empty ())
tn = tenant;
- // Return the list of distinct toolchain name/version pairs. The build db
- // transaction must be started.
+ // Return the list of distinct toolchain name/version pairs. If no builds
+ // are present for the tenant, then fallback to the toolchain recorded in
+ // the tenant object, if present.
+ //
+ // Note: the build db transaction must be started.
//
using toolchains = vector<pair<string, version>>;
@@ -415,11 +413,19 @@ handle (request& rq, response& rs)
false /* first */)))
r.emplace_back (move (t.name), move (t.version));
+ if (r.empty ())
+ {
+ shared_ptr<build_tenant> t (build_db_->find<build_tenant> (tenant));
+
+ if (t != nullptr && t->toolchain)
+ r.emplace_back (t->toolchain->name, t->toolchain->version);
+ }
+
return r;
};
auto print_form = [&s, &params, this] (const toolchains& toolchains,
- size_t build_count)
+ optional<size_t> build_count)
{
// Print the package builds filter form on the first page only.
//
@@ -430,16 +436,16 @@ handle (request& rq, response& rs)
// the selected toolchain is still present in the database. Otherwise
// fallback to the * wildcard selection.
//
- string ctc ("*");
+ string cth ("*");
vector<pair<string, string>> toolchain_opts ({{"*", "*"}});
{
for (const auto& t: toolchains)
{
- string tc (t.first + '-' + t.second.string ());
- toolchain_opts.emplace_back (tc, tc);
+ string th (t.first + '-' + t.second.string ());
+ toolchain_opts.emplace_back (th, th);
- if (tc == params.toolchain ())
- ctc = move (tc);
+ if (th == params.toolchain ())
+ cth = move (th);
}
}
@@ -455,28 +461,42 @@ handle (request& rq, response& rs)
<< TBODY
<< TR_INPUT ("name", "builds", params.name (), "*", true)
<< TR_INPUT ("version", "pv", params.version (), "*")
- << TR_SELECT ("toolchain", "tc", ctc, toolchain_opts)
+ << TR_SELECT ("toolchain", "th", cth, toolchain_opts)
+ << TR_INPUT ("target", "tg", params.target (), "*")
- << TR(CLASS="config")
- << TH << "config" << ~TH
+ << TR(CLASS="tgt-config")
+ << TH << "tgt config" << ~TH
<< TD
<< *INPUT(TYPE="text",
- NAME="cf",
- VALUE=params.configuration (),
+ NAME="tc",
+ VALUE=params.target_config (),
PLACEHOLDER="*",
- LIST="configs")
- << DATALIST(ID="configs")
+ LIST="target-configs")
+ << DATALIST(ID="target-configs")
<< *OPTION(VALUE="*");
- for (const auto& c: *build_conf_names_)
- s << *OPTION(VALUE=c);
+ // Print unique config names from the target config map.
+ //
+ set<const char*, butl::compare_c_string> conf_names;
+ for (const auto& c: *target_conf_map_)
+ {
+ if (conf_names.insert (c.first.config.get ().c_str ()).second)
+ s << *OPTION(VALUE=c.first.config.get ());
+ }
s << ~DATALIST
<< ~TD
<< ~TR
- << TR_INPUT ("machine", "mn", params.machine (), "*")
- << TR_INPUT ("target", "tg", params.target (), "*")
+ << TR(CLASS="pkg-config")
+ << TH << "pkg config" << ~TH
+ << TD
+ << *INPUT(TYPE="text",
+ NAME="pc",
+ VALUE=params.package_config (),
+ PLACEHOLDER="*")
+ << ~TD
+ << ~TR
<< TR_SELECT ("result", "rs", params.result (), build_results)
<< ~TBODY
<< ~TABLE
@@ -498,26 +518,25 @@ handle (request& rq, response& rs)
s << DIV_COUNTER (build_count, "Build", "Builds");
};
+ const string& tgt (params.target ());
+ const string& tgt_cfg (params.target_config ());
+ const string& pkg_cfg (params.package_config ());
+
// We will not display hidden configurations, unless the configuration is
// specified explicitly.
//
- bool exclude_hidden (params.configuration ().empty () ||
- path_pattern (params.configuration ()));
+ bool exclude_hidden (tgt_cfg.empty () || path_pattern (tgt_cfg));
- cstrings conf_names;
+ vector<build_target_config_id> conf_ids;
+ conf_ids.reserve (target_conf_map_->size ());
- if (exclude_hidden)
+ for (const auto& c: *target_conf_map_)
{
- for (const auto& c: *build_conf_map_)
- {
- if (belongs (*c.second, "all"))
- conf_names.push_back (c.first);
- }
+ if (!exclude_hidden || !belongs (*c.second, "hidden"))
+ conf_ids.push_back (c.first);
}
- else
- conf_names = *build_conf_names_;
- size_t count;
+ optional<size_t> count;
size_t page (params.page ());
if (params.result () != "unbuilt") // Print package build configurations.
@@ -532,37 +551,22 @@ handle (request& rq, response& rs)
// printing the builds.
//
count = 0;
- vector<shared_ptr<build>> builds;
+ vector<package_build> builds;
builds.reserve (page_configs);
- // Prepare the package build prepared query.
+ // Prepare the package build query.
//
using query = query<package_build>;
- using prep_query = prepared_query<package_build>;
- query q (build_query<package_build> (
- &conf_names, params, tn, nullopt /* archived */));
-
- // Specify the portion. Note that we will be querying builds in chunks,
- // not to hold locks for too long.
- //
- // Also note that for each build we also load the corresponding
- // package. Nevertheless, we use a fairly large portion to speed-up the
- // builds traversal but also cache the package objects (see below).
- //
- size_t offset (0);
+ query q (build_query<package_build> (&conf_ids, params, tn));
// Print package build configurations ordered by the timestamp (later goes
// first).
//
- q += "ORDER BY" + query::build::timestamp + "DESC" +
- "OFFSET" + query::_ref (offset) + "LIMIT 500";
+ q += "ORDER BY" + query::build::timestamp + "DESC";
connection_ptr conn (build_db_->connection ());
- prep_query pq (
- conn->prepare_query<package_build> ("mod-builds-query", q));
-
// Note that we can't skip the proper number of builds in the database
// query for a page numbers greater than one. So we will query builds from
// the very beginning and skip the appropriate number of them while
@@ -578,81 +582,101 @@ handle (request& rq, response& rs)
//
session sn;
- for (bool ne (true); ne; )
+ transaction t (conn->begin ());
+
+ // For some reason PostgreSQL (as of 9.4) picks the nested loop join
+ // strategy for the below package_build query, which executes quite slow
+ // even for reasonably small number of builds. Thus, we just discourage
+ // PostgreSQL from using this strategy in the current transaction.
+ //
+ // @@ TMP Re-check for the later PostgreSQL versions if we can drop this
+ // hint. If drop, then also grep for other places where this hint
+ // is used.
+ //
+ conn->execute ("SET LOCAL enable_nestloop=off");
+
+ // Iterate over builds and cache build objects that should be printed.
+ // Skip the appropriate number of them (for page number greater than
+ // one).
+ //
+ for (auto& pb: build_db_->query<package_build> (q))
{
- transaction t (conn->begin ());
+ shared_ptr<build>& b (pb.build);
+
+ auto i (
+ target_conf_map_->find (
+ build_target_config_id {b->target, b->target_config_name}));
- // Query package builds (and cache the result).
+ assert (i != target_conf_map_->end ());
+
+ // Match the target configuration against the package build
+ // configuration expressions/constraints.
//
- auto bs (pq.execute ());
+ shared_ptr<build_package> p (
+ build_db_->load<build_package> (b->id.package));
+
+ const build_package_config* pc (find (b->package_config_name,
+ p->configs));
- if ((ne = !bs.empty ()))
+ // The package configuration should be present since the configurations
+ // set cannot change if the package version doesn't change. If that's
+ // not the case, then the database has probably been manually amended.
+ // In this case let's just skip such a build as if it excluded and log
+ // the warning.
+ //
+ if (pc == nullptr)
{
- offset += bs.size ();
+ warn << "cannot find configuration '" << b->package_config_name
+ << "' for package " << p->id.name << '/' << p->version;
- // Iterate over builds and cache build objects that should be printed.
- // Skip the appropriate number of them (for page number greater than
- // one).
- //
- for (auto& pb: bs)
- {
- shared_ptr<build>& b (pb.build);
+ continue;
+ }
- auto i (build_conf_map_->find (b->configuration.c_str ()));
- assert (i != build_conf_map_->end ());
+ if (!p->constraints_section.loaded ())
+ build_db_->load (*p, p->constraints_section);
- // Match the configuration against the package build
- // expressions/constraints.
+ if (!exclude (*pc, p->builds, p->constraints, *i->second))
+ {
+ if (skip != 0)
+ --skip;
+ else if (print != 0)
+ {
+ // As we query builds in multiple transactions we may see the same
+ // build multiple times. Let's skip the duplicates. Note: we don't
+ // increment the counter in this case.
//
- shared_ptr<build_package> p (
- build_db_->load<build_package> (b->id.package));
-
- if (!exclude (p->builds, p->constraints, *i->second))
+ if (find_if (builds.begin (), builds.end (),
+ [&b] (const package_build& pb)
+ {
+ return b->id == pb.build->id;
+ }) != builds.end ())
+ continue;
+
+ if (b->state == build_state::built)
{
- if (skip != 0)
- --skip;
- else if (print != 0)
- {
- // As we query builds in multiple transactions we may see the
- // same build multiple times. Let's skip the duplicates. Note:
- // we don't increment the counter in this case.
- //
- if (find_if (builds.begin (),
- builds.end (),
- [&b] (const shared_ptr<build>& pb)
- {
- return b->id == pb->id;
- }) != builds.end ())
- continue;
-
- if (b->state == build_state::built)
- {
- build_db_->load (*b, b->results_section);
+ build_db_->load (*b, b->results_section);
- // Let's clear unneeded result logs for builds being cached.
- //
- for (operation_result& r: b->results)
- r.log.clear ();
- }
+ // Let's clear unneeded result logs for builds being cached.
+ //
+ for (operation_result& r: b->results)
+ r.log.clear ();
+ }
- builds.push_back (move (b));
+ builds.push_back (move (pb));
- --print;
- }
-
- ++count;
- }
+ --print;
}
+
+ ++(*count);
}
+ }
- // Print the filter form after the build count is calculated. Note:
- // query_toolchains() must be called inside the build db transaction.
- //
- else
- print_form (query_toolchains (), count);
+ // Print the filter form after the build count is calculated. Note:
+ // query_toolchains() must be called inside the build db transaction.
+ //
+ print_form (query_toolchains (), count);
- t.commit ();
- }
+ t.commit ();
// Finally, print the cached package build configurations.
//
@@ -661,34 +685,43 @@ handle (request& rq, response& rs)
// Enclose the subsequent tables to be able to use nth-child CSS selector.
//
s << DIV;
- for (const shared_ptr<build>& pb: builds)
+ for (const package_build& pb: builds)
{
- const build& b (*pb);
+ const build& b (*pb.build);
string ts (butl::to_string (b.timestamp,
"%Y-%m-%d %H:%M:%S %Z",
true /* special */,
true /* local */) +
- " (" + butl::to_string (now - b.timestamp, false) + " ago)");
+ " (" + butl::to_string (now - b.timestamp, false) + " ago");
+
+ if (pb.archived)
+ ts += ", archived";
+
+ ts += ')';
s << TABLE(CLASS="proplist build")
<< TBODY
- << TR_NAME (b.package_name, string (), root, b.tenant)
+ << TR_NAME (b.package_name, root, b.tenant)
<< TR_VERSION (b.package_name, b.package_version, root, b.tenant)
<< TR_VALUE ("toolchain",
b.toolchain_name + '-' +
b.toolchain_version.string ())
- << TR_VALUE ("config", b.configuration)
- << TR_VALUE ("machine", b.machine)
<< TR_VALUE ("target", b.target.string ())
- << TR_VALUE ("timestamp", ts)
- << TR_BUILD_RESULT (b, host, root);
+ << TR_VALUE ("tgt config", b.target_config_name)
+ << TR_VALUE ("pkg config", b.package_config_name)
+ << TR_VALUE ("timestamp", ts);
+
+ if (b.interactive) // Note: can only be present for the building state.
+ s << TR_VALUE ("login", *b.interactive);
+
+ s << TR_BUILD_RESULT (b, pb.archived, host, root);
// In the global view mode add the tenant builds link. Note that the
// global view (and the link) makes sense only in the multi-tenant mode.
//
if (!tn && !b.tenant.empty ())
- s << TR_TENANT (tenant_name, "builds", root, b.tenant);
+ s << TR_TENANT (tenant_name, "builds", root, b.tenant);
s << ~TBODY
<< ~TABLE;
@@ -698,47 +731,73 @@ handle (request& rq, response& rs)
else // Print unbuilt package configurations.
{
// Parameters to use for package build configurations queries. Note that
- // we cleanup the machine and the result filter arguments, as they are
- // irrelevant for unbuilt configurations.
+ // we cleanup the result filter argument, as it is irrelevant for unbuilt
+ // configurations.
//
params::builds bld_params (params);
- bld_params.machine ().clear ();
bld_params.result () = "*";
- // Query toolchains, filter build configurations and toolchains, and
- // create the set of configuration/toolchain combinations, that we will
- // print for packages. Also calculate the number of unbuilt package
- // configurations.
+ // Query toolchains, filter build target configurations and toolchains,
+ // and create the set of target configuration/toolchain combinations, that
+ // we will print for package configurations. Also calculate the number of
+ // unbuilt package configurations.
//
toolchains toolchains;
- // Note that config_toolchains contains shallow references to the
- // toolchain names and versions.
+ // Target configuration/toolchain combination.
+ //
+ // Note: all members are the shallow references.
+ //
+ struct target_config_toolchain
+ {
+ const butl::target_triplet& target;
+ const string& target_config;
+ const string& toolchain_name;
+ const bpkg::version& toolchain_version;
+ };
+
+ // Cache the build package objects that would otherwise be loaded twice:
+ // first time during calculating the builds count and then during printing
+ // the builds. Note that the build package is a subset of the package
+ // object and normally has a small memory footprint.
+ //
+ // @@ TMP It feels that we can try to combine the mentioned steps and
+ // improve the performance a bit. We won't need the session in this
+ // case.
+ //
+ session sn;
+
+ connection_ptr conn (build_db_->connection ());
+ transaction t (conn->begin ());
+
+ // Discourage PostgreSQL from using the nested loop join strategy in the
+ // current transaction (see above for details).
//
- set<config_toolchain> config_toolchains;
+ conn->execute ("SET LOCAL enable_nestloop=off");
+
+ vector<target_config_toolchain> config_toolchains;
{
- transaction t (build_db_->begin ());
toolchains = query_toolchains ();
- string tc_name;
- version tc_version;
- const string& tc (params.toolchain ());
+ string th_name;
+ version th_version;
+ const string& th (params.toolchain ());
- if (tc != "*")
+ if (th != "*")
try
{
- size_t p (tc.find ('-'));
+ size_t p (th.find ('-'));
if (p == string::npos) // Invalid format.
throw invalid_argument ("");
- tc_name.assign (tc, 0, p);
+ th_name.assign (th, 0, p);
// May throw invalid_argument.
//
// Note that an absent and zero revisions have the same semantics,
// so the zero revision is folded (see above for details).
//
- tc_version = version (string (tc, p + 1));
+ th_version = version (string (th, p + 1));
}
catch (const invalid_argument&)
{
@@ -748,63 +807,63 @@ handle (request& rq, response& rs)
throw invalid_request (400, "invalid toolchain");
}
- const string& pc (params.configuration ());
- const string& tg (params.target ());
- vector<const build_config*> configs;
+ vector<const build_target_config*> target_configs;
- for (const auto& c: *build_conf_)
+ for (const auto& c: *target_conf_)
{
- if ((pc.empty () || path_match (c.name, pc)) && // Filter by name.
+ // Filter by name.
+ //
+ if ((tgt_cfg.empty () || path_match (c.name, tgt_cfg)) &&
// Filter by target.
//
- (tg.empty () || path_match (c.target.string (), tg)) &&
+ (tgt.empty () || path_match (c.target.string (), tgt)) &&
- (!exclude_hidden || belongs (c, "all"))) // Filter hidden.
+ (!exclude_hidden || !belongs (c, "hidden"))) // Filter hidden.
{
- configs.push_back (&c);
+ target_configs.push_back (&c);
for (const auto& t: toolchains)
{
// Filter by toolchain.
//
- if (tc == "*" || (t.first == tc_name && t.second == tc_version))
- config_toolchains.insert ({c.name, t.first, t.second});
+ if (th == "*" || (t.first == th_name && t.second == th_version))
+ config_toolchains.push_back (
+ target_config_toolchain {c.target, c.name, t.first, t.second});
}
}
}
- // Calculate the number of unbuilt package configurations as a
- // difference between the maximum possible number of unbuilt
- // configurations and the number of existing package builds.
- //
- // Note that we also need to deduct the package-excluded configurations
- // count from the maximum possible number of unbuilt configurations. The
- // only way to achieve this is to traverse through the packages and
- // match their build expressions/constraints against our configurations.
- //
- // Also note that some existing builds can now be excluded by packages
- // due to the build configuration target or class set change. We should
- // deduct such builds count from the number of existing package builds.
- //
- size_t nmax (
- config_toolchains.size () *
- build_db_->query_value<buildable_package_count> (
- package_query<buildable_package_count> (
- params, tn, false /* archived */)));
-
- size_t ncur = build_db_->query_value<package_build_count> (
- build_query<package_build_count> (
- &conf_names, bld_params, tn, false /* archived */));
-
- // From now we will be using specific package name and version for each
- // build database query.
- //
- bld_params.name ().clear ();
- bld_params.version ().clear ();
-
if (!config_toolchains.empty ())
{
+ // Calculate the number of unbuilt package configurations as a
+ // difference between the possible number of unbuilt configurations
+ // and the number of existing package builds.
+ //
+ // Note that some existing builds can now be excluded by package
+ // configurations due to the build target configuration class set
+ // change. We should deduct such builds count from the number of
+ // existing package configurations builds.
+ //
+ // The only way to calculate both numbers is to traverse through the
+ // package configurations and match their build
+ // expressions/constraints against our target configurations.
+ //
+ size_t npos (0);
+
+ size_t ncur (build_db_->query_value<package_build_count> (
+ build_query<package_build_count> (&conf_ids, bld_params, tn)));
+
+ // From now we will be using specific values for the below filters for
+ // each build database query. Note that the toolchain is the only
+ // filter left in bld_params.
+ //
+ bld_params.name ().clear ();
+ bld_params.version ().clear ();
+ bld_params.target ().clear ();
+ bld_params.target_config ().clear ();
+ bld_params.package_config ().clear ();
+
// Prepare the build count prepared query.
//
// For each package-excluded configuration we will query the number of
@@ -814,59 +873,82 @@ handle (request& rq, response& rs)
using prep_bld_query = prepared_query<package_build_count>;
package_id id;
- string config;
+ target_triplet target;
+ string target_config_name;
+ string package_config_name;
const auto& bid (bld_query::build::id);
bld_query bq (
- package_id_eq<package_build_count> (bid.package, id) &&
- bid.configuration == bld_query::_ref (config) &&
+ equal<package_build_count> (bid.package, id) &&
+ bid.target == bld_query::_ref (target) &&
+ bid.target_config_name == bld_query::_ref (target_config_name) &&
+ bid.package_config_name == bld_query::_ref (package_config_name) &&
// Note that the query already constrains configurations via the
- // configuration name and the tenant via the build package id.
+ // configuration name and target.
//
- build_query<package_build_count> (nullptr /* configs */,
+ // Also note that while the query already constrains the tenant via
+ // the build package id, we still need to pass the tenant not to
+ // erroneously filter out the private tenants.
+ //
+ build_query<package_build_count> (nullptr /* config_ids */,
bld_params,
- nullopt /* tenant */,
- false /* archived */));
+ tn));
prep_bld_query bld_prep_query (
build_db_->prepare_query<package_build_count> (
"mod-builds-build-count-query", bq));
- size_t nt (tc == "*" ? toolchains.size () : 1);
+ // Number of possible builds per package configuration.
+ //
+ size_t nt (th == "*" ? toolchains.size () : 1);
// The number of packages can potentially be large, and we may
// implement some caching in the future. However, the caching will not
// be easy as the cached values depend on the filter form parameters.
//
query<buildable_package> q (
- package_query<buildable_package> (
- params, tn, false /* archived */));
+ package_query<buildable_package> (params, tn));
for (auto& bp: build_db_->query<buildable_package> (q))
{
- id = move (bp.id);
+ shared_ptr<build_package>& p (bp.package);
- shared_ptr<build_package> p (build_db_->load<build_package> (id));
+ id = p->id;
- for (const auto& c: configs)
+ // Note: load the constrains section lazily.
+ //
+ for (const build_package_config& c: p->configs)
{
- if (exclude (p->builds, p->constraints, *c))
+ // Filter by package config name.
+ //
+ if (pkg_cfg.empty () || path_match (c.name, pkg_cfg))
{
- nmax -= nt;
-
- config = c->name;
- ncur -= bld_prep_query.execute_value ();
+ for (const auto& tc: target_configs)
+ {
+ if (!p->constraints_section.loaded ())
+ build_db_->load (*p, p->constraints_section);
+
+ if (exclude (c, p->builds, p->constraints, *tc))
+ {
+ target = tc->target;
+ target_config_name = tc->name;
+ package_config_name = c.name;
+ ncur -= bld_prep_query.execute_value ();
+ }
+ else
+ npos += nt;
+ }
}
}
}
- }
-
- assert (nmax >= ncur);
- count = nmax - ncur;
- t.commit ();
+ assert (npos >= ncur);
+ count = npos - ncur;
+ }
+ else
+ count = nullopt; // Unknown count.
}
// Print the filter form.
@@ -880,9 +962,11 @@ handle (request& rq, response& rs)
// 3: package tenant
// 4: toolchain name
// 5: toolchain version (descending)
- // 6: configuration name
+ // 6: target
+ // 7: target configuration name
+ // 8: package configuration name
//
- // Prepare the build package prepared query.
+ // Prepare the build package query.
//
// Note that we can't skip the proper number of packages in the database
// query for a page numbers greater than one. So we will query packages
@@ -897,28 +981,14 @@ handle (request& rq, response& rs)
// URL query parameter. Alternatively, we can invent the page number cap.
//
using pkg_query = query<buildable_package>;
- using prep_pkg_query = prepared_query<buildable_package>;
- pkg_query pq (
- package_query<buildable_package> (params, tn, false /* archived */));
-
- // Specify the portion. Note that we will still be querying packages in
- // chunks, not to hold locks for too long. For each package we will query
- // its builds, so let's keep the portion small.
- //
- size_t offset (0);
+ pkg_query pq (package_query<buildable_package> (params, tn));
pq += "ORDER BY" +
pkg_query::build_package::id.name +
order_by_version_desc (pkg_query::build_package::id.version,
false /* first */) + "," +
- pkg_query::build_package::id.tenant +
- "OFFSET" + pkg_query::_ref (offset) + "LIMIT 50";
-
- connection_ptr conn (build_db_->connection ());
-
- prep_pkg_query pkg_prep_query (
- conn->prepare_query<buildable_package> ("mod-builds-package-query", pq));
+ pkg_query::build_package::id.tenant;
// Prepare the build prepared query.
//
@@ -932,14 +1002,13 @@ handle (request& rq, response& rs)
package_id id;
- bld_query bq (
- package_id_eq<package_build> (bld_query::build::id.package, id) &&
+ bld_query bq (equal<package_build> (bld_query::build::id.package, id) &&
- // Note that the query already constrains the tenant via the build
- // package id.
- //
- build_query<package_build> (
- &conf_names, bld_params, nullopt /* tenant */, false /* archived */));
+ // Note that while the query already constrains the tenant
+ // via the build package id, we still need to pass the
+ // tenant not to erroneously filter out the private tenants.
+ //
+ build_query<package_build> (&conf_ids, bld_params, tn));
prep_bld_query bld_prep_query (
conn->prepare_query<package_build> ("mod-builds-build-query", bq));
@@ -950,99 +1019,115 @@ handle (request& rq, response& rs)
// Enclose the subsequent tables to be able to use nth-child CSS selector.
//
s << DIV;
- while (print != 0)
- {
- transaction t (conn->begin ());
- // Query (and cache) buildable packages.
- //
- auto packages (pkg_prep_query.execute ());
+ // Query (and cache) buildable packages.
+ //
+ auto packages (build_db_->query<buildable_package> (pq));
- if (packages.empty ())
- print = 0;
- else
+ if (packages.empty ())
+ print = 0;
+ else
+ {
+ // Iterate over packages and print unbuilt configurations. Skip the
+ // appropriate number of them first (for page number greater than one).
+ //
+ for (auto& bp: packages)
{
- offset += packages.size ();
+ shared_ptr<build_package>& p (bp.package);
+
+ id = p->id;
- // Iterate over packages and print unbuilt configurations. Skip the
- // appropriate number of them first (for page number greater than one).
+ // Copy configuration/toolchain combinations for this package,
+ // skipping excluded configurations.
//
- for (auto& p: packages)
- {
- id = move (p.id);
+ set<config_toolchain> unbuilt_configs;
- // Copy configuration/toolchain combinations for this package,
- // skipping excluded configurations.
+ // Load the constrains section lazily.
+ //
+ for (const build_package_config& pc: p->configs)
+ {
+ // Filter by package config name.
//
- set<config_toolchain> unbuilt_configs;
+ if (pkg_cfg.empty () || path_match (pc.name, pkg_cfg))
{
- shared_ptr<build_package> p (build_db_->load<build_package> (id));
-
- for (const auto& ct: config_toolchains)
+ for (const target_config_toolchain& ct: config_toolchains)
{
- auto i (build_conf_map_->find (ct.configuration.c_str ()));
- assert (i != build_conf_map_->end ());
-
- if (!exclude (p->builds, p->constraints, *i->second))
- unbuilt_configs.insert (ct);
+ auto i (
+ target_conf_map_->find (
+ build_target_config_id {ct.target, ct.target_config}));
+
+ assert (i != target_conf_map_->end ());
+
+ if (!p->constraints_section.loaded ())
+ build_db_->load (*p, p->constraints_section);
+
+ if (!exclude (pc, p->builds, p->constraints, *i->second))
+ unbuilt_configs.insert (
+ config_toolchain {ct.target,
+ ct.target_config,
+ pc.name,
+ ct.toolchain_name,
+ ct.toolchain_version});
}
}
+ }
- // Iterate through the package configuration builds and erase them
- // from the unbuilt configurations set.
- //
- for (const auto& pb: bld_prep_query.execute ())
- {
- const build& b (*pb.build);
+ // Iterate through the package configuration builds and erase them
+ // from the unbuilt configurations set.
+ //
+ for (const auto& pb: bld_prep_query.execute ())
+ {
+ const build& b (*pb.build);
- unbuilt_configs.erase ({
- b.id.configuration, b.toolchain_name, b.toolchain_version});
- }
+ unbuilt_configs.erase (config_toolchain {b.target,
+ b.target_config_name,
+ b.package_config_name,
+ b.toolchain_name,
+ b.toolchain_version});
+ }
- // Print unbuilt package configurations.
- //
- for (const auto& ct: unbuilt_configs)
+ // Print unbuilt package configurations.
+ //
+ for (const auto& ct: unbuilt_configs)
+ {
+ if (skip != 0)
{
- if (skip != 0)
- {
- --skip;
- continue;
- }
-
- auto i (build_conf_map_->find (ct.configuration.c_str ()));
- assert (i != build_conf_map_->end ());
-
- s << TABLE(CLASS="proplist build")
- << TBODY
- << TR_NAME (id.name, string (), root, id.tenant)
- << TR_VERSION (id.name, p.version, root, id.tenant)
- << TR_VALUE ("toolchain",
- string (ct.toolchain_name) + '-' +
- ct.toolchain_version.string ())
- << TR_VALUE ("config", ct.configuration)
- << TR_VALUE ("target", i->second->target.string ());
-
- // In the global view mode add the tenant builds link. Note that
- // the global view (and the link) makes sense only in the
- // multi-tenant mode.
- //
- if (!tn && !id.tenant.empty ())
- s << TR_TENANT (tenant_name, "builds", root, id.tenant);
+ --skip;
+ continue;
+ }
- s << ~TBODY
- << ~TABLE;
+ s << TABLE(CLASS="proplist build")
+ << TBODY
+ << TR_NAME (id.name, root, id.tenant)
+ << TR_VERSION (id.name, p->version, root, id.tenant)
+ << TR_VALUE ("toolchain",
+ string (ct.toolchain_name) + '-' +
+ ct.toolchain_version.string ())
+ << TR_VALUE ("target", ct.target.string ())
+ << TR_VALUE ("tgt config", ct.target_config)
+ << TR_VALUE ("pkg config", ct.package_config);
+
+ // In the global view mode add the tenant builds link. Note that
+ // the global view (and the link) makes sense only in the
+ // multi-tenant mode.
+ //
+ if (!tn && !id.tenant.empty ())
+ s << TR_TENANT (tenant_name, "builds", root, id.tenant);
- if (--print == 0) // Bail out the configuration loop.
- break;
- }
+ s << ~TBODY
+ << ~TABLE;
- if (print == 0) // Bail out the package loop.
+ if (--print == 0) // Bail out the configuration loop.
break;
}
- }
- t.commit ();
+ if (print == 0) // Bail out the package loop.
+ break;
+ }
}
+
+ t.commit ();
+
s << ~DIV;
}
@@ -1068,13 +1153,17 @@ handle (request& rq, response& rs)
};
add_filter ("pv", params.version ());
- add_filter ("tc", params.toolchain (), "*");
- add_filter ("cf", params.configuration ());
- add_filter ("mn", params.machine ());
- add_filter ("tg", params.target ());
+ add_filter ("th", params.toolchain (), "*");
+ add_filter ("tg", tgt);
+ add_filter ("tc", tgt_cfg);
+ add_filter ("pc", pkg_cfg);
add_filter ("rs", params.result (), "*");
- s << DIV_PAGER (page, count, page_configs, options_->build_pages (), u)
+ s << DIV_PAGER (page,
+ count ? *count : 0,
+ page_configs,
+ options_->build_pages (),
+ u)
<< ~DIV
<< ~BODY
<< ~HTML;
generated by cgit v1.1 at 2024-04-20 01:22:30 +0000