From 443088f6093d3420212be0e1af3b9e802dca9362 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Tue, 6 Aug 2024 22:03:31 +0300 Subject: Add support for advanced package search --- etc/brep-module.conf | 1 + etc/private/install/brep-module.conf | 1 + libbrep/package.hxx | 16 +- mod/mod-advanced-search.cxx | 342 +++++++++++++++++++++++++++++++++++ mod/mod-advanced-search.hxx | 41 +++++ mod/mod-builds.cxx | 69 +------ mod/mod-package-details.cxx | 10 +- mod/mod-repository-root.cxx | 23 ++- mod/mod-repository-root.hxx | 2 + mod/module.cli | 47 ++++- mod/utility.cxx | 69 +++++++ mod/utility.hxx | 7 + www/advanced-search-body.css | 84 +++++++++ www/advanced-search.css | 3 + www/advanced-search.scss | 3 + 15 files changed, 644 insertions(+), 74 deletions(-) create mode 100644 mod/mod-advanced-search.cxx create mode 100644 mod/mod-advanced-search.hxx create mode 100644 mod/utility.cxx create mode 100644 www/advanced-search-body.css create mode 100644 www/advanced-search.css create mode 100644 www/advanced-search.scss diff --git a/etc/brep-module.conf b/etc/brep-module.conf index 3963379..560227e 100644 --- a/etc/brep-module.conf +++ b/etc/brep-module.conf @@ -37,6 +37,7 @@ menu Packages= # menu Configs=?build-configs # menu Submit=?submit # menu CI=?ci +# menu Advanced Search=?advanced-search menu About=?about diff --git a/etc/private/install/brep-module.conf b/etc/private/install/brep-module.conf index bad5ede..0c7f065 100644 --- a/etc/private/install/brep-module.conf +++ b/etc/private/install/brep-module.conf @@ -37,6 +37,7 @@ menu Packages= # menu Configs=?build-configs menu Submit=?submit # menu CI=?ci +# menu Advanced Search=?advanced-search menu About=?about diff --git a/libbrep/package.hxx b/libbrep/package.hxx index 668dc5c..e2d2da5 100644 --- a/libbrep/package.hxx +++ b/libbrep/package.hxx @@ -983,6 +983,20 @@ namespace brep search_text (const weighted_text&) {} }; + // Packages count. + // + #pragma db view object(package) + struct package_count + { + size_t result; + + operator size_t () const {return result;} + + // Database mapping. + // + #pragma db member(result) column("count(" + package::id.tenant + ")") + }; + // Package search query matching rank. // #pragma db view query("/*CALL*/ SELECT * FROM search_latest_packages(?)") @@ -1009,7 +1023,7 @@ namespace brep }; #pragma db view query("/*CALL*/ SELECT count(*) FROM search_packages(?)") - struct package_count + struct package_search_count { size_t result; diff --git a/mod/mod-advanced-search.cxx b/mod/mod-advanced-search.cxx new file mode 100644 index 0000000..5de3b9a --- /dev/null +++ b/mod/mod-advanced-search.cxx @@ -0,0 +1,342 @@ +// file : mod/mod-advanced-search.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include + +#include +#include + +#include +#include + +#include + +#include +#include + +#include +#include // wildcard_to_similar_to_pattern() +#include + +using namespace std; +using namespace butl; +using namespace web; +using namespace odb::core; +using namespace brep::cli; + +// While currently the user-defined copy constructor is not required (we don't +// need to deep copy nullptr's), it is a good idea to keep the placeholder +// ready for less trivial cases. +// +brep::advanced_search:: +advanced_search (const advanced_search& r) + : database_module (r), + options_ (r.initialized_ ? r.options_ : nullptr) +{ +} + +void brep::advanced_search:: +init (scanner& s) +{ + HANDLER_DIAG; + + options_ = make_shared ( + s, unknown_mode::fail, unknown_mode::fail); + + database_module::init (*options_, options_->package_db_retry ()); + + if (options_->root ().empty ()) + options_->root (dir_path ("/")); +} + +template +static inline query +match (const C qc, const string& pattern) +{ + return qc + + "SIMILAR TO" + + query::_val (brep::wildcard_to_similar_to_pattern (pattern)); +} + +template +static inline query +package_query (const brep::params::advanced_search& params) +{ + using namespace brep; + using query = query; + + query q (query::internal_repository.canonical_name.is_not_null ()); + + // Note that there is no error reported if the filter parameters parsing + // fails. Instead, it is considered that no package builds match such a + // query. + // + try + { + // Package name. + // + if (!params.name ().empty ()) + q = q && match (query::id.name, params.name ()); + + // Package version. + // + if (!params.version ().empty () && params.version () != "*") + { + // May throw invalid_argument. + // + version v (params.version (), version::none); + + q = q && compare_version_eq (query::id.version, + canonical_version (v), + v.revision.has_value ()); + } + + // Package project. + // + if (!params.project ().empty ()) + q = q && match (query::project, params.project ()); + + // Reviews. + // + const string& rs (params.reviews ()); + + if (rs != "*") + { + if (rs == "reviewed") + q = q && query::reviews.pass.is_not_null (); + else if (rs == "unreviewed") + q = q && query::reviews.pass.is_null (); + else + throw invalid_argument (""); + } + } + catch (const invalid_argument&) + { + return query (false); + } + + return q; +} + +static const vector> reviews ({ + {"*", "*"}, + {"reviewed", "reviewed"}, + {"unreviewed", "unreviewed"}}); + +bool brep::advanced_search:: +handle (request& rq, response& rs) +{ + using namespace web::xhtml; + + HANDLER_DIAG; + + // Note that while we could potentially support the multi-tenant mode, that + // would require to invent the package/tenant view to filter out the private + // tenants from the search. This doesn't look of much use at the moment. + // Thus, let's keep it simple for now and just respond with the 501 status + // code (not implemented) if such a mode is detected. + // + if (!tenant.empty ()) + throw invalid_request (501, "not implemented"); + + const size_t res_page (options_->search_page_entries ()); + const dir_path& root (options_->root ()); + + params::advanced_search params; + + try + { + name_value_scanner s (rq.parameters (8 * 1024)); + params = params::advanced_search (s, + unknown_mode::fail, + unknown_mode::fail); + } + catch (const cli::exception& e) + { + throw invalid_request (400, e.what ()); + } + + const char* title ("Advanced Package Search"); + + xml::serializer s (rs.content (), title); + + s << HTML + << HEAD + << TITLE << title << ~TITLE + << CSS_LINKS (path ("advanced-search.css"), root) + << ~HEAD + << BODY + << DIV_HEADER (options_->logo (), options_->menu (), root, tenant) + << DIV(ID="content"); + + transaction t (package_db_->begin ()); + + size_t count ( + package_db_->query_value ( + package_query (params))); + + // Print the package builds filter form on the first page only. + // + size_t page (params.page ()); + + if (page == 0) + { + // The 'action' attribute is optional in HTML5. While the standard + // doesn't specify browser behavior explicitly for the case the + // attribute is omitted, the only reasonable behavior is to default it + // to the current document URL. + // + s << FORM + << TABLE(ID="filter", CLASS="proplist") + << TBODY + << TR_INPUT ("name", "advanced-search", params.name (), "*", true) + << TR_INPUT ("version", "pv", params.version (), "*") + << TR_INPUT ("project", "pr", params.project (), "*"); + + if (options_->reviews_url_specified ()) + s << TR_SELECT ("reviews", "rv", params.reviews (), reviews); + + s << ~TBODY + << ~TABLE + << TABLE(CLASS="form-table") + << TBODY + << TR + << TD(ID="package-version-count") + << DIV_COUNTER (count, "Package Version", "Package Versions") + << ~TD + << TD(ID="filter-btn") + << *INPUT(TYPE="submit", VALUE="Filter") + << ~TD + << ~TR + << ~TBODY + << ~TABLE + << ~FORM; + } + else + s << DIV_COUNTER (count, "Package Version", "Package Versions"); + + using query = query; + + // Note that we query an additional package version which we will not + // display, but will use to check if it belongs to the same package and/or + // project as the last displayed package version. If that's the case we will + // display the '...' mark(s) at the end of the page, indicating that there a + // more package versions from this package/project on the next page(s). + // + query q (package_query (params) + + "ORDER BY tenant, project, name, version_epoch DESC, " + "version_canonical_upstream DESC, version_canonical_release DESC, " + "version_revision DESC" + + "OFFSET" + to_string (page * res_page) + + "LIMIT" + to_string (res_page + 1)); + + package_name prj; + package_name pkg; + size_t n (0); + + for (package& p: package_db_->query (q)) + { + if (!p.id.tenant.empty ()) + throw invalid_request (501, "not implemented"); + + if (n++ == res_page) + { + if (p.project == prj) + { + if (p.name == pkg) + s << DIV(ID="package-break") << "..." << ~DIV; + + s << DIV(ID="project-break") << "..." << ~DIV; + } + + break; + } + + if (p.project != prj) + { + prj = move (p.project); + pkg = package_name (); + + s << TABLE(CLASS="proplist project") + << TBODY + << TR_VALUE ("project", prj.string ()) + << ~TBODY + << ~TABLE; + } + + if (p.name != pkg) + { + pkg = move (p.name); + + s << TABLE(CLASS="proplist package") + << TBODY + << TR_NAME (pkg, root, p.tenant) + << TR_SUMMARY (p.summary) + << TR_LICENSE (p.license_alternatives) + << ~TBODY + << ~TABLE; + } + + s << TABLE(CLASS="proplist version") + << TBODY + << TR_VERSION (pkg, p.version, root, tenant, p.upstream_version); + + assert (p.internal ()); + + const repository_location& rl (p.internal_repository.load ()->location); + + s << TR_REPOSITORY (rl, root, tenant) + << TR_DEPENDS (p.dependencies, root, tenant) + << TR_REQUIRES (p.requirements); + + if (options_->reviews_url_specified ()) + { + package_db_->load (p, p.reviews_section); + + s << TR_REVIEWS_SUMMARY (p.reviews, options_->reviews_url ()); + } + + s << ~TBODY + << ~TABLE; + } + + t.commit (); + + string u (root.string () + "?advanced-search"); + + if (!params.name ().empty ()) + { + u += '='; + u += mime_url_encode (params.name ()); + } + + auto add_filter = [&u] (const char* pn, + const string& pv, + const char* def = "") + { + if (pv != def) + { + u += '&'; + u += pn; + u += '='; + u += mime_url_encode (pv); + } + }; + + add_filter ("pv", params.version ()); + add_filter ("pr", params.project ()); + add_filter ("rv", params.reviews (), "*"); + + s << DIV_PAGER (page, + count, + res_page, + options_->search_pages (), + u) + << ~DIV + << ~BODY + << ~HTML; + + return true; +} diff --git a/mod/mod-advanced-search.hxx b/mod/mod-advanced-search.hxx new file mode 100644 index 0000000..4ab4d42 --- /dev/null +++ b/mod/mod-advanced-search.hxx @@ -0,0 +1,41 @@ +// file : mod/mod-advanced-search.hxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef MOD_MOD_ADVANCED_SEARCH_HXX +#define MOD_MOD_ADVANCED_SEARCH_HXX + +#include +#include + +#include +#include + +namespace brep +{ + class advanced_search: public database_module + { + public: + advanced_search () = default; + + // Create a shallow copy (handling instance) if initialized and a deep + // copy (context exemplar) otherwise. + // + explicit + advanced_search (const advanced_search&); + + virtual bool + handle (request&, response&); + + virtual const cli::options& + cli_options () const {return options::advanced_search::description ();} + + private: + virtual void + init (cli::scanner&); + + private: + shared_ptr options_; + }; +} + +#endif // MOD_MOD_ADVANCED_SEARCH_HXX diff --git a/mod/mod-builds.cxx b/mod/mod-builds.cxx index 81d4649..0155c2e 100644 --- a/mod/mod-builds.cxx +++ b/mod/mod-builds.cxx @@ -27,6 +27,7 @@ #include #include +#include // wildcard_to_similar_to_pattern() #include using namespace std; @@ -63,71 +64,13 @@ init (scanner& s) options_->root (dir_path ("/")); } -// Transform the wildcard to the SIMILAR TO-pattern. -// -static string -transform (const string& pattern) -{ - if (pattern.empty ()) - return "%"; - - string r; - for (const path_pattern_term& pt: path_pattern_iterator (pattern)) - { - switch (pt.type) - { - case path_pattern_term_type::question: r += '_'; break; - case path_pattern_term_type::star: r += '%'; break; - case path_pattern_term_type::bracket: - { - // Copy the bracket expression translating the inverse character, if - // present. - // - size_t n (r.size ()); - r.append (pt.begin, pt.end); - - if (r[n + 1] == '!') // ...[!... ? - r[n + 1] = '^'; - - break; - } - case path_pattern_term_type::literal: - { - char c (get_literal (pt)); - - // Escape the special characters. - // - // Note that '.' is not a special character for SIMILAR TO. - // - switch (c) - { - case '\\': - case '%': - case '_': - case '|': - case '+': - case '{': - case '}': - case '(': - case ')': - case '[': - case ']': r += '\\'; break; - } - - r += c; - break; - } - } - } - - return r; -} - template static inline query match (const C qc, const string& pattern) { - return qc + "SIMILAR TO" + query::_val (transform (pattern)); + return qc + + "SIMILAR TO" + + query::_val (brep::wildcard_to_similar_to_pattern (pattern)); } // If tenant is absent, then query builds from all the public tenants. @@ -450,9 +393,7 @@ handle (request& rq, response& rs) // The 'action' attribute is optional in HTML5. While the standard // doesn't specify browser behavior explicitly for the case the // attribute is omitted, the only reasonable behavior is to default it - // to the current document URL. Note that we specify the function name - // using the "hidden" element since the action url must not - // contain the query part. + // to the current document URL. // s << FORM << TABLE(ID="filter", CLASS="proplist") diff --git a/mod/mod-package-details.cxx b/mod/mod-package-details.cxx index 15a4115..ceb23c5 100644 --- a/mod/mod-package-details.cxx +++ b/mod/mod-package-details.cxx @@ -119,7 +119,7 @@ handle (request& rq, response& rs) throw invalid_request (400, "invalid package name format"); } - const package_name& name (pkg->name); + const package_name& name (pkg->name); const string ename (mime_url_encode (name.string (), false)); auto url = [&ename] (bool f = false, @@ -226,8 +226,8 @@ handle (request& rq, response& rs) } size_t pkg_count ( - package_db_->query_value ( - search_params (squery, tenant, name))); + package_db_->query_value ( + search_params (squery, tenant, name))); // Let's disable autofocus in the full page mode since clicking the full or // more link the user most likely intends to read rather than search, while @@ -244,8 +244,8 @@ handle (request& rq, response& rs) search_params (squery, tenant, name) + "ORDER BY rank DESC, version_epoch DESC, " "version_canonical_upstream DESC, version_canonical_release DESC, " - "version_revision DESC" + - "OFFSET" + to_string (page * res_page) + + "version_revision DESC" + + "OFFSET" + to_string (page * res_page) + "LIMIT" + to_string (res_page))) { shared_ptr p (package_db_->load (pr.id)); diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx index bc861a8..165302d 100644 --- a/mod/mod-repository-root.cxx +++ b/mod/mod-repository-root.cxx @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -118,6 +119,7 @@ namespace brep // tenant_service_map_ (make_shared ()), packages_ (make_shared ()), + advanced_search_ (make_shared ()), package_details_ (make_shared ()), package_version_details_ (make_shared ()), repository_details_ (make_shared ()), @@ -153,6 +155,10 @@ namespace brep r.initialized_ ? r.packages_ : make_shared (*r.packages_)), + advanced_search_ ( + r.initialized_ + ? r.advanced_search_ + : make_shared (*r.advanced_search_)), package_details_ ( r.initialized_ ? r.package_details_ @@ -225,6 +231,7 @@ namespace brep { option_descriptions r (handler::options ()); append (r, packages_->options ()); + append (r, advanced_search_->options ()); append (r, package_details_->options ()); append (r, package_version_details_->options ()); append (r, repository_details_->options ()); @@ -272,6 +279,7 @@ namespace brep // Initialize sub-handlers. // sub_init (*packages_, "packages"); + sub_init (*advanced_search_, "advanced_search"); sub_init (*package_details_, "package_details"); sub_init (*package_version_details_, "package_version_details"); sub_init (*repository_details_, "repository_details"); @@ -305,7 +313,13 @@ namespace brep auto verify = [&fail] (const string& v, const char* what) { cstrings vs ({ - "packages", "builds", "build-configs", "about", "submit", "ci"}); + "packages", + "advanced-search", + "builds", + "build-configs", + "about", + "submit", + "ci"}); if (find (vs.begin (), vs.end (), v) == vs.end ()) fail << what << " value '" << v << "' is invalid"; @@ -459,6 +473,13 @@ namespace brep return handle ("packages", param); } + else if (func == "advanced-search") + { + if (handler_ == nullptr) + handler_.reset (new advanced_search (*advanced_search_)); + + return handle ("advanced_search", param); + } else if (func == "about") { if (handler_ == nullptr) diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx index 990587e..5a57403 100644 --- a/mod/mod-repository-root.hxx +++ b/mod/mod-repository-root.hxx @@ -14,6 +14,7 @@ namespace brep { class packages; + class advanced_search; class package_details; class package_version_details; class repository_details; @@ -64,6 +65,7 @@ namespace brep shared_ptr tenant_service_map_; shared_ptr packages_; + shared_ptr advanced_search_; shared_ptr package_details_; shared_ptr package_version_details_; shared_ptr repository_details_; diff --git a/mod/module.cli b/mod/module.cli index 7b0c0d2..41684bd 100644 --- a/mod/module.cli +++ b/mod/module.cli @@ -545,6 +545,15 @@ namespace brep } }; + class advanced_search: package_db, + search, + page, + repository_url, + package_version_metadata, + handler + { + }; + class package_details: package, package_db, search, page, @@ -871,11 +880,11 @@ namespace brep // Web handler HTTP request parameters. // + // Use parameters long names in the C++ code, short aliases (if present) in + // HTTP URL. + // namespace params { - // Use parameters long names in the C++ code, short aliases (if present) - // in HTTP URL. - // class packages { // Display package search result list starting from this page. @@ -890,6 +899,38 @@ namespace brep string q | _; }; + class advanced_search + { + // Display advanced package search result list starting from this page. + // + uint16_t page | p; + + // Advanced package search filter options. + // + + // Package name wildcard. An empty value is treated the same way as *. + // + // Note that the advanced-search parameter is renamed to '_' by the root + // handler (see the request_proxy class for details). + // + string name | _; + + // Package version. If empty or *, then no version constraint is applied. + // Otherwise the package version must match the value exactly. + // + string version | pv; + + // Package project wildcard. An empty value is treated the same way as *. + // + string project | pr; + + // Package version reviews. If *, then no reviews-related constraint is + // applied. Otherwise the value is supposed to be the one of the + // following statuses: reviewed and unreviewed. + // + string reviews | rv = "*"; + }; + class package_details { // Display package version search result list starting from this page. diff --git a/mod/utility.cxx b/mod/utility.cxx new file mode 100644 index 0000000..5ca16a0 --- /dev/null +++ b/mod/utility.cxx @@ -0,0 +1,69 @@ +// file : mod/utility.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include + +#include + +namespace brep +{ + string + wildcard_to_similar_to_pattern (const string& wildcard) + { + using namespace butl; + + if (wildcard.empty ()) + return "%"; + + string r; + for (const path_pattern_term& pt: path_pattern_iterator (wildcard)) + { + switch (pt.type) + { + case path_pattern_term_type::question: r += '_'; break; + case path_pattern_term_type::star: r += '%'; break; + case path_pattern_term_type::bracket: + { + // Copy the bracket expression translating the inverse character, if + // present. + // + size_t n (r.size ()); + r.append (pt.begin, pt.end); + + if (r[n + 1] == '!') // ...[!... ? + r[n + 1] = '^'; + + break; + } + case path_pattern_term_type::literal: + { + char c (get_literal (pt)); + + // Escape the special characters. + // + // Note that '.' is not a special character for SIMILAR TO. + // + switch (c) + { + case '\\': + case '%': + case '_': + case '|': + case '+': + case '{': + case '}': + case '(': + case ')': + case '[': + case ']': r += '\\'; break; + } + + r += c; + break; + } + } + } + + return r; + } +} diff --git a/mod/utility.hxx b/mod/utility.hxx index 43527ae..07fbf8b 100644 --- a/mod/utility.hxx +++ b/mod/utility.hxx @@ -19,6 +19,13 @@ namespace brep ? path_cast (dir / ('@' + tenant)) : dir; } + + // Transform the wildcard to the `SIMILAR TO` pattern. + // + // Note that the empty wildcard is transformed to the '%' pattern. + // + string + wildcard_to_similar_to_pattern (const string&); } #endif // MOD_UTILITY_HXX diff --git a/www/advanced-search-body.css b/www/advanced-search-body.css new file mode 100644 index 0000000..b2a23f6 --- /dev/null +++ b/www/advanced-search-body.css @@ -0,0 +1,84 @@ +/* + * Filter form (based on proplist and form-table) + */ +#filter input, #filter select, +#package-version-count, #package-version-count #count +{ + width: 100%; + margin:0; +} + +#filter-btn {padding-left: .4em;} + +/* + * Package version count. + */ +#count +{ + font-size: 1.32em; + line-height: 1.4em; + color: #555; + + margin: 1.2em 0 0 0; +} + +/* + * Project, Package, and Version tables. + */ +table.project, .package, table.version, #filter +{ + margin-top: .8em; + margin-bottom: .8em; + + padding-top: .4em; + padding-bottom: .4em; +} + +.package, #project-break +{ + padding-left: 2.25em; +} + +table.version, #package-break +{ + padding-left: 4.5em; +} + +table.version:nth-child(even) {background-color: rgba(0, 0, 0, 0.07);} + +table.project th, .package th, #filter th +{ + width: 5.5em; +} + +table.version th +{ + width: 7.3em; +} + +table.project tr.project td .value, +.package tr.name td .value, +.package tr.summary td .value, +.package tr.license td .value, +table.version tr.version td .value, +table.version tr.depends td .value, +table.version tr.requires td .value, +table.version tr.reviews td .value +{ + /* style. */ + font-family: monospace; + font-size: 0.94em; +} + +table.version tr.reviews td .none {color: #fe7c04;} +table.version tr.reviews td .fail {color: #ff0000;} +table.version tr.reviews td .pass {color: #00bb00;} + +#package-break, #project-break +{ + margin-left: -.4rem; + + /* style. */ + font-family: monospace; + font-weight: 500; +} diff --git a/www/advanced-search.css b/www/advanced-search.css new file mode 100644 index 0000000..b594f97 --- /dev/null +++ b/www/advanced-search.css @@ -0,0 +1,3 @@ +@import url(common.css); +@import url(brep-common.css); +@import url(advanced-search-body.css); diff --git a/www/advanced-search.scss b/www/advanced-search.scss new file mode 100644 index 0000000..77cbe34 --- /dev/null +++ b/www/advanced-search.scss @@ -0,0 +1,3 @@ +@import "common"; +@import "brep-common"; +@import "advanced-search-body"; -- cgit v1.1