diff options
| -rw-r--r-- | INSTALL | 8 | ||||
| -rw-r--r-- | INSTALL-DEV | 4 | ||||
| -rw-r--r-- | etc/systemd/brep-load.service | 2 | ||||
| -rw-r--r-- | load/load.cli | 6 | ||||
| -rw-r--r-- | load/load.cxx | 154 | ||||
| -rw-r--r-- | tests/load/buildfile | 2 | ||||
| -rw-r--r-- | tests/load/driver.cxx | 2 | ||||
| -rw-r--r-- | tests/load/loadtab (renamed from tests/load/r.conf) | 0 |
8 files changed, 86 insertions, 92 deletions
@@ -114,10 +114,10 @@ To troubleshoot, see PostgreSQL logs. 5. Create Database Schema and Load Repositories $ mkdir config -$ edit config/brep-load.conf # Loader configuration, see brep-load(1). +$ edit config/loadtab # Loader configuration, see brep-load(1). $ install/bin/brep-migrate -$ install/bin/brep-load config/brep-load.conf +$ install/bin/brep-load config/loadtab To verify: @@ -250,7 +250,7 @@ The following crontab entry will execute the loader every five minutes: $ crontab - MAILTO=<brep-admin-email> PATH=/usr/local/bin:/bin:/usr/bin -*/5 * * * * $HOME/install/bin/brep-load $HOME/config/brep-load.conf +*/5 * * * * $HOME/install/bin/brep-load $HOME/config/loadtab ^D Note that here we assume that bpkg (which is executed by brep-load) is in one @@ -360,7 +360,7 @@ If using cron, then simply wait for the next run. You can also do a manual load: -$ install/bin/brep-load config/brep-load.conf +$ install/bin/brep-load config/loadtab Next, start apache: diff --git a/INSTALL-DEV b/INSTALL-DEV index 66d554c..8ad9ae7 100644 --- a/INSTALL-DEV +++ b/INSTALL-DEV @@ -52,7 +52,7 @@ $ migrate/brep-migrate # Or use some other loader config. # -$ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/r.conf +$ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab To verify: @@ -121,7 +121,7 @@ To do a "complete reload" (i.e., recreate database schema, load the repository data, and reload the Apache2 plugin), execute the following from brep/: migrate/brep-migrate --recreate -load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/r.conf +load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab sudo /etc/init.d/apache2 restart sudo systemctl restart apache2 diff --git a/etc/systemd/brep-load.service b/etc/systemd/brep-load.service index bc12e4b..f9d4e31 100644 --- a/etc/systemd/brep-load.service +++ b/etc/systemd/brep-load.service @@ -5,7 +5,7 @@ Description=brep repository loader service Type=oneshot #User=brep #Group=brep -ExecStart=/home/brep/install/bin/brep-load /home/brep/config/brep-load.conf +ExecStart=/home/brep/install/bin/brep-load /home/brep/config/loadtab [Install] WantedBy=default.target diff --git a/load/load.cli b/load/load.cli index a3fcdb4..7850d86 100644 --- a/load/load.cli +++ b/load/load.cli @@ -13,18 +13,18 @@ include <brep/types>; "\summary=load build2 repositories into database" { - "<options> <file>", + "<options> <loadtab-file>", "\h|SYNOPSIS| \cb{brep-load --help}\n \cb{brep-load --version}\n - \c{\b{brep-load} [<options>] <file>} + \c{\b{brep-load} [<options>] <loadtab-file>} \h|DESCRIPTION| \cb{brep-load} reads the list of repositories from the specified - configuration <file>, fetches their manifest files, and loads the + configuration <loadtab-file>, fetches their manifest files, and loads the repository and package information into the database, suitable for consumption by the \cb{brep} web module. diff --git a/load/load.cxx b/load/load.cxx index 926f173..8856e38 100644 --- a/load/load.cxx +++ b/load/load.cxx @@ -22,6 +22,7 @@ #include <butl/process> #include <butl/fdstream> #include <butl/filesystem> +#include <butl/tab-parser> #include <butl/manifest-parser> #include <bpkg/manifest> @@ -72,6 +73,12 @@ struct internal_repository using internal_repositories = vector<internal_repository>; + // Parse loadtab file. + // + // loadtab consists of lines in the following format: + // + // <remote-repository-location> <display-name> cache:<local-repository-location> [fingerprint:<fingerprint>] + // static internal_repositories load_repositories (path p) { @@ -82,124 +89,121 @@ load_repositories (path p) try { - ifdstream ifs (p, ifdstream::in, ifdstream::badbit); + ifdstream ifs (p); + tab_parser parser (ifs, p.string ()); - string s; - for (uint64_t l (1); getline (ifs, s); ++l) + tab_fields tl; + while (!(tl = parser.next ()).empty ()) { - auto b (s.cbegin ()); - auto i (b); - auto e (s.cend ()); + size_t n (tl.size ()); // Fields count. + size_t i (0); // The field currently being processed. - // Skip until first non-space (true) or space (false). + // Report an error for the field currently being processed. If i == n + // then we refer to the end-of-line column (presumably reporting a missed + // field). // - auto skip = [&i, &e] (bool s = true) -> decltype (i) + auto bad_line = [&p, &tl, &i, n] (const string& d, size_t offset = 0) { - for (; i != e && space (*i) == s; ++i) ; - return i; - }; - - skip (); // Skip leading spaces. - - if (i == e || *i == '#') // Empty line or comment. - continue; - - // From now on pb will track the begining of the next part - // while i -- the end. - // - auto pb (i); // Location begin. - skip (false); // Find end of location. + // Offset beyond the end-of-line is meaningless. + // + assert (i < n || (i == n && offset == 0)); - auto bad_line = [&p, l, &pb, &b] (const string& d) - { - cerr << p << ':' << l << ':' << pb - b + 1 << ": error: " << d - << endl; + cerr << p << ':' << tl.line << ':' + << (i == n + ? tl.end_column + : tl[i].column + offset) + << ": error: " << d << endl; throw failed (); }; - repository_location location; + internal_repository r; try { - location = repository_location (string (pb, i)); + r.location = repository_location (tl[i].value); } catch (const invalid_argument& e) { bad_line (e.what ()); } - if (location.local ()) + if (r.location.local ()) bad_line ("local repository location"); - for (const auto& r: repos) - if (r.location.canonical_name () == location.canonical_name ()) + for (const auto& rp: repos) + if (rp.location.canonical_name () == r.location.canonical_name ()) bad_line ("duplicate canonical name"); - pb = skip (); // Find begin of display name. - - if (pb == e) + // Display name field is a required one. + // + if (++i == n) bad_line ("no display name found"); - skip (false); // Find end of display name. - - string name (pb, i); - repository_location cache_location; - optional<string> fingerprint; + r.display_name = move (tl[i++].value); // Parse options, that have <name>:<value> form. Currently defined // options are cache (mandatory for now) and fingerprint. // - while ((pb = skip ()) != e) + for (; i < n; ++i) { - skip (false); // Find end of the option (no spaces allowed). - - string nv (pb, i); + string nv (tl[i].value); size_t vp; if (strncmp (nv.c_str (), "cache:", vp = 6) == 0) { - if (!cache_location.empty ()) + if (!r.cache_location.empty ()) bad_line ("cache option redefinition"); - // If the internal repository cache path is relative, then calculate - // its absolute path. Such path is considered to be relative to the - // configuration file directory path so result is independent from - // whichever directory is current for the loader process. Note that - // the resulting absolute path should be a valid repository location. - // - dir_path cache_path = dir_path (string (nv, vp)); - if (cache_path.relative ()) - cache_path = p.directory () / cache_path; - try { - cache_location = repository_location (cache_path.string ()); + // If the internal repository cache path is relative, then + // calculate its absolute path. Such path is considered to be + // relative to the configuration file directory path so result is + // independent from whichever directory is current for the loader + // process. Note that the resulting absolute path should be a valid + // repository location. + // + dir_path cache_path = dir_path (string (nv, vp)); + if (cache_path.relative ()) + cache_path = p.directory () / cache_path; + + r.cache_location = repository_location (cache_path.string ()); // Created from the absolute path repository location can not be // other than absolute. // - assert (cache_location.absolute ()); + assert (r.cache_location.absolute ()); + } + catch (const invalid_path& e) // Thrown by dir_path(). + { + bad_line (string ("invalid cache path: ") + e.what ()); } - catch (const invalid_argument&) + catch (const invalid_argument& e) // Thrown by repository_location(). { - bad_line ("invalid cache path"); + bad_line (string ("invalid cache path: ") + e.what ()); } + + if (!file_exists (r.packages_path ())) + bad_line ("'packages' file does not exist"); + + if (!file_exists (r.repositories_path ())) + bad_line ("'repositories' file does not exist"); } else if (strncmp (nv.c_str (), "fingerprint:", vp = 12) == 0) { - if (fingerprint) + if (r.fingerprint) bad_line ("fingerprint option redefinition"); - fingerprint = string (nv, vp); + r.fingerprint = string (nv, vp); // Sanity check. // - if (!fingerprint->empty ()) + if (!r.fingerprint->empty ()) { try { - fingerprint_to_sha256 (*fingerprint); + fingerprint_to_sha256 (*r.fingerprint); } catch (const invalid_argument&) { @@ -211,29 +215,19 @@ load_repositories (path p) bad_line ("invalid option '" + nv + "'"); } - if (cache_location.empty ()) // For now cache option is mandatory. + // For now cache option is mandatory. + // + if (r.cache_location.empty ()) bad_line ("no cache option found"); - internal_repository r { - move (location), - move (name), - move (cache_location), - move (fingerprint)}; - - if (!file_exists (r.packages_path ())) - bad_line ("'packages' file does not exist"); - - if (!file_exists (r.repositories_path ())) - bad_line ("'repositories' file does not exist"); - repos.emplace_back (move (r)); - - // Check that there is no non-whitespace junk at the end. - // - if (skip () != e) - bad_line ("junk after filesystem path"); } } + catch (const tab_parsing& e) + { + cerr << e << endl; + throw failed (); + } catch (const io_error& e) { cerr << "error: unable to read " << p << ": " << e << endl; diff --git a/tests/load/buildfile b/tests/load/buildfile index a4b0ffb..2088ee1 100644 --- a/tests/load/buildfile +++ b/tests/load/buildfile @@ -18,4 +18,4 @@ exe{driver}: test = false # precondition: PostgreSQL server running port 8432 with brep schema created. # test: -# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./r.conf +# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./loadtab diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx index 396d89f..a0ab10f 100644 --- a/tests/load/driver.cxx +++ b/tests/load/driver.cxx @@ -52,7 +52,7 @@ main (int argc, char* argv[]) auto print_usage = [argv]() { cerr << "usage: " << argv[0] - << " <loader_path> [loader_options] <loader_conf_file>" << endl; + << " <loader_path> [loader_options] <loadtab_file>" << endl; }; if (argc < 3) diff --git a/tests/load/r.conf b/tests/load/loadtab index fe06984..fe06984 100644 --- a/tests/load/r.conf +++ b/tests/load/loadtab |