Rename brep-load.conf to loadtab, make use of butl::tab_parser

8 files changed, 86 insertions, 92 deletions

diff --git a/INSTALL b/INSTALL
index de25c4d..b738683 100644
--- a/INSTALL
+++ b/INSTALL
@@ -114,10 +114,10 @@ To troubleshoot, see PostgreSQL logs.
 5. Create Database Schema and Load Repositories
 
 $ mkdir config
-$ edit config/brep-load.conf # Loader configuration, see brep-load(1).
+$ edit config/loadtab # Loader configuration, see brep-load(1).
 
 $ install/bin/brep-migrate
-$ install/bin/brep-load config/brep-load.conf
+$ install/bin/brep-load config/loadtab
 
 To verify:
 
@@ -250,7 +250,7 @@ The following crontab entry will execute the loader every five minutes:
 $ crontab -
 MAILTO=<brep-admin-email>
 PATH=/usr/local/bin:/bin:/usr/bin
-*/5 * * * * $HOME/install/bin/brep-load $HOME/config/brep-load.conf
+*/5 * * * * $HOME/install/bin/brep-load $HOME/config/loadtab
 ^D
 
 Note that here we assume that bpkg (which is executed by brep-load) is in one
@@ -360,7 +360,7 @@ If using cron, then simply wait for the next run.
 
 You can also do a manual load:
 
-$ install/bin/brep-load config/brep-load.conf
+$ install/bin/brep-load config/loadtab
 
 Next, start apache:
 
diff --git a/INSTALL-DEV b/INSTALL-DEV
index 66d554c..8ad9ae7 100644
--- a/INSTALL-DEV
+++ b/INSTALL-DEV
@@ -52,7 +52,7 @@ $ migrate/brep-migrate
 
 # Or use some other loader config.
 #
-$ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/r.conf
+$ load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab
 
 To verify:
 
@@ -121,7 +121,7 @@ To do a "complete reload" (i.e., recreate database schema, load the repository
 data, and reload the Apache2 plugin), execute the following from brep/:
 
 migrate/brep-migrate --recreate
-load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/r.conf
+load/brep-load --bpkg ../bpkg/bpkg/bpkg tests/load/loadtab
 sudo /etc/init.d/apache2 restart
 sudo systemctl restart apache2
 
diff --git a/etc/systemd/brep-load.service b/etc/systemd/brep-load.service
index bc12e4b..f9d4e31 100644
--- a/etc/systemd/brep-load.service
+++ b/etc/systemd/brep-load.service
@@ -5,7 +5,7 @@ Description=brep repository loader service
 Type=oneshot
 #User=brep
 #Group=brep
-ExecStart=/home/brep/install/bin/brep-load /home/brep/config/brep-load.conf
+ExecStart=/home/brep/install/bin/brep-load /home/brep/config/loadtab
 
 [Install]
 WantedBy=default.target
diff --git a/load/load.cli b/load/load.cli
index a3fcdb4..7850d86 100644
--- a/load/load.cli
+++ b/load/load.cli
@@ -13,18 +13,18 @@ include <brep/types>;
 "\summary=load build2 repositories into database"
 
 {
-  "<options> <file>",
+  "<options> <loadtab-file>",
 
   "\h|SYNOPSIS|
 
    \cb{brep-load --help}\n
    \cb{brep-load --version}\n
-   \c{\b{brep-load} [<options>] <file>}
+   \c{\b{brep-load} [<options>] <loadtab-file>}
 
    \h|DESCRIPTION|
 
    \cb{brep-load} reads the list of repositories from the specified
-   configuration <file>, fetches their manifest files, and loads the
+   configuration <loadtab-file>, fetches their manifest files, and loads the
    repository and package information into the database, suitable for
    consumption by the \cb{brep} web module.
 
diff --git a/load/load.cxx b/load/load.cxx
index 926f173..8856e38 100644
--- a/load/load.cxx
+++ b/load/load.cxx
@@ -22,6 +22,7 @@
 #include <butl/process>
 #include <butl/fdstream>
 #include <butl/filesystem>
+#include <butl/tab-parser>
 #include <butl/manifest-parser>
 
 #include <bpkg/manifest>
@@ -72,6 +73,12 @@ struct internal_repository
 
 using internal_repositories = vector<internal_repository>;
 
+  // Parse loadtab file.
+  //
+  // loadtab consists of lines in the following format:
+  //
+  // <remote-repository-location> <display-name> cache:<local-repository-location> [fingerprint:<fingerprint>]
+  //
 static internal_repositories
 load_repositories (path p)
 {
@@ -82,124 +89,121 @@ load_repositories (path p)
 
   try
   {
-    ifdstream ifs (p, ifdstream::in, ifdstream::badbit);
+    ifdstream ifs (p);
+    tab_parser parser (ifs, p.string ());
 
-    string s;
-    for (uint64_t l (1); getline (ifs, s); ++l)
+    tab_fields tl;
+    while (!(tl = parser.next ()).empty ())
     {
-      auto b (s.cbegin ());
-      auto i (b);
-      auto e (s.cend ());
+      size_t n (tl.size ()); // Fields count.
+      size_t i (0);          // The field currently being processed.
 
-      // Skip until first non-space (true) or space (false).
+      // Report an error for the field currently being processed. If i == n
+      // then we refer to the end-of-line column (presumably reporting a missed
+      // field).
       //
-      auto skip = [&i, &e] (bool s = true) -> decltype (i)
+      auto bad_line = [&p, &tl, &i, n] (const string& d, size_t offset = 0)
       {
-        for (; i != e && space (*i) == s; ++i) ;
-        return i;
-      };
-
-      skip (); // Skip leading spaces.
-
-      if (i == e || *i == '#') // Empty line or comment.
-        continue;
-
-      // From now on pb will track the begining of the next part
-      // while i -- the end.
-      //
-      auto pb (i);  // Location begin.
-      skip (false); // Find end of location.
+        // Offset beyond the end-of-line is meaningless.
+        //
+        assert (i < n || (i == n && offset == 0));
 
-      auto bad_line = [&p, l, &pb, &b] (const string& d)
-      {
-        cerr << p << ':' << l << ':' << pb - b + 1 << ": error: " << d
-             << endl;
+        cerr << p << ':' << tl.line << ':'
+             << (i == n
+                 ? tl.end_column
+                 : tl[i].column + offset)
+             << ": error: " << d << endl;
 
         throw failed ();
       };
 
-      repository_location location;
+      internal_repository r;
 
       try
       {
-        location = repository_location (string (pb, i));
+        r.location = repository_location (tl[i].value);
       }
       catch (const invalid_argument& e)
       {
         bad_line (e.what ());
       }
 
-      if (location.local ())
+      if (r.location.local ())
         bad_line ("local repository location");
 
-      for (const auto& r: repos)
-        if (r.location.canonical_name () == location.canonical_name ())
+      for (const auto& rp: repos)
+        if (rp.location.canonical_name () == r.location.canonical_name ())
           bad_line ("duplicate canonical name");
 
-      pb = skip (); // Find begin of display name.
-
-      if (pb == e)
+      // Display name field is a required one.
+      //
+      if (++i == n)
         bad_line ("no display name found");
 
-      skip (false); // Find end of display name.
-
-      string name (pb, i);
-      repository_location cache_location;
-      optional<string> fingerprint;
+      r.display_name = move (tl[i++].value);
 
       // Parse options, that have <name>:<value> form. Currently defined
       // options are cache (mandatory for now) and fingerprint.
       //
-      while ((pb = skip ()) != e)
+      for (; i < n; ++i)
       {
-        skip (false); // Find end of the option (no spaces allowed).
-
-        string nv (pb, i);
+        string nv (tl[i].value);
         size_t vp;
 
         if (strncmp (nv.c_str (), "cache:", vp = 6) == 0)
         {
-          if (!cache_location.empty ())
+          if (!r.cache_location.empty ())
             bad_line ("cache option redefinition");
 
-          // If the internal repository cache path is relative, then calculate
-          // its absolute path. Such path is considered to be relative to the
-          // configuration file directory path so result is independent from
-          // whichever directory is current for the loader process. Note that
-          // the resulting absolute path should be a valid repository location.
-          //
-          dir_path cache_path = dir_path (string (nv, vp));
-          if (cache_path.relative ())
-            cache_path = p.directory () / cache_path;
-
           try
           {
-            cache_location = repository_location (cache_path.string ());
+            // If the internal repository cache path is relative, then
+            // calculate its absolute path. Such path is considered to be
+            // relative to the configuration file directory path so result is
+            // independent from whichever directory is current for the loader
+            // process. Note that the resulting absolute path should be a valid
+            // repository location.
+            //
+            dir_path cache_path = dir_path (string (nv, vp));
+            if (cache_path.relative ())
+              cache_path = p.directory () / cache_path;
+
+            r.cache_location = repository_location (cache_path.string ());
 
             // Created from the absolute path repository location can not be
             // other than absolute.
             //
-            assert (cache_location.absolute ());
+            assert (r.cache_location.absolute ());
+          }
+          catch (const invalid_path& e)     // Thrown by dir_path().
+          {
+            bad_line (string ("invalid cache path: ") + e.what ());
           }
-          catch (const invalid_argument&)
+          catch (const invalid_argument& e) // Thrown by repository_location().
           {
-            bad_line ("invalid cache path");
+            bad_line (string ("invalid cache path: ") + e.what ());
           }
+
+          if (!file_exists (r.packages_path ()))
+            bad_line ("'packages' file does not exist");
+
+          if (!file_exists (r.repositories_path ()))
+            bad_line ("'repositories' file does not exist");
         }
         else if (strncmp (nv.c_str (), "fingerprint:", vp = 12) == 0)
         {
-          if (fingerprint)
+          if (r.fingerprint)
             bad_line ("fingerprint option redefinition");
 
-          fingerprint = string (nv, vp);
+          r.fingerprint = string (nv, vp);
 
           // Sanity check.
           //
-          if (!fingerprint->empty ())
+          if (!r.fingerprint->empty ())
           {
             try
             {
-              fingerprint_to_sha256 (*fingerprint);
+              fingerprint_to_sha256 (*r.fingerprint);
             }
             catch (const invalid_argument&)
             {
@@ -211,29 +215,19 @@ load_repositories (path p)
           bad_line ("invalid option '" + nv + "'");
       }
 
-      if (cache_location.empty ()) // For now cache option is mandatory.
+      // For now cache option is mandatory.
+      //
+      if (r.cache_location.empty ())
         bad_line ("no cache option found");
 
-      internal_repository r {
-        move (location),
-        move (name),
-        move (cache_location),
-        move (fingerprint)};
-
-      if (!file_exists (r.packages_path ()))
-        bad_line ("'packages' file does not exist");
-
-      if (!file_exists (r.repositories_path ()))
-        bad_line ("'repositories' file does not exist");
-
       repos.emplace_back (move (r));
-
-      // Check that there is no non-whitespace junk at the end.
-      //
-      if (skip () != e)
-        bad_line ("junk after filesystem path");
     }
   }
+  catch (const tab_parsing& e)
+  {
+    cerr << e << endl;
+    throw failed ();
+  }
   catch (const io_error& e)
   {
     cerr << "error: unable to read " << p << ": " << e << endl;
diff --git a/tests/load/buildfile b/tests/load/buildfile
index a4b0ffb..2088ee1 100644
--- a/tests/load/buildfile
+++ b/tests/load/buildfile
@@ -18,4 +18,4 @@ exe{driver}: test = false
 
 # precondition: PostgreSQL server running port 8432 with brep schema created.
 # test:
-# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./r.conf
+# ./driver ../../load/brep-load --db-host localhost --db-port 8432 ./loadtab
diff --git a/tests/load/driver.cxx b/tests/load/driver.cxx
index 396d89f..a0ab10f 100644
--- a/tests/load/driver.cxx
+++ b/tests/load/driver.cxx
@@ -52,7 +52,7 @@ main (int argc, char* argv[])
   auto print_usage = [argv]()
   {
     cerr << "usage: " << argv[0]
-         << " <loader_path> [loader_options] <loader_conf_file>" << endl;
+         << " <loader_path> [loader_options] <loadtab_file>" << endl;
   };
 
   if (argc < 3)
diff --git a/tests/load/r.conf b/tests/load/loadtab
index fe06984..fe06984 100644
--- a/tests/load/r.conf
+++ b/tests/load/loadtab