Implement rep-fetch command

18 files changed, 486 insertions, 32 deletions

diff --git a/bpkg/bpkg-options.cli b/bpkg/bpkg-options.cli
index 6ea7f91..029c5a8 100644
--- a/bpkg/bpkg-options.cli
+++ b/bpkg/bpkg-options.cli
@@ -98,6 +98,13 @@ namespace bpkg
       ""
     };
 
+    bool rep-fetch
+    {
+      "",
+      "Fetch available packages list.",
+      ""
+    };
+
     bool rep-create
     {
       "[<dir>]",
diff --git a/bpkg/bpkg.cxx b/bpkg/bpkg.cxx
index c84a1f2..ef74144 100644
--- a/bpkg/bpkg.cxx
+++ b/bpkg/bpkg.cxx
@@ -2,11 +2,11 @@
 // copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
 // license   : MIT; see accompanying LICENSE file
 
-#include <cassert>
 #include <iostream>
 #include <exception>
 
 #include <bpkg/types>
+#include <bpkg/utility>
 #include <bpkg/diagnostics>
 
 #include <bpkg/bpkg-options>
@@ -28,6 +28,7 @@
 #include <bpkg/cfg-create>
 
 #include <bpkg/rep-add>
+#include <bpkg/rep-fetch>
 #include <bpkg/rep-create>
 
 using namespace std;
@@ -197,6 +198,7 @@ try
 #define REP_COMMAND(CMD) ANY_COMMAND(rep, CMD)
 
   REP_COMMAND (add);
+  REP_COMMAND (fetch);
   REP_COMMAND (create);
 
   // @@ Would be nice to check that args doesn't contain any junk left.
diff --git a/bpkg/buildfile b/bpkg/buildfile
index c016e09..4e5bd23 100644
--- a/bpkg/buildfile
+++ b/bpkg/buildfile
@@ -26,6 +26,7 @@ exe{bpkg}: cxx{package package-odb database diagnostics utility} \
                       cli.cxx{pkg-clean-options}                 \
   cxx{cfg-create}     cli.cxx{cfg-create-options}                \
   cxx{rep-add}        cli.cxx{rep-add-options}                   \
+  cxx{rep-fetch}      cli.cxx{rep-fetch-options}                 \
   cxx{rep-create}     cli.cxx{rep-create-options}                \
   $libs
 
@@ -72,5 +73,8 @@ cli.cxx{cfg-create-options}: cli.options += --exclude-base
 cli.cxx{rep-add-options}: cli{rep-add-options}
 cli.cxx{rep-add-options}: cli.options += --exclude-base
 
+cli.cxx{rep-fetch-options}: cli{rep-fetch-options}
+cli.cxx{rep-fetch-options}: cli.options += --exclude-base
+
 cli.cxx{rep-create-options}: cli{rep-create-options}
 cli.cxx{rep-create-options}: cli.options += --exclude-base
diff --git a/bpkg/cfg-create.cxx b/bpkg/cfg-create.cxx
index be85d92..df7a1bf 100644
--- a/bpkg/cfg-create.cxx
+++ b/bpkg/cfg-create.cxx
@@ -4,8 +4,6 @@
 
 #include <bpkg/cfg-create>
 
-#include <utility> // move()
-#include <cassert>
 #include <fstream>
 
 #include <bpkg/types>
diff --git a/bpkg/database b/bpkg/database
index 05ffb04..f7de4d5 100644
--- a/bpkg/database
+++ b/bpkg/database
@@ -5,6 +5,9 @@
 #ifndef BPKG_DATABASE
 #define BPKG_DATABASE
 
+#include <utility> // forward()
+
+#include <odb/session.hxx>
 #include <odb/sqlite/database.hxx>
 
 #include <bpkg/types>
@@ -12,6 +15,8 @@
 
 namespace bpkg
 {
+  using odb::session;
+
   using odb::sqlite::database;
   using odb::sqlite::transaction;
 
@@ -28,6 +33,43 @@ namespace bpkg
     database& db_;
     odb::tracer* t_;
   };
+
+  // Range-based for-loop iteration over query result that returns
+  // object pointers. For example:
+  //
+  // for (shared_ptr<object> o: pointer_result (db.query<object> (...)))
+  //
+  template <typename R>
+  class pointer_result_range
+  {
+    R r_;
+
+  public:
+    pointer_result_range (R&& r): r_ (std::forward<R> (r)) {}
+
+    using base_iterator = typename R::iterator;
+
+    struct iterator: base_iterator
+    {
+      iterator () = default;
+
+      explicit
+      iterator (base_iterator i): base_iterator (std::move (i)) {}
+
+      typename base_iterator::pointer_type
+      operator* () {return this->load ();}
+    };
+
+    iterator begin () {return iterator (r_.begin ());}
+    iterator end () {return iterator (r_.end ());}
+  };
+
+  template <typename R>
+  inline pointer_result_range<R>
+  pointer_result (R&& r)
+  {
+    return pointer_result_range<R> (std::forward<R> (r));
+  }
 }
 
 #endif // BPKG_DATABASE
diff --git a/bpkg/diagnostics b/bpkg/diagnostics
index e48d4f2..565a9e5 100644
--- a/bpkg/diagnostics
+++ b/bpkg/diagnostics
@@ -7,8 +7,7 @@
 
 #include <string>
 #include <cstdint>
-#include <utility> // move(), forward()
-#include <cassert>
+#include <utility> // forward()
 #include <sstream>
 #include <ostream>
 #include <exception>
@@ -16,6 +15,7 @@
 #include <odb/tracer.hxx>
 
 #include <bpkg/types>
+#include <bpkg/utility>
 
 namespace bpkg
 {
diff --git a/bpkg/help.cxx b/bpkg/help.cxx
index 89ce93f..66b68aa 100644
--- a/bpkg/help.cxx
+++ b/bpkg/help.cxx
@@ -4,10 +4,10 @@
 
 #include <bpkg/help>
 
-#include <cassert>
 #include <iostream>
 
 #include <bpkg/types>
+#include <bpkg/utility>
 #include <bpkg/diagnostics>
 
 #include <bpkg/bpkg-options>
diff --git a/bpkg/package b/bpkg/package
index 1c2a420..34d917e 100644
--- a/bpkg/package
+++ b/bpkg/package
@@ -9,8 +9,6 @@
 #include <vector>
 #include <cstdint> // uint16
 #include <ostream>
-#include <utility> // move()
-#include <cstdint> // uint16
 
 #include <odb/core.hxx>
 
@@ -99,6 +97,15 @@ namespace bpkg
     complements_type complements;
     prerequisites_type prerequisites;
 
+    const string&
+    name () const {return location.canonical_name ();}
+
+    // Used to detect recursive fecthing. Will probably be replaced
+    // by the 'repositories' file timestamp or hashsum later.
+    //
+    #pragma db transient
+    bool fetched = false;
+
   public:
     explicit
     repository (repository_location l): location (move (l)) {}
@@ -134,6 +141,13 @@ namespace bpkg
     repository () = default;
   };
 
+  #pragma db view object(repository) query(repository::id.name != "" && (?))
+  struct repository_count
+  {
+    #pragma db column("count(" + repository::id.name + ")")
+    size_t result;
+  };
+
   // package_version_id
   //
   #pragma db value
@@ -144,6 +158,13 @@ namespace bpkg
     string        upstream; // Canonical upstream.
     std::uint16_t revision;
 
+    package_version_id () = default;
+    package_version_id (string n, const version& v)
+        : name (move (n)),
+          epoch (v.epoch ()),
+          upstream (v.canonical_upstream ()),
+          revision (v.revision ()) {}
+
     #pragma db member(epoch)    column("version_epoch")
     #pragma db member(upstream) column("version_upstream")
     #pragma db member(revision) column("version_revision")
@@ -215,6 +236,13 @@ namespace bpkg
     available_package () = default;
   };
 
+  #pragma db view object(available_package)
+  struct available_package_count
+  {
+    #pragma db column("count(" + available_package::id.data.name + ")")
+    size_t result;
+  };
+
   // state
   //
   enum class state
diff --git a/bpkg/package.cxx b/bpkg/package.cxx
index 9528288..e955a3f 100644
--- a/bpkg/package.cxx
+++ b/bpkg/package.cxx
@@ -52,15 +52,7 @@ namespace bpkg
   available_package::_id_type available_package::
   _id () const
   {
-    return _id_type {
-      {
-        name,
-        version.epoch (),
-        version.canonical_upstream (),
-        version.revision ()
-      },
-      version.upstream ()
-    };
+    return _id_type {package_version_id (name, version), version.upstream ()};
   }
 
   void available_package::
diff --git a/bpkg/pkg-disfigure.cxx b/bpkg/pkg-disfigure.cxx
index e064cba..6c357f4 100644
--- a/bpkg/pkg-disfigure.cxx
+++ b/bpkg/pkg-disfigure.cxx
@@ -22,9 +22,9 @@ namespace bpkg
                  const shared_ptr<package>& p)
   {
     tracer trace ("pkg_disfigure");
-    t.tracer (trace); // "Tail" call, never restored.
 
     database& db (t.database ());
+    tracer_guard tg (db, trace);
 
     // Calculate package's src_root and out_root.
     //
diff --git a/bpkg/rep-add.cxx b/bpkg/rep-add.cxx
index 803c68d..924cd65 100644
--- a/bpkg/rep-add.cxx
+++ b/bpkg/rep-add.cxx
@@ -32,19 +32,19 @@ namespace bpkg
 
     // Figure out the repository location.
     //
-    const char* s (args.next ());
+    const char* arg (args.next ());
     repository_location rl;
     try
     {
-      rl = repository_location (s, repository_location ());
+      rl = repository_location (arg, repository_location ());
 
       if (rl.relative ()) // Throws if location is empty.
         rl = repository_location (
-          dir_path (s).complete ().normalize ().string ());
+          dir_path (arg).complete ().normalize ().string ());
     }
     catch (const invalid_argument& e)
     {
-      fail << "invalid repository location '" << s << "': " << e.what ();
+      fail << "invalid repository location '" << arg << "': " << e.what ();
     }
 
     const string& rn (rl.canonical_name ());
@@ -53,6 +53,7 @@ namespace bpkg
     //
     database db (open (c, trace));
     transaction t (db.begin ());
+    session s; // Repository dependencies can have cycles.
 
     // It is possible that this repository is already in the database.
     // For example, it might be a prerequisite of one of the already
diff --git a/bpkg/rep-create.cxx b/bpkg/rep-create.cxx
index 4546224..6ee0e87 100644
--- a/bpkg/rep-create.cxx
+++ b/bpkg/rep-create.cxx
@@ -5,8 +5,6 @@
 #include <bpkg/rep-create>
 
 #include <map>
-#include <utility>  // pair, move()
-#include <cassert>
 #include <fstream>
 #include <iostream>
 #include <system_error>
@@ -145,6 +143,10 @@ namespace bpkg
     // Load the 'repositories' file to make sure it is there and
     // is valid.
     //
+    // @@ The same code as in rep-fetch.
+    // @@ Should we check for duplicates? Or should this be done at
+    //    the manifest level?
+    //
     path rf (d / path ("repositories"));
 
     if (!exists (rf))
diff --git a/bpkg/rep-fetch b/bpkg/rep-fetch
new file mode 100644
index 0000000..9977571
--- /dev/null
+++ b/bpkg/rep-fetch
@@ -0,0 +1,17 @@
+// file      : bpkg/rep-fetch -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BPKG_REP_FETCH
+#define BPKG_REP_FETCH
+
+#include <bpkg/types>
+#include <bpkg/rep-fetch-options>
+
+namespace bpkg
+{
+  void
+  rep_fetch (const rep_fetch_options&, cli::scanner& args);
+}
+
+#endif // BPKG_REP_FETCH
diff --git a/bpkg/rep-fetch-options.cli b/bpkg/rep-fetch-options.cli
new file mode 100644
index 0000000..0e43eda
--- /dev/null
+++ b/bpkg/rep-fetch-options.cli
@@ -0,0 +1,33 @@
+// file      : bpkg/rep-fetch-options.cli
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+include <bpkg/common-options.cli>;
+
+/*
+"\section=1"
+"\name=bpkg-rep-fetch"
+
+"\h{SYNOPSIS}
+
+bpkg rep-fetch [<options>]"
+
+"\h{DESCRIPTION}
+
+The \cb{rep-fetch} command recursively fetches the prerequisite repository
+and available package lists for all the repositories that were added
+(\cb{rep-add}) to the configuration."
+*/
+
+namespace bpkg
+{
+  class rep_fetch_options: common_options
+  {
+    dir_path --directory|-d (".")
+    {
+      "<dir>",
+      "Assume configuration is in <dir> rather than in the current working
+       directory."
+    };
+  };
+}
diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
new file mode 100644
index 0000000..7bb5652
--- /dev/null
+++ b/bpkg/rep-fetch.cxx
@@ -0,0 +1,265 @@
+// file      : bpkg/rep-fetch.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <bpkg/rep-fetch>
+
+#include <fstream>
+#include <stdexcept>
+
+#include <bpkg/manifest>
+#include <bpkg/manifest-parser>
+
+#include <bpkg/types>
+#include <bpkg/package>
+#include <bpkg/package-odb>
+#include <bpkg/utility>
+#include <bpkg/database>
+#include <bpkg/diagnostics>
+
+using namespace std;
+using namespace butl;
+
+namespace bpkg
+{
+  static void
+  rep_fetch (transaction& t, const shared_ptr<repository>& r)
+  {
+    tracer trace ("rep_fetch(rep)");
+
+    database& db (t.database ());
+    tracer_guard tg (db, trace);
+
+    if (verb >= 2)
+      text << "fetching " << r->name ();
+
+    const repository_location& rl (r->location);
+    level4 ([&]{trace << r->name () << " " << rl;});
+
+    assert (rl.absolute () /*|| rl.remote ()*/);
+
+    r->fetched = true; // Mark as being fetched.
+
+    //@@ The same code as in rep-create.
+    //
+
+    // Load the 'repositories' file and use it to populate the
+    // prerequisite and complement repository sets.
+    //
+    repository_manifests rms;
+    {
+      path f (rl.path () / path ("repositories"));
+
+      if (!exists (f))
+        fail << "file " << f << " does not exist";
+
+      try
+      {
+        ifstream ifs;
+        ifs.exceptions (ofstream::badbit | ofstream::failbit);
+        ifs.open (f.string ());
+
+        manifest_parser mp (ifs, f.string ());
+        rms = repository_manifests (mp);
+      }
+      catch (const manifest_parsing& e)
+      {
+        fail (e.name, e.line, e.column) << e.description;
+      }
+      catch (const ifstream::failure&)
+      {
+        fail << "unable to read from " << f;
+      }
+    }
+
+    for (repository_manifest& rm: rms)
+    {
+      if (rm.location.empty ())
+        continue; // Entry for this repository.
+
+      // If the location is relative, complete it using this repository
+      // as a base.
+      //
+      if (rm.location.relative ())
+      {
+        try
+        {
+          rm.location = repository_location (rm.location, rl);
+        }
+        catch (const invalid_argument& e)
+        {
+          fail << "invalid relative repository location '" << rm.location
+               << "': " << e.what () <<
+            info << "base repository location is " << rl;
+        }
+      }
+
+      // We might already have this repository in the database.
+      //
+      shared_ptr<repository> pr (
+        db.find<repository> (
+          rm.location.canonical_name ()));
+
+      if (pr == nullptr)
+      {
+        pr.reset (new repository (move (rm.location)));
+        db.persist (pr); // Enter into session, important if recursive.
+      }
+
+      // Load the prerequisite repository unless it has already been
+      // (or is already being) fetched.
+      //
+      if (!pr->fetched)
+        rep_fetch (t, pr);
+
+      level4 ([&]{trace << pr->name () << " prerequisite of " << r->name ();});
+
+      if (!r->prerequisites.insert (lazy_weak_ptr<repository> (db, pr)).second)
+      {
+        fail << "duplicate prerequisite repository " << pr->location << " "
+             << "in " << r->name ();
+      }
+    }
+
+    // Load the 'packages' file.
+    //
+    // @@ We need to check that that 'repositories' file hasn't
+    //    changed since.
+    //
+    package_manifests pms;
+    {
+      path f (rl.path () / path ("packages"));
+
+      if (!exists (f))
+        fail << "file " << f << " does not exist";
+
+      try
+      {
+        ifstream ifs;
+        ifs.exceptions (ofstream::badbit | ofstream::failbit);
+        ifs.open (f.string ());
+
+        manifest_parser mp (ifs, f.string ());
+        pms = package_manifests (mp);
+      }
+      catch (const manifest_parsing& e)
+      {
+        fail (e.name, e.line, e.column) << e.description;
+      }
+      catch (const ifstream::failure&)
+      {
+        fail << "unable to read from " << f;
+      }
+    }
+
+    // "Suspend" session while persisting packages to reduce memory
+    // consumption.
+    //
+    session& s (session::current ());
+    session::reset_current ();
+
+    for (package_manifest& pm: pms)
+    {
+      // We might already have this package in the database.
+      //
+      bool persist (false);
+
+      shared_ptr<available_package> p (
+        db.find<available_package> (
+          package_version_id (pm.name, pm.version)));
+
+      if (p == nullptr)
+      {
+        p.reset (new available_package {move (pm.name),
+                                        move (pm.version),
+                                        {}});
+        persist = true;
+      }
+
+      // This repository shouldn't already be in the location set since
+      // that would mean it has already been loaded and we shouldn't be
+      // here.
+      //
+      p->locations.push_back (
+        package_location {lazy_shared_ptr<repository> (db, r),
+                          move (*pm.location)});
+
+      if (persist)
+        db.persist (p);
+      else
+        db.update (p);
+    }
+
+    session::current (s); // "Resume".
+
+    // Save the changes to the repository object.
+    //
+    db.update (r);
+  }
+
+  void
+  rep_fetch (const rep_fetch_options& o, cli::scanner&)
+  {
+    tracer trace ("rep_fetch");
+
+    dir_path c (o.directory ());
+    level4 ([&]{trace << "configuration: " << c;});
+
+    database db (open (c, trace));
+    transaction t (db.begin ());
+    session s; // Repository dependencies can have cycles.
+
+    shared_ptr<repository> root (db.load<repository> (""));
+    const auto& ua (root->complements); // User-added repositories.
+
+    if (ua.empty ())
+      fail << "configuration has no repositories" <<
+        info << "use 'bpkg rep-add' to add a repository";
+
+    // Clean repositories and available packages. At the end only
+    // repositories that were explicitly added by the user and the
+    // special root repository should remain.
+    //
+    db.erase_query<available_package> ();
+
+    for (shared_ptr<repository> r: pointer_result (db.query<repository> ()))
+    {
+      if (r == root)
+      {
+        level5 ([&]{trace << "skipping root";});
+      }
+      else if (ua.find (lazy_shared_ptr<repository> (db, r)) != ua.end ())
+      {
+        level4 ([&]{trace << "cleaning " << r->name ();});
+
+        r->complements.clear ();
+        r->prerequisites.clear ();
+        r->fetched = false;
+        db.update (r);
+      }
+      else
+      {
+        level4 ([&]{trace << "erasing " << r->name ();});
+        db.erase (r);
+      }
+    }
+
+    // Now recursively fetch prerequisite/complement repositories and
+    // their packages.
+    //
+    for (const lazy_shared_ptr<repository>& lp: ua)
+      rep_fetch (t, lp.load ());
+
+    size_t rcount, pcount;
+    if (verb)
+    {
+      rcount = db.query_value<repository_count> ().result;
+      pcount = db.query_value<available_package_count> ().result;
+    }
+
+    t.commit ();
+
+    if (verb)
+      text << pcount << " package(s) in " << rcount << " repository(s)";
+  }
+}
diff --git a/bpkg/test.sh b/bpkg/test.sh
index e25d2bb..c252a8a 100755
--- a/bpkg/test.sh
+++ b/bpkg/test.sh
@@ -10,6 +10,7 @@ ver=1.0.0
 pkga=../../hello/dist/$pkg-$ver.tar.bz2
 pkgd=../../hello/dist/$pkg-$ver
 out=$cfg/`basename $pkgd`
+rep=../../hello/1/hello
 
 function error ()
 {
@@ -19,25 +20,33 @@ function error ()
 
 function test ()
 {
-  local cmd=$1
-  shift
+  local cmd=$1; shift
+  local ops=
 
-  $bpkg $cmd -d $cfg $*
+  if [ "$cmd" != "rep-create" ]; then
+    ops="-d $cfg"
+  fi
+
+  $bpkg $cmd $ops $*
 
   if [ $? -ne 0 ]; then
-    error "failed: $bpkg $cmd -d $cfg $*"
+    error "failed: $bpkg $cmd $ops $*"
   fi
 }
 
 function fail ()
 {
-  local cmd=$1
-  shift
+  local cmd=$1; shift
+  local ops=
+
+  if [ "$cmd" != "rep-create" ]; then
+    ops="-d $cfg"
+  fi
 
-  $bpkg $cmd -d $cfg $*
+  $bpkg $cmd $ops $*
 
   if [ $? -eq 0 ]; then
-    error "succeeded: $bpkg $cmd -d $cfg $*"
+    error "succeeded: $bpkg $cmd $ops $*"
   fi
 
   return 0
@@ -64,6 +73,19 @@ function gone ()
 }
 
 ##
+## rep-create
+##
+
+fail rep-create # no 'repositories' file
+
+test rep-create ../tests/repository/1/misc/stable
+test rep-create ../tests/repository/1/misc/testing
+
+test rep-create ../tests/repository/1/math/stable
+test rep-create ../tests/repository/1/math/testing
+test rep-create ../tests/repository/1/math/unstable
+
+##
 ## cfg-create
 ##
 
@@ -74,6 +96,8 @@ stat unknown
 ## rep-add
 ##
 
+test cfg-create --wipe
+
 fail rep-add         # repository location expected
 fail rep-add stable  # invalid location
 fail rep-add http:// # invalid location
@@ -93,6 +117,42 @@ fail rep-add /tmp/1/../1/misc/stable # duplicate
 test rep-add http://pkg.example.org/1/testing
 fail rep-add http://www.example.org/1/testing # duplicate
 
+##
+## rep-fetch
+##
+
+test cfg-create --wipe
+
+fail rep-fetch # no repositories
+
+# hello repository
+#
+test cfg-create --wipe
+test rep-add $rep
+test rep-fetch
+test rep-fetch
+
+# math/unstable repository
+#
+test cfg-create --wipe
+test rep-add ../tests/repository/1/math/unstable
+test rep-fetch
+test rep-fetch
+
+# both
+#
+test cfg-create --wipe
+test rep-add $rep
+test rep-add ../tests/repository/1/math/unstable
+test rep-fetch
+test rep-fetch
+
+## @@
+##
+##
+
+test cfg-create --wipe config.cxx=g++-4.9 cxx config.install.root=/tmp/install
+stat unknown
 
 ##
 ## pkg-fetch
diff --git a/bpkg/types b/bpkg/types
index 97b9ed9..e4e1340 100644
--- a/bpkg/types
+++ b/bpkg/types
@@ -8,6 +8,7 @@
 #include <vector>
 #include <string>
 #include <memory>   // shared_ptr, unique_ptr
+#include <cstddef>  // size_t
 #include <ostream>
 
 #include <odb/lazy-ptr.hxx>
@@ -19,6 +20,7 @@ namespace bpkg
 {
   // Commonly-used types.
   //
+  using std::size_t;
   using std::string;
 
   using strings = std::vector<string>;
diff --git a/bpkg/utility b/bpkg/utility
index 32ae252..72e02ba 100644
--- a/bpkg/utility
+++ b/bpkg/utility
@@ -5,6 +5,7 @@
 #ifndef BPKG_UTILITY
 #define BPKG_UTILITY
 
+#include <cassert>
 #include <utility>   // move()
 #include <exception> // uncaught_exception ()