Add support for dir repository

1 files changed, 239 insertions, 126 deletions

diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx
index 31e7b11..2e884c4 100644
--- a/bpkg/rep-fetch.cxx
+++ b/bpkg/rep-fetch.cxx
@@ -127,119 +127,60 @@ namespace bpkg
     }
   }
 
-  static rep_fetch_data
-  rep_fetch_git (const common_options& co,
-                 const dir_path* conf,
-                 const repository_location& rl,
-                 bool ignore_unknown)
+  // Parse the repositories manifest file if exists. Otherwise return the
+  // repository manifest list containing the only (trivial) base repository.
+  //
+  template <typename M>
+  static M
+  parse_repository_manifests (const path& f,
+                              bool iu,
+                              const repository_location& rl)
   {
-    // Plan:
-    //
-    // 1. Check repos_dir/<hash>/:
-    //
-    // 1.a If does not exist, git-clone into temp_dir/<hash>/<fragment>/.
-    //
-    // 1.a Otherwise, move as temp_dir/<hash>/ and git-fetch.
-    //
-    // 2. Move from temp_dir/<hash>/ to repos_dir/<hash>/<fragment>/
-    //
-    // 3. Check if repos_dir/<hash>/<fragment>/repositories.manifest exists:
-    //
-    // 3.a If exists, load.
-    //
-    // 3.b Otherwise, synthesize repository list with base repository.
-    //
-    // 4. Check if repos_dir/<hash>/<fragment>/packages.manifest exists:
-    //
-    // 4.a If exists, load. (into "skeleton" packages list to be filled?)
-    //
-    // 4.b Otherwise, synthesize as if single 'location: ./'.
-    //
-    // 5. For each package location obtained on step 4:
-    //
-    // 5.a Load repos_dir/<hash>/<fragment>/<location>/manifest.
-    //
-    // 5.b Run 'b info: repos_dir/<hash>/<fragment>/<location>/' and fix-up
-    //     package version.
-    //
-    // 6. Return repository and package manifests (certificate is NULL).
-    //
-
-    if (conf != nullptr && conf->empty ())
-      conf = dir_exists (bpkg_dir) ? &current_dir : nullptr;
-
-    assert (conf == nullptr || !conf->empty ());
-
-    // Clone or fetch the repository.
-    //
-    dir_path sd (repository_state (rl));
-
-    auto_rmdir rm (temp_dir / sd);
-    dir_path& td (rm.path);
-
-    if (exists (td))
-      rm_r (td);
-
-    // If the git repository directory already exists, then we are fetching
-    // an already cloned repository. Move it to the temporary directory.
-    //
-    // In this case also set the filesystem_state_changed flag since we are
-    // modifying the repository filesystem state.
-    //
-    // In the future we can probably do something smarter about the flag,
-    // keeping it unset unless the repository state directory is really
-    // changed.
-    //
-    dir_path rd;
-    bool fetch (false);
-    if (conf != nullptr)
-    {
-      rd = *conf / repos_dir / sd;
-
-      if (exists (rd))
-      {
-        mv (rd, td);
-        filesystem_state_changed = true;
-        fetch = true;
-      }
-    }
+    M r;
+    if (exists (f))
+      r = parse_manifest<M> (f, iu, rl);
+    else
+      r.emplace_back (repository_manifest ()); // Add the base repository.
 
-    dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td));
-    dir_path fd (td / nm); // Full directory path.
+    return r;
+  }
 
-    // Produce repository manifest list.
-    //
-    git_repository_manifests rms;
+  // Parse the package directories manifest file if exists. Otherwise treat
+  // the current directory as a package and return the manifest list with the
+  // single entry referencing this package.
+  //
+  template <typename M>
+  static M
+  parse_directory_manifests (const path& f,
+                             bool iu,
+                             const repository_location& rl)
+  {
+    M r;
+    if (exists (f))
+      r = parse_manifest<M> (f, iu, rl);
+    else
     {
-      path f (fd / repositories_file);
-
-      if (exists (f))
-        rms = parse_manifest<git_repository_manifests> (f, ignore_unknown, rl);
-      else
-        rms.emplace_back (repository_manifest ()); // Add the base repository.
+      r.push_back (package_manifest ());
+      r.back ().location = current_dir;
     }
 
-    // Produce the "skeleton" package manifest list.
-    //
-    git_package_manifests pms;
-    {
-      path f (fd / packages_file);
-
-      if (exists (f))
-        pms = parse_manifest<git_package_manifests> (f, ignore_unknown, rl);
-      else
-      {
-        pms.push_back (package_manifest ());
-        pms.back ().location = current_dir;
-      }
-    }
+    return r;
+  }
 
+  // Parse package manifests referenced by the package directory manifests.
+  //
+  static vector<rep_fetch_data::package>
+  parse_package_manifests (const common_options& co,
+                           const dir_path& repo_dir,
+                           const string& repo_fragment,
+                           vector<package_manifest>&& sms,
+                           bool iu,
+                           const repository_location& rl)
+  {
     vector<rep_fetch_data::package> fps;
-    fps.reserve (pms.size ());
+    fps.reserve (sms.size ());
 
-    // Parse package manifests.
-    //
-    for (package_manifest& sm: pms)
+    for (package_manifest& sm: sms)
     {
       assert (sm.location);
 
@@ -260,7 +201,7 @@ namespace bpkg
         package_info (dr);
       };
 
-      dir_path d (fd / path_cast<dir_path> (*sm.location));
+      dir_path d (repo_dir / path_cast<dir_path> (*sm.location));
       path f (d / path ("manifest"));
 
       if (!exists (f))
@@ -270,7 +211,7 @@ namespace bpkg
       {
         ifdstream ifs (f);
         manifest_parser mp (ifs, f.string ());
-        package_manifest m (pkg_package_manifest (mp, ignore_unknown));
+        package_manifest m (pkg_package_manifest (mp, iu));
 
         // Save the package manifest, preserving its location.
         //
@@ -290,7 +231,9 @@ namespace bpkg
         package_info (dr);
       }
 
-      // Fix-up the package version.
+      // Fix-up the package version. Note that the package may have the
+      // version module enable and the directory repository may well be a git
+      // repository.
       //
       const char* b (name_b (co));
 
@@ -361,7 +304,7 @@ namespace bpkg
           if (pr.wait ())
           {
             fps.emplace_back (rep_fetch_data::package {move (sm),
-                                                       nm.string ()});
+                                                       repo_fragment});
             continue;
           }
 
@@ -387,6 +330,157 @@ namespace bpkg
       }
     }
 
+    return fps;
+  }
+
+  static rep_fetch_data
+  rep_fetch_dir (const common_options& co,
+                 const repository_location& rl,
+                 bool ignore_unknown)
+  {
+    assert (rl.absolute ());
+
+    dir_path rd (path_cast<dir_path> (rl.path ()));
+
+    dir_repository_manifests rms (
+      parse_repository_manifests<dir_repository_manifests> (
+        rd / repositories_file,
+        ignore_unknown,
+        rl));
+
+    dir_package_manifests pms (
+      parse_directory_manifests<dir_package_manifests> (
+        rd / packages_file,
+        ignore_unknown,
+        rl));
+
+    vector<rep_fetch_data::package> fps (
+      parse_package_manifests (co,
+                               rd,
+                               string () /* repo_fragment */,
+                               move (pms),
+                               ignore_unknown,
+                               rl));
+
+    // @@ Here we will need to go through packages and check if there is the
+    //    selected package of the same version (without regards to the
+    //    iteration component), and having the different package manifest
+    //    hash. If that's the case then set the manifest version iteration
+    //    component as the selected package version iteation + 1.
+    //
+    // @@ It also seems that the manifest hash should be stored in the (new)
+    //    member of the package_location struct. Later this value will be
+    //    transmitted into the selected package objects by pkg_unpack().
+    //
+    // @@ Should we later check and fail if any available package contains
+    //    several package locations with different non-zero (for those that
+    //    come from the directory repo) manifest hashes?
+    //
+    return rep_fetch_data {move (rms), move (fps), nullptr};
+  }
+
+  static rep_fetch_data
+  rep_fetch_git (const common_options& co,
+                 const dir_path* conf,
+                 const repository_location& rl,
+                 bool ignore_unknown)
+  {
+    // Plan:
+    //
+    // 1. Check repos_dir/<hash>/:
+    //
+    // 1.a If does not exist, git-clone into temp_dir/<hash>/<fragment>/.
+    //
+    // 1.a Otherwise, move as temp_dir/<hash>/ and git-fetch.
+    //
+    // 2. Move from temp_dir/<hash>/ to repos_dir/<hash>/<fragment>/
+    //
+    // 3. Check if repos_dir/<hash>/<fragment>/repositories.manifest exists:
+    //
+    // 3.a If exists, load.
+    //
+    // 3.b Otherwise, synthesize repository list with base repository.
+    //
+    // 4. Check if repos_dir/<hash>/<fragment>/packages.manifest exists:
+    //
+    // 4.a If exists, load. (into "skeleton" packages list to be filled?)
+    //
+    // 4.b Otherwise, synthesize as if single 'location: ./'.
+    //
+    // 5. For each package location obtained on step 4:
+    //
+    // 5.a Load repos_dir/<hash>/<fragment>/<location>/manifest.
+    //
+    // 5.b Run 'b info: repos_dir/<hash>/<fragment>/<location>/' and fix-up
+    //     package version.
+    //
+    // 6. Return repository and package manifests (certificate is NULL).
+    //
+
+    if (conf != nullptr && conf->empty ())
+      conf = dir_exists (bpkg_dir) ? &current_dir : nullptr;
+
+    assert (conf == nullptr || !conf->empty ());
+
+    // Clone or fetch the repository.
+    //
+    dir_path sd (repository_state (rl));
+
+    auto_rmdir rm (temp_dir / sd);
+    dir_path& td (rm.path);
+
+    if (exists (td))
+      rm_r (td);
+
+    // If the git repository directory already exists, then we are fetching
+    // an already cloned repository. Move it to the temporary directory.
+    //
+    // In this case also set the filesystem_state_changed flag since we are
+    // modifying the repository filesystem state.
+    //
+    // In the future we can probably do something smarter about the flag,
+    // keeping it unset unless the repository state directory is really
+    // changed.
+    //
+    dir_path rd;
+    bool fetch (false);
+    if (conf != nullptr)
+    {
+      rd = *conf / repos_dir / sd;
+
+      if (exists (rd))
+      {
+        mv (rd, td);
+        filesystem_state_changed = true;
+        fetch = true;
+      }
+    }
+
+    dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td));
+    dir_path fd (td / nm); // Full directory path.
+
+    // Parse manifests.
+    //
+    git_repository_manifests rms (
+      parse_repository_manifests<git_repository_manifests> (
+        fd / repositories_file,
+        ignore_unknown,
+        rl));
+
+    git_package_manifests pms (
+      parse_directory_manifests<git_package_manifests> (
+        fd / packages_file,
+        ignore_unknown,
+        rl));
+
+    vector<rep_fetch_data::package> fps (
+      parse_package_manifests (co,
+                               fd,
+                               nm.string (),
+                               move (pms),
+                               ignore_unknown,
+                               rl));
+
     // Move the state directory to its proper place.
     //
     // If there is no configuration directory then we let auto_rmdir clean it
@@ -411,6 +505,7 @@ namespace bpkg
     switch (rl.type ())
     {
     case repository_type::pkg: return rep_fetch_pkg (co, conf, rl, iu);
+    case repository_type::dir: return rep_fetch_dir (co, rl, iu);
     case repository_type::git: return rep_fetch_git (co, conf, rl, iu);
     }
 
@@ -592,19 +687,33 @@ namespace bpkg
       }
     }
 
-    // For git repositories that have neither prerequisites nor complements
-    // we use the root repository as the default complement.
+    // For dir and git repositories that have neither prerequisites nor
+    // complements we use the root repository as the default complement.
     //
     // This supports the common use case where the user has a single-package
-    // git repository and doesn't want to bother with the
-    // repositories.manifest file. This way their package will still pick up
-    // its dependencies from the configuration, without regards from which
-    // repositories they came from.
+    // repository and doesn't want to bother with the repositories.manifest
+    // file. This way their package will still pick up its dependencies from
+    // the configuration, without regards from which repositories they came
+    // from.
     //
-    if (rl.type () == repository_type::git &&
-        r->complements.empty ()            &&
-        r->prerequisites.empty ())
-      r->complements.insert (lazy_shared_ptr<repository> (db, string ()));
+    switch (rl.type ())
+    {
+    case repository_type::git:
+    case repository_type::dir:
+      {
+        if (r->complements.empty () && r->prerequisites.empty ())
+          r->complements.insert (lazy_shared_ptr<repository> (db, string ()));
+
+        break;
+      }
+    case repository_type::pkg:
+      {
+        // Pkg repository is a "strict" one, that requires all the
+        // prerequisites and complements to be listed.
+        //
+        break;
+      }
+    }
 
     // Save the changes to the repository object.
     //
@@ -805,20 +914,24 @@ namespace bpkg
 
         if (repository_name (a))
         {
-          lazy_shared_ptr<repository> rp (db, a);
+          r = lazy_shared_ptr<repository> (db, a);
 
-          if (ua.find (rp) != ua.end ())
-            r = move (rp);
-          else
+          if (ua.find (r) == ua.end ())
             fail << "repository '" << a << "' does not exist in this "
                  << "configuration";
         }
         else
-          //@@ TODO: check if exists in root & same location and avoid
-          // calling rep_add. Get rid of quiet mode.
+        {
+          repository_location rl (parse_location (a, nullopt /* type */));
+          r = lazy_shared_ptr<repository> (db, rl.canonical_name ());
+
+          // If the repository is not the root complement yet or has
+          // a different location then we add it to the configuration.
           //
-          r = lazy_shared_ptr<repository> (
-            db, rep_add (t, parse_location (a, nullopt /* type */)));
+          auto i (ua.find (r));
+          if (i == ua.end () || i->load ()->location.url () != rl.url ())
+            r = lazy_shared_ptr<repository> (db, rep_add (t, rl));
+        }
 
         repos.emplace_back (move (r));
       }