From 7e4b2dcd0e5ddd37276879e699fd84059183f5e2 Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Tue, 6 Mar 2018 23:52:11 +0300 Subject: Add support for dir repository --- bpkg/rep-fetch.cxx | 365 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 239 insertions(+), 126 deletions(-) (limited to 'bpkg/rep-fetch.cxx') diff --git a/bpkg/rep-fetch.cxx b/bpkg/rep-fetch.cxx index 31e7b11..2e884c4 100644 --- a/bpkg/rep-fetch.cxx +++ b/bpkg/rep-fetch.cxx @@ -127,119 +127,60 @@ namespace bpkg } } - static rep_fetch_data - rep_fetch_git (const common_options& co, - const dir_path* conf, - const repository_location& rl, - bool ignore_unknown) + // Parse the repositories manifest file if exists. Otherwise return the + // repository manifest list containing the only (trivial) base repository. + // + template + static M + parse_repository_manifests (const path& f, + bool iu, + const repository_location& rl) { - // Plan: - // - // 1. Check repos_dir//: - // - // 1.a If does not exist, git-clone into temp_dir///. - // - // 1.a Otherwise, move as temp_dir// and git-fetch. - // - // 2. Move from temp_dir// to repos_dir/// - // - // 3. Check if repos_dir///repositories.manifest exists: - // - // 3.a If exists, load. - // - // 3.b Otherwise, synthesize repository list with base repository. - // - // 4. Check if repos_dir///packages.manifest exists: - // - // 4.a If exists, load. (into "skeleton" packages list to be filled?) - // - // 4.b Otherwise, synthesize as if single 'location: ./'. - // - // 5. For each package location obtained on step 4: - // - // 5.a Load repos_dir////manifest. - // - // 5.b Run 'b info: repos_dir////' and fix-up - // package version. - // - // 6. Return repository and package manifests (certificate is NULL). - // - - if (conf != nullptr && conf->empty ()) - conf = dir_exists (bpkg_dir) ? ¤t_dir : nullptr; - - assert (conf == nullptr || !conf->empty ()); - - // Clone or fetch the repository. - // - dir_path sd (repository_state (rl)); - - auto_rmdir rm (temp_dir / sd); - dir_path& td (rm.path); - - if (exists (td)) - rm_r (td); - - // If the git repository directory already exists, then we are fetching - // an already cloned repository. Move it to the temporary directory. - // - // In this case also set the filesystem_state_changed flag since we are - // modifying the repository filesystem state. - // - // In the future we can probably do something smarter about the flag, - // keeping it unset unless the repository state directory is really - // changed. - // - dir_path rd; - bool fetch (false); - if (conf != nullptr) - { - rd = *conf / repos_dir / sd; - - if (exists (rd)) - { - mv (rd, td); - filesystem_state_changed = true; - fetch = true; - } - } + M r; + if (exists (f)) + r = parse_manifest (f, iu, rl); + else + r.emplace_back (repository_manifest ()); // Add the base repository. - dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td)); - dir_path fd (td / nm); // Full directory path. + return r; + } - // Produce repository manifest list. - // - git_repository_manifests rms; + // Parse the package directories manifest file if exists. Otherwise treat + // the current directory as a package and return the manifest list with the + // single entry referencing this package. + // + template + static M + parse_directory_manifests (const path& f, + bool iu, + const repository_location& rl) + { + M r; + if (exists (f)) + r = parse_manifest (f, iu, rl); + else { - path f (fd / repositories_file); - - if (exists (f)) - rms = parse_manifest (f, ignore_unknown, rl); - else - rms.emplace_back (repository_manifest ()); // Add the base repository. + r.push_back (package_manifest ()); + r.back ().location = current_dir; } - // Produce the "skeleton" package manifest list. - // - git_package_manifests pms; - { - path f (fd / packages_file); - - if (exists (f)) - pms = parse_manifest (f, ignore_unknown, rl); - else - { - pms.push_back (package_manifest ()); - pms.back ().location = current_dir; - } - } + return r; + } + // Parse package manifests referenced by the package directory manifests. + // + static vector + parse_package_manifests (const common_options& co, + const dir_path& repo_dir, + const string& repo_fragment, + vector&& sms, + bool iu, + const repository_location& rl) + { vector fps; - fps.reserve (pms.size ()); + fps.reserve (sms.size ()); - // Parse package manifests. - // - for (package_manifest& sm: pms) + for (package_manifest& sm: sms) { assert (sm.location); @@ -260,7 +201,7 @@ namespace bpkg package_info (dr); }; - dir_path d (fd / path_cast (*sm.location)); + dir_path d (repo_dir / path_cast (*sm.location)); path f (d / path ("manifest")); if (!exists (f)) @@ -270,7 +211,7 @@ namespace bpkg { ifdstream ifs (f); manifest_parser mp (ifs, f.string ()); - package_manifest m (pkg_package_manifest (mp, ignore_unknown)); + package_manifest m (pkg_package_manifest (mp, iu)); // Save the package manifest, preserving its location. // @@ -290,7 +231,9 @@ namespace bpkg package_info (dr); } - // Fix-up the package version. + // Fix-up the package version. Note that the package may have the + // version module enable and the directory repository may well be a git + // repository. // const char* b (name_b (co)); @@ -361,7 +304,7 @@ namespace bpkg if (pr.wait ()) { fps.emplace_back (rep_fetch_data::package {move (sm), - nm.string ()}); + repo_fragment}); continue; } @@ -387,6 +330,157 @@ namespace bpkg } } + return fps; + } + + static rep_fetch_data + rep_fetch_dir (const common_options& co, + const repository_location& rl, + bool ignore_unknown) + { + assert (rl.absolute ()); + + dir_path rd (path_cast (rl.path ())); + + dir_repository_manifests rms ( + parse_repository_manifests ( + rd / repositories_file, + ignore_unknown, + rl)); + + dir_package_manifests pms ( + parse_directory_manifests ( + rd / packages_file, + ignore_unknown, + rl)); + + vector fps ( + parse_package_manifests (co, + rd, + string () /* repo_fragment */, + move (pms), + ignore_unknown, + rl)); + + // @@ Here we will need to go through packages and check if there is the + // selected package of the same version (without regards to the + // iteration component), and having the different package manifest + // hash. If that's the case then set the manifest version iteration + // component as the selected package version iteation + 1. + // + // @@ It also seems that the manifest hash should be stored in the (new) + // member of the package_location struct. Later this value will be + // transmitted into the selected package objects by pkg_unpack(). + // + // @@ Should we later check and fail if any available package contains + // several package locations with different non-zero (for those that + // come from the directory repo) manifest hashes? + // + return rep_fetch_data {move (rms), move (fps), nullptr}; + } + + static rep_fetch_data + rep_fetch_git (const common_options& co, + const dir_path* conf, + const repository_location& rl, + bool ignore_unknown) + { + // Plan: + // + // 1. Check repos_dir//: + // + // 1.a If does not exist, git-clone into temp_dir///. + // + // 1.a Otherwise, move as temp_dir// and git-fetch. + // + // 2. Move from temp_dir// to repos_dir/// + // + // 3. Check if repos_dir///repositories.manifest exists: + // + // 3.a If exists, load. + // + // 3.b Otherwise, synthesize repository list with base repository. + // + // 4. Check if repos_dir///packages.manifest exists: + // + // 4.a If exists, load. (into "skeleton" packages list to be filled?) + // + // 4.b Otherwise, synthesize as if single 'location: ./'. + // + // 5. For each package location obtained on step 4: + // + // 5.a Load repos_dir////manifest. + // + // 5.b Run 'b info: repos_dir////' and fix-up + // package version. + // + // 6. Return repository and package manifests (certificate is NULL). + // + + if (conf != nullptr && conf->empty ()) + conf = dir_exists (bpkg_dir) ? ¤t_dir : nullptr; + + assert (conf == nullptr || !conf->empty ()); + + // Clone or fetch the repository. + // + dir_path sd (repository_state (rl)); + + auto_rmdir rm (temp_dir / sd); + dir_path& td (rm.path); + + if (exists (td)) + rm_r (td); + + // If the git repository directory already exists, then we are fetching + // an already cloned repository. Move it to the temporary directory. + // + // In this case also set the filesystem_state_changed flag since we are + // modifying the repository filesystem state. + // + // In the future we can probably do something smarter about the flag, + // keeping it unset unless the repository state directory is really + // changed. + // + dir_path rd; + bool fetch (false); + if (conf != nullptr) + { + rd = *conf / repos_dir / sd; + + if (exists (rd)) + { + mv (rd, td); + filesystem_state_changed = true; + fetch = true; + } + } + + dir_path nm (fetch ? git_fetch (co, rl, td) : git_clone (co, rl, td)); + dir_path fd (td / nm); // Full directory path. + + // Parse manifests. + // + git_repository_manifests rms ( + parse_repository_manifests ( + fd / repositories_file, + ignore_unknown, + rl)); + + git_package_manifests pms ( + parse_directory_manifests ( + fd / packages_file, + ignore_unknown, + rl)); + + vector fps ( + parse_package_manifests (co, + fd, + nm.string (), + move (pms), + ignore_unknown, + rl)); + // Move the state directory to its proper place. // // If there is no configuration directory then we let auto_rmdir clean it @@ -411,6 +505,7 @@ namespace bpkg switch (rl.type ()) { case repository_type::pkg: return rep_fetch_pkg (co, conf, rl, iu); + case repository_type::dir: return rep_fetch_dir (co, rl, iu); case repository_type::git: return rep_fetch_git (co, conf, rl, iu); } @@ -592,19 +687,33 @@ namespace bpkg } } - // For git repositories that have neither prerequisites nor complements - // we use the root repository as the default complement. + // For dir and git repositories that have neither prerequisites nor + // complements we use the root repository as the default complement. // // This supports the common use case where the user has a single-package - // git repository and doesn't want to bother with the - // repositories.manifest file. This way their package will still pick up - // its dependencies from the configuration, without regards from which - // repositories they came from. + // repository and doesn't want to bother with the repositories.manifest + // file. This way their package will still pick up its dependencies from + // the configuration, without regards from which repositories they came + // from. // - if (rl.type () == repository_type::git && - r->complements.empty () && - r->prerequisites.empty ()) - r->complements.insert (lazy_shared_ptr (db, string ())); + switch (rl.type ()) + { + case repository_type::git: + case repository_type::dir: + { + if (r->complements.empty () && r->prerequisites.empty ()) + r->complements.insert (lazy_shared_ptr (db, string ())); + + break; + } + case repository_type::pkg: + { + // Pkg repository is a "strict" one, that requires all the + // prerequisites and complements to be listed. + // + break; + } + } // Save the changes to the repository object. // @@ -805,20 +914,24 @@ namespace bpkg if (repository_name (a)) { - lazy_shared_ptr rp (db, a); + r = lazy_shared_ptr (db, a); - if (ua.find (rp) != ua.end ()) - r = move (rp); - else + if (ua.find (r) == ua.end ()) fail << "repository '" << a << "' does not exist in this " << "configuration"; } else - //@@ TODO: check if exists in root & same location and avoid - // calling rep_add. Get rid of quiet mode. + { + repository_location rl (parse_location (a, nullopt /* type */)); + r = lazy_shared_ptr (db, rl.canonical_name ()); + + // If the repository is not the root complement yet or has + // a different location then we add it to the configuration. // - r = lazy_shared_ptr ( - db, rep_add (t, parse_location (a, nullopt /* type */))); + auto i (ua.find (r)); + if (i == ua.end () || i->load ()->location.url () != rl.url ()) + r = lazy_shared_ptr (db, rep_add (t, rl)); + } repos.emplace_back (move (r)); } -- cgit v1.1