diff options
Diffstat (limited to 'bpkg/system-package-manager-archive.cxx')
| -rw-r--r-- | bpkg/system-package-manager-archive.cxx | 794 |
1 files changed, 794 insertions, 0 deletions
diff --git a/bpkg/system-package-manager-archive.cxx b/bpkg/system-package-manager-archive.cxx new file mode 100644 index 0000000..d46e6d6 --- /dev/null +++ b/bpkg/system-package-manager-archive.cxx @@ -0,0 +1,794 @@ +// file : bpkg/system-package-manager-archive.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#include <bpkg/system-package-manager-archive.hxx> + +#include <map> + +#include <bpkg/diagnostics.hxx> + +#include <bpkg/pkg-bindist-options.hxx> + +using namespace butl; + +namespace bpkg +{ + system_package_manager_archive:: + system_package_manager_archive (bpkg::os_release&& osr, + const target_triplet& h, + string a, + optional<bool> progress, + const pkg_bindist_options* options) + : system_package_manager (move (osr), h, "", progress), ops (options) + { + if (!a.empty ()) + { + assert (ops != nullptr); + + try + { + target = target_triplet (a); + } + catch (const invalid_argument& e) + { + fail << "invalid --architecture target triplet value '" << a << "': " + << e; + } + + if (!ops->os_release_id_specified ()) + fail << "--architecture requires explict --os-release-id"; + + if (!ops->archive_install_root_specified () && + !ops->archive_install_config ()) + fail << "--architecture requires explict --archive-install-root"; + } + else + target = host; + + arch = target.string (); // Set since queried (e.g., JSON value). + } + + // env --chdir=<root> tar|zip ... <base>.<ext> <base> + // + // Return the archive file path. + // + static path + archive (const dir_path& root, + const string& base, + const string& e /* ext */) + { + // NOTE: similar code in build2 (libbuild2/dist/operation.cxx). + + path an (base + '.' + e); + path ap (root / an); + + // Use zip for .zip archives. Also recognize and handle a few well-known + // tar.xx cases (in case tar doesn't support -a or has other issues like + // MSYS). Everything else goes to tar in the auto-compress mode (-a). + // + // Note also that we pass the archive path as name (an) instead of path + // (ap) since we are running from the root directory (see below). + // + cstrings args; + + // Separate compressor (gzip, xz, etc) state. + // + size_t i (0); // Command line start or 0 if not used. + auto_rmfile out_rm; // Output file cleanup (must come first). + auto_fd out_fd; // Output file. + + if (e == "zip") + { + // On Windows we use libarchive's bsdtar (zip is an MSYS executable). + // + // While not explicitly stated, the compression-level option works + // for zip archives. + // +#ifdef _WIN32 + args = {"bsdtar", + "-a", // -a with the .zip extension seems to be the only way. + "--options=compression-level=9", + "-cf", an.string ().c_str (), + base.c_str (), + nullptr}; +#else + args = {"zip", + "-9", + "-rq", an.string ().c_str (), + base.c_str (), + nullptr}; +#endif + } + else + { + // On Windows we use libarchive's bsdtar with auto-compression (tar + // itself and quite a few compressors are MSYS executables). + // + // OpenBSD tar does not support --format but it appear ustar is the + // default (while this is not said explicitly in tar(1), it is said in + // pax(1) and confirmed on the mailing list). Nor does it support -a, at + // least as of 7.1 but we will let this play out naturally, in case this + // support gets added. + // + // Note also that in the future we may switch to libarchive in order to + // generate reproducible archives. + // + const char* l (nullptr); // Compression level (option). + +#ifdef _WIN32 + args = {"bsdtar", "--format", "ustar"}; + + if (e == "tar.gz" || e == "tar.xz") + l = "--options=compression-level=9"; +#else + args = {"tar" +#ifndef __OpenBSD__ + , "--format", "ustar" +#endif + }; + + // For gzip it's a good idea to use -9 by default. While for xz, -9 is + // not recommended as the default due to memory requirements, in our + // case (large binary archives on development machines), this is + // unlikely to be an issue. + // + // Note also that the compression level can be altered via the GZIP + // (GZIP_OPT also seems to work) and XZ_OPT environment variables, + // respectively. + // + const char* c (nullptr); + + if (e == "tar.gz") { c = "gzip"; l = "-9"; } + else if (e == "tar.xz") + { + // At least as of Mac OS 13 and Xcode 15, there is no standalone xz + // utility but tar seem to be capable of producing .tar.xz. + // +#ifdef __APPLE__ + l = "--options=compression-level=9"; +#else + c = "xz"; l = "-9"; +#endif + } + + if (c != nullptr) + { + args.push_back ("-cf"); + args.push_back ("-"); + args.push_back (base.c_str ()); + args.push_back (nullptr); i = args.size (); + args.push_back (c); + if (l != nullptr) + args.push_back (l); + args.push_back (nullptr); + args.push_back (nullptr); // Pipe end. + + try + { + out_fd = fdopen (ap, + fdopen_mode::out | fdopen_mode::binary | + fdopen_mode::truncate | fdopen_mode::create); + out_rm = auto_rmfile (ap); + } + catch (const io_error& e) + { + fail << "unable to open " << ap << ": " << e; + } + } + else +#endif + { + if (e != "tar") + { + args.push_back ("-a"); + if (l != nullptr) + args.push_back (l); + } + + args.push_back ("-cf"); + args.push_back (an.string ().c_str ()); + args.push_back (base.c_str ()); + args.push_back (nullptr); + } + } + + size_t what (0); // Failed program name index in args. + try + { + process_path app; // Archiver path. + process_path cpp; // Compressor path. + + app = process::path_search (args[what = 0]); + + if (i != 0) + cpp = process::path_search (args[what = i]); + + // Change the archiver's working directory to root. + // + process_env ape (app, root); + + // Note: print the command line unless quiet similar to other package + // manager implementations. + // + if (verb >= 1) + print_process (ape, args); + + what = 0; + process apr (app, + args.data (), // No auto-pipe. + 0 /* stdin */, + (i != 0 ? -1 : 1) /* stdout */, + 2 /* stderr */, + ape.cwd->string ().c_str (), + ape.vars); + + // Start the compressor if required. + // + process cpr; + if (i != 0) + { + what = i; + cpr = process (cpp, + args.data () + i, + apr.in_ofd.get () /* stdin */, + out_fd.get () /* stdout */, + 2 /* stderr */); + + cpr.in_ofd.reset (); // Close the archiver's stdout on our side. + } + + // Delay throwing until we diagnose both ends of the pipe. + // + bool fail (false); + + what = 0; + if (!apr.wait ()) + { + diag_record dr (error); + dr << args[0] << " exited with non-zero code"; + + if (verb == 0) + { + info << "command line: "; + print_process (dr, ape, args.data ()); + } + + fail = true; + } + + if (i != 0) + { + what = i; + if (!cpr.wait ()) + { + diag_record dr (error); + dr << args[i] << " exited with non-zero code"; + + if (verb == 0) + { + info << "command line: "; + print_process (dr, args.data () + i); + } + + fail = true; + } + } + + if (fail) + throw failed (); + } + catch (const process_error& e) + { + error << "unable to execute " << args[what] << ": " << e; + + if (e.child) + exit (1); + + throw failed (); + } + + out_rm.cancel (); + return ap; + } + + // NOTE: THE BELOW DESCRIPTION IS ALSO REWORDED IN BPKG-PKG-BINDIST(1). + // + // The overall plan is to invoke the build system and install all the + // packages directly from their bpkg locations into the binary package + // directory as a chroot. Then tar/zip this directory to produce one or more + // binary package archives. + // + auto system_package_manager_archive:: + generate (const packages& pkgs, + const packages& deps, + const strings& vars, + const dir_path& /*cfg_dir*/, + const package_manifest& pm, + const string& pt, + const small_vector<language, 1>& langs, + optional<bool> recursive_full, + bool first) -> binary_files + { + tracer trace ("system_package_manager_archive::generate"); + + assert (!langs.empty ()); // Should be effective. + + // We require explicit output root. + // + if (!ops->output_root_specified ()) + fail << "output root directory must be specified explicitly with " + << "--output-root|-o"; + + const dir_path& out (ops->output_root ()); // Cannot be empty. + + const shared_ptr<selected_package>& sp (pkgs.front ().selected); + const package_name& pn (sp->name); + const version& pv (sp->version); + + // Use version without iteration in paths, etc. + // + string pvs (pv.string (false /* ignore_revision */, + true /* ignore_iteration */)); + + bool lib (pt == "lib"); + bool priv (ops->private_ ()); // Private installation. + + // Return true if this package uses the specified language, only as + // interface language if intf_only is true. + // + auto lang = [&langs] (const char* n, bool intf_only = false) -> bool + { + return find_if (langs.begin (), langs.end (), + [n, intf_only] (const language& l) + { + return (!intf_only || !l.impl) && l.name == n; + }) != langs.end (); + }; + + bool lang_c (lang ("c")); + bool lang_cxx (lang ("c++")); + bool lang_cc (lang ("cc")); + + if (verb >= 3) + { + auto print_status = [] (diag_record& dr, const selected_package& p) + { + dr << (p.substate == package_substate::system ? "sys:" : "") + << p.name << ' ' << p.version; + }; + + { + diag_record dr (trace); + dr << "package: " ; + print_status (dr, *sp); + } + + for (const package& p: deps) + { + diag_record dr (trace); + dr << "dependency: "; + print_status (dr, *p.selected); + } + } + + // Should we override config.install.* or just use whatever configured + // (sans the root)? While using whatever configure seemed like a good idea + // at first, it's also a good idea to have the ability to tweak the + // installation directory structure on the per-platform basis (like, say, + // lib/libexec split or pkgconfig/ location on FreeBSD; in a sense, the + // user may choose to install to /usr and it would be good if things ended + // up in the expected places -- this is still a @@ TODO). + // + // So unless instructed otherwise with --archive-install-config, we + // override every config.install.* variable in order not to pick anything + // configured. Note that we add some more in the command line below. + // + // We make use of the <project> substitution since in the recursive mode + // we may be installing multiple projects. Note that the <private> + // directory component is automatically removed if this functionality is + // not enabled. + // + bool ovr_install (!ops->archive_install_config ()); + + strings config; + { + const string& c (target.class_); + + dir_path root; + if (ops->archive_install_root_specified ()) + { + // If specified, we override it even with --archive-install-config. + // + root = ops->archive_install_root (); // Cannot be empty. + } + else if (ovr_install) + { + if (c == "windows") + { + // Using C:\<project>\ looks like the best we can do (if the + // installation is not relocatable, at least related packages will + // be grouped together). + // + root = dir_path ("C:\\" + pm.effective_project ().string ()); + } + else + root = dir_path ("/usr/local"); + } + + auto add = [&config] (auto&& v) + { + config.push_back (string ("config.install.") + v); + }; + + if (!root.empty ()) + add ("root='" + root.representation () + '\''); + + if (ovr_install) + { + add ("data_root=root/"); + add ("exec_root=root/"); + + add ("bin=exec_root/bin/"); + add ("sbin=exec_root/sbin/"); + + add ("lib=exec_root/lib/<private>/"); + add ("libexec=exec_root/libexec/<private>/<project>/"); + add ("pkgconfig=lib/pkgconfig/"); + + add ("etc=data_root/etc/"); + add ("include=data_root/include/<private>/"); + add ("include_arch=include/"); + add ("share=data_root/share/"); + add ("data=share/<private>/<project>/"); + add ("buildfile=share/build2/export/<project>/"); + + add ("doc=share/doc/<private>/<project>/"); + add ("legal=doc/"); + add ("man=share/man/"); + add ("man1=man/man1/"); + add ("man2=man/man2/"); + add ("man3=man/man3/"); + add ("man4=man/man4/"); + add ("man5=man/man5/"); + add ("man6=man/man6/"); + add ("man7=man/man7/"); + add ("man8=man/man8/"); + + add ("private=" + (priv ? pn.string () : "[null]")); + + // If this is a C-based language, add rpath for private installation, + // unless targeting Windows. + // + if (priv && (lang_c || lang_cxx || lang_cc) && c != "windows") + { + dir_path l ((dir_path (root) /= "lib") /= pn.string ()); + config.push_back ("config.bin.rpath='" + l.representation () + '\''); + } + } + } + + // Add user-specified configuration variables last to allow them to + // override anything. + // + for (const string& v: vars) + config.push_back (v); + + // Note that we can use weak install scope for the auto recursive mode + // since we know dependencies cannot be spread over multiple linked + // configurations. + // + string scope (!recursive_full || *recursive_full ? "project" : "weak"); + + // The plan is to create the archive directory (with the same name as the + // archive base; we call it "destination directory") inside the output + // directory and then tar/zip it up placing the resulting archives next to + // it. + // + // Let's require clean output directory to keep things simple. + // + // Also, by default, we are going to keep all the intermediate files on + // failure for troubleshooting. + // + if (first && exists (out) && !empty (out)) + { + if (!ops->wipe_output ()) + fail << "output root directory " << out << " is not empty" << + info << "use --wipe-output to clean it up but be careful"; + + rm_r (out, false); + } + + // NOTE: THE BELOW DESCRIPTION IS ALSO REWORDED IN BPKG-PKG-BINDIST(1). + // + // Our archive directory/file base have the following form: + // + // <package>-<version>-<build_metadata> + // + // Where <build_metadata> in turn has the following form (unless overriden + // with --archive-build-mata): + // + // <cpu>-<os>[-<langrt>...] + // + // For example: + // + // hello-1.2.3-x86_64-windows10 + // libhello-1.2.3-x86_64-windows10-msvc17.4 + // libhello-1.2.3-x86_64-debian11-gcc12-rust1.62 + // + bool md_s (ops->archive_build_meta_specified ()); + const string& md (ops->archive_build_meta ()); + + bool md_f (false); + bool md_b (false); + if (md_s && !md.empty ()) + { + md_f = md.front () == '+'; + md_b = md.back () == '+'; + + if (md_f && md_b) // Note: covers just `+`. + fail << "invalid build metadata '" << md << "'"; + } + + vector<reference_wrapper<const pair<const string, string>>> langrt; + if (!md_s || md_f || md_b) + { + // First collect the interface languages and then add implementation. + // This way if different languages map to the same runtimes (e.g., C and + // C++ mapped to gcc12), then we will always prefer the interface + // version over the implementation (which could be different, for + // example, libstdc++6 vs libstdc++-12-dev; but it's not clear how this + // will be specified, won't they end up with different names as opposed + // to gcc6 and gcc12 -- still fuzzy/unclear). + // + // @@ We will need to split id and version to be able to pick the + // highest version. + // + // @@ Maybe we should just do "soft" version like in <distribution>? + // + // Note that we allow multiple values for the same language to support + // cases like --archive-lang cc=gcc12 --archive-lang cc=g++12. But + // we treat an empty value as a request to clear all the previous + // entries. + // + + auto find = [] (const std::multimap<string, string>& m, const string& n) + { + auto p (m.equal_range (n)); + + if (p.first == p.second) + { + // If no mapping for c/c++, fallback to cc. + // + if (n == "c" || n == "c++") + p = m.equal_range ("cc"); + } + + return p; + }; + + // We don't want to clear entries specified with --*-lang with an empty + // value specified with --*-lang-impl. + // + size_t clear_limit (0); + + auto add = [&langrt, &clear_limit] (const pair<const string, string>& p) + { + // Suppress duplicates. + // + if (!p.second.empty ()) + { + if (find_if (langrt.begin (), langrt.end (), + [&p] (const pair<const string, string>& x) + { + // @@ TODO: keep highest version. + return p.second == x.second; + }) == langrt.end ()) + { + langrt.push_back (p); + } + } + else if (clear_limit != langrt.size ()) + { + for (auto i (langrt.begin () + clear_limit); i != langrt.end (); ) + { + if (i->get ().first == p.first) + i = langrt.erase (i); + else + ++i; + } + } + }; + + auto& implm (ops->archive_lang_impl ()); + + // The interface/implementation distinction is only relevant to + // libraries. For everything else we treat all the languages as + // implementation. + // + if (lib) + { + auto& intfm (ops->archive_lang ()); + + for (const language& l: langs) + { + if (l.impl) + continue; + + auto p (find (intfm, l.name)); + + if (p.first == p.second) + p = find (implm, l.name); + + if (p.first == p.second) + fail << "no runtime mapping for language " << l.name << + info << "consider specifying with --archive-lang[-impl]" << + info << "or alternatively specify --archive-build-meta"; + + for (auto i (p.first); i != p.second; ++i) + add (*i); + } + + clear_limit = langrt.size (); + } + + for (const language& l: langs) + { + if (lib && !l.impl) + continue; + + auto p (find (implm, l.name)); + + if (p.first == p.second) + continue; // Unimportant. + + for (auto i (p.first); i != p.second; ++i) + add (*i); + } + } + + // If there is no split, reduce to empty key and empty filter. + // + binary_files r; + for (const pair<const string, string>& kf: + ops->archive_split_specified () + ? ops->archive_split () + : std::map<string, string> {{string (), string ()}}) + { + string sys_name (pn.string ()); + + if (!kf.first.empty ()) + sys_name += '-' + kf.first; + + string base (sys_name); + + base += '-' + pvs; + + if (md_s && !(md_f || md_b)) + { + if (!md.empty ()) + base += '-' + md; + } + else + { + if (md_b) + { + base += '-'; + base.append (md, 0, md.size () - 1); + } + + if (!ops->archive_no_cpu ()) + base += '-' + target.cpu; + + if (!ops->archive_no_os ()) + base += '-' + os_release.name_id + os_release.version_id; + + for (const pair<const string, string>& p: langrt) + base += '-' + p.second; + + if (md_f) + { + base += '-'; + base.append (md, 1, md.size () - 1); + } + } + + dir_path dst (out / dir_path (base)); + mk_p (dst); + + // Install. + // + // In a sense, this is a special version of pkg-install. + // + { + strings dirs; + for (const package& p: pkgs) + dirs.push_back (p.out_root.representation ()); + + string filter; + if (!kf.second.empty ()) + filter = "config.install.filter=" + kf.second; + + run_b (*ops, + verb_b::normal, + (ops->jobs_specified () + ? strings ({"--jobs", to_string (ops->jobs ())}) + : strings ()), + "config.install.chroot='" + dst.representation () + '\'', + (ovr_install ? "config.install.sudo=[null]" : nullptr), + (!filter.empty () ? filter.c_str () : nullptr), + config, + "!config.install.scope=" + scope, + "install:", + dirs); + + // @@ TODO: call install.json? Or manifest-install.json. Place in + // data/ (would need support in build2 to use install.* values)? + // +#if 0 + args.push_back ("!config.install.manifest=-"); +#endif + } + + if (ops->archive_prepare_only ()) + { + if (verb >= 1) + text << "prepared " << dst; + + continue; + } + + // Create the archive. + // + // Should the default archive type be based on host or target? I guess + // that depends on where the result will be unpacked, and it feels like + // target is more likely. + // + // @@ What about the ownerhip of the resulting file in the archive? + // We don't do anything for source archives, not sure why we should + // do something here. + // + for (string t: (ops->archive_type_specified () + ? ops->archive_type () + : strings {target.class_ == "windows" ? "zip" : "tar.xz"})) + { + // Help the user out if the extension is specified with the leading + // dot. + // + if (t.size () > 1 && t.front () == '.') + t.erase (0, 1); + + path f (archive (out, base, t)); + + // Using archive type as file type seems appropriate. Add key before + // the archive type, if any. + // + if (!kf.first.empty ()) + t = kf.first + '.' + t; + + r.push_back (binary_file {move (t), move (f), sys_name}); + } + + // Cleanup intermediate files unless requested not to. + // + if (!ops->keep_output ()) + { + rm_r (dst); + } + } + + return r; + } + + optional<const system_package_status*> system_package_manager_archive:: + status (const package_name&, const available_packages*) + { + assert (false); + return nullopt; + } + + void system_package_manager_archive:: + install (const vector<package_name>&) + { + assert (false); + } +} |