aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBoris Kolpackov <boris@codesynthesis.com>2024-02-26 09:21:47 +0200
committerBoris Kolpackov <boris@codesynthesis.com>2024-02-26 09:21:47 +0200
commit82ff2e7df2243b679aadbc6cc120a2b5f7ee73b3 (patch)
tree616e33051cf012f9790641f0127901f0c7d6ed38
parent4a2a3bd5033744c31377d31ca54be00622280a1b (diff)
Add ability to serialize compilation/linking in cc rules
Specifically, both the C/C++ compiler and link rules now recognize the cc.serialize boolean variable which instructs them to compiler/link serially with regards to any other recipe. This is primarily useful when compiling large translation units or linking large binaries that require so much memory that doing that in parallel with other compilation/linking jobs is likely to summon the OOM killer. For example: obj{memory-hog}: cc.serialize = true
-rw-r--r--libbuild2/c/init.cxx1
-rw-r--r--libbuild2/cc/common.hxx1
-rw-r--r--libbuild2/cc/compile-rule.cxx10
-rw-r--r--libbuild2/cc/init.cxx7
-rw-r--r--libbuild2/cc/link-rule.cxx33
-rw-r--r--libbuild2/cxx/init.cxx1
6 files changed, 45 insertions, 8 deletions
diff --git a/libbuild2/c/init.cxx b/libbuild2/c/init.cxx
index 10b3d6a..8bc2f7d 100644
--- a/libbuild2/c/init.cxx
+++ b/libbuild2/c/init.cxx
@@ -345,6 +345,7 @@ namespace build2
vp["cc.module_name"],
vp["cc.importable"],
vp["cc.reprocess"],
+ vp["cc.serialize"],
vp.insert<string> ("c.preprocessed"), // See cxx.preprocessed.
nullptr, // No __symexport (no modules).
diff --git a/libbuild2/cc/common.hxx b/libbuild2/cc/common.hxx
index 6347005..cb85632 100644
--- a/libbuild2/cc/common.hxx
+++ b/libbuild2/cc/common.hxx
@@ -117,6 +117,7 @@ namespace build2
const variable& c_module_name; // cc.module_name
const variable& c_importable; // cc.importable
const variable& c_reprocess; // cc.reprocess
+ const variable& c_serialize; // cc.serialize
const variable& x_preprocessed; // x.preprocessed
const variable* x_symexport; // x.features.symexport
diff --git a/libbuild2/cc/compile-rule.cxx b/libbuild2/cc/compile-rule.cxx
index d490c8e..2e4775e 100644
--- a/libbuild2/cc/compile-rule.cxx
+++ b/libbuild2/cc/compile-rule.cxx
@@ -7830,6 +7830,14 @@ namespace build2
if (!env.empty ())
env.push_back (nullptr);
+ // We have no choice but to serialize early if we want the command line
+ // printed shortly before actually executing the compiler. Failed that,
+ // it may look like we are still executing in parallel.
+ //
+ scheduler::alloc_guard jobs_ag;
+ if (!ctx.dry_run && cast_false<bool> (t[c_serialize]))
+ jobs_ag = scheduler::alloc_guard (*ctx.sched, phase_unlock (nullptr));
+
// With verbosity level 2 print the command line as if we are compiling
// the source file, not its preprocessed version (so that it's easy to
// copy and re-run, etc). Only at level 3 and above print the real deal.
@@ -7994,6 +8002,8 @@ namespace build2
throw failed ();
}
+ jobs_ag.deallocate ();
+
if (md.deferred_failure)
fail << "expected error exit status from " << x_lang << " compiler";
}
diff --git a/libbuild2/cc/init.cxx b/libbuild2/cc/init.cxx
index 1bddbb2..e124450 100644
--- a/libbuild2/cc/init.cxx
+++ b/libbuild2/cc/init.cxx
@@ -185,6 +185,13 @@ namespace build2
//
vp.insert<bool> ("cc.reprocess");
+ // Execute serially with regards to any other recipe. This is primarily
+ // useful when compiling large translation units or linking large
+ // binaries that require so much memory that doing that in parallel with
+ // other compilation/linking jobs is likely to summon the OOM killer.
+ //
+ vp.insert<bool> ("cc.serialize");
+
// Register scope operation callback.
//
// It feels natural to clean up sidebuilds as a post operation but that
diff --git a/libbuild2/cc/link-rule.cxx b/libbuild2/cc/link-rule.cxx
index 704fb47..08a60b9 100644
--- a/libbuild2/cc/link-rule.cxx
+++ b/libbuild2/cc/link-rule.cxx
@@ -3969,6 +3969,14 @@ namespace build2
try_rmfile (relt, true);
}
+ // We have no choice but to serialize early if we want the command line
+ // printed shortly before actually executing the linker. Failed that, it
+ // may look like we are still executing in parallel.
+ //
+ scheduler::alloc_guard jobs_ag;
+ if (!ctx.dry_run && cast_false<bool> (t[c_serialize]))
+ jobs_ag = scheduler::alloc_guard (*ctx.sched, phase_unlock (nullptr));
+
if (verb == 1)
print_diag (lt.static_library () ? "ar" : "ld", t);
else if (verb == 2)
@@ -3989,10 +3997,15 @@ namespace build2
//
// Note that we are not going to bother with oargs for this.
//
+ // Note also that we now have scheduler::serialize() which allows us to
+ // block until full parallelism is available (this mode can currently
+ // be forced with cc.serialize=true; maybe we should invent something
+ // like config.cc.link_serialize or some such which can be used when
+ // LTO is enabled).
+ //
string jobs_arg;
- scheduler::alloc_guard jobs_extra;
- if (!lt.static_library ())
+ if (!ctx.dry_run && !lt.static_library ())
{
switch (ctype)
{
@@ -4008,8 +4021,10 @@ namespace build2
auto i (find_option_prefix ("-flto", args.rbegin (), args.rend ()));
if (i != args.rend () && strcmp (*i, "-flto=auto") == 0)
{
- jobs_extra = scheduler::alloc_guard (*ctx.sched, 0);
- jobs_arg = "-flto=" + to_string (1 + jobs_extra.n);
+ if (jobs_ag.n == 0) // Might already have (see above).
+ jobs_ag = scheduler::alloc_guard (*ctx.sched, 0);
+
+ jobs_arg = "-flto=" + to_string (1 + jobs_ag.n);
*i = jobs_arg.c_str ();
}
break;
@@ -4027,8 +4042,10 @@ namespace build2
strcmp (*i, "-flto=thin") == 0 &&
!find_option_prefix ("-flto-jobs=", args))
{
- jobs_extra = scheduler::alloc_guard (*ctx.sched, 0);
- jobs_arg = "-flto-jobs=" + to_string (1 + jobs_extra.n);
+ if (jobs_ag.n == 0) // Might already have (see above).
+ jobs_ag = scheduler::alloc_guard (*ctx.sched, 0);
+
+ jobs_arg = "-flto-jobs=" + to_string (1 + jobs_ag.n);
args.insert (i.base (), jobs_arg.c_str ()); // After -flto=thin.
}
break;
@@ -4200,8 +4217,6 @@ namespace build2
if (!e)
throw failed ();
}
-
- jobs_extra.deallocate ();
}
catch (const process_error& e)
{
@@ -4281,6 +4296,8 @@ namespace build2
}
}
+ jobs_ag.deallocate ();
+
// For Windows generate (or clean up) rpath-emulating assembly.
//
if (tclass == "windows")
diff --git a/libbuild2/cxx/init.cxx b/libbuild2/cxx/init.cxx
index 7eedb4a..8159d18 100644
--- a/libbuild2/cxx/init.cxx
+++ b/libbuild2/cxx/init.cxx
@@ -951,6 +951,7 @@ namespace build2
vp["cc.module_name"],
vp["cc.importable"],
vp["cc.reprocess"],
+ vp["cc.serialize"],
// Ability to signal that source is already (partially) preprocessed.
// Valid values are 'none' (not preprocessed), 'includes' (no #include