https://github.com/ninja-build/ninja/pull/2634 From 1eb9398cf204885745e6f749bd2c06390699ba84 Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Mon, 16 Jun 2025 14:14:31 +0200 Subject: [PATCH 1/3] Add JobserverPool class. A new class implementing a GNU Jobserver pool of job slots. This only supports FIFO mode on Posix, to match the implementation of the client Jobserver class that uses the pool. --- CMakeLists.txt | 2 + configure.py | 2 + src/jobserver_pool.cc | 218 +++++++++++++++++++++++++++++++++++++ src/jobserver_pool.h | 47 ++++++++ src/jobserver_pool_test.cc | 67 ++++++++++++ 5 files changed, 336 insertions(+) create mode 100644 src/jobserver_pool.cc create mode 100644 src/jobserver_pool.h create mode 100644 src/jobserver_pool_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index cbb0f1aa28..c785afe4bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,6 +140,7 @@ add_library(libninja OBJECT src/graph.cc src/graphviz.cc src/jobserver.cc + src/jobserver_pool.cc src/json.cc src/line_printer.cc src/manifest_parser.cc @@ -285,6 +286,7 @@ if(BUILD_TESTING) src/explanations_test.cc src/graph_test.cc src/jobserver_test.cc + src/jobserver_pool_test.cc src/json_test.cc src/lexer_test.cc src/manifest_parser_test.cc diff --git a/configure.py b/configure.py index 03b5ce4272..0ffe5da8c6 100755 --- a/configure.py +++ b/configure.py @@ -551,6 +551,7 @@ def has_re2c() -> bool: 'graph', 'graphviz', 'jobserver', + 'jobserver_pool', 'json', 'line_printer', 'manifest_parser', @@ -655,6 +656,7 @@ def has_re2c() -> bool: 'explanations_test', 'graph_test', 'jobserver_test', + 'jobserver_pool_test', 'json_test', 'lexer_test', 'manifest_parser_test', diff --git a/src/jobserver_pool.cc b/src/jobserver_pool.cc new file mode 100644 index 0000000000..8465c5f6be --- /dev/null +++ b/src/jobserver_pool.cc @@ -0,0 +1,218 @@ +// Copyright 2025 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jobserver_pool.h" + +#include + +#include "util.h" + +#ifdef _WIN32 + +#include + +class Win32JobserverPool : public JobserverPool { + public: + static std::unique_ptr Create(size_t slot_count, + std::string* error) { + assert(slot_count > 1 && "slot_count must be 2 or higher"); + auto pool = std::unique_ptr(new Win32JobserverPool()); + if (!pool->InitWithSemaphore(slot_count, error)) + pool.reset(); + return pool; + } + + std::string GetEnvMakeFlagsValue() const override { + std::string result; + result.resize(sem_name_.size() + 32); + int ret = + snprintf(const_cast(result.data()), result.size(), + " -j%zd --jobserver-auth=%s", job_count_, sem_name_.c_str()); + if (ret < 0 || ret > static_cast(result.size())) + Fatal("Could not format Win32JobserverPool MAKEFLAGS!"); + + return result; + } + + virtual ~Win32JobserverPool() { + if (IsValid()) + ::CloseHandle(handle_); + } + + private: + Win32JobserverPool() = default; + + // CreateSemaphore returns NULL on failure. + bool IsValid() const { + // CreateSemaphoreA() returns NULL on failure, not INVALID_HANDLE_VALUE. + return handle_ != NULL; + } + + // Compute semaphore name for new instance. + static std::string GetSemaphoreName() { + // Use a per-process global counter to allow multiple instances of this + // class to run in the same process. Useful for unit-tests. + static int counter = 0; + counter += 1; + char name[64]; + snprintf(name, sizeof(name), "ninja_jobserver_pool_%d_%d", + GetCurrentProcessId(), counter); + return std::string(name); + } + + bool InitWithSemaphore(size_t slot_count, std::string* error) { + job_count_ = slot_count; + sem_name_ = GetSemaphoreName(); + LONG count = static_cast(slot_count - 1); + handle_ = ::CreateSemaphoreA(NULL, count, count, sem_name_.c_str()); + if (!IsValid()) { + *error = "Could not create semaphore: " + GetLastErrorString(); + return false; + } + return true; + } + + // Semaphore handle. + HANDLE handle_ = NULL; + + // Saved slot count. + size_t job_count_ = 0; + + // Semaphore name. + std::string sem_name_; +}; + +#else // !_WIN32 + +#include +#include +#include +#include +#include + +class PosixJobserverPool : public JobserverPool { + public: + static std::unique_ptr Create(size_t slot_count, + std::string* error) { + assert(slot_count > 1 && "slot_count must be 2 or higher"); + auto pool = std::unique_ptr(new PosixJobserverPool()); + if (!pool->InitWithFifo(slot_count, error)) { + pool.reset(); + } + return pool; + } + + std::string GetEnvMakeFlagsValue() const override { + std::string result; + if (!fifo_.empty()) { + result.resize(fifo_.size() + 32); + int ret = snprintf(const_cast(result.data()), result.size(), + " -j%zd --jobserver-auth=fifo:%s", job_count_, + fifo_.c_str()); + if (ret < 0 || ret > static_cast(result.size())) + Fatal("Could not format PosixJobserverPool MAKEFLAGS!"); + result.resize(static_cast(ret)); + } + return result; + } + + virtual ~PosixJobserverPool() { + if (write_fd_ >= 0) + ::close(write_fd_); + if (!fifo_.empty()) + ::unlink(fifo_.c_str()); + } + + private: + PosixJobserverPool() = default; + + // Fill the pool to satisfy |slot_count| job slots. This + // writes |slot_count - 1| bytes to the pipe to satisfy the + // implicit job slot requirement. + bool FillSlots(size_t slot_count, std::string* error) { + job_count_ = slot_count; + for (; slot_count > 1; --slot_count) { + // Write '+' into the pipe, just like GNU Make. Note that some + // implementations write '|' instead, but so far no client or pool + // implementation cares about the exact value, though the official spec + // says this might change in the future. + const char slot_char = '+'; + ssize_t ret = ::write(write_fd_, &slot_char, 1); + if (ret != 1) { + if (ret < 0 && errno == EINTR) + continue; + *error = + std::string("Could not fill job slots pool: ") + strerror(errno); + return false; + } + } + return true; + } + + bool InitWithFifo(size_t slot_count, std::string* error) { + const char* tmp_dir = getenv("TMPDIR"); + if (!tmp_dir) + tmp_dir = "/tmp"; + + fifo_.resize(strlen(tmp_dir) + 32); + int len = snprintf(const_cast(fifo_.data()), fifo_.size(), + "%s/NinjaFIFO%d", tmp_dir, getpid()); + if (len < 0) { + *error = "Cannot create fifo path!"; + return false; + } + fifo_.resize(static_cast(len)); + + int ret = mknod(fifo_.c_str(), S_IFIFO | 0666, 0); + if (ret < 0) { + *error = std::string("Cannot create fifo: ") + strerror(errno); + return false; + } + + do { + write_fd_ = ::open(fifo_.c_str(), O_RDWR | O_CLOEXEC); + } while (write_fd_ < 0 && errno == EINTR); + if (write_fd_ < 0) { + *error = std::string("Could not open fifo: ") + strerror(errno); + // Let destructor remove the fifo. + return false; + } + return FillSlots(slot_count, error); + } + + // Number of parallel job slots (including implicit one). + size_t job_count_ = 0; + + // A non-inheritable file descriptor to keep the pool alive. + int write_fd_ = -1; + + // Path to fifo, this will be empty when using an anonymous pipe. + std::string fifo_; +}; +#endif // !_WIN32 + +// static +std::unique_ptr JobserverPool::Create(size_t num_job_slots, + std::string* error) { + if (num_job_slots < 2) { + *error = "At least 2 job slots needed"; + return nullptr; + } + +#ifdef _WIN32 + return Win32JobserverPool::Create(num_job_slots, error); +#else // !_WIN32 + return PosixJobserverPool::Create(num_job_slots, error); +#endif // !_WIN32 +} diff --git a/src/jobserver_pool.h b/src/jobserver_pool.h new file mode 100644 index 0000000000..bb22242809 --- /dev/null +++ b/src/jobserver_pool.h @@ -0,0 +1,47 @@ +// Copyright 2025 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_JOBSERVER_POOL_H_ +#define NINJA_JOBSERVER_POOL_H_ + +#include +#include +#include + +/// JobserverPool implements a jobserver pool of job slots according +/// to the GNU Make protocol. Usage is the following: +/// +/// - Use Create() method to create new instances. +/// +/// - Retrieve the value of the MAKEFLAGS environment variable, and +/// ensure it is passed to each client. +/// +class JobserverPool { + public: + /// Destructor. + virtual ~JobserverPool() {} + + /// Create new instance to use |num_slots| job slots, using a specific + /// implementation mode. On failure, set |*error| and return null. + /// + /// It is an error to use a value of |num_slots| that is <= 1. + static std::unique_ptr Create(size_t num_job_slots, + std::string* error); + + /// Return the value of the MAKEFLAGS variable, corresponding to this + /// instance, to pass to sub-processes. + virtual std::string GetEnvMakeFlagsValue() const = 0; +}; + +#endif // NINJA_JOBSERVER_POOL_H_ diff --git a/src/jobserver_pool_test.cc b/src/jobserver_pool_test.cc new file mode 100644 index 0000000000..0882c4f441 --- /dev/null +++ b/src/jobserver_pool_test.cc @@ -0,0 +1,67 @@ +// Copyright 2025 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jobserver_pool.h" + +#include "jobserver.h" +#include "test.h" + +#ifndef _WIN32 +#include +#include +#endif + +TEST(JobserverPoolTest, DefaultPool) { + const size_t kSlotCount = 10; + std::string error; + auto pool = JobserverPool::Create(kSlotCount, &error); + ASSERT_TRUE(pool.get()) << error; + EXPECT_TRUE(error.empty()); + + std::string makeflags = pool->GetEnvMakeFlagsValue(); +#ifdef _WIN32 + std::string auth_prefix = " -j10 --jobserver-auth="; +#else // !_WIN32 + std::string auth_prefix = " -j10 --jobserver-auth=fifo:"; +#endif // !_WIN32 + ASSERT_EQ(auth_prefix, makeflags.substr(0, auth_prefix.size())); + + // Parse the MAKEFLAGS value to create a JobServer::Config + Jobserver::Config config; + ASSERT_TRUE( + Jobserver::ParseMakeFlagsValue(makeflags.c_str(), &config, &error)); + EXPECT_EQ(config.mode, Jobserver::Config::kModeDefault); + + // Create a client from the Config, and try to read all slots. + std::unique_ptr client = + Jobserver::Client::Create(config, &error); + EXPECT_TRUE(client.get()); + EXPECT_TRUE(error.empty()) << error; + + // First slot is always implicit. + Jobserver::Slot slot = client->TryAcquire(); + EXPECT_TRUE(slot.IsValid()); + EXPECT_TRUE(slot.IsImplicit()); + + // Then read kSlotCount - 1 slots from the pipe. + for (size_t n = 1; n < kSlotCount; ++n) { + slot = client->TryAcquire(); + EXPECT_TRUE(slot.IsValid()) << "Slot #" << n + 1; + EXPECT_TRUE(slot.IsExplicit()) << "Slot #" << n + 1; + } + + // Pool should be empty now, so next TryAcquire() will fail. + slot = client->TryAcquire(); + EXPECT_FALSE(slot.IsValid()); +} From 272132db92bc90eddade9a1987980e791409a5aa Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Mon, 16 Jun 2025 14:44:56 +0200 Subject: [PATCH 2/3] Implement GNU jobserver pool mode. Make Ninja provide a pool of GNU Make jobserver slots when invoked with the `--jobserver-pool` command-line option. - Introduce JobserverState class to manage the state of the jobserver pool and client instances for a given Ninja build. In particular, the methods ShouldSetupClient() and ShouldSetupPool() clarify under which conditions the pool or client should be created, and provide explanations for the decision. - All jobserver-related info / warnings are moved to the VERBOSE level, keeping the output of normal invocations small, and prevents modifying the unit-tests accordingly. - Update manual accordingly, detailing how everything works. --- doc/manual.asciidoc | 55 +++++++-- misc/jobserver_test.py | 79 ++++++++++++- src/build.h | 6 +- src/jobserver_pool.cc | 16 +-- src/ninja.cc | 245 +++++++++++++++++++++++++++++------------ 5 files changed, 309 insertions(+), 92 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 9bbabc205d..a8f7287d1d 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -192,12 +192,33 @@ GNU Jobserver support Since version 1.13., Ninja builds can follow the https://https://www.gnu.org/software/make/manual/html_node/Job-Slots.html[GNU Make jobserver] -client protocol. This is useful when Ninja is invoked as part of a larger -build system controlled by a top-level GNU Make instance, or any other -jobserver pool implementation, as it allows better coordination between -concurrent build tasks. +protocol. -This feature is automatically enabled under the following conditions: +The protocol is useful to efficiently control parallelism across a set of +concurrent and cooperating processes. This is useful when Ninja is invoked +as part of a larger build system controlled by a top-level Ninja or +GNU Make instance, or any other jobserver pool implementation. + +Ninja becomes a protocol client automatically if it detects the right +values in the `MAKEFLAGS` environment variable (see exact conditions below). + +Since version 1.14, Ninja can also be a protocol server, if needed, using +the `--jobserver-pool` command-line flag. + +In jobserver-enabled builds, there is one top-level "server" process which: + +- Sets up a shared pool of job tokens. +- Sets the `MAKEFLAGS` environment variable with special values + to reference the pool. +- Launches child processes (concurrent sub-commands). + +Said child processes can be protocol clients if they: + +- Recognize the special `MAKEFLAGS` values specific to the protocol. +- Use it to access the shared pool to acquire and release job tokens + during the build. + +Ninja automatically becomes a protocol client during builds when: - Dry-run (i.e. `-n` or `--dry-run`) is not enabled. @@ -208,18 +229,30 @@ This feature is automatically enabled under the following conditions: jobserver mode using `--jobserver-auth=SEMAPHORE_NAME` on Windows, or `--jobserver-auth=fifo:PATH` on Posix. -In this case, Ninja will use the jobserver pool of job slots to control -parallelism, instead of its default parallel implementation. - -Note that load-average limitations (i.e. when using `-l`) -are still being enforced in this mode. - IMPORTANT: On Posix, only the FIFO-based version of the protocol, which is implemented by GNU Make 4.4 and higher, is supported. Ninja will detect when a pipe-based jobserver is being used (i.e. when `MAKEFLAGS` contains `--jobserver-auth=,`) and will print a warning, but will otherwise ignore it. +Using `--jobserver-pool` will make Ninja act as a protocol server, unless +any of these are true: + +- An existing pool was detected, as this keeps all processes cooperating + properly. + +- `-j1` is used on the command-line, as this is asking Ninja to explicitly + not perform parallel builds. + +- Dry-run is enabled. + +The size of the pool setup by Ninja matches its parallel count, determined +by the `-j` option, or auto-detected if that one is not provided. + +The load-average limitations (i.e. when using `-l`) are still being +enforced in both modes. + + Environment variables ~~~~~~~~~~~~~~~~~~~~~ diff --git a/misc/jobserver_test.py b/misc/jobserver_test.py index 0378c98870..53410ec8e2 100755 --- a/misc/jobserver_test.py +++ b/misc/jobserver_test.py @@ -31,6 +31,7 @@ # Set this to True to debug command invocations. _DEBUG = False +_DEBUG = True default_env = dict(os.environ) default_env.pop("NINJA_STATUS", None) @@ -272,7 +273,7 @@ def run_ninja_with_jobserver_pipe(args): ret.check_returncode() return ret.stdout, ret.stderr - output, error = run_ninja_with_jobserver_pipe(["all"]) + output, error = run_ninja_with_jobserver_pipe(["-v", "all"]) if _DEBUG: print(f"OUTPUT [{output}]\nERROR [{error}]\n", file=sys.stderr) self.assertTrue(error.find("Pipe-based protocol is not supported!") >= 0) @@ -282,7 +283,7 @@ def run_ninja_with_jobserver_pipe(args): # Using an explicit -j ignores the jobserver pool. b.ninja_clean() - output, error = run_ninja_with_jobserver_pipe(["-j1", "all"]) + output, error = run_ninja_with_jobserver_pipe(["-v", "-j1", "all"]) if _DEBUG: print(f"OUTPUT [{output}]\nERROR [{error}]\n", file=sys.stderr) self.assertFalse(error.find("Pipe-based protocol is not supported!") >= 0) @@ -290,6 +291,80 @@ def run_ninja_with_jobserver_pipe(args): max_overlaps = compute_max_overlapped_spans(b.path, task_count) self.assertEqual(max_overlaps, 1) + def test_jobserver_pool_mode(self): + task_count = 4 + build_plan = generate_build_plan(task_count) + with BuildDir(build_plan) as b: + # First, run the full tasks with with {task_count} tokens, this should allow all + # tasks to run in parallel. + ret = b.ninja_run( + ninja_args=["--jobserver-pool", "all"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, task_count) + + # Second, use 2 tokens only, and verify that this was enforced by Ninja and + # that both a pool and a client were setup by Ninja. + b.ninja_clean() + ret = b.ninja_spawn( + ["-j2", "--jobserver-pool", "--verbose", "all"], + capture_output=True, + ) + self.assertEqual(ret.returncode, 0) + self.assertTrue( + "ninja: Creating jobserver pool for 2 parallel jobs" in ret.stdout, + msg="Ninja failed to setup jobserver pool!", + ) + self.assertTrue( + "ninja: Jobserver mode detected: " in ret.stdout, + msg="Ninja failed to setup jobserver client!", + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, 2) + + # Third, verify that --jobs=1 serializes all tasks. + b.ninja_clean() + b.ninja_run( + ["--jobserver-pool", "-j1", "all"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, 1) + + # On Linux, use taskset to limit the number of available cores to 1 + # and verify that the jobserver overrides the default Ninja parallelism + # and that {task_count} tasks are still spawned in parallel. + if platform.system() == "Linux": + # First, run without a jobserver, with a single CPU, Ninja will + # use a parallelism of 2 in this case (GuessParallelism() in ninja.cc) + b.ninja_clean() + b.ninja_run( + ["all"], + prefix_args=["taskset", "-c", "0"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, 2) + + # Now with a jobserver with {task_count} tasks. + b.ninja_clean() + b.ninja_run( + ["--jobserver-pool", f"-j{task_count}", "all"], + prefix_args=["taskset", "-c", "0"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, task_count) + + def test_jobserver_pool_mode_ignored_with_existing_pool(self): + task_count = 4 + build_plan = generate_build_plan(task_count) + with BuildDir(build_plan) as b: + # Setup a top-level pool with 2 jobs, and verify that `--jobserver-pool` respected it. + ret = b.ninja_run( + ninja_args=["--jobserver-pool", "all"], + prefix_args=[sys.executable, "-S", _JOBSERVER_POOL_SCRIPT, "--jobs=2"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, 2) + def _test_MAKEFLAGS_value( self, ninja_args: T.List[str] = [], prefix_args: T.List[str] = [] ): diff --git a/src/build.h b/src/build.h index 0531747be1..afdbc8e83e 100644 --- a/src/build.h +++ b/src/build.h @@ -184,8 +184,12 @@ struct BuildConfig { }; Verbosity verbosity = NORMAL; bool dry_run = false; + /// Number of concurrent jobs, auto-detected or specified explicitly. int parallelism = 1; - bool disable_jobserver_client = false; + /// True if -j was used on the command line. + bool explicit_parallelism = false; + /// True if --jobserver-pool was used on the command line. + bool jobserver_pool = false; int failures_allowed = 1; /// The maximum load average we must not exceed. A negative value /// means that we do not have any limit. diff --git a/src/jobserver_pool.cc b/src/jobserver_pool.cc index 8465c5f6be..0e9faa64ed 100644 --- a/src/jobserver_pool.cc +++ b/src/jobserver_pool.cc @@ -142,20 +142,22 @@ class PosixJobserverPool : public JobserverPool { // implicit job slot requirement. bool FillSlots(size_t slot_count, std::string* error) { job_count_ = slot_count; - for (; slot_count > 1; --slot_count) { + while (slot_count > 1) { // Write '+' into the pipe, just like GNU Make. Note that some // implementations write '|' instead, but so far no client or pool // implementation cares about the exact value, though the official spec // says this might change in the future. const char slot_char = '+'; ssize_t ret = ::write(write_fd_, &slot_char, 1); - if (ret != 1) { - if (ret < 0 && errno == EINTR) - continue; - *error = - std::string("Could not fill job slots pool: ") + strerror(errno); - return false; + if (ret == 1) { + slot_count--; + continue; } + if (ret < 0 && errno == EINTR) + continue; + + *error = std::string("Could not fill job slots pool: ") + strerror(errno); + return false; } return true; } diff --git a/src/ninja.cc b/src/ninja.cc index 92d0761aa3..70af71c3d4 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -47,6 +47,7 @@ #include "graph.h" #include "graphviz.h" #include "jobserver.h" +#include "jobserver_pool.h" #include "json.h" #include "manifest_parser.h" #include "metrics.h" @@ -85,6 +86,37 @@ struct Options { bool phony_cycle_should_err; }; +/// Helper class used to manage the state of jobserver pool and client +/// handling in a given NinjaMain instance. +struct JobserverState { + JobserverState(const BuildConfig& config, Status* status) { + SetupPool(config, status); + SetupClient(config, status); + } + + /// Return pointer to client instance or nullptr. + Jobserver::Client* client() { return client_.get(); } + + /// Transfer ownership of client to caller. + std::unique_ptr TakeClient() { return std::move(client_); } + + private: + /// Detect whether an external and supported jobserver pool is available. + /// On success, set |*config| and return true. + /// On failure, set |*error| and return false. + /// A pool with an unsupported scheme is an error. + bool HasExternalJobserverPool(std::string* error); + bool ShouldSetupPool(const BuildConfig& config, std::string* reason); + bool ShouldSetupClient(const BuildConfig& config, std::string* reason); + void SetupPool(const BuildConfig& config, Status* status); + void SetupClient(const BuildConfig& config, Status* status); + + std::string makeflags_; + Jobserver::Config jobserver_config_; + std::unique_ptr pool_; + std::unique_ptr client_; +}; + /// The Ninja main() loads up a series of data structures; various tools need /// to poke into these, so store them as fields on an object. struct NinjaMain : public BuildLogUser { @@ -164,10 +196,6 @@ struct NinjaMain : public BuildLogUser { /// and record that in the edge itself. It will be used for ETA prediction. void ParsePreviousElapsedTimes(); - /// Create a jobserver client if needed. Return a nullptr value if - /// not. Prints info and warnings to \a status. - std::unique_ptr SetupJobserverClient(Status* status); - /// Build the targets listed on the command line. /// @return an exit code. ExitStatus RunBuild(int argc, char** argv, Status* status); @@ -225,29 +253,37 @@ struct Tool { /// Print usage information. void Usage(const BuildConfig& config) { - fprintf(stderr, -"usage: ninja [options] [targets...]\n" -"\n" -"if targets are unspecified, builds the 'default' target (see manual).\n" -"\n" -"options:\n" -" --version print ninja version (\"%s\")\n" -" -v, --verbose show all command lines while building\n" -" --quiet don't show progress status, just command output\n" -"\n" -" -C DIR change to DIR before doing anything else\n" -" -f FILE specify input build file [default=build.ninja]\n" -"\n" -" -j N run N jobs in parallel (0 means infinity) [default=%d on this system]\n" -" -k N keep going until N jobs fail (0 means infinity) [default=1]\n" -" -l N do not start new jobs if the load average is greater than N\n" -" -n dry run (don't run commands but act like they succeeded)\n" -"\n" -" -d MODE enable debugging (use '-d list' to list modes)\n" -" -t TOOL run a subtool (use '-t list' to list subtools)\n" -" terminates toplevel options; further flags are passed to the tool\n" -" -w FLAG adjust warnings (use '-w list' to list warnings)\n", - kNinjaVersion, config.parallelism); + fprintf( + stderr, + "usage: ninja [options] [targets...]\n" + "\n" + "if targets are unspecified, builds the 'default' target (see manual).\n" + "\n" + "options:\n" + " --version print ninja version (\"%s\")\n" + " -v, --verbose show all command lines while building\n" + " --quiet don't show progress status, just command output\n" + "\n" + " -C DIR change to DIR before doing anything else\n" + " -f FILE specify input build file [default=build.ninja]\n" + "\n" + " -j N run N jobs in parallel (0 means infinity) [default=%d on " + "this system]\n" + " -k N keep going until N jobs fail (0 means infinity) [default=1]\n" + " -l N do not start new jobs if the load average is greater than N\n" + " -n dry run (don't run commands but act like they succeeded)\n" + "\n" + " -d MODE enable debugging (use '-d list' to list modes)\n" + " -t TOOL run a subtool (use '-t list' to list subtools)\n" + " terminates toplevel options; further flags are passed to the tool\n" + " -w FLAG adjust warnings (use '-w list' to list warnings)\n\n" + + " --jobserver-pool\n" + " setup a GNU jobserver pool of job slots matching the\n" + " current parallel job configuration. Ignored if -j1 is\n" + " specified explicitly, or if an existing pool is detected\n\n", + + kNinjaVersion, config.parallelism); } /// Choose a default value for the -j (parallelism) flag. @@ -1546,47 +1582,115 @@ bool NinjaMain::EnsureBuildDirExists() { return true; } -std::unique_ptr NinjaMain::SetupJobserverClient( - Status* status) { - // Empty result by default. - std::unique_ptr result; - - // If dry-run or explicit job count, don't even look at MAKEFLAGS - if (config_.disable_jobserver_client) - return result; - +bool JobserverState::HasExternalJobserverPool(std::string* error) { const char* makeflags = getenv("MAKEFLAGS"); + makeflags_ = makeflags ? makeflags : ""; if (!makeflags) { - // MAKEFLAGS is not defined. - return result; + return false; } - std::string err; - Jobserver::Config jobserver_config; - if (!Jobserver::ParseNativeMakeFlagsValue(makeflags, &jobserver_config, - &err)) { + if (!Jobserver::ParseNativeMakeFlagsValue(makeflags, &jobserver_config_, + error)) { // MAKEFLAGS is defined but could not be parsed correctly. - if (config_.verbosity > BuildConfig::QUIET) - status->Warning("Ignoring jobserver: %s [%s]", err.c_str(), makeflags); - return result; + return false; } + if (!jobserver_config_.HasMode()) { + // This happens when the feature is disabled explicitly in MAKEFLAGS + // e.g. using "--jobserver-fds=-1,-1" + *error = "external pool is disabled"; + return false; + } + return true; +} - if (!jobserver_config.HasMode()) { - // MAKEFLAGS is defined, but does not describe a jobserver mode. - return result; +bool JobserverState::ShouldSetupPool(const BuildConfig& config, + std::string* reason) { + if (config.parallelism == 1) { + *reason = "no parallelism (-j1) specified"; + return false; + } + if (config.dry_run) { + *reason = "dry-run mode"; + return false; } + if (HasExternalJobserverPool(reason)) { + *reason = "external pool detected"; + return false; + } + if (!reason->empty()) + return false; - if (config_.verbosity > BuildConfig::NO_STATUS_UPDATE) { - status->Info("Jobserver mode detected: %s", makeflags); + if (!config.jobserver_pool) { + return false; } + *reason = ""; + return true; +} - result = Jobserver::Client::Create(jobserver_config, &err); - if (!result.get()) { +bool JobserverState::ShouldSetupClient(const BuildConfig& config, + std::string* reason) { + if (config.dry_run) { + *reason = "Dry-run mode"; + return false; + } + if (config.explicit_parallelism && !config.jobserver_pool) { + *reason = "Explicit parallelism specified"; + return false; + } + return HasExternalJobserverPool(reason); +} + +void JobserverState::SetupPool(const BuildConfig& config, Status* status) { + std::string err; + if (!ShouldSetupPool(config, &err)) { + if (!err.empty() && config.verbosity >= BuildConfig::VERBOSE) + status->Info("not creating a jobserver pool: %s", err.c_str()); + return; + } + + if (config.verbosity >= BuildConfig::VERBOSE) + status->Info("Creating jobserver pool for %d parallel jobs", + config.parallelism); + + err.clear(); + pool_ = JobserverPool::Create(static_cast(config.parallelism), &err); + if (!pool_.get()) { + if (config.verbosity > BuildConfig::QUIET) + status->Warning("Jobserver pool creation failed: %s", err.c_str()); + return; + } + + std::string makeflags = pool_->GetEnvMakeFlagsValue(); + + // Set or override the MAKEFLAGS environment variable in + // the current process. This ensures it is passed to sub-commands + // as well. +#ifdef _WIN32 + std::string env = "MAKEFLAGS=" + makeflags; + _putenv(env.c_str()); +#else // !_WIN32 + setenv("MAKEFLAGS", makeflags.c_str(), 1); +#endif // !_WIN32 +} + +void JobserverState::SetupClient(const BuildConfig& config, Status* status) { + std::string err; + if (!ShouldSetupClient(config, &err)) { + if (!err.empty() && config.verbosity >= BuildConfig::VERBOSE) + status->Warning("ignoring jobserver: %s [%s]", err.c_str(), + makeflags_.c_str()); + return; + } + if (config.verbosity > BuildConfig::NO_STATUS_UPDATE) { + status->Info("Jobserver mode detected: %s", makeflags_.c_str()); + } + + client_ = Jobserver::Client::Create(jobserver_config_, &err); + if (!client_.get()) { // Jobserver client initialization failed !? - if (config_.verbosity > BuildConfig::QUIET) + if (config.verbosity > BuildConfig::QUIET) status->Error("Could not initialize jobserver: %s", err.c_str()); } - return result; } ExitStatus NinjaMain::RunBuild(int argc, char** argv, Status* status) { @@ -1599,16 +1703,14 @@ ExitStatus NinjaMain::RunBuild(int argc, char** argv, Status* status) { disk_interface_.AllowStatCache(g_experimental_statcache); - // Detect jobserver context and inject Jobserver::Client into the builder - // if needed. - std::unique_ptr jobserver_client = - SetupJobserverClient(status); + // Setup jobserver pool and client if needed. + JobserverState jobserver_state(config_, status); Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, status, start_time_millis_); - if (jobserver_client.get()) { - builder.SetJobserverClient(std::move(jobserver_client)); + if (jobserver_state.client()) { + builder.SetJobserverClient(jobserver_state.TakeClient()); } for (size_t i = 0; i < targets.size(); ++i) { @@ -1690,14 +1792,14 @@ int ReadFlags(int* argc, char*** argv, Options* options, BuildConfig* config) { DeferGuessParallelism deferGuessParallelism(config); - enum { OPT_VERSION = 1, OPT_QUIET = 2 }; - const option kLongOptions[] = { - { "help", no_argument, NULL, 'h' }, - { "version", no_argument, NULL, OPT_VERSION }, - { "verbose", no_argument, NULL, 'v' }, - { "quiet", no_argument, NULL, OPT_QUIET }, - { NULL, 0, NULL, 0 } - }; + enum { OPT_VERSION = 1, OPT_QUIET = 2, OPT_JOBSERVER_POOL = 3 }; + const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, OPT_VERSION }, + { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, OPT_QUIET }, + { "jobserver-pool", no_argument, NULL, + OPT_JOBSERVER_POOL }, + { NULL, 0, NULL, 0 } }; int opt; while (!options->tool && @@ -1721,7 +1823,7 @@ int ReadFlags(int* argc, char*** argv, // is close enough to infinite for most sane builds. config->parallelism = static_cast((value > 0 && value < INT_MAX) ? value : INT_MAX); - config->disable_jobserver_client = true; + config->explicit_parallelism = true; deferGuessParallelism.needGuess = false; break; } @@ -1748,7 +1850,6 @@ int ReadFlags(int* argc, char*** argv, } case 'n': config->dry_run = true; - config->disable_jobserver_client = true; break; case 't': options->tool = ChooseTool(optarg); @@ -1771,6 +1872,9 @@ int ReadFlags(int* argc, char*** argv, case OPT_VERSION: printf("%s\n", kNinjaVersion); return 0; + case OPT_JOBSERVER_POOL: + config->jobserver_pool = true; + break; case 'h': default: deferGuessParallelism.Refresh(); @@ -1780,7 +1884,6 @@ int ReadFlags(int* argc, char*** argv, } *argv += optind; *argc -= optind; - return -1; } From eb1825b0538ba1af93f359155c178d4551a876b0 Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Tue, 1 Jul 2025 18:36:05 +0200 Subject: [PATCH 3/3] Support `enable_jobserver_pool = 1` in `build.ninja`. Another way to enable jobserver pool mode is to set `enable_jobserver_pool = 1` in the `build.ninja` file directly, which can be determined by the generator directly. This is equivalent to using `--jobserver-pool` on the command-line. Note that: - Any value other than 1 is ignored, intentionally (the size of the pool is determined when Ninja is invoked only). - There is no way to disable the feature from the command-line when enabled in the build plan. - Just like `--jobserver-pool`, this is ignored if a parent jobserver pool is detected when Ninja is invoked. --- doc/manual.asciidoc | 14 ++++++--- misc/jobserver_test.py | 57 +++++++++++++++++++++++++++++++++++-- src/manifest_parser.cc | 4 +++ src/manifest_parser.h | 7 ++++- src/manifest_parser_test.cc | 25 ++++++++++++++++ src/ninja.cc | 10 +++++++ 6 files changed, 109 insertions(+), 8 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index a8f7287d1d..95b606e2f8 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -203,7 +203,8 @@ Ninja becomes a protocol client automatically if it detects the right values in the `MAKEFLAGS` environment variable (see exact conditions below). Since version 1.14, Ninja can also be a protocol server, if needed, using -the `--jobserver-pool` command-line flag. +the `--jobserver-pool` command-line flag, or if `enable_jobserver_pool = 1` +is set in the Ninja build plan. In jobserver-enabled builds, there is one top-level "server" process which: @@ -235,8 +236,8 @@ when a pipe-based jobserver is being used (i.e. when `MAKEFLAGS` contains `--jobserver-auth=,`) and will print a warning, but will otherwise ignore it. -Using `--jobserver-pool` will make Ninja act as a protocol server, unless -any of these are true: +Using `--jobserver-pool` or `enable_jobserver_pool = 1` will make Ninja +act as a protocol server, unless any of these are true: - An existing pool was detected, as this keeps all processes cooperating properly. @@ -983,7 +984,7 @@ previous one, it closes the previous scope. Top-level variables ~~~~~~~~~~~~~~~~~~~ -Two variables are significant when declared in the outermost file scope. +Three variables are significant when declared in the outermost file scope. `builddir`:: a directory for some Ninja output files. See <>. (You can also store other build output @@ -992,6 +993,11 @@ Two variables are significant when declared in the outermost file scope. `ninja_required_version`:: the minimum version of Ninja required to process the build correctly. See <>. +`enable_jobserver_pool`:: If set to `1` (any other value is ignored), enable + jobserver pool mode, as if `--jobserver-pool` was passed on the command + line. Note that `0` does not disable the feature, and that the size of + the pool is determined by the parallel job count that is either auto-detected + or controlled by the `-j` command-line option. [[ref_rule]] Rule variables diff --git a/misc/jobserver_test.py b/misc/jobserver_test.py index 53410ec8e2..834437f7a6 100755 --- a/misc/jobserver_test.py +++ b/misc/jobserver_test.py @@ -111,13 +111,13 @@ def span_output_file(span_n: int) -> str: return "out%02d" % span_n -def generate_build_plan(command_count: int) -> str: +def generate_build_plan(command_count: int, prefix: str = "") -> str: """Generate a Ninja build plan for |command_count| parallel tasks. Each task calls the test helper script which waits for 50ms then writes its own start and end time to its output file. """ - result = f""" + result = prefix + f""" rule span command = {sys.executable} -S {_JOBSERVER_TEST_HELPER_SCRIPT} --duration-ms=50 $out @@ -291,7 +291,7 @@ def run_ninja_with_jobserver_pipe(args): max_overlaps = compute_max_overlapped_spans(b.path, task_count) self.assertEqual(max_overlaps, 1) - def test_jobserver_pool_mode(self): + def test_jobserver_pool_mode_with_flag(self): task_count = 4 build_plan = generate_build_plan(task_count) with BuildDir(build_plan) as b: @@ -365,6 +365,57 @@ def test_jobserver_pool_mode_ignored_with_existing_pool(self): max_overlaps = compute_max_overlapped_spans(b.path, task_count) self.assertEqual(max_overlaps, 2) + def test_jobserver_pool_mode_with_variable(self): + task_count = 4 + build_plan = generate_build_plan(task_count, prefix = "enable_jobserver_pool = 1\n") + with BuildDir(build_plan) as b: + # First, run the full tasks with with {task_count} tokens, this should allow all + # tasks to run in parallel. + ret = b.ninja_run( + ninja_args=["all"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, task_count) + + # Second, use 2 tokens only, and verify that this was enforced by Ninja. + b.ninja_clean() + b.ninja_run( + ["-j2", "all"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, 2) + + # Third, verify that --jobs=1 serializes all tasks. + b.ninja_clean() + b.ninja_run( + ["-j1", "all"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, 1) + + # On Linux, use taskset to limit the number of available cores to 1 + # and verify that the jobserver overrides the default Ninja parallelism + # and that {task_count} tasks are still spawned in parallel. + if platform.system() == "Linux": + # First, run without a jobserver, with a single CPU, Ninja will + # use a parallelism of 2 in this case (GuessParallelism() in ninja.cc) + b.ninja_clean() + b.ninja_run( + ["all"], + prefix_args=["taskset", "-c", "0"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, 2) + + # Now with a jobserver with {task_count} tasks. + b.ninja_clean() + b.ninja_run( + [f"-j{task_count}", "all"], + prefix_args=["taskset", "-c", "0"], + ) + max_overlaps = compute_max_overlapped_spans(b.path, task_count) + self.assertEqual(max_overlaps, task_count) + def _test_MAKEFLAGS_value( self, ninja_args: T.List[str] = [], prefix_args: T.List[str] = [] ): diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 30c4c151b5..33031966e0 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -35,6 +35,10 @@ ManifestParser::ManifestParser(State* state, FileReader* file_reader, env_ = &state->bindings_; } +std::string ManifestParser::LookupVariable(const std::string& varname) { + return env_->LookupVariable(varname); +} + bool ManifestParser::Parse(const string& filename, const string& input, string* err) { lexer_.Start(filename, input); diff --git a/src/manifest_parser.h b/src/manifest_parser.h index ce37759676..7eb525a776 100644 --- a/src/manifest_parser.h +++ b/src/manifest_parser.h @@ -48,7 +48,12 @@ struct ManifestParser : public Parser { return Parse("input", input, err); } -private: + /// Retrieve the expanded value of a top-level variable from the + /// manifest. Returns an empty string if the variable is not defined. + /// Must be called only after a successful Load() call. + std::string LookupVariable(const std::string& varname); + + private: /// Parse a file, given its contents as a string. bool Parse(const std::string& filename, const std::string& input, std::string* err); diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 03ce0b1b80..2d47f8ab13 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -1136,3 +1136,28 @@ TEST_F(ParserTest, DyndepRuleInput) { EXPECT_TRUE(edge->dyndep_->dyndep_pending()); EXPECT_EQ(edge->dyndep_->path(), "in"); } + +struct ManifestParserTest : public testing::Test { + ManifestParserTest() : parser(&state, &fs_) {} + + void AssertParse(const char* input) { + string err; + EXPECT_TRUE(parser.ParseTest(input, &err)); + ASSERT_EQ("", err); + VerifyGraph(state); + } + + State state; + VirtualFileSystem fs_; + ManifestParser parser; +}; + +TEST_F(ManifestParserTest, LookupVariable) { + ASSERT_NO_FATAL_FAILURE( + AssertParse("foo = World\n" + "bar = Hello $foo\n")); + + ASSERT_EQ(parser.LookupVariable("foo"), "World"); + ASSERT_EQ(parser.LookupVariable("bar"), "Hello World"); + ASSERT_EQ(parser.LookupVariable("zoo"), ""); +} diff --git a/src/ninja.cc b/src/ninja.cc index 70af71c3d4..d763d4fe51 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -1964,6 +1964,16 @@ NORETURN void real_main(int argc, char** argv) { exit(1); } + // If enable_jobserver_pool is set to 1, enable jobserver pool mode as + // if --jobserver-pool had been passed on the command line. Note that + // any other value is ignored (thus 0 does not disable the flag if it is + // used). + std::string enable_jobserver_pool = + parser.LookupVariable("enable_jobserver_pool"); + if (enable_jobserver_pool == "1") { + const_cast(ninja.config_).jobserver_pool = true; + } + ninja.ParsePreviousElapsedTimes(); ExitStatus result = ninja.RunBuild(argc, argv, status);