Restat build.ninja after gen
As of version 1.8, ninja caches mtimes in the .ninja_log file and will
not restat a file if it has an entry in the .ninja_log. This becomes
problematic for generator files such as build.ninja. In a scenario where
the .ninja_log has an entry for build.ninja (ie, after ninja has had to
trigger a re-gen) and a manual run of of `gn gen` has updated the
build.ninja file, a subsequent invocation of ninja will trigger yet
another re-gen of build.ninja.
This can be reproduced like this:
# Initial build
gn gen out && ninja -C out
# Trigger ninja re-gen so there's a build.ninja log entry
touch BUILD.gn && ninja -C out
# Manual re-gen
gn gen out
# The following command will re-gen again before building
ninja -C out
To work around this, the recommended approach from ninja is to use the
ninja restat tool to get ninja to update the mtime in the .ninja_log.
This CL does that by attempting to invoke `ninja -t restat build.ninja`
immediately after the build.ninja is generated by `gn gen`. Because this
tool was only introduced in 1.10, the usage is gated on having a ninja
binary of at least that version. The ninja binary can be provided via
the --ninja-executable switch.
Bug: 136
Change-Id: If40391aa13e6ef71c4e8ab26aff57d66a8137b8e
Reviewed-on: https://gn-review.googlesource.com/c/gn/+/10400
Commit-Queue: RJ Ascani <rjascani@google.com>
Reviewed-by: Brett Wilson <brettw@chromium.org>
diff --git a/build/gen.py b/build/gen.py
index 486354b..0f40b75 100755
--- a/build/gen.py
+++ b/build/gen.py
@@ -586,6 +586,7 @@
'src/gn/ninja_target_command_util.cc',
'src/gn/ninja_target_writer.cc',
'src/gn/ninja_toolchain_writer.cc',
+ 'src/gn/ninja_tools.cc',
'src/gn/ninja_utils.cc',
'src/gn/ninja_writer.cc',
'src/gn/operators.cc',
diff --git a/docs/reference.md b/docs/reference.md
index 06ab5a0..8eb1510 100644
--- a/docs/reference.md
+++ b/docs/reference.md
@@ -718,6 +718,16 @@
See "gn help switches" for the common command-line switches.
```
+#### **General options**
+
+```
+ --ninja-executable=<string>
+ Can be used to specify the ninja executable to use. This executable will
+ be used as an IDE option to indicate which ninja to use for building. This
+ executable will also be used as part of the gen process for triggering a
+ restat on generated ninja files.
+```
+
#### **IDE options**
```
@@ -7760,6 +7770,7 @@
* --dotfile: Override the name of the ".gn" file.
* --fail-on-unused-args: Treat unused build args as fatal errors.
* --markdown: Write help output in the Markdown format.
+ * --ninja-executable: Set the Ninja executable.
* --nocolor: Force non-colored output.
* -q: Quiet mode. Don't print output on success.
* --root: Explicitly specify source root.
diff --git a/src/gn/command_gen.cc b/src/gn/command_gen.cc
index 7c7fd01..4437236 100644
--- a/src/gn/command_gen.cc
+++ b/src/gn/command_gen.cc
@@ -12,8 +12,10 @@
#include "gn/commands.h"
#include "gn/compile_commands_writer.h"
#include "gn/eclipse_writer.h"
+#include "gn/filesystem_utils.h"
#include "gn/json_project_writer.h"
#include "gn/ninja_target_writer.h"
+#include "gn/ninja_tools.h"
#include "gn/ninja_writer.h"
#include "gn/qt_creator_writer.h"
#include "gn/runtime_deps.h"
@@ -352,6 +354,38 @@
return res;
}
+bool RunNinjaPostProcessTools(const BuildSettings* build_settings,
+ base::FilePath ninja_executable,
+ bool is_regeneration,
+ Err* err) {
+ // If the user did not specify an executable, skip running the post processing
+ // tools. Since these tools can re-write ninja build log and dep logs, it is
+ // really important that ninja executable used for tools matches the
+ // executable that is used for builds.
+ if (ninja_executable.empty()) {
+ return true;
+ }
+
+ base::FilePath build_dir =
+ build_settings->GetFullPath(build_settings->build_dir());
+
+ // If we have a ninja version that supports restat, we should restat the
+ // build.ninja file so the next ninja invocation will use the right mtime. If
+ // gen is being invoked as part of a re-gen (ie, ninja is invoking gn gen),
+ // then we can elide this restat, as ninja will restat build.ninja anyways
+ // after it is complete.
+ if (!is_regeneration &&
+ build_settings->ninja_required_version() >= Version{1, 10, 0}) {
+ std::vector<base::FilePath> files_to_restat{
+ base::FilePath(FILE_PATH_LITERAL("build.ninja"))};
+ if (!InvokeNinjaRestatTool(ninja_executable, build_dir, files_to_restat,
+ err)) {
+ return false;
+ }
+ }
+ return true;
+}
+
} // namespace
const char kGen[] = "gen";
@@ -373,6 +407,14 @@
See "gn help switches" for the common command-line switches.
+General options
+
+ --ninja-executable=<string>
+ Can be used to specify the ninja executable to use. This executable will
+ be used as an IDE option to indicate which ninja to use for building. This
+ executable will also be used as part of the gen process for triggering a
+ restat on generated ninja files.
+
IDE options
GN optionally generates files for IDE. Files won't be overwritten if their
@@ -558,6 +600,14 @@
return 1;
}
+ if (!RunNinjaPostProcessTools(
+ &setup->build_settings(),
+ command_line->GetSwitchValuePath(switches::kNinjaExecutable),
+ command_line->HasSwitch(switches::kRegeneration), &err)) {
+ err.PrintToStdout();
+ return 1;
+ }
+
if (!WriteRuntimeDepsFilesIfNecessary(&setup->build_settings(),
setup->builder(), &err)) {
err.PrintToStdout();
diff --git a/src/gn/ninja_build_writer.cc b/src/gn/ninja_build_writer.cc
index 164c1d5..04e0754 100644
--- a/src/gn/ninja_build_writer.cc
+++ b/src/gn/ninja_build_writer.cc
@@ -113,6 +113,15 @@
}
}
+ // Add the regeneration switch if not already present. This is so that when
+ // the regeneration is invoked by ninja, the gen command is aware that it is a
+ // regeneration invocation and not an user invocation. This allows the gen
+ // command to elide ninja post processing steps that ninja will perform
+ // itself.
+ if (!cmdline.HasSwitch(switches::kRegeneration)) {
+ cmdline.AppendSwitch(switches::kRegeneration);
+ }
+
return cmdline;
}
diff --git a/src/gn/ninja_tools.cc b/src/gn/ninja_tools.cc
new file mode 100644
index 0000000..ed45544
--- /dev/null
+++ b/src/gn/ninja_tools.cc
@@ -0,0 +1,64 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "gn/ninja_tools.h"
+
+#include <vector>
+
+#include "base/command_line.h"
+#include "base/files/file_path.h"
+#include "base/strings/string_number_conversions.h"
+#include "gn/err.h"
+#include "gn/exec_process.h"
+#include "gn/filesystem_utils.h"
+
+namespace {
+
+base::CommandLine CreateNinjaToolCommandLine(const base::FilePath& ninja_executable,
+ const std::string& tool) {
+ base::CommandLine cmdline(ninja_executable);
+ cmdline.SetParseSwitches(false);
+ cmdline.AppendArg("-t");
+ cmdline.AppendArg(tool);
+ return cmdline;
+}
+
+bool RunNinja(const base::CommandLine& cmdline,
+ const base::FilePath& startup_dir,
+ std::string* output,
+ Err* err) {
+ std::string stderr_output;
+
+ int exit_code = 0;
+ if (!internal::ExecProcess(cmdline, startup_dir, output, &stderr_output,
+ &exit_code)) {
+ *err = Err(Location(), "Could not execute Ninja.",
+ "I was trying to execute \"" +
+ FilePathToUTF8(cmdline.GetProgram()) + "\".");
+ return false;
+ }
+
+ if (exit_code != 0) {
+ *err = Err(Location(), "Ninja has quit with exit code " +
+ base::IntToString(exit_code) + ".");
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace
+
+bool InvokeNinjaRestatTool(const base::FilePath& ninja_executable,
+ const base::FilePath& build_dir,
+ const std::vector<base::FilePath>& files_to_restat,
+ Err* err) {
+ base::CommandLine cmdline =
+ CreateNinjaToolCommandLine(ninja_executable, "restat");
+ for (const base::FilePath& file : files_to_restat) {
+ cmdline.AppendArgPath(file);
+ }
+ std::string output;
+ return RunNinja(cmdline, build_dir, &output, err);
+}
diff --git a/src/gn/ninja_tools.h b/src/gn/ninja_tools.h
new file mode 100644
index 0000000..4eda1e1
--- /dev/null
+++ b/src/gn/ninja_tools.h
@@ -0,0 +1,30 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef TOOLS_GN_NINJA_TOOLS_H_
+#define TOOLS_GN_NINJA_TOOLS_H_
+
+#include <vector>
+
+#include "base/files/file_path.h"
+#include "gn/err.h"
+
+// Invokes the ninja restat tool (ie, ninja -C build_dir -t restat). This tool
+// tells ninja that it should check the mtime of the provided files and update
+// the .ninja_log accordingly. This is useful when GN knows that an output file
+// in the ninja graph has been updated without invoking ninja.
+//
+// The best example of this is after gn gen runs, we know that build.ninja has
+// been potentially updated, but ninja will still use the mtime from the
+// .ninja_log and could trigger another re-gen. By telling ninja to restat
+// build.ninja, we can eliminate the extra re-gen.
+//
+// If files_to_restat is empty, ninja will restat all files that have an entry
+// in the .ninja_log.
+bool InvokeNinjaRestatTool(const base::FilePath& ninja_executable,
+ const base::FilePath& build_dir,
+ const std::vector<base::FilePath>& files_to_restat,
+ Err* err);
+
+#endif // TOOLS_GN_NINJA_TOOLS_H_
diff --git a/src/gn/switches.cc b/src/gn/switches.cc
index 0ad3a6b..fb63b30 100644
--- a/src/gn/switches.cc
+++ b/src/gn/switches.cc
@@ -99,6 +99,16 @@
const char kNoColor_HelpShort[] = "--nocolor: Force non-colored output.";
const char kNoColor_Help[] = COLOR_HELP_LONG;
+const char kNinjaExecutable[] = "ninja-executable";
+const char kNinjaExecutable_HelpShort[] =
+ "--ninja-executable: Set the Ninja executable.";
+const char kNinjaExecutable_Help[] =
+ R"(--ninja-executable: Set the Ninja executable.
+
+ When set specifies the Ninja executable that will be used to perform some
+ post-processing on the generated files for more consistent builds.
+)";
+
const char kScriptExecutable[] = "script-executable";
const char kScriptExecutable_HelpShort[] =
"--script-executable: Set the executable used to execute scripts.";
@@ -306,6 +316,7 @@
const char kDefaultToolchain[] = "default-toolchain";
+const char kRegeneration[] = "regeneration";
// -----------------------------------------------------------------------------
SwitchInfo::SwitchInfo() : short_help(""), long_help("") {}
@@ -324,6 +335,7 @@
INSERT_VARIABLE(Dotfile)
INSERT_VARIABLE(FailOnUnusedArgs)
INSERT_VARIABLE(Markdown)
+ INSERT_VARIABLE(NinjaExecutable)
INSERT_VARIABLE(NoColor)
INSERT_VARIABLE(Root)
INSERT_VARIABLE(RootTarget)
diff --git a/src/gn/switches.h b/src/gn/switches.h
index 2f89e50..2382bb7 100644
--- a/src/gn/switches.h
+++ b/src/gn/switches.h
@@ -58,6 +58,10 @@
extern const char kMetaRebaseFiles_HelpShort[];
extern const char kMetaRebaseFiles_Help[];
+extern const char kNinjaExecutable[];
+extern const char kNinjaExecutable_HelpShort[];
+extern const char kNinjaExecutable_Help[];
+
extern const char kNoColor[];
extern const char kNoColor_HelpShort[];
extern const char kNoColor_Help[];
@@ -115,6 +119,13 @@
" Non-wildcard inputs with no explicit toolchain specification will\n" \
" always match only a target in the default toolchain if one exists.\n"
+// This switch is used to signal to the gen command that it is being invoked on
+// a regeneration step. Ie, ninja has realized that build.ninja needs to be
+// generated again and has invoked gn gen. There is no help associated with it
+// because users should not be setting this switch. It is located in this file
+// so it can be shared between command_gen and ninja_build_writer.
+extern const char kRegeneration[];
+
} // namespace switches
#endif // TOOLS_GN_SWITCHES_H_
