From 0b6be4353e4ee330e9b72228ba768b4f2cb56399 Mon Sep 17 00:00:00 2001 From: tzik Date: Wed, 25 Oct 2017 22:17:40 +0900 Subject: [PATCH 01/96] Look up header dependencies on the first-output build Ninja has special syntax to specify the first output of the given node. E.g. it builds foo.o for foo.cc^. However, it doesn't work for headers, as headers usually doesn't appear in the regular dependency tree. After this change, Ninja looks up header dependencies from .ninja_deps to pick up a build target, so that it builds foo.o for foo.h^. --- src/deps_log.cc | 13 +++++++++++++ src/deps_log.h | 1 + src/deps_log_test.cc | 27 +++++++++++++++++++++++++++ src/ninja.cc | 21 +++++++++++++-------- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/deps_log.cc b/src/deps_log.cc index eb81a376f4..42e5326879 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -310,6 +310,19 @@ DepsLog::Deps* DepsLog::GetDeps(Node* node) { return deps_[node->id()]; } +Node* DepsLog::GetFirstReverseDepsNode(Node* node) { + for (size_t id = 0; id < deps_.size(); ++id) { + Deps* deps = deps_[id]; + if (!deps) + continue; + for (int i = 0; i < deps->node_count; ++i) { + if (deps->nodes[i] == node) + return nodes_[id]; + } + } + return NULL; +} + bool DepsLog::Recompact(const string& path, string* err) { METRIC_RECORD(".ninja_deps recompact"); diff --git a/src/deps_log.h b/src/deps_log.h index 3812a28a80..1336078af3 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -86,6 +86,7 @@ struct DepsLog { }; bool Load(const string& path, State* state, string* err); Deps* GetDeps(Node* node); + Node* GetFirstReverseDepsNode(Node* node); /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, string* err); diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index 89f7be159e..7daca90402 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -476,4 +476,31 @@ TEST_F(DepsLogTest, TruncatedRecovery) { } } +TEST_F(DepsLogTest, ReverseDepsNodes) { + State state; + DepsLog log; + string err; + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector deps; + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); + + deps.clear(); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); + + log.Close(); + + Node* rev_deps = log.GetFirstReverseDepsNode(state.GetNode("foo.h", 0)); + EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0) || + rev_deps == state.GetNode("out2.o", 0)); + + rev_deps = log.GetFirstReverseDepsNode(state.GetNode("bar.h", 0)); + EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0)); +} + } // anonymous namespace diff --git a/src/ninja.cc b/src/ninja.cc index 30f89c27f6..3a7ebbcba8 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -282,15 +282,20 @@ Node* NinjaMain::CollectTarget(const char* cpath, string* err) { if (node) { if (first_dependent) { if (node->out_edges().empty()) { - *err = "'" + path + "' has no out edge"; - return NULL; - } - Edge* edge = node->out_edges()[0]; - if (edge->outputs_.empty()) { - edge->Dump(); - Fatal("edge has no outputs"); + Node* rev_deps = deps_log_.GetFirstReverseDepsNode(node); + if (!rev_deps) { + *err = "'" + path + "' has no out edge"; + return NULL; + } + node = rev_deps; + } else { + Edge* edge = node->out_edges()[0]; + if (edge->outputs_.empty()) { + edge->Dump(); + Fatal("edge has no outputs"); + } + node = edge->outputs_[0]; } - node = edge->outputs_[0]; } return node; } else { From de6485646ddf3248871e1af6dcd1de180742b8e6 Mon Sep 17 00:00:00 2001 From: Cameron Desrochers Date: Tue, 12 Nov 2019 15:47:27 -0500 Subject: [PATCH 02/96] Fixed processor count detection on Windows when multiple processor groups (i.e. >64 processors) are present --- src/util.cc | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/util.cc b/src/util.cc index 4df2bb23c1..7893f61a69 100644 --- a/src/util.cc +++ b/src/util.cc @@ -481,6 +481,32 @@ string StripAnsiEscapeCodes(const string& in) { int GetProcessorCount() { #ifdef _WIN32 +#if _WIN32_WINNT >= 0x0601 + // Need to use GetLogicalProcessorInformationEx to get real core count on + // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 + DWORD len = 0; + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len) + && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + std::vector buf(len); + int cores = 0; + if (GetLogicalProcessorInformationEx(RelationProcessorCore, + (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)buf.data(), &len)) { + for (DWORD i = 0; i < len; ) { + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = + (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)(buf.data() + i); + if (info->Relationship == RelationProcessorCore && + info->Processor.GroupCount == 1) { + for (KAFFINITY core_mask = info->Processor.GroupMask[0].Mask; + core_mask; core_mask >>= 1) + cores += (core_mask & 1); + } + i += info->Size; + } + if (cores) + return cores; + } + } +#endif return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); #else #ifdef CPU_COUNT From 48a8a7ae95f10d70a4c2f117df820e9ce5f4a0f2 Mon Sep 17 00:00:00 2001 From: Cameron Date: Tue, 12 Nov 2019 21:13:36 -0500 Subject: [PATCH 03/96] Use symbolic constant as suggested in code review Co-Authored-By: Takuto Ikuta --- src/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index 7893f61a69..ba63ab3e6d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -481,7 +481,7 @@ string StripAnsiEscapeCodes(const string& in) { int GetProcessorCount() { #ifdef _WIN32 -#if _WIN32_WINNT >= 0x0601 +#if _WIN32_WINNT >= _WIN32_WINNT_WIN7 // Need to use GetLogicalProcessorInformationEx to get real core count on // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 DWORD len = 0; From 9d502da06d179d598c2167ca0b97756e19d687c2 Mon Sep 17 00:00:00 2001 From: Cameron Desrochers Date: Wed, 13 Nov 2019 12:17:31 -0500 Subject: [PATCH 04/96] Removed unnecessary #ifdef following code review --- src/util.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/util.cc b/src/util.cc index ba63ab3e6d..ac4f24727f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -481,7 +481,6 @@ string StripAnsiEscapeCodes(const string& in) { int GetProcessorCount() { #ifdef _WIN32 -#if _WIN32_WINNT >= _WIN32_WINNT_WIN7 // Need to use GetLogicalProcessorInformationEx to get real core count on // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 DWORD len = 0; @@ -506,7 +505,7 @@ int GetProcessorCount() { return cores; } } -#endif + // fallback just in case return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); #else #ifdef CPU_COUNT From 546e27c6459b862e3394caad1c71d63da0891041 Mon Sep 17 00:00:00 2001 From: Cameron Desrochers Date: Wed, 20 Nov 2019 16:31:14 -0500 Subject: [PATCH 05/96] Style changes following code review --- src/util.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/util.cc b/src/util.cc index ac4f24727f..9049f90e37 100644 --- a/src/util.cc +++ b/src/util.cc @@ -489,20 +489,24 @@ int GetProcessorCount() { std::vector buf(len); int cores = 0; if (GetLogicalProcessorInformationEx(RelationProcessorCore, - (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)buf.data(), &len)) { + reinterpret_cast( + buf.data()), &len)) { for (DWORD i = 0; i < len; ) { PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = - (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)(buf.data() + i); + reinterpret_cast( + buf.data() + i); if (info->Relationship == RelationProcessorCore && info->Processor.GroupCount == 1) { for (KAFFINITY core_mask = info->Processor.GroupMask[0].Mask; - core_mask; core_mask >>= 1) + core_mask; core_mask >>= 1) { cores += (core_mask & 1); + } } i += info->Size; } - if (cores) + if (cores) { return cores; + } } } // fallback just in case From b1afd603beeed7ac387faa8fc958db622b984043 Mon Sep 17 00:00:00 2001 From: Nicolas Arciniega Date: Mon, 3 Feb 2020 11:40:46 -0800 Subject: [PATCH 06/96] Add 'inputs' tool to print out all inputs for a set of targets --- doc/manual.asciidoc | 4 ++++ src/ninja.cc | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 9976ce4c24..cdff9c0c7d 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -257,6 +257,10 @@ than the _depth_ mode. executed in order, may be used to rebuild those targets, assuming that all output files are out of date. +`inputs`:: given a list of targets, print a list of all inputs which are used +to rebuild those targets. +_Available since Ninja 1.10._ + `clean`:: remove built files. By default it removes all built files except for those created by the generator. Adding the `-g` flag also removes built files created by the generator (see <* seen, PrintCommandMode mode) { } int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { - // The clean tool uses getopt, and expects argv[0] to contain the name of + // The commands tool uses getopt, and expects argv[0] to contain the name of // the tool, i.e. "commands". ++argc; --argv; @@ -669,6 +670,35 @@ int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { return 0; } +void PrintInputs(Edge* edge, set* seen) { + if (!edge) + return; + if (!seen->insert(edge).second) + return; + + for (vector::iterator in = edge->inputs_.begin(); + in != edge->inputs_.end(); ++in) + PrintInputs((*in)->in_edge(), seen); + + if (!edge->is_phony()) + puts(edge->GetBinding("in_newline").c_str()); +} + +int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) { + vector nodes; + string err; + if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { + Error("%s", err.c_str()); + return 1; + } + + set seen; + for (vector::iterator in = nodes.begin(); in != nodes.end(); ++in) + PrintInputs((*in)->in_edge(), &seen); + + return 0; +} + int NinjaMain::ToolClean(const Options* options, int argc, char* argv[]) { // The clean tool uses getopt, and expects argv[0] to contain the name of // the tool, i.e. "clean". @@ -956,6 +986,8 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolClean }, { "commands", "list all commands required to rebuild given targets", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCommands }, + { "inputs", "list all inputs required to rebuild given targets", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolInputs}, { "deps", "show dependencies stored in the deps log", Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps }, { "graph", "output graphviz dot file for targets", From 9e1e51aeb56115b94d3cf14e67934dcd51a3f5c6 Mon Sep 17 00:00:00 2001 From: Cameron Date: Wed, 10 Jun 2020 07:32:04 -0400 Subject: [PATCH 07/96] Tweaks following code review --- src/util.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/util.cc b/src/util.cc index 9049f90e37..e8455ecc15 100644 --- a/src/util.cc +++ b/src/util.cc @@ -492,9 +492,8 @@ int GetProcessorCount() { reinterpret_cast( buf.data()), &len)) { for (DWORD i = 0; i < len; ) { - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = - reinterpret_cast( - buf.data() + i); + auto info = reinterpret_cast( + buf.data() + i); if (info->Relationship == RelationProcessorCore && info->Processor.GroupCount == 1) { for (KAFFINITY core_mask = info->Processor.GroupMask[0].Mask; @@ -504,7 +503,7 @@ int GetProcessorCount() { } i += info->Size; } - if (cores) { + if (cores != 0) { return cores; } } From 03cbfc65220e8f98548684e785d95bf5734c3f59 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 14 Nov 2016 17:13:16 -0800 Subject: [PATCH 08/96] Add unique IDs to edges Edges are nominally ordered by order in the build manifest, but in fact are ordered by memory address. In most cases the memory address will be monontonically increasing. Since serialized build output will need unique IDs, add a monotonically increasing ID to edges, and use that for sorting instead of memory address. --- src/build.cc | 2 +- src/build.h | 3 +-- src/graph.h | 18 ++++++++++++++---- src/graphviz.h | 3 ++- src/ninja.cc | 4 ++-- src/state.cc | 11 ++--------- src/state.h | 16 ++++++++++++---- 7 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/build.cc b/src/build.cc index e3131e2024..3c2d0b734d 100644 --- a/src/build.cc +++ b/src/build.cc @@ -381,7 +381,7 @@ void Plan::EdgeWanted(const Edge* edge) { Edge* Plan::FindWork() { if (ready_.empty()) return NULL; - set::iterator e = ready_.begin(); + EdgeSet::iterator e = ready_.begin(); Edge* edge = *e; ready_.erase(e); return edge; diff --git a/src/build.h b/src/build.h index 2798693491..2fe71fc84e 100644 --- a/src/build.h +++ b/src/build.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -122,7 +121,7 @@ struct Plan { /// we want for the edge. std::map want_; - std::set ready_; + EdgeSet ready_; Builder* builder_; diff --git a/src/graph.h b/src/graph.h index 4833f49b03..8c51782a6e 100644 --- a/src/graph.h +++ b/src/graph.h @@ -15,6 +15,7 @@ #ifndef NINJA_GRAPH_H_ #define NINJA_GRAPH_H_ +#include #include #include @@ -143,10 +144,11 @@ struct Edge { VisitDone }; - Edge() : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), - mark_(VisitNone), outputs_ready_(false), deps_loaded_(false), - deps_missing_(false), implicit_deps_(0), order_only_deps_(0), - implicit_outs_(0) {} + Edge() + : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone), + id_(0), outputs_ready_(false), deps_loaded_(false), + deps_missing_(false), implicit_deps_(0), order_only_deps_(0), + implicit_outs_(0) {} /// Return true if all inputs' in-edges are ready. bool AllInputsReady() const; @@ -176,6 +178,7 @@ struct Edge { Node* dyndep_; BindingEnv* env_; VisitMark mark_; + size_t id_; bool outputs_ready_; bool deps_loaded_; bool deps_missing_; @@ -218,6 +221,13 @@ struct Edge { bool maybe_phonycycle_diagnostic() const; }; +struct EdgeCmp { + bool operator()(const Edge* a, const Edge* b) const { + return a->id_ < b->id_; + } +}; + +typedef std::set EdgeSet; /// ImplicitDepLoader loads implicit dependencies, as referenced via the /// "depfile" attribute in build files. diff --git a/src/graphviz.h b/src/graphviz.h index 601c9b2ea0..3a3282e290 100644 --- a/src/graphviz.h +++ b/src/graphviz.h @@ -18,6 +18,7 @@ #include #include "dyndep.h" +#include "graph.h" struct DiskInterface; struct Node; @@ -34,7 +35,7 @@ struct GraphViz { DyndepLoader dyndep_loader_; std::set visited_nodes_; - std::set visited_edges_; + EdgeSet visited_edges_; }; #endif // NINJA_GRAPHVIZ_H_ diff --git a/src/ninja.cc b/src/ninja.cc index 471a023cef..eb9732058b 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -613,7 +613,7 @@ int NinjaMain::ToolRules(const Options* options, int argc, char* argv[]) { } enum PrintCommandMode { PCM_Single, PCM_All }; -void PrintCommands(Edge* edge, set* seen, PrintCommandMode mode) { +void PrintCommands(Edge* edge, EdgeSet* seen, PrintCommandMode mode) { if (!edge) return; if (!seen->insert(edge).second) @@ -664,7 +664,7 @@ int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { return 1; } - set seen; + EdgeSet seen; for (vector::iterator in = nodes.begin(); in != nodes.end(); ++in) PrintCommands((*in)->in_edge(), &seen, mode); diff --git a/src/state.cc b/src/state.cc index d3a9e29f1e..a33d5a88ae 100644 --- a/src/state.cc +++ b/src/state.cc @@ -39,7 +39,7 @@ void Pool::DelayEdge(Edge* edge) { delayed_.insert(edge); } -void Pool::RetrieveReadyEdges(set* ready_queue) { +void Pool::RetrieveReadyEdges(EdgeSet* ready_queue) { DelayedEdges::iterator it = delayed_.begin(); while (it != delayed_.end()) { Edge* edge = *it; @@ -62,14 +62,6 @@ void Pool::Dump() const { } } -// static -bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) { - if (!a) return b; - if (!b) return false; - int weight_diff = a->weight() - b->weight(); - return ((weight_diff < 0) || (weight_diff == 0 && a < b)); -} - Pool State::kDefaultPool("", 0); Pool State::kConsolePool("console", 1); const Rule State::kPhonyRule("phony"); @@ -97,6 +89,7 @@ Edge* State::AddEdge(const Rule* rule) { edge->rule_ = rule; edge->pool_ = &State::kDefaultPool; edge->env_ = &bindings_; + edge->id_ = edges_.size(); edges_.push_back(edge); return edge; } diff --git a/src/state.h b/src/state.h index f553ed49ce..72c5b33667 100644 --- a/src/state.h +++ b/src/state.h @@ -21,6 +21,7 @@ #include #include "eval_env.h" +#include "graph.h" #include "hash_map.h" #include "util.h" @@ -38,7 +39,7 @@ struct Rule; /// completes). struct Pool { Pool(const std::string& name, int depth) - : name_(name), current_use_(0), depth_(depth), delayed_(&WeightedEdgeCmp) {} + : name_(name), current_use_(0), depth_(depth), delayed_() {} // A depth of 0 is infinite bool is_valid() const { return depth_ >= 0; } @@ -61,7 +62,7 @@ struct Pool { void DelayEdge(Edge* edge); /// Pool will add zero or more edges to the ready_queue - void RetrieveReadyEdges(std::set* ready_queue); + void RetrieveReadyEdges(EdgeSet* ready_queue); /// Dump the Pool and its edges (useful for debugging). void Dump() const; @@ -74,9 +75,16 @@ struct Pool { int current_use_; int depth_; - static bool WeightedEdgeCmp(const Edge* a, const Edge* b); + struct WeightedEdgeCmp { + bool operator()(const Edge* a, const Edge* b) const { + if (!a) return b; + if (!b) return false; + int weight_diff = a->weight() - b->weight(); + return ((weight_diff < 0) || (weight_diff == 0 && EdgeCmp()(a, b))); + } + }; - typedef std::set DelayedEdges; + typedef std::set DelayedEdges; DelayedEdges delayed_; }; From c09122cc3d5024b76405b54e7e1ea4564c29bc2f Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Fri, 6 Nov 2020 19:02:46 +0000 Subject: [PATCH 09/96] Replace Travis CI with GitHub Actions --- .github/workflows/linux.yml | 21 +++++++++++++++++++++ .travis.yml | 36 ------------------------------------ 2 files changed, 21 insertions(+), 36 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9062d9800e..cd55262322 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -123,3 +123,24 @@ jobs: - name: clang-tidy run: /usr/lib/llvm-10/share/clang/run-clang-tidy.py -header-filter=src working-directory: build-clang + + build-with-python: + runs-on: [ubuntu-latest] + container: + image: ${{ matrix.image }} + strategy: + matrix: + image: ['ubuntu:14.04', 'ubuntu:16.04', 'ubuntu:18.04'] + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + apt update + apt install -y g++ python3 + - name: ${{ matrix.image }} + run: | + python3 configure.py --bootstrap + ./ninja all + ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots + python3 misc/ninja_syntax_test.py + ./misc/output_test.py diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e5d7d2b8ee..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -matrix: - include: - - os: linux - dist: precise - compiler: gcc - - os: linux - dist: precise - compiler: clang - - os: linux - dist: trusty - compiler: gcc - - os: linux - dist: trusty - compiler: clang - - os: linux - dist: xenial - compiler: gcc - - os: linux - dist: xenial - compiler: clang - - os: osx - osx_image: xcode10 - - os: osx - osx_image: xcode10.1 -sudo: false -language: cpp -before_install: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install re2c ; fi - - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then choco install re2c python ; fi -script: - - ./misc/ci.py - - python3 configure.py --bootstrap - - ./ninja all - - ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots - - ./misc/ninja_syntax_test.py - - ./misc/output_test.py From d709a37d14dd37610eea32991c6df8a1a62a25ee Mon Sep 17 00:00:00 2001 From: Orgad Shaneh Date: Wed, 11 Nov 2020 13:52:54 +0200 Subject: [PATCH 10/96] Suppress MSVC warnings * D9025: overriding '/GR' with '/GR-' * C4100: 'argc': unreferenced formal parameter * C4244: '=': conversion from 'int' to 'char', possible loss of data (on tolower) * C4267: 'initializing': conversion from 'size_t' to 'unsigned int', possible loss of data * C4702: unreachable code (after Fatal) * C4706: assignment within conditional expression * C4996: 'strcpy': This function or variable may be unsafe --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e02849dc8e..d772e15100 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,9 @@ endif() # --- compiler flags if(MSVC) set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") - string(APPEND CMAKE_CXX_FLAGS " /W4 /GR- /Zc:__cplusplus") + string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + string(APPEND CMAKE_CXX_FLAGS " /W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus") + add_definitions(-D_CRT_SECURE_NO_WARNINGS) else() include(CheckCXXCompilerFlag) check_cxx_compiler_flag(-Wno-deprecated flag_no_deprecated) From 9d4ea5a8e81395672e0c3b59f8926ba48a223000 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Sat, 28 Nov 2020 23:54:09 +0000 Subject: [PATCH 11/96] Add FreeBSD support to GetProcessorCount --- src/util.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index c76f730949..05bdb2d467 100644 --- a/src/util.cc +++ b/src/util.cc @@ -51,6 +51,10 @@ #include #endif +#if defined(__FreeBSD__) +#include +#endif + #include "edit_distance.h" #include "metrics.h" @@ -485,10 +489,17 @@ int GetProcessorCount() { #ifdef _WIN32 return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); #else -#ifdef CPU_COUNT // The number of exposed processors might not represent the actual number of // processors threads can run on. This happens when a CPU set limitation is // active, see https://github.com/ninja-build/ninja/issues/1278 +#if defined(__FreeBSD__) + cpuset_t mask; + CPU_ZERO(&mask); + if (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(mask), + &mask) == 0) { + return CPU_COUNT(&mask); + } +#elif defined(CPU_COUNT) cpu_set_t set; if (sched_getaffinity(getpid(), sizeof(set), &set) == 0) { return CPU_COUNT(&set); From 76ecd305411e6fe942ee4efc5cc098b5c5c79d96 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 2 Jun 2020 18:02:15 -0700 Subject: [PATCH 12/96] [clang-tidy] fix inconsistent declarations Found with readability-inconsistent-declaration-parameter-name Signed-off-by: Rosen Penev --- .clang-tidy | 2 ++ src/build.cc | 12 ++++++------ src/build.h | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index df4c1edbd6..18ffaac8a1 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,12 +1,14 @@ --- Checks: ' ,readability-avoid-const-params-in-decls, + ,readability-inconsistent-declaration-parameter-name, ,readability-non-const-parameter, ,readability-redundant-string-cstr, ,readability-redundant-string-init, ' WarningsAsErrors: ' ,readability-avoid-const-params-in-decls, + ,readability-inconsistent-declaration-parameter-name, ,readability-non-const-parameter, ,readability-redundant-string-cstr, ,readability-redundant-string-init, diff --git a/src/build.cc b/src/build.cc index 2fb2aa431e..d08535d980 100644 --- a/src/build.cc +++ b/src/build.cc @@ -318,8 +318,8 @@ void Plan::Reset() { want_.clear(); } -bool Plan::AddTarget(const Node* node, string* err) { - return AddSubTarget(node, NULL, err, NULL); +bool Plan::AddTarget(const Node* target, string* err) { + return AddSubTarget(target, NULL, err, NULL); } bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err, @@ -782,16 +782,16 @@ Node* Builder::AddTarget(const string& name, string* err) { return node; } -bool Builder::AddTarget(Node* node, string* err) { - if (!scan_.RecomputeDirty(node, err)) +bool Builder::AddTarget(Node* target, string* err) { + if (!scan_.RecomputeDirty(target, err)) return false; - if (Edge* in_edge = node->in_edge()) { + if (Edge* in_edge = target->in_edge()) { if (in_edge->outputs_ready()) return true; // Nothing to do. } - if (!plan_.AddTarget(node, err)) + if (!plan_.AddTarget(target, err)) return false; return true; diff --git a/src/build.h b/src/build.h index 2798693491..fd6b6f69c4 100644 --- a/src/build.h +++ b/src/build.h @@ -46,7 +46,7 @@ struct Plan { /// Add a target to our plan (including all its dependencies). /// Returns false if we don't need to build this target; may /// fill in |err| with an error message if there's a problem. - bool AddTarget(const Node* node, std::string* err); + bool AddTarget(const Node* target, std::string* err); // Pop a ready edge off the queue of edges to build. // Returns NULL if there's no work to do. From 160531764cd04be7327ae99d4d703e14fc8ead47 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Thu, 3 Dec 2020 20:05:25 +0100 Subject: [PATCH 13/96] Ignore clangd files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index dca11295b5..b9a45a1015 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ # Qt Creator project files /CMakeLists.txt.user + +# clangd +/.clangd/ +/compile_commands.json From 7c36b8871ac0cd976461b19b8d302ba58121fbfd Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 2 Jun 2020 18:23:54 -0700 Subject: [PATCH 14/96] [clang-tidy] simplify boolean expression Found with readability-simplify-boolean-expr Signed-off-by: Rosen Penev --- .clang-tidy | 2 ++ src/manifest_parser.cc | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index df4c1edbd6..54f579b4c7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -4,10 +4,12 @@ Checks: ' ,readability-non-const-parameter, ,readability-redundant-string-cstr, ,readability-redundant-string-init, + ,readability-simplify-boolean-expr, ' WarningsAsErrors: ' ,readability-avoid-const-params-in-decls, ,readability-non-const-parameter, ,readability-redundant-string-cstr, ,readability-redundant-string-init, + ,readability-simplify-boolean-expr, ' diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 860a8fc6a9..54ad3f44c8 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -202,10 +202,7 @@ bool ManifestParser::ParseDefault(string* err) { return false; } while (!eval.empty()); - if (!ExpectToken(Lexer::NEWLINE, err)) - return false; - - return true; + return ExpectToken(Lexer::NEWLINE, err); } bool ManifestParser::ParseEdge(string* err) { From 0351d68f0a6bf326126b8e537c788c486b46aa14 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 2 Jun 2020 19:39:55 -0700 Subject: [PATCH 15/96] [clang-tidy] add explicit to single argument constructors Found with google-explicit-constructor Signed-off-by: Rosen Penev --- src/build_test.cc | 7 +++---- src/graph.cc | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/build_test.cc b/src/build_test.cc index 078080dae9..0baabc4bd9 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -490,10 +490,9 @@ struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { status_(config_) { } - BuildTest(DepsLog* log) : config_(MakeConfig()), command_runner_(&fs_), - builder_(&state_, config_, NULL, log, &fs_), - status_(config_) { - } + explicit BuildTest(DepsLog* log) + : config_(MakeConfig()), command_runner_(&fs_), + builder_(&state_, config_, NULL, log, &fs_), status_(config_) {} virtual void SetUp() { StateTestWithBuiltinRules::SetUp(); diff --git a/src/graph.cc b/src/graph.cc index ea113606d4..78d0d49024 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -515,7 +515,7 @@ bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { } struct matches { - matches(std::vector::iterator i) : i_(i) {} + explicit matches(std::vector::iterator i) : i_(i) {} bool operator()(const Node* node) const { StringPiece opath = StringPiece(node->path()); From 58f77f972bcece256c16ca1cc4a933270b1ea1c1 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Sun, 6 Dec 2020 15:49:29 +0100 Subject: [PATCH 16/96] Ignore new clangd cache directory, too --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b9a45a1015..fdca01513f 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ # clangd /.clangd/ /compile_commands.json +/.cache/ From 2d7f7e55c05a92033a1a62fbb6163e2e32aa81c3 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Mon, 7 Dec 2020 19:46:10 +0100 Subject: [PATCH 17/96] Delete read-only files on Windows, too Fixes main complaint of #1886. --- src/disk_interface.cc | 21 +++++++++++++++++++-- src/disk_interface_test.cc | 6 ++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 49af001322..8d4cc7f65d 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -265,6 +265,23 @@ FileReader::Status RealDiskInterface::ReadFile(const string& path, } int RealDiskInterface::RemoveFile(const string& path) { +#ifdef _WIN32 + DWORD attributes = GetFileAttributes(path.c_str()); + if (attributes == INVALID_FILE_ATTRIBUTES && + GetLastError() == ERROR_FILE_NOT_FOUND) { + return 1; + } + if (attributes & FILE_ATTRIBUTE_READONLY) { + // On non-Windows systems remove will happily delete read-only files. On + // Windows Ninja should behave the same. See + // https://github.com/ninja-build/ninja/issues/1886 + SetFileAttributes(path.c_str(), attributes & ~FILE_ATTRIBUTE_READONLY); + } + if (!DeleteFile(path.c_str())) { + Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); + return -1; + } +#else if (remove(path.c_str()) < 0) { switch (errno) { case ENOENT: @@ -273,9 +290,9 @@ int RealDiskInterface::RemoveFile(const string& path) { Error("remove(%s): %s", path.c_str(), strerror(errno)); return -1; } - } else { - return 0; } +#endif + return 0; } void RealDiskInterface::AllowStatCache(bool allow) { diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 066c770cb8..b424243a94 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -211,6 +211,12 @@ TEST_F(DiskInterfaceTest, RemoveFile) { EXPECT_EQ(0, disk_.RemoveFile(kFileName)); EXPECT_EQ(1, disk_.RemoveFile(kFileName)); EXPECT_EQ(1, disk_.RemoveFile("does not exist")); +#ifdef _WIN32 + ASSERT_TRUE(Touch(kFileName)); + EXPECT_EQ(0, system((std::string("attrib +R ") + kFileName).c_str())); + EXPECT_EQ(0, disk_.RemoveFile(kFileName)); + EXPECT_EQ(1, disk_.RemoveFile(kFileName)); +#endif } struct StatTest : public StateTestWithBuiltinRules, From 9c801316baade0b27b62dbdd4767c10e1d11f411 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Tue, 8 Dec 2020 20:15:21 +0000 Subject: [PATCH 18/96] Added fuzzer with build script and seed --- misc/build.sh | 29 +++++++++++++++++++++++++++++ misc/manifest_fuzzer.cc | 41 +++++++++++++++++++++++++++++++++++++++++ misc/sample_ninja_build | 14 ++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 misc/build.sh create mode 100644 misc/manifest_fuzzer.cc create mode 100644 misc/sample_ninja_build diff --git a/misc/build.sh b/misc/build.sh new file mode 100644 index 0000000000..4328feb662 --- /dev/null +++ b/misc/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash -eu +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +cmake -Bbuild-cmake -H. +cmake --build build-cmake + +cd $SRC/ninja/misc + +$CXX $CXXFLAGS -fdiagnostics-color -I/src/ninja/src -o fuzzer.o -c manifest_fuzzer.cc + +find .. -name "*.o" -exec ar rcs fuzz_lib.a {} \; + +$CXX $CXXFLAGS $LIB_FUZZING_ENGINE fuzzer.o -o $OUT/fuzzer fuzz_lib.a + +zip $OUT/fuzzer_seed_corpus.zip $SRC/sample_ninja_build diff --git a/misc/manifest_fuzzer.cc b/misc/manifest_fuzzer.cc new file mode 100644 index 0000000000..0e1261a804 --- /dev/null +++ b/misc/manifest_fuzzer.cc @@ -0,0 +1,41 @@ +// Copyright 2020 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "stdint.h" +#include +#include "disk_interface.h" +#include "state.h" +#include "manifest_parser.h" +#include + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + char build_file[256]; + sprintf(build_file, "/tmp/build.ninja"); + FILE *fp = fopen(build_file, "wb"); + if (!fp) + return 0; + fwrite(data, size, 1, fp); + fclose(fp); + + std::string err; + RealDiskInterface disk_interface; + State state; + ManifestParser parser(&state, &disk_interface); + + parser.Load("/tmp/build.ninja", &err); + + std::__fs::filesystem::remove_all("/tmp/build.ninja"); + return 0; +} diff --git a/misc/sample_ninja_build b/misc/sample_ninja_build new file mode 100644 index 0000000000..7b513be2dd --- /dev/null +++ b/misc/sample_ninja_build @@ -0,0 +1,14 @@ +# build.ninja +cc = clang +cflags = -Weverything + +rule compile + command = $cc $cflags -c $in -o $out + +rule link + command = $cc $in -o $out + +build hello.o: compile hello.c +build hello: link hello.o + +default hello From 365a149069792e97a168557c02a61c59c3fe62fe Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Thu, 10 Dec 2020 18:00:26 +0000 Subject: [PATCH 19/96] Created oss-fuzz folder and moved build.sh and sample ninja file into it --- misc/{ => oss-fuzz}/build.sh | 0 misc/{ => oss-fuzz}/sample_ninja_build | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename misc/{ => oss-fuzz}/build.sh (100%) rename misc/{ => oss-fuzz}/sample_ninja_build (100%) diff --git a/misc/build.sh b/misc/oss-fuzz/build.sh similarity index 100% rename from misc/build.sh rename to misc/oss-fuzz/build.sh diff --git a/misc/sample_ninja_build b/misc/oss-fuzz/sample_ninja_build similarity index 100% rename from misc/sample_ninja_build rename to misc/oss-fuzz/sample_ninja_build From 721db105e6bd5844f007d1c414363e3ca406bfb8 Mon Sep 17 00:00:00 2001 From: Gregor Jasny Date: Thu, 10 Dec 2020 09:37:42 +0100 Subject: [PATCH 20/96] cmake: Use modern add_test signature Using this modern signature makes CMake/CTest use the effective binary path and works also with multi-config generators which put binaries in extra `Release` folders. --- .github/workflows/macos.yml | 2 +- CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4ea958fd1d..af79080a73 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -26,7 +26,7 @@ jobs: cmake --build build --config Release - name: Test ninja - run: ctest -vv + run: ctest -C Release -vv working-directory: build - name: Create ninja archive diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e4bafaaba..39348c9ca4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -209,7 +209,7 @@ if(BUILD_TESTING) target_link_libraries(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000") endif() - add_test(NinjaTest ninja_test) + add_test(NAME NinjaTest COMMAND ninja_test) endif() install(TARGETS ninja DESTINATION bin) From dede9ac780ca63d2e479e8a94096aa67ea668a83 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 16 Nov 2016 10:05:17 -0800 Subject: [PATCH 21/96] Move edge time from Status to Builder The times that end up in the build log currently originate in the status printer, and are propagated back out to the Builder. Move the edge times into the Builder instead, and move the overall start time into NinjaMain so it doesn't get reset during manifest rebuilds. --- src/build.cc | 79 +++++++++++++++++++++++------------------------ src/build.h | 54 ++++++++++---------------------- src/build_test.cc | 34 ++++++++++---------- src/ninja.cc | 11 +++++-- 4 files changed, 80 insertions(+), 98 deletions(-) diff --git a/src/build.cc b/src/build.cc index 2007d82b48..f04e2f2ff2 100644 --- a/src/build.cc +++ b/src/build.cc @@ -36,6 +36,7 @@ #include "deps_log.h" #include "disk_interface.h" #include "graph.h" +#include "metrics.h" #include "state.h" #include "subprocess.h" #include "util.h" @@ -79,8 +80,9 @@ bool DryRunCommandRunner::WaitForCommand(Result* result) { } // namespace BuildStatus::BuildStatus(const BuildConfig& config) - : config_(config), start_time_millis_(GetTimeMillis()), started_edges_(0), - finished_edges_(0), total_edges_(0), progress_status_format_(NULL), + : config_(config), + started_edges_(0), finished_edges_(0), total_edges_(0), + time_millis_(0), progress_status_format_(NULL), current_rate_(config.parallelism) { // Don't do anything fancy in verbose mode. if (config_.verbosity != BuildConfig::NORMAL) @@ -95,33 +97,24 @@ void BuildStatus::PlanHasTotalEdges(int total) { total_edges_ = total; } -void BuildStatus::BuildEdgeStarted(const Edge* edge) { - assert(running_edges_.find(edge) == running_edges_.end()); - int start_time = (int)(GetTimeMillis() - start_time_millis_); - running_edges_.insert(make_pair(edge, start_time)); +void BuildStatus::BuildEdgeStarted(const Edge* edge, + int64_t start_time_millis) { ++started_edges_; + time_millis_ = start_time_millis; if (edge->use_console() || printer_.is_smart_terminal()) - PrintStatus(edge, kEdgeStarted); + PrintStatus(edge, start_time_millis, kEdgeStarted); if (edge->use_console()) printer_.SetConsoleLocked(true); } -void BuildStatus::BuildEdgeFinished(Edge* edge, +void BuildStatus::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, bool success, - const string& output, - int* start_time, - int* end_time) { - int64_t now = GetTimeMillis(); - + const string& output) { + time_millis_ = end_time_millis; ++finished_edges_; - RunningEdgeMap::iterator i = running_edges_.find(edge); - *start_time = i->second; - *end_time = (int)(now - start_time_millis_); - running_edges_.erase(i); - if (edge->use_console()) printer_.SetConsoleLocked(false); @@ -129,7 +122,7 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, return; if (!edge->use_console()) - PrintStatus(edge, kEdgeFinished); + PrintStatus(edge, end_time_millis, kEdgeFinished); // Print the command that is spewing before printing its output. if (!success) { @@ -192,8 +185,6 @@ void BuildStatus::BuildLoadDyndeps() { } void BuildStatus::BuildStarted() { - overall_rate_.Restart(); - current_rate_.Restart(); } void BuildStatus::BuildFinished() { @@ -202,10 +193,9 @@ void BuildStatus::BuildFinished() { } string BuildStatus::FormatProgressStatus( - const char* progress_status_format, EdgeStatus status) const { + const char* progress_status_format, int64_t time, EdgeStatus status) const { string out; char buf[32]; - int percent; for (const char* s = progress_status_format; *s != '\0'; ++s) { if (*s == '%') { ++s; @@ -251,28 +241,27 @@ string BuildStatus::FormatProgressStatus( // Overall finished edges per second. case 'o': - overall_rate_.UpdateRate(finished_edges_); - SnprintfRate(overall_rate_.rate(), buf, "%.1f"); + SnprintfRate(finished_edges_ / (time_millis_ / 1e3), buf, "%.1f"); out += buf; break; // Current rate, average over the last '-j' jobs. case 'c': - current_rate_.UpdateRate(finished_edges_); + current_rate_.UpdateRate(finished_edges_, time_millis_); SnprintfRate(current_rate_.rate(), buf, "%.1f"); out += buf; break; // Percentage - case 'p': - percent = (100 * finished_edges_) / total_edges_; + case 'p': { + int percent = (100 * finished_edges_) / total_edges_; snprintf(buf, sizeof(buf), "%3i%%", percent); out += buf; break; + } case 'e': { - double elapsed = overall_rate_.Elapsed(); - snprintf(buf, sizeof(buf), "%.3f", elapsed); + snprintf(buf, sizeof(buf), "%.3f", time_millis_ / 1e3); out += buf; break; } @@ -289,7 +278,7 @@ string BuildStatus::FormatProgressStatus( return out; } -void BuildStatus::PrintStatus(const Edge* edge, EdgeStatus status) { +void BuildStatus::PrintStatus(const Edge* edge, int64_t time, EdgeStatus status) { if (config_.verbosity == BuildConfig::QUIET) return; @@ -299,7 +288,7 @@ void BuildStatus::PrintStatus(const Edge* edge, EdgeStatus status) { if (to_print.empty() || force_full_command) to_print = edge->GetBinding("command"); - to_print = FormatProgressStatus(progress_status_format_, status) + to_print; + to_print = FormatProgressStatus(progress_status_format_, time, status) + to_print; printer_.Print(to_print, force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); @@ -729,9 +718,9 @@ bool RealCommandRunner::WaitForCommand(Result* result) { Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface) - : state_(state), config_(config), - plan_(this), disk_interface_(disk_interface), + DiskInterface* disk_interface, int64_t start_time_millis) + : state_(state), config_(config), plan_(this), + start_time_millis_(start_time_millis), disk_interface_(disk_interface), scan_(state, build_log, deps_log, disk_interface, &config_.depfile_parser_options) { status_ = new BuildStatus(config); @@ -904,7 +893,10 @@ bool Builder::StartEdge(Edge* edge, string* err) { if (edge->is_phony()) return true; - status_->BuildEdgeStarted(edge); + int64_t start_time_millis = GetTimeMillis() - start_time_millis_; + running_edges_.insert(make_pair(edge, start_time_millis)); + + status_->BuildEdgeStarted(edge, start_time_millis); // Create directories necessary for outputs. // XXX: this will block; do we care? @@ -957,9 +949,14 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { } } - int start_time, end_time; - status_->BuildEdgeFinished(edge, result->success(), result->output, - &start_time, &end_time); + int64_t start_time_millis, end_time_millis; + RunningEdgeMap::iterator it = running_edges_.find(edge); + start_time_millis = it->second; + end_time_millis = GetTimeMillis() - start_time_millis_; + running_edges_.erase(it); + + status_->BuildEdgeFinished(edge, end_time_millis, result->success(), + result->output); // The rest of this function only applies to successful commands. if (!result->success()) { @@ -1028,8 +1025,8 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { disk_interface_->RemoveFile(rspfile); if (scan_.build_log()) { - if (!scan_.build_log()->RecordCommand(edge, start_time, end_time, - output_mtime)) { + if (!scan_.build_log()->RecordCommand(edge, start_time_millis, + end_time_millis, output_mtime)) { *err = string("Error writing to build log: ") + strerror(errno); return false; } diff --git a/src/build.h b/src/build.h index 0a68478aef..6935086719 100644 --- a/src/build.h +++ b/src/build.h @@ -26,7 +26,6 @@ #include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" #include "line_printer.h" -#include "metrics.h" #include "util.h" // int64_t struct BuildLog; @@ -179,7 +178,7 @@ struct BuildConfig { struct Builder { Builder(State* state, const BuildConfig& config, BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface); + DiskInterface* disk_interface, int64_t start_time_millis); ~Builder(); /// Clean up after interrupted commands by deleting output files. @@ -227,6 +226,13 @@ struct Builder { const std::string& deps_prefix, std::vector* deps_nodes, std::string* err); + /// Map of running edge to time the edge started running. + typedef std::map RunningEdgeMap; + RunningEdgeMap running_edges_; + + /// Time the build started. + int64_t start_time_millis_; + DiskInterface* disk_interface_; DependencyScan scan_; @@ -239,9 +245,9 @@ struct Builder { struct BuildStatus { explicit BuildStatus(const BuildConfig& config); void PlanHasTotalEdges(int total); - void BuildEdgeStarted(const Edge* edge); - void BuildEdgeFinished(Edge* edge, bool success, const std::string& output, - int* start_time, int* end_time); + void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis); + void BuildEdgeFinished(Edge* edge, int64_t end_time_millis, bool success, + const std::string& output); void BuildLoadDyndeps(); void BuildStarted(); void BuildFinished(); @@ -257,21 +263,15 @@ struct BuildStatus { /// @param progress_status_format The format of the progress status. /// @param status The status of the edge. std::string FormatProgressStatus(const char* progress_status_format, - EdgeStatus status) const; + int64_t time, EdgeStatus status) const; private: - void PrintStatus(const Edge* edge, EdgeStatus status); + void PrintStatus(const Edge* edge, int64_t time, EdgeStatus status); const BuildConfig& config_; - /// Time the build started. - int64_t start_time_millis_; - int started_edges_, finished_edges_, total_edges_; - - /// Map of running edge to time the edge started running. - typedef std::map RunningEdgeMap; - RunningEdgeMap running_edges_; + int64_t time_millis_; /// Prints progress output. LinePrinter printer_; @@ -287,50 +287,30 @@ struct BuildStatus { snprintf(buf, S, format, rate); } - struct RateInfo { - RateInfo() : rate_(-1) {} - - void Restart() { stopwatch_.Restart(); } - double Elapsed() const { return stopwatch_.Elapsed(); } - double rate() { return rate_; } - - void UpdateRate(int edges) { - if (edges && stopwatch_.Elapsed()) - rate_ = edges / stopwatch_.Elapsed(); - } - - private: - double rate_; - Stopwatch stopwatch_; - }; - struct SlidingRateInfo { SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {} - void Restart() { stopwatch_.Restart(); } double rate() { return rate_; } - void UpdateRate(int update_hint) { + void UpdateRate(int update_hint, int64_t time_millis_) { if (update_hint == last_update_) return; last_update_ = update_hint; if (times_.size() == N) times_.pop(); - times_.push(stopwatch_.Elapsed()); + times_.push(time_millis_); if (times_.back() != times_.front()) - rate_ = times_.size() / (times_.back() - times_.front()); + rate_ = times_.size() / ((times_.back() - times_.front()) / 1e3); } private: double rate_; - Stopwatch stopwatch_; const size_t N; std::queue times_; int last_update_; }; - mutable RateInfo overall_rate_; mutable SlidingRateInfo current_rate_; }; diff --git a/src/build_test.cc b/src/build_test.cc index 0baabc4bd9..4ccb2c4bb1 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -486,13 +486,13 @@ struct FakeCommandRunner : public CommandRunner { struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { BuildTest() : config_(MakeConfig()), command_runner_(&fs_), - builder_(&state_, config_, NULL, NULL, &fs_), + builder_(&state_, config_, NULL, NULL, &fs_, 0), status_(config_) { } explicit BuildTest(DepsLog* log) : config_(MakeConfig()), command_runner_(&fs_), - builder_(&state_, config_, NULL, log, &fs_), status_(config_) {} + builder_(&state_, config_, NULL, log, &fs_, 0), status_(config_) {} virtual void SetUp() { StateTestWithBuiltinRules::SetUp(); @@ -563,7 +563,7 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, pdeps_log = &deps_log; } - Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_); + Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_, 0); EXPECT_TRUE(builder.AddTarget(target, &err)); command_runner_.commands_ran_.clear(); @@ -1400,7 +1400,7 @@ TEST_F(BuildWithLogTest, RestatTest) { ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); ASSERT_EQ("", err); - EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]", + EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]", 0, BuildStatus::kEdgeStarted)); command_runner_.commands_ran_.clear(); state_.Reset(); @@ -1843,13 +1843,13 @@ TEST_F(BuildTest, StatusFormatElapsed) { status_.BuildStarted(); // Before any task is done, the elapsed time must be zero. EXPECT_EQ("[%/e0.000]", - status_.FormatProgressStatus("[%%/e%e]", + status_.FormatProgressStatus("[%%/e%e]", 0, BuildStatus::kEdgeStarted)); } TEST_F(BuildTest, StatusFormatReplacePlaceholder) { EXPECT_EQ("[%/s0/t0/r0/u0/f0]", - status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", + status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", 0, BuildStatus::kEdgeStarted)); } @@ -2120,7 +2120,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2150,7 +2150,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2191,7 +2191,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2220,7 +2220,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2256,7 +2256,7 @@ TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) { // The deps log is NULL in dry runs. config_.dry_run = true; - Builder builder(&state, config_, NULL, NULL, &fs_); + Builder builder(&state, config_, NULL, NULL, &fs_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2314,7 +2314,7 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2340,7 +2340,7 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2373,7 +2373,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); @@ -2394,7 +2394,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); Edge* edge = state.edges_.back(); @@ -2435,7 +2435,7 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); ASSERT_EQ("", err); @@ -2458,7 +2458,7 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); builder.command_runner_.reset(&command_runner_); Edge* edge = state.edges_.back(); diff --git a/src/ninja.cc b/src/ninja.cc index eb9732058b..57690bef29 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -82,7 +82,8 @@ struct Options { /// to poke into these, so store them as fields on an object. struct NinjaMain : public BuildLogUser { NinjaMain(const char* ninja_command, const BuildConfig& config) : - ninja_command_(ninja_command), config_(config) {} + ninja_command_(ninja_command), config_(config), + start_time_millis_(GetTimeMillis()) {} /// Command line used to run Ninja. const char* ninja_command_; @@ -172,6 +173,8 @@ struct NinjaMain : public BuildLogUser { Error("%s", err.c_str()); // Log and ignore Stat() errors. return mtime == 0; } + + int64_t start_time_millis_; }; /// Subtools, accessible via "-t foo". @@ -249,7 +252,8 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err) { if (!node) return false; - Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_); + Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, + start_time_millis_); if (!builder.AddTarget(node, err)) return false; @@ -1199,7 +1203,8 @@ int NinjaMain::RunBuild(int argc, char** argv) { disk_interface_.AllowStatCache(g_experimental_statcache); - Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_); + Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, + start_time_millis_); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { if (!err.empty()) { From 045890cee352be512beea96da0d142f219472fbb Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 18 Nov 2016 13:22:48 -0800 Subject: [PATCH 22/96] Simplify running edges status Store the number of running edges instead of trying to compute it from the started and finshed edge counts, which may be different for starting and ending status messages. Allows removing the status parameter to PrintStatus and the EdgeStatus enum. --- src/build.cc | 26 +++++++++++++++----------- src/build.h | 11 +++-------- src/build_test.cc | 9 +++------ 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/build.cc b/src/build.cc index f04e2f2ff2..3b1a3f49ce 100644 --- a/src/build.cc +++ b/src/build.cc @@ -81,7 +81,7 @@ bool DryRunCommandRunner::WaitForCommand(Result* result) { BuildStatus::BuildStatus(const BuildConfig& config) : config_(config), - started_edges_(0), finished_edges_(0), total_edges_(0), + started_edges_(0), finished_edges_(0), total_edges_(0), running_edges_(0), time_millis_(0), progress_status_format_(NULL), current_rate_(config.parallelism) { // Don't do anything fancy in verbose mode. @@ -103,7 +103,7 @@ void BuildStatus::BuildEdgeStarted(const Edge* edge, time_millis_ = start_time_millis; if (edge->use_console() || printer_.is_smart_terminal()) - PrintStatus(edge, start_time_millis, kEdgeStarted); + PrintStatus(edge, start_time_millis); if (edge->use_console()) printer_.SetConsoleLocked(true); @@ -122,7 +122,12 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, return; if (!edge->use_console()) - PrintStatus(edge, end_time_millis, kEdgeFinished); + PrintStatus(edge, end_time_millis); + + // Update running_edges_ after PrintStatus so that the number of running + // edges doesn't oscillate between config.parallelism_ and + // config.parallelism_ - 1. + --running_edges_; // Print the command that is spewing before printing its output. if (!success) { @@ -185,6 +190,9 @@ void BuildStatus::BuildLoadDyndeps() { } void BuildStatus::BuildStarted() { + started_edges_ = 0; + finished_edges_ = 0; + running_edges_ = 0; } void BuildStatus::BuildFinished() { @@ -193,7 +201,7 @@ void BuildStatus::BuildFinished() { } string BuildStatus::FormatProgressStatus( - const char* progress_status_format, int64_t time, EdgeStatus status) const { + const char* progress_status_format, int64_t time) const { string out; char buf[32]; for (const char* s = progress_status_format; *s != '\0'; ++s) { @@ -218,11 +226,7 @@ string BuildStatus::FormatProgressStatus( // Running edges. case 'r': { - int running_edges = started_edges_ - finished_edges_; - // count the edge that just finished as a running edge - if (status == kEdgeFinished) - running_edges++; - snprintf(buf, sizeof(buf), "%d", running_edges); + snprintf(buf, sizeof(buf), "%d", running_edges_); out += buf; break; } @@ -278,7 +282,7 @@ string BuildStatus::FormatProgressStatus( return out; } -void BuildStatus::PrintStatus(const Edge* edge, int64_t time, EdgeStatus status) { +void BuildStatus::PrintStatus(const Edge* edge, int64_t time) { if (config_.verbosity == BuildConfig::QUIET) return; @@ -288,7 +292,7 @@ void BuildStatus::PrintStatus(const Edge* edge, int64_t time, EdgeStatus status) if (to_print.empty() || force_full_command) to_print = edge->GetBinding("command"); - to_print = FormatProgressStatus(progress_status_format_, time, status) + to_print; + to_print = FormatProgressStatus(progress_status_format_, time) + to_print; printer_.Print(to_print, force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); diff --git a/src/build.h b/src/build.h index 6935086719..265723aeb9 100644 --- a/src/build.h +++ b/src/build.h @@ -252,25 +252,20 @@ struct BuildStatus { void BuildStarted(); void BuildFinished(); - enum EdgeStatus { - kEdgeStarted, - kEdgeFinished, - }; - /// Format the progress status string by replacing the placeholders. /// See the user manual for more information about the available /// placeholders. /// @param progress_status_format The format of the progress status. /// @param status The status of the edge. std::string FormatProgressStatus(const char* progress_status_format, - int64_t time, EdgeStatus status) const; + int64_t time) const; private: - void PrintStatus(const Edge* edge, int64_t time, EdgeStatus status); + void PrintStatus(const Edge* edge, int64_t time); const BuildConfig& config_; - int started_edges_, finished_edges_, total_edges_; + int started_edges_, finished_edges_, total_edges_, running_edges_; int64_t time_millis_; /// Prints progress output. diff --git a/src/build_test.cc b/src/build_test.cc index 4ccb2c4bb1..1baec65e5d 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -1400,8 +1400,7 @@ TEST_F(BuildWithLogTest, RestatTest) { ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); ASSERT_EQ("", err); - EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]", 0, - BuildStatus::kEdgeStarted)); + EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]", 0)); command_runner_.commands_ran_.clear(); state_.Reset(); @@ -1843,14 +1842,12 @@ TEST_F(BuildTest, StatusFormatElapsed) { status_.BuildStarted(); // Before any task is done, the elapsed time must be zero. EXPECT_EQ("[%/e0.000]", - status_.FormatProgressStatus("[%%/e%e]", 0, - BuildStatus::kEdgeStarted)); + status_.FormatProgressStatus("[%%/e%e]", 0)); } TEST_F(BuildTest, StatusFormatReplacePlaceholder) { EXPECT_EQ("[%/s0/t0/r0/u0/f0]", - status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", 0, - BuildStatus::kEdgeStarted)); + status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", 0)); } TEST_F(BuildTest, FailedDepsParse) { From 589f5b2497929a50a1c74786478cc6fea7a2e1c6 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 14 Nov 2016 15:59:41 -0800 Subject: [PATCH 23/96] Turn BuildStatus into an interface Make BuildStatus an abstract interface, and move the current implementation to StatusPrinter, to make way for a serialized Status output. --- CMakeLists.txt | 1 + configure.py | 2 + src/build.cc | 231 +----------------------------------------- src/build.h | 76 +------------- src/build_test.cc | 40 ++++---- src/ninja.cc | 20 ++-- src/status.cc | 244 +++++++++++++++++++++++++++++++++++++++++++++ src/status.h | 107 ++++++++++++++++++++ src/status_test.cc | 35 +++++++ 9 files changed, 429 insertions(+), 327 deletions(-) create mode 100644 src/status.cc create mode 100644 src/status.h create mode 100644 src/status_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 39348c9ca4..4d99a0c5f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,7 @@ add_library(libninja OBJECT src/metrics.cc src/parser.cc src/state.cc + src/status.cc src/string_piece_util.cc src/util.cc src/version.cc diff --git a/configure.py b/configure.py index cded265742..1a3aca5846 100755 --- a/configure.py +++ b/configure.py @@ -513,6 +513,7 @@ def has_re2c(): 'metrics', 'parser', 'state', + 'status', 'string_piece_util', 'util', 'version']: @@ -579,6 +580,7 @@ def has_re2c(): 'manifest_parser_test', 'ninja_test', 'state_test', + 'status_test', 'string_piece_util_test', 'subprocess_test', 'test', diff --git a/src/build.cc b/src/build.cc index 3b1a3f49ce..3d1e76b45a 100644 --- a/src/build.cc +++ b/src/build.cc @@ -20,11 +20,6 @@ #include #include -#ifdef _WIN32 -#include -#include -#endif - #if defined(__SVR4) && defined(__sun) #include #endif @@ -38,6 +33,7 @@ #include "graph.h" #include "metrics.h" #include "state.h" +#include "status.h" #include "subprocess.h" #include "util.h" @@ -79,225 +75,6 @@ bool DryRunCommandRunner::WaitForCommand(Result* result) { } // namespace -BuildStatus::BuildStatus(const BuildConfig& config) - : config_(config), - started_edges_(0), finished_edges_(0), total_edges_(0), running_edges_(0), - time_millis_(0), progress_status_format_(NULL), - current_rate_(config.parallelism) { - // Don't do anything fancy in verbose mode. - if (config_.verbosity != BuildConfig::NORMAL) - printer_.set_smart_terminal(false); - - progress_status_format_ = getenv("NINJA_STATUS"); - if (!progress_status_format_) - progress_status_format_ = "[%f/%t] "; -} - -void BuildStatus::PlanHasTotalEdges(int total) { - total_edges_ = total; -} - -void BuildStatus::BuildEdgeStarted(const Edge* edge, - int64_t start_time_millis) { - ++started_edges_; - time_millis_ = start_time_millis; - - if (edge->use_console() || printer_.is_smart_terminal()) - PrintStatus(edge, start_time_millis); - - if (edge->use_console()) - printer_.SetConsoleLocked(true); -} - -void BuildStatus::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, - bool success, - const string& output) { - time_millis_ = end_time_millis; - ++finished_edges_; - - if (edge->use_console()) - printer_.SetConsoleLocked(false); - - if (config_.verbosity == BuildConfig::QUIET) - return; - - if (!edge->use_console()) - PrintStatus(edge, end_time_millis); - - // Update running_edges_ after PrintStatus so that the number of running - // edges doesn't oscillate between config.parallelism_ and - // config.parallelism_ - 1. - --running_edges_; - - // Print the command that is spewing before printing its output. - if (!success) { - string outputs; - for (vector::const_iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) - outputs += (*o)->path() + " "; - - if (printer_.supports_color()) { - printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n"); - } else { - printer_.PrintOnNewLine("FAILED: " + outputs + "\n"); - } - printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n"); - } - - if (!output.empty()) { - // ninja sets stdout and stderr of subprocesses to a pipe, to be able to - // check if the output is empty. Some compilers, e.g. clang, check - // isatty(stderr) to decide if they should print colored output. - // To make it possible to use colored output with ninja, subprocesses should - // be run with a flag that forces them to always print color escape codes. - // To make sure these escape codes don't show up in a file if ninja's output - // is piped to a file, ninja strips ansi escape codes again if it's not - // writing to a |smart_terminal_|. - // (Launching subprocesses in pseudo ttys doesn't work because there are - // only a few hundred available on some systems, and ninja can launch - // thousands of parallel compile commands.) - string final_output; - if (!printer_.supports_color()) - final_output = StripAnsiEscapeCodes(output); - else - final_output = output; - -#ifdef _WIN32 - // Fix extra CR being added on Windows, writing out CR CR LF (#773) - _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix -#endif - - printer_.PrintOnNewLine(final_output); - -#ifdef _WIN32 - _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix -#endif - } -} - -void BuildStatus::BuildLoadDyndeps() { - // The DependencyScan calls EXPLAIN() to print lines explaining why - // it considers a portion of the graph to be out of date. Normally - // this is done before the build starts, but our caller is about to - // load a dyndep file during the build. Doing so may generate more - // explanation lines (via fprintf directly to stderr), but in an - // interactive console the cursor is currently at the end of a status - // line. Start a new line so that the first explanation does not - // append to the status line. After the explanations are done a - // new build status line will appear. - if (g_explaining) - printer_.PrintOnNewLine(""); -} - -void BuildStatus::BuildStarted() { - started_edges_ = 0; - finished_edges_ = 0; - running_edges_ = 0; -} - -void BuildStatus::BuildFinished() { - printer_.SetConsoleLocked(false); - printer_.PrintOnNewLine(""); -} - -string BuildStatus::FormatProgressStatus( - const char* progress_status_format, int64_t time) const { - string out; - char buf[32]; - for (const char* s = progress_status_format; *s != '\0'; ++s) { - if (*s == '%') { - ++s; - switch (*s) { - case '%': - out.push_back('%'); - break; - - // Started edges. - case 's': - snprintf(buf, sizeof(buf), "%d", started_edges_); - out += buf; - break; - - // Total edges. - case 't': - snprintf(buf, sizeof(buf), "%d", total_edges_); - out += buf; - break; - - // Running edges. - case 'r': { - snprintf(buf, sizeof(buf), "%d", running_edges_); - out += buf; - break; - } - - // Unstarted edges. - case 'u': - snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_); - out += buf; - break; - - // Finished edges. - case 'f': - snprintf(buf, sizeof(buf), "%d", finished_edges_); - out += buf; - break; - - // Overall finished edges per second. - case 'o': - SnprintfRate(finished_edges_ / (time_millis_ / 1e3), buf, "%.1f"); - out += buf; - break; - - // Current rate, average over the last '-j' jobs. - case 'c': - current_rate_.UpdateRate(finished_edges_, time_millis_); - SnprintfRate(current_rate_.rate(), buf, "%.1f"); - out += buf; - break; - - // Percentage - case 'p': { - int percent = (100 * finished_edges_) / total_edges_; - snprintf(buf, sizeof(buf), "%3i%%", percent); - out += buf; - break; - } - - case 'e': { - snprintf(buf, sizeof(buf), "%.3f", time_millis_ / 1e3); - out += buf; - break; - } - - default: - Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); - return ""; - } - } else { - out.push_back(*s); - } - } - - return out; -} - -void BuildStatus::PrintStatus(const Edge* edge, int64_t time) { - if (config_.verbosity == BuildConfig::QUIET) - return; - - bool force_full_command = config_.verbosity == BuildConfig::VERBOSE; - - string to_print = edge->GetBinding("description"); - if (to_print.empty() || force_full_command) - to_print = edge->GetBinding("command"); - - to_print = FormatProgressStatus(progress_status_format_, time) + to_print; - - printer_.Print(to_print, - force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); -} - Plan::Plan(Builder* builder) : builder_(builder) , command_edges_(0) @@ -722,12 +499,12 @@ bool RealCommandRunner::WaitForCommand(Result* result) { Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface, int64_t start_time_millis) - : state_(state), config_(config), plan_(this), + DiskInterface* disk_interface, Status *status, + int64_t start_time_millis) + : state_(state), config_(config), plan_(this), status_(status), start_time_millis_(start_time_millis), disk_interface_(disk_interface), scan_(state, build_log, deps_log, disk_interface, &config_.depfile_parser_options) { - status_ = new BuildStatus(config); } Builder::~Builder() { diff --git a/src/build.h b/src/build.h index 265723aeb9..06823c2e50 100644 --- a/src/build.h +++ b/src/build.h @@ -25,16 +25,15 @@ #include "depfile_parser.h" #include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" -#include "line_printer.h" #include "util.h" // int64_t struct BuildLog; -struct BuildStatus; struct Builder; struct DiskInterface; struct Edge; struct Node; struct State; +struct Status; /// Plan stores the state of a build plan: what we intend to build, /// which steps we're ready to execute. @@ -178,7 +177,8 @@ struct BuildConfig { struct Builder { Builder(State* state, const BuildConfig& config, BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface, int64_t start_time_millis); + DiskInterface* disk_interface, Status* status, + int64_t start_time_millis); ~Builder(); /// Clean up after interrupted commands by deleting output files. @@ -219,7 +219,7 @@ struct Builder { #else std::unique_ptr command_runner_; // auto_ptr was removed in C++17. #endif - BuildStatus* status_; + Status* status_; private: bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type, @@ -241,72 +241,4 @@ struct Builder { void operator=(const Builder &other); // DO NOT IMPLEMENT }; -/// Tracks the status of a build: completion fraction, printing updates. -struct BuildStatus { - explicit BuildStatus(const BuildConfig& config); - void PlanHasTotalEdges(int total); - void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis); - void BuildEdgeFinished(Edge* edge, int64_t end_time_millis, bool success, - const std::string& output); - void BuildLoadDyndeps(); - void BuildStarted(); - void BuildFinished(); - - /// Format the progress status string by replacing the placeholders. - /// See the user manual for more information about the available - /// placeholders. - /// @param progress_status_format The format of the progress status. - /// @param status The status of the edge. - std::string FormatProgressStatus(const char* progress_status_format, - int64_t time) const; - - private: - void PrintStatus(const Edge* edge, int64_t time); - - const BuildConfig& config_; - - int started_edges_, finished_edges_, total_edges_, running_edges_; - int64_t time_millis_; - - /// Prints progress output. - LinePrinter printer_; - - /// The custom progress status format to use. - const char* progress_status_format_; - - template - void SnprintfRate(double rate, char(&buf)[S], const char* format) const { - if (rate == -1) - snprintf(buf, S, "?"); - else - snprintf(buf, S, format, rate); - } - - struct SlidingRateInfo { - SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {} - - double rate() { return rate_; } - - void UpdateRate(int update_hint, int64_t time_millis_) { - if (update_hint == last_update_) - return; - last_update_ = update_hint; - - if (times_.size() == N) - times_.pop(); - times_.push(time_millis_); - if (times_.back() != times_.front()) - rate_ = times_.size() / ((times_.back() - times_.front()) / 1e3); - } - - private: - double rate_; - const size_t N; - std::queue times_; - int last_update_; - }; - - mutable SlidingRateInfo current_rate_; -}; - #endif // NINJA_BUILD_H_ diff --git a/src/build_test.cc b/src/build_test.cc index 1baec65e5d..f58a7de870 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -19,6 +19,7 @@ #include "build_log.h" #include "deps_log.h" #include "graph.h" +#include "status.h" #include "test.h" using namespace std; @@ -485,14 +486,13 @@ struct FakeCommandRunner : public CommandRunner { }; struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { - BuildTest() : config_(MakeConfig()), command_runner_(&fs_), - builder_(&state_, config_, NULL, NULL, &fs_, 0), - status_(config_) { + BuildTest() : config_(MakeConfig()), command_runner_(&fs_), status_(config_), + builder_(&state_, config_, NULL, NULL, &fs_, &status_, 0) { } explicit BuildTest(DepsLog* log) - : config_(MakeConfig()), command_runner_(&fs_), - builder_(&state_, config_, NULL, log, &fs_, 0), status_(config_) {} + : config_(MakeConfig()), command_runner_(&fs_), status_(config_), + builder_(&state_, config_, NULL, log, &fs_, &status_, 0) {} virtual void SetUp() { StateTestWithBuiltinRules::SetUp(); @@ -532,9 +532,8 @@ struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { BuildConfig config_; FakeCommandRunner command_runner_; VirtualFileSystem fs_; + StatusPrinter status_; Builder builder_; - - BuildStatus status_; }; void BuildTest::RebuildTarget(const string& target, const char* manifest, @@ -563,7 +562,7 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, pdeps_log = &deps_log; } - Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_, 0); + Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_, &status_, 0); EXPECT_TRUE(builder.AddTarget(target, &err)); command_runner_.commands_ran_.clear(); @@ -1400,7 +1399,8 @@ TEST_F(BuildWithLogTest, RestatTest) { ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); ASSERT_EQ("", err); - EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]", 0)); + EXPECT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ(3u, builder_.plan_.command_edge_count()); command_runner_.commands_ran_.clear(); state_.Reset(); @@ -2117,7 +2117,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2147,7 +2147,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2188,7 +2188,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2217,7 +2217,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2253,7 +2253,7 @@ TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) { // The deps log is NULL in dry runs. config_.dry_run = true; - Builder builder(&state, config_, NULL, NULL, &fs_, 0); + Builder builder(&state, config_, NULL, NULL, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); @@ -2311,7 +2311,7 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); @@ -2337,7 +2337,7 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2370,7 +2370,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); @@ -2391,7 +2391,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); Edge* edge = state.edges_.back(); @@ -2432,7 +2432,7 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); ASSERT_EQ("", err); @@ -2455,7 +2455,7 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_, 0); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); Edge* edge = state.edges_.back(); diff --git a/src/ninja.cc b/src/ninja.cc index 57690bef29..d92e95b072 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -43,6 +43,7 @@ #include "manifest_parser.h" #include "metrics.h" #include "state.h" +#include "status.h" #include "util.h" #include "version.h" @@ -145,11 +146,11 @@ struct NinjaMain : public BuildLogUser { /// Rebuild the manifest, if necessary. /// Fills in \a err on error. /// @return true if the manifest was rebuilt. - bool RebuildManifest(const char* input_file, string* err); + bool RebuildManifest(const char* input_file, string* err, Status* status); /// Build the targets listed on the command line. /// @return an exit code. - int RunBuild(int argc, char** argv); + int RunBuild(int argc, char** argv, Status* status); /// Dump the output requested by '-d stats'. void DumpMetrics(); @@ -243,7 +244,8 @@ int GuessParallelism() { /// Rebuild the build manifest, if necessary. /// Returns true if the manifest was rebuilt. -bool NinjaMain::RebuildManifest(const char* input_file, string* err) { +bool NinjaMain::RebuildManifest(const char* input_file, string* err, + Status *status) { string path = input_file; uint64_t slash_bits; // Unused because this path is only used for lookup. if (!CanonicalizePath(&path, &slash_bits, err)) @@ -253,7 +255,7 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err) { return false; Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, - start_time_millis_); + status, start_time_millis_); if (!builder.AddTarget(node, err)) return false; @@ -1193,7 +1195,7 @@ bool NinjaMain::EnsureBuildDirExists() { return true; } -int NinjaMain::RunBuild(int argc, char** argv) { +int NinjaMain::RunBuild(int argc, char** argv, Status* status) { string err; vector targets; if (!CollectTargetsFromArgs(argc, argv, &targets, &err)) { @@ -1204,7 +1206,7 @@ int NinjaMain::RunBuild(int argc, char** argv) { disk_interface_.AllowStatCache(g_experimental_statcache); Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, - start_time_millis_); + status, start_time_millis_); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { if (!err.empty()) { @@ -1364,6 +1366,8 @@ NORETURN void real_main(int argc, char** argv) { if (exit_code >= 0) exit(exit_code); + Status* status = new StatusPrinter(config); + if (options.working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for @@ -1416,7 +1420,7 @@ NORETURN void real_main(int argc, char** argv) { exit((ninja.*options.tool->func)(&options, argc, argv)); // Attempt to rebuild the manifest before building anything else - if (ninja.RebuildManifest(options.input_file, &err)) { + if (ninja.RebuildManifest(options.input_file, &err, status)) { // In dry_run mode the regeneration will succeed without changing the // manifest forever. Better to return immediately. if (config.dry_run) @@ -1428,7 +1432,7 @@ NORETURN void real_main(int argc, char** argv) { exit(1); } - int result = ninja.RunBuild(argc, argv); + int result = ninja.RunBuild(argc, argv, status); if (g_metrics) ninja.DumpMetrics(); exit(result); diff --git a/src/status.cc b/src/status.cc new file mode 100644 index 0000000000..a308f62c81 --- /dev/null +++ b/src/status.cc @@ -0,0 +1,244 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "status.h" + +#include + +#ifdef _WIN32 +#include +#include +#endif + +#include "debug_flags.h" + +using namespace std; + +StatusPrinter::StatusPrinter(const BuildConfig& config) + : config_(config), + started_edges_(0), finished_edges_(0), total_edges_(0), running_edges_(0), + time_millis_(0), progress_status_format_(NULL), + current_rate_(config.parallelism) { + + // Don't do anything fancy in verbose mode. + if (config_.verbosity != BuildConfig::NORMAL) + printer_.set_smart_terminal(false); + + progress_status_format_ = getenv("NINJA_STATUS"); + if (!progress_status_format_) + progress_status_format_ = "[%f/%t] "; +} + +void StatusPrinter::PlanHasTotalEdges(int total) { + total_edges_ = total; +} + +void StatusPrinter::BuildEdgeStarted(const Edge* edge, + int64_t start_time_millis) { + ++started_edges_; + ++running_edges_; + time_millis_ = start_time_millis; + + if (edge->use_console() || printer_.is_smart_terminal()) + PrintStatus(edge, start_time_millis); + + if (edge->use_console()) + printer_.SetConsoleLocked(true); +} + +void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, + bool success, const string& output) { + time_millis_ = end_time_millis; + ++finished_edges_; + + if (edge->use_console()) + printer_.SetConsoleLocked(false); + + if (config_.verbosity == BuildConfig::QUIET) + return; + + if (!edge->use_console()) + PrintStatus(edge, end_time_millis); + + --running_edges_; + + // Print the command that is spewing before printing its output. + if (!success) { + string outputs; + for (vector::const_iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) + outputs += (*o)->path() + " "; + + if (printer_.supports_color()) { + printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n"); + } else { + printer_.PrintOnNewLine("FAILED: " + outputs + "\n"); + } + printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n"); + } + + if (!output.empty()) { + // ninja sets stdout and stderr of subprocesses to a pipe, to be able to + // check if the output is empty. Some compilers, e.g. clang, check + // isatty(stderr) to decide if they should print colored output. + // To make it possible to use colored output with ninja, subprocesses should + // be run with a flag that forces them to always print color escape codes. + // To make sure these escape codes don't show up in a file if ninja's output + // is piped to a file, ninja strips ansi escape codes again if it's not + // writing to a |smart_terminal_|. + // (Launching subprocesses in pseudo ttys doesn't work because there are + // only a few hundred available on some systems, and ninja can launch + // thousands of parallel compile commands.) + string final_output; + if (!printer_.supports_color()) + final_output = StripAnsiEscapeCodes(output); + else + final_output = output; + +#ifdef _WIN32 + // Fix extra CR being added on Windows, writing out CR CR LF (#773) + _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix +#endif + + printer_.PrintOnNewLine(final_output); + +#ifdef _WIN32 + _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix +#endif + } +} + +void StatusPrinter::BuildLoadDyndeps() { + // The DependencyScan calls EXPLAIN() to print lines explaining why + // it considers a portion of the graph to be out of date. Normally + // this is done before the build starts, but our caller is about to + // load a dyndep file during the build. Doing so may generate more + // explanation lines (via fprintf directly to stderr), but in an + // interactive console the cursor is currently at the end of a status + // line. Start a new line so that the first explanation does not + // append to the status line. After the explanations are done a + // new build status line will appear. + if (g_explaining) + printer_.PrintOnNewLine(""); +} + +void StatusPrinter::BuildStarted() { + started_edges_ = 0; + finished_edges_ = 0; + running_edges_ = 0; +} + +void StatusPrinter::BuildFinished() { + printer_.SetConsoleLocked(false); + printer_.PrintOnNewLine(""); +} + +string StatusPrinter::FormatProgressStatus(const char* progress_status_format, + int64_t time_millis) const { + string out; + char buf[32]; + for (const char* s = progress_status_format; *s != '\0'; ++s) { + if (*s == '%') { + ++s; + switch (*s) { + case '%': + out.push_back('%'); + break; + + // Started edges. + case 's': + snprintf(buf, sizeof(buf), "%d", started_edges_); + out += buf; + break; + + // Total edges. + case 't': + snprintf(buf, sizeof(buf), "%d", total_edges_); + out += buf; + break; + + // Running edges. + case 'r': { + snprintf(buf, sizeof(buf), "%d", running_edges_); + out += buf; + break; + } + + // Unstarted edges. + case 'u': + snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_); + out += buf; + break; + + // Finished edges. + case 'f': + snprintf(buf, sizeof(buf), "%d", finished_edges_); + out += buf; + break; + + // Overall finished edges per second. + case 'o': + SnprintfRate(finished_edges_ / (time_millis_ / 1e3), buf, "%.1f"); + out += buf; + break; + + // Current rate, average over the last '-j' jobs. + case 'c': + current_rate_.UpdateRate(finished_edges_, time_millis_); + SnprintfRate(current_rate_.rate(), buf, "%.1f"); + out += buf; + break; + + // Percentage + case 'p': { + int percent = (100 * finished_edges_) / total_edges_; + snprintf(buf, sizeof(buf), "%3i%%", percent); + out += buf; + break; + } + + case 'e': { + snprintf(buf, sizeof(buf), "%.3f", time_millis_ / 1e3); + out += buf; + break; + } + + default: + Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); + return ""; + } + } else { + out.push_back(*s); + } + } + + return out; +} + +void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) { + if (config_.verbosity == BuildConfig::QUIET) + return; + + bool force_full_command = config_.verbosity == BuildConfig::VERBOSE; + + string to_print = edge->GetBinding("description"); + if (to_print.empty() || force_full_command) + to_print = edge->GetBinding("command"); + + to_print = FormatProgressStatus(progress_status_format_, time_millis) + + to_print; + + printer_.Print(to_print, + force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); +} diff --git a/src/status.h b/src/status.h new file mode 100644 index 0000000000..8344b8e9c2 --- /dev/null +++ b/src/status.h @@ -0,0 +1,107 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_STATUS_H_ +#define NINJA_STATUS_H_ + +#include +#include + +#include "build.h" +#include "line_printer.h" + +/// Abstract interface to object that tracks the status of a build: +/// completion fraction, printing updates. +struct Status { + virtual void PlanHasTotalEdges(int total) = 0; + virtual void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis) = 0; + virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis, + bool success, const std::string& output) = 0; + virtual void BuildLoadDyndeps() = 0; + virtual void BuildStarted() = 0; + virtual void BuildFinished() = 0; + virtual ~Status() { } +}; + +/// Implementation of the BuildStatus interface that prints the status as +/// human-readable strings to stdout +struct StatusPrinter : Status { + explicit StatusPrinter(const BuildConfig& config); + virtual void PlanHasTotalEdges(int total); + virtual void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis); + virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis, + bool success, const std::string& output); + virtual void BuildLoadDyndeps(); + virtual void BuildStarted(); + virtual void BuildFinished(); + virtual ~StatusPrinter() { } + + /// Format the progress status string by replacing the placeholders. + /// See the user manual for more information about the available + /// placeholders. + /// @param progress_status_format The format of the progress status. + /// @param status The status of the edge. + std::string FormatProgressStatus(const char* progress_status_format, + int64_t time_millis) const; + + private: + void PrintStatus(const Edge* edge, int64_t time_millis); + + const BuildConfig& config_; + + int started_edges_, finished_edges_, total_edges_, running_edges_; + int64_t time_millis_; + + /// Prints progress output. + LinePrinter printer_; + + /// The custom progress status format to use. + const char* progress_status_format_; + + template + void SnprintfRate(double rate, char(&buf)[S], const char* format) const { + if (rate == -1) + snprintf(buf, S, "?"); + else + snprintf(buf, S, format, rate); + } + + struct SlidingRateInfo { + SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {} + + double rate() { return rate_; } + + void UpdateRate(int update_hint, int64_t time_millis_) { + if (update_hint == last_update_) + return; + last_update_ = update_hint; + + if (times_.size() == N) + times_.pop(); + times_.push(time_millis_); + if (times_.back() != times_.front()) + rate_ = times_.size() / ((times_.back() - times_.front()) / 1e3); + } + + private: + double rate_; + const size_t N; + std::queue times_; + int last_update_; + }; + + mutable SlidingRateInfo current_rate_; +}; + +#endif // NINJA_STATUS_H_ diff --git a/src/status_test.cc b/src/status_test.cc new file mode 100644 index 0000000000..6e42490ab1 --- /dev/null +++ b/src/status_test.cc @@ -0,0 +1,35 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "status.h" + +#include "test.h" + +TEST(StatusTest, StatusFormatElapsed) { + BuildConfig config; + StatusPrinter status(config); + + status.BuildStarted(); + // Before any task is done, the elapsed time must be zero. + EXPECT_EQ("[%/e0.000]", + status.FormatProgressStatus("[%%/e%e]", 0)); +} + +TEST(StatusTest, StatusFormatReplacePlaceholder) { + BuildConfig config; + StatusPrinter status(config); + + EXPECT_EQ("[%/s0/t0/r0/u0/f0]", + status.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", 0)); +} From 4b07ca35eb56b572720521fef7c202ec4cc1e325 Mon Sep 17 00:00:00 2001 From: Cameron Date: Thu, 31 Dec 2020 14:45:59 -0500 Subject: [PATCH 24/96] Added preprocessor if to clarify processor count workaround code is only needed on 32-bit builds (following code review) --- src/util.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index e8455ecc15..6ea854bf34 100644 --- a/src/util.cc +++ b/src/util.cc @@ -481,6 +481,7 @@ string StripAnsiEscapeCodes(const string& in) { int GetProcessorCount() { #ifdef _WIN32 +#ifndef _WIN64 // Need to use GetLogicalProcessorInformationEx to get real core count on // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 DWORD len = 0; @@ -508,7 +509,7 @@ int GetProcessorCount() { } } } - // fallback just in case +#endif return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); #else #ifdef CPU_COUNT From 97b4008aa788116683bb752d7c79b44128c2c485 Mon Sep 17 00:00:00 2001 From: Martin Boye Petersen Date: Sat, 2 Jan 2021 21:20:00 +0100 Subject: [PATCH 25/96] disk_interface: Improve the stat cache handling for case sensitive folders on Windows The path used as argument to FindFirstFileExA is no longer being converted to lowercase. By not converting the path to lower case, case sensitive folders will now be handled correctly. Case insensitive folders still works as expected because casing doesn't matter on those. All entries in the stat cache remains lowercase to avoid potential cache misses. --- src/disk_interface.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 49af001322..001b5b2e79 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -180,12 +180,13 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { dir = path; } - transform(dir.begin(), dir.end(), dir.begin(), ::tolower); + string dir_lowercase = dir; + transform(dir.begin(), dir.end(), dir_lowercase.begin(), ::tolower); transform(base.begin(), base.end(), base.begin(), ::tolower); - Cache::iterator ci = cache_.find(dir); + Cache::iterator ci = cache_.find(dir_lowercase); if (ci == cache_.end()) { - ci = cache_.insert(make_pair(dir, DirCache())).first; + ci = cache_.insert(make_pair(dir_lowercase, DirCache())).first; if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, err)) { cache_.erase(ci); return -1; From c161199646ec6de76d92d653b6ce9f10f2b43d07 Mon Sep 17 00:00:00 2001 From: Nathan Ringo Date: Tue, 12 Jan 2021 10:36:45 -0800 Subject: [PATCH 26/96] Adds dyndep support to ninja_syntax.py. --- misc/ninja_syntax.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index ab5c0d42c1..ca73b5ba99 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -74,7 +74,7 @@ def rule(self, name, command, description=None, depfile=None, self.variable('deps', deps, indent=1) def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, - variables=None, implicit_outputs=None, pool=None): + variables=None, implicit_outputs=None, pool=None, dyndep=None): outputs = as_list(outputs) out_outputs = [escape_path(x) for x in outputs] all_inputs = [escape_path(x) for x in as_list(inputs)] @@ -97,6 +97,8 @@ def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, ' '.join([rule] + all_inputs))) if pool is not None: self._line(' pool = %s' % pool) + if dyndep is not None: + self._line(' dyndep = %s' % dyndep) if variables: if isinstance(variables, dict): From fba4f7b1e5ba74b8d2f6d9e180dd2bd20aff3f08 Mon Sep 17 00:00:00 2001 From: Augustin Cavalier Date: Sat, 16 Sep 2017 12:25:48 +0200 Subject: [PATCH 27/96] Add Haiku support. --- src/util.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/util.cc b/src/util.cc index ae2e3c24d9..1e0d147dd6 100644 --- a/src/util.cc +++ b/src/util.cc @@ -625,6 +625,10 @@ double GetLoadAverage() { return -0.0f; return 1.0 / (1 << SI_LOAD_SHIFT) * si.loads[0]; } +#elif defined(__HAIKU__) +double GetLoadAverage() { + return -0.0f; +} #else double GetLoadAverage() { double loadavg[3] = { 0.0f, 0.0f, 0.0f }; From 942a523a7336da410e74fe5f72015fbc5ae2067a Mon Sep 17 00:00:00 2001 From: Nixon Enraght-Moony Date: Fri, 29 Jan 2021 18:34:58 +0000 Subject: [PATCH 28/96] Clarify purpose for implicit dependencies --- doc/manual.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index e1ae083b05..8fd5d8681d 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -949,8 +949,9 @@ source file of a compile command. + This is for expressing dependencies that don't show up on the command line of the command; for example, for a rule that runs a -script, the script itself should be an implicit dependency, as -changes to the script should cause the output to rebuild. +script that reads a hardcoded file, the hardcoded file should +be an implicit dependency, as changes to the file should cause +the output to rebuild, even though it doesn't show up in the arguments. + Note that dependencies as loaded through depfiles have slightly different semantics, as described in the <>. From ad3d29fb5375c3122b2318ea5efad170b83e74e5 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 15 Nov 2016 16:05:34 -0800 Subject: [PATCH 29/96] Put builder output through status interface Send all output after manifest parsing is finished to the Status interface, so that when status frontends are added they can handle build messages. --- src/build.cc | 2 +- src/ninja.cc | 18 +++++++++--------- src/status.cc | 22 ++++++++++++++++++++++ src/status.h | 12 +++++++++++- src/util.cc | 29 +++++++++++++++++++++++++---- src/util.h | 8 ++++++++ 6 files changed, 76 insertions(+), 15 deletions(-) diff --git a/src/build.cc b/src/build.cc index 3d1e76b45a..fb5890a76d 100644 --- a/src/build.cc +++ b/src/build.cc @@ -531,7 +531,7 @@ void Builder::Cleanup() { string err; TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err); if (new_mtime == -1) // Log and ignore Stat() errors. - Error("%s", err.c_str()); + status_->Error("%s", err.c_str()); if (!depfile.empty() || (*o)->mtime() != new_mtime) disk_interface_->RemoveFile((*o)->path()); } diff --git a/src/ninja.cc b/src/ninja.cc index d92e95b072..5053fcd075 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -245,7 +245,7 @@ int GuessParallelism() { /// Rebuild the build manifest, if necessary. /// Returns true if the manifest was rebuilt. bool NinjaMain::RebuildManifest(const char* input_file, string* err, - Status *status) { + Status* status) { string path = input_file; uint64_t slash_bits; // Unused because this path is only used for lookup. if (!CanonicalizePath(&path, &slash_bits, err)) @@ -1199,7 +1199,7 @@ int NinjaMain::RunBuild(int argc, char** argv, Status* status) { string err; vector targets; if (!CollectTargetsFromArgs(argc, argv, &targets, &err)) { - Error("%s", err.c_str()); + status->Error("%s", err.c_str()); return 1; } @@ -1210,7 +1210,7 @@ int NinjaMain::RunBuild(int argc, char** argv, Status* status) { for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { if (!err.empty()) { - Error("%s", err.c_str()); + status->Error("%s", err.c_str()); return 1; } else { // Added a target that is already up-to-date; not really @@ -1223,12 +1223,12 @@ int NinjaMain::RunBuild(int argc, char** argv, Status* status) { disk_interface_.AllowStatCache(false); if (builder.AlreadyUpToDate()) { - printf("ninja: no work to do.\n"); + status->Info("no work to do."); return 0; } if (!builder.Build(&err)) { - printf("ninja: build stopped: %s.\n", err.c_str()); + status->Info("build stopped: %s.", err.c_str()); if (err.find("interrupted by user") != string::npos) { return 2; } @@ -1375,7 +1375,7 @@ NORETURN void real_main(int argc, char** argv) { // Don't print this if a tool is being used, so that tool output // can be piped into a file without this string showing up. if (!options.tool) - printf("ninja: Entering directory `%s'\n", options.working_dir); + status->Info("Entering directory `%s'", options.working_dir); if (chdir(options.working_dir) < 0) { Fatal("chdir to '%s' - %s", options.working_dir, strerror(errno)); } @@ -1403,7 +1403,7 @@ NORETURN void real_main(int argc, char** argv) { ManifestParser parser(&ninja.state_, &ninja.disk_interface_, parser_opts); string err; if (!parser.Load(options.input_file, &err)) { - Error("%s", err.c_str()); + status->Error("%s", err.c_str()); exit(1); } @@ -1428,7 +1428,7 @@ NORETURN void real_main(int argc, char** argv) { // Start the build over with the new manifest. continue; } else if (!err.empty()) { - Error("rebuilding '%s': %s", options.input_file, err.c_str()); + status->Error("rebuilding '%s': %s", options.input_file, err.c_str()); exit(1); } @@ -1438,7 +1438,7 @@ NORETURN void real_main(int argc, char** argv) { exit(result); } - Error("manifest '%s' still dirty after %d tries\n", + status->Error("manifest '%s' still dirty after %d tries", options.input_file, kCycleLimit); exit(1); } diff --git a/src/status.cc b/src/status.cc index a308f62c81..171cbebf12 100644 --- a/src/status.cc +++ b/src/status.cc @@ -14,6 +14,7 @@ #include "status.h" +#include #include #ifdef _WIN32 @@ -242,3 +243,24 @@ void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) { printer_.Print(to_print, force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); } + +void StatusPrinter::Warning(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + ::Warning(msg, ap); + va_end(ap); +} + +void StatusPrinter::Error(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + ::Error(msg, ap); + va_end(ap); +} + +void StatusPrinter::Info(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + ::Info(msg, ap); + va_end(ap); +} diff --git a/src/status.h b/src/status.h index 8344b8e9c2..e211ba3487 100644 --- a/src/status.h +++ b/src/status.h @@ -31,10 +31,15 @@ struct Status { virtual void BuildLoadDyndeps() = 0; virtual void BuildStarted() = 0; virtual void BuildFinished() = 0; + + virtual void Info(const char* msg, ...) = 0; + virtual void Warning(const char* msg, ...) = 0; + virtual void Error(const char* msg, ...) = 0; + virtual ~Status() { } }; -/// Implementation of the BuildStatus interface that prints the status as +/// Implementation of the Status interface that prints the status as /// human-readable strings to stdout struct StatusPrinter : Status { explicit StatusPrinter(const BuildConfig& config); @@ -45,6 +50,11 @@ struct StatusPrinter : Status { virtual void BuildLoadDyndeps(); virtual void BuildStarted(); virtual void BuildFinished(); + + virtual void Info(const char* msg, ...); + virtual void Warning(const char* msg, ...); + virtual void Error(const char* msg, ...); + virtual ~StatusPrinter() { } /// Format the progress status string by replacing the placeholders. diff --git a/src/util.cc b/src/util.cc index 05bdb2d467..76bc2629a8 100644 --- a/src/util.cc +++ b/src/util.cc @@ -78,24 +78,45 @@ void Fatal(const char* msg, ...) { #endif } +void Warning(const char* msg, va_list ap) { + fprintf(stderr, "ninja: warning: "); + vfprintf(stderr, msg, ap); + fprintf(stderr, "\n"); +} + void Warning(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: warning: "); va_start(ap, msg); - vfprintf(stderr, msg, ap); + Warning(msg, ap); va_end(ap); +} + +void Error(const char* msg, va_list ap) { + fprintf(stderr, "ninja: error: "); + vfprintf(stderr, msg, ap); fprintf(stderr, "\n"); } void Error(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: error: "); va_start(ap, msg); - vfprintf(stderr, msg, ap); + Error(msg, ap); va_end(ap); +} + +void Info(const char* msg, va_list ap) { + fprintf(stderr, "ninja: "); + vfprintf(stderr, msg, ap); fprintf(stderr, "\n"); } +void Info(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + Info(msg, ap); + va_end(ap); +} + bool CanonicalizePath(string* path, uint64_t* slash_bits, string* err) { METRIC_RECORD("canonicalize str"); size_t len = path->size(); diff --git a/src/util.h b/src/util.h index 4e6ebb8810..15414e169f 100644 --- a/src/util.h +++ b/src/util.h @@ -21,6 +21,8 @@ #include #endif +#include + #include #include @@ -49,9 +51,15 @@ NORETURN void Fatal(const char* msg, ...); /// Log a warning message. void Warning(const char* msg, ...); +void Warning(const char* msg, va_list ap); /// Log an error message. void Error(const char* msg, ...); +void Error(const char* msg, va_list ap); + +/// Log an informational message. +void Info(const char* msg, ...); +void Info(const char* msg, va_list ap); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". /// |slash_bits| has bits set starting from lowest for a backslash that was From 6118f748385723b37da1b19696211e80b37df720 Mon Sep 17 00:00:00 2001 From: Peter Eszlari Date: Tue, 9 Feb 2021 13:57:23 +0100 Subject: [PATCH 30/96] cmake: add_compile_options / PROJECT_SOURCE_DIR --- CMakeLists.txt | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 39348c9ca4..0a79994cc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,17 +19,17 @@ endif() if(MSVC) set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) - string(APPEND CMAKE_CXX_FLAGS " /W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus") - add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_compile_options(/W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus) + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) else() include(CheckCXXCompilerFlag) check_cxx_compiler_flag(-Wno-deprecated flag_no_deprecated) if(flag_no_deprecated) - string(APPEND CMAKE_CXX_FLAGS " -Wno-deprecated") + add_compile_options(-Wno-deprecated) endif() check_cxx_compiler_flag(-fdiagnostics-color flag_color_diag) if(flag_color_diag) - string(APPEND CMAKE_CXX_FLAGS " -fdiagnostics-color") + add_compile_options(-fdiagnostics-color) endif() endif() @@ -57,7 +57,7 @@ function(check_platform_supports_browse_mode RESULT) # It uses the shell commands such as 'od', which may not be available. execute_process( COMMAND sh -c "echo 'TEST' | src/inline.sh var" - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} RESULT_VARIABLE inline_result OUTPUT_QUIET ERROR_QUIET @@ -130,7 +130,7 @@ endif() # On IBM i (identified as "OS400" for compatibility reasons) and AIX, this fixes missing # PRId64 (and others) at compile time in C++ sources if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") - string(APPEND CMAKE_CXX_FLAGS " -D__STDC_FORMAT_MACROS") + add_compile_definitions(__STDC_FORMAT_MACROS) endif() # Main executable is library plus main() function. @@ -145,11 +145,11 @@ if(platform_supports_ninja_browse) OUTPUT build/browse_py.h MAIN_DEPENDENCY src/browse.py DEPENDS src/inline.sh - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/build + COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/build COMMAND src/inline.sh kBrowsePy < src/browse.py - > ${CMAKE_BINARY_DIR}/build/browse_py.h - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + > ${PROJECT_BINARY_DIR}/build/browse_py.h + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} VERBATIM ) @@ -157,8 +157,8 @@ if(platform_supports_ninja_browse) target_sources(ninja PRIVATE src/browse.cc) set_source_files_properties(src/browse.cc PROPERTIES - OBJECT_DEPENDS "${CMAKE_BINARY_DIR}/build/browse_py.h" - INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}" + OBJECT_DEPENDS "${PROJECT_BINARY_DIR}/build/browse_py.h" + INCLUDE_DIRECTORIES "${PROJECT_BINARY_DIR}" COMPILE_DEFINITIONS NINJA_PYTHON="python" ) endif() @@ -205,8 +205,8 @@ if(BUILD_TESTING) if(CMAKE_SYSTEM_NAME STREQUAL "AIX" AND CMAKE_SIZEOF_VOID_P EQUAL 4) # These tests require more memory than will fit in the standard AIX shared stack/heap (256M) - target_link_libraries(hash_collision_bench PRIVATE "-Wl,-bmaxdata:0x80000000") - target_link_libraries(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000") + target_link_options(hash_collision_bench PRIVATE "-Wl,-bmaxdata:0x80000000") + target_link_options(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000") endif() add_test(NAME NinjaTest COMMAND ninja_test) From b0662970ba2cc69a64d7d2ebe3e07dcef948dabe Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Thu, 11 Feb 2021 18:18:02 +0100 Subject: [PATCH 31/96] GitHub Actions: Test both Debug and Release with MSVC --- .github/workflows/windows.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 04fc2f6d30..e4fe7bdb7e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -19,10 +19,15 @@ jobs: - name: Build ninja shell: bash run: | - cmake -DCMAKE_BUILD_TYPE=Release -B build + cmake -Bbuild + cmake --build build --parallel --config Debug cmake --build build --parallel --config Release - - name: Test ninja + - name: Test ninja (Debug) + run: .\ninja_test.exe + working-directory: build/Debug + + - name: Test ninja (Release) run: .\ninja_test.exe working-directory: build/Release From a33b81305889e45823b36b3c96cc8db97e5d3b41 Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 12 Feb 2021 10:17:36 -0500 Subject: [PATCH 32/96] disk_interface: Improve wrapping of comment in RemoveFile --- src/disk_interface.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 8d4cc7f65d..47532019a9 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -272,9 +272,9 @@ int RealDiskInterface::RemoveFile(const string& path) { return 1; } if (attributes & FILE_ATTRIBUTE_READONLY) { - // On non-Windows systems remove will happily delete read-only files. On - // Windows Ninja should behave the same. See - // https://github.com/ninja-build/ninja/issues/1886 + // On non-Windows systems, remove() will happily delete read-only files. + // On Windows Ninja should behave the same: + // https://github.com/ninja-build/ninja/issues/1886 SetFileAttributes(path.c_str(), attributes & ~FILE_ATTRIBUTE_READONLY); } if (!DeleteFile(path.c_str())) { From c7e3e5ef45c1877d12d6365fbfb68fa840c1518b Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 12 Feb 2021 11:34:22 -0500 Subject: [PATCH 33/96] disk_interface: Do not query bits of INVALID_FILE_ATTRIBUTES --- src/disk_interface.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 47532019a9..6290680bcb 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -267,11 +267,11 @@ FileReader::Status RealDiskInterface::ReadFile(const string& path, int RealDiskInterface::RemoveFile(const string& path) { #ifdef _WIN32 DWORD attributes = GetFileAttributes(path.c_str()); - if (attributes == INVALID_FILE_ATTRIBUTES && - GetLastError() == ERROR_FILE_NOT_FOUND) { - return 1; - } - if (attributes & FILE_ATTRIBUTE_READONLY) { + if (attributes == INVALID_FILE_ATTRIBUTES) { + if (GetLastError() == ERROR_FILE_NOT_FOUND) { + return 1; + } + } else if (attributes & FILE_ATTRIBUTE_READONLY) { // On non-Windows systems, remove() will happily delete read-only files. // On Windows Ninja should behave the same: // https://github.com/ninja-build/ninja/issues/1886 From 5392e0e7bf7095039ae6bd69c0a8dfc77ef80a05 Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 12 Feb 2021 11:37:23 -0500 Subject: [PATCH 34/96] disk_interface: Restore toleration of missing files in RemoveFile on Windows Revise the logic from commit 2d7f7e55 (Delete read-only files on Windows, too, 2020-12-07) to check if `GetFileAttributes` or `DeleteFile` failed due either variant of the file/path-not-found error. Issue: #1886 --- src/disk_interface.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 6290680bcb..a9497cb8a6 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -268,16 +268,23 @@ int RealDiskInterface::RemoveFile(const string& path) { #ifdef _WIN32 DWORD attributes = GetFileAttributes(path.c_str()); if (attributes == INVALID_FILE_ATTRIBUTES) { - if (GetLastError() == ERROR_FILE_NOT_FOUND) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { return 1; } } else if (attributes & FILE_ATTRIBUTE_READONLY) { // On non-Windows systems, remove() will happily delete read-only files. // On Windows Ninja should behave the same: // https://github.com/ninja-build/ninja/issues/1886 + // Skip error checking. If this fails, accept whatever happens below. SetFileAttributes(path.c_str(), attributes & ~FILE_ATTRIBUTE_READONLY); } if (!DeleteFile(path.c_str())) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + // Report as remove(), not DeleteFile(), for cross-platform consistency. Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); return -1; } From 00459e2b44fe1ee1a508f562bdf05acbea99c181 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Wed, 17 Feb 2021 21:59:20 +0100 Subject: [PATCH 35/96] Use UTF-8 on Windows 10 Version 1903, fix #1195 Allows Ninja to use descriptions, filenames and environment variables with characters outside of the ANSI codepage on Windows. Build manifests are now UTF-8 by default (this change needs to be emphasized in the release notes). WriteConsoleOutput doesn't support UTF-8, but it's deprecated on newer Windows 10 versions anyway (or as Microsoft likes to put it: "no longer a part of our ecosystem roadmap"). We'll use the VT100 sequence just as we do on Linux and macOS. https://docs.microsoft.com/en-us/windows/uwp/design/globalizing/use-utf8-code-page https://docs.microsoft.com/en-us/windows/console/writeconsoleoutput https://docs.microsoft.com/de-de/windows/console/console-virtual-terminal-sequences --- CMakeLists.txt | 4 ++++ src/line_printer.cc | 35 ++++++++++++++++++++--------------- windows/ninja.manifest | 8 ++++++++ 3 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 windows/ninja.manifest diff --git a/CMakeLists.txt b/CMakeLists.txt index 39348c9ca4..89bccb2fa3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,6 +137,10 @@ endif() add_executable(ninja src/ninja.cc) target_link_libraries(ninja PRIVATE libninja libninja-re2c) +if(WIN32) + target_sources(ninja PRIVATE windows/ninja.manifest) +endif() + # Adds browse mode into the ninja binary if it's supported by the host platform. if(platform_supports_ninja_browse) # Inlines src/browse.py into the browse_py.h header, so that it can be included diff --git a/src/line_printer.cc b/src/line_printer.cc index 68c58ad3e8..3138960366 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -87,22 +87,27 @@ void LinePrinter::Print(string to_print, LineType type) { GetConsoleScreenBufferInfo(console_, &csbi); to_print = ElideMiddle(to_print, static_cast(csbi.dwSize.X)); - // We don't want to have the cursor spamming back and forth, so instead of - // printf use WriteConsoleOutput which updates the contents of the buffer, - // but doesn't move the cursor position. - COORD buf_size = { csbi.dwSize.X, 1 }; - COORD zero_zero = { 0, 0 }; - SMALL_RECT target = { - csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, - static_cast(csbi.dwCursorPosition.X + csbi.dwSize.X - 1), - csbi.dwCursorPosition.Y - }; - vector char_data(csbi.dwSize.X); - for (size_t i = 0; i < static_cast(csbi.dwSize.X); ++i) { - char_data[i].Char.AsciiChar = i < to_print.size() ? to_print[i] : ' '; - char_data[i].Attributes = csbi.wAttributes; + if (supports_color_) { // this means ENABLE_VIRTUAL_TERMINAL_PROCESSING + // succeeded + printf("%s\x1B[K", to_print.c_str()); // Clear to end of line. + fflush(stdout); + } else { + // We don't want to have the cursor spamming back and forth, so instead of + // printf use WriteConsoleOutput which updates the contents of the buffer, + // but doesn't move the cursor position. + COORD buf_size = { csbi.dwSize.X, 1 }; + COORD zero_zero = { 0, 0 }; + SMALL_RECT target = { csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, + static_cast(csbi.dwCursorPosition.X + + csbi.dwSize.X - 1), + csbi.dwCursorPosition.Y }; + vector char_data(csbi.dwSize.X); + for (size_t i = 0; i < static_cast(csbi.dwSize.X); ++i) { + char_data[i].Char.AsciiChar = i < to_print.size() ? to_print[i] : ' '; + char_data[i].Attributes = csbi.wAttributes; + } + WriteConsoleOutput(console_, &char_data[0], buf_size, zero_zero, &target); } - WriteConsoleOutput(console_, &char_data[0], buf_size, zero_zero, &target); #else // Limit output to width of the terminal if provided so we don't cause // line-wrapping. diff --git a/windows/ninja.manifest b/windows/ninja.manifest new file mode 100644 index 0000000000..dab929e151 --- /dev/null +++ b/windows/ninja.manifest @@ -0,0 +1,8 @@ + + + + + UTF-8 + + + From d0489c3863f6b2a0d0b0e15880217da3fd4e6d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=9Aniatowski?= Date: Mon, 15 Feb 2021 13:06:30 +0100 Subject: [PATCH 36/96] Refactor depfile loading in preparation for the missingdeps tool Extract an usable helper to load depfile dependencies without adding them to the graph. --- src/graph.cc | 11 ++++++++--- src/graph.h | 8 +++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/graph.cc b/src/graph.cc index 78d0d49024..90e8d0877e 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -588,13 +588,18 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, } } + return ProcessDepfileDeps(edge, &depfile.ins_, err); +} + +bool ImplicitDepLoader::ProcessDepfileDeps( + Edge* edge, std::vector* depfile_ins, std::string* err) { // Preallocate space in edge->inputs_ to be filled in below. vector::iterator implicit_dep = - PreallocateSpace(edge, depfile.ins_.size()); + PreallocateSpace(edge, depfile_ins->size()); // Add all its in-edges. - for (vector::iterator i = depfile.ins_.begin(); - i != depfile.ins_.end(); ++i, ++implicit_dep) { + for (std::vector::iterator i = depfile_ins->begin(); + i != depfile_ins->end(); ++i, ++implicit_dep) { uint64_t slash_bits; if (!CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits, err)) diff --git a/src/graph.h b/src/graph.h index 8c51782a6e..6756378a4b 100644 --- a/src/graph.h +++ b/src/graph.h @@ -247,7 +247,13 @@ struct ImplicitDepLoader { return deps_log_; } - private: + protected: + /// Process loaded implicit dependencies for \a edge and update the graph + /// @return false on error (without filling \a err if info is just missing) + virtual bool ProcessDepfileDeps(Edge* edge, + std::vector* depfile_ins, + std::string* err); + /// Load implicit dependencies for \a edge from a depfile attribute. /// @return false on error (without filling \a err if info is just missing). bool LoadDepFile(Edge* edge, const std::string& path, std::string* err); From 3030254733f0baae1353b99e72e85babfbf5fbce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=9Aniatowski?= Date: Mon, 15 Feb 2021 13:06:30 +0100 Subject: [PATCH 37/96] Add a -t missingdeps tool to detect some classes of build flakes The tool looks for targets that depend on a generated file, but do not properly specify a dependency on the generator. It needs to be run after a successful build, and will list all potential flakes that may have broken the build, but didn't due to accidental build step ordering. The search relies on the correctness of depfile and generator output information, but these are usually easier to get right than dependencies. The errors found can usually be verified as actual build flakes by trying to build the listed problematic files alone in a clean build directory. Such builds usually fail with a compile job lacking a generated file. There is some overlap between this tool and 'gn check', but not everyone uses gn, not everyone using gn uses gn check, and most importantly, gn check is more about modularity, and less about actual build-time deps without flakes. The tool needs to be run after a build completes and depfile data is collected. It may take several seconds to process, up to a dozen or two on a large, chromium-sized build. --- CMakeLists.txt | 2 + configure.py | 2 + src/missing_deps.cc | 181 +++++++++++++++++++++++++++++++++++++++ src/missing_deps.h | 105 +++++++++++++++++++++++ src/missing_deps_test.cc | 162 +++++++++++++++++++++++++++++++++++ src/ninja.cc | 27 +++++- 6 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 src/missing_deps.cc create mode 100644 src/missing_deps.h create mode 100644 src/missing_deps_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 39348c9ca4..e6377e0013 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,6 +93,7 @@ add_library(libninja OBJECT src/line_printer.cc src/manifest_parser.cc src/metrics.cc + src/missing_deps.cc src/parser.cc src/state.cc src/string_piece_util.cc @@ -179,6 +180,7 @@ if(BUILD_TESTING) src/graph_test.cc src/lexer_test.cc src/manifest_parser_test.cc + src/missing_deps_test.cc src/ninja_test.cc src/state_test.cc src/string_piece_util_test.cc diff --git a/configure.py b/configure.py index cded265742..647b5b037d 100755 --- a/configure.py +++ b/configure.py @@ -511,6 +511,7 @@ def has_re2c(): 'line_printer', 'manifest_parser', 'metrics', + 'missing_deps', 'parser', 'state', 'string_piece_util', @@ -577,6 +578,7 @@ def has_re2c(): 'graph_test', 'lexer_test', 'manifest_parser_test', + 'missing_deps_test', 'ninja_test', 'state_test', 'string_piece_util_test', diff --git a/src/missing_deps.cc b/src/missing_deps.cc new file mode 100644 index 0000000000..a0fd048807 --- /dev/null +++ b/src/missing_deps.cc @@ -0,0 +1,181 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "missing_deps.h" + +#include + +#include + +#include "depfile_parser.h" +#include "deps_log.h" +#include "disk_interface.h" +#include "graph.h" +#include "state.h" +#include "util.h" + +namespace { + +/// ImplicitDepLoader variant that stores dep nodes into the given output +/// without updating graph deps like the base loader does. +struct NodeStoringImplicitDepLoader : public ImplicitDepLoader { + NodeStoringImplicitDepLoader( + State* state, DepsLog* deps_log, DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options, + std::vector* dep_nodes_output) + : ImplicitDepLoader(state, deps_log, disk_interface, + depfile_parser_options), + dep_nodes_output_(dep_nodes_output) {} + + protected: + virtual bool ProcessDepfileDeps(Edge* edge, + std::vector* depfile_ins, + std::string* err); + + private: + std::vector* dep_nodes_output_; +}; + +bool NodeStoringImplicitDepLoader::ProcessDepfileDeps( + Edge* edge, std::vector* depfile_ins, std::string* err) { + for (std::vector::iterator i = depfile_ins->begin(); + i != depfile_ins->end(); ++i) { + uint64_t slash_bits; + if (!CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits, + err)) + return false; + Node* node = state_->GetNode(*i, slash_bits); + dep_nodes_output_->push_back(node); + } + return true; +} + +} // namespace + +MissingDependencyScannerDelegate::~MissingDependencyScannerDelegate() {} + +void MissingDependencyPrinter::OnMissingDep(Node* node, const std::string& path, + const Rule& generator) { + std::cout << "Missing dep: " << node->path() << " uses " << path + << " (generated by " << generator.name() << ")\n"; +} + +MissingDependencyScanner::MissingDependencyScanner( + MissingDependencyScannerDelegate* delegate, DepsLog* deps_log, State* state, + DiskInterface* disk_interface) + : delegate_(delegate), deps_log_(deps_log), state_(state), + disk_interface_(disk_interface), missing_dep_path_count_(0) {} + +void MissingDependencyScanner::ProcessNode(Node* node) { + if (!node) + return; + Edge* edge = node->in_edge(); + if (!edge) + return; + if (!seen_.insert(node).second) + return; + + for (std::vector::iterator in = edge->inputs_.begin(); + in != edge->inputs_.end(); ++in) { + ProcessNode(*in); + } + + std::string deps_type = edge->GetBinding("deps"); + if (!deps_type.empty()) { + DepsLog::Deps* deps = deps_log_->GetDeps(node); + if (deps) + ProcessNodeDeps(node, deps->nodes, deps->node_count); + } else { + DepfileParserOptions parser_opts; + std::vector depfile_deps; + NodeStoringImplicitDepLoader dep_loader(state_, deps_log_, disk_interface_, + &parser_opts, &depfile_deps); + std::string err; + dep_loader.LoadDeps(edge, &err); + if (!depfile_deps.empty()) + ProcessNodeDeps(node, &depfile_deps[0], depfile_deps.size()); + } +} + +void MissingDependencyScanner::ProcessNodeDeps(Node* node, Node** dep_nodes, + int dep_nodes_count) { + Edge* edge = node->in_edge(); + std::set deplog_edges; + for (int i = 0; i < dep_nodes_count; ++i) { + Node* deplog_node = dep_nodes[i]; + Edge* deplog_edge = deplog_node->in_edge(); + if (deplog_edge) { + deplog_edges.insert(deplog_edge); + } + } + std::vector missing_deps; + for (std::set::iterator de = deplog_edges.begin(); + de != deplog_edges.end(); ++de) { + if (!PathExistsBetween(*de, edge)) { + missing_deps.push_back(*de); + } + } + + if (!missing_deps.empty()) { + std::set missing_deps_rule_names; + for (std::vector::iterator ne = missing_deps.begin(); + ne != missing_deps.end(); ++ne) { + for (int i = 0; i < dep_nodes_count; ++i) { + if (dep_nodes[i]->in_edge() == *ne) { + generated_nodes_.insert(dep_nodes[i]); + generator_rules_.insert(&(*ne)->rule()); + missing_deps_rule_names.insert((*ne)->rule().name()); + delegate_->OnMissingDep(node, dep_nodes[i]->path(), (*ne)->rule()); + } + } + } + missing_dep_path_count_ += missing_deps_rule_names.size(); + nodes_missing_deps_.insert(node); + } +} + +void MissingDependencyScanner::PrintStats() { + std::cout << "Processed " << seen_.size() << " nodes.\n"; + if (HadMissingDeps()) { + std::cout << "Error: There are " << missing_dep_path_count_ + << " missing dependency paths.\n"; + std::cout << nodes_missing_deps_.size() + << " targets had depfile dependencies on " + << generated_nodes_.size() << " distinct generated inputs " + << "(from " << generator_rules_.size() << " rules) " + << " without a non-depfile dep path to the generator.\n"; + std::cout << "There might be build flakiness if any of the targets listed " + "above are built alone, or not late enough, in a clean output " + "directory.\n"; + } else { + std::cout << "No missing dependencies on generated files found.\n"; + } +} + +bool MissingDependencyScanner::PathExistsBetween(Edge* from, Edge* to) { + EdgePair key(from, to); + EdgeAdjacencyMap::iterator it = edge_adjacency_map_.find(key); + if (it != edge_adjacency_map_.end()) + return it->second; + bool found = false; + for (size_t i = 0; i < to->inputs_.size(); ++i) { + Edge* e = to->inputs_[i]->in_edge(); + if (e && (e == from || PathExistsBetween(from, e))) { + found = true; + break; + } + } + edge_adjacency_map_.insert(std::make_pair(key, found)); + return found; +} diff --git a/src/missing_deps.h b/src/missing_deps.h new file mode 100644 index 0000000000..211a865988 --- /dev/null +++ b/src/missing_deps.h @@ -0,0 +1,105 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_MISSING_DEPS_H_ +#define NINJA_MISSING_DEPS_H_ + +#include +#include +#include + +#if __cplusplus >= 201103L +#include +#endif + +struct DepsLog; +struct DiskInterface; +struct Edge; +struct Node; +struct Rule; +struct State; + +class MissingDependencyScannerDelegate { + public: + virtual ~MissingDependencyScannerDelegate(); + virtual void OnMissingDep(Node* node, const std::string& path, + const Rule& generator) = 0; +}; + +class MissingDependencyPrinter : public MissingDependencyScannerDelegate { + void OnMissingDep(Node* node, const std::string& path, const Rule& generator); + void OnStats(int nodes_processed, int nodes_missing_deps, + int missing_dep_path_count, int generated_nodes, + int generator_rules); +}; + +struct EdgePair { + EdgePair(Edge* from, Edge* to) : from_(from), to_(to) {} + bool operator==(const EdgePair& other) const { + return from_ == other.from_ && to_ == other.to_; + } + bool operator<(const EdgePair& other) const { + return (from_ < other.from_) || + ((from_ == other.from_) && (to_ < other.to_)); + } + Edge* from_; + Edge* to_; +}; + +#if __cplusplus >= 201103L +namespace std { +template <> +struct hash { + size_t operator()(const EdgePair& k) const { + uintptr_t uint_from = uintptr_t(k.from_); + uintptr_t uint_to = uintptr_t(k.to_); + return hash()(uint_from ^ (uint_to >> 3)); + } +}; +} // namespace std +#endif // __cplusplus >= 201103L + +struct MissingDependencyScanner { + public: + MissingDependencyScanner(MissingDependencyScannerDelegate* delegate, + DepsLog* deps_log, State* state, + DiskInterface* disk_interface); + void ProcessNode(Node* node); + void PrintStats(); + bool HadMissingDeps() { return !nodes_missing_deps_.empty(); } + + void ProcessNodeDeps(Node* node, Node** dep_nodes, int dep_nodes_count); + + bool PathExistsBetween(Edge* from, Edge* to); + + MissingDependencyScannerDelegate* delegate_; + DepsLog* deps_log_; + State* state_; + DiskInterface* disk_interface_; + std::set seen_; + std::set nodes_missing_deps_; + std::set generated_nodes_; + std::set generator_rules_; + int missing_dep_path_count_; + + private: +#if __cplusplus >= 201103L + using EdgeAdjacencyMap = std::unordered_map; +#else + typedef std::map EdgeAdjacencyMap; +#endif + EdgeAdjacencyMap edge_adjacency_map_; +}; + +#endif // NINJA_MISSING_DEPS_H_ diff --git a/src/missing_deps_test.cc b/src/missing_deps_test.cc new file mode 100644 index 0000000000..7b62e6cc48 --- /dev/null +++ b/src/missing_deps_test.cc @@ -0,0 +1,162 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "deps_log.h" +#include "graph.h" +#include "missing_deps.h" +#include "state.h" +#include "test.h" + +const char kTestDepsLogFilename[] = "MissingDepTest-tempdepslog"; + +class MissingDependencyTestDelegate : public MissingDependencyScannerDelegate { + void OnMissingDep(Node* node, const std::string& path, + const Rule& generator) {} +}; + +struct MissingDependencyScannerTest : public testing::Test { + MissingDependencyScannerTest() + : generator_rule_("generator_rule"), compile_rule_("compile_rule"), + scanner_(&delegate_, &deps_log_, &state_, &filesystem_) { + std::string err; + deps_log_.OpenForWrite(kTestDepsLogFilename, &err); + ASSERT_EQ("", err); + } + + MissingDependencyScanner& scanner() { return scanner_; } + + void RecordDepsLogDep(const std::string& from, const std::string& to) { + Node* node_deps[] = { state_.LookupNode(to) }; + deps_log_.RecordDeps(state_.LookupNode(from), 0, 1, node_deps); + } + + void ProcessAllNodes() { + std::string err; + std::vector nodes = state_.RootNodes(&err); + EXPECT_EQ("", err); + for (std::vector::iterator it = nodes.begin(); it != nodes.end(); + ++it) { + scanner().ProcessNode(*it); + } + } + + void CreateInitialState() { + EvalString deps_type; + deps_type.AddText("gcc"); + compile_rule_.AddBinding("deps", deps_type); + generator_rule_.AddBinding("deps", deps_type); + Edge* header_edge = state_.AddEdge(&generator_rule_); + state_.AddOut(header_edge, "generated_header", 0); + Edge* compile_edge = state_.AddEdge(&compile_rule_); + state_.AddOut(compile_edge, "compiled_object", 0); + } + + void CreateGraphDependencyBetween(const char* from, const char* to) { + Node* from_node = state_.LookupNode(from); + Edge* from_edge = from_node->in_edge(); + state_.AddIn(from_edge, to, 0); + } + + void AssertMissingDependencyBetween(const char* flaky, const char* generated, + Rule* rule) { + Node* flaky_node = state_.LookupNode(flaky); + ASSERT_EQ(1u, scanner().nodes_missing_deps_.count(flaky_node)); + Node* generated_node = state_.LookupNode(generated); + ASSERT_EQ(1u, scanner().generated_nodes_.count(generated_node)); + ASSERT_EQ(1u, scanner().generator_rules_.count(rule)); + } + + MissingDependencyTestDelegate delegate_; + Rule generator_rule_; + Rule compile_rule_; + DepsLog deps_log_; + State state_; + VirtualFileSystem filesystem_; + MissingDependencyScanner scanner_; +}; + +TEST_F(MissingDependencyScannerTest, EmptyGraph) { + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, NoMissingDep) { + CreateInitialState(); + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, MissingDepPresent) { + CreateInitialState(); + // compiled_object uses generated_header, without a proper dependency + RecordDepsLogDep("compiled_object", "generated_header"); + ProcessAllNodes(); + ASSERT_TRUE(scanner().HadMissingDeps()); + ASSERT_EQ(1u, scanner().nodes_missing_deps_.size()); + ASSERT_EQ(1u, scanner().missing_dep_path_count_); + AssertMissingDependencyBetween("compiled_object", "generated_header", + &generator_rule_); +} + +TEST_F(MissingDependencyScannerTest, MissingDepFixedDirect) { + CreateInitialState(); + // Adding the direct dependency fixes the missing dep + CreateGraphDependencyBetween("compiled_object", "generated_header"); + RecordDepsLogDep("compiled_object", "generated_header"); + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, MissingDepFixedIndirect) { + CreateInitialState(); + // Adding an indirect dependency also fixes the issue + Edge* intermediate_edge = state_.AddEdge(&generator_rule_); + state_.AddOut(intermediate_edge, "intermediate", 0); + CreateGraphDependencyBetween("compiled_object", "intermediate"); + CreateGraphDependencyBetween("intermediate", "generated_header"); + RecordDepsLogDep("compiled_object", "generated_header"); + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, CyclicMissingDep) { + CreateInitialState(); + RecordDepsLogDep("generated_header", "compiled_object"); + RecordDepsLogDep("compiled_object", "generated_header"); + // In case of a cycle, both paths are reported (and there is + // no way to fix the issue by adding deps). + ProcessAllNodes(); + ASSERT_TRUE(scanner().HadMissingDeps()); + ASSERT_EQ(2u, scanner().nodes_missing_deps_.size()); + ASSERT_EQ(2u, scanner().missing_dep_path_count_); + AssertMissingDependencyBetween("compiled_object", "generated_header", + &generator_rule_); + AssertMissingDependencyBetween("generated_header", "compiled_object", + &compile_rule_); +} + +TEST_F(MissingDependencyScannerTest, CycleInGraph) { + CreateInitialState(); + CreateGraphDependencyBetween("compiled_object", "generated_header"); + CreateGraphDependencyBetween("generated_header", "compiled_object"); + // The missing-deps tool doesn't deal with cycles in the graph, beacuse + // there will be an error loading the graph before we get to the tool. + // This test is to illustrate that. + std::string err; + std::vector nodes = state_.RootNodes(&err); + ASSERT_NE("", err); +} + diff --git a/src/ninja.cc b/src/ninja.cc index eb9732058b..7af094137b 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -37,11 +37,13 @@ #include "deps_log.h" #include "clean.h" #include "debug_flags.h" +#include "depfile_parser.h" #include "disk_interface.h" #include "graph.h" #include "graphviz.h" #include "manifest_parser.h" #include "metrics.h" +#include "missing_deps.h" #include "state.h" #include "util.h" #include "version.h" @@ -117,6 +119,7 @@ struct NinjaMain : public BuildLogUser { int ToolGraph(const Options* options, int argc, char* argv[]); int ToolQuery(const Options* options, int argc, char* argv[]); int ToolDeps(const Options* options, int argc, char* argv[]); + int ToolMissingDeps(const Options* options, int argc, char* argv[]); int ToolBrowse(const Options* options, int argc, char* argv[]); int ToolMSVC(const Options* options, int argc, char* argv[]); int ToolTargets(const Options* options, int argc, char* argv[]); @@ -523,6 +526,26 @@ int NinjaMain::ToolDeps(const Options* options, int argc, char** argv) { return 0; } +int NinjaMain::ToolMissingDeps(const Options* options, int argc, char** argv) { + vector nodes; + string err; + if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { + Error("%s", err.c_str()); + return 1; + } + RealDiskInterface disk_interface; + MissingDependencyPrinter printer; + MissingDependencyScanner scanner(&printer, &deps_log_, &state_, + &disk_interface); + for (vector::iterator it = nodes.begin(); it != nodes.end(); ++it) { + scanner.ProcessNode(*it); + } + scanner.PrintStats(); + if (scanner.HadMissingDeps()) + return 3; + return 0; +} + int NinjaMain::ToolTargets(const Options* options, int argc, char* argv[]) { int depth = 1; if (argc >= 1) { @@ -960,6 +983,8 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCommands }, { "deps", "show dependencies stored in the deps log", Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps }, + { "missingdeps", "check deps log dependencies on generated files", + Tool::RUN_AFTER_LOGS, &NinjaMain::ToolMissingDeps }, { "graph", "output graphviz dot file for targets", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolGraph }, { "query", "show inputs/outputs for a path", @@ -985,7 +1010,7 @@ const Tool* ChooseTool(const string& tool_name) { printf("ninja subtools:\n"); for (const Tool* tool = &kTools[0]; tool->name; ++tool) { if (tool->desc) - printf("%10s %s\n", tool->name, tool->desc); + printf("%11s %s\n", tool->name, tool->desc); } return NULL; } From bc69a640de3d22512fca69b1f997fb7b832addb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=9Aniatowski?= Date: Mon, 15 Feb 2021 13:06:30 +0100 Subject: [PATCH 38/96] Add manual entry about the missingdeps tool --- doc/manual.asciidoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index e1ae083b05..2cbe6a3700 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -284,6 +284,21 @@ _Available since Ninja 1.2._ `deps`:: show all dependencies stored in the `.ninja_deps` file. When given a target, show just the target's dependencies. _Available since Ninja 1.4._ +`missingdeps`:: given a list of targets, look for targets that depend on +a generated file, but do not have a properly (possibly transitive) dependency +on the generator. Such targets may cause build flakiness on clean builds. + +The broken targets can be found assuming deps log / depfile dependency +information is correct. Any target that depends on a generated file (output +of a generator-target) implicitly, but does not have an explicit or order-only +dependency path to the generator-target, is considered broken. + +The tool's findings can be verified by trying to build the listed targets in +a clean outdir without buidling any other targets. The build should fail for +each of them with a missing include error or equivalent pointing to the +generated file. +_Available since Ninja 1.11._ + `recompact`:: recompact the `.ninja_deps` file. _Available since Ninja 1.4._ `restat`:: updates all recorded file modification timestamps in the `.ninja_log` From b94a891ac9fcd0cee80652197c03633d752ad068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=9Aniatowski?= Date: Mon, 15 Feb 2021 13:06:30 +0100 Subject: [PATCH 39/96] missingdeps: add exception for targets that dep on build.ninja A "missing dep path" to build.ninja is a false positive, skip reporting it. --- src/missing_deps.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/missing_deps.cc b/src/missing_deps.cc index a0fd048807..adce2d2377 100644 --- a/src/missing_deps.cc +++ b/src/missing_deps.cc @@ -114,6 +114,14 @@ void MissingDependencyScanner::ProcessNodeDeps(Node* node, Node** dep_nodes, std::set deplog_edges; for (int i = 0; i < dep_nodes_count; ++i) { Node* deplog_node = dep_nodes[i]; + // Special exception: A dep on build.ninja can be used to mean "always + // rebuild this target when the build is reconfigured", but build.ninja is + // often generated by a configuration tool like cmake or gn. The rest of + // the build "implicitly" depends on the entire build being reconfigured, + // so a missing dep path to build.ninja is not an actual missing dependecy + // problem. + if (deplog_node->path() == "build.ninja") + return; Edge* deplog_edge = deplog_node->in_edge(); if (deplog_edge) { deplog_edges.insert(deplog_edge); From 6c89e596eef50a4aa53f3cdc0f40a7071638769f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20=C5=9Aniatowski?= Date: Mon, 15 Feb 2021 13:06:30 +0100 Subject: [PATCH 40/96] missingdeps: use nested maps for edge adjacency cache In my tests, nested maps outperform a large map of pairs by 10-20% on a sample Chromium missingdeps run, and are arguably simpler due to fewer ifdefs needed. --- src/missing_deps.cc | 15 ++++++++++----- src/missing_deps.h | 34 +++++----------------------------- 2 files changed, 15 insertions(+), 34 deletions(-) diff --git a/src/missing_deps.cc b/src/missing_deps.cc index adce2d2377..eaa3f73cc5 100644 --- a/src/missing_deps.cc +++ b/src/missing_deps.cc @@ -172,10 +172,15 @@ void MissingDependencyScanner::PrintStats() { } bool MissingDependencyScanner::PathExistsBetween(Edge* from, Edge* to) { - EdgePair key(from, to); - EdgeAdjacencyMap::iterator it = edge_adjacency_map_.find(key); - if (it != edge_adjacency_map_.end()) - return it->second; + AdjacencyMap::iterator it = adjacency_map_.find(from); + if (it != adjacency_map_.end()) { + InnerAdjacencyMap::iterator inner_it = it->second.find(to); + if (inner_it != it->second.end()) { + return inner_it->second; + } + } else { + it = adjacency_map_.insert(std::make_pair(from, InnerAdjacencyMap())).first; + } bool found = false; for (size_t i = 0; i < to->inputs_.size(); ++i) { Edge* e = to->inputs_[i]->in_edge(); @@ -184,6 +189,6 @@ bool MissingDependencyScanner::PathExistsBetween(Edge* from, Edge* to) { break; } } - edge_adjacency_map_.insert(std::make_pair(key, found)); + it->second.insert(std::make_pair(to, found)); return found; } diff --git a/src/missing_deps.h b/src/missing_deps.h index 211a865988..ae5707424c 100644 --- a/src/missing_deps.h +++ b/src/missing_deps.h @@ -44,32 +44,6 @@ class MissingDependencyPrinter : public MissingDependencyScannerDelegate { int generator_rules); }; -struct EdgePair { - EdgePair(Edge* from, Edge* to) : from_(from), to_(to) {} - bool operator==(const EdgePair& other) const { - return from_ == other.from_ && to_ == other.to_; - } - bool operator<(const EdgePair& other) const { - return (from_ < other.from_) || - ((from_ == other.from_) && (to_ < other.to_)); - } - Edge* from_; - Edge* to_; -}; - -#if __cplusplus >= 201103L -namespace std { -template <> -struct hash { - size_t operator()(const EdgePair& k) const { - uintptr_t uint_from = uintptr_t(k.from_); - uintptr_t uint_to = uintptr_t(k.to_); - return hash()(uint_from ^ (uint_to >> 3)); - } -}; -} // namespace std -#endif // __cplusplus >= 201103L - struct MissingDependencyScanner { public: MissingDependencyScanner(MissingDependencyScannerDelegate* delegate, @@ -95,11 +69,13 @@ struct MissingDependencyScanner { private: #if __cplusplus >= 201103L - using EdgeAdjacencyMap = std::unordered_map; + using InnerAdjacencyMap = std::unordered_map; + using AdjacencyMap = std::unordered_map; #else - typedef std::map EdgeAdjacencyMap; + typedef std::map InnerAdjacencyMap; + typedef std::map AdjacencyMap; #endif - EdgeAdjacencyMap edge_adjacency_map_; + AdjacencyMap adjacency_map_; }; #endif // NINJA_MISSING_DEPS_H_ From aa9317e42fadb7f84498f555620a8f89c9afd9f3 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Wed, 17 Feb 2021 22:29:45 -0500 Subject: [PATCH 41/96] cmake: add browse check status --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e0696ff8d8..8de69e89c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ target_include_directories(libninja-re2c PRIVATE src) function(check_platform_supports_browse_mode RESULT) # Make sure the inline.sh script works on this platform. # It uses the shell commands such as 'od', which may not be available. + execute_process( COMMAND sh -c "echo 'TEST' | src/inline.sh var" WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} @@ -65,12 +66,19 @@ function(check_platform_supports_browse_mode RESULT) if(NOT inline_result EQUAL "0") # The inline script failed, so browse mode is not supported. set(${RESULT} "0" PARENT_SCOPE) + if(NOT WIN32) + message(WARNING "browse feature omitted due to inline script failure") + endif() return() endif() # Now check availability of the unistd header check_include_file_cxx(unistd.h PLATFORM_HAS_UNISTD_HEADER) set(${RESULT} "${PLATFORM_HAS_UNISTD_HEADER}" PARENT_SCOPE) + if(NOT PLATFORM_HAS_UNISTD_HEADER) + message(WARNING "browse feature omitted due to missing unistd.h") + endif() + endfunction() check_platform_supports_browse_mode(platform_supports_ninja_browse) From 212cd97679e783de77230b009b23dd11344644ac Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 26 Feb 2021 10:30:33 -0500 Subject: [PATCH 42/96] doc: fix format of 'missingdeps' documentation Group all the paragraphs together in the definition list entry. --- doc/manual.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 2cbe6a3700..0ac0ebb33e 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -287,12 +287,12 @@ target, show just the target's dependencies. _Available since Ninja 1.4._ `missingdeps`:: given a list of targets, look for targets that depend on a generated file, but do not have a properly (possibly transitive) dependency on the generator. Such targets may cause build flakiness on clean builds. - ++ The broken targets can be found assuming deps log / depfile dependency information is correct. Any target that depends on a generated file (output of a generator-target) implicitly, but does not have an explicit or order-only dependency path to the generator-target, is considered broken. - ++ The tool's findings can be verified by trying to build the listed targets in a clean outdir without buidling any other targets. The build should fail for each of them with a missing include error or equivalent pointing to the From 706e16bee5f6b5fccd284476a9157ecc8c39f12c Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 26 Feb 2021 09:21:03 -0500 Subject: [PATCH 43/96] Add tool to print code page information on Windows Since commit 00459e2b (Use UTF-8 on Windows 10 Version 1903, fix #1195, 2021-02-17), `ninja` does not always expect `build.ninja` to be encoded in the system's ANSI code page. The expected encoding now depends on how `ninja` is built and the version of Windows on which it is running. Add a `-t wincodepage` tool that generators can use to ask `ninja` what encoding it expects. Issue: #1195 --- doc/manual.asciidoc | 12 ++++++++++++ src/ninja.cc | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 0ac0ebb33e..204cc6db39 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -308,6 +308,18 @@ file. _Available since Ninja 1.10._ if they have one). It can be used to know which rule name to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. +`wincodepage`:: available on Windows hosts. Prints the ANSI code page +used by `ninja`, whose encoding is expected in `build.ninja`. Also prints +the Console code page for reference. The output has the form: ++ +---- +ANSI code page: %u +Console code page: %u +---- ++ +where each `%u` is an integer code page identifier, expressed in decimal. +_Available since Ninja 1.11._ + Writing your own Ninja files ---------------------------- diff --git a/src/ninja.cc b/src/ninja.cc index 96eac1de50..3172ee5a6f 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -133,6 +133,7 @@ struct NinjaMain : public BuildLogUser { int ToolRestat(const Options* options, int argc, char* argv[]); int ToolUrtle(const Options* options, int argc, char** argv); int ToolRules(const Options* options, int argc, char* argv[]); + int ToolWinCodePage(const Options* options, int argc, char* argv[]); /// Open the build log. /// @return LOAD_ERROR on error. @@ -641,6 +642,18 @@ int NinjaMain::ToolRules(const Options* options, int argc, char* argv[]) { return 0; } +#ifdef _WIN32 +int NinjaMain::ToolWinCodePage(const Options* options, int argc, char* argv[]) { + if (argc != 0) { + printf("usage: ninja -t wincodepage\n"); + return 1; + } + printf("ANSI code page: %u\n", GetACP()); + printf("Console code page: %u\n", GetConsoleOutputCP()); + return 0; +} +#endif + enum PrintCommandMode { PCM_Single, PCM_All }; void PrintCommands(Edge* edge, EdgeSet* seen, PrintCommandMode mode) { if (!edge) @@ -1009,6 +1022,10 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOGS, &NinjaMain::ToolCleanDead }, { "urtle", NULL, Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolUrtle }, +#ifdef _WIN32 + { "wincodepage", "print the Windows ANSI code page identifier", + Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolWinCodePage }, +#endif { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } }; From a510f532f6ac0ec05703afade6e2bfdecbc6d00a Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 1 Mar 2021 12:35:51 -0500 Subject: [PATCH 44/96] wincodepage: minimize to indicate UTF-8 or not The ANSI code page identifier is more information than generator programs actually need. The encoding of `build.ninja` is always either UTF-8 or the system-wide ANSI code page. Reduce the output to provide no more information than the generator programs need. The Console code page can be obtained in other ways, so drop it. --- doc/manual.asciidoc | 18 +++++++++++------- src/ninja.cc | 5 ++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 204cc6db39..54403ac092 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -308,17 +308,21 @@ file. _Available since Ninja 1.10._ if they have one). It can be used to know which rule name to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. -`wincodepage`:: available on Windows hosts. Prints the ANSI code page -used by `ninja`, whose encoding is expected in `build.ninja`. Also prints -the Console code page for reference. The output has the form: +`wincodepage`:: Available on Windows hosts (_since Ninja 1.11_). +Prints the Windows code page whose encoding is expected in the build file. +The output has the form: + ---- -ANSI code page: %u -Console code page: %u +Build file encoding: ---- + -where each `%u` is an integer code page identifier, expressed in decimal. -_Available since Ninja 1.11._ +Additional lines may be added in future versions of Ninja. ++ +The `` is one of: + +`UTF-8`::: Encode as UTF-8. + +`ANSI`::: Encode to the system-wide ANSI code page. Writing your own Ninja files ---------------------------- diff --git a/src/ninja.cc b/src/ninja.cc index 3172ee5a6f..45fc8ea417 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -648,8 +648,7 @@ int NinjaMain::ToolWinCodePage(const Options* options, int argc, char* argv[]) { printf("usage: ninja -t wincodepage\n"); return 1; } - printf("ANSI code page: %u\n", GetACP()); - printf("Console code page: %u\n", GetConsoleOutputCP()); + printf("Build file encoding: %s\n", GetACP() == CP_UTF8? "UTF-8" : "ANSI"); return 0; } #endif @@ -1023,7 +1022,7 @@ const Tool* ChooseTool(const string& tool_name) { { "urtle", NULL, Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolUrtle }, #ifdef _WIN32 - { "wincodepage", "print the Windows ANSI code page identifier", + { "wincodepage", "print the Windows code page used by ninja", Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolWinCodePage }, #endif { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } From 67fbbeeec91ec171da7d4e297b8f9b319f3424c8 Mon Sep 17 00:00:00 2001 From: John Drouhard Date: Sun, 15 Mar 2020 14:56:54 -0500 Subject: [PATCH 45/96] Change build log to always log the most recent input mtime If an edge's output files' mtimes are compared to the most recent input's mtime, edges might be calculated as clean even if they are actually dirty. While an edge's command is running its rule to produce its outputs and an input to the edge is updated before the outputs are written to disk, then subsequent runs will think that the outputs are newer than the inputs, even though the inputs have actually been updated and may be different than what were used to produce those outputs. Ninja will now restat all inputs just prior to running an edge's command and remember the most recent input mtime. When the command completes, it will stat any discovered dependencies from dep files (if necessary), recalculate the most recent input mtime, and log it to the build log file. On subsequent runs, ninja will use this value to compare to the edge's most recent input's mtime to determine whether the outputs are dirty. This extends the methodology used by restat rules to work in all cases. Restat rules are still unique in that they will clean the edge's output nodes recursively if the edge's command did not change the output, but in all cases, the mtime recorded in the log file is now the most recent input mtime. See the new tests for more clarification. --- src/build.cc | 75 ++++++++++++---------- src/build_log.cc | 10 +-- src/build_log.h | 2 +- src/build_test.cc | 160 ++++++++++++++++++++++++++++++++++++++++++++++ src/graph.cc | 43 ++++--------- src/graph.h | 10 +-- 6 files changed, 226 insertions(+), 74 deletions(-) diff --git a/src/build.cc b/src/build.cc index fb5890a76d..2e292322ff 100644 --- a/src/build.cc +++ b/src/build.cc @@ -270,19 +270,11 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { #define MEM_FN mem_fn // mem_fun was removed in C++17. #endif if (find_if(begin, end, MEM_FN(&Node::dirty)) == end) { - // Recompute most_recent_input. - Node* most_recent_input = NULL; - for (vector::iterator i = begin; i != end; ++i) { - if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) - most_recent_input = *i; - } - // Now, this edge is dirty if any of the outputs are dirty. // If the edge isn't dirty, clean the outputs and mark the edge as not // wanted. bool outputs_dirty = false; - if (!scan->RecomputeOutputsDirty(*oe, most_recent_input, - &outputs_dirty, err)) { + if (!scan->RecomputeOutputsDirty(*oe, &outputs_dirty, err)) { return false; } if (!outputs_dirty) { @@ -696,6 +688,20 @@ bool Builder::StartEdge(Edge* edge, string* err) { return false; } + // Find the most recent mtime of any (existing) non-order-only input + Node* most_recent_input = NULL; + for (vector::iterator i = edge->inputs_.begin(); + i != edge->inputs_.end() - edge->order_only_deps_; ++i) { + if (!(*i)->Stat(disk_interface_, err)) + return false; + if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) + most_recent_input = *i; + } + + edge->most_recent_input_ = most_recent_input; + if (most_recent_input) + edge->most_recent_input_mtime_ = most_recent_input->mtime(); + // start command computing and run it if (!command_runner_->StartCommand(edge)) { err->assign("command '" + edge->EvaluateCommand() + "' failed."); @@ -744,20 +750,18 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { return plan_.EdgeFinished(edge, Plan::kEdgeFailed, err); } - // Restat the edge outputs - TimeStamp output_mtime = 0; - bool restat = edge->GetBindingBool("restat"); + TimeStamp most_recent_input_mtime = 0; if (!config_.dry_run) { + // Restat the edge outputs bool node_cleaned = false; - for (vector::iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) { - TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err); + TimeStamp old_mtime = (*o)->mtime(); + (*o)->Stat(disk_interface_, err); + TimeStamp new_mtime = (*o)->mtime(); if (new_mtime == -1) return false; - if (new_mtime > output_mtime) - output_mtime = new_mtime; - if ((*o)->mtime() == new_mtime && restat) { + if (old_mtime == new_mtime && edge->GetBindingBool("restat")) { // The rule command did not change the output. Propagate the clean // state through the build graph. // Note that this also applies to nonexistent outputs (mtime == 0). @@ -767,33 +771,34 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { } } - if (node_cleaned) { - TimeStamp restat_mtime = 0; - // If any output was cleaned, find the most recent mtime of any - // (existing) non-order-only input or the depfile. - for (vector::iterator i = edge->inputs_.begin(); - i != edge->inputs_.end() - edge->order_only_deps_; ++i) { - TimeStamp input_mtime = disk_interface_->Stat((*i)->path(), err); - if (input_mtime == -1) - return false; - if (input_mtime > restat_mtime) - restat_mtime = input_mtime; - } + // Use the time from the most recent input that was computed when the edge was + // started, not the mtime of the node as it is now. There could have been other edges + // that restat'd the input node and detected a change, but for *this* edge, we want + // the mtime as it was when the command began. + most_recent_input_mtime = edge->most_recent_input_mtime_; + + // If there were any added deps, compute the most recent input mtime + for (vector::iterator i = deps_nodes.begin(); + i != deps_nodes.end(); ++i) { + (*i)->StatIfNecessary(disk_interface_, err); + if ((*i)->mtime() > most_recent_input_mtime) + most_recent_input_mtime = (*i)->mtime(); + } + if (node_cleaned) { + // If any output was cleaned, take into account the mtime of the depfile string depfile = edge->GetUnescapedDepfile(); - if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) { + if (most_recent_input_mtime != 0 && deps_type.empty() && !depfile.empty()) { TimeStamp depfile_mtime = disk_interface_->Stat(depfile, err); if (depfile_mtime == -1) return false; - if (depfile_mtime > restat_mtime) - restat_mtime = depfile_mtime; + if (depfile_mtime > most_recent_input_mtime) + most_recent_input_mtime = depfile_mtime; } // The total number of edges in the plan may have changed as a result // of a restat. status_->PlanHasTotalEdges(plan_.command_edge_count()); - - output_mtime = restat_mtime; } } @@ -807,7 +812,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { if (scan_.build_log()) { if (!scan_.build_log()->RecordCommand(edge, start_time_millis, - end_time_millis, output_mtime)) { + end_time_millis, most_recent_input_mtime)) { *err = string("Error writing to build log: ") + strerror(errno); return false; } diff --git a/src/build_log.cc b/src/build_log.cc index 4dcd6cee53..b35279d410 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -116,9 +116,9 @@ BuildLog::LogEntry::LogEntry(const string& output) : output(output) {} BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, - int start_time, int end_time, TimeStamp restat_mtime) + int start_time, int end_time, TimeStamp mtime) : output(output), command_hash(command_hash), - start_time(start_time), end_time(end_time), mtime(restat_mtime) + start_time(start_time), end_time(end_time), mtime(mtime) {} BuildLog::BuildLog() @@ -303,7 +303,7 @@ LoadStatus BuildLog::Load(const string& path, string* err) { *end = 0; int start_time = 0, end_time = 0; - TimeStamp restat_mtime = 0; + TimeStamp mtime = 0; start_time = atoi(start); start = end + 1; @@ -319,7 +319,7 @@ LoadStatus BuildLog::Load(const string& path, string* err) { if (!end) continue; *end = 0; - restat_mtime = strtoll(start, NULL, 10); + mtime = strtoll(start, NULL, 10); start = end + 1; end = (char*)memchr(start, kFieldSeparator, line_end - start); @@ -343,7 +343,7 @@ LoadStatus BuildLog::Load(const string& path, string* err) { entry->start_time = start_time; entry->end_time = end_time; - entry->mtime = restat_mtime; + entry->mtime = mtime; if (log_version >= 5) { char c = *end; *end = '\0'; entry->command_hash = (uint64_t)strtoull(start, NULL, 16); diff --git a/src/build_log.h b/src/build_log.h index 88551e3217..dd72c4c772 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -73,7 +73,7 @@ struct BuildLog { explicit LogEntry(const std::string& output); LogEntry(const std::string& output, uint64_t command_hash, - int start_time, int end_time, TimeStamp restat_mtime); + int start_time, int end_time, TimeStamp mtime); }; /// Lookup a previously-run command by its output path. diff --git a/src/build_test.cc b/src/build_test.cc index f58a7de870..474dd196a0 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -609,6 +609,15 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) == DiskInterface::Okay) fs_->WriteFile(edge->outputs_[0]->path(), content); + } else if (edge->rule().name() == "long-cc" || + edge->rule().name() == "long-cc2") { + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + fs_->Tick(); + fs_->Tick(); + fs_->Tick(); + fs_->Create((*out)->path(), ""); + } } else { printf("unknown command\n"); return false; @@ -664,6 +673,15 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { else result->status = ExitSuccess; + // These rules simulate an external process modifying files while the build command + // runs. See TestInputMtimeRaceCondition and TestInputMtimeRaceConditionWithDepFile. + // Note: only the "first" time the rule is run per test is the file modified, so the + // test can verify that subsequent runs without the race have no work to do. + if (edge->rule().name() == "long-cc" && fs_->now_ == 4) + fs_->files_["in1"].mtime = 3; + if (edge->rule().name() == "long-cc2" && fs_->now_ == 4) + fs_->files_["header.h"].mtime = 3; + // Provide a way for test cases to verify when an edge finishes that // some other edge is still active. This is useful for test cases // covering behavior involving multiple active edges. @@ -2266,6 +2284,148 @@ TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) { builder.command_runner_.release(); } +TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceCondition) { + string err; + const char* manifest = + "rule long-cc\n" + " command = long-cc\n" + "build out: long-cc in1\n"; + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + BuildLog build_log; + ASSERT_TRUE(build_log.Load("build_log", &err)); + ASSERT_TRUE(build_log.OpenForWrite("build_log", *this, &err)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + + BuildLog::LogEntry* log_entry = NULL; + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + // Run the build, out gets built, dep file is created + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.Build(&err)); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + // See that an entry in the logfile is created. the input_mtime is 1 since that was + // the mtime of in1 when the command was started + log_entry = build_log.LookupByOutput("out"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(1u, log_entry->mtime); + + builder.command_runner_.release(); + } + + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + // Trigger the build again - "out" should rebuild despite having a newer mtime than + // "in1", since "in1" was touched during the build of out (simulated by changing its + // mtime in the the test builder's WaitForCommand() which runs before FinishCommand() + command_runner_.commands_ran_.clear(); + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.Build(&err)); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + // Check that the logfile entry is still correct + log_entry = build_log.LookupByOutput("out"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(fs_.files_["in1"].mtime, log_entry->mtime); + builder.command_runner_.release(); + } + + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + // And a subsequent run should not have any work to do + command_runner_.commands_ran_.clear(); + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + builder.command_runner_.release(); + } +} + +TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { + string err; + const char* manifest = + "rule long-cc2\n" + " command = long-cc2\n" + "build out: long-cc2 in1\n" + " deps = gcc\n" + " depfile = in1.d\n"; + + fs_.Create("header.h", ""); + fs_.Create("in1.d", "out: header.h"); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + BuildLog build_log; + ASSERT_TRUE(build_log.Load("build_log", &err)); + ASSERT_TRUE(build_log.OpenForWrite("build_log", *this, &err)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + // Run the build, out gets built, dep file is created + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.Build(&err)); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + // See that an entry in the logfile is created. the mtime is 3 due to header.h being + // updated during the build of out (simulated by changing its mtime in the the test + // builder's WaitForCommand() which runs before FinishCommand() + BuildLog::LogEntry* log_entry = build_log.LookupByOutput("out"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(3u, log_entry->mtime); + + builder.command_runner_.release(); + } + + { + // Trigger the build again - "out" won't rebuild since its newest mtime (header.h) + // wasn't known when out was originally built and was only discovered via the deps file + // when the command completed. Subsequent runs will see out's recorded mtime equal to + // the actual most recent mtime. + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + command_runner_.commands_ran_.clear(); + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + builder.command_runner_.release(); + } +} + /// Check that a restat rule generating a header cancels compilations correctly. TEST_F(BuildTest, RestatDepfileDependency) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, diff --git a/src/graph.cc b/src/graph.cc index 90e8d0877e..9e8eda69de 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -143,10 +143,12 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, } } + edge->most_recent_input_ = most_recent_input; + // We may also be dirty due to output state: missing outputs, out of // date outputs, etc. Visit all outputs and determine whether they're dirty. if (!dirty) - if (!RecomputeOutputsDirty(edge, most_recent_input, &dirty, err)) + if (!RecomputeOutputsDirty(edge, &dirty, err)) return false; // Finally, visit each output and update their dirty state if necessary. @@ -212,12 +214,11 @@ bool DependencyScan::VerifyDAG(Node* node, vector* stack, string* err) { return false; } -bool DependencyScan::RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, - bool* outputs_dirty, string* err) { +bool DependencyScan::RecomputeOutputsDirty(Edge* edge, bool* outputs_dirty, string* err) { string command = edge->EvaluateCommand(/*incl_rsp_file=*/true); for (vector::iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) { - if (RecomputeOutputDirty(edge, most_recent_input, command, *o)) { + if (RecomputeOutputDirty(edge, command, *o)) { *outputs_dirty = true; return true; } @@ -226,7 +227,6 @@ bool DependencyScan::RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, } bool DependencyScan::RecomputeOutputDirty(const Edge* edge, - const Node* most_recent_input, const string& command, Node* output) { if (edge->is_phony()) { @@ -248,30 +248,7 @@ bool DependencyScan::RecomputeOutputDirty(const Edge* edge, return true; } - // Dirty if the output is older than the input. - if (most_recent_input && output->mtime() < most_recent_input->mtime()) { - TimeStamp output_mtime = output->mtime(); - - // If this is a restat rule, we may have cleaned the output with a restat - // rule in a previous run and stored the most recent input mtime in the - // build log. Use that mtime instead, so that the file will only be - // considered dirty if an input was modified since the previous run. - bool used_restat = false; - if (edge->GetBindingBool("restat") && build_log() && - (entry = build_log()->LookupByOutput(output->path()))) { - output_mtime = entry->mtime; - used_restat = true; - } - - if (output_mtime < most_recent_input->mtime()) { - EXPLAIN("%soutput %s older than most recent input %s " - "(%" PRId64 " vs %" PRId64 ")", - used_restat ? "restat of " : "", output->path().c_str(), - most_recent_input->path().c_str(), - output_mtime, most_recent_input->mtime()); - return true; - } - } + const Node* most_recent_input = edge->most_recent_input_; if (build_log()) { bool generator = edge->GetBindingBool("generator"); @@ -299,6 +276,14 @@ bool DependencyScan::RecomputeOutputDirty(const Edge* edge, EXPLAIN("command line not found in log for %s", output->path().c_str()); return true; } + } else if (most_recent_input && + output->mtime() < most_recent_input->mtime()) { + EXPLAIN( + "output %s older than most recent input %s " + "(%" PRId64 " vs %" PRId64 ")", + output->path().c_str(), most_recent_input->path().c_str(), + output->mtime(), most_recent_input->mtime()); + return true; } return false; diff --git a/src/graph.h b/src/graph.h index 6756378a4b..b7187ad667 100644 --- a/src/graph.h +++ b/src/graph.h @@ -146,6 +146,7 @@ struct Edge { Edge() : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone), + most_recent_input_(NULL), most_recent_input_mtime_(0), id_(0), outputs_ready_(false), deps_loaded_(false), deps_missing_(false), implicit_deps_(0), order_only_deps_(0), implicit_outs_(0) {} @@ -178,6 +179,8 @@ struct Edge { Node* dyndep_; BindingEnv* env_; VisitMark mark_; + Node* most_recent_input_; + TimeStamp most_recent_input_mtime_; size_t id_; bool outputs_ready_; bool deps_loaded_; @@ -298,8 +301,7 @@ struct DependencyScan { /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. /// Returns false on failure. - bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, - bool* dirty, std::string* err); + bool RecomputeOutputsDirty(Edge* edge, bool* dirty, std::string* err); BuildLog* build_log() const { return build_log_; @@ -325,8 +327,8 @@ struct DependencyScan { /// Recompute whether a given single output should be marked dirty. /// Returns true if so. - bool RecomputeOutputDirty(const Edge* edge, const Node* most_recent_input, - const std::string& command, Node* output); + bool RecomputeOutputDirty(const Edge* edge, const std::string& command, + Node* output); BuildLog* build_log_; DiskInterface* disk_interface_; From db6c4ac47601c88984822e1e7b5f4b9044dd6d81 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Sat, 20 Mar 2021 15:18:53 +0100 Subject: [PATCH 46/96] Remove `-w dupbuild` from error message and help output This is step 4 on #931. --- src/manifest_parser.cc | 11 +++++------ src/manifest_parser_test.cc | 5 ++--- src/ninja.cc | 1 - 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 54ad3f44c8..f77109fa7e 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -323,15 +323,14 @@ bool ManifestParser::ParseEdge(string* err) { return lexer_.Error(path_err, err); if (!state_->AddOut(edge, path, slash_bits)) { if (options_.dupe_edge_action_ == kDupeEdgeActionError) { - lexer_.Error("multiple rules generate " + path + " [-w dupbuild=err]", - err); + lexer_.Error("multiple rules generate " + path, err); return false; } else { if (!quiet_) { - Warning("multiple rules generate %s. " - "builds involving this target will not be correct; " - "continuing anyway [-w dupbuild=warn]", - path.c_str()); + Warning( + "multiple rules generate %s. builds involving this target will " + "not be correct; continuing anyway", + path.c_str()); } if (e - i <= static_cast(implicit_outs)) --implicit_outs; diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index ec2eeed2e8..5b0eddf7c4 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -365,7 +365,7 @@ TEST_F(ParserTest, DuplicateEdgeWithMultipleOutputsError) { ManifestParser parser(&state, &fs_, parser_opts); string err; EXPECT_FALSE(parser.ParseTest(kInput, &err)); - EXPECT_EQ("input:5: multiple rules generate out1 [-w dupbuild=err]\n", err); + EXPECT_EQ("input:5: multiple rules generate out1\n", err); } TEST_F(ParserTest, DuplicateEdgeInIncludedFile) { @@ -382,8 +382,7 @@ TEST_F(ParserTest, DuplicateEdgeInIncludedFile) { ManifestParser parser(&state, &fs_, parser_opts); string err; EXPECT_FALSE(parser.ParseTest(kInput, &err)); - EXPECT_EQ("sub.ninja:5: multiple rules generate out1 [-w dupbuild=err]\n", - err); + EXPECT_EQ("sub.ninja:5: multiple rules generate out1\n", err); } TEST_F(ParserTest, PhonySelfReferenceIgnored) { diff --git a/src/ninja.cc b/src/ninja.cc index 45fc8ea417..1cff6e8ca8 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -1104,7 +1104,6 @@ bool DebugEnable(const string& name) { bool WarningEnable(const string& name, Options* options) { if (name == "list") { printf("warning flags:\n" -" dupbuild={err,warn} multiple build lines for one target\n" " phonycycle={err,warn} phony build statement references itself\n" ); return false; From b51a3e83ff25ca95464b179344cfe4a98e242a21 Mon Sep 17 00:00:00 2001 From: John Drouhard Date: Tue, 23 Mar 2021 15:10:43 -0500 Subject: [PATCH 47/96] Revert "Change build log to always log the most recent input mtime" This reverts commit 67fbbeeec91ec171da7d4e297b8f9b319f3424c8. There were a few missing test cases exposed by CMake's test suite that need slight adjustments. Better to revert the original attempt, add the test cases, and then re-land the change with the fixes. Fixes #1932 --- src/build.cc | 75 ++++++++++------------ src/build_log.cc | 10 +-- src/build_log.h | 2 +- src/build_test.cc | 160 ---------------------------------------------- src/graph.cc | 43 +++++++++---- src/graph.h | 10 ++- 6 files changed, 74 insertions(+), 226 deletions(-) diff --git a/src/build.cc b/src/build.cc index 2e292322ff..fb5890a76d 100644 --- a/src/build.cc +++ b/src/build.cc @@ -270,11 +270,19 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { #define MEM_FN mem_fn // mem_fun was removed in C++17. #endif if (find_if(begin, end, MEM_FN(&Node::dirty)) == end) { + // Recompute most_recent_input. + Node* most_recent_input = NULL; + for (vector::iterator i = begin; i != end; ++i) { + if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) + most_recent_input = *i; + } + // Now, this edge is dirty if any of the outputs are dirty. // If the edge isn't dirty, clean the outputs and mark the edge as not // wanted. bool outputs_dirty = false; - if (!scan->RecomputeOutputsDirty(*oe, &outputs_dirty, err)) { + if (!scan->RecomputeOutputsDirty(*oe, most_recent_input, + &outputs_dirty, err)) { return false; } if (!outputs_dirty) { @@ -688,20 +696,6 @@ bool Builder::StartEdge(Edge* edge, string* err) { return false; } - // Find the most recent mtime of any (existing) non-order-only input - Node* most_recent_input = NULL; - for (vector::iterator i = edge->inputs_.begin(); - i != edge->inputs_.end() - edge->order_only_deps_; ++i) { - if (!(*i)->Stat(disk_interface_, err)) - return false; - if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) - most_recent_input = *i; - } - - edge->most_recent_input_ = most_recent_input; - if (most_recent_input) - edge->most_recent_input_mtime_ = most_recent_input->mtime(); - // start command computing and run it if (!command_runner_->StartCommand(edge)) { err->assign("command '" + edge->EvaluateCommand() + "' failed."); @@ -750,18 +744,20 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { return plan_.EdgeFinished(edge, Plan::kEdgeFailed, err); } - TimeStamp most_recent_input_mtime = 0; + // Restat the edge outputs + TimeStamp output_mtime = 0; + bool restat = edge->GetBindingBool("restat"); if (!config_.dry_run) { - // Restat the edge outputs bool node_cleaned = false; + for (vector::iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) { - TimeStamp old_mtime = (*o)->mtime(); - (*o)->Stat(disk_interface_, err); - TimeStamp new_mtime = (*o)->mtime(); + TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err); if (new_mtime == -1) return false; - if (old_mtime == new_mtime && edge->GetBindingBool("restat")) { + if (new_mtime > output_mtime) + output_mtime = new_mtime; + if ((*o)->mtime() == new_mtime && restat) { // The rule command did not change the output. Propagate the clean // state through the build graph. // Note that this also applies to nonexistent outputs (mtime == 0). @@ -771,34 +767,33 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { } } - // Use the time from the most recent input that was computed when the edge was - // started, not the mtime of the node as it is now. There could have been other edges - // that restat'd the input node and detected a change, but for *this* edge, we want - // the mtime as it was when the command began. - most_recent_input_mtime = edge->most_recent_input_mtime_; - - // If there were any added deps, compute the most recent input mtime - for (vector::iterator i = deps_nodes.begin(); - i != deps_nodes.end(); ++i) { - (*i)->StatIfNecessary(disk_interface_, err); - if ((*i)->mtime() > most_recent_input_mtime) - most_recent_input_mtime = (*i)->mtime(); - } - if (node_cleaned) { - // If any output was cleaned, take into account the mtime of the depfile + TimeStamp restat_mtime = 0; + // If any output was cleaned, find the most recent mtime of any + // (existing) non-order-only input or the depfile. + for (vector::iterator i = edge->inputs_.begin(); + i != edge->inputs_.end() - edge->order_only_deps_; ++i) { + TimeStamp input_mtime = disk_interface_->Stat((*i)->path(), err); + if (input_mtime == -1) + return false; + if (input_mtime > restat_mtime) + restat_mtime = input_mtime; + } + string depfile = edge->GetUnescapedDepfile(); - if (most_recent_input_mtime != 0 && deps_type.empty() && !depfile.empty()) { + if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) { TimeStamp depfile_mtime = disk_interface_->Stat(depfile, err); if (depfile_mtime == -1) return false; - if (depfile_mtime > most_recent_input_mtime) - most_recent_input_mtime = depfile_mtime; + if (depfile_mtime > restat_mtime) + restat_mtime = depfile_mtime; } // The total number of edges in the plan may have changed as a result // of a restat. status_->PlanHasTotalEdges(plan_.command_edge_count()); + + output_mtime = restat_mtime; } } @@ -812,7 +807,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { if (scan_.build_log()) { if (!scan_.build_log()->RecordCommand(edge, start_time_millis, - end_time_millis, most_recent_input_mtime)) { + end_time_millis, output_mtime)) { *err = string("Error writing to build log: ") + strerror(errno); return false; } diff --git a/src/build_log.cc b/src/build_log.cc index b35279d410..4dcd6cee53 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -116,9 +116,9 @@ BuildLog::LogEntry::LogEntry(const string& output) : output(output) {} BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, - int start_time, int end_time, TimeStamp mtime) + int start_time, int end_time, TimeStamp restat_mtime) : output(output), command_hash(command_hash), - start_time(start_time), end_time(end_time), mtime(mtime) + start_time(start_time), end_time(end_time), mtime(restat_mtime) {} BuildLog::BuildLog() @@ -303,7 +303,7 @@ LoadStatus BuildLog::Load(const string& path, string* err) { *end = 0; int start_time = 0, end_time = 0; - TimeStamp mtime = 0; + TimeStamp restat_mtime = 0; start_time = atoi(start); start = end + 1; @@ -319,7 +319,7 @@ LoadStatus BuildLog::Load(const string& path, string* err) { if (!end) continue; *end = 0; - mtime = strtoll(start, NULL, 10); + restat_mtime = strtoll(start, NULL, 10); start = end + 1; end = (char*)memchr(start, kFieldSeparator, line_end - start); @@ -343,7 +343,7 @@ LoadStatus BuildLog::Load(const string& path, string* err) { entry->start_time = start_time; entry->end_time = end_time; - entry->mtime = mtime; + entry->mtime = restat_mtime; if (log_version >= 5) { char c = *end; *end = '\0'; entry->command_hash = (uint64_t)strtoull(start, NULL, 16); diff --git a/src/build_log.h b/src/build_log.h index dd72c4c772..88551e3217 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -73,7 +73,7 @@ struct BuildLog { explicit LogEntry(const std::string& output); LogEntry(const std::string& output, uint64_t command_hash, - int start_time, int end_time, TimeStamp mtime); + int start_time, int end_time, TimeStamp restat_mtime); }; /// Lookup a previously-run command by its output path. diff --git a/src/build_test.cc b/src/build_test.cc index 474dd196a0..f58a7de870 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -609,15 +609,6 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) == DiskInterface::Okay) fs_->WriteFile(edge->outputs_[0]->path(), content); - } else if (edge->rule().name() == "long-cc" || - edge->rule().name() == "long-cc2") { - for (vector::iterator out = edge->outputs_.begin(); - out != edge->outputs_.end(); ++out) { - fs_->Tick(); - fs_->Tick(); - fs_->Tick(); - fs_->Create((*out)->path(), ""); - } } else { printf("unknown command\n"); return false; @@ -673,15 +664,6 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { else result->status = ExitSuccess; - // These rules simulate an external process modifying files while the build command - // runs. See TestInputMtimeRaceCondition and TestInputMtimeRaceConditionWithDepFile. - // Note: only the "first" time the rule is run per test is the file modified, so the - // test can verify that subsequent runs without the race have no work to do. - if (edge->rule().name() == "long-cc" && fs_->now_ == 4) - fs_->files_["in1"].mtime = 3; - if (edge->rule().name() == "long-cc2" && fs_->now_ == 4) - fs_->files_["header.h"].mtime = 3; - // Provide a way for test cases to verify when an edge finishes that // some other edge is still active. This is useful for test cases // covering behavior involving multiple active edges. @@ -2284,148 +2266,6 @@ TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) { builder.command_runner_.release(); } -TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceCondition) { - string err; - const char* manifest = - "rule long-cc\n" - " command = long-cc\n" - "build out: long-cc in1\n"; - - State state; - ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); - ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); - - BuildLog build_log; - ASSERT_TRUE(build_log.Load("build_log", &err)); - ASSERT_TRUE(build_log.OpenForWrite("build_log", *this, &err)); - - DepsLog deps_log; - ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - - BuildLog::LogEntry* log_entry = NULL; - { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); - builder.command_runner_.reset(&command_runner_); - command_runner_.commands_ran_.clear(); - - // Run the build, out gets built, dep file is created - EXPECT_TRUE(builder.AddTarget("out", &err)); - ASSERT_EQ("", err); - EXPECT_TRUE(builder.Build(&err)); - ASSERT_EQ(1u, command_runner_.commands_ran_.size()); - - // See that an entry in the logfile is created. the input_mtime is 1 since that was - // the mtime of in1 when the command was started - log_entry = build_log.LookupByOutput("out"); - ASSERT_TRUE(NULL != log_entry); - ASSERT_EQ(1u, log_entry->mtime); - - builder.command_runner_.release(); - } - - { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); - builder.command_runner_.reset(&command_runner_); - command_runner_.commands_ran_.clear(); - - // Trigger the build again - "out" should rebuild despite having a newer mtime than - // "in1", since "in1" was touched during the build of out (simulated by changing its - // mtime in the the test builder's WaitForCommand() which runs before FinishCommand() - command_runner_.commands_ran_.clear(); - state.Reset(); - EXPECT_TRUE(builder.AddTarget("out", &err)); - ASSERT_EQ("", err); - EXPECT_TRUE(builder.Build(&err)); - ASSERT_EQ(1u, command_runner_.commands_ran_.size()); - - // Check that the logfile entry is still correct - log_entry = build_log.LookupByOutput("out"); - ASSERT_TRUE(NULL != log_entry); - ASSERT_EQ(fs_.files_["in1"].mtime, log_entry->mtime); - builder.command_runner_.release(); - } - - { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); - builder.command_runner_.reset(&command_runner_); - command_runner_.commands_ran_.clear(); - - // And a subsequent run should not have any work to do - command_runner_.commands_ran_.clear(); - state.Reset(); - EXPECT_TRUE(builder.AddTarget("out", &err)); - ASSERT_EQ("", err); - EXPECT_TRUE(builder.AlreadyUpToDate()); - - builder.command_runner_.release(); - } -} - -TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { - string err; - const char* manifest = - "rule long-cc2\n" - " command = long-cc2\n" - "build out: long-cc2 in1\n" - " deps = gcc\n" - " depfile = in1.d\n"; - - fs_.Create("header.h", ""); - fs_.Create("in1.d", "out: header.h"); - - State state; - ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); - ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); - - BuildLog build_log; - ASSERT_TRUE(build_log.Load("build_log", &err)); - ASSERT_TRUE(build_log.OpenForWrite("build_log", *this, &err)); - - DepsLog deps_log; - ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); - - { - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); - builder.command_runner_.reset(&command_runner_); - command_runner_.commands_ran_.clear(); - - // Run the build, out gets built, dep file is created - EXPECT_TRUE(builder.AddTarget("out", &err)); - ASSERT_EQ("", err); - EXPECT_TRUE(builder.Build(&err)); - ASSERT_EQ(1u, command_runner_.commands_ran_.size()); - - // See that an entry in the logfile is created. the mtime is 3 due to header.h being - // updated during the build of out (simulated by changing its mtime in the the test - // builder's WaitForCommand() which runs before FinishCommand() - BuildLog::LogEntry* log_entry = build_log.LookupByOutput("out"); - ASSERT_TRUE(NULL != log_entry); - ASSERT_EQ(3u, log_entry->mtime); - - builder.command_runner_.release(); - } - - { - // Trigger the build again - "out" won't rebuild since its newest mtime (header.h) - // wasn't known when out was originally built and was only discovered via the deps file - // when the command completed. Subsequent runs will see out's recorded mtime equal to - // the actual most recent mtime. - Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); - builder.command_runner_.reset(&command_runner_); - command_runner_.commands_ran_.clear(); - - command_runner_.commands_ran_.clear(); - state.Reset(); - EXPECT_TRUE(builder.AddTarget("out", &err)); - ASSERT_EQ("", err); - EXPECT_TRUE(builder.AlreadyUpToDate()); - - builder.command_runner_.release(); - } -} - /// Check that a restat rule generating a header cancels compilations correctly. TEST_F(BuildTest, RestatDepfileDependency) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, diff --git a/src/graph.cc b/src/graph.cc index 9e8eda69de..90e8d0877e 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -143,12 +143,10 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, } } - edge->most_recent_input_ = most_recent_input; - // We may also be dirty due to output state: missing outputs, out of // date outputs, etc. Visit all outputs and determine whether they're dirty. if (!dirty) - if (!RecomputeOutputsDirty(edge, &dirty, err)) + if (!RecomputeOutputsDirty(edge, most_recent_input, &dirty, err)) return false; // Finally, visit each output and update their dirty state if necessary. @@ -214,11 +212,12 @@ bool DependencyScan::VerifyDAG(Node* node, vector* stack, string* err) { return false; } -bool DependencyScan::RecomputeOutputsDirty(Edge* edge, bool* outputs_dirty, string* err) { +bool DependencyScan::RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, + bool* outputs_dirty, string* err) { string command = edge->EvaluateCommand(/*incl_rsp_file=*/true); for (vector::iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) { - if (RecomputeOutputDirty(edge, command, *o)) { + if (RecomputeOutputDirty(edge, most_recent_input, command, *o)) { *outputs_dirty = true; return true; } @@ -227,6 +226,7 @@ bool DependencyScan::RecomputeOutputsDirty(Edge* edge, bool* outputs_dirty, stri } bool DependencyScan::RecomputeOutputDirty(const Edge* edge, + const Node* most_recent_input, const string& command, Node* output) { if (edge->is_phony()) { @@ -248,7 +248,30 @@ bool DependencyScan::RecomputeOutputDirty(const Edge* edge, return true; } - const Node* most_recent_input = edge->most_recent_input_; + // Dirty if the output is older than the input. + if (most_recent_input && output->mtime() < most_recent_input->mtime()) { + TimeStamp output_mtime = output->mtime(); + + // If this is a restat rule, we may have cleaned the output with a restat + // rule in a previous run and stored the most recent input mtime in the + // build log. Use that mtime instead, so that the file will only be + // considered dirty if an input was modified since the previous run. + bool used_restat = false; + if (edge->GetBindingBool("restat") && build_log() && + (entry = build_log()->LookupByOutput(output->path()))) { + output_mtime = entry->mtime; + used_restat = true; + } + + if (output_mtime < most_recent_input->mtime()) { + EXPLAIN("%soutput %s older than most recent input %s " + "(%" PRId64 " vs %" PRId64 ")", + used_restat ? "restat of " : "", output->path().c_str(), + most_recent_input->path().c_str(), + output_mtime, most_recent_input->mtime()); + return true; + } + } if (build_log()) { bool generator = edge->GetBindingBool("generator"); @@ -276,14 +299,6 @@ bool DependencyScan::RecomputeOutputDirty(const Edge* edge, EXPLAIN("command line not found in log for %s", output->path().c_str()); return true; } - } else if (most_recent_input && - output->mtime() < most_recent_input->mtime()) { - EXPLAIN( - "output %s older than most recent input %s " - "(%" PRId64 " vs %" PRId64 ")", - output->path().c_str(), most_recent_input->path().c_str(), - output->mtime(), most_recent_input->mtime()); - return true; } return false; diff --git a/src/graph.h b/src/graph.h index b7187ad667..6756378a4b 100644 --- a/src/graph.h +++ b/src/graph.h @@ -146,7 +146,6 @@ struct Edge { Edge() : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone), - most_recent_input_(NULL), most_recent_input_mtime_(0), id_(0), outputs_ready_(false), deps_loaded_(false), deps_missing_(false), implicit_deps_(0), order_only_deps_(0), implicit_outs_(0) {} @@ -179,8 +178,6 @@ struct Edge { Node* dyndep_; BindingEnv* env_; VisitMark mark_; - Node* most_recent_input_; - TimeStamp most_recent_input_mtime_; size_t id_; bool outputs_ready_; bool deps_loaded_; @@ -301,7 +298,8 @@ struct DependencyScan { /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. /// Returns false on failure. - bool RecomputeOutputsDirty(Edge* edge, bool* dirty, std::string* err); + bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, + bool* dirty, std::string* err); BuildLog* build_log() const { return build_log_; @@ -327,8 +325,8 @@ struct DependencyScan { /// Recompute whether a given single output should be marked dirty. /// Returns true if so. - bool RecomputeOutputDirty(const Edge* edge, const std::string& command, - Node* output); + bool RecomputeOutputDirty(const Edge* edge, const Node* most_recent_input, + const std::string& command, Node* output); BuildLog* build_log_; DiskInterface* disk_interface_; From ef2ae300a8f589483bd4435fabfbcfcb05e64fb0 Mon Sep 17 00:00:00 2001 From: John Drouhard Date: Tue, 23 Mar 2021 15:15:03 -0500 Subject: [PATCH 48/96] Add some regression tests for additional test cases These expose some behavior related to implicit deps unknown to ninja and timestamps with generating them as part of a rule. See discussions in #1932 and #1933. --- src/build_test.cc | 186 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/src/build_test.cc b/src/build_test.cc index f58a7de870..a92f7c5840 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -609,6 +609,32 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) == DiskInterface::Okay) fs_->WriteFile(edge->outputs_[0]->path(), content); + } else if (edge->rule().name() == "touch-implicit-dep-out") { + string dep = edge->GetBinding("test_dependency"); + fs_->Create(dep, ""); + fs_->Tick(); + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + fs_->Create((*out)->path(), ""); + } + } else if (edge->rule().name() == "touch-out-implicit-dep") { + string dep = edge->GetBinding("test_dependency"); + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + fs_->Create((*out)->path(), ""); + } + fs_->Tick(); + fs_->Create(dep, ""); + } else if (edge->rule().name() == "generate-depfile") { + string dep = edge->GetBinding("test_dependency"); + string depfile = edge->GetUnescapedDepfile(); + string contents; + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + contents += (*out)->path() + ": " + dep + "\n"; + fs_->Create((*out)->path(), ""); + } + fs_->Create(depfile, contents); } else { printf("unknown command\n"); return false; @@ -1270,6 +1296,55 @@ struct BuildWithLogTest : public BuildTest { BuildLog build_log_; }; +TEST_F(BuildWithLogTest, ImplicitGeneratedOutOfDate) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +" generator = 1\n" +"build out.imp: touch | in\n")); + fs_.Create("out.imp", ""); + fs_.Tick(); + fs_.Create("in", ""); + + string err; + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_TRUE(GetNode("out.imp")->dirty()); +} + +TEST_F(BuildWithLogTest, ImplicitGeneratedOutOfDate2) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch-implicit-dep-out\n" +" command = touch $test_dependency ; sleep 1 ; touch $out\n" +" generator = 1\n" +"build out.imp: touch-implicit-dep-out | inimp inimp2\n" +" test_dependency = inimp\n")); + fs_.Create("inimp", ""); + fs_.Create("out.imp", ""); + fs_.Tick(); + fs_.Create("inimp2", ""); + fs_.Tick(); + + string err; + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + builder_.Cleanup(); + builder_.plan_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + EXPECT_FALSE(GetNode("out.imp")->dirty()); +} + TEST_F(BuildWithLogTest, NotInLogButOnDisk) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule cc\n" @@ -1571,6 +1646,33 @@ TEST_F(BuildWithLogTest, RestatMissingInput) { ASSERT_EQ(restat_mtime, log_entry->mtime); } +TEST_F(BuildWithLogTest, GeneratedPlainDepfileMtime) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule generate-depfile\n" +" command = touch $out ; echo \"$out: $test_dependency\" > $depfile\n" +"build out: generate-depfile\n" +" test_dependency = inimp\n" +" depfile = out.d\n")); + fs_.Create("inimp", ""); + fs_.Tick(); + + string err; + + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + builder_.Cleanup(); + builder_.plan_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + struct BuildDryRun : public BuildWithLogTest { BuildDryRun() { config_.dry_run = true; @@ -2414,6 +2516,90 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { } } +TEST_F(BuildWithDepsLogTest, DiscoveredDepDuringBuildChanged) { + string err; + const char* manifest = + "rule touch-out-implicit-dep\n" + " command = touch $out ; sleep 1 ; touch $test_dependency\n" + "rule generate-depfile\n" + " command = touch $out ; echo \"$out: $test_dependency\" > $depfile\n" + "build out1: touch-out-implicit-dep in1\n" + " test_dependency = inimp\n" + "build out2: generate-depfile in1 || out1\n" + " test_dependency = inimp\n" + " depfile = out2.d\n" + " deps = gcc\n"; + + fs_.Create("in1", ""); + fs_.Tick(); + + BuildLog build_log; + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out2", &err)); + EXPECT_FALSE(builder.AlreadyUpToDate()); + + EXPECT_TRUE(builder.Build(&err)); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + deps_log.Close(); + builder.command_runner_.release(); + } + + fs_.Tick(); + fs_.Create("in1", ""); + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out2", &err)); + EXPECT_FALSE(builder.AlreadyUpToDate()); + + EXPECT_TRUE(builder.Build(&err)); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + deps_log.Close(); + builder.command_runner_.release(); + } + + fs_.Tick(); + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out2", &err)); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + deps_log.Close(); + builder.command_runner_.release(); + } +} + #ifdef _WIN32 TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { string err; From a5b178048746383ac8be231c074b11fbfce6f6c8 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 24 Mar 2021 14:06:19 -0400 Subject: [PATCH 49/96] dyndep: add dedicated test for dyndep-discovered implicit dependencies Previously this was covered only as part of more complex tests. --- src/graph_test.cc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/graph_test.cc b/src/graph_test.cc index 14f6375765..6b4bb517e8 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -511,6 +511,37 @@ TEST_F(GraphTest, DyndepLoadTrivial) { EXPECT_FALSE(edge->GetBindingBool("restat")); } +TEST_F(GraphTest, DyndepLoadImplicit) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1: r in || dd\n" +" dyndep = dd\n" +"build out2: r in\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out1: dyndep | out2\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("", err); + EXPECT_FALSE(GetNode("dd")->dyndep_pending()); + + Edge* edge = GetNode("out1")->in_edge(); + ASSERT_EQ(1u, edge->outputs_.size()); + EXPECT_EQ("out1", edge->outputs_[0]->path()); + ASSERT_EQ(3u, edge->inputs_.size()); + EXPECT_EQ("in", edge->inputs_[0]->path()); + EXPECT_EQ("out2", edge->inputs_[1]->path()); + EXPECT_EQ("dd", edge->inputs_[2]->path()); + EXPECT_EQ(1u, edge->implicit_deps_); + EXPECT_EQ(1u, edge->order_only_deps_); + EXPECT_FALSE(edge->GetBindingBool("restat")); +} + TEST_F(GraphTest, DyndepLoadMissingFile) { AssertParse(&state_, "rule r\n" From fa577b6d5364d103e57784ba5d6efa973b96a5c3 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 24 Mar 2021 15:38:35 -0400 Subject: [PATCH 50/96] dyndep: reconcile dyndep-specified outputs with depfile-specified inputs When a path loaded from a depfile does not have a node, we create a new node with a phony edge producing it. If we later load a dyndep file that specifies the same node as an output of a known edge, we previously failed with a "multiple rules generate ..." error. Instead, since the conflicting edge was internally generated, replace the node's input edge with the now-known real edge that produces it. --- src/build_test.cc | 42 ++++++++++++++++++++++++++++++++++++++++++ src/dyndep.cc | 12 +++++++++--- src/graph.cc | 1 + src/graph.h | 5 +++-- 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/build_test.cc b/src/build_test.cc index a92f7c5840..e0c43b11af 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -3115,6 +3115,48 @@ TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) { EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]); } +TEST_F(BuildTest, DyndepBuildDiscoverOutputAndDepfileInput) { + // Verify that a dyndep file can be built and loaded to discover + // that one edge has an implicit output that is also reported by + // a depfile as an input of another edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build tmp: touch || dd\n" +" dyndep = dd\n" +"build out: cp tmp\n" +" depfile = out.d\n" +)); + fs_.Create("out.d", "out: tmp.imp\n"); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build tmp | tmp.imp: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + // Loading the depfile gave tmp.imp a phony input edge. + ASSERT_TRUE(GetNode("tmp.imp")->in_edge()->is_phony()); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + // Loading the dyndep file gave tmp.imp a real input edge. + ASSERT_FALSE(GetNode("tmp.imp")->in_edge()->is_phony()); + + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]); + EXPECT_EQ("cp tmp out", command_runner_.commands_ran_[2]); + EXPECT_EQ(1u, fs_.files_created_.count("tmp.imp")); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdge) { // Verify that a dyndep file can be built and loaded to discover // that an edge is actually wanted due to a missing implicit output. diff --git a/src/dyndep.cc b/src/dyndep.cc index b388e9bf4b..dd4ed099a9 100644 --- a/src/dyndep.cc +++ b/src/dyndep.cc @@ -97,9 +97,15 @@ bool DyndepLoader::UpdateEdge(Edge* edge, Dyndeps const* dyndeps, for (std::vector::const_iterator i = dyndeps->implicit_outputs_.begin(); i != dyndeps->implicit_outputs_.end(); ++i) { - if ((*i)->in_edge() != NULL) { - *err = "multiple rules generate " + (*i)->path(); - return false; + if (Edge* old_in_edge = (*i)->in_edge()) { + // This node already has an edge producing it. Fail with an error + // unless the edge was generated by ImplicitDepLoader, in which + // case we can replace it with the now-known real producer. + if (!old_in_edge->generated_by_dep_loader_) { + *err = "multiple rules generate " + (*i)->path(); + return false; + } + old_in_edge->outputs_.clear(); } (*i)->set_in_edge(edge); } diff --git a/src/graph.cc b/src/graph.cc index 90e8d0877e..822b7c5bcf 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -654,6 +654,7 @@ void ImplicitDepLoader::CreatePhonyInEdge(Node* node) { return; Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); + phony_edge->generated_by_dep_loader_ = true; node->set_in_edge(phony_edge); phony_edge->outputs_.push_back(node); diff --git a/src/graph.h b/src/graph.h index 6756378a4b..bb4f10c479 100644 --- a/src/graph.h +++ b/src/graph.h @@ -147,8 +147,8 @@ struct Edge { Edge() : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), mark_(VisitNone), id_(0), outputs_ready_(false), deps_loaded_(false), - deps_missing_(false), implicit_deps_(0), order_only_deps_(0), - implicit_outs_(0) {} + deps_missing_(false), generated_by_dep_loader_(false), + implicit_deps_(0), order_only_deps_(0), implicit_outs_(0) {} /// Return true if all inputs' in-edges are ready. bool AllInputsReady() const; @@ -182,6 +182,7 @@ struct Edge { bool outputs_ready_; bool deps_loaded_; bool deps_missing_; + bool generated_by_dep_loader_; const Rule& rule() const { return *rule_; } Pool* pool() const { return pool_; } From 5780728a60c032929c03a5b478d6d93948602846 Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Mon, 29 Mar 2021 01:30:25 +0200 Subject: [PATCH 51/96] CI: macOS: Unfix Xcode version to use the latest stable one Removes the selection of a fixed Xcode version added in 242b7dd which assured Xcode version 12.2 was used when the default was still 12.1. The GitHub default is now 12.4 and newer versions will be made the default a few weeks after the stable release. This commit ensures an up to date Xcode version is used in the CI. --- .github/workflows/macos.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index af79080a73..0797433267 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -21,7 +21,6 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: 10.12 run: | - sudo xcode-select -s /Applications/Xcode_12.2.app cmake -Bbuild -GXcode '-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64' cmake --build build --config Release From 721f5e9342be3bb83ff30edba420d2c508ff0cad Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Fri, 9 Apr 2021 13:45:25 +0200 Subject: [PATCH 52/96] Fix misleading usage of return type of Open*Log In an earlier version of 791c887e22046e5e7a2d05ecb5ff27701d56895d those functions returned LoadStatus, too, but it was changed back to bool. --- src/ninja.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 1cff6e8ca8..56e31e0d0e 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -136,11 +136,11 @@ struct NinjaMain : public BuildLogUser { int ToolWinCodePage(const Options* options, int argc, char* argv[]); /// Open the build log. - /// @return LOAD_ERROR on error. + /// @return false on error. bool OpenBuildLog(bool recompact_only = false); /// Open the deps log: load it, then open for writing. - /// @return LOAD_ERROR on error. + /// @return false on error. bool OpenDepsLog(bool recompact_only = false); /// Ensure the build directory exists, creating it if necessary. @@ -894,8 +894,8 @@ int NinjaMain::ToolRecompact(const Options* options, int argc, char* argv[]) { if (!EnsureBuildDirExists()) return 1; - if (OpenBuildLog(/*recompact_only=*/true) == LOAD_ERROR || - OpenDepsLog(/*recompact_only=*/true) == LOAD_ERROR) + if (!OpenBuildLog(/*recompact_only=*/true) || + !OpenDepsLog(/*recompact_only=*/true)) return 1; return 0; From ce700488e01af33bc478bc986e261e306180b993 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Fri, 9 Apr 2021 17:08:39 +0200 Subject: [PATCH 53/96] Avoid implicit cast LoadStatus -> bool --- src/build_log_perftest.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index ced0df9df7..5a936198fb 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -112,7 +112,7 @@ int main() { { // Read once to warm up disk cache. BuildLog log; - if (!log.Load(kTestFilename, &err)) { + if (log.Load(kTestFilename, &err) == LOAD_ERROR) { fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); return 1; } @@ -121,7 +121,7 @@ int main() { for (int i = 0; i < kNumRepetitions; ++i) { int64_t start = GetTimeMillis(); BuildLog log; - if (!log.Load(kTestFilename, &err)) { + if (log.Load(kTestFilename, &err) == LOAD_ERROR) { fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); return 1; } @@ -148,4 +148,3 @@ int main() { return 0; } - From a567ebb6ef75bfda9204900e4fcbcd7ee2785f61 Mon Sep 17 00:00:00 2001 From: Henner Zeller Date: Mon, 20 Jul 2020 12:39:07 -0700 Subject: [PATCH 54/96] Add --quiet option that suppresses status updates. Refined pull request after discussion in #1816 Signed-off-by: Henner Zeller --- src/build.h | 3 ++- src/ninja.cc | 7 ++++++- src/status.cc | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/build.h b/src/build.h index 06823c2e50..d697dfb89e 100644 --- a/src/build.h +++ b/src/build.h @@ -159,8 +159,9 @@ struct BuildConfig { failures_allowed(1), max_load_average(-0.0f) {} enum Verbosity { - NORMAL, QUIET, // No output -- used when testing. + NO_STATUS_UPDATE, // just regular output but suppress status update + NORMAL, // regular output and status update VERBOSE }; Verbosity verbosity; diff --git a/src/ninja.cc b/src/ninja.cc index 56e31e0d0e..ca63f2520c 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -217,6 +217,7 @@ void Usage(const BuildConfig& config) { "options:\n" " --version print ninja version (\"%s\")\n" " -v, --verbose show all command lines while building\n" +" --quiet don't show progress updates.\n" "\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" @@ -1307,11 +1308,12 @@ int ReadFlags(int* argc, char*** argv, Options* options, BuildConfig* config) { config->parallelism = GuessParallelism(); - enum { OPT_VERSION = 1 }; + enum { OPT_VERSION = 1, OPT_QUIET = 2 }; const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, OPT_VERSION }, { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, OPT_QUIET }, { NULL, 0, NULL, 0 } }; @@ -1369,6 +1371,9 @@ int ReadFlags(int* argc, char*** argv, case 'v': config->verbosity = BuildConfig::VERBOSE; break; + case OPT_QUIET: + config->verbosity = BuildConfig::NO_STATUS_UPDATE; + break; case 'w': if (!WarningEnable(optarg, options)) return 1; diff --git a/src/status.cc b/src/status.cc index 171cbebf12..88b77815b3 100644 --- a/src/status.cc +++ b/src/status.cc @@ -228,7 +228,8 @@ string StatusPrinter::FormatProgressStatus(const char* progress_status_format, } void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) { - if (config_.verbosity == BuildConfig::QUIET) + if (config_.verbosity == BuildConfig::QUIET + || config_.verbosity == BuildConfig::NO_STATUS_UPDATE) return; bool force_full_command = config_.verbosity == BuildConfig::VERBOSE; From 21a0834e897ef0dda6bf5509e92c000808ae8e09 Mon Sep 17 00:00:00 2001 From: Henner Zeller Date: Mon, 20 Jul 2020 13:15:23 -0700 Subject: [PATCH 55/96] Update help description for --quiet Signed-off-by: Henner Zeller --- src/ninja.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ninja.cc b/src/ninja.cc index ca63f2520c..759ccd7d05 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -217,7 +217,7 @@ void Usage(const BuildConfig& config) { "options:\n" " --version print ninja version (\"%s\")\n" " -v, --verbose show all command lines while building\n" -" --quiet don't show progress updates.\n" +" --quiet don't show progress status, just command output\n" "\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" From 770459d26f9985e4dbe4bdb1223bd21834458d4e Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Thu, 6 May 2021 09:23:44 -0700 Subject: [PATCH 56/96] Add jhasse's suggestion to suppress output on '--quiet'. This just ensures that we also don't get the "Entering directory..." message when the new '--quiet' flag is added. --- src/ninja.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ninja.cc b/src/ninja.cc index 759ccd7d05..c7182dff66 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -1419,7 +1419,7 @@ NORETURN void real_main(int argc, char** argv) { // subsequent commands. // Don't print this if a tool is being used, so that tool output // can be piped into a file without this string showing up. - if (!options.tool) + if (!options.tool && config.verbosity != BuildConfig::NO_STATUS_UPDATE) status->Info("Entering directory `%s'", options.working_dir); if (chdir(options.working_dir) < 0) { Fatal("chdir to '%s' - %s", options.working_dir, strerror(errno)); From f926cd4503f8d70d116752c3f5f3db9ea3d47eca Mon Sep 17 00:00:00 2001 From: Eli Ribble Date: Thu, 6 May 2021 09:31:11 -0700 Subject: [PATCH 57/96] Add test for status suppression under '--quiet'. This just tests that the flag works. --- misc/output_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/misc/output_test.py b/misc/output_test.py index b63520f6e9..6858e218ed 100755 --- a/misc/output_test.py +++ b/misc/output_test.py @@ -48,6 +48,15 @@ def run(build_ninja, flags='', pipe=False, env=default_env): @unittest.skipIf(platform.system() == 'Windows', 'These test methods do not work on Windows') class Output(unittest.TestCase): + BUILD_SIMPLE_ECHO = '\n'.join(( + 'rule echo', + ' command = printf "do thing"', + ' description = echo $out', + '', + 'build a: echo', + '', + )) + def test_issue_1418(self): self.assertEqual(run( '''rule echo @@ -111,5 +120,14 @@ def test_pr_1685(self): def test_status(self): self.assertEqual(run(''), 'ninja: no work to do.\n') + def test_ninja_status_default(self): + 'Do we show the default status by default?' + self.assertEqual(run(Output.BUILD_SIMPLE_ECHO), '[1/1] echo a\x1b[K\ndo thing\n') + + def test_ninja_status_quiet(self): + 'Do we suppress the status information when --quiet is specified?' + output = run(Output.BUILD_SIMPLE_ECHO, flags='--quiet') + self.assertEqual(output, 'do thing\n') + if __name__ == '__main__': unittest.main() From 8a3af294fd278a7ed378b36db677c46716583ff3 Mon Sep 17 00:00:00 2001 From: Andrew Scheller Date: Sun, 16 May 2021 21:09:38 +0100 Subject: [PATCH 58/96] Attempt to fix Linux CI build --- .github/workflows/linux.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index cd55262322..80c88c6387 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -18,10 +18,10 @@ jobs: curl -L -O https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-Linux-x86_64.sh chmod +x cmake-3.16.4-Linux-x86_64.sh ./cmake-3.16.4-Linux-x86_64.sh --skip-license --prefix=/usr/local - curl -L -O https://www.mirrorservice.org/sites/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/p7zip-16.02-10.el7.x86_64.rpm - curl -L -O https://www.mirrorservice.org/sites/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/p7zip-plugins-16.02-10.el7.x86_64.rpm - rpm -U --quiet p7zip-16.02-10.el7.x86_64.rpm - rpm -U --quiet p7zip-plugins-16.02-10.el7.x86_64.rpm + curl -L -O https://www.mirrorservice.org/sites/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/p7zip-16.02-20.el7.x86_64.rpm + curl -L -O https://www.mirrorservice.org/sites/dl.fedoraproject.org/pub/epel/7/x86_64/Packages/p/p7zip-plugins-16.02-20.el7.x86_64.rpm + rpm -U --quiet p7zip-16.02-20.el7.x86_64.rpm + rpm -U --quiet p7zip-plugins-16.02-20.el7.x86_64.rpm yum install -y make gcc-c++ libasan clang-analyzer - name: Build debug ninja From 54c5759471895f820735c232c36e81af96201912 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 2 Jun 2021 11:08:10 -0400 Subject: [PATCH 59/96] cmake: use `check_symbol_exists` for browse support Just because `unistd.h` exists does not mean it has the APIs that are needed. This occurs when cross-compiling using Fedora's packaged MinGW toolchain. --- CMakeLists.txt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8de69e89c9..bc02c4d9c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.15) -include(CheckIncludeFileCXX) +include(CheckSymbolExists) include(CheckIPOSupported) project(ninja) @@ -73,10 +73,15 @@ function(check_platform_supports_browse_mode RESULT) endif() # Now check availability of the unistd header - check_include_file_cxx(unistd.h PLATFORM_HAS_UNISTD_HEADER) - set(${RESULT} "${PLATFORM_HAS_UNISTD_HEADER}" PARENT_SCOPE) - if(NOT PLATFORM_HAS_UNISTD_HEADER) - message(WARNING "browse feature omitted due to missing unistd.h") + check_symbol_exists(fork "unistd.h" HAVE_FORK) + check_symbol_exists(pipe "unistd.h" HAVE_PIPE) + set(browse_supported 0) + if (HAVE_FORK AND HAVE_PIPE) + set(browse_supported 1) + endif () + set(${RESULT} "${browse_supported}" PARENT_SCOPE) + if(NOT browse_supported) + message(WARNING "browse feature omitted due to missing `fork` and `pipe` functions") endif() endfunction() From f2f62e295ad1254b7e3cdd5d78fa8afb0e86f1ce Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 3 Jun 2021 17:16:20 -0400 Subject: [PATCH 60/96] Add explicit "empty path" errors before calling CanonicalizePath Update call sites that might have empty paths to explicitly check for them before calling CanonicalizePath. Note that the depfile parser ensures all parsed outs and deps are non-empty. --- src/clean.cc | 5 +++++ src/dyndep_parser.cc | 6 ++++++ src/manifest_parser.cc | 6 ++++++ src/ninja.cc | 8 ++++++++ 4 files changed, 25 insertions(+) diff --git a/src/clean.cc b/src/clean.cc index 3e57437d10..1e97182b9a 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -189,6 +189,11 @@ int Cleaner::CleanTargets(int target_count, char* targets[]) { LoadDyndeps(); for (int i = 0; i < target_count; ++i) { string target_name = targets[i]; + if (target_name.empty()) { + Error("failed to canonicalize '': empty path"); + status_ = 1; + continue; + } uint64_t slash_bits; string err; if (!CanonicalizePath(&target_name, &slash_bits, &err)) { diff --git a/src/dyndep_parser.cc b/src/dyndep_parser.cc index 56da16f5b7..45d1c31595 100644 --- a/src/dyndep_parser.cc +++ b/src/dyndep_parser.cc @@ -115,6 +115,8 @@ bool DyndepParser::ParseEdge(string* err) { return lexer_.Error("expected path", err); string path = out0.Evaluate(&env_); + if (path.empty()) + return lexer_.Error("empty path", err); string path_err; uint64_t slash_bits; if (!CanonicalizePath(&path, &slash_bits, &path_err)) @@ -202,6 +204,8 @@ bool DyndepParser::ParseEdge(string* err) { dyndeps->implicit_inputs_.reserve(ins.size()); for (vector::iterator i = ins.begin(); i != ins.end(); ++i) { string path = i->Evaluate(&env_); + if (path.empty()) + return lexer_.Error("empty path", err); string path_err; uint64_t slash_bits; if (!CanonicalizePath(&path, &slash_bits, &path_err)) @@ -213,6 +217,8 @@ bool DyndepParser::ParseEdge(string* err) { dyndeps->implicit_outputs_.reserve(outs.size()); for (vector::iterator i = outs.begin(); i != outs.end(); ++i) { string path = i->Evaluate(&env_); + if (path.empty()) + return lexer_.Error("empty path", err); string path_err; uint64_t slash_bits; if (!CanonicalizePath(&path, &slash_bits, &path_err)) diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index f77109fa7e..8f0528a898 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -190,6 +190,8 @@ bool ManifestParser::ParseDefault(string* err) { do { string path = eval.Evaluate(env_); + if (path.empty()) + return lexer_.Error("empty path", err); string path_err; uint64_t slash_bits; // Unused because this only does lookup. if (!CanonicalizePath(&path, &slash_bits, &path_err)) @@ -317,6 +319,8 @@ bool ManifestParser::ParseEdge(string* err) { edge->outputs_.reserve(outs.size()); for (size_t i = 0, e = outs.size(); i != e; ++i) { string path = outs[i].Evaluate(env); + if (path.empty()) + return lexer_.Error("empty path", err); string path_err; uint64_t slash_bits; if (!CanonicalizePath(&path, &slash_bits, &path_err)) @@ -349,6 +353,8 @@ bool ManifestParser::ParseEdge(string* err) { edge->inputs_.reserve(ins.size()); for (vector::iterator i = ins.begin(); i != ins.end(); ++i) { string path = i->Evaluate(env); + if (path.empty()) + return lexer_.Error("empty path", err); string path_err; uint64_t slash_bits; if (!CanonicalizePath(&path, &slash_bits, &path_err)) diff --git a/src/ninja.cc b/src/ninja.cc index c7182dff66..d55290c6c8 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -252,6 +252,10 @@ int GuessParallelism() { bool NinjaMain::RebuildManifest(const char* input_file, string* err, Status* status) { string path = input_file; + if (path.empty()) { + *err = "empty path"; + return false; + } uint64_t slash_bits; // Unused because this path is only used for lookup. if (!CanonicalizePath(&path, &slash_bits, err)) return false; @@ -284,6 +288,10 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err, Node* NinjaMain::CollectTarget(const char* cpath, string* err) { string path = cpath; + if (path.empty()) { + *err = "empty path"; + return NULL; + } uint64_t slash_bits; if (!CanonicalizePath(&path, &slash_bits, err)) return NULL; From 91706b323fcb2da9a87656d00ebbaf939ed8dad7 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 3 Jun 2021 17:19:05 -0400 Subject: [PATCH 61/96] util: Remove unnecessary CanonicalizePath error handling Since commit 86f606fe (Remove path component limit from input of CanonicalizePath in windows, 2017-08-30, v1.8.0^2~2^2), the only failure case in the `CanonicalizePath` implementation is the "empty path" error. All call sites have been updated to ensure `CanonicalizePath` is never called with an empty path. Remove error handling from the signature to simplify call sites. --- src/build.cc | 4 +- src/canon_perftest.cc | 3 +- src/clean.cc | 21 ++--- src/clparser.cc | 3 +- src/dyndep_parser.cc | 11 +-- src/graph.cc | 12 +-- src/includes_normalize-win32.cc | 3 +- src/manifest_parser.cc | 20 ++--- src/missing_deps.cc | 4 +- src/ninja.cc | 6 +- src/util.cc | 13 +-- src/util.h | 6 +- src/util_test.cc | 148 +++++++++++++++----------------- 13 files changed, 103 insertions(+), 151 deletions(-) diff --git a/src/build.cc b/src/build.cc index fb5890a76d..cf078461e8 100644 --- a/src/build.cc +++ b/src/build.cc @@ -878,9 +878,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, for (vector::iterator i = deps.ins_.begin(); i != deps.ins_.end(); ++i) { uint64_t slash_bits; - if (!CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits, - err)) - return false; + CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits); deps_nodes->push_back(state_->GetNode(*i, slash_bits)); } diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index 088bd45dfb..6b5e3822d8 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -26,7 +26,6 @@ const char kPath[] = int main() { vector times; - string err; char buf[200]; size_t len = strlen(kPath); @@ -37,7 +36,7 @@ int main() { int64_t start = GetTimeMillis(); uint64_t slash_bits; for (int i = 0; i < kNumRepetitions; ++i) { - CanonicalizePath(buf, &len, &slash_bits, &err); + CanonicalizePath(buf, &len, &slash_bits); } int delta = (int)(GetTimeMillis() - start); times.push_back(delta); diff --git a/src/clean.cc b/src/clean.cc index 1e97182b9a..72dee1faee 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -195,20 +195,15 @@ int Cleaner::CleanTargets(int target_count, char* targets[]) { continue; } uint64_t slash_bits; - string err; - if (!CanonicalizePath(&target_name, &slash_bits, &err)) { - Error("failed to canonicalize '%s': %s", target_name.c_str(), err.c_str()); - status_ = 1; + CanonicalizePath(&target_name, &slash_bits); + Node* target = state_->LookupNode(target_name); + if (target) { + if (IsVerbose()) + printf("Target %s\n", target_name.c_str()); + DoCleanTarget(target); } else { - Node* target = state_->LookupNode(target_name); - if (target) { - if (IsVerbose()) - printf("Target %s\n", target_name.c_str()); - DoCleanTarget(target); - } else { - Error("unknown target '%s'", target_name.c_str()); - status_ = 1; - } + Error("unknown target '%s'", target_name.c_str()); + status_ = 1; } } PrintFooter(); diff --git a/src/clparser.cc b/src/clparser.cc index 275641eaa7..40e9407375 100644 --- a/src/clparser.cc +++ b/src/clparser.cc @@ -103,8 +103,7 @@ bool CLParser::Parse(const string& output, const string& deps_prefix, // TODO: should this make the path relative to cwd? normalized = include; uint64_t slash_bits; - if (!CanonicalizePath(&normalized, &slash_bits, err)) - return false; + CanonicalizePath(&normalized, &slash_bits); #endif if (!IsSystemInclude(normalized)) includes_.insert(normalized); diff --git a/src/dyndep_parser.cc b/src/dyndep_parser.cc index 45d1c31595..1b4dddd337 100644 --- a/src/dyndep_parser.cc +++ b/src/dyndep_parser.cc @@ -117,10 +117,8 @@ bool DyndepParser::ParseEdge(string* err) { string path = out0.Evaluate(&env_); if (path.empty()) return lexer_.Error("empty path", err); - string path_err; uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); Node* node = state_->LookupNode(path); if (!node || !node->in_edge()) return lexer_.Error("no build statement exists for '" + path + "'", err); @@ -206,10 +204,8 @@ bool DyndepParser::ParseEdge(string* err) { string path = i->Evaluate(&env_); if (path.empty()) return lexer_.Error("empty path", err); - string path_err; uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); Node* n = state_->GetNode(path, slash_bits); dyndeps->implicit_inputs_.push_back(n); } @@ -221,8 +217,7 @@ bool DyndepParser::ParseEdge(string* err) { return lexer_.Error("empty path", err); string path_err; uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); Node* n = state_->GetNode(path, slash_bits); dyndeps->implicit_outputs_.push_back(n); } diff --git a/src/graph.cc b/src/graph.cc index 822b7c5bcf..c142f0c155 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -562,11 +562,8 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, uint64_t unused; std::vector::iterator primary_out = depfile.outs_.begin(); - if (!CanonicalizePath(const_cast(primary_out->str_), - &primary_out->len_, &unused, err)) { - *err = path + ": " + *err; - return false; - } + CanonicalizePath(const_cast(primary_out->str_), &primary_out->len_, + &unused); // Check that this depfile matches the edge's output, if not return false to // mark the edge as dirty. @@ -601,10 +598,7 @@ bool ImplicitDepLoader::ProcessDepfileDeps( for (std::vector::iterator i = depfile_ins->begin(); i != depfile_ins->end(); ++i, ++implicit_dep) { uint64_t slash_bits; - if (!CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits, - err)) - return false; - + CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits); Node* node = state_->GetNode(*i, slash_bits); *implicit_dep = node; node->AddOutEdge(edge); diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 9f8dfc2d0c..5d529437f7 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -191,8 +191,7 @@ bool IncludesNormalize::Normalize(const string& input, } strncpy(copy, input.c_str(), input.size() + 1); uint64_t slash_bits; - if (!CanonicalizePath(copy, &len, &slash_bits, err)) - return false; + CanonicalizePath(copy, &len, &slash_bits); StringPiece partially_fixed(copy, len); string abs_input = AbsPath(partially_fixed, err); if (!err->empty()) diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 8f0528a898..521edb4f09 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -192,12 +192,11 @@ bool ManifestParser::ParseDefault(string* err) { string path = eval.Evaluate(env_); if (path.empty()) return lexer_.Error("empty path", err); - string path_err; uint64_t slash_bits; // Unused because this only does lookup. - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); - if (!state_->AddDefault(path, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); + std::string default_err; + if (!state_->AddDefault(path, &default_err)) + return lexer_.Error(default_err, err); eval.Clear(); if (!lexer_.ReadPath(&eval, err)) @@ -321,10 +320,8 @@ bool ManifestParser::ParseEdge(string* err) { string path = outs[i].Evaluate(env); if (path.empty()) return lexer_.Error("empty path", err); - string path_err; uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); if (!state_->AddOut(edge, path, slash_bits)) { if (options_.dupe_edge_action_ == kDupeEdgeActionError) { lexer_.Error("multiple rules generate " + path, err); @@ -355,10 +352,8 @@ bool ManifestParser::ParseEdge(string* err) { string path = i->Evaluate(env); if (path.empty()) return lexer_.Error("empty path", err); - string path_err; uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); state_->AddIn(edge, path, slash_bits); } edge->implicit_deps_ = implicit; @@ -389,8 +384,7 @@ bool ManifestParser::ParseEdge(string* err) { string dyndep = edge->GetUnescapedDyndep(); if (!dyndep.empty()) { uint64_t slash_bits; - if (!CanonicalizePath(&dyndep, &slash_bits, err)) - return false; + CanonicalizePath(&dyndep, &slash_bits); edge->dyndep_ = state_->GetNode(dyndep, slash_bits); edge->dyndep_->set_dyndep_pending(true); vector::iterator dgi = diff --git a/src/missing_deps.cc b/src/missing_deps.cc index eaa3f73cc5..78feb4967f 100644 --- a/src/missing_deps.cc +++ b/src/missing_deps.cc @@ -52,9 +52,7 @@ bool NodeStoringImplicitDepLoader::ProcessDepfileDeps( for (std::vector::iterator i = depfile_ins->begin(); i != depfile_ins->end(); ++i) { uint64_t slash_bits; - if (!CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits, - err)) - return false; + CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits); Node* node = state_->GetNode(*i, slash_bits); dep_nodes_output_->push_back(node); } diff --git a/src/ninja.cc b/src/ninja.cc index d55290c6c8..32cf00eafd 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -257,8 +257,7 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err, return false; } uint64_t slash_bits; // Unused because this path is only used for lookup. - if (!CanonicalizePath(&path, &slash_bits, err)) - return false; + CanonicalizePath(&path, &slash_bits); Node* node = state_.LookupNode(path); if (!node) return false; @@ -293,8 +292,7 @@ Node* NinjaMain::CollectTarget(const char* cpath, string* err) { return NULL; } uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, err)) - return NULL; + CanonicalizePath(&path, &slash_bits); // Special syntax: "foo.cc^" means "the first output of foo.cc". bool first_dependent = false; diff --git a/src/util.cc b/src/util.cc index b40a636df9..46a948d702 100644 --- a/src/util.cc +++ b/src/util.cc @@ -117,16 +117,14 @@ void Info(const char* msg, ...) { va_end(ap); } -bool CanonicalizePath(string* path, uint64_t* slash_bits, string* err) { +void CanonicalizePath(string* path, uint64_t* slash_bits) { METRIC_RECORD("canonicalize str"); size_t len = path->size(); char* str = 0; if (len > 0) str = &(*path)[0]; - if (!CanonicalizePath(str, &len, slash_bits, err)) - return false; + CanonicalizePath(str, &len, slash_bits); path->resize(len); - return true; } static bool IsPathSeparator(char c) { @@ -137,14 +135,12 @@ static bool IsPathSeparator(char c) { #endif } -bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, - string* err) { +void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. METRIC_RECORD("canonicalize path"); if (*len == 0) { - *err = "empty path"; - return false; + return; } const int kMaxPathComponents = 60; @@ -234,7 +230,6 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, #else *slash_bits = 0; #endif - return true; } static inline bool IsKnownShellSafeCharacter(char ch) { diff --git a/src/util.h b/src/util.h index 15414e169f..4a7fea2258 100644 --- a/src/util.h +++ b/src/util.h @@ -64,10 +64,8 @@ void Info(const char* msg, va_list ap); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". /// |slash_bits| has bits set starting from lowest for a backslash that was /// normalized to a forward slash. (only used on Windows) -bool CanonicalizePath(std::string* path, uint64_t* slash_bits, - std::string* err); -bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, - std::string* err); +void CanonicalizePath(std::string* path, uint64_t* slash_bits); +void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits); /// Appends |input| to |*result|, escaping according to the whims of either /// Bash, or Win32's CommandLineToArgvW(). diff --git a/src/util_test.cc b/src/util_test.cc index 1621c91e5b..d58b17088c 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -20,70 +20,69 @@ using namespace std; namespace { -bool CanonicalizePath(string* path, string* err) { +void CanonicalizePath(string* path) { uint64_t unused; - return ::CanonicalizePath(path, &unused, err); + ::CanonicalizePath(path, &unused); } } // namespace TEST(CanonicalizePath, PathSamples) { string path; - string err; - EXPECT_FALSE(CanonicalizePath(&path, &err)); - EXPECT_EQ("empty path", err); + CanonicalizePath(&path); + EXPECT_EQ("", path); - path = "foo.h"; err = ""; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + path = "foo.h"; + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = "./foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = "./foo/./bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar.h", path); path = "./x/foo/../bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("x/bar.h", path); path = "./x/foo/../../bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar.h", path); path = "foo//bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar", path); path = "foo//.//..///bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar", path); path = "./x/../foo/../../bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../bar.h", path); path = "foo/./."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo/bar/.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo/.hidden_bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/.hidden_bar", path); path = "/foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("/foo", path); path = "//foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); #ifdef _WIN32 EXPECT_EQ("//foo", path); #else @@ -91,173 +90,171 @@ TEST(CanonicalizePath, PathSamples) { #endif path = "/"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("", path); path = "/foo/.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("", path); path = "."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ(".", path); path = "./."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ(".", path); path = "foo/.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ(".", path); } #ifdef _WIN32 TEST(CanonicalizePath, PathSamplesWindows) { string path; - string err; - EXPECT_FALSE(CanonicalizePath(&path, &err)); - EXPECT_EQ("empty path", err); + CanonicalizePath(&path); + EXPECT_EQ("", path); - path = "foo.h"; err = ""; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + path = "foo.h"; + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = ".\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = ".\\foo\\.\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar.h", path); path = ".\\x\\foo\\..\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("x/bar.h", path); path = ".\\x\\foo\\..\\..\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar.h", path); path = "foo\\\\bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar", path); path = "foo\\\\.\\\\..\\\\\\bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar", path); path = ".\\x\\..\\foo\\..\\..\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../bar.h", path); path = "foo\\.\\."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo\\bar\\.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo\\.hidden_bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/.hidden_bar", path); path = "\\foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("/foo", path); path = "\\\\foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("//foo", path); path = "\\"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("", path); } TEST(CanonicalizePath, SlashTracking) { string path; - string err; uint64_t slash_bits; - path = "foo.h"; err = ""; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + path = "foo.h"; + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("foo.h", path); EXPECT_EQ(0, slash_bits); path = "a\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a/bcd/efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(4, slash_bits); path = "a\\bcd/efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(5, slash_bits); path = "a\\bcd\\efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(7, slash_bits); path = "a/bcd/efh/foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(0, slash_bits); path = "a\\./efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/efh/foo.h", path); EXPECT_EQ(3, slash_bits); path = "a\\../efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("efh/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\b\\c\\d\\e\\f\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/b/c/d/e/f/g/foo.h", path); EXPECT_EQ(127, slash_bits); path = "a\\b\\c\\..\\..\\..\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("g/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\b/c\\../../..\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("g/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\b/c\\./../..\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/g/foo.h", path); EXPECT_EQ(3, slash_bits); path = "a\\b/c\\./../..\\g/foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/g/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\\\\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a/\\\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(0, slash_bits); path = "a\\//foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(1, slash_bits); } @@ -266,22 +263,20 @@ TEST(CanonicalizePath, CanonicalizeNotExceedingLen) { // Make sure searching \/ doesn't go past supplied len. char buf[] = "foo/bar\\baz.h\\"; // Last \ past end. uint64_t slash_bits; - string err; size_t size = 13; - EXPECT_TRUE(::CanonicalizePath(buf, &size, &slash_bits, &err)); + ::CanonicalizePath(buf, &size, &slash_bits); EXPECT_EQ(0, strncmp("foo/bar/baz.h", buf, size)); EXPECT_EQ(2, slash_bits); // Not including the trailing one. } TEST(CanonicalizePath, TooManyComponents) { string path; - string err; uint64_t slash_bits; // 64 is OK. path = "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./" "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./x.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x0); // Backslashes version. @@ -291,44 +286,40 @@ TEST(CanonicalizePath, TooManyComponents) { "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\x.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0xffffffff); // 65 is OK if #component is less than 60 after path canonicalization. - err = ""; path = "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./" "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./x/y.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x0); // Backslashes version. - err = ""; path = "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\x\\y.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x1ffffffff); // 59 after canonicalization is OK. - err = ""; path = "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/x/y.h"; EXPECT_EQ(58, std::count(path.begin(), path.end(), '/')); - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x0); // Backslashes version. - err = ""; path = "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" "a\\a\\a\\a\\a\\a\\a\\a\\a\\x\\y.h"; EXPECT_EQ(58, std::count(path.begin(), path.end(), '\\')); - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x3ffffffffffffff); } #endif @@ -336,36 +327,35 @@ TEST(CanonicalizePath, TooManyComponents) { TEST(CanonicalizePath, UpDir) { string path, err; path = "../../foo/bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../../foo/bar.h", path); path = "test/../../foo/bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../foo/bar.h", path); } TEST(CanonicalizePath, AbsolutePath) { string path = "/usr/include/stdio.h"; string err; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("/usr/include/stdio.h", path); } TEST(CanonicalizePath, NotNullTerminated) { string path; - string err; size_t len; uint64_t unused; path = "foo/. bar/."; len = strlen("foo/."); // Canonicalize only the part before the space. - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); + CanonicalizePath(&path[0], &len, &unused); EXPECT_EQ(strlen("foo"), len); EXPECT_EQ("foo/. bar/.", string(path)); path = "foo/../file bar/."; len = strlen("foo/../file"); - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); + CanonicalizePath(&path[0], &len, &unused); EXPECT_EQ(strlen("file"), len); EXPECT_EQ("file ./file bar/.", string(path)); } From 0843ae874643a39a42a11a91d1fbb72c93d43d9b Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Thu, 10 Jun 2021 21:25:04 +0200 Subject: [PATCH 62/96] Put Info output back on stdout instead of stderr See comment in #1899. Also adds two tests to output_test.py which check this behaviour by relying on Python's suprocess.check_output not piping stderr. --- misc/output_test.py | 5 +++++ src/util.cc | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) mode change 100755 => 100644 misc/output_test.py diff --git a/misc/output_test.py b/misc/output_test.py old mode 100755 new mode 100644 index 6858e218ed..45698f1e57 --- a/misc/output_test.py +++ b/misc/output_test.py @@ -119,6 +119,7 @@ def test_pr_1685(self): def test_status(self): self.assertEqual(run(''), 'ninja: no work to do.\n') + self.assertEqual(run('', pipe=True), 'ninja: no work to do.\n') def test_ninja_status_default(self): 'Do we show the default status by default?' @@ -129,5 +130,9 @@ def test_ninja_status_quiet(self): output = run(Output.BUILD_SIMPLE_ECHO, flags='--quiet') self.assertEqual(output, 'do thing\n') + def test_entering_directory_on_stdout(self): + output = run(Output.BUILD_SIMPLE_ECHO, flags='-C$PWD', pipe=True) + self.assertEqual(output.splitlines()[0][:25], "ninja: Entering directory") + if __name__ == '__main__': unittest.main() diff --git a/src/util.cc b/src/util.cc index b40a636df9..f34656c44f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -105,9 +105,9 @@ void Error(const char* msg, ...) { } void Info(const char* msg, va_list ap) { - fprintf(stderr, "ninja: "); - vfprintf(stderr, msg, ap); - fprintf(stderr, "\n"); + fprintf(stdout, "ninja: "); + vfprintf(stdout, msg, ap); + fprintf(stdout, "\n"); } void Info(const char* msg, ...) { From cd967ab21eef51819473b0cea22135fead76ffc2 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Thu, 10 Jun 2021 21:30:10 +0200 Subject: [PATCH 63/96] Make output_test.py executable again (mistake thanks to WSL) --- misc/output_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 misc/output_test.py diff --git a/misc/output_test.py b/misc/output_test.py old mode 100644 new mode 100755 From 1fbae7fe9800ff02e5104ae583e1d5166a36e90f Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Sat, 12 Jun 2021 22:50:56 -0700 Subject: [PATCH 64/96] win: Use cl /help to test for /FS requirement --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index ffa75c7f2f..0ca7ec3983 100755 --- a/configure.py +++ b/configure.py @@ -84,7 +84,7 @@ def is_msvc(self): return self._platform == 'msvc' def msvc_needs_fs(self): - popen = subprocess.Popen(['cl', '/nologo', '/?'], + popen = subprocess.Popen(['cl', '/nologo', '/help'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = popen.communicate() From c7b2cf60f4c07440728307c0e68f99c464ac7f7b Mon Sep 17 00:00:00 2001 From: Hans Wennborg Date: Fri, 25 Jun 2021 13:59:34 +0200 Subject: [PATCH 65/96] CLParser: Don't filter filename lines after seeing /showIncludes The /showIncludes output always comes after cl.exe echos the filename, so there is no need to filter out filename lines afterwards. This makes it less likely that CLParser will "over filter" the compiler output. For example, using the -H flag with clang-cl may lead to output lines ending in .cc, which are not supposed to be filtered out. --- src/clparser.cc | 4 +++- src/clparser_test.cc | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/clparser.cc b/src/clparser.cc index 40e9407375..070bcfdc8c 100644 --- a/src/clparser.cc +++ b/src/clparser.cc @@ -83,6 +83,7 @@ bool CLParser::Parse(const string& output, const string& deps_prefix, // Loop over all lines in the output to process them. assert(&output != filtered_output); size_t start = 0; + bool seen_show_includes = false; #ifdef _WIN32 IncludesNormalize normalizer("."); #endif @@ -95,6 +96,7 @@ bool CLParser::Parse(const string& output, const string& deps_prefix, string include = FilterShowIncludes(line, deps_prefix); if (!include.empty()) { + seen_show_includes = true; string normalized; #ifdef _WIN32 if (!normalizer.Normalize(include, &normalized, err)) @@ -107,7 +109,7 @@ bool CLParser::Parse(const string& output, const string& deps_prefix, #endif if (!IsSystemInclude(normalized)) includes_.insert(normalized); - } else if (FilterInputFilename(line)) { + } else if (!seen_show_includes && FilterInputFilename(line)) { // Drop it. // TODO: if we support compiling multiple output files in a single // cl.exe invocation, we should stash the filename. diff --git a/src/clparser_test.cc b/src/clparser_test.cc index 0b829c1cca..f1416809c3 100644 --- a/src/clparser_test.cc +++ b/src/clparser_test.cc @@ -70,6 +70,17 @@ TEST(CLParserTest, ParseFilenameFilter) { ASSERT_EQ("cl: warning\n", output); } +TEST(CLParserTest, NoFilenameFilterAfterShowIncludes) { + CLParser parser; + string output, err; + ASSERT_TRUE(parser.Parse( + "foo.cc\r\n" + "Note: including file: foo.h\r\n" + "something something foo.cc\r\n", + "", &output, &err)); + ASSERT_EQ("something something foo.cc\n", output); +} + TEST(CLParserTest, ParseSystemInclude) { CLParser parser; string output, err; From eba7a50456dfbbba8a3b0db7cc94214e7c105783 Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Wed, 21 Jul 2021 17:38:49 +0200 Subject: [PATCH 66/96] cleandead: Fix the logic to preserve inputs. In the rare case when the build log contains an entry for a file path that used to be an output in a previous build, but was moved as an input in the new one, the cleandead tool would incorrectly consider the input file stale and remove it. This patch fixes the logic used in Cleaner::CleanDead to not remove any path that is an input in the build graph. --- src/clean.cc | 11 ++++++++- src/clean_test.cc | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/clean.cc b/src/clean.cc index 72dee1faee..575bf6bd94 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -129,7 +129,16 @@ int Cleaner::CleanDead(const BuildLog::Entries& entries) { PrintHeader(); for (BuildLog::Entries::const_iterator i = entries.begin(); i != entries.end(); ++i) { Node* n = state_->LookupNode(i->first); - if (!n || !n->in_edge()) { + // Detecting stale outputs works as follows: + // + // - If it has no Node, it is not in the build graph, or the deps log + // anymore, hence is stale. + // + // - If it isn't an output or input for any edge, it comes from a stale + // entry in the deps log, but no longer referenced from the build + // graph. + // + if (!n || (!n->in_edge() && n->out_edges().empty())) { Remove(i->first.AsString()); } } diff --git a/src/clean_test.cc b/src/clean_test.cc index 1b843a28e2..e99909c0d0 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -537,4 +537,65 @@ TEST_F(CleanDeadTest, CleanDead) { EXPECT_NE(0, fs_.Stat("out2", &err)); log2.Close(); } + +TEST_F(CleanDeadTest, CleanDeadPreservesInputs) { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, +"rule cat\n" +" command = cat $in > $out\n" +"build out1: cat in\n" +"build out2: cat in\n" +)); + // This manifest does not build out1 anymore, but makes + // it an implicit input. CleanDead should detect this + // and preserve it. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out2: cat in | out1\n" +)); + fs_.Create("in", ""); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + + BuildLog log1; + string err; + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); + ASSERT_EQ("", err); + log1.RecordCommand(state.edges_[0], 15, 18); + log1.RecordCommand(state.edges_[1], 20, 25); + log1.Close(); + + BuildLog log2; + EXPECT_TRUE(log2.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + ASSERT_EQ(2u, log2.entries().size()); + ASSERT_TRUE(log2.LookupByOutput("out1")); + ASSERT_TRUE(log2.LookupByOutput("out2")); + + // First use the manifest that describe how to build out1. + Cleaner cleaner1(&state, config_, &fs_); + EXPECT_EQ(0, cleaner1.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner1.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Then use the manifest that does not build out1 anymore. + Cleaner cleaner2(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Nothing to do now. + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + log2.Close(); +} } // anonymous namespace From 45751face232a1d72926e83ac7ee51ef25ea03a3 Mon Sep 17 00:00:00 2001 From: Li-Yu Yu Date: Tue, 20 Jul 2021 12:25:45 +0800 Subject: [PATCH 67/96] compdb: escape control characters in JSON strings --- CMakeLists.txt | 2 ++ configure.py | 2 ++ src/json.cc | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ src/json.h | 26 ++++++++++++++++++++++++ src/json_test.cc | 40 ++++++++++++++++++++++++++++++++++++ src/ninja.cc | 18 +++++----------- 6 files changed, 128 insertions(+), 13 deletions(-) create mode 100644 src/json.cc create mode 100644 src/json.h create mode 100644 src/json_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index bc02c4d9c7..b49c5b01f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,6 +103,7 @@ add_library(libninja OBJECT src/eval_env.cc src/graph.cc src/graphviz.cc + src/json.cc src/line_printer.cc src/manifest_parser.cc src/metrics.cc @@ -196,6 +197,7 @@ if(BUILD_TESTING) src/dyndep_parser_test.cc src/edit_distance_test.cc src/graph_test.cc + src/json_test.cc src/lexer_test.cc src/manifest_parser_test.cc src/missing_deps_test.cc diff --git a/configure.py b/configure.py index ffa75c7f2f..4ca78fb404 100755 --- a/configure.py +++ b/configure.py @@ -507,6 +507,7 @@ def has_re2c(): 'eval_env', 'graph', 'graphviz', + 'json', 'lexer', 'line_printer', 'manifest_parser', @@ -577,6 +578,7 @@ def has_re2c(): 'disk_interface_test', 'edit_distance_test', 'graph_test', + 'json_test', 'lexer_test', 'manifest_parser_test', 'missing_deps_test', diff --git a/src/json.cc b/src/json.cc new file mode 100644 index 0000000000..4bbf6e15ad --- /dev/null +++ b/src/json.cc @@ -0,0 +1,53 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "json.h" + +#include +#include + +std::string EncodeJSONString(const std::string& in) { + static const char* hex_digits = "0123456789abcdef"; + std::string out; + out.reserve(in.length() * 1.2); + for (std::string::const_iterator it = in.begin(); it != in.end(); ++it) { + char c = *it; + if (c == '\b') + out += "\\b"; + else if (c == '\f') + out += "\\f"; + else if (c == '\n') + out += "\\n"; + else if (c == '\r') + out += "\\r"; + else if (c == '\t') + out += "\\t"; + else if (0x0 <= c && c < 0x20) { + out += "\\u00"; + out += hex_digits[c >> 4]; + out += hex_digits[c & 0xf]; + } else if (c == '\\') + out += "\\\\"; + else if (c == '\"') + out += "\\\""; + else + out += c; + } + return out; +} + +void PrintJSONString(const std::string& in) { + std::string out = EncodeJSONString(in); + fwrite(out.c_str(), 1, out.length(), stdout); +} diff --git a/src/json.h b/src/json.h new file mode 100644 index 0000000000..f39c759236 --- /dev/null +++ b/src/json.h @@ -0,0 +1,26 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_JSON_H_ +#define NINJA_JSON_H_ + +#include + +// Encode a string in JSON format without encolsing quotes +std::string EncodeJSONString(const std::string& in); + +// Print a string in JSON format to stdout without enclosing quotes +void PrintJSONString(const std::string& in); + +#endif diff --git a/src/json_test.cc b/src/json_test.cc new file mode 100644 index 0000000000..b4afc73695 --- /dev/null +++ b/src/json_test.cc @@ -0,0 +1,40 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "json.h" + +#include "test.h" + +TEST(JSONTest, RegularAscii) { + EXPECT_EQ(EncodeJSONString("foo bar"), "foo bar"); +} + +TEST(JSONTest, EscapedChars) { + EXPECT_EQ(EncodeJSONString("\"\\\b\f\n\r\t"), + "\\\"" + "\\\\" + "\\b\\f\\n\\r\\t"); +} + +// codepoints between 0 and 0x1f should be escaped +TEST(JSONTest, ControlChars) { + EXPECT_EQ(EncodeJSONString("\x01\x1f"), "\\u0001\\u001f"); +} + +// Leave them alone as JSON accepts unicode literals +// out of control character range +TEST(JSONTest, UTF8) { + const char* utf8str = "\xe4\xbd\xa0\xe5\xa5\xbd"; + EXPECT_EQ(EncodeJSONString(utf8str), utf8str); +} diff --git a/src/ninja.cc b/src/ninja.cc index 32cf00eafd..1d94442f1b 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -41,6 +41,7 @@ #include "disk_interface.h" #include "graph.h" #include "graphviz.h" +#include "json.h" #include "manifest_parser.h" #include "metrics.h" #include "missing_deps.h" @@ -773,15 +774,6 @@ int NinjaMain::ToolCleanDead(const Options* options, int argc, char* argv[]) { return cleaner.CleanDead(build_log_.entries()); } -void EncodeJSONString(const char *str) { - while (*str) { - if (*str == '"' || *str == '\\') - putchar('\\'); - putchar(*str); - str++; - } -} - enum EvaluateCommandMode { ECM_NORMAL, ECM_EXPAND_RSPFILE @@ -814,13 +806,13 @@ std::string EvaluateCommandWithRspfile(const Edge* edge, void printCompdb(const char* const directory, const Edge* const edge, const EvaluateCommandMode eval_mode) { printf("\n {\n \"directory\": \""); - EncodeJSONString(directory); + PrintJSONString(directory); printf("\",\n \"command\": \""); - EncodeJSONString(EvaluateCommandWithRspfile(edge, eval_mode).c_str()); + PrintJSONString(EvaluateCommandWithRspfile(edge, eval_mode)); printf("\",\n \"file\": \""); - EncodeJSONString(edge->inputs_[0]->path().c_str()); + PrintJSONString(edge->inputs_[0]->path()); printf("\",\n \"output\": \""); - EncodeJSONString(edge->outputs_[0]->path().c_str()); + PrintJSONString(edge->outputs_[0]->path()); printf("\"\n }"); } From 0bc608077c7cf0d4e5fc84ef0e754d80d934ac64 Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Mon, 26 Jul 2021 16:31:07 -0700 Subject: [PATCH 68/96] Optimize ninja -d stats -d stats enables instrumented profiling of key functions in ninja. However, some of those functions are invoked 6+ million times in a NOP build of Chromium and the cost of measuring those functions dwarfs the cost of the functions. Here is typical -d stats output for a Chromium build: metric count avg (us) total (ms) .ninja parse 6580 4197.5 27619.5 canonicalize str 6240450 0.0 47.3 canonicalize path 6251390 0.0 33.5 lookup node 6339402 0.0 37.2 .ninja_log load 1 176226.0 176.2 .ninja_deps load 1 465407.0 465.4 node stat 168997 8.8 1482.9 depfile load 327 352.7 115.3 99% of the measurements are in three functions. The total measurement cost (per ETW sampled profiling) is 700-1200 ms, which is many times greater than the costs of the functions. With this change typical output looks like this: metric count avg (us) total (ms) .ninja parse 6580 3526.3 23203.2 .ninja_log load 1 227305.0 227.3 .ninja_deps load 1 485693.0 485.7 node stat 168997 9.6 1615.0 depfile load 327 383.1 125.3 This resolves issue #1998. --- src/state.cc | 2 -- src/util.cc | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/state.cc b/src/state.cc index a33d5a88ae..fc37c8a894 100644 --- a/src/state.cc +++ b/src/state.cc @@ -19,7 +19,6 @@ #include "edit_distance.h" #include "graph.h" -#include "metrics.h" #include "util.h" using namespace std; @@ -104,7 +103,6 @@ Node* State::GetNode(StringPiece path, uint64_t slash_bits) { } Node* State::LookupNode(StringPiece path) const { - METRIC_RECORD("lookup node"); Paths::const_iterator i = paths_.find(path); if (i != paths_.end()) return i->second; diff --git a/src/util.cc b/src/util.cc index 3dfa8dd034..080883e066 100644 --- a/src/util.cc +++ b/src/util.cc @@ -56,7 +56,6 @@ #endif #include "edit_distance.h" -#include "metrics.h" using namespace std; @@ -118,7 +117,6 @@ void Info(const char* msg, ...) { } void CanonicalizePath(string* path, uint64_t* slash_bits) { - METRIC_RECORD("canonicalize str"); size_t len = path->size(); char* str = 0; if (len > 0) @@ -138,7 +136,6 @@ static bool IsPathSeparator(char c) { void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. - METRIC_RECORD("canonicalize path"); if (*len == 0) { return; } From 48ed0bb4b433eba43583379336ba13b5489ab2aa Mon Sep 17 00:00:00 2001 From: Richard Geary Date: Sun, 4 Aug 2013 22:36:14 -0400 Subject: [PATCH 69/96] Set output mtime of phony edges to the latest inputs This commit fixes issue #478. Observed: Real edges depending on a phony edge will not be marked as dirty or rebuilt if the phony's (real) inputs are updated. Expected: An edge should always be rebuilt if its inputs or transitive inputs are newer than the output's mtime. Change: Node::mtime_ was overloaded, 0 represented "does not exist". This change disambiguates it by adding Node::exists_. Then to fix the observed behaviour, Node::UpdatePhonyMtime was added to update the mtime if the node does not exist. Add tests BuildTest.PhonyUseCase# GraphTest.PhonyDepsMtimes. Unit tests will also test for always-dirty behaviour if a phony rule has no inputs. --- doc/manual.asciidoc | 5 ++ src/build_test.cc | 145 ++++++++++++++++++++++++++++++++++++++++++++ src/graph.cc | 24 +++++++- src/graph.h | 27 +++++++-- src/graph_test.cc | 37 +++++++++++ 5 files changed, 232 insertions(+), 6 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index a5012b4560..2b948b94b3 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -483,6 +483,11 @@ nothing, but phony rules are handled specially in that they aren't printed when run, logged (see below), nor do they contribute to the command count printed as part of the build process. +When a `phony` target is used as an input to another build rule, the +other build rule will, semantically, consider the inputs of the +`phony` rule as its own. Therefore, `phony` rules can be used to group +inputs, e.g. header files. + `phony` can also be used to create dummy targets for files which may not exist at build time. If a phony build statement is written without any dependencies, the target will be considered out of date if diff --git a/src/build_test.cc b/src/build_test.cc index e0c43b11af..8b6dca247d 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -1182,6 +1182,151 @@ TEST_F(BuildTest, PhonySelfReference) { EXPECT_TRUE(builder_.AlreadyUpToDate()); } +// There are 6 different cases for phony rules: +// +// 1. output edge does not exist, inputs are not real +// 2. output edge does not exist, no inputs +// 3. output edge does not exist, inputs are real, newest mtime is M +// 4. output edge is real, inputs are not real +// 5. output edge is real, no inputs +// 6. output edge is real, inputs are real, newest mtime is M +// +// Expected results : +// 1. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 2. Edge is marked as dirty, causing dependent edges to always rebuild +// 3. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 4. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 5. Edge is marked as dirty, causing dependent edges to always rebuild +// 6. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +void TestPhonyUseCase(BuildTest* t, int i) { + State& state_ = t->state_; + Builder& builder_ = t->builder_; + FakeCommandRunner& command_runner_ = t->command_runner_; + VirtualFileSystem& fs_ = t->fs_; + + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build notreal: phony blank\n" +"build phony1: phony notreal\n" +"build phony2: phony\n" +"build phony3: phony blank\n" +"build phony4: phony notreal\n" +"build phony5: phony\n" +"build phony6: phony blank\n" +"\n" +"build test1: touch phony1\n" +"build test2: touch phony2\n" +"build test3: touch phony3\n" +"build test4: touch phony4\n" +"build test5: touch phony5\n" +"build test6: touch phony6\n" +)); + + // Set up test. + builder_.command_runner_.reset(&command_runner_); + + fs_.Create("blank", ""); // a "real" file + EXPECT_TRUE(builder_.AddTarget("test1", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test3", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test4", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test5", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test6", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + + string ci; + ci += static_cast('0' + i); + + // Tests 1, 3, 4, and 6 should rebuild when the input is updated. + if (i != 2 && i != 5) { + Node* testNode = t->GetNode("test" + ci); + Node* phonyNode = t->GetNode("phony" + ci); + Node* inputNode = t->GetNode("blank"); + + state_.Reset(); + TimeStamp startTime = fs_.now_; + + // Build number 1 + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + if (!builder_.AlreadyUpToDate()) + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + + // Touch the input file + state_.Reset(); + command_runner_.commands_ran_.clear(); + fs_.Tick(); + fs_.Create("blank", ""); // a "real" file + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + + // Second build, expect testN edge to be rebuilt + // and phonyN node's mtime to be updated. + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ(string("touch test") + ci, command_runner_.commands_ran_[0]); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + TimeStamp inputTime = inputNode->mtime(); + + EXPECT_FALSE(phonyNode->exists()); + EXPECT_FALSE(phonyNode->dirty()); + + EXPECT_GT(phonyNode->mtime(), startTime); + EXPECT_EQ(phonyNode->mtime(), inputTime); + ASSERT_TRUE(testNode->Stat(&fs_, &err)); + EXPECT_TRUE(testNode->exists()); + EXPECT_GT(testNode->mtime(), startTime); + } else { + // Tests 2 and 5: Expect dependents to always rebuild. + + state_.Reset(); + command_runner_.commands_ran_.clear(); + fs_.Tick(); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("touch test" + ci, command_runner_.commands_ran_[0]); + + state_.Reset(); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("touch test" + ci, command_runner_.commands_ran_[0]); + } +} + +TEST_F(BuildTest, PhonyUseCase1) { TestPhonyUseCase(this, 1); } +TEST_F(BuildTest, PhonyUseCase2) { TestPhonyUseCase(this, 2); } +TEST_F(BuildTest, PhonyUseCase3) { TestPhonyUseCase(this, 3); } +TEST_F(BuildTest, PhonyUseCase4) { TestPhonyUseCase(this, 4); } +TEST_F(BuildTest, PhonyUseCase5) { TestPhonyUseCase(this, 5); } +TEST_F(BuildTest, PhonyUseCase6) { TestPhonyUseCase(this, 6); } + TEST_F(BuildTest, Fail) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule fail\n" diff --git a/src/graph.cc b/src/graph.cc index 822b7c5bcf..ceeaf13e22 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -31,7 +31,19 @@ using namespace std; bool Node::Stat(DiskInterface* disk_interface, string* err) { - return (mtime_ = disk_interface->Stat(path_, err)) != -1; + METRIC_RECORD("node stat"); + mtime_ = disk_interface->Stat(path_, err); + if (mtime_ == -1) { + return false; + } + exists_ = (mtime_ != 0) ? ExistenceStatusExists : ExistenceStatusMissing; + return true; +} + +void Node::UpdatePhonyMtime(TimeStamp mtime) { + if (!exists()) { + mtime_ = std::max(mtime_, mtime); + } } bool DependencyScan::RecomputeDirty(Node* node, string* err) { @@ -237,6 +249,14 @@ bool DependencyScan::RecomputeOutputDirty(const Edge* edge, output->path().c_str()); return true; } + + // Update the mtime with the newest input. Dependents can thus call mtime() + // on the fake node and get the latest mtime of the dependencies + if (most_recent_input) { + output->UpdatePhonyMtime(most_recent_input->mtime()); + } + + // Phony edges are clean, nothing to do return false; } @@ -487,7 +507,7 @@ string Node::PathDecanonicalized(const string& path, uint64_t slash_bits) { void Node::Dump(const char* prefix) const { printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ", prefix, path().c_str(), this, - mtime(), mtime() ? "" : " (:missing)", + mtime(), exists() ? "" : " (:missing)", dirty() ? " dirty" : " clean"); if (in_edge()) { in_edge()->Dump("in-edge: "); diff --git a/src/graph.h b/src/graph.h index bb4f10c479..fac8059e3e 100644 --- a/src/graph.h +++ b/src/graph.h @@ -15,6 +15,7 @@ #ifndef NINJA_GRAPH_H_ #define NINJA_GRAPH_H_ +#include #include #include #include @@ -40,6 +41,7 @@ struct Node { : path_(path), slash_bits_(slash_bits), mtime_(-1), + exists_(ExistenceStatusUnknown), dirty_(false), dyndep_pending_(false), in_edge_(NULL), @@ -48,6 +50,9 @@ struct Node { /// Return false on error. bool Stat(DiskInterface* disk_interface, std::string* err); + /// If the file doesn't exist, set the mtime_ from its dependencies + void UpdatePhonyMtime(TimeStamp mtime); + /// Return false on error. bool StatIfNecessary(DiskInterface* disk_interface, std::string* err) { if (status_known()) @@ -58,20 +63,24 @@ struct Node { /// Mark as not-yet-stat()ed and not dirty. void ResetState() { mtime_ = -1; + exists_ = ExistenceStatusUnknown; dirty_ = false; } /// Mark the Node as already-stat()ed and missing. void MarkMissing() { - mtime_ = 0; + if (mtime_ == -1) { + mtime_ = 0; + } + exists_ = ExistenceStatusMissing; } bool exists() const { - return mtime_ != 0; + return exists_ == ExistenceStatusExists; } bool status_known() const { - return mtime_ != -1; + return exists_ != ExistenceStatusUnknown; } const std::string& path() const { return path_; } @@ -113,9 +122,19 @@ struct Node { /// Possible values of mtime_: /// -1: file hasn't been examined /// 0: we looked, and file doesn't exist - /// >0: actual file's mtime + /// >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist TimeStamp mtime_; + enum ExistenceStatus { + /// The file hasn't been examined. + ExistenceStatusUnknown, + /// The file doesn't exist. mtime_ will be the latest mtime of its dependencies. + ExistenceStatusMissing, + /// The path is an actual file. mtime_ will be the file's mtime. + ExistenceStatusExists + }; + ExistenceStatus exists_; + /// Dirty is true when the underlying file is out-of-date. /// But note that Edge::outputs_ready_ is also used in judging which /// edges to build. diff --git a/src/graph_test.cc b/src/graph_test.cc index 6b4bb517e8..4f0de9864b 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -889,3 +889,40 @@ TEST_F(GraphTest, DyndepFileCircular) { EXPECT_EQ(1u, edge->implicit_deps_); EXPECT_EQ(1u, edge->order_only_deps_); } + +// Check that phony's dependencies' mtimes are propagated. +TEST_F(GraphTest, PhonyDepsMtimes) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build in_ph: phony in1\n" +"build out1: touch in_ph\n" +)); + fs_.Create("in1", ""); + fs_.Create("out1", ""); + Node* out1 = GetNode("out1"); + Node* in1 = GetNode("in1"); + + EXPECT_TRUE(scan_.RecomputeDirty(out1, &err)); + EXPECT_TRUE(!out1->dirty()); + + // Get the mtime of out1 + ASSERT_TRUE(in1->Stat(&fs_, &err)); + ASSERT_TRUE(out1->Stat(&fs_, &err)); + TimeStamp out1Mtime1 = out1->mtime(); + TimeStamp in1Mtime1 = in1->mtime(); + + // Touch in1. This should cause out1 to be dirty + state_.Reset(); + fs_.Tick(); + fs_.Create("in1", ""); + + ASSERT_TRUE(in1->Stat(&fs_, &err)); + EXPECT_GT(in1->mtime(), in1Mtime1); + + EXPECT_TRUE(scan_.RecomputeDirty(out1, &err)); + EXPECT_GT(in1->mtime(), in1Mtime1); + EXPECT_EQ(out1->mtime(), out1Mtime1); + EXPECT_TRUE(out1->dirty()); +} From a58b058a39aa34f33affac27433e2bc708b32f23 Mon Sep 17 00:00:00 2001 From: Mischa Jonker Date: Mon, 13 Sep 2021 13:30:38 +0200 Subject: [PATCH 70/96] Fix ninja -t clean for directories on Windows remove() deletes both files and directories. On Windows we have to select the correct function (DeleteFile will yield Permission Denied when used on a directory) This fixes the behavior of ninja -t clean in some cases https://github.com/ninja-build/ninja/issues/828 --- src/disk_interface.cc | 31 ++++++++++++++++++++++++------- src/disk_interface_test.cc | 8 ++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/disk_interface.cc b/src/disk_interface.cc index a9497cb8a6..a37c5703ba 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -279,14 +279,31 @@ int RealDiskInterface::RemoveFile(const string& path) { // Skip error checking. If this fails, accept whatever happens below. SetFileAttributes(path.c_str(), attributes & ~FILE_ATTRIBUTE_READONLY); } - if (!DeleteFile(path.c_str())) { - DWORD win_err = GetLastError(); - if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { - return 1; + if (attributes & FILE_ATTRIBUTE_DIRECTORY) { + // remove() deletes both files and directories. On Windows we have to + // select the correct function (DeleteFile will yield Permission Denied when + // used on a directory) + // This fixes the behavior of ninja -t clean in some cases + // https://github.com/ninja-build/ninja/issues/828 + if (!RemoveDirectory(path.c_str())) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + // Report remove(), not RemoveDirectory(), for cross-platform consistency. + Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); + return -1; + } + } else { + if (!DeleteFile(path.c_str())) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + // Report as remove(), not DeleteFile(), for cross-platform consistency. + Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); + return -1; } - // Report as remove(), not DeleteFile(), for cross-platform consistency. - Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); - return -1; } #else if (remove(path.c_str()) < 0) { diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index b424243a94..339aea143d 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -219,6 +219,14 @@ TEST_F(DiskInterfaceTest, RemoveFile) { #endif } +TEST_F(DiskInterfaceTest, RemoveDirectory) { + const char* kDirectoryName = "directory-to-remove"; + EXPECT_TRUE(disk_.MakeDir(kDirectoryName)); + EXPECT_EQ(0, disk_.RemoveFile(kDirectoryName)); + EXPECT_EQ(1, disk_.RemoveFile(kDirectoryName)); + EXPECT_EQ(1, disk_.RemoveFile("does not exist")); +} + struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { StatTest() : scan_(&state_, NULL, NULL, this, NULL) {} From a280868e9c2c791a0d1529c7002786a117bd16fc Mon Sep 17 00:00:00 2001 From: Ronald <3665590+playgithub@users.noreply.github.com> Date: Mon, 20 Sep 2021 15:14:39 +0800 Subject: [PATCH 71/96] Remove -H CMake flag from README.md (#2023) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d11fd3351f..d7637663e7 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ to build Ninja with itself. ### CMake ``` -cmake -Bbuild-cmake -H. +cmake -Bbuild-cmake cmake --build build-cmake ``` From 95d7e865a15e22e516ce7cc681a7f801d6650cfb Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Fri, 1 Oct 2021 14:39:03 -0700 Subject: [PATCH 72/96] Disable stdout buffering in real_main Previously stdout buffering was disabled in the LinePrinter constructor. This worked for a long time but ultimately this side-effect caused a performance bug (issue #2018) in tools such as -t deps. Moving the disabling of buffering into real_main and only disabling buffering when a tool is not used makes the desired semantics clearer and restores the lost performance. This fixes issue #2018. It has been tested and a 10x speedup was seen relative to the tip-of-tree version. --- src/line_printer.cc | 5 ----- src/ninja.cc | 11 +++++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/line_printer.cc b/src/line_printer.cc index 3138960366..a3d0528606 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -37,14 +37,9 @@ LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { #ifndef _WIN32 smart_terminal_ = isatty(1) && term && string(term) != "dumb"; #else - // Disable output buffer. It'd be nice to use line buffering but - // MSDN says: "For some systems, [_IOLBF] provides line - // buffering. However, for Win32, the behavior is the same as _IOFBF - // - Full Buffering." if (term && string(term) == "dumb") { smart_terminal_ = false; } else { - setvbuf(stdout, NULL, _IONBF, 0); console_ = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO csbi; smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi); diff --git a/src/ninja.cc b/src/ninja.cc index 07b1f1e497..3e5c971fae 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -1436,6 +1436,17 @@ NORETURN void real_main(int argc, char** argv) { exit((ninja.*options.tool->func)(&options, argc, argv)); } +#ifdef WIN32 + // It'd be nice to use line buffering but MSDN says: "For some systems, + // [_IOLBF] provides line buffering. However, for Win32, the behavior is the + // same as _IOFBF - Full Buffering." + // Buffering used to be disabled in the LinePrinter constructor but that + // now disables it too early and breaks -t deps performance (see issue #2018) + // so we disable it here instead, but only when not running a tool. + if (!options.tool) + setvbuf(stdout, NULL, _IONBF, 0); +#endif + // Limit number of rebuilds, to prevent infinite loops. const int kCycleLimit = 100; for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { From d2dae79908e890222d4fd4fa81c669c40f102c47 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 21 Apr 2021 10:04:44 -0700 Subject: [PATCH 73/96] Remove early return from Builder::AddTarget Refactor Builder::AddTarget to remove an early return in a non-error case. The next CL will add code that needs to be executed even if the node is clean. Change-Id: I953dc54b60b635dd75d75f8f3931970faefc5ecf --- src/build.cc | 10 +++++----- src/build_test.cc | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/build.cc b/src/build.cc index cf078461e8..735922b8e7 100644 --- a/src/build.cc +++ b/src/build.cc @@ -556,13 +556,13 @@ bool Builder::AddTarget(Node* target, string* err) { if (!scan_.RecomputeDirty(target, err)) return false; - if (Edge* in_edge = target->in_edge()) { - if (in_edge->outputs_ready()) - return true; // Nothing to do. + Edge* in_edge = target->in_edge(); + if (!in_edge || !in_edge->outputs_ready()) { + if (!plan_.AddTarget(target, err)) { + return false; + } } - if (!plan_.AddTarget(target, err)) - return false; return true; } diff --git a/src/build_test.cc b/src/build_test.cc index 8b6dca247d..8da4698339 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -897,6 +897,14 @@ TEST_F(BuildTest, MissingTarget) { EXPECT_EQ("unknown target: 'meow'", err); } +TEST_F(BuildTest, MissingInputTarget) { + // Target is a missing input file + string err; + Dirty("in1"); + EXPECT_FALSE(builder_.AddTarget("in1", &err)); + EXPECT_EQ("'in1' missing and no known rule to make it", err); +} + TEST_F(BuildTest, MakeDirs) { string err; From 04c410b15b70fb321928ffba19d697db15cb0121 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Tue, 9 Jun 2020 18:41:27 -0700 Subject: [PATCH 74/96] Add validation nodes to ninja A common problem in the Android build is inserting rules that perform some sort of error checking that doesn't produce any artifacts needed by the build, for example static analysis tools. There are a few patterns currently used, both of which have downsides. The first is to have a rule that depends on all of the static analysis results. This ensures they run, but requires running static analysis over everything, and not just the active parts of the build graph. The second is to insert the static analysis rule into the build graph between the artifact producing rule and anything that depends on it, often copying the artifact as the output of the static analysis rule. This increases the critical path of the build, often reducing parallelism. In the case of copying the artifact, it also wastes disk space. This patch adds "validation nodes" to edges in Ninja. A build statement can specify validation nodes using "|@" in the edge inputs. The validation nodes are not used as an input to the edge (the edge can run before the validation node is ready), but are added to the initial nodes of the build graph whenever the edge is part of the build graph. The edge that outputs the validation node can depend on the output of the edge that is being validated if desired. Test: ninja_test Change-Id: Ife27086c50c1b257a26509373199664680b2b247 --- doc/manual.asciidoc | 27 ++++ src/build.cc | 29 +++- src/build_test.cc | 305 ++++++++++++++++++++++++++++++++++++ src/depfile_parser.cc | 2 +- src/disk_interface_test.cc | 8 +- src/graph.cc | 66 +++++++- src/graph.h | 15 +- src/graph_test.cc | 82 ++++++---- src/lexer.cc | 225 +++++++++++++------------- src/lexer.h | 1 + src/lexer.in.cc | 2 + src/manifest_parser.cc | 26 ++- src/manifest_parser_test.cc | 10 ++ src/ninja.cc | 18 +++ src/state.cc | 6 + src/state.h | 1 + 16 files changed, 663 insertions(+), 160 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 2b948b94b3..bdfa99b55c 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -748,6 +748,8 @@ A file is a series of declarations. A declaration can be one of: Order-only dependencies may be tacked on the end with +|| _dependency1_ _dependency2_+. (See <>.) + Validations may be taked on the end with +|@ _validation1_ _validation2_+. + (See <>.) + Implicit outputs _(available since Ninja 1.7)_ may be added before the `:` with +| _output1_ _output2_+ and do not appear in `$out`. @@ -1006,6 +1008,31 @@ express the implicit dependency.) File paths are compared as is, which means that an absolute path and a relative path, pointing to the same file, are considered different by Ninja. +[[validations]] +Validations +~~~~~~~~~~~ +Validations listed on the build line cause the specified files to be +added to the top level of the build graph (as if they were specified +on the Ninja command line) whenever the build line is a transitive +dependency of one of the targets specified on the command line or a +default target. + +Validations are added to the build graph regardless of whether the output +files of the build statement are dirty are not, and the dirty state of +the build statement that outputs the file being used as a validation +has no effect on the dirty state of the build statement that requested it. + +A build edge can list another build edge as a validation even if the second +edge depends on the first. + +Validations are designed to handle rules that perform error checking but +don't produce any artifacts needed by the build, for example static +analysis tools. Marking the static analysis rule as an implicit input +of the main build rule of the source files or of the rules that depend +on the main build rule would slow down the critical path of the build, +but using a validation would allow the build to proceed in parallel with +the static analysis rule once the main build rule is complete. + Variable expansion ~~~~~~~~~~~~~~~~~~ diff --git a/src/build.cc b/src/build.cc index 735922b8e7..6f11ed7a3c 100644 --- a/src/build.cc +++ b/src/build.cc @@ -384,8 +384,21 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node, Node* n = *i; // Check if this dependent node is now dirty. Also checks for new cycles. - if (!scan->RecomputeDirty(n, err)) + std::vector validation_nodes; + if (!scan->RecomputeDirty(n, &validation_nodes, err)) return false; + + // Add any validation nodes found during RecomputeDirty as new top level + // targets. + for (std::vector::iterator v = validation_nodes.begin(); + v != validation_nodes.end(); ++v) { + if (Edge* in_edge = (*v)->in_edge()) { + if (!in_edge->outputs_ready() && + !AddTarget(*v, err)) { + return false; + } + } + } if (!n->dirty()) continue; @@ -553,7 +566,8 @@ Node* Builder::AddTarget(const string& name, string* err) { } bool Builder::AddTarget(Node* target, string* err) { - if (!scan_.RecomputeDirty(target, err)) + std::vector validation_nodes; + if (!scan_.RecomputeDirty(target, &validation_nodes, err)) return false; Edge* in_edge = target->in_edge(); @@ -563,6 +577,17 @@ bool Builder::AddTarget(Node* target, string* err) { } } + // Also add any validation nodes found during RecomputeDirty as top level + // targets. + for (std::vector::iterator n = validation_nodes.begin(); + n != validation_nodes.end(); ++n) { + if (Edge* validation_in_edge = (*n)->in_edge()) { + if (!validation_in_edge->outputs_ready() && + !plan_.AddTarget(*n, err)) { + return false; + } + } + } return true; } diff --git a/src/build_test.cc b/src/build_test.cc index 8da4698339..36b723affb 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -3236,6 +3236,67 @@ TEST_F(BuildTest, DyndepBuildDiscoverNewInput) { EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); } +TEST_F(BuildTest, DyndepBuildDiscoverNewInputWithValidation) { + // Verify that a dyndep file cannot contain the |@ validation + // syntax. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep |@ validation\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_FALSE(builder_.Build(&err)); + + string err_first_line = err.substr(0, err.find("\n")); + EXPECT_EQ("dd:2: expected newline, got '|@'", err_first_line); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNewInputWithTransitiveValidation) { + // Verify that a dyndep file can be built and loaded to discover + // a new input to an edge that has a validation edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build in: touch |@ validation\n" +"build validation: touch in out\n" +"build out: touch || dd\n" +" dyndep = dd\n" + )); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep | in\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(4u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch in", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); + EXPECT_EQ("touch validation", command_runner_.commands_ran_[3]); +} + TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) { // Verify that a dyndep file can be built and loaded to discover // that one edge has an implicit output that is also an implicit @@ -3679,3 +3740,247 @@ TEST_F(BuildTest, DyndepTwoLevelDiscoveredDirty) { EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]); EXPECT_EQ("touch out", command_runner_.commands_ran_[4]); } + +TEST_F(BuildTest, Validation) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat in2\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" only rebuilds "out" ("validate" doesn't depend on + // "out"). + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on + // "validate"). + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > validate", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildTest, ValidationDependsOnOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat in2 | out\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" rebuilds "out" and "validate". + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on + // "validate"). + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > validate", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildWithDepsLogTest, ValidationThroughDepfile) { + const char* manifest = + "build out: cat in |@ validate\n" + "build validate: cat in2 | out\n" + "build out2: cat in3\n" + " deps = gcc\n" + " depfile = out2.d\n"; + + string err; + + { + fs_.Create("in", ""); + fs_.Create("in2", ""); + fs_.Create("in3", ""); + fs_.Create("out2.d", "out: out"); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + + EXPECT_TRUE(builder.AddTarget("out2", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + // On the first build, only the out2 command is run. + ASSERT_EQ(command_runner_.commands_ran_.size(), 1); + EXPECT_EQ("cat in3 > out2", command_runner_.commands_ran_[0]); + + // The deps file should have been removed. + EXPECT_EQ(0, fs_.Stat("out2.d", &err)); + + deps_log.Close(); + builder.command_runner_.release(); + } + + fs_.Tick(); + command_runner_.commands_ran_.clear(); + + { + fs_.Create("in2", ""); + fs_.Create("in3", ""); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + + EXPECT_TRUE(builder.AddTarget("out2", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + // The out and validate actions should have been run as well as out2. + ASSERT_EQ(command_runner_.commands_ran_.size(), 3); + // out has to run first, as both out2 and validate depend on it. + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + deps_log.Close(); + builder.command_runner_.release(); + } +} + +TEST_F(BuildTest, ValidationCircular) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ out2\n" + "build out2: cat in2 |@ out\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" rebuilds "out". + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + // Test touching "in2" rebuilds "out2". + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > out2", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildTest, ValidationWithCircularDependency) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat validate_in | out\n" + "build validate_in: cat validate\n")); + + fs_.Create("in", ""); + + string err; + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err); +} diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index bffeb76c40..2eca108789 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 1.3 */ +/* Generated by re2c 2.2 */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 339aea143d..5e952edde5 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -272,7 +272,7 @@ TEST_F(StatTest, Simple) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(2u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_EQ("in", stats_[1]); @@ -288,7 +288,7 @@ TEST_F(StatTest, TwoStep) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(3u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_TRUE(GetNode("out")->dirty()); @@ -308,7 +308,7 @@ TEST_F(StatTest, Tree) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(1u + 6u, stats_.size()); ASSERT_EQ("mid1", stats_[1]); ASSERT_TRUE(GetNode("mid1")->dirty()); @@ -329,7 +329,7 @@ TEST_F(StatTest, Middle) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_FALSE(GetNode("in")->dirty()); ASSERT_TRUE(GetNode("mid")->dirty()); ASSERT_TRUE(GetNode("out")->dirty()); diff --git a/src/graph.cc b/src/graph.cc index c875d3b271..8d58587784 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -15,6 +15,7 @@ #include "graph.h" #include +#include #include #include @@ -46,13 +47,42 @@ void Node::UpdatePhonyMtime(TimeStamp mtime) { } } -bool DependencyScan::RecomputeDirty(Node* node, string* err) { - vector stack; - return RecomputeDirty(node, &stack, err); +bool DependencyScan::RecomputeDirty(Node* initial_node, + std::vector* validation_nodes, + string* err) { + std::vector stack; + std::vector new_validation_nodes; + + std::deque nodes(1, initial_node); + + // RecomputeNodeDirty might return new validation nodes that need to be + // checked for dirty state, keep a queue of nodes to visit. + while (!nodes.empty()) { + Node* node = nodes.front(); + nodes.pop_front(); + + stack.clear(); + new_validation_nodes.clear(); + + if (!RecomputeNodeDirty(node, &stack, &new_validation_nodes, err)) + return false; + nodes.insert(nodes.end(), new_validation_nodes.begin(), + new_validation_nodes.end()); + if (!new_validation_nodes.empty()) { + assert(validation_nodes && + "validations require RecomputeDirty to be called with validation_nodes"); + validation_nodes->insert(validation_nodes->end(), + new_validation_nodes.begin(), + new_validation_nodes.end()); + } + } + + return true; } -bool DependencyScan::RecomputeDirty(Node* node, vector* stack, - string* err) { +bool DependencyScan::RecomputeNodeDirty(Node* node, std::vector* stack, + std::vector* validation_nodes, + string* err) { Edge* edge = node->in_edge(); if (!edge) { // If we already visited this leaf node then we are done. @@ -96,7 +126,7 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, // Later during the build the dyndep file will become ready and be // loaded to update this edge before it can possibly be scheduled. if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) { - if (!RecomputeDirty(edge->dyndep_, stack, err)) + if (!RecomputeNodeDirty(edge->dyndep_, stack, validation_nodes, err)) return false; if (!edge->dyndep_->in_edge() || @@ -127,12 +157,20 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, } } + // Store any validation nodes from the edge for adding to the initial + // nodes. Don't recurse into them, that would trigger the dependency + // cycle detector if the validation node depends on this node. + // RecomputeDirty will add the validation nodes to the initial nodes + // and recurse into them. + validation_nodes->insert(validation_nodes->end(), + edge->validations_.begin(), edge->validations_.end()); + // Visit all inputs; we're dirty if any of the inputs are dirty. Node* most_recent_input = NULL; for (vector::iterator i = edge->inputs_.begin(); i != edge->inputs_.end(); ++i) { // Visit this input. - if (!RecomputeDirty(*i, stack, err)) + if (!RecomputeNodeDirty(*i, stack, validation_nodes, err)) return false; // If an input is not ready, neither are our outputs. @@ -463,6 +501,13 @@ void Edge::Dump(const char* prefix) const { i != outputs_.end() && *i != NULL; ++i) { printf("%s ", (*i)->path().c_str()); } + if (!validations_.empty()) { + printf(" validations "); + for (std::vector::const_iterator i = validations_.begin(); + i != validations_.end() && *i != NULL; ++i) { + printf("%s ", (*i)->path().c_str()); + } + } if (pool_) { if (!pool_->name().empty()) { printf("(in pool '%s')", pool_->name().c_str()); @@ -519,6 +564,13 @@ void Node::Dump(const char* prefix) const { e != out_edges().end() && *e != NULL; ++e) { (*e)->Dump(" +- "); } + if (!validation_out_edges().empty()) { + printf(" validation out edges:\n"); + for (std::vector::const_iterator e = validation_out_edges().begin(); + e != validation_out_edges().end() && *e != NULL; ++e) { + (*e)->Dump(" +- "); + } + } } bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { diff --git a/src/graph.h b/src/graph.h index fac8059e3e..141b43993e 100644 --- a/src/graph.h +++ b/src/graph.h @@ -108,7 +108,9 @@ struct Node { void set_id(int id) { id_ = id; } const std::vector& out_edges() const { return out_edges_; } + const std::vector& validation_out_edges() const { return validation_out_edges_; } void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); } + void AddValidationOutEdge(Edge* edge) { validation_out_edges_.push_back(edge); } void Dump(const char* prefix="") const; @@ -151,6 +153,9 @@ struct Node { /// All Edges that use this Node as an input. std::vector out_edges_; + /// All Edges that use this Node as a validation. + std::vector validation_out_edges_; + /// A dense integer id for the node, assigned and used by DepsLog. int id_; }; @@ -194,6 +199,7 @@ struct Edge { Pool* pool_; std::vector inputs_; std::vector outputs_; + std::vector validations_; Node* dyndep_; BindingEnv* env_; VisitMark mark_; @@ -309,12 +315,14 @@ struct DependencyScan { dep_loader_(state, deps_log, disk_interface, depfile_parser_options), dyndep_loader_(state, disk_interface) {} - /// Update the |dirty_| state of the given node by inspecting its input edge. + /// Update the |dirty_| state of the given nodes by transitively inspecting + /// their input edges. /// Examine inputs, outputs, and command lines to judge whether an edge /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| /// state accordingly. + /// Appends any validation nodes found to the nodes parameter. /// Returns false on failure. - bool RecomputeDirty(Node* node, std::string* err); + bool RecomputeDirty(Node* node, std::vector* validation_nodes, std::string* err); /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. /// Returns false on failure. @@ -340,7 +348,8 @@ struct DependencyScan { bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const; private: - bool RecomputeDirty(Node* node, std::vector* stack, std::string* err); + bool RecomputeNodeDirty(Node* node, std::vector* stack, + std::vector* validation_nodes, std::string* err); bool VerifyDAG(Node* node, std::vector* stack, std::string* err); /// Recompute whether a given single output should be marked dirty. diff --git a/src/graph_test.cc b/src/graph_test.cc index 4f0de9864b..5314bc5f5f 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -33,7 +33,7 @@ TEST_F(GraphTest, MissingImplicit) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); // A missing implicit dep *should* make the output dirty. @@ -51,7 +51,7 @@ TEST_F(GraphTest, ModifiedImplicit) { fs_.Create("implicit", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); // A modified implicit dep should make the output dirty. @@ -71,7 +71,7 @@ TEST_F(GraphTest, FunkyMakefilePath) { fs_.Create("implicit.h", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); // implicit.h has changed, though our depfile refers to it with a @@ -94,7 +94,7 @@ TEST_F(GraphTest, ExplicitImplicit) { fs_.Create("data", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); // We have both an implicit and an explicit dep on implicit.h. @@ -122,7 +122,7 @@ TEST_F(GraphTest, ImplicitOutputMissing) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out")->dirty()); @@ -138,7 +138,7 @@ TEST_F(GraphTest, ImplicitOutputOutOfDate) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out")->dirty()); @@ -162,7 +162,7 @@ TEST_F(GraphTest, ImplicitOutputOnlyMissing) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.imp")->dirty()); @@ -176,7 +176,7 @@ TEST_F(GraphTest, ImplicitOutputOnlyOutOfDate) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.imp")->dirty()); @@ -193,7 +193,7 @@ TEST_F(GraphTest, PathWithCurrentDirectory) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -241,7 +241,7 @@ TEST_F(GraphTest, DepfileWithCanonicalizablePath) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -261,13 +261,13 @@ TEST_F(GraphTest, DepfileRemoved) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); state_.Reset(); fs_.RemoveFile("out.o.d"); - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.o")->dirty()); } @@ -314,7 +314,7 @@ TEST_F(GraphTest, NestedPhonyPrintsDone) { "build n2: phony n1\n" ); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), NULL, &err)); ASSERT_EQ("", err); Plan plan_; @@ -333,7 +333,7 @@ TEST_F(GraphTest, PhonySelfReferenceError) { parser_opts); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a [-w phonycycle=err]", err); } @@ -345,7 +345,7 @@ TEST_F(GraphTest, DependencyCycle) { "build pre: cat out\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } @@ -353,7 +353,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes1) { string err; AssertParse(&state_, "build a b: cat a\n"); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a", err); } @@ -361,7 +361,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes2) { string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build b a: cat a\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a", err); } @@ -370,7 +370,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes3) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build a b: cat c\n" "build c: cat a\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> c -> a", err); } @@ -382,7 +382,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes4) { "build b: cat a\n" "build a e: cat d\n" "build f: cat e\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> d -> c -> b -> a", err); } @@ -398,7 +398,7 @@ TEST_F(GraphTest, CycleWithLengthZeroFromDepfile) { fs_.Create("dep.d", "a: b\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> b", err); // Despite the depfile causing edge to be a cycle (it has outputs a and b, @@ -423,7 +423,7 @@ TEST_F(GraphTest, CycleWithLengthOneFromDepfile) { fs_.Create("dep.d", "a: c\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> c -> b", err); // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, @@ -450,7 +450,7 @@ TEST_F(GraphTest, CycleWithLengthOneFromDepfileOneHopAway) { fs_.Create("dep.d", "a: c\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> c -> b", err); // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, @@ -705,7 +705,7 @@ TEST_F(GraphTest, DyndepFileMissing) { ); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("loading 'dd': No such file or directory", err); } @@ -721,7 +721,7 @@ TEST_F(GraphTest, DyndepFileError) { ); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("'out' not mentioned in its dyndep file 'dd'", err); } @@ -741,7 +741,7 @@ TEST_F(GraphTest, DyndepImplicitInputNewer) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("in")->dirty()); @@ -769,7 +769,7 @@ TEST_F(GraphTest, DyndepFileReady) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("in")->dirty()); @@ -794,7 +794,7 @@ TEST_F(GraphTest, DyndepFileNotClean) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("dd")->dirty()); @@ -820,7 +820,7 @@ TEST_F(GraphTest, DyndepFileNotReady) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("dd")->dirty()); @@ -848,7 +848,7 @@ TEST_F(GraphTest, DyndepFileSecondNotReady) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("dd1")->dirty()); @@ -877,7 +877,7 @@ TEST_F(GraphTest, DyndepFileCircular) { Edge* edge = GetNode("out")->in_edge(); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); EXPECT_EQ("dependency cycle: circ -> in -> circ", err); // Verify that "out.d" was loaded exactly once despite @@ -890,6 +890,24 @@ TEST_F(GraphTest, DyndepFileCircular) { EXPECT_EQ(1u, edge->order_only_deps_); } +TEST_F(GraphTest, Validation) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out: cat in |@ validate\n" +"build validate: cat in\n")); + + fs_.Create("in", ""); + string err; + std::vector validation_nodes; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &validation_nodes, &err)); + ASSERT_EQ("", err); + + ASSERT_EQ(validation_nodes.size(), 1); + EXPECT_EQ(validation_nodes[0]->path(), "validate"); + + EXPECT_TRUE(GetNode("out")->dirty()); + EXPECT_TRUE(GetNode("validate")->dirty()); +} + // Check that phony's dependencies' mtimes are propagated. TEST_F(GraphTest, PhonyDepsMtimes) { string err; @@ -904,7 +922,7 @@ TEST_F(GraphTest, PhonyDepsMtimes) { Node* out1 = GetNode("out1"); Node* in1 = GetNode("in1"); - EXPECT_TRUE(scan_.RecomputeDirty(out1, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(out1, NULL, &err)); EXPECT_TRUE(!out1->dirty()); // Get the mtime of out1 @@ -921,7 +939,7 @@ TEST_F(GraphTest, PhonyDepsMtimes) { ASSERT_TRUE(in1->Stat(&fs_, &err)); EXPECT_GT(in1->mtime(), in1Mtime1); - EXPECT_TRUE(scan_.RecomputeDirty(out1, &err)); + EXPECT_TRUE(scan_.RecomputeDirty(out1, NULL, &err)); EXPECT_GT(in1->mtime(), in1Mtime1); EXPECT_EQ(out1->mtime(), out1Mtime1); EXPECT_TRUE(out1->dirty()); diff --git a/src/lexer.cc b/src/lexer.cc index 6e4a470245..af861ae322 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 1.1.1 */ +/* Generated by re2c 2.2 */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -85,6 +85,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case PIPEAT: return "'|@'"; case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; @@ -291,7 +292,8 @@ Lexer::Token Lexer::ReadToken() { goto yy14; yy26: yych = *++p; - if (yych == '|') goto yy42; + if (yych == '@') goto yy42; + if (yych == '|') goto yy44; { token = PIPE; break; } yy28: ++p; @@ -317,126 +319,129 @@ Lexer::Token Lexer::ReadToken() { { continue; } yy36: yych = *++p; - if (yych == 'i') goto yy44; + if (yych == 'i') goto yy46; goto yy14; yy37: yych = *++p; - if (yych == 'f') goto yy45; + if (yych == 'f') goto yy47; goto yy14; yy38: yych = *++p; - if (yych == 'c') goto yy46; + if (yych == 'c') goto yy48; goto yy14; yy39: yych = *++p; - if (yych == 'o') goto yy47; + if (yych == 'o') goto yy49; goto yy14; yy40: yych = *++p; - if (yych == 'l') goto yy48; + if (yych == 'l') goto yy50; goto yy14; yy41: yych = *++p; - if (yych == 'b') goto yy49; + if (yych == 'b') goto yy51; goto yy14; yy42: ++p; - { token = PIPE2; break; } + { token = PIPEAT; break; } yy44: - yych = *++p; - if (yych == 'l') goto yy50; - goto yy14; -yy45: - yych = *++p; - if (yych == 'a') goto yy51; - goto yy14; + ++p; + { token = PIPE2; break; } yy46: yych = *++p; if (yych == 'l') goto yy52; goto yy14; yy47: yych = *++p; - if (yych == 'l') goto yy53; + if (yych == 'a') goto yy53; goto yy14; yy48: yych = *++p; - if (yych == 'e') goto yy55; + if (yych == 'l') goto yy54; goto yy14; yy49: yych = *++p; - if (yych == 'n') goto yy57; + if (yych == 'l') goto yy55; goto yy14; yy50: yych = *++p; - if (yych == 'd') goto yy58; + if (yych == 'e') goto yy57; goto yy14; yy51: yych = *++p; - if (yych == 'u') goto yy60; + if (yych == 'n') goto yy59; goto yy14; yy52: yych = *++p; - if (yych == 'u') goto yy61; + if (yych == 'd') goto yy60; goto yy14; yy53: + yych = *++p; + if (yych == 'u') goto yy62; + goto yy14; +yy54: + yych = *++p; + if (yych == 'u') goto yy63; + goto yy14; +yy55: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = POOL; break; } -yy55: +yy57: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = RULE; break; } -yy57: +yy59: yych = *++p; - if (yych == 'i') goto yy62; + if (yych == 'i') goto yy64; goto yy14; -yy58: +yy60: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = BUILD; break; } -yy60: - yych = *++p; - if (yych == 'l') goto yy63; - goto yy14; -yy61: - yych = *++p; - if (yych == 'd') goto yy64; - goto yy14; yy62: yych = *++p; - if (yych == 'n') goto yy65; + if (yych == 'l') goto yy65; goto yy14; yy63: yych = *++p; - if (yych == 't') goto yy66; + if (yych == 'd') goto yy66; goto yy14; yy64: yych = *++p; - if (yych == 'e') goto yy68; + if (yych == 'n') goto yy67; goto yy14; yy65: yych = *++p; - if (yych == 'j') goto yy70; + if (yych == 't') goto yy68; goto yy14; yy66: + yych = *++p; + if (yych == 'e') goto yy70; + goto yy14; +yy67: + yych = *++p; + if (yych == 'j') goto yy72; + goto yy14; +yy68: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = DEFAULT; break; } -yy68: +yy70: yych = *++p; if (yybm[0+yych] & 64) { goto yy13; } { token = INCLUDE; break; } -yy70: +yy72: yych = *++p; if (yych != 'a') goto yy14; yych = *++p; @@ -507,38 +512,38 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yybm[0+yych] & 128) { - goto yy79; + goto yy81; } - if (yych <= 0x00) goto yy75; - if (yych == '$') goto yy82; - goto yy77; -yy75: - ++p; - { break; } + if (yych <= 0x00) goto yy77; + if (yych == '$') goto yy84; + goto yy79; yy77: ++p; -yy78: { break; } yy79: + ++p; +yy80: + { break; } +yy81: yych = *++p; if (yybm[0+yych] & 128) { - goto yy79; + goto yy81; } { continue; } -yy82: +yy84: yych = *(q = ++p); - if (yych == '\n') goto yy83; - if (yych == '\r') goto yy85; - goto yy78; -yy83: + if (yych == '\n') goto yy85; + if (yych == '\r') goto yy87; + goto yy80; +yy85: ++p; { continue; } -yy85: +yy87: yych = *++p; - if (yych == '\n') goto yy87; + if (yych == '\n') goto yy89; p = q; - goto yy78; -yy87: + goto yy80; +yy89: ++p; { continue; } } @@ -590,17 +595,17 @@ bool Lexer::ReadIdent(string* out) { }; yych = *p; if (yybm[0+yych] & 128) { - goto yy93; + goto yy95; } ++p; { last_token_ = start; return false; } -yy93: +yy95: yych = *++p; if (yybm[0+yych] & 128) { - goto yy93; + goto yy95; } { out->assign(start, p - start); @@ -660,33 +665,33 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { }; yych = *p; if (yybm[0+yych] & 16) { - goto yy100; + goto yy102; } if (yych <= '\r') { - if (yych <= 0x00) goto yy98; - if (yych <= '\n') goto yy103; - goto yy105; + if (yych <= 0x00) goto yy100; + if (yych <= '\n') goto yy105; + goto yy107; } else { - if (yych <= ' ') goto yy103; - if (yych <= '$') goto yy107; - goto yy103; + if (yych <= ' ') goto yy105; + if (yych <= '$') goto yy109; + goto yy105; } -yy98: +yy100: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy100: +yy102: yych = *++p; if (yybm[0+yych] & 16) { - goto yy100; + goto yy102; } { eval->AddText(StringPiece(start, p - start)); continue; } -yy103: +yy105: ++p; { if (path) { @@ -699,112 +704,112 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { continue; } } -yy105: +yy107: yych = *++p; - if (yych == '\n') goto yy108; + if (yych == '\n') goto yy110; { last_token_ = start; return Error(DescribeLastError(), err); } -yy107: +yy109: yych = *++p; if (yybm[0+yych] & 64) { - goto yy120; + goto yy122; } if (yych <= ' ') { if (yych <= '\f') { - if (yych == '\n') goto yy112; - goto yy110; + if (yych == '\n') goto yy114; + goto yy112; } else { - if (yych <= '\r') goto yy115; - if (yych <= 0x1F) goto yy110; - goto yy116; + if (yych <= '\r') goto yy117; + if (yych <= 0x1F) goto yy112; + goto yy118; } } else { if (yych <= '/') { - if (yych == '$') goto yy118; - goto yy110; + if (yych == '$') goto yy120; + goto yy112; } else { - if (yych <= ':') goto yy123; - if (yych <= '`') goto yy110; - if (yych <= '{') goto yy125; - goto yy110; + if (yych <= ':') goto yy125; + if (yych <= '`') goto yy112; + if (yych <= '{') goto yy127; + goto yy112; } } -yy108: +yy110: ++p; { if (path) p = start; break; } -yy110: +yy112: ++p; -yy111: +yy113: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy112: +yy114: yych = *++p; if (yybm[0+yych] & 32) { - goto yy112; + goto yy114; } { continue; } -yy115: +yy117: yych = *++p; - if (yych == '\n') goto yy126; - goto yy111; -yy116: + if (yych == '\n') goto yy128; + goto yy113; +yy118: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy118: +yy120: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy120: +yy122: yych = *++p; if (yybm[0+yych] & 64) { - goto yy120; + goto yy122; } { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy123: +yy125: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy125: +yy127: yych = *(q = ++p); if (yybm[0+yych] & 128) { - goto yy129; + goto yy131; } - goto yy111; -yy126: + goto yy113; +yy128: yych = *++p; - if (yych == ' ') goto yy126; + if (yych == ' ') goto yy128; { continue; } -yy129: +yy131: yych = *++p; if (yybm[0+yych] & 128) { - goto yy129; + goto yy131; } - if (yych == '}') goto yy132; + if (yych == '}') goto yy134; p = q; - goto yy111; -yy132: + goto yy113; +yy134: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); diff --git a/src/lexer.h b/src/lexer.h index 788d9488c6..683fd6c6ab 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -41,6 +41,7 @@ struct Lexer { NEWLINE, PIPE, PIPE2, + PIPEAT, POOL, RULE, SUBNINJA, diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 88007e7bb6..6f1d8e7937 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -84,6 +84,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case PIPEAT: return "'|@'"; case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; @@ -142,6 +143,7 @@ Lexer::Token Lexer::ReadToken() { "default" { token = DEFAULT; break; } "=" { token = EQUALS; break; } ":" { token = COLON; break; } + "|@" { token = PIPEAT; break; } "||" { token = PIPE2; break; } "|" { token = PIPE; break; } "include" { token = INCLUDE; break; } diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 521edb4f09..8db6eb3009 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -207,7 +207,7 @@ bool ManifestParser::ParseDefault(string* err) { } bool ManifestParser::ParseEdge(string* err) { - vector ins, outs; + vector ins, outs, validations; { EvalString out; @@ -288,6 +288,18 @@ bool ManifestParser::ParseEdge(string* err) { } } + // Add all validations, counting how many as we go. + if (lexer_.PeekToken(Lexer::PIPEAT)) { + for (;;) { + EvalString validation; + if (!lexer_.ReadPath(&validation, err)) + return false; + if (validation.empty()) + break; + validations.push_back(validation); + } + } + if (!ExpectToken(Lexer::NEWLINE, err)) return false; @@ -338,6 +350,7 @@ bool ManifestParser::ParseEdge(string* err) { } } } + if (edge->outputs_.empty()) { // All outputs of the edge are already created by other edges. Don't add // this edge. Do this check before input nodes are connected to the edge. @@ -359,6 +372,17 @@ bool ManifestParser::ParseEdge(string* err) { edge->implicit_deps_ = implicit; edge->order_only_deps_ = order_only; + edge->validations_.reserve(validations.size()); + for (std::vector::iterator v = validations.begin(); + v != validations.end(); ++v) { + string path = v->Evaluate(env); + if (path.empty()) + return lexer_.Error("empty path", err); + uint64_t slash_bits; + CanonicalizePath(&path, &slash_bits); + state_->AddValidation(edge, path, slash_bits); + } + if (options_.phony_cycle_action_ == kPhonyCycleActionWarn && edge->maybe_phonycycle_diagnostic()) { // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 5b0eddf7c4..66b72e2d4a 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -965,6 +965,16 @@ TEST_F(ParserTest, OrderOnly) { ASSERT_TRUE(edge->is_order_only(1)); } +TEST_F(ParserTest, Validations) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n command = cat $in > $out\n" +"build foo: cat bar |@ baz\n")); + + Edge* edge = state.LookupNode("foo")->in_edge(); + ASSERT_EQ(edge->validations_.size(), 1); + EXPECT_EQ(edge->validations_[0]->path(), "baz"); +} + TEST_F(ParserTest, ImplicitOutput) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule cat\n" diff --git a/src/ninja.cc b/src/ninja.cc index 3e5c971fae..89580ae94f 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -404,6 +404,13 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { label = "|| "; printf(" %s%s\n", label, edge->inputs_[in]->path().c_str()); } + if (!edge->validations_.empty()) { + printf(" validations:\n"); + for (std::vector::iterator validation = edge->validations_.begin(); + validation != edge->validations_.end(); ++validation) { + printf(" %s\n", (*validation)->path().c_str()); + } + } } printf(" outputs:\n"); for (vector::const_iterator edge = node->out_edges().begin(); @@ -413,6 +420,17 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { printf(" %s\n", (*out)->path().c_str()); } } + const std::vector validation_edges = node->validation_out_edges(); + if (!validation_edges.empty()) { + printf(" validation for:\n"); + for (std::vector::const_iterator edge = validation_edges.begin(); + edge != validation_edges.end(); ++edge) { + for (vector::iterator out = (*edge)->outputs_.begin(); + out != (*edge)->outputs_.end(); ++out) { + printf(" %s\n", (*out)->path().c_str()); + } + } + } } return 0; } diff --git a/src/state.cc b/src/state.cc index fc37c8a894..556b0d8802 100644 --- a/src/state.cc +++ b/src/state.cc @@ -141,6 +141,12 @@ bool State::AddOut(Edge* edge, StringPiece path, uint64_t slash_bits) { return true; } +void State::AddValidation(Edge* edge, StringPiece path, uint64_t slash_bits) { + Node* node = GetNode(path, slash_bits); + edge->validations_.push_back(node); + node->AddValidationOutEdge(edge); +} + bool State::AddDefault(StringPiece path, string* err) { Node* node = LookupNode(path); if (!node) { diff --git a/src/state.h b/src/state.h index 72c5b33667..878ac6d991 100644 --- a/src/state.h +++ b/src/state.h @@ -107,6 +107,7 @@ struct State { void AddIn(Edge* edge, StringPiece path, uint64_t slash_bits); bool AddOut(Edge* edge, StringPiece path, uint64_t slash_bits); + void AddValidation(Edge* edge, StringPiece path, uint64_t slash_bits); bool AddDefault(StringPiece path, std::string* error); /// Reset state. Keeps all nodes and edges, but restores them to the From 7222513f47e41eff30d3f267e4c1ce10ea859fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?okhowang=28=E7=8E=8B=E6=B2=9B=E6=96=87=29?= Date: Tue, 18 Aug 2020 12:04:16 +0800 Subject: [PATCH 75/96] feat: support cpu limit by job api on windows --- src/util.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/util.cc b/src/util.cc index 080883e066..2e85513c8d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -500,6 +500,7 @@ string StripAnsiEscapeCodes(const string& in) { int GetProcessorCount() { #ifdef _WIN32 + DWORD cpuCount = 0; #ifndef _WIN64 // Need to use GetLogicalProcessorInformationEx to get real core count on // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 @@ -524,12 +525,25 @@ int GetProcessorCount() { i += info->Size; } if (cores != 0) { - return cores; + cpuCount = cores; } } } #endif - return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); + if (cpuCount == 0) { + cpuCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); + } + JOBOBJECT_CPU_RATE_CONTROL_INFORMATION info; + // reference: + // https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_cpu_rate_control_information + if (QueryInformationJobObject(NULL, JobObjectCpuRateControlInformation, &info, + sizeof(info), NULL)) { + if (info.ControlFlags & (JOB_OBJECT_CPU_RATE_CONTROL_ENABLE | + JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP)) { + return cpuCount * info.CpuRate / 10000; + } + } + return cpuCount; #else // The number of exposed processors might not represent the actual number of // processors threads can run on. This happens when a CPU set limitation is From 540be336f5639ee6a89e959e6f9f434c01900ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?okhowang=28=E7=8E=8B=E6=B2=9B=E6=96=87=29?= Date: Tue, 18 Aug 2020 14:38:04 +0800 Subject: [PATCH 76/96] feat: support cpu limit by cgroups on linux --- src/util.cc | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 2 deletions(-) diff --git a/src/util.cc b/src/util.cc index 2e85513c8d..287debc12e 100644 --- a/src/util.cc +++ b/src/util.cc @@ -49,6 +49,9 @@ #include #elif defined(linux) || defined(__GLIBC__) #include +#include +#include +#include "string_piece_util.h" #endif #if defined(__FreeBSD__) @@ -498,6 +501,155 @@ string StripAnsiEscapeCodes(const string& in) { return stripped; } +#if defined(linux) || defined(__GLIBC__) +std::pair readCount(const std::string& path) { + std::ifstream file(path.c_str()); + if (!file.is_open()) + return std::make_pair(0, false); + int64_t n = 0; + file >> n; + if (file.good()) + return std::make_pair(n, true); + return std::make_pair(0, false); +} + +struct MountPoint { + int mountId; + int parentId; + StringPiece deviceId; + StringPiece root; + StringPiece mountPoint; + vector options; + vector optionalFields; + StringPiece fsType; + StringPiece mountSource; + vector superOptions; + bool parse(const string& line) { + vector pieces = SplitStringPiece(line, ' '); + if (pieces.size() < 10) + return false; + size_t optionalStart = 0; + for (size_t i = 6; i < pieces.size(); i++) { + if (pieces[i] == "-") { + optionalStart = i + 1; + break; + } + } + if (optionalStart == 0) + return false; + if (optionalStart + 3 != pieces.size()) + return false; + mountId = atoi(pieces[0].AsString().c_str()); + parentId = atoi(pieces[1].AsString().c_str()); + deviceId = pieces[2]; + root = pieces[3]; + mountPoint = pieces[4]; + options = SplitStringPiece(pieces[5], ','); + optionalFields = + vector(&pieces[6], &pieces[optionalStart - 1]); + fsType = pieces[optionalStart]; + mountSource = pieces[optionalStart + 1]; + superOptions = SplitStringPiece(pieces[optionalStart + 2], ','); + return true; + } + string translate(string& path) const { + // path must be sub dir of root + if (path.compare(0, root.len_, root.str_, root.len_) != 0) { + return string(); + } + path.erase(0, root.len_); + if (path == ".." || (path.length() > 2 && path.compare(0, 3, "../") == 0)) { + return string(); + } + return mountPoint.AsString() + "/" + path; + } +}; + +struct CGroupSubSys { + int id; + string name; + vector subsystems; + bool parse(string& line) { + size_t first = line.find(':'); + if (first == string::npos) + return false; + line[first] = '\0'; + size_t second = line.find(':', first + 1); + if (second == string::npos) + return false; + line[second] = '\0'; + id = atoi(line.c_str()); + name = line.substr(second + 1); + vector pieces = + SplitStringPiece(StringPiece(line.c_str() + first + 1), ','); + for (size_t i = 0; i < pieces.size(); i++) { + subsystems.push_back(pieces[i].AsString()); + } + return true; + } +}; + +map ParseMountInfo(map& subsystems) { + map cgroups; + ifstream mountinfo("/proc/self/mountinfo"); + if (!mountinfo.is_open()) + return cgroups; + while (!mountinfo.eof()) { + string line; + getline(mountinfo, line); + MountPoint mp; + if (!mp.parse(line)) + continue; + if (mp.fsType != "cgroup") + continue; + for (size_t i = 0; i < mp.superOptions.size(); i++) { + string opt = mp.superOptions[i].AsString(); + map::iterator subsys = subsystems.find(opt); + if (subsys == subsystems.end()) + continue; + string newPath = mp.translate(subsys->second.name); + if (!newPath.empty()) + cgroups.insert(make_pair(opt, newPath)); + } + } + return cgroups; +} + +map ParseSelfCGroup() { + map cgroups; + ifstream cgroup("/proc/self/cgroup"); + if (!cgroup.is_open()) + return cgroups; + string line; + while (!cgroup.eof()) { + getline(cgroup, line); + CGroupSubSys subsys; + if (!subsys.parse(line)) + continue; + for (size_t i = 0; i < subsys.subsystems.size(); i++) { + cgroups.insert(make_pair(subsys.subsystems[i], subsys)); + } + } + return cgroups; +} + +int ParseCPUFromCGroup() { + map subsystems = ParseSelfCGroup(); + map cgroups = ParseMountInfo(subsystems); + map::iterator cpu = cgroups.find("cpu"); + if (cpu == cgroups.end()) + return -1; + std::pair quota = readCount(cpu->second + "/cpu.cfs_quota_us"); + if (!quota.second || quota.first == -1) + return -1; + std::pair period = + readCount(cpu->second + "/cpu.cfs_period_us"); + if (!period.second) + return -1; + return quota.first / period.first; +} +#endif + int GetProcessorCount() { #ifdef _WIN32 DWORD cpuCount = 0; @@ -545,6 +697,11 @@ int GetProcessorCount() { } return cpuCount; #else + int cgroupCount = -1; + int schedCount = -1; +#if defined(linux) || defined(__GLIBC__) + cgroupCount = ParseCPUFromCGroup(); +#endif // The number of exposed processors might not represent the actual number of // processors threads can run on. This happens when a CPU set limitation is // active, see https://github.com/ninja-build/ninja/issues/1278 @@ -558,10 +715,12 @@ int GetProcessorCount() { #elif defined(CPU_COUNT) cpu_set_t set; if (sched_getaffinity(getpid(), sizeof(set), &set) == 0) { - return CPU_COUNT(&set); + schedCount = CPU_COUNT(&set); } #endif - return sysconf(_SC_NPROCESSORS_ONLN); + if (cgroupCount >= 0 && schedCount >= 0) return std::min(cgroupCount, schedCount); + if (cgroupCount < 0 && schedCount < 0) return sysconf(_SC_NPROCESSORS_ONLN); + return std::max(cgroupCount, schedCount); #endif } From 2187403594b951c50bfb9ffe7fedaf4178e9455a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?okhowang=28=E7=8E=8B=E6=B2=9B=E6=96=87=29?= Date: Mon, 24 Aug 2020 14:14:07 +0800 Subject: [PATCH 77/96] feat: don't detect cpu if -j set --- src/ninja.cc | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ninja.cc b/src/ninja.cc index 3e5c971fae..2d43b1d953 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -1305,11 +1305,28 @@ int ExceptionFilter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { #endif // _MSC_VER +class DeferGuessParallelism { + public: + bool needGuess; + BuildConfig* config; + + DeferGuessParallelism(BuildConfig* config) + : config(config), needGuess(true) {} + + void Refresh() { + if (needGuess) { + needGuess = false; + config->parallelism = GuessParallelism(); + } + } + ~DeferGuessParallelism() { Refresh(); } +}; + /// Parse argv for command-line options. /// Returns an exit code, or -1 if Ninja should continue. int ReadFlags(int* argc, char*** argv, Options* options, BuildConfig* config) { - config->parallelism = GuessParallelism(); + DeferGuessParallelism deferGuessParallelism(config); enum { OPT_VERSION = 1, OPT_QUIET = 2 }; const option kLongOptions[] = { @@ -1341,6 +1358,7 @@ int ReadFlags(int* argc, char*** argv, // We want to run N jobs in parallel. For N = 0, INT_MAX // is close enough to infinite for most sane builds. config->parallelism = value > 0 ? value : INT_MAX; + deferGuessParallelism.needGuess = false; break; } case 'k': { @@ -1389,6 +1407,7 @@ int ReadFlags(int* argc, char*** argv, return 0; case 'h': default: + deferGuessParallelism.Refresh(); Usage(*config); return 1; } From c0f1c0fc085007780e6a451c0d5fce26ea28f499 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Wed, 3 Nov 2021 10:26:11 -0400 Subject: [PATCH 78/96] minidump-win32 is needed by native GNU-like Clang on Windows The entire minidump-win32.cc is guarded by ifdef _MSC_VER, so MSYS2/MinGW will just ignore the contents. fixes #2006 --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b49c5b01f6..fe1e03f132 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,10 +122,8 @@ if(WIN32) src/msvc_helper-win32.cc src/msvc_helper_main-win32.cc src/getopt.c + src/minidump-win32.cc ) - if(MSVC) - target_sources(libninja PRIVATE src/minidump-win32.cc) - endif() else() target_sources(libninja PRIVATE src/subprocess-posix.cc) if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") From c7da3e64f97ef61f9201eff7ad70a8a652f506bb Mon Sep 17 00:00:00 2001 From: Arthur Milchior Date: Sun, 7 Nov 2021 18:00:27 +0100 Subject: [PATCH 79/96] NF: clarify documentation's part about rules description The word "eventually" was quite strange here and should probably removed. I suspect "potentially" was meant (which in French is "eventuellement", a standard false friend). In any way, after a quick look at the source/ninja message, I chose to even clarify this part of the doc, indicating that the `-d` option is required here. --- doc/manual.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 2b948b94b3..ce36d9bfb1 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -304,9 +304,9 @@ _Available since Ninja 1.11._ `restat`:: updates all recorded file modification timestamps in the `.ninja_log` file. _Available since Ninja 1.10._ -`rules`:: output the list of all rules (eventually with their description -if they have one). It can be used to know which rule name to pass to -+ninja -t targets rule _name_+ or +ninja -t compdb+. +`rules`:: output the list of all rules. It can be used to know which rule name +to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. Adding the `-d` +flag also prints the description of the rules. `wincodepage`:: Available on Windows hosts (_since Ninja 1.11_). Prints the Windows code page whose encoding is expected in the build file. From 112f85e1536b82d5f0bd84ceea5df4634151dfa9 Mon Sep 17 00:00:00 2001 From: Dimitris Apostolou Date: Wed, 3 Nov 2021 19:59:22 +0200 Subject: [PATCH 80/96] Fix typo --- doc/manual.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 2b948b94b3..67f89a1ede 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -294,7 +294,7 @@ of a generator-target) implicitly, but does not have an explicit or order-only dependency path to the generator-target, is considered broken. + The tool's findings can be verified by trying to build the listed targets in -a clean outdir without buidling any other targets. The build should fail for +a clean outdir without building any other targets. The build should fail for each of them with a missing include error or equivalent pointing to the generated file. _Available since Ninja 1.11._ From 2dab655765499e9b05b8f097ab446ddee8f9adca Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 11 Nov 2021 09:53:53 +0100 Subject: [PATCH 81/96] xrange() was removed in Python 3 in favor of range() (#2036) https://portingguide.readthedocs.io --- misc/write_fake_manifests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py index b3594de0bb..abcb677e18 100644 --- a/misc/write_fake_manifests.py +++ b/misc/write_fake_manifests.py @@ -65,7 +65,7 @@ def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1): def _n_unique_strings(self, n): seen = set([None]) return [self._unique_string(seen, avg_options=3, p_suffix=0.4) - for _ in xrange(n)] + for _ in range(n)] def target_name(self): return self._unique_string(p_suffix=0, seen=self.seen_names) @@ -73,7 +73,7 @@ def target_name(self): def path(self): return os.path.sep.join([ self._unique_string(self.seen_names, avg_options=1, p_suffix=0) - for _ in xrange(1 + paretoint(0.6, alpha=4))]) + for _ in range(1 + paretoint(0.6, alpha=4))]) def src_obj_pairs(self, path, name): num_sources = paretoint(55, alpha=2) + 1 @@ -84,7 +84,7 @@ def src_obj_pairs(self, path, name): def defines(self): return [ '-DENABLE_' + self._unique_string(self.seen_defines).upper() - for _ in xrange(paretoint(20, alpha=3))] + for _ in range(paretoint(20, alpha=3))] LIB, EXE = 0, 1 @@ -227,7 +227,7 @@ def random_targets(num_targets, src_dir): gen = GenRandom(src_dir) # N-1 static libraries, and 1 executable depending on all of them. - targets = [Target(gen, LIB) for i in xrange(num_targets - 1)] + targets = [Target(gen, LIB) for i in range(num_targets - 1)] for i in range(len(targets)): targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05] From 102ef25fdf27504f3ac7392ce8dbfe83485fb25a Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 12 Nov 2021 09:39:02 +0100 Subject: [PATCH 82/96] Fix typos discovered by codespell (#2047) % [`codespell --count --ignore-words-list=fo .`](https://pypi.org/project/codespell/) ``` ./ninja/src/missing_deps.cc:119: dependecy ==> dependency ./ninja/src/includes_normalize-win32.cc:51: funcation ==> function ./ninja/src/eval_env.h:58: invokable ==> invocable ./ninja/src/missing_deps_test.cc:155: beacuse ==> because ./ninja/src/deps_log_test.cc:393: unparseable ==> unparsable ``` --- .github/workflows/linux.yml | 3 +++ src/deps_log_test.cc | 2 +- src/eval_env.h | 2 +- src/includes_normalize-win32.cc | 2 +- src/missing_deps.cc | 2 +- src/missing_deps_test.cc | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 80c88c6387..3c93e00745 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -13,6 +13,9 @@ jobs: image: centos:7 steps: - uses: actions/checkout@v2 + - uses: codespell-project/actions-codespell@master + with: + ignore_words_list: fo,wee - name: Install dependencies run: | curl -L -O https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-Linux-x86_64.sh diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index 1c29d89e5e..13fcc788b6 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -390,7 +390,7 @@ TEST_F(DepsLogTest, Truncated) { DepsLog log; EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); if (!err.empty()) { - // At some point the log will be so short as to be unparseable. + // At some point the log will be so short as to be unparsable. break; } diff --git a/src/eval_env.h b/src/eval_env.h index ca7daa4d75..677dc217a2 100644 --- a/src/eval_env.h +++ b/src/eval_env.h @@ -55,7 +55,7 @@ struct EvalString { TokenList parsed_; }; -/// An invokable build command and associated metadata (description, etc.). +/// An invocable build command and associated metadata (description, etc.). struct Rule { explicit Rule(const std::string& name) : name_(name) {} diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 5d529437f7..081e364ac3 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -48,7 +48,7 @@ bool IsPathSeparator(char c) { } // Return true if paths a and b are on the same windows drive. -// Return false if this funcation cannot check +// Return false if this function cannot check // whether or not on the same windows drive. bool SameDriveFast(StringPiece a, StringPiece b) { if (a.size() < 3 || b.size() < 3) { diff --git a/src/missing_deps.cc b/src/missing_deps.cc index 78feb4967f..de76620ac8 100644 --- a/src/missing_deps.cc +++ b/src/missing_deps.cc @@ -116,7 +116,7 @@ void MissingDependencyScanner::ProcessNodeDeps(Node* node, Node** dep_nodes, // rebuild this target when the build is reconfigured", but build.ninja is // often generated by a configuration tool like cmake or gn. The rest of // the build "implicitly" depends on the entire build being reconfigured, - // so a missing dep path to build.ninja is not an actual missing dependecy + // so a missing dep path to build.ninja is not an actual missing dependency // problem. if (deplog_node->path() == "build.ninja") return; diff --git a/src/missing_deps_test.cc b/src/missing_deps_test.cc index 7b62e6cc48..db66885d07 100644 --- a/src/missing_deps_test.cc +++ b/src/missing_deps_test.cc @@ -152,7 +152,7 @@ TEST_F(MissingDependencyScannerTest, CycleInGraph) { CreateInitialState(); CreateGraphDependencyBetween("compiled_object", "generated_header"); CreateGraphDependencyBetween("generated_header", "compiled_object"); - // The missing-deps tool doesn't deal with cycles in the graph, beacuse + // The missing-deps tool doesn't deal with cycles in the graph, because // there will be an error loading the graph before we get to the tool. // This test is to illustrate that. std::string err; From e5935b63757f3a788bc56d2c7afd9e390daf2f07 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Tue, 16 Nov 2021 14:53:41 +0100 Subject: [PATCH 83/96] Fix crash when FakeCommandRunner is deleted twice --- src/build_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/build_test.cc b/src/build_test.cc index 8b6dca247d..f6e12158b2 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -1229,6 +1229,7 @@ void TestPhonyUseCase(BuildTest* t, int i) { )); // Set up test. + builder_.command_runner_.release(); // BuildTest owns the CommandRunner builder_.command_runner_.reset(&command_runner_); fs_.Create("blank", ""); // a "real" file From 5339c62bd63691d8030c6195cc83c50d5ccd9282 Mon Sep 17 00:00:00 2001 From: Steve Wills Date: Tue, 23 Nov 2021 13:49:05 -0500 Subject: [PATCH 84/96] Suggest possible cause for hitting cycle limit --- src/ninja.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ninja.cc b/src/ninja.cc index 3e5c971fae..8e5a44abc2 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -1497,7 +1497,7 @@ NORETURN void real_main(int argc, char** argv) { exit(result); } - status->Error("manifest '%s' still dirty after %d tries", + status->Error("manifest '%s' still dirty after %d tries, perhaps system time is not set", options.input_file, kCycleLimit); exit(1); } From 4a13d59820bf215232c12a34c1e0bef9e31fb04d Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Tue, 21 Dec 2021 19:35:22 +0100 Subject: [PATCH 85/96] Use correct version number for inputs tool --- doc/manual.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index cdff9c0c7d..55f16d8c48 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -259,7 +259,7 @@ output files are out of date. `inputs`:: given a list of targets, print a list of all inputs which are used to rebuild those targets. -_Available since Ninja 1.10._ +_Available since Ninja 1.11._ `clean`:: remove built files. By default it removes all built files except for those created by the generator. Adding the `-g` flag also From 4f363609e04cf842930c2d5276f0ad93c2945e7b Mon Sep 17 00:00:00 2001 From: Marc-Antoine Ruel Date: Tue, 28 Dec 2021 15:12:48 -0500 Subject: [PATCH 86/96] Fix two typos One in ninja_test usage text, the other in a unit test comment. --- src/build_test.cc | 2 +- src/ninja_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build_test.cc b/src/build_test.cc index 326a7d633b..4ef62b2113 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -2355,7 +2355,7 @@ struct BuildWithDepsLogTest : public BuildTest { void* builder_; }; -/// Run a straightforwad build where the deps log is used. +/// Run a straightforward build where the deps log is used. TEST_F(BuildWithDepsLogTest, Straightforward) { string err; // Note: in1 was created by the superclass SetUp(). diff --git a/src/ninja_test.cc b/src/ninja_test.cc index b40e1769c6..6720dec4ff 100644 --- a/src/ninja_test.cc +++ b/src/ninja_test.cc @@ -67,7 +67,7 @@ void Usage() { "usage: ninja_tests [options]\n" "\n" "options:\n" -" --gtest_filter=POSTIVE_PATTERN[-NEGATIVE_PATTERN]\n" +" --gtest_filter=POSITIVE_PATTERN[-NEGATIVE_PATTERN]\n" " Run tests whose names match the positive but not the negative pattern.\n" " '*' matches any substring. (gtest's ':', '?' are not implemented).\n"); } From 054ca7569a42904a11694f222da6a6db7d74da0a Mon Sep 17 00:00:00 2001 From: Marc-Antoine Ruel Date: Fri, 31 Dec 2021 07:53:35 -0500 Subject: [PATCH 87/96] Disable re2c from embedding the version number in the source file The checked in sources (depfile_parser.cc and lexer.cc) have the re2c version embedded in it. The output didn't change since re2c 1.3, except for the different version number. A reproducible build would use a pinned version of re2c anyway, so there is no need to hardcode its value in the checked in generated files. --- CMakeLists.txt | 2 +- configure.py | 2 +- src/depfile_parser.cc | 2 +- src/lexer.cc | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe1e03f132..57ae548f5b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ if(RE2C) # the depfile parser and ninja lexers are generated using re2c. function(re2c IN OUT) add_custom_command(DEPENDS ${IN} OUTPUT ${OUT} - COMMAND ${RE2C} -b -i --no-generation-date -o ${OUT} ${IN} + COMMAND ${RE2C} -b -i --no-generation-date --no-version -o ${OUT} ${IN} ) endfunction() re2c(${PROJECT_SOURCE_DIR}/src/depfile_parser.in.cc ${PROJECT_BINARY_DIR}/depfile_parser.cc) diff --git a/configure.py b/configure.py index e0a5a22d97..43904349a8 100755 --- a/configure.py +++ b/configure.py @@ -479,7 +479,7 @@ def has_re2c(): return False if has_re2c(): n.rule('re2c', - command='re2c -b -i --no-generation-date -o $out $in', + command='re2c -b -i --no-generation-date --no-version -o $out $in', description='RE2C $out') # Generate the .cc files in the source directory so we can check them in. n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 2eca108789..98fba2eda6 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 2.2 */ +/* Generated by re2c */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/lexer.cc b/src/lexer.cc index af861ae322..e5729f00a0 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 2.2 */ +/* Generated by re2c */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); From 1a99b250c9ddb78ada2785b50a969580ef3dd4ee Mon Sep 17 00:00:00 2001 From: "DESKTOP-8CNEPM9\\sune" Date: Wed, 26 Jan 2022 08:57:47 +0100 Subject: [PATCH 88/96] Filter lines ending with ".c++" in clparser Projects like cap n proto uses c++ extensions and generates a lot of status noise when running ninja. --- src/clparser.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clparser.cc b/src/clparser.cc index 070bcfdc8c..3d3e7de54e 100644 --- a/src/clparser.cc +++ b/src/clparser.cc @@ -72,7 +72,8 @@ bool CLParser::FilterInputFilename(string line) { return EndsWith(line, ".c") || EndsWith(line, ".cc") || EndsWith(line, ".cxx") || - EndsWith(line, ".cpp"); + EndsWith(line, ".cpp") || + EndsWith(line, ".c++"); } // static From 1f5653358813e2dc2656910d2de7ed9d355bad3f Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Wed, 16 Feb 2022 13:56:26 -1000 Subject: [PATCH 89/96] Don't disable stdout buffering on Windows Long ago ninja disabled stdout buffering. Since then much has changed and this appears to no longer be needed, while also being actively harmful. Disabling of stdout buffering is not needed because printing is done (in LinePrinter::Print) either with WriteConsoleOutput (which is unaffected by stdout buffering) or by printf followed by fflush. Meanwhile, the unbuffered printing in the printf case causes dramatic slowdowns which are most visible in dry-run builds, as documented in issue #2084. This fixes issue #2084, finishing off the incomplete buffering fix done in pull request #2031. --- src/ninja.cc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index df39ba92d1..71dea210df 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -1486,17 +1486,6 @@ NORETURN void real_main(int argc, char** argv) { exit((ninja.*options.tool->func)(&options, argc, argv)); } -#ifdef WIN32 - // It'd be nice to use line buffering but MSDN says: "For some systems, - // [_IOLBF] provides line buffering. However, for Win32, the behavior is the - // same as _IOFBF - Full Buffering." - // Buffering used to be disabled in the LinePrinter constructor but that - // now disables it too early and breaks -t deps performance (see issue #2018) - // so we disable it here instead, but only when not running a tool. - if (!options.tool) - setvbuf(stdout, NULL, _IONBF, 0); -#endif - // Limit number of rebuilds, to prevent infinite loops. const int kCycleLimit = 100; for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { From 9116613e39ba4e18f50494e97eb968a874effdcf Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Fri, 18 Feb 2022 15:00:07 +0100 Subject: [PATCH 90/96] Fix ReadFile() handle leak on read error on Windows. Small change to fix a file handle leak in case of error. Also return -EIO instead of -1 to signal read errors, since it is more consistent with what is happening here. --- src/util.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util.cc b/src/util.cc index 080883e066..a2a0f278d8 100644 --- a/src/util.cc +++ b/src/util.cc @@ -350,7 +350,8 @@ int ReadFile(const string& path, string* contents, string* err) { if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) { err->assign(GetLastErrorString()); contents->clear(); - return -1; + ::CloseHandle(f); + return -EIO; } if (len == 0) break; From 2dd343b8649c8834d42421d5664d8241cabd8f8a Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Sun, 20 Feb 2022 20:29:38 -0500 Subject: [PATCH 91/96] cmake: remove unnecessary install parameter The install(TARGETS ... DESTINATION bin) is not necessary as this subdirectory is implicit for executable targets. ref: https://cmake.org/cmake/help/v3.15/command/install.html --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 57ae548f5b..70fc5e99f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -232,4 +232,4 @@ if(BUILD_TESTING) add_test(NAME NinjaTest COMMAND ninja_test) endif() -install(TARGETS ninja DESTINATION bin) +install(TARGETS ninja) From f0fd305a5772fe06be9627ddcdcf2950449900b9 Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Sat, 5 Mar 2022 15:45:11 +0100 Subject: [PATCH 92/96] Ignore Visual Studios folders --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index fdca01513f..ca36ec8810 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ /.clangd/ /compile_commands.json /.cache/ + +# Visual Studio files +/.vs/ +/out/ From 7923d736c108cf19e15aa53f5a3fa30582530abb Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Tue, 15 Feb 2022 12:38:58 +0100 Subject: [PATCH 93/96] Document the `msvc` tool --- doc/manual.asciidoc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 9e2bec5d49..00b293cd66 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -312,6 +312,37 @@ file. _Available since Ninja 1.10._ to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. Adding the `-d` flag also prints the description of the rules. +`msvc`:: Available on Windows hosts only. +Helper tool to invoke the `cl.exe` compiler with a pre-defined set of +environment variables, as in: ++ +---- +ninja -t msvc -e ENVFILE -- cl.exe +---- ++ +Where `ENVFILE` is a binary file that contains an environment block suitable +for CreateProcessA() on Windows (i.e. a series of zero-terminated strings that +look like NAME=VALUE, followed by an extra zero terminator). Note that this uses +the local codepage encoding. + +This tool also supports a deprecated way of parsing the compiler's output when +the `/showIncludes` flag is used, and generating a GCC-compatible depfile from it. ++ +--- +ninja -t msvc -o DEPFILE [-p STRING] -- cl.exe /showIncludes +--- ++ + +When using this option, `-p STRING` can be used to pass the localized line prefix +that `cl.exe` uses to output dependency information. For English-speaking regions +this is `"Note: including file: "` without the double quotes, but will be different +for other regions. + +Note that Ninja supports this natively now, with the use of `deps = msvc` and +`msvc_deps_prefix` in Ninja files. Native support also avoids launching an extra +tool process each time the compiler must be called, which can speed up builds +noticeably on Windows. + `wincodepage`:: Available on Windows hosts (_since Ninja 1.11_). Prints the Windows code page whose encoding is expected in the build file. The output has the form: From 66b05496ff0ef6114e3534418c970dd92d34accb Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Fri, 18 Feb 2022 15:05:10 +0100 Subject: [PATCH 94/96] Ensure the `msvc` tool is built in all Win32 Ninja binaries. It was only previously available when Ninja was built when `_MSVC` is defined (i.e. when compiling with the Microsoft compiler of with `clang-cl`). + Tag the tool as DEPRECATED --- src/ninja.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ninja.cc b/src/ninja.cc index 71dea210df..ad0912ecf5 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -52,7 +52,7 @@ using namespace std; -#ifdef _MSC_VER +#ifdef _WIN32 // Defined in msvc_helper_main-win32.cc. int MSVCHelperMain(int argc, char** argv); @@ -449,7 +449,7 @@ int NinjaMain::ToolBrowse(const Options*, int, char**) { } #endif -#if defined(_MSC_VER) +#if defined(_WIN32) int NinjaMain::ToolMSVC(const Options* options, int argc, char* argv[]) { // Reset getopt: push one argument onto the front of argv, reset optind. argc++; @@ -1043,8 +1043,8 @@ const Tool* ChooseTool(const string& tool_name) { static const Tool kTools[] = { { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolBrowse }, -#if defined(_MSC_VER) - { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)", +#ifdef _WIN32 + { "msvc", "build helper for MSVC cl.exe (DEPRECATED)", Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolMSVC }, #endif { "clean", "clean built files", From 988c847ee728de5520d07051843c02a17dd44777 Mon Sep 17 00:00:00 2001 From: David 'Digit' Turner Date: Mon, 21 Feb 2022 19:32:35 +0100 Subject: [PATCH 95/96] Make the output of `ninja -t inputs` deterministic This sorts the output of `ninja -t inputs` to make it deterministic and remove duplicates, and adds a regression test in output_test.py + Ensure all inputs are listed, not only explicit ones. + Document the `inputs` tool in doc/manual.asciidoc. --- doc/manual.asciidoc | 4 ++-- misc/output_test.py | 18 ++++++++++++++++ src/graph.cc | 22 +++++++++++++++++++ src/graph.h | 3 +++ src/graph_test.cc | 33 +++++++++++++++++++++++++++++ src/ninja.cc | 51 +++++++++++++++++++++++++++++++++++++++------ 6 files changed, 123 insertions(+), 8 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 00b293cd66..2062a2a210 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -257,8 +257,8 @@ than the _depth_ mode. executed in order, may be used to rebuild those targets, assuming that all output files are out of date. -`inputs`:: given a list of targets, print a list of all inputs which are used -to rebuild those targets. +`inputs`:: given a list of targets, print a list of all inputs used to +rebuild those targets. _Available since Ninja 1.11._ `clean`:: remove built files. By default it removes all built files diff --git a/misc/output_test.py b/misc/output_test.py index 45698f1e57..141716c136 100755 --- a/misc/output_test.py +++ b/misc/output_test.py @@ -134,5 +134,23 @@ def test_entering_directory_on_stdout(self): output = run(Output.BUILD_SIMPLE_ECHO, flags='-C$PWD', pipe=True) self.assertEqual(output.splitlines()[0][:25], "ninja: Entering directory") + def test_tool_inputs(self): + plan = ''' +rule cat + command = cat $in $out +build out1 : cat in1 +build out2 : cat in2 out1 +build out3 : cat out2 out1 | implicit || order_only +''' + self.assertEqual(run(plan, flags='-t inputs out3'), +'''implicit +in1 +in2 +order_only +out1 +out2 +''') + + if __name__ == '__main__': unittest.main() diff --git a/src/graph.cc b/src/graph.cc index 8d58587784..43ba45ae3d 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -456,6 +456,28 @@ std::string EdgeEnv::MakePathList(const Node* const* const span, return result; } +void Edge::CollectInputs(bool shell_escape, + std::vector* out) const { + for (std::vector::const_iterator it = inputs_.begin(); + it != inputs_.end(); ++it) { + std::string path = (*it)->PathDecanonicalized(); + if (shell_escape) { + std::string unescaped; + unescaped.swap(path); +#ifdef _WIN32 + GetWin32EscapedString(unescaped, &path); +#else + GetShellEscapedString(unescaped, &path); +#endif + } +#if __cplusplus >= 201103L + out->push_back(std::move(path)); +#else + out->push_back(path); +#endif + } +} + std::string Edge::EvaluateCommand(const bool incl_rsp_file) const { string command = GetBinding("command"); if (incl_rsp_file) { diff --git a/src/graph.h b/src/graph.h index 141b43993e..9de67d2718 100644 --- a/src/graph.h +++ b/src/graph.h @@ -195,6 +195,9 @@ struct Edge { void Dump(const char* prefix="") const; + // Append all edge explicit inputs to |*out|. Possibly with shell escaping. + void CollectInputs(bool shell_escape, std::vector* out) const; + const Rule* rule_; Pool* pool_; std::vector inputs_; diff --git a/src/graph_test.cc b/src/graph_test.cc index 5314bc5f5f..9dba8afc3c 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -215,6 +215,39 @@ TEST_F(GraphTest, RootNodes) { } } +TEST_F(GraphTest, CollectInputs) { + ASSERT_NO_FATAL_FAILURE(AssertParse( + &state_, + "build out$ 1: cat in1 in2 in$ with$ space | implicit || order_only\n")); + + std::vector inputs; + Edge* edge = GetNode("out 1")->in_edge(); + + // Test without shell escaping. + inputs.clear(); + edge->CollectInputs(false, &inputs); + EXPECT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("in2", inputs[1]); + EXPECT_EQ("in with space", inputs[2]); + EXPECT_EQ("implicit", inputs[3]); + EXPECT_EQ("order_only", inputs[4]); + + // Test with shell escaping. + inputs.clear(); + edge->CollectInputs(true, &inputs); + EXPECT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("in2", inputs[1]); +#ifdef _WIN32 + EXPECT_EQ("\"in with space\"", inputs[2]); +#else + EXPECT_EQ("'in with space'", inputs[2]); +#endif + EXPECT_EQ("implicit", inputs[3]); + EXPECT_EQ("order_only", inputs[4]); +} + TEST_F(GraphTest, VarInOutPathEscaping) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build a$ b: cat no'space with$ space$$ no\"space2\n")); diff --git a/src/ninja.cc b/src/ninja.cc index ad0912ecf5..aea430d112 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -17,6 +17,8 @@ #include #include #include + +#include #include #ifdef _WIN32 @@ -744,7 +746,8 @@ int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { return 0; } -void PrintInputs(Edge* edge, set* seen) { +void CollectInputs(Edge* edge, std::set* seen, + std::vector* result) { if (!edge) return; if (!seen->insert(edge).second) @@ -752,13 +755,41 @@ void PrintInputs(Edge* edge, set* seen) { for (vector::iterator in = edge->inputs_.begin(); in != edge->inputs_.end(); ++in) - PrintInputs((*in)->in_edge(), seen); + CollectInputs((*in)->in_edge(), seen, result); - if (!edge->is_phony()) - puts(edge->GetBinding("in_newline").c_str()); + if (!edge->is_phony()) { + edge->CollectInputs(true, result); + } } int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) { + // The inputs tool uses getopt, and expects argv[0] to contain the name of + // the tool, i.e. "inputs". + argc++; + argv--; + optind = 1; + int opt; + const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } }; + while ((opt = getopt_long(argc, argv, "h", kLongOptions, NULL)) != -1) { + switch (opt) { + case 'h': + default: + // clang-format off + printf( +"Usage '-t inputs [options] [targets]\n" +"\n" +"List all inputs used for a set of targets. Note that this includes\n" +"explicit, implicit and order-only inputs, but not validation ones.\n\n" +"Options:\n" +" -h, --help Print this message.\n"); + // clang-format on + return 1; + } + } + argv += optind; + argc -= optind; + vector nodes; string err; if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { @@ -766,9 +797,17 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) { return 1; } - set seen; + std::set seen; + std::vector result; for (vector::iterator in = nodes.begin(); in != nodes.end(); ++in) - PrintInputs((*in)->in_edge(), &seen); + CollectInputs((*in)->in_edge(), &seen, &result); + + // Make output deterministic by sorting then removing duplicates. + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + + for (size_t n = 0; n < result.size(); ++n) + puts(result[n].c_str()); return 0; } From bb471e235a83fd2b146299cd7d4d3a95163de10a Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Sun, 15 May 2022 16:53:55 +0200 Subject: [PATCH 96/96] mark this 1.11.0.git --- src/version.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.cc b/src/version.cc index 97afa7e56e..bdcbc53e50 100644 --- a/src/version.cc +++ b/src/version.cc @@ -20,7 +20,7 @@ using namespace std; -const char* kNinjaVersion = "1.10.2.git"; +const char* kNinjaVersion = "1.11.0.git"; void ParseVersion(const string& version, int* major, int* minor) { size_t end = version.find('.');