From eb32ba6f51d0c21d58cd7d89785285ff8fa64624 Mon Sep 17 00:00:00 2001 From: Zeno Albisser Date: Thu, 21 Nov 2013 14:09:57 +0100 Subject: Update chromium to branch 1599. Change-Id: I04e775a946a208bb4500d3b722bcb05c82b9d7cb Reviewed-by: Andras Becsi --- ninja/bootstrap.py | 6 +- ninja/configure.py | 4 +- ninja/doc/manual.asciidoc | 6 +- ninja/platform_helper.py | 8 ++- ninja/src/build.cc | 56 ++++++++++------- ninja/src/build.h | 7 ++- ninja/src/build_log.cc | 48 +++++++++------ ninja/src/build_log.h | 4 +- ninja/src/build_test.cc | 40 ++++++++++++ ninja/src/deps_log.cc | 57 +++++++++++++----- ninja/src/graph_test.cc | 20 ++++++ ninja/src/hash_map.h | 4 +- ninja/src/line_printer.cc | 6 +- ninja/src/manifest_parser.cc | 32 +++++----- ninja/src/manifest_parser.h | 2 +- ninja/src/manifest_parser_test.cc | 124 ++++++++++++++++++++++++++++++++++++++ ninja/src/ninja.cc | 83 +++++++++++++++++++++++-- ninja/src/subprocess-posix.cc | 6 +- ninja/src/subprocess_test.cc | 4 +- 19 files changed, 418 insertions(+), 99 deletions(-) (limited to 'ninja') diff --git a/ninja/bootstrap.py b/ninja/bootstrap.py index 5682bf1d250..66ec85bccba 100755 --- a/ninja/bootstrap.py +++ b/ninja/bootstrap.py @@ -37,7 +37,7 @@ parser.add_option('--platform', help='target platform (' + '/'.join(platform_helper.platforms()) + ')', choices=platform_helper.platforms()) parser.add_option('--force-pselect', action='store_true', - help="ppoll() is used by default on Linux and OpenBSD, but older versions might need to use pselect instead",) + help="ppoll() is used by default on Linux, OpenBSD and Bitrig, but older versions might need to use pselect instead",) (options, conf_args) = parser.parse_args() @@ -53,7 +53,7 @@ def run(*args, **kwargs): # g++ call as well as in the later configure.py. cflags = os.environ.get('CFLAGS', '').split() ldflags = os.environ.get('LDFLAGS', '').split() -if platform.is_freebsd() or platform.is_openbsd(): +if platform.is_freebsd() or platform.is_openbsd() or platform.is_bitrig(): cflags.append('-I/usr/local/include') ldflags.append('-L/usr/local/lib') @@ -109,7 +109,7 @@ else: cflags.append('-D_WIN32_WINNT=0x0501') if options.x64: cflags.append('-m64') -if (platform.is_linux() or platform.is_openbsd()) and not options.force_pselect: +if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and not options.force_pselect: cflags.append('-DUSE_PPOLL') if options.force_pselect: conf_args.append("--force-pselect") diff --git a/ninja/configure.py b/ninja/configure.py index 22eb1e57566..c838392895d 100755 --- a/ninja/configure.py +++ b/ninja/configure.py @@ -48,7 +48,7 @@ parser.add_option('--with-python', metavar='EXE', help='use EXE as the Python interpreter', default=os.path.basename(sys.executable)) parser.add_option('--force-pselect', action='store_true', - help="ppoll() is used by default on Linux and OpenBSD, but older versions might need to use pselect instead",) + help="ppoll() is used by default where available, but some platforms may need to use pselect instead",) (options, args) = parser.parse_args() if args: print('ERROR: extra unparsed command-line arguments:', args) @@ -165,7 +165,7 @@ else: cflags.append('-fno-omit-frame-pointer') libs.extend(['-Wl,--no-as-needed', '-lprofiler']) -if (platform.is_linux() or platform.is_openbsd()) and not options.force_pselect: +if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and not options.force_pselect: cflags.append('-DUSE_PPOLL') def shell_escape(str): diff --git a/ninja/doc/manual.asciidoc b/ninja/doc/manual.asciidoc index aa5644d6451..a7352571c84 100644 --- a/ninja/doc/manual.asciidoc +++ b/ninja/doc/manual.asciidoc @@ -54,7 +54,7 @@ Here are the design goals of Ninja: higher-level build systems have different opinions about how code should be built; for example, should built objects live alongside the sources or should all build output go into a separate directory? - Is there an "package" rule that builds a distributable package of + Is there a "package" rule that builds a distributable package of the project? Sidestep these decisions by trying to allow either to be implemented, rather than choosing, even if that results in more verbosity. @@ -588,6 +588,7 @@ rule cc command = cl /showIncludes -c $in /Fo$out ---- +[[ref_pool]] Pools ~~~~~ @@ -668,6 +669,9 @@ A file is a series of declarations. A declaration can be one of: +include _path_+. The difference between these is explained below <>. +6. A pool declaration, which looks like +pool _poolname_+. Pools are explained + <>. + Lexical syntax ~~~~~~~~~~~~~~ diff --git a/ninja/platform_helper.py b/ninja/platform_helper.py index 5097f498baf..b7447a1a122 100644 --- a/ninja/platform_helper.py +++ b/ninja/platform_helper.py @@ -19,7 +19,7 @@ import sys def platforms(): return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5', - 'mingw', 'msvc', 'gnukfreebsd8'] + 'mingw', 'msvc', 'gnukfreebsd8', 'bitrig'] class Platform( object ): def __init__( self, platform): @@ -41,7 +41,8 @@ class Platform( object ): self._platform = 'mingw' elif self._platform.startswith('win'): self._platform = 'msvc' - + elif self._platform.startswith('bitrig'): + self._platform = 'bitrig' def platform(self): return self._platform @@ -69,3 +70,6 @@ class Platform( object ): def is_sunos5(self): return self._platform == 'sunos5' + + def is_bitrig(self): + return self._platform == 'bitrig' diff --git a/ninja/src/build.cc b/ninja/src/build.cc index 5cf9d27dfc6..6d23f3bb40f 100644 --- a/ninja/src/build.cc +++ b/ninja/src/build.cc @@ -362,8 +362,11 @@ void Plan::ResumeDelayedJobs(Edge* edge) { void Plan::EdgeFinished(Edge* edge) { map::iterator i = want_.find(edge); assert(i != want_.end()); - if (i->second) + if (i->second) { --wanted_edges_; + if (!edge->is_phony()) + --command_edges_; + } want_.erase(i); edge->outputs_ready_ = true; @@ -422,24 +425,22 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) { } string command = (*ei)->EvaluateCommand(true); - // Now, recompute the dirty state of each output. - bool all_outputs_clean = true; + // Now, this edge is dirty if any of the outputs are dirty. + bool dirty = false; for (vector::iterator ni = (*ei)->outputs_.begin(); - ni != (*ei)->outputs_.end(); ++ni) { - if (!(*ni)->dirty()) - continue; - - if (scan->RecomputeOutputDirty(*ei, most_recent_input, 0, - command, *ni)) { - (*ni)->MarkDirty(); - all_outputs_clean = false; - } else { + !dirty && ni != (*ei)->outputs_.end(); ++ni) { + dirty = scan->RecomputeOutputDirty(*ei, most_recent_input, 0, + command, *ni); + } + + // If the edge isn't dirty, clean the outputs and mark the edge as not + // wanted. + if (!dirty) { + for (vector::iterator ni = (*ei)->outputs_.begin(); + ni != (*ei)->outputs_.end(); ++ni) { CleanNode(scan, *ni); } - } - // If we cleaned all outputs, mark the node as not wanted. - if (all_outputs_clean) { want_i->second = false; --wanted_edges_; if (!(*ei)->is_phony()) @@ -641,7 +642,10 @@ bool Builder::Build(string* err) { } --pending_commands; - FinishCommand(&result); + if (!FinishCommand(&result, err)) { + status_->BuildFinished(); + return false; + } if (!result.success()) { if (failures_allowed) @@ -704,7 +708,7 @@ bool Builder::StartEdge(Edge* edge, string* err) { return true; } -void Builder::FinishCommand(CommandRunner::Result* result) { +bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { METRIC_RECORD("FinishCommand"); Edge* edge = result->edge; @@ -733,7 +737,7 @@ void Builder::FinishCommand(CommandRunner::Result* result) { // The rest of this function only applies to successful commands. if (!result->success()) - return; + return true; // Restat the edge outputs, if necessary. TimeStamp restat_mtime = 0; @@ -763,7 +767,7 @@ void Builder::FinishCommand(CommandRunner::Result* result) { } string depfile = edge->GetBinding("depfile"); - if (restat_mtime != 0 && !depfile.empty()) { + if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) { TimeStamp depfile_mtime = disk_interface_->Stat(depfile); if (depfile_mtime > restat_mtime) restat_mtime = depfile_mtime; @@ -783,17 +787,23 @@ void Builder::FinishCommand(CommandRunner::Result* result) { disk_interface_->RemoveFile(rspfile); if (scan_.build_log()) { - scan_.build_log()->RecordCommand(edge, start_time, end_time, - restat_mtime); + if (!scan_.build_log()->RecordCommand(edge, start_time, end_time, + restat_mtime)) { + *err = string("Error writing to build log: ") + strerror(errno); + return false; + } } if (!deps_type.empty() && !config_.dry_run) { assert(edge->outputs_.size() == 1 && "should have been rejected by parser"); Node* out = edge->outputs_[0]; TimeStamp deps_mtime = disk_interface_->Stat(out->path()); - scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes); + if (!scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes)) { + *err = string("Error writing to deps log: ") + strerror(errno); + return false; + } } - + return true; } bool Builder::ExtractDeps(CommandRunner::Result* result, diff --git a/ninja/src/build.h b/ninja/src/build.h index 2715c0cf722..a872f6cf282 100644 --- a/ninja/src/build.h +++ b/ninja/src/build.h @@ -51,7 +51,7 @@ struct Plan { Edge* FindWork(); /// Returns true if there's more work to be done. - bool more_to_do() const { return wanted_edges_; } + bool more_to_do() const { return (command_edges_ > 0); } /// Dumps the current state of the plan. void Dump(); @@ -163,7 +163,10 @@ struct Builder { bool Build(string* err); bool StartEdge(Edge* edge, string* err); - void FinishCommand(CommandRunner::Result* result); + + /// Update status ninja logs following a command termination. + /// @return false if the build can not proceed further due to a fatal error. + bool FinishCommand(CommandRunner::Result* result, string* err); /// Used for tests. void SetBuildLog(BuildLog* log) { diff --git a/ninja/src/build_log.cc b/ninja/src/build_log.cc index 6b730028510..1374bd068f8 100644 --- a/ninja/src/build_log.cc +++ b/ninja/src/build_log.cc @@ -54,26 +54,27 @@ uint64_t MurmurHash64A(const void* key, size_t len) { const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); const int r = 47; uint64_t h = seed ^ (len * m); - const uint64_t * data = (const uint64_t *)key; - const uint64_t * end = data + (len/8); - while (data != end) { - uint64_t k = *data++; + const unsigned char * data = (const unsigned char *)key; + while (len >= 8) { + uint64_t k; + memcpy(&k, data, sizeof k); k *= m; k ^= k >> r; k *= m; h ^= k; h *= m; + data += 8; + len -= 8; } - const unsigned char* data2 = (const unsigned char*)data; switch (len & 7) { - case 7: h ^= uint64_t(data2[6]) << 48; - case 6: h ^= uint64_t(data2[5]) << 40; - case 5: h ^= uint64_t(data2[4]) << 32; - case 4: h ^= uint64_t(data2[3]) << 24; - case 3: h ^= uint64_t(data2[2]) << 16; - case 2: h ^= uint64_t(data2[1]) << 8; - case 1: h ^= uint64_t(data2[0]); + case 7: h ^= uint64_t(data[6]) << 48; + case 6: h ^= uint64_t(data[5]) << 40; + case 5: h ^= uint64_t(data[4]) << 32; + case 4: h ^= uint64_t(data[3]) << 24; + case 3: h ^= uint64_t(data[2]) << 16; + case 2: h ^= uint64_t(data[1]) << 8; + case 1: h ^= uint64_t(data[0]); h *= m; }; h ^= h >> r; @@ -109,7 +110,6 @@ BuildLog::~BuildLog() { bool BuildLog::OpenForWrite(const string& path, string* err) { if (needs_recompaction_) { - Close(); if (!Recompact(path, err)) return false; } @@ -136,7 +136,7 @@ bool BuildLog::OpenForWrite(const string& path, string* err) { return true; } -void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, +bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp restat_mtime) { string command = edge->EvaluateCommand(true); uint64_t command_hash = LogEntry::HashCommand(command); @@ -156,9 +156,12 @@ void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, log_entry->end_time = end_time; log_entry->restat_mtime = restat_mtime; - if (log_file_) - WriteEntry(log_file_, *log_entry); + if (log_file_) { + if (!WriteEntry(log_file_, *log_entry)) + return false; + } } + return true; } void BuildLog::Close() { @@ -341,16 +344,17 @@ BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { return NULL; } -void BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { - fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n", +bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { + return fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n", entry.start_time, entry.end_time, entry.restat_mtime, - entry.output.c_str(), entry.command_hash); + entry.output.c_str(), entry.command_hash) > 0; } bool BuildLog::Recompact(const string& path, string* err) { METRIC_RECORD(".ninja_log recompact"); printf("Recompacting log...\n"); + Close(); string temp_path = path + ".recompact"; FILE* f = fopen(temp_path.c_str(), "wb"); if (!f) { @@ -365,7 +369,11 @@ bool BuildLog::Recompact(const string& path, string* err) { } for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { - WriteEntry(f, *i->second); + if (!WriteEntry(f, *i->second)) { + *err = strerror(errno); + fclose(f); + return false; + } } fclose(f); diff --git a/ninja/src/build_log.h b/ninja/src/build_log.h index 6eae89f87ee..eeac5b3b0b4 100644 --- a/ninja/src/build_log.h +++ b/ninja/src/build_log.h @@ -37,7 +37,7 @@ struct BuildLog { ~BuildLog(); bool OpenForWrite(const string& path, string* err); - void RecordCommand(Edge* edge, int start_time, int end_time, + bool RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp restat_mtime = 0); void Close(); @@ -69,7 +69,7 @@ struct BuildLog { LogEntry* LookupByOutput(const string& path); /// Serialize an entry into a log file. - void WriteEntry(FILE* f, const LogEntry& entry); + bool WriteEntry(FILE* f, const LogEntry& entry); /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, string* err); diff --git a/ninja/src/build_test.cc b/ninja/src/build_test.cc index 313a386fc43..d1ac0efb8bb 100644 --- a/ninja/src/build_test.cc +++ b/ninja/src/build_test.cc @@ -1094,6 +1094,46 @@ TEST_F(BuildWithLogTest, RestatMissingFile) { ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } +TEST_F(BuildWithLogTest, RestatSingleDependentOutputDirty) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "rule true\n" + " command = true\n" + " restat = 1\n" + "rule touch\n" + " command = touch\n" + "build out1: true in\n" + "build out2 out3: touch out1\n" + "build out4: touch out2\n" + )); + + // Create the necessary files + fs_.Create("in", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out4", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + + fs_.Tick(); + fs_.Create("in", ""); + fs_.RemoveFile("out3"); + + // Since "in" is missing, out1 will be built. Since "out3" is missing, + // out2 and out3 will be built even though "in" is not touched when built. + // Then, since out2 is rebuilt, out4 should be rebuilt -- the restat on the + // "true" rule should not lead to the "touch" edge writing out2 and out3 being + // cleard. + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out4", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); +} + // Test scenario, in which an input file is removed, but output isn't changed // https://github.com/martine/ninja/issues/295 TEST_F(BuildWithLogTest, RestatMissingInput) { diff --git a/ninja/src/deps_log.cc b/ninja/src/deps_log.cc index ce9bf06b3f3..2c4e3c29a1f 100644 --- a/ninja/src/deps_log.cc +++ b/ninja/src/deps_log.cc @@ -37,13 +37,15 @@ const int kCurrentVersion = 1; // buffer after every record to make sure records aren't written partially. const int kMaxBufferSize = 1 << 15; +// Record size is currently limited to 15 bit +const size_t kMaxRecordSize = (1 << 15) - 1; + DepsLog::~DepsLog() { Close(); } bool DepsLog::OpenForWrite(const string& path, string* err) { if (needs_recompaction_) { - Close(); if (!Recompact(path, err)) return false; } @@ -70,8 +72,10 @@ bool DepsLog::OpenForWrite(const string& path, string* err) { return false; } } - fflush(file_); - + if (fflush(file_) != 0) { + *err = strerror(errno); + return false; + } return true; } @@ -88,12 +92,14 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, // Assign ids to all nodes that are missing one. if (node->id() < 0) { - RecordId(node); + if (!RecordId(node)) + return false; made_change = true; } for (int i = 0; i < node_count; ++i) { if (nodes[i]->id() < 0) { - RecordId(nodes[i]); + if (!RecordId(nodes[i])) + return false; made_change = true; } } @@ -120,18 +126,28 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, return true; // Update on-disk representation. - uint16_t size = 4 * (1 + 1 + (uint16_t)node_count); + size_t size = 4 * (1 + 1 + (uint16_t)node_count); + if (size > kMaxRecordSize) { + errno = ERANGE; + return false; + } size |= 0x8000; // Deps record: set high bit. - fwrite(&size, 2, 1, file_); + uint16_t size16 = (uint16_t)size; + if (fwrite(&size16, 2, 1, file_) < 1) + return false; int id = node->id(); - fwrite(&id, 4, 1, file_); + if (fwrite(&id, 4, 1, file_) < 1) + return false; int timestamp = mtime; - fwrite(×tamp, 4, 1, file_); + if (fwrite(×tamp, 4, 1, file_) < 1) + return false; for (int i = 0; i < node_count; ++i) { id = nodes[i]->id(); - fwrite(&id, 4, 1, file_); + if (fwrite(&id, 4, 1, file_) < 1) + return false; } - fflush(file_); + if (fflush(file_) != 0) + return false; // Update in-memory representation. Deps* deps = new Deps(mtime, node_count); @@ -265,6 +281,7 @@ bool DepsLog::Recompact(const string& path, string* err) { METRIC_RECORD(".ninja_deps recompact"); printf("Recompacting deps...\n"); + Close(); string temp_path = path + ".recompact"; // OpenForWrite() opens for append. Make sure it's not appending to a @@ -323,10 +340,20 @@ bool DepsLog::UpdateDeps(int out_id, Deps* deps) { } bool DepsLog::RecordId(Node* node) { - uint16_t size = (uint16_t)node->path().size(); - fwrite(&size, 2, 1, file_); - fwrite(node->path().data(), node->path().size(), 1, file_); - fflush(file_); + size_t size = node->path().size(); + if (size > kMaxRecordSize) { + errno = ERANGE; + return false; + } + uint16_t size16 = (uint16_t)size; + if (fwrite(&size16, 2, 1, file_) < 1) + return false; + if (fwrite(node->path().data(), node->path().size(), 1, file_) < 1) { + assert(node->path().size() > 0); + return false; + } + if (fflush(file_) != 0) + return false; node->set_id(nodes_.size()); nodes_.push_back(node); diff --git a/ninja/src/graph_test.cc b/ninja/src/graph_test.cc index 63d5757da92..85212167e95 100644 --- a/ninja/src/graph_test.cc +++ b/ninja/src/graph_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "graph.h" +#include "build.h" #include "test.h" @@ -226,3 +227,22 @@ TEST_F(GraphTest, DepfileOverrideParent) { Edge* edge = GetNode("out")->in_edge(); EXPECT_EQ("depfile is y", edge->GetBinding("command")); } + +// Verify that building a nested phony rule prints "no work to do" +TEST_F(GraphTest, NestedPhonyPrintsDone) { + AssertParse(&state_, +"build n1: phony \n" +"build n2: phony n1\n" + ); + string err; + Edge* edge = GetNode("n2")->in_edge(); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); + ASSERT_EQ("", err); + + Plan plan_; + EXPECT_TRUE(plan_.AddTarget(GetNode("n2"), &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(0, plan_.command_edge_count()); + ASSERT_FALSE(plan_.more_to_do()); +} diff --git a/ninja/src/hash_map.h b/ninja/src/hash_map.h index 076f6c02198..919b6fc5d14 100644 --- a/ninja/src/hash_map.h +++ b/ninja/src/hash_map.h @@ -15,6 +15,7 @@ #ifndef NINJA_MAP_H_ #define NINJA_MAP_H_ +#include #include "string_piece.h" // MurmurHash2, by Austin Appleby @@ -26,7 +27,8 @@ unsigned int MurmurHash2(const void* key, size_t len) { unsigned int h = seed ^ len; const unsigned char * data = (const unsigned char *)key; while (len >= 4) { - unsigned int k = *(unsigned int *)data; + unsigned int k; + memcpy(&k, data, sizeof k); k *= m; k ^= k >> r; k *= m; diff --git a/ninja/src/line_printer.cc b/ninja/src/line_printer.cc index a75eb0505f4..3537e886e69 100644 --- a/ninja/src/line_printer.cc +++ b/ninja/src/line_printer.cc @@ -104,6 +104,10 @@ void LinePrinter::Print(string to_print, LineType type) { void LinePrinter::PrintOnNewLine(const string& to_print) { if (!have_blank_line_) printf("\n"); - printf("%s", to_print.c_str()); + if (!to_print.empty()) { + // Avoid printf and C strings, since the actual output might contain null + // bytes like UTF-16 does (yuck). + fwrite(&to_print[0], sizeof(char), to_print.size(), stdout); + } have_blank_line_ = to_print.empty() || *to_print.rbegin() == '\n'; } diff --git a/ninja/src/manifest_parser.cc b/ninja/src/manifest_parser.cc index d742331bb76..20be7f3777f 100644 --- a/ninja/src/manifest_parser.cc +++ b/ninja/src/manifest_parser.cc @@ -28,20 +28,30 @@ ManifestParser::ManifestParser(State* state, FileReader* file_reader) : state_(state), file_reader_(file_reader) { env_ = &state->bindings_; } -bool ManifestParser::Load(const string& filename, string* err) { + +bool ManifestParser::Load(const string& filename, string* err, Lexer* parent) { + METRIC_RECORD(".ninja parse"); string contents; string read_err; if (!file_reader_->ReadFile(filename, &contents, &read_err)) { *err = "loading '" + filename + "': " + read_err; + if (parent) + parent->Error(string(*err), err); return false; } - contents.resize(contents.size() + 10); + + // The lexer needs a nul byte at the end of its input, to know when it's done. + // It takes a StringPiece, and StringPiece's string constructor uses + // string::data(). data()'s return value isn't guaranteed to be + // null-terminated (although in practice - libc++, libstdc++, msvc's stl -- + // it is, and C++11 demands that too), so add an explicit nul byte. + contents.resize(contents.size() + 1); + return Parse(filename, contents, err); } bool ManifestParser::Parse(const string& filename, const string& input, string* err) { - METRIC_RECORD(".ninja parse"); lexer_.Start(filename, input); for (;;) { @@ -146,10 +156,8 @@ bool ManifestParser::ParseRule(string* err) { if (!ExpectToken(Lexer::NEWLINE, err)) return false; - if (state_->LookupRule(name) != NULL) { - *err = "duplicate rule '" + name + "'"; - return false; - } + if (state_->LookupRule(name) != NULL) + return lexer_.Error("duplicate rule '" + name + "'", err); Rule* rule = new Rule(name); // XXX scoped_ptr @@ -307,7 +315,7 @@ bool ManifestParser::ParseEdge(string* err) { if (!pool_name.empty()) { Pool* pool = state_->LookupPool(pool_name); if (pool == NULL) - return lexer_.Error("unknown pool name", err); + return lexer_.Error("unknown pool name '" + pool_name + "'", err); edge->pool_ = pool; } @@ -340,17 +348,11 @@ bool ManifestParser::ParseEdge(string* err) { } bool ManifestParser::ParseFileInclude(bool new_scope, string* err) { - // XXX this should use ReadPath! EvalString eval; if (!lexer_.ReadPath(&eval, err)) return false; string path = eval.Evaluate(env_); - string contents; - string read_err; - if (!file_reader_->ReadFile(path, &contents, &read_err)) - return lexer_.Error("loading '" + path + "': " + read_err, err); - ManifestParser subparser(state_, file_reader_); if (new_scope) { subparser.env_ = new BindingEnv(env_); @@ -358,7 +360,7 @@ bool ManifestParser::ParseFileInclude(bool new_scope, string* err) { subparser.env_ = env_; } - if (!subparser.Parse(path, contents, err)) + if (!subparser.Load(path, err, &lexer_)) return false; if (!ExpectToken(Lexer::NEWLINE, err)) diff --git a/ninja/src/manifest_parser.h b/ninja/src/manifest_parser.h index 967dfddc306..5212f72ba1b 100644 --- a/ninja/src/manifest_parser.h +++ b/ninja/src/manifest_parser.h @@ -35,7 +35,7 @@ struct ManifestParser { ManifestParser(State* state, FileReader* file_reader); /// Load and parse a file. - bool Load(const string& filename, string* err); + bool Load(const string& filename, string* err, Lexer* parent=NULL); /// Parse a text string of input. Used by tests. bool ParseTest(const string& input, string* err) { diff --git a/ninja/src/manifest_parser_test.cc b/ninja/src/manifest_parser_test.cc index 2638edc488f..5ed158408fc 100644 --- a/ninja/src/manifest_parser_test.cc +++ b/ninja/src/manifest_parser_test.cc @@ -298,6 +298,17 @@ TEST_F(ParserTest, ReservedWords) { } TEST_F(ParserTest, Errors) { + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest(string("subn", 4), &err)); + EXPECT_EQ("input:1: expected '=', got eof\n" + "subn\n" + " ^ near here" + , err); + } + { State state; ManifestParser parser(&state, NULL); @@ -373,6 +384,17 @@ TEST_F(ParserTest, Errors) { , err); } + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("build\n", &err)); + EXPECT_EQ("input:1: expected path\n" + "build\n" + " ^ near here" + , err); + } + { State state; ManifestParser parser(&state, NULL); @@ -417,6 +439,32 @@ TEST_F(ParserTest, Errors) { EXPECT_EQ("input:2: expected 'command =' line\n", err); } + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n" + " command = echo\n" + "rule cat\n" + " command = echo\n", &err)); + EXPECT_EQ("input:3: duplicate rule 'cat'\n" + "rule cat\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n" + " command = echo\n" + " rspfile = cat.rsp\n", &err)); + EXPECT_EQ( + "input:4: rspfile and rspfile_content need to be both specified\n", + err); + } + { State state; ManifestParser parser(&state, NULL); @@ -583,6 +631,71 @@ TEST_F(ParserTest, Errors) { " generator = 1\n", &err)); EXPECT_EQ("input:4: unexpected indent\n", err); } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("pool\n", &err)); + EXPECT_EQ("input:1: expected pool name\n", err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("pool foo\n", &err)); + EXPECT_EQ("input:2: expected 'depth =' line\n", err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("pool foo\n" + " depth = 4\n" + "pool foo\n", &err)); + EXPECT_EQ("input:3: duplicate pool 'foo'\n" + "pool foo\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("pool foo\n" + " depth = -1\n", &err)); + EXPECT_EQ("input:2: invalid pool depth\n" + " depth = -1\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("pool foo\n" + " bar = 1\n", &err)); + EXPECT_EQ("input:2: unexpected variable 'bar'\n" + " bar = 1\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + // Pool names are dereferenced at edge parsing time. + EXPECT_FALSE(parser.ParseTest("rule run\n" + " command = echo\n" + " pool = unnamed_pool\n" + "build out: run in\n", &err)); + EXPECT_EQ("input:5: unknown pool name 'unnamed_pool'\n", err); + } } TEST_F(ParserTest, MissingInput) { @@ -660,6 +773,17 @@ TEST_F(ParserTest, Include) { EXPECT_EQ("inner", state.bindings_.LookupVariable("var")); } +TEST_F(ParserTest, BrokenInclude) { + files_["include.ninja"] = "build\n"; + ManifestParser parser(&state, this); + string err; + EXPECT_FALSE(parser.ParseTest("include include.ninja\n", &err)); + EXPECT_EQ("include.ninja:1: expected path\n" + "build\n" + " ^ near here" + , err); +} + TEST_F(ParserTest, Implicit) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule cat\n" diff --git a/ninja/src/ninja.cc b/ninja/src/ninja.cc index 3b381b762d7..0586bdca907 100644 --- a/ninja/src/ninja.cc +++ b/ninja/src/ninja.cc @@ -104,21 +104,23 @@ struct NinjaMain { // The various subcommands, run via "-t XXX". int ToolGraph(int argc, char* argv[]); int ToolQuery(int argc, char* argv[]); + int ToolDeps(int argc, char* argv[]); int ToolBrowse(int argc, char* argv[]); int ToolMSVC(int argc, char* argv[]); int ToolTargets(int argc, char* argv[]); int ToolCommands(int argc, char* argv[]); int ToolClean(int argc, char* argv[]); int ToolCompilationDatabase(int argc, char* argv[]); + int ToolRecompact(int argc, char* argv[]); int ToolUrtle(int argc, char** argv); /// Open the build log. /// @return false on error. - bool OpenBuildLog(); + bool OpenBuildLog(bool recompact_only = false); /// Open the deps log: load it, then open for writing. /// @return false on error. - bool OpenDepsLog(); + bool OpenDepsLog(bool recompact_only = false); /// Ensure the build directory exists, creating it if necessary. /// @return false on error. @@ -437,6 +439,45 @@ int ToolTargetsList(State* state) { return 0; } +int NinjaMain::ToolDeps(int argc, char** argv) { + vector nodes; + if (argc == 0) { + for (vector::const_iterator ni = deps_log_.nodes().begin(); + ni != deps_log_.nodes().end(); ++ni) { + // Only query for targets with an incoming edge and deps + Edge* e = (*ni)->in_edge(); + if (e && !e->GetBinding("deps").empty()) + nodes.push_back(*ni); + } + } else { + string err; + if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { + Error("%s", err.c_str()); + return 1; + } + } + + RealDiskInterface disk_interface; + for (vector::iterator it = nodes.begin(), end = nodes.end(); + it != end; ++it) { + DepsLog::Deps* deps = deps_log_.GetDeps(*it); + if (!deps) { + printf("%s: deps not found\n", (*it)->path().c_str()); + continue; + } + + TimeStamp mtime = disk_interface.Stat((*it)->path()); + printf("%s: #deps %d, deps mtime %d (%s)\n", + (*it)->path().c_str(), deps->node_count, deps->mtime, + (!mtime || mtime > deps->mtime ? "STALE":"VALID")); + for (int i = 0; i < deps->node_count; ++i) + printf(" %s\n", deps->nodes[i]->path().c_str()); + printf("\n"); + } + + return 0; +} + int NinjaMain::ToolTargets(int argc, char* argv[]) { int depth = 1; if (argc >= 1) { @@ -602,6 +643,17 @@ int NinjaMain::ToolCompilationDatabase(int argc, char* argv[]) { return 0; } +int NinjaMain::ToolRecompact(int argc, char* argv[]) { + if (!EnsureBuildDirExists()) + return 1; + + if (!OpenBuildLog(/*recompact_only=*/true) || + !OpenDepsLog(/*recompact_only=*/true)) + return 1; + + return 0; +} + int NinjaMain::ToolUrtle(int argc, char** argv) { // RLE encoded. const char* urtle = @@ -644,6 +696,8 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolClean }, { "commands", "list all commands required to rebuild given targets", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCommands }, + { "deps", "show dependencies stored in the deps log", + Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps }, { "graph", "output graphviz dot file for targets", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolGraph }, { "query", "show inputs/outputs for a path", @@ -652,6 +706,8 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolTargets }, { "compdb", "dump JSON compilation database to stdout", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabase }, + { "recompact", "recompacts ninja-internal data structures", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRecompact }, { "urtle", NULL, Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolUrtle }, { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } @@ -712,7 +768,7 @@ bool DebugEnable(const string& name) { } } -bool NinjaMain::OpenBuildLog() { +bool NinjaMain::OpenBuildLog(bool recompact_only) { string log_path = ".ninja_log"; if (!build_dir_.empty()) log_path = build_dir_ + "/" + log_path; @@ -728,6 +784,13 @@ bool NinjaMain::OpenBuildLog() { err.clear(); } + if (recompact_only) { + bool success = build_log_.Recompact(log_path, &err); + if (!success) + Error("failed recompaction: %s", err.c_str()); + return success; + } + if (!config_.dry_run) { if (!build_log_.OpenForWrite(log_path, &err)) { Error("opening build log: %s", err.c_str()); @@ -740,7 +803,7 @@ bool NinjaMain::OpenBuildLog() { /// Open the deps log: load it, then open for writing. /// @return false on error. -bool NinjaMain::OpenDepsLog() { +bool NinjaMain::OpenDepsLog(bool recompact_only) { string path = ".ninja_deps"; if (!build_dir_.empty()) path = build_dir_ + "/" + path; @@ -756,6 +819,13 @@ bool NinjaMain::OpenDepsLog() { err.clear(); } + if (recompact_only) { + bool success = deps_log_.Recompact(path, &err); + if (!success) + Error("failed recompaction: %s", err.c_str()); + return success; + } + if (!config_.dry_run) { if (!deps_log_.OpenForWrite(path, &err)) { Error("opening deps log: %s", err.c_str()); @@ -936,6 +1006,7 @@ int real_main(int argc, char** argv) { options.input_file = "build.ninja"; setvbuf(stdout, NULL, _IOLBF, BUFSIZ); + const char* ninja_command = argv[0]; int exit_code = ReadFlags(&argc, &argv, &options, &config); if (exit_code >= 0) @@ -944,7 +1015,7 @@ int real_main(int argc, char** argv) { if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) { // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed // by other tools. - NinjaMain ninja(argv[0], config); + NinjaMain ninja(ninja_command, config); return (ninja.*options.tool->func)(argc, argv); } @@ -964,7 +1035,7 @@ int real_main(int argc, char** argv) { // The build can take up to 2 passes: one to rebuild the manifest, then // another to build the desired target. for (int cycle = 0; cycle < 2; ++cycle) { - NinjaMain ninja(argv[0], config); + NinjaMain ninja(ninja_command, config); RealFileReader file_reader; ManifestParser parser(&ninja.state_, &file_reader); diff --git a/ninja/src/subprocess-posix.cc b/ninja/src/subprocess-posix.cc index b396f84b2b4..a9af756dca4 100644 --- a/ninja/src/subprocess-posix.cc +++ b/ninja/src/subprocess-posix.cc @@ -41,7 +41,7 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { Fatal("pipe: %s", strerror(errno)); fd_ = output_pipe[0]; #if !defined(USE_PPOLL) - // On Linux and OpenBSD, we use ppoll in DoWork(); elsewhere we use pselect + // If available, we use ppoll in DoWork(); otherwise we use pselect // and so must avoid overly-large FDs. if (fd_ >= static_cast(FD_SETSIZE)) Fatal("pipe: %s", strerror(EMFILE)); @@ -224,7 +224,7 @@ bool SubprocessSet::DoWork() { return interrupted_; } -#else // linux || __OpenBSD__ +#else // !defined(USE_PPOLL) bool SubprocessSet::DoWork() { fd_set set; int nfds = 0; @@ -266,7 +266,7 @@ bool SubprocessSet::DoWork() { return interrupted_; } -#endif // linux || __OpenBSD__ +#endif // !defined(USE_PPOLL) Subprocess* SubprocessSet::NextFinished() { if (finished_.empty()) diff --git a/ninja/src/subprocess_test.cc b/ninja/src/subprocess_test.cc index afd90089e5f..9f8dcea68cb 100644 --- a/ninja/src/subprocess_test.cc +++ b/ninja/src/subprocess_test.cc @@ -152,7 +152,7 @@ TEST_F(SubprocessTest, SetWithMulti) { // OS X's process limit is less than 1025 by default // (|sysctl kern.maxprocperuid| is 709 on 10.7 and 10.8 and less prior to that). -#if defined(linux) || defined(__OpenBSD__) +#if !defined(__APPLE__) && !defined(_WIN32) TEST_F(SubprocessTest, SetWithLots) { // Arbitrary big number; needs to be over 1024 to confirm we're no longer // hostage to pselect. @@ -179,7 +179,7 @@ TEST_F(SubprocessTest, SetWithLots) { } ASSERT_EQ(kNumProcs, subprocs_.finished_.size()); } -#endif // linux || __OpenBSD__ +#endif // !__APPLE__ && !_WIN32 // TODO: this test could work on Windows, just not sure how to simply // read stdin. -- cgit v1.2.1