diff --git a/.gitignore b/.gitignore index a86205b708..11150c9beb 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ TAGS # Ninja output .ninja_deps .ninja_log + +# Visual Studio Code project files +/.vscode/ diff --git a/.travis.yml b/.travis.yml index 093139b83e..19a9b28423 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,15 @@ +matrix: + include: + - os: linux + compiler: gcc + - os: linux + compiler: clang + - os: osx sudo: false language: cpp -compiler: - - gcc - - clang -script: ./configure.py --bootstrap && ./ninja all && ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots && ./misc/ninja_syntax_test.py +script: + - ./configure.py --bootstrap + - ./ninja all + - ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots + - ./misc/ninja_syntax_test.py + - ./misc/output_test.py diff --git a/HACKING.md b/HACKING.md index e7c91efdf5..bd6fec7d18 100644 --- a/HACKING.md +++ b/HACKING.md @@ -13,14 +13,50 @@ run `ninja_test` when developing. Ninja is built using itself. To bootstrap the first binary, run the configure script as `./configure.py --bootstrap`. This first compiles all non-test source files together, then re-builds Ninja using itself. -You should end up with a `ninja` binary (or `ninja.exe`) in the source root. +You should end up with a `ninja` binary (or `ninja.exe`) in the project root. #### Windows On Windows, you'll need to install Python to run `configure.py`, and run everything under a Visual Studio Tools Command Prompt (or after -running `vcvarsall` in a normal command prompt). See below if you -want to use mingw or some other compiler instead of Visual Studio. +running `vcvarsall` in a normal command prompt). + +For other combinations such as gcc/clang you will need the compiler +(gcc/cl) in your PATH and you will have to set the appropriate +platform configuration script. + +See below if you want to use mingw or some other compiler instead of +Visual Studio. + +##### Using Visual Studio +Assuming that you now have Python installed, then the steps for building under +Windows using Visual Studio are: + +Clone and checkout the latest release (or whatever branch you want). You +can do this in either a command prompt or by opening a git bash prompt: + +``` + $ git clone git://github.com/ninja-build/ninja.git && cd ninja + $ git checkout release +``` + +Then: + +1. Open a Windows command prompt in the folder where you checked out ninja. +2. Select the Microsoft build environment by running +`vcvarsall.bat` with the appropriate environment. +3. Build ninja and test it. + +The steps for a Visual Studio 2015 64-bit build are outlined here: + +``` + > "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 + > python configure.py --bootstrap + > ninja --help +``` +Copy the ninja executable to another location, if desired, e.g. C:\local\Ninja. + +Finally add the path where ninja.exe is to the PATH variable. ### Adjusting build flags @@ -73,17 +109,9 @@ build "all" before committing to verify the other source still works! ## Testing performance impact of changes -If you have a Chrome build handy, it's a good test case. Otherwise, -[the github downoads page](https://github.com/ninja-build/ninja/releases) -has a copy of the Chrome build files (and depfiles). You can untar -that, then run - - path/to/my/ninja chrome - -and compare that against a baseline Ninja. - -There's a script at `misc/measure.py` that repeatedly runs a command like -the above (to address variance) and summarizes its runtime. E.g. +If you have a Chrome build handy, it's a good test case. There's a +script at `misc/measure.py` that repeatedly runs a command (to address +variance) and summarizes its runtime. E.g. path/to/misc/measure.py path/to/my/ninja chrome @@ -95,7 +123,7 @@ and run that directly on some representative input files. Generally it's the [Google C++ coding style][], but in brief: * Function name are camelcase. -* Member methods are camelcase, expect for trivial getters which are +* Member methods are camelcase, except for trivial getters which are underscore separated. * Local variables are underscore separated. * Member variables are underscore separated and suffixed by an extra diff --git a/RELEASING b/RELEASING index 5f51b736e3..da4dbdd0f7 100644 --- a/RELEASING +++ b/RELEASING @@ -1,19 +1,20 @@ Notes to myself on all the steps to make for a Ninja release. Push new release branch: -1. Consider sending a heads-up to the ninja-build mailing list first -2. Make sure branches 'master' and 'release' are synced up locally -3. update src/version.cc with new version (with ".git"), then +1. Run afl-fuzz for a day or so (see HACKING.md) and run ninja_test +2. Consider sending a heads-up to the ninja-build mailing list first +3. Make sure branches 'master' and 'release' are synced up locally +4. Update src/version.cc with new version (with ".git"), then git commit -am 'mark this 1.5.0.git' -4. git checkout release; git merge master -5. fix version number in src/version.cc (it will likely conflict in the above) -6. fix version in doc/manual.asciidoc (exists only on release branch) -7. commit, tag, push (don't forget to push --tags) +5. git checkout release; git merge master +6. Fix version number in src/version.cc (it will likely conflict in the above) +7. Fix version in doc/manual.asciidoc (exists only on release branch) +8. commit, tag, push (don't forget to push --tags) git commit -am v1.5.0; git push origin release git tag v1.5.0; git push --tags # Push the 1.5.0.git change on master too: git checkout master; git push origin master -8. construct release notes from prior notes +9. Construct release notes from prior notes credits: git shortlog -s --no-merges REV.. Release on github: diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000000..4c64f291d5 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,40 @@ +version: 1.0.{build} +image: Visual Studio 2017 + +environment: + CLICOLOR_FORCE: 1 + CHERE_INVOKING: 1 # Tell Bash to inherit the current working directory + matrix: + - MSYSTEM: MINGW64 + - MSYSTEM: MSVC + +for: + - + matrix: + only: + - MSYSTEM: MINGW64 + build_script: + ps: "C:\\msys64\\usr\\bin\\bash -lc @\"\n + pacman -S --quiet --noconfirm --needed re2c 2>&1\n + sed -i 's|cmd /c $ar cqs $out.tmp $in && move /Y $out.tmp $out|$ar crs $out $in|g' configure.py\n + ./configure.py --bootstrap --platform mingw 2>&1\n + ./ninja all\n + ./ninja_test 2>&1\n + ./misc/ninja_syntax_test.py 2>&1\n\"@" + - + matrix: + only: + - MSYSTEM: MSVC + build_script: + - cmd: >- + call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" + + python configure.py --bootstrap + + ninja.bootstrap.exe all + + ninja_test + + python misc/ninja_syntax_test.py + +test: off diff --git a/configure.py b/configure.py index a443748942..78cd1deb0d 100755 --- a/configure.py +++ b/configure.py @@ -98,7 +98,7 @@ def is_aix(self): return self._platform == 'aix' def uses_usr_local(self): - return self._platform in ('freebsd', 'openbsd', 'bitrig', 'dragonfly') + return self._platform in ('freebsd', 'openbsd', 'bitrig', 'dragonfly', 'netbsd') def supports_ppoll(self): return self._platform in ('freebsd', 'linux', 'openbsd', 'bitrig', @@ -256,7 +256,7 @@ def _run_command(self, cmdline): if '--bootstrap' in configure_args: configure_args.remove('--bootstrap') n.variable('configure_args', ' '.join(configure_args)) -env_keys = set(['CXX', 'AR', 'CFLAGS', 'LDFLAGS']) +env_keys = set(['CXX', 'AR', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']) configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys) if configure_env: config_str = ' '.join([k + '=' + pipes.quote(configure_env[k]) @@ -356,6 +356,11 @@ def binary(name): if platform.uses_usr_local(): cflags.append('-I/usr/local/include') ldflags.append('-L/usr/local/lib') + if platform.is_aix(): + # printf formats for int64_t, uint64_t; large file support + cflags.append('-D__STDC_FORMAT_MACROS') + cflags.append('-D_LARGE_FILES') + libs = [] @@ -397,6 +402,10 @@ def shell_escape(str): if 'CFLAGS' in configure_env: cflags.append(configure_env['CFLAGS']) + ldflags.append(configure_env['CFLAGS']) +if 'CXXFLAGS' in configure_env: + cflags.append(configure_env['CXXFLAGS']) + ldflags.append(configure_env['CXXFLAGS']) n.variable('cflags', ' '.join(shell_escape(flag) for flag in cflags)) if 'LDFLAGS' in configure_env: ldflags.append(configure_env['LDFLAGS']) @@ -405,7 +414,7 @@ def shell_escape(str): if platform.is_msvc(): n.rule('cxx', - command='$cxx $cflags -c $in /Fo$out', + command='$cxx $cflags -c $in /Fo$out /Fd' + built('$pdb'), description='CXX $out', deps='msvc' # /showIncludes is included in $cflags. ) @@ -476,6 +485,9 @@ def has_re2c(): n.newline() n.comment('Core source files all build into ninja library.') +cxxvariables = [] +if platform.is_msvc(): + cxxvariables = [('pdb', 'ninja.pdb')] for name in ['build', 'build_log', 'clean', @@ -496,15 +508,15 @@ def has_re2c(): 'string_piece_util', 'util', 'version']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) if platform.is_windows(): for name in ['subprocess-win32', 'includes_normalize-win32', 'msvc_helper-win32', 'msvc_helper_main-win32']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) if platform.is_msvc(): - objs += cxx('minidump-win32') + objs += cxx('minidump-win32', variables=cxxvariables) objs += cc('getopt') else: objs += cxx('subprocess-posix') @@ -527,7 +539,7 @@ def has_re2c(): all_targets = [] n.comment('Main executable is library plus main() function.') -objs = cxx('ninja') +objs = cxx('ninja', variables=cxxvariables) ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) n.newline() @@ -542,6 +554,8 @@ def has_re2c(): n.comment('Tests all build into ninja_test executable.') objs = [] +if platform.is_msvc(): + cxxvariables = [('pdb', 'ninja_test.pdb')] for name in ['build_log_test', 'build_test', @@ -560,10 +574,10 @@ def has_re2c(): 'subprocess_test', 'test', 'util_test']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) if platform.is_windows(): for name in ['includes_normalize_test', 'msvc_helper_test']: - objs += cxx(name) + objs += cxx(name, variables=cxxvariables) ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) @@ -579,7 +593,9 @@ def has_re2c(): 'hash_collision_bench', 'manifest_parser_perftest', 'clparser_perftest']: - objs = cxx(name) + if platform.is_msvc(): + cxxvariables = [('pdb', name + '.pdb')] + objs = cxx(name, variables=cxxvariables) all_targets += n.build(binary(name), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 17d607a325..3c42aa740e 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -155,11 +155,10 @@ design is quite clever. Ninja's benefit comes from using it in conjunction with a smarter meta-build system. -http://code.google.com/p/gyp/[gyp]:: The meta-build system used to +https://gn.googlesource.com/gn/[gn]:: The meta-build system used to generate build files for Google Chrome and related projects (v8, -node.js). gyp can generate Ninja files for all platforms supported by -Chrome. See the -https://chromium.googlesource.com/chromium/src/+/master/docs/ninja_build.md[Chromium Ninja documentation for more details]. +node.js), as well as Google Fuschia. gn can generate Ninja files for +all platforms supported by Chrome. https://cmake.org/[CMake]:: A widely used meta-build system that can generate Ninja files on Linux as of CMake version 2.8.8. Newer versions @@ -594,7 +593,7 @@ Ninja supports this processing in two forms. to its stdout. Ninja then filters these lines from the displayed output. No `depfile` attribute is necessary, but the localized string in front of the the header file path. For instance - `msvc_deps_prefix = Note: including file: ` + `msvc_deps_prefix = Note: including file:` for a English Visual Studio (the default). Should be globally defined. + ---- @@ -881,7 +880,8 @@ quoting rules are deterimined by the called program, which on Windows are usually provided by the C library. If you need shell interpretation of the command (such as the use of `&&` to chain multiple commands), make the command execute the Windows shell by -prefixing the command with `cmd /c`. +prefixing the command with `cmd /c`. Ninja may error with "invalid parameter" +which usually indicates that the command line length has been exceeded. [[ref_outputs]] Build outputs diff --git a/misc/ninja-mode.el b/misc/ninja-mode.el index 639e5375bb..8b975d5156 100644 --- a/misc/ninja-mode.el +++ b/misc/ninja-mode.el @@ -56,7 +56,7 @@ (save-excursion (goto-char (line-end-position 0)) (or - ;; If we're continuting the previous line, it's not a + ;; If we're continuing the previous line, it's not a ;; comment. (not (eq ?$ (char-before))) ;; Except if the previous line is a comment as well, as the diff --git a/misc/ninja.vim b/misc/ninja.vim index 190d9ceb8a..c1ffd50b1c 100644 --- a/misc/ninja.vim +++ b/misc/ninja.vim @@ -1,8 +1,8 @@ " ninja build file syntax. " Language: ninja build file as described at " http://ninja-build.org/manual.html -" Version: 1.4 -" Last Change: 2014/05/13 +" Version: 1.5 +" Last Change: 2018/04/05 " Maintainer: Nicolas Weber " Version 1.4 of this script is in the upstream vim repository and will be " included in the next vim release. If you change this, please send your change @@ -21,7 +21,10 @@ set cpo&vim syn case match -syn match ninjaComment /#.*/ contains=@Spell +" Comments are only matched when the # is at the beginning of the line (with +" optional whitespace), as long as the prior line didn't end with a $ +" continuation. +syn match ninjaComment /\(\$\n\)\@" " limited set of magic variables, 'build' allows general " let assignments. " manifest_parser.cc, ParseRule() -syn region ninjaRule start="^rule" end="^\ze\S" contains=ALL transparent -syn keyword ninjaRuleCommand contained command deps depfile description generator +syn region ninjaRule start="^rule" end="^\ze\S" contains=TOP transparent +syn keyword ninjaRuleCommand contained containedin=ninjaRule command + \ deps depfile description generator \ pool restat rspfile rspfile_content -syn region ninjaPool start="^pool" end="^\ze\S" contains=ALL transparent -syn keyword ninjaPoolCommand contained depth +syn region ninjaPool start="^pool" end="^\ze\S" contains=TOP transparent +syn keyword ninjaPoolCommand contained containedin=ninjaPool depth " Strings are parsed as follows: " lexer.in.cc, ReadEvalString() diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index 5c52ea23f8..ebe6490d8d 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -21,7 +21,7 @@ def __init__(self, output, width=78): def newline(self): self.output.write('\n') - def comment(self, text, has_path=False): + def comment(self, text): for line in textwrap.wrap(text, self.width - 2, break_long_words=False, break_on_hyphens=False): self.output.write('# ' + line + '\n') @@ -60,7 +60,7 @@ def rule(self, name, command, description=None, depfile=None, self.variable('deps', deps, indent=1) def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, - variables=None, implicit_outputs=None): + variables=None, implicit_outputs=None, pool=None): outputs = as_list(outputs) out_outputs = [escape_path(x) for x in outputs] all_inputs = [escape_path(x) for x in as_list(inputs)] @@ -81,6 +81,8 @@ def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, self._line('build %s: %s' % (' '.join(out_outputs), ' '.join([rule] + all_inputs))) + if pool is not None: + self._line(' pool = %s' % pool) if variables: if isinstance(variables, dict): diff --git a/misc/ninja_syntax_test.py b/misc/ninja_syntax_test.py index 07e3ed3843..90ff9c6bdb 100755 --- a/misc/ninja_syntax_test.py +++ b/misc/ninja_syntax_test.py @@ -46,13 +46,13 @@ def test_few_long_words(self): self.out.getvalue()) def test_comment_wrap(self): - # Filenames shoud not be wrapped + # Filenames should not be wrapped self.n.comment('Hello /usr/local/build-tools/bin') self.assertEqual('# Hello\n# /usr/local/build-tools/bin\n', self.out.getvalue()) def test_short_words_indented(self): - # Test that indent is taking into acount when breaking subsequent lines. + # Test that indent is taking into account when breaking subsequent lines. # The second line should not be ' to tree', as that's longer than the # test layout width of 8. self.n._line('line_one to tree') diff --git a/misc/output_test.py b/misc/output_test.py new file mode 100755 index 0000000000..1dcde10b03 --- /dev/null +++ b/misc/output_test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +"""Runs ./ninja and checks if the output is correct. + +In order to simulate a smart terminal it uses the 'script' command. +""" + +import os +import platform +import subprocess +import sys +import tempfile +import unittest + +default_env = dict(os.environ) +if 'NINJA_STATUS' in default_env: + del default_env['NINJA_STATUS'] +if 'CLICOLOR_FORCE' in default_env: + del default_env['CLICOLOR_FORCE'] +default_env['TERM'] = '' + +def run(build_ninja, flags='', pipe=False, env=default_env): + with tempfile.NamedTemporaryFile('w') as f: + f.write(build_ninja) + f.flush() + ninja_cmd = './ninja {} -f {}'.format(flags, f.name) + try: + if pipe: + output = subprocess.check_output([ninja_cmd], shell=True, env=env) + elif platform.system() == 'Darwin': + output = subprocess.check_output(['script', '-q', '/dev/null', 'bash', '-c', ninja_cmd], + env=env) + else: + output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'], + env=env) + except subprocess.CalledProcessError as err: + sys.stdout.buffer.write(err.output) + raise err + final_output = '' + for line in output.decode('utf-8').splitlines(True): + if len(line) > 0 and line[-1] == '\r': + continue + final_output += line.replace('\r', '') + return final_output + +class Output(unittest.TestCase): + def test_issue_1418(self): + self.assertEqual(run( +'''rule echo + command = sleep $delay && echo $out + description = echo $out + +build a: echo + delay = 3 +build b: echo + delay = 2 +build c: echo + delay = 1 +'''), +'''[1/3] echo c\x1b[K +c +[2/3] echo b\x1b[K +b +[3/3] echo a\x1b[K +a +''') + + def test_issue_1214(self): + print_red = '''rule echo + command = printf '\x1b[31mred\x1b[0m' + description = echo $out + +build a: echo +''' + # Only strip color when ninja's output is piped. + self.assertEqual(run(print_red), +'''[1/1] echo a\x1b[K +\x1b[31mred\x1b[0m +''') + self.assertEqual(run(print_red, pipe=True), +'''[1/1] echo a +red +''') + # Even in verbose mode, colors should still only be stripped when piped. + self.assertEqual(run(print_red, flags='-v'), +'''[1/1] printf '\x1b[31mred\x1b[0m' +\x1b[31mred\x1b[0m +''') + self.assertEqual(run(print_red, flags='-v', pipe=True), +'''[1/1] printf '\x1b[31mred\x1b[0m' +red +''') + + # CLICOLOR_FORCE=1 can be used to disable escape code stripping. + env = default_env.copy() + env['CLICOLOR_FORCE'] = '1' + self.assertEqual(run(print_red, pipe=True, env=env), +'''[1/1] echo a +\x1b[31mred\x1b[0m +''') + +if __name__ == '__main__': + unittest.main() diff --git a/misc/zsh-completion b/misc/zsh-completion index bf23face5f..4cee3b8631 100644 --- a/misc/zsh-completion +++ b/misc/zsh-completion @@ -14,7 +14,7 @@ # limitations under the License. # Add the following to your .zshrc to tab-complete ninja targets -# . path/to/ninja/misc/zsh-completion +# fpath=(path/to/ninja/misc/zsh-completion $fpath) __get_targets() { dir="." diff --git a/src/browse.cc b/src/browse.cc index 14900f8464..c08c9f4d40 100644 --- a/src/browse.cc +++ b/src/browse.cc @@ -14,6 +14,7 @@ #include "browse.h" +#include #include #include #include @@ -57,7 +58,11 @@ void RunBrowsePython(State* state, const char* ninja_command, } command.push_back(NULL); execvp(command[0], (char**)&command[0]); - perror("ninja: execvp"); + if (errno == ENOENT) { + printf("ninja: %s is required for the browse tool\n", NINJA_PYTHON); + } else { + perror("ninja: execvp"); + } } while (false); _exit(1); } else { // Child. diff --git a/src/browse.py b/src/browse.py index 64a16f2a27..1c9c39b8e8 100755 --- a/src/browse.py +++ b/src/browse.py @@ -24,8 +24,10 @@ try: import http.server as httpserver + import socketserver except ImportError: import BaseHTTPServer as httpserver + import SocketServer as socketserver import argparse import cgi import os @@ -205,10 +207,14 @@ def log_message(self, format, *args): parser.add_argument('initial_target', default='all', nargs='?', help='Initial target to show (default %(default)s)') +class HTTPServer(socketserver.ThreadingMixIn, httpserver.HTTPServer): + # terminate server immediately when Python exits. + daemon_threads = True + args = parser.parse_args() port = args.port hostname = args.hostname -httpd = httpserver.HTTPServer((hostname,port), RequestHandler) +httpd = HTTPServer((hostname,port), RequestHandler) try: if hostname == "": hostname = socket.gethostname() diff --git a/src/build.cc b/src/build.cc index 61ef0e849a..b3928033e1 100644 --- a/src/build.cc +++ b/src/build.cc @@ -154,9 +154,8 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, // (Launching subprocesses in pseudo ttys doesn't work because there are // only a few hundred available on some systems, and ninja can launch // thousands of parallel compile commands.) - // TODO: There should be a flag to disable escape code stripping. string final_output; - if (!printer_.is_smart_terminal()) + if (!printer_.supports_color()) final_output = StripAnsiEscapeCodes(output); else final_output = output; @@ -318,18 +317,18 @@ bool Plan::AddSubTarget(Node* node, Node* dependent, string* err) { return false; // Don't need to do anything. // If an entry in want_ does not already exist for edge, create an entry which - // maps to false, indicating that we do not want to build this entry itself. - pair::iterator, bool> want_ins = - want_.insert(make_pair(edge, false)); - bool& want = want_ins.first->second; + // maps to kWantNothing, indicating that we do not want to build this entry itself. + pair::iterator, bool> want_ins = + want_.insert(make_pair(edge, kWantNothing)); + Want& want = want_ins.first->second; // If we do need to build edge and we haven't already marked it as wanted, // mark it now. - if (node->dirty() && !want) { - want = true; + if (node->dirty() && want == kWantNothing) { + want = kWantToStart; ++wanted_edges_; if (edge->AllInputsReady()) - ScheduleWork(edge); + ScheduleWork(want_ins.first); if (!edge->is_phony()) ++command_edges_; } @@ -355,30 +354,32 @@ Edge* Plan::FindWork() { return edge; } -void Plan::ScheduleWork(Edge* edge) { - set::iterator e = ready_.lower_bound(edge); - if (e != ready_.end() && !ready_.key_comp()(edge, *e)) { +void Plan::ScheduleWork(map::iterator want_e) { + if (want_e->second == kWantToFinish) { // This edge has already been scheduled. We can get here again if an edge // and one of its dependencies share an order-only input, or if a node // duplicates an out edge (see https://github.com/ninja-build/ninja/pull/519). // Avoid scheduling the work again. return; } + assert(want_e->second == kWantToStart); + want_e->second = kWantToFinish; + Edge* edge = want_e->first; Pool* pool = edge->pool(); if (pool->ShouldDelayEdge()) { pool->DelayEdge(edge); pool->RetrieveReadyEdges(&ready_); } else { pool->EdgeScheduled(*edge); - ready_.insert(e, edge); + ready_.insert(edge); } } void Plan::EdgeFinished(Edge* edge, EdgeResult result) { - map::iterator e = want_.find(edge); + map::iterator e = want_.find(edge); assert(e != want_.end()); - bool directly_wanted = e->second; + bool directly_wanted = e->second != kWantNothing; // See if this job frees up any delayed jobs. if (directly_wanted) @@ -405,14 +406,14 @@ void Plan::NodeFinished(Node* node) { // See if we we want any edges from this node. for (vector::const_iterator oe = node->out_edges().begin(); oe != node->out_edges().end(); ++oe) { - map::iterator want_e = want_.find(*oe); + map::iterator want_e = want_.find(*oe); if (want_e == want_.end()) continue; // See if the edge is now ready. if ((*oe)->AllInputsReady()) { - if (want_e->second) { - ScheduleWork(*oe); + if (want_e->second != kWantNothing) { + ScheduleWork(want_e); } else { // We do not need to build this edge, but we might need to build one of // its dependents. @@ -428,8 +429,8 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { for (vector::const_iterator oe = node->out_edges().begin(); oe != node->out_edges().end(); ++oe) { // Don't process edges that we don't actually want. - map::iterator want_e = want_.find(*oe); - if (want_e == want_.end() || !want_e->second) + map::iterator want_e = want_.find(*oe); + if (want_e == want_.end() || want_e->second == kWantNothing) continue; // Don't attempt to clean an edge if it failed to load deps. @@ -441,7 +442,12 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { vector::iterator begin = (*oe)->inputs_.begin(), end = (*oe)->inputs_.end() - (*oe)->order_only_deps_; - if (find_if(begin, end, mem_fun(&Node::dirty)) == end) { +#if __cplusplus < 201703L +#define MEM_FN mem_fun +#else +#define MEM_FN mem_fn // mem_fun was removed in C++17. +#endif + if (find_if(begin, end, MEM_FN(&Node::dirty)) == end) { // Recompute most_recent_input. Node* most_recent_input = NULL; for (vector::iterator i = begin; i != end; ++i) { @@ -464,7 +470,7 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { return false; } - want_e->second = false; + want_e->second = kWantNothing; --wanted_edges_; if (!(*oe)->is_phony()) --command_edges_; @@ -476,8 +482,8 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { void Plan::Dump() { printf("pending: %d\n", (int)want_.size()); - for (map::iterator e = want_.begin(); e != want_.end(); ++e) { - if (e->second) + for (map::iterator e = want_.begin(); e != want_.end(); ++e) { + if (e->second != kWantNothing) printf("want "); e->first->Dump(); } @@ -551,7 +557,8 @@ Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface) : state_(state), config_(config), disk_interface_(disk_interface), - scan_(state, build_log, deps_log, disk_interface) { + scan_(state, build_log, deps_log, disk_interface, + &config_.depfile_parser_options) { status_ = new BuildStatus(config); } @@ -900,7 +907,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, if (content.empty()) return true; - DepfileParser deps; + DepfileParser deps(config_.depfile_parser_options); if (!deps.Parse(&content, err)) return false; diff --git a/src/build.h b/src/build.h index 43786f1c92..a42b8d403e 100644 --- a/src/build.h +++ b/src/build.h @@ -23,6 +23,7 @@ #include #include +#include "depfile_parser.h" #include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" #include "line_printer.h" @@ -78,17 +79,29 @@ struct Plan { bool AddSubTarget(Node* node, Node* dependent, string* err); void NodeFinished(Node* node); + /// Enumerate possible steps we want for an edge. + enum Want + { + /// We do not want to build the edge, but we might want to build one of + /// its dependents. + kWantNothing, + /// We want to build the edge, but have not yet scheduled it. + kWantToStart, + /// We want to build the edge, have scheduled it, and are waiting + /// for it to complete. + kWantToFinish + }; + /// Submits a ready edge as a candidate for execution. /// The edge may be delayed from running, for example if it's a member of a /// currently-full pool. - void ScheduleWork(Edge* edge); + void ScheduleWork(map::iterator want_e); /// Keep track of which edges we want to build in this plan. If this map does /// not contain an entry for an edge, we do not want to build the entry or its - /// dependents. If an entry maps to false, we do not want to build it, but we - /// might want to build one of its dependents. If the entry maps to true, we - /// want to build it. - map want_; + /// dependents. If it does contain an entry, the enumeration indicates what + /// we want for the edge. + map want_; set ready_; @@ -139,6 +152,7 @@ struct BuildConfig { /// The maximum load average we must not exceed. A negative value /// means that we do not have any limit. double max_load_average; + DepfileParserOptions depfile_parser_options; }; /// Builder wraps the build process: starting commands, updating status. @@ -178,7 +192,11 @@ struct Builder { State* state_; const BuildConfig& config_; Plan plan_; +#if __cplusplus < 201703L auto_ptr command_runner_; +#else + unique_ptr command_runner_; // auto_ptr was removed in C++17. +#endif BuildStatus* status_; private: diff --git a/src/build_log.cc b/src/build_log.cc index 333915af9f..c4a08a079a 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -35,6 +35,9 @@ #include "graph.h" #include "metrics.h" #include "util.h" +#if defined(_MSC_VER) && (_MSC_VER < 1800) +#define strtoll _strtoi64 +#endif // Implementation details: // Each run's log appends to the log file. @@ -76,11 +79,17 @@ uint64_t MurmurHash64A(const void* key, size_t len) { switch (len & 7) { case 7: h ^= uint64_t(data[6]) << 48; + NINJA_FALLTHROUGH; case 6: h ^= uint64_t(data[5]) << 40; + NINJA_FALLTHROUGH; case 5: h ^= uint64_t(data[4]) << 32; + NINJA_FALLTHROUGH; case 4: h ^= uint64_t(data[3]) << 24; + NINJA_FALLTHROUGH; case 3: h ^= uint64_t(data[2]) << 16; + NINJA_FALLTHROUGH; case 2: h ^= uint64_t(data[1]) << 8; + NINJA_FALLTHROUGH; case 1: h ^= uint64_t(data[0]); h *= m; }; @@ -167,6 +176,9 @@ bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, if (log_file_) { if (!WriteEntry(log_file_, *log_entry)) return false; + if (fflush(log_file_) != 0) { + return false; + } } } return true; @@ -290,7 +302,7 @@ bool BuildLog::Load(const string& path, string* err) { if (!end) continue; *end = 0; - restat_mtime = atol(start); + restat_mtime = strtoll(start, NULL, 10); start = end + 1; end = (char*)memchr(start, kFieldSeparator, line_end - start); @@ -353,7 +365,7 @@ BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { } bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { - return fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n", + return fprintf(f, "%d\t%d\t%" PRId64 "\t%s\t%" PRIx64 "\n", entry.start_time, entry.end_time, entry.mtime, entry.output.c_str(), entry.command_hash) > 0; } diff --git a/src/clean.cc b/src/clean.cc index 1d6ba9e967..ce6a5753ed 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -101,6 +101,7 @@ void Cleaner::PrintHeader() { printf("\n"); else printf(" "); + fflush(stdout); } void Cleaner::PrintFooter() { @@ -180,15 +181,22 @@ int Cleaner::CleanTargets(int target_count, char* targets[]) { Reset(); PrintHeader(); for (int i = 0; i < target_count; ++i) { - const char* target_name = targets[i]; - Node* target = state_->LookupNode(target_name); - if (target) { - if (IsVerbose()) - printf("Target %s\n", target_name); - DoCleanTarget(target); - } else { - Error("unknown target '%s'", target_name); + string target_name = targets[i]; + uint64_t slash_bits; + string err; + if (!CanonicalizePath(&target_name, &slash_bits, &err)) { + Error("failed to canonicalize '%s': %s", target_name.c_str(), err.c_str()); status_ = 1; + } else { + Node* target = state_->LookupNode(target_name); + if (target) { + if (IsVerbose()) + printf("Target %s\n", target_name.c_str()); + DoCleanTarget(target); + } else { + Error("unknown target '%s'", target_name.c_str()); + status_ = 1; + } } } PrintFooter(); diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 7cee892109..405289ff9d 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.13.5 */ +/* Generated by re2c 1.1.1 */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,12 @@ // limitations under the License. #include "depfile_parser.h" +#include "util.h" + +DepfileParser::DepfileParser(DepfileParserOptions options) + : options_(options) +{ +} // A note on backslashes in Makefiles, from reading the docs: // Backslash-newline is the line continuation character. @@ -35,8 +41,13 @@ bool DepfileParser::Parse(string* content, string* err) { // parsing_targets: whether we are parsing targets or dependencies. char* in = &(*content)[0]; char* end = in + content->size(); + bool have_target = false; + bool have_secondary_target_on_this_rule = false; + bool have_newline_since_primary_target = false; + bool warned_distinct_target_lines = false; bool parsing_targets = true; while (in < end) { + bool have_newline = false; // out: current output point (typically same as in, but can fall behind // as we de-escape backslashes). char* out = in; @@ -45,6 +56,7 @@ bool DepfileParser::Parse(string* content, string* err) { for (;;) { // start: beginning of the current parsed span. const char* start = in; + char* yymarker = NULL; { unsigned char yych; @@ -53,7 +65,7 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 128, 0, 0, 0, 0, 0, 0, + 0, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 128, 0, 0, @@ -82,88 +94,55 @@ bool DepfileParser::Parse(string* content, string* err) { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, }; - yych = *in; - if (yych <= '=') { - if (yych <= '$') { - if (yych <= ' ') { - if (yych <= 0x00) goto yy7; - goto yy9; - } else { - if (yych <= '!') goto yy5; - if (yych <= '#') goto yy9; - goto yy4; - } + if (yybm[0+yych] & 128) { + goto yy9; + } + if (yych <= '\r') { + if (yych <= '\t') { + if (yych >= 0x01) goto yy4; } else { - if (yych <= '*') { - if (yych <= '\'') goto yy9; - if (yych <= ')') goto yy5; - goto yy9; - } else { - if (yych <= ':') goto yy5; - if (yych <= '<') goto yy9; - goto yy5; - } + if (yych <= '\n') goto yy6; + if (yych <= '\f') goto yy4; + goto yy8; } } else { - if (yych <= '_') { - if (yych <= '[') { - if (yych <= '?') goto yy9; - if (yych <= 'Z') goto yy5; - goto yy9; - } else { - if (yych <= '\\') goto yy2; - if (yych <= '^') goto yy9; - goto yy5; - } + if (yych <= '$') { + if (yych <= '#') goto yy4; + goto yy12; } else { - if (yych <= '|') { - if (yych <= '`') goto yy9; - if (yych <= '{') goto yy5; - goto yy9; - } else { - if (yych == 0x7F) goto yy9; - goto yy5; - } + if (yych == '\\') goto yy13; + goto yy4; } } -yy2: ++in; - if ((yych = *in) <= '"') { - if (yych <= '\f') { - if (yych <= 0x00) goto yy3; - if (yych != '\n') goto yy14; - } else { - if (yych <= '\r') goto yy3; - if (yych == ' ') goto yy16; - goto yy14; - } - } else { - if (yych <= 'Z') { - if (yych <= '#') goto yy16; - if (yych == '*') goto yy16; - goto yy14; - } else { - if (yych <= '\\') goto yy16; - if (yych == '|') goto yy16; - goto yy14; - } + { + break; } -yy3: +yy4: + ++in; +yy5: { // For any other character (e.g. whitespace), swallow it here, // allowing the outer logic to loop around again. break; } -yy4: - yych = *++in; - if (yych == '$') goto yy12; - goto yy3; -yy5: - ++in; - yych = *in; - goto yy11; yy6: + ++in; + { + // A newline ends the current file name and the current rule. + have_newline = true; + break; + } +yy8: + yych = *++in; + if (yych == '\n') goto yy6; + goto yy5; +yy9: + yych = *++in; + if (yybm[0+yych] & 128) { + goto yy9; + } { // Got a span of plain text. int len = (int)(in - start); @@ -173,30 +152,41 @@ bool DepfileParser::Parse(string* content, string* err) { out += len; continue; } -yy7: - ++in; - { - break; - } -yy9: +yy12: yych = *++in; - goto yy3; -yy10: - ++in; - yych = *in; -yy11: - if (yybm[0+yych] & 128) { - goto yy10; + if (yych == '$') goto yy14; + goto yy5; +yy13: + yych = *(yymarker = ++in); + if (yych <= '"') { + if (yych <= '\f') { + if (yych <= 0x00) goto yy5; + if (yych == '\n') goto yy18; + goto yy16; + } else { + if (yych <= '\r') goto yy20; + if (yych == ' ') goto yy22; + goto yy16; + } + } else { + if (yych <= 'Z') { + if (yych <= '#') goto yy22; + if (yych == '*') goto yy22; + goto yy16; + } else { + if (yych <= ']') goto yy22; + if (yych == '|') goto yy22; + goto yy16; + } } - goto yy6; -yy12: +yy14: ++in; { // De-escape dollar character. *out++ = '$'; continue; } -yy14: +yy16: ++in; { // Let backslash before other characters through verbatim. @@ -204,7 +194,18 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = yych; continue; } -yy16: +yy18: + ++in; + { + // A line continuation ends the current file name. + break; + } +yy20: + yych = *++in; + if (yych == '\n') goto yy18; + in = yymarker; + goto yy5; +yy22: ++in; { // De-escape backslashed character. @@ -216,25 +217,52 @@ bool DepfileParser::Parse(string* content, string* err) { } int len = (int)(out - filename); - const bool is_target = parsing_targets; + const bool is_dependency = !parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. parsing_targets = false; + have_target = true; } - if (len == 0) - continue; + if (len > 0) { + if (is_dependency) { + if (have_secondary_target_on_this_rule) { + if (!have_newline_since_primary_target) { + *err = "depfile has multiple output paths"; + return false; + } else if (options_.depfile_distinct_target_lines_action_ == + kDepfileDistinctTargetLinesActionError) { + *err = + "depfile has multiple output paths (on separate lines)" + " [-w depfilemulti=err]"; + return false; + } else { + if (!warned_distinct_target_lines) { + warned_distinct_target_lines = true; + Warning("depfile has multiple output paths (on separate lines); " + "continuing anyway [-w depfilemulti=warn]"); + } + continue; + } + } + ins_.push_back(StringPiece(filename, len)); + } else if (!out_.str_) { + out_ = StringPiece(filename, len); + } else if (out_ != StringPiece(filename, len)) { + have_secondary_target_on_this_rule = true; + } + } - if (!is_target) { - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths"; - return false; + if (have_newline) { + // A newline ends a rule so the next filename will be a new target. + parsing_targets = true; + have_secondary_target_on_this_rule = false; + if (have_target) { + have_newline_since_primary_target = true; + } } } - if (parsing_targets) { + if (!have_target) { *err = "expected ':' in depfile"; return false; } diff --git a/src/depfile_parser.h b/src/depfile_parser.h index 1e6ebb5795..be203746d6 100644 --- a/src/depfile_parser.h +++ b/src/depfile_parser.h @@ -21,8 +21,24 @@ using namespace std; #include "string_piece.h" +enum DepfileDistinctTargetLinesAction { + kDepfileDistinctTargetLinesActionWarn, + kDepfileDistinctTargetLinesActionError, +}; + +struct DepfileParserOptions { + DepfileParserOptions() + : depfile_distinct_target_lines_action_( + kDepfileDistinctTargetLinesActionWarn) {} + DepfileDistinctTargetLinesAction + depfile_distinct_target_lines_action_; +}; + /// Parser for the dependency information emitted by gcc's -M flags. struct DepfileParser { + explicit DepfileParser(DepfileParserOptions options = + DepfileParserOptions()); + /// Parse an input file. Input must be NUL-terminated. /// Warning: may mutate the content in-place and parsed StringPieces are /// pointers within it. @@ -30,6 +46,7 @@ struct DepfileParser { StringPiece out_; vector ins_; + DepfileParserOptions options_; }; #endif // NINJA_DEPFILE_PARSER_H_ diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 98c1621d46..f8c94b3c96 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -13,6 +13,12 @@ // limitations under the License. #include "depfile_parser.h" +#include "util.h" + +DepfileParser::DepfileParser(DepfileParserOptions options) + : options_(options) +{ +} // A note on backslashes in Makefiles, from reading the docs: // Backslash-newline is the line continuation character. @@ -34,8 +40,13 @@ bool DepfileParser::Parse(string* content, string* err) { // parsing_targets: whether we are parsing targets or dependencies. char* in = &(*content)[0]; char* end = in + content->size(); + bool have_target = false; + bool have_secondary_target_on_this_rule = false; + bool have_newline_since_primary_target = false; + bool warned_distinct_target_lines = false; bool parsing_targets = true; while (in < end) { + bool have_newline = false; // out: current output point (typically same as in, but can fall behind // as we de-escape backslashes). char* out = in; @@ -44,10 +55,12 @@ bool DepfileParser::Parse(string* content, string* err) { for (;;) { // start: beginning of the current parsed span. const char* start = in; + char* yymarker = NULL; /*!re2c re2c:define:YYCTYPE = "unsigned char"; re2c:define:YYCURSOR = in; re2c:define:YYLIMIT = end; + re2c:define:YYMARKER = yymarker; re2c:yyfill:enable = 0; @@ -55,7 +68,8 @@ bool DepfileParser::Parse(string* content, string* err) { re2c:indent:string = " "; nul = "\000"; - escape = [ \\#*[|]; + escape = [ \\#*[|\]]; + newline = '\r'?'\n'; '\\' escape { // De-escape backslashed character. @@ -73,7 +87,7 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = yych; continue; } - [a-zA-Z0-9+,/_:.~()}{@=!\x80-\xFF-]+ { + [a-zA-Z0-9+,/_:.~()}{%@=!\x80-\xFF-]+ { // Got a span of plain text. int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. @@ -85,6 +99,15 @@ bool DepfileParser::Parse(string* content, string* err) { nul { break; } + '\\' newline { + // A line continuation ends the current file name. + break; + } + newline { + // A newline ends the current file name and the current rule. + have_newline = true; + break; + } [^] { // For any other character (e.g. whitespace), swallow it here, // allowing the outer logic to loop around again. @@ -94,25 +117,52 @@ bool DepfileParser::Parse(string* content, string* err) { } int len = (int)(out - filename); - const bool is_target = parsing_targets; + const bool is_dependency = !parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. parsing_targets = false; + have_target = true; } - if (len == 0) - continue; + if (len > 0) { + if (is_dependency) { + if (have_secondary_target_on_this_rule) { + if (!have_newline_since_primary_target) { + *err = "depfile has multiple output paths"; + return false; + } else if (options_.depfile_distinct_target_lines_action_ == + kDepfileDistinctTargetLinesActionError) { + *err = + "depfile has multiple output paths (on separate lines)" + " [-w depfilemulti=err]"; + return false; + } else { + if (!warned_distinct_target_lines) { + warned_distinct_target_lines = true; + Warning("depfile has multiple output paths (on separate lines); " + "continuing anyway [-w depfilemulti=warn]"); + } + continue; + } + } + ins_.push_back(StringPiece(filename, len)); + } else if (!out_.str_) { + out_ = StringPiece(filename, len); + } else if (out_ != StringPiece(filename, len)) { + have_secondary_target_on_this_rule = true; + } + } - if (!is_target) { - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - *err = "depfile has multiple output paths"; - return false; + if (have_newline) { + // A newline ends a rule so the next filename will be a new target. + parsing_targets = true; + have_secondary_target_on_this_rule = false; + if (have_target) { + have_newline_since_primary_target = true; + } } } - if (parsing_targets) { + if (!have_target) { *err = "expected ':' in depfile"; return false; } diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index ee798f8239..52fe7cd789 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -119,15 +119,16 @@ TEST_F(DepfileParserTest, SpecialChars) { // https://github.com/google/libcxx/tree/master/test/iterators/stream.iterators/istreambuf.iterator/ string err; EXPECT_TRUE(Parse( -"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" -" en@quot.header~ t+t-x!=1 \n" -" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif\n" -" Fu\303\244ball", +"C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \\\n" +" en@quot.header~ t+t-x!=1 \\\n" +" openldap/slapd.d/cn=config/cn=schema/cn={0}core.ldif\\\n" +" Fu\303\244ball\\\n" +" a\\[1\\]b@2%c", &err)); ASSERT_EQ("", err); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", parser_.out_.AsString()); - ASSERT_EQ(4u, parser_.ins_.size()); + ASSERT_EQ(5u, parser_.ins_.size()); EXPECT_EQ("en@quot.header~", parser_.ins_[0].AsString()); EXPECT_EQ("t+t-x!=1", @@ -136,6 +137,8 @@ TEST_F(DepfileParserTest, SpecialChars) { parser_.ins_[2].AsString()); EXPECT_EQ("Fu\303\244ball", parser_.ins_[3].AsString()); + EXPECT_EQ("a[1]b@2%c", + parser_.ins_[4].AsString()); } TEST_F(DepfileParserTest, UnifyMultipleOutputs) { @@ -155,3 +158,133 @@ TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { EXPECT_FALSE(Parse("foo bar: x y z", &err)); ASSERT_EQ("depfile has multiple output paths", err); } + +TEST_F(DepfileParserTest, MultipleEmptyRules) { + string err; + EXPECT_TRUE(Parse("foo: x\n" + "foo: \n" + "foo:\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMultipleRulesLF) { + string err; + EXPECT_TRUE(Parse("foo: x\n" + "foo: y\n" + "foo \\\n" + "foo: z\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMultipleRulesCRLF) { + string err; + EXPECT_TRUE(Parse("foo: x\r\n" + "foo: y\r\n" + "foo \\\r\n" + "foo: z\r\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMixedRulesLF) { + string err; + EXPECT_TRUE(Parse("foo: x\\\n" + " y\n" + "foo \\\n" + "foo: z\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, UnifyMixedRulesCRLF) { + string err; + EXPECT_TRUE(Parse("foo: x\\\r\n" + " y\r\n" + "foo \\\r\n" + "foo: z\r\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, IndentedRulesLF) { + string err; + EXPECT_TRUE(Parse(" foo: x\n" + " foo: y\n" + " foo: z\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, IndentedRulesCRLF) { + string err; + EXPECT_TRUE(Parse(" foo: x\r\n" + " foo: y\r\n" + " foo: z\r\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, TolerateMP) { + string err; + EXPECT_TRUE(Parse("foo: x y z\n" + "x:\n" + "y:\n" + "z:\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, MultipleRulesTolerateMP) { + string err; + EXPECT_TRUE(Parse("foo: x\n" + "x:\n" + "foo: y\n" + "y:\n" + "foo: z\n" + "z:\n", &err)); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, MultipleRulesRejectDifferentOutputs) { + // check that multiple different outputs are rejected by the parser + // when spread across multiple rules + DepfileParserOptions parser_opts; + parser_opts.depfile_distinct_target_lines_action_ = + kDepfileDistinctTargetLinesActionError; + DepfileParser parser(parser_opts); + string err; + string input = + "foo: x y\n" + "bar: y z\n"; + EXPECT_FALSE(parser.Parse(&input, &err)); + ASSERT_EQ("depfile has multiple output paths (on separate lines)" + " [-w depfilemulti=err]", err); +} diff --git a/src/deps_log.cc b/src/deps_log.cc index 89c60232b7..0bb96f3759 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -20,6 +20,9 @@ #include #ifndef _WIN32 #include +#elif defined(_MSC_VER) && (_MSC_VER < 1900) +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; #endif #include "graph.h" @@ -30,7 +33,7 @@ // The version is stored as 4 bytes after the signature and also serves as a // byte order mark. Signature and version combined are 16 bytes long. const char kFileSignature[] = "# ninjadeps\n"; -const int kCurrentVersion = 3; +const int kCurrentVersion = 4; // Record size is currently limited to less than the full 32 bit, due to // internal buffers having to have this size. @@ -124,7 +127,7 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, return true; // Update on-disk representation. - unsigned size = 4 * (1 + 1 + node_count); + unsigned size = 4 * (1 + 2 + node_count); if (size > kMaxRecordSize) { errno = ERANGE; return false; @@ -135,8 +138,11 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, int id = node->id(); if (fwrite(&id, 4, 1, file_) < 1) return false; - int timestamp = mtime; - if (fwrite(×tamp, 4, 1, file_) < 1) + uint32_t mtime_part = static_cast(mtime & 0xffffffff); + if (fwrite(&mtime_part, 4, 1, file_) < 1) + return false; + mtime_part = static_cast((mtime >> 32) & 0xffffffff); + if (fwrite(&mtime_part, 4, 1, file_) < 1) return false; for (int i = 0; i < node_count; ++i) { id = nodes[i]->id(); @@ -209,7 +215,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { bool is_deps = (size >> 31) != 0; size = size & 0x7FFFFFFF; - if (fread(buf, size, 1, f) < 1 || size > kMaxRecordSize) { + if (size > kMaxRecordSize || fread(buf, size, 1, f) < 1) { read_failed = true; break; } @@ -218,9 +224,11 @@ bool DepsLog::Load(const string& path, State* state, string* err) { assert(size % 4 == 0); int* deps_data = reinterpret_cast(buf); int out_id = deps_data[0]; - int mtime = deps_data[1]; - deps_data += 2; - int deps_count = (size / 4) - 2; + TimeStamp mtime; + mtime = (TimeStamp)(((uint64_t)(unsigned int)deps_data[2] << 32) | + (uint64_t)(unsigned int)deps_data[1]); + deps_data += 3; + int deps_count = (size / 4) - 3; Deps* deps = new Deps(mtime, deps_count); for (int i = 0; i < deps_count; ++i) { diff --git a/src/deps_log.h b/src/deps_log.h index cec0257cef..3812a28a80 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -57,7 +57,9 @@ struct State; /// one's complement of the expected index of the record (to detect /// concurrent writes of multiple ninja processes to the log). /// dependency records are an array of 4-byte integers -/// [output path id, output path mtime, input path id, input path id...] +/// [output path id, +/// output path mtime (lower 4 bytes), output path mtime (upper 4 bytes), +/// input path id, input path id...] /// (The mtime is compared against the on-disk output path mtime /// to verify the stored data is up-to-date.) /// If two records reference the same output the latter one in the file @@ -75,10 +77,10 @@ struct DepsLog { // Reading (startup-time) interface. struct Deps { - Deps(int mtime, int node_count) + Deps(int64_t mtime, int node_count) : mtime(mtime), node_count(node_count), nodes(new Node*[node_count]) {} ~Deps() { delete [] nodes; } - int mtime; + TimeStamp mtime; int node_count; Node** nodes; }; diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index 89f7be159e..0cdeb45bed 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -143,7 +143,7 @@ TEST_F(DepsLogTest, DoubleEntry) { ASSERT_GT(file_size, 0); } - // Now reload the file, and readd the same deps. + // Now reload the file, and read the same deps. { State state; DepsLog log; @@ -203,7 +203,7 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_GT(file_size, 0); } - // Now reload the file, and add slighly different deps. + // Now reload the file, and add slightly different deps. int file_size_2; { State state; diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 28530b19d0..d4c2fb0878 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -35,14 +35,15 @@ namespace { string DirName(const string& path) { #ifdef _WIN32 - const char kPathSeparators[] = "\\/"; + static const char kPathSeparators[] = "\\/"; #else - const char kPathSeparators[] = "/"; + static const char kPathSeparators[] = "/"; #endif + static const char* const kEnd = kPathSeparators + sizeof(kPathSeparators) - 1; + string::size_type slash_pos = path.find_last_of(kPathSeparators); if (slash_pos == string::npos) return string(); // Nothing to do. - const char* const kEnd = kPathSeparators + strlen(kPathSeparators); while (slash_pos > 0 && std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd) --slash_pos; @@ -61,17 +62,16 @@ int MakeDir(const string& path) { TimeStamp TimeStampFromFileTime(const FILETIME& filetime) { // FILETIME is in 100-nanosecond increments since the Windows epoch. // We don't much care about epoch correctness but we do want the - // resulting value to fit in an integer. + // resulting value to fit in a 64-bit integer. uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | ((uint64_t)filetime.dwLowDateTime); - mtime /= 1000000000LL / 100; // 100ns -> s. - mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). - return (TimeStamp)mtime; + // 1600 epoch -> 2000 epoch (subtract 400 years). + return (TimeStamp)mtime - 12622770400LL * (1000000000LL / 100); } TimeStamp StatSingleFile(const string& path, string* err) { WIN32_FILE_ATTRIBUTE_DATA attrs; - if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { + if (!GetFileAttributesExA(path.c_str(), GetFileExInfoStandard, &attrs)) { DWORD win_err = GetLastError(); if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) return 0; @@ -113,6 +113,11 @@ bool StatAllFilesInDir(const string& dir, map* stamps, } do { string lowername = ffd.cFileName; + if (lowername == "..") { + // Seems to just copy the timestamp for ".." from ".", which is wrong. + // This is the case at least on NTFS under Windows 7. + continue; + } transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower); stamps->insert(make_pair(lowername, TimeStampFromFileTime(ffd.ftLastWriteTime))); @@ -165,6 +170,11 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { string dir = DirName(path); string base(path.substr(dir.size() ? dir.size() + 1 : 0)); + if (base == "..") { + // StatAllFilesInDir does not report any information for base = "..". + base = "."; + dir = path; + } transform(dir.begin(), dir.end(), dir.begin(), ::tolower); transform(base.begin(), base.end(), base.begin(), ::tolower); @@ -192,7 +202,22 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { // that it doesn't exist. if (st.st_mtime == 0) return 1; - return st.st_mtime; +#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE) + return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL + + st.st_mtimespec.tv_nsec); +#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ + defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || defined(__FreeBSD__)) + // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html + // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above. + // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar + // For bionic, C and POSIX API is always enabled. + // For solaris, see https://docs.oracle.com/cd/E88353_01/html/E37841/stat-2.html. + return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec; +#elif defined(_AIX) + return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n; +#else + return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec; +#endif #endif } diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index d7fb8f8eb6..bac515d235 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -87,6 +87,8 @@ TEST_F(DiskInterfaceTest, StatExistingDir) { string err; ASSERT_TRUE(disk_.MakeDir("subdir")); ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir")); + EXPECT_GT(disk_.Stat("..", &err), 1); + EXPECT_EQ("", err); EXPECT_GT(disk_.Stat(".", &err), 1); EXPECT_EQ("", err); EXPECT_GT(disk_.Stat("subdir", &err), 1); @@ -105,7 +107,6 @@ TEST_F(DiskInterfaceTest, StatExistingDir) { #ifdef _WIN32 TEST_F(DiskInterfaceTest, StatCache) { string err; - disk_.AllowStatCache(true); ASSERT_TRUE(Touch("file1")); ASSERT_TRUE(Touch("fiLE2")); @@ -115,6 +116,10 @@ TEST_F(DiskInterfaceTest, StatCache) { ASSERT_TRUE(Touch("subdir\\SUBFILE2")); ASSERT_TRUE(Touch("subdir\\SUBFILE3")); + disk_.AllowStatCache(false); + TimeStamp parent_stat_uncached = disk_.Stat("..", &err); + disk_.AllowStatCache(true); + EXPECT_GT(disk_.Stat("FIle1", &err), 1); EXPECT_EQ("", err); EXPECT_GT(disk_.Stat("file1", &err), 1); @@ -125,6 +130,8 @@ TEST_F(DiskInterfaceTest, StatCache) { EXPECT_GT(disk_.Stat("sUbdir\\suBFile1", &err), 1); EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("..", &err), 1); + EXPECT_EQ("", err); EXPECT_GT(disk_.Stat(".", &err), 1); EXPECT_EQ("", err); EXPECT_GT(disk_.Stat("subdir", &err), 1); @@ -132,11 +139,15 @@ TEST_F(DiskInterfaceTest, StatCache) { EXPECT_GT(disk_.Stat("subdir/subsubdir", &err), 1); EXPECT_EQ("", err); +#ifndef _MSC_VER // TODO: Investigate why. Also see https://github.com/ninja-build/ninja/pull/1423 EXPECT_EQ(disk_.Stat("subdir", &err), disk_.Stat("subdir/.", &err)); EXPECT_EQ("", err); EXPECT_EQ(disk_.Stat("subdir", &err), disk_.Stat("subdir/subsubdir/..", &err)); +#endif + EXPECT_EQ("", err); + EXPECT_EQ(disk_.Stat("..", &err), parent_stat_uncached); EXPECT_EQ("", err); EXPECT_EQ(disk_.Stat("subdir/subsubdir", &err), disk_.Stat("subdir/subsubdir/.", &err)); @@ -202,7 +213,7 @@ TEST_F(DiskInterfaceTest, RemoveFile) { struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { - StatTest() : scan_(&state_, NULL, NULL, this) {} + StatTest() : scan_(&state_, NULL, NULL, this, NULL) {} // DiskInterface implementation. virtual TimeStamp Stat(const string& path, string* err) const; diff --git a/src/graph.cc b/src/graph.cc index ce4ea774f2..9c2f784ffd 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -233,7 +233,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, if (output_mtime < most_recent_input->mtime()) { EXPLAIN("%soutput %s older than most recent input %s " - "(%d vs %d)", + "(%" PRId64 " vs %" PRId64 ")", used_restat ? "restat of " : "", output->path().c_str(), most_recent_input->path().c_str(), output_mtime, most_recent_input->mtime()); @@ -257,7 +257,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, // mtime of the most recent input. This can occur even when the mtime // on disk is newer if a previous run wrote to the output file but // exited with an error or was interrupted. - EXPLAIN("recorded mtime of %s older than most recent input %s (%d vs %d)", + EXPLAIN("recorded mtime of %s older than most recent input %s (%" PRId64 " vs %" PRId64 ")", output->path().c_str(), most_recent_input->path().c_str(), entry->mtime, most_recent_input->mtime()); return true; @@ -441,7 +441,7 @@ string Node::PathDecanonicalized(const string& path, uint64_t slash_bits) { } void Node::Dump(const char* prefix) const { - printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", + printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ", prefix, path().c_str(), this, mtime(), mtime() ? "" : " (:missing)", dirty() ? " dirty" : " clean"); @@ -491,7 +491,9 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, return false; } - DepfileParser depfile; + DepfileParser depfile(depfile_parser_options_ + ? *depfile_parser_options_ + : DepfileParserOptions()); string depfile_err; if (!depfile.Parse(&content, &depfile_err)) { *err = path + ": " + depfile_err; @@ -547,7 +549,7 @@ bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, string* err) { // Deps are invalid if the output is newer than the deps. if (output->mtime() > deps->mtime) { - EXPLAIN("stored deps info out of date for '%s' (%d vs %d)", + EXPLAIN("stored deps info out of date for '%s' (%" PRId64 " vs %" PRId64 ")", output->path().c_str(), deps->mtime, output->mtime()); return false; } diff --git a/src/graph.h b/src/graph.h index a8f0641d5f..d58fecd2c6 100644 --- a/src/graph.h +++ b/src/graph.h @@ -24,6 +24,7 @@ using namespace std; #include "util.h" struct BuildLog; +struct DepfileParserOptions; struct DiskInterface; struct DepsLog; struct Edge; @@ -209,8 +210,10 @@ struct Edge { /// "depfile" attribute in build files. struct ImplicitDepLoader { ImplicitDepLoader(State* state, DepsLog* deps_log, - DiskInterface* disk_interface) - : state_(state), disk_interface_(disk_interface), deps_log_(deps_log) {} + DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options) + : state_(state), disk_interface_(disk_interface), deps_log_(deps_log), + depfile_parser_options_(depfile_parser_options) {} /// Load implicit dependencies for \a edge. /// @return false on error (without filling \a err if info is just missing @@ -242,6 +245,7 @@ struct ImplicitDepLoader { State* state_; DiskInterface* disk_interface_; DepsLog* deps_log_; + DepfileParserOptions const* depfile_parser_options_; }; @@ -249,10 +253,11 @@ struct ImplicitDepLoader { /// and updating the dirty/outputs_ready state of all the nodes and edges. struct DependencyScan { DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface) + DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options) : build_log_(build_log), disk_interface_(disk_interface), - dep_loader_(state, deps_log, disk_interface) {} + dep_loader_(state, deps_log, disk_interface, depfile_parser_options) {} /// Update the |dirty_| state of the given node by inspecting its input edge. /// Examine inputs, outputs, and command lines to judge whether an edge diff --git a/src/graph_test.cc b/src/graph_test.cc index 422bc9a053..4a66831ae0 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -18,7 +18,7 @@ #include "test.h" struct GraphTest : public StateTestWithBuiltinRules { - GraphTest() : scan_(&state_, NULL, NULL, &fs_) {} + GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL) {} VirtualFileSystem fs_; DependencyScan scan_; diff --git a/src/hash_map.h b/src/hash_map.h index a91aeb9960..55d2c9d46d 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -18,6 +18,7 @@ #include #include #include "string_piece.h" +#include "util.h" // MurmurHash2, by Austin Appleby static inline @@ -40,7 +41,9 @@ unsigned int MurmurHash2(const void* key, size_t len) { } switch (len) { case 3: h ^= data[2] << 16; + NINJA_FALLTHROUGH; case 2: h ^= data[1] << 8; + NINJA_FALLTHROUGH; case 1: h ^= data[0]; h *= m; }; diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 459329bc99..79bf5b46a9 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -26,6 +26,21 @@ namespace { +bool InternalGetFullPathName(const StringPiece& file_name, char* buffer, + size_t buffer_length, string *err) { + DWORD result_size = GetFullPathNameA(file_name.AsString().c_str(), + buffer_length, buffer, NULL); + if (result_size == 0) { + *err = "GetFullPathNameA(" + file_name.AsString() + "): " + + GetLastErrorString(); + return false; + } else if (result_size > buffer_length) { + *err = "path too long"; + return false; + } + return true; +} + bool IsPathSeparator(char c) { return c == '/' || c == '\\'; } @@ -54,15 +69,19 @@ bool SameDriveFast(StringPiece a, StringPiece b) { } // Return true if paths a and b are on the same Windows drive. -bool SameDrive(StringPiece a, StringPiece b) { +bool SameDrive(StringPiece a, StringPiece b, string* err) { if (SameDriveFast(a, b)) { return true; } char a_absolute[_MAX_PATH]; char b_absolute[_MAX_PATH]; - GetFullPathName(a.AsString().c_str(), sizeof(a_absolute), a_absolute, NULL); - GetFullPathName(b.AsString().c_str(), sizeof(b_absolute), b_absolute, NULL); + if (!InternalGetFullPathName(a, a_absolute, sizeof(a_absolute), err)) { + return false; + } + if (!InternalGetFullPathName(b, b_absolute, sizeof(b_absolute), err)) { + return false; + } char a_drive[_MAX_DIR]; char b_drive[_MAX_DIR]; _splitpath(a_absolute, a_drive, NULL, NULL, NULL); @@ -106,11 +125,15 @@ bool IsFullPathName(StringPiece s) { } // anonymous namespace IncludesNormalize::IncludesNormalize(const string& relative_to) { - relative_to_ = AbsPath(relative_to); + string err; + relative_to_ = AbsPath(relative_to, &err); + if (!err.empty()) { + Fatal("Initializing IncludesNormalize(): %s", err.c_str()); + } split_relative_to_ = SplitStringPiece(relative_to_, '/'); } -string IncludesNormalize::AbsPath(StringPiece s) { +string IncludesNormalize::AbsPath(StringPiece s, string* err) { if (IsFullPathName(s)) { string result = s.AsString(); for (size_t i = 0; i < result.size(); ++i) { @@ -122,7 +145,9 @@ string IncludesNormalize::AbsPath(StringPiece s) { } char result[_MAX_PATH]; - GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + if (!InternalGetFullPathName(s, result, sizeof(result), err)) { + return ""; + } for (char* c = result; *c; ++c) if (*c == '\\') *c = '/'; @@ -130,8 +155,10 @@ string IncludesNormalize::AbsPath(StringPiece s) { } string IncludesNormalize::Relativize( - StringPiece path, const vector& start_list) { - string abs_path = AbsPath(path); + StringPiece path, const vector& start_list, string* err) { + string abs_path = AbsPath(path, err); + if (!err->empty()) + return ""; vector path_list = SplitStringPiece(abs_path, '/'); int i; for (i = 0; i < static_cast(min(start_list.size(), path_list.size())); @@ -165,12 +192,18 @@ bool IncludesNormalize::Normalize(const string& input, if (!CanonicalizePath(copy, &len, &slash_bits, err)) return false; StringPiece partially_fixed(copy, len); - string abs_input = AbsPath(partially_fixed); + string abs_input = AbsPath(partially_fixed, err); + if (!err->empty()) + return false; - if (!SameDrive(abs_input, relative_to_)) { + if (!SameDrive(abs_input, relative_to_, err)) { + if (!err->empty()) + return false; *result = partially_fixed.AsString(); return true; } - *result = Relativize(abs_input, split_relative_to_); + *result = Relativize(abs_input, split_relative_to_, err); + if (!err->empty()) + return false; return true; } diff --git a/src/includes_normalize.h b/src/includes_normalize.h index 3811e53840..0339581ec8 100644 --- a/src/includes_normalize.h +++ b/src/includes_normalize.h @@ -25,9 +25,9 @@ struct IncludesNormalize { IncludesNormalize(const string& relative_to); // Internal utilities made available for testing, maybe useful otherwise. - static string AbsPath(StringPiece s); + static string AbsPath(StringPiece s, string* err); static string Relativize(StringPiece path, - const vector& start_list); + const vector& start_list, string* err); /// Normalize by fixing slashes style, fixing redundant .. and . and makes the /// path |input| relative to |this->relative_to_| and store to |result|. diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index eac36fd2d7..dbcdbe0eb8 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -58,9 +58,12 @@ TEST(IncludesNormalize, Simple) { } TEST(IncludesNormalize, WithRelative) { + string err; string currentdir = GetCurDir(); EXPECT_EQ("c", NormalizeRelativeAndCheckNoError("a/b/c", "a/b")); - EXPECT_EQ("a", NormalizeAndCheckNoError(IncludesNormalize::AbsPath("a"))); + EXPECT_EQ("a", + NormalizeAndCheckNoError(IncludesNormalize::AbsPath("a", &err))); + EXPECT_EQ("", err); EXPECT_EQ(string("../") + currentdir + string("/a"), NormalizeRelativeAndCheckNoError("a", "../b")); EXPECT_EQ(string("../") + currentdir + string("/a/b"), @@ -138,3 +141,27 @@ TEST(IncludesNormalize, LongInvalidPath) { EXPECT_EQ(forward_slashes.substr(cwd_len + 1), NormalizeAndCheckNoError(kExactlyMaxPath)); } + +TEST(IncludesNormalize, ShortRelativeButTooLongAbsolutePath) { + string result, err; + IncludesNormalize normalizer("."); + // A short path should work + EXPECT_TRUE(normalizer.Normalize("a", &result, &err)); + EXPECT_EQ("", err); + + // Construct max size path having cwd prefix. + // kExactlyMaxPath = "aaaa\\aaaa...aaaa\0"; + char kExactlyMaxPath[_MAX_PATH + 1]; + for (int i = 0; i < _MAX_PATH; ++i) { + if (i < _MAX_PATH - 1 && i % 10 == 4) + kExactlyMaxPath[i] = '\\'; + else + kExactlyMaxPath[i] = 'a'; + } + kExactlyMaxPath[_MAX_PATH] = '\0'; + EXPECT_EQ(strlen(kExactlyMaxPath), _MAX_PATH); + + // Make sure a path that's exactly _MAX_PATH long fails with a proper error. + EXPECT_FALSE(normalizer.Normalize(kExactlyMaxPath, &result, &err)); + EXPECT_TRUE(err.find("GetFullPathName") != string::npos); +} diff --git a/src/lexer.cc b/src/lexer.cc index 37b867885c..35ae97bc58 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.13.5 */ +/* Generated by re2c 0.16 */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,14 +23,14 @@ bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; - const char* context = input_.str_; + const char* line_start = input_.str_; for (const char* p = input_.str_; p < last_token_; ++p) { if (*p == '\n') { ++line; - context = p + 1; + line_start = p + 1; } } - int col = last_token_ ? (int)(last_token_ - context) : 0; + int col = last_token_ ? (int)(last_token_ - line_start) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); @@ -43,12 +43,12 @@ bool Lexer::Error(const string& message, string* err) { int len; bool truncated = true; for (len = 0; len < kTruncateColumn; ++len) { - if (context[len] == 0 || context[len] == '\n') { + if (line_start[len] == 0 || line_start[len] == '\n') { truncated = false; break; } } - *err += string(context, len); + *err += string(line_start, len); if (truncated) *err += "..."; *err += "\n"; @@ -126,305 +126,325 @@ Lexer::Token Lexer::ReadToken() { unsigned char yych; unsigned int yyaccept = 0; static const unsigned char yybm[] = { - 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 192, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 96, 96, 64, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 64, 64, 64, 64, 64, 64, - 64, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 64, 64, 64, 64, 96, - 64, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 96, 96, 96, 96, 96, - 96, 96, 96, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 64, 64, + 0, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 160, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 192, 192, 128, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 128, 128, 128, 128, 128, 128, + 128, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 128, 128, 128, 128, 192, + 128, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, }; - yych = *p; - if (yych <= 'Z') { - if (yych <= '#') { + if (yybm[0+yych] & 32) { + goto yy9; + } + if (yych <= '^') { + if (yych <= ',') { if (yych <= '\f') { - if (yych <= 0x00) goto yy23; - if (yych == '\n') goto yy7; - goto yy25; + if (yych <= 0x00) goto yy2; + if (yych == '\n') goto yy6; + goto yy4; } else { - if (yych <= 0x1F) { - if (yych <= '\r') goto yy6; - goto yy25; - } else { - if (yych <= ' ') goto yy2; - if (yych <= '"') goto yy25; - goto yy4; - } + if (yych <= '\r') goto yy8; + if (yych == '#') goto yy12; + goto yy4; } } else { - if (yych <= '9') { - if (yych <= ',') goto yy25; - if (yych == '/') goto yy25; - goto yy22; + if (yych <= ':') { + if (yych == '/') goto yy4; + if (yych <= '9') goto yy13; + goto yy16; } else { - if (yych <= '<') { - if (yych <= ':') goto yy16; - goto yy25; + if (yych <= '=') { + if (yych <= '<') goto yy4; + goto yy18; } else { - if (yych <= '=') goto yy14; - if (yych <= '@') goto yy25; - goto yy22; + if (yych <= '@') goto yy4; + if (yych <= 'Z') goto yy13; + goto yy4; } } } } else { if (yych <= 'i') { - if (yych <= 'a') { - if (yych == '_') goto yy22; - if (yych <= '`') goto yy25; - goto yy22; + if (yych <= 'b') { + if (yych == '`') goto yy4; + if (yych <= 'a') goto yy13; + goto yy20; } else { - if (yych <= 'c') { - if (yych <= 'b') goto yy9; - goto yy22; - } else { - if (yych <= 'd') goto yy13; - if (yych <= 'h') goto yy22; - goto yy20; - } + if (yych == 'd') goto yy21; + if (yych <= 'h') goto yy13; + goto yy22; } } else { if (yych <= 'r') { - if (yych == 'p') goto yy11; - if (yych <= 'q') goto yy22; - goto yy12; + if (yych == 'p') goto yy23; + if (yych <= 'q') goto yy13; + goto yy24; } else { if (yych <= 'z') { - if (yych <= 's') goto yy21; - goto yy22; + if (yych <= 's') goto yy25; + goto yy13; } else { - if (yych == '|') goto yy18; - goto yy25; + if (yych == '|') goto yy26; + goto yy4; } } } } yy2: - yyaccept = 0; - yych = *(q = ++p); - goto yy73; -yy3: - { token = INDENT; break; } + ++p; + { token = TEOF; break; } yy4: - yyaccept = 1; - yych = *(q = ++p); - if (yych >= 0x01) goto yy68; + ++p; yy5: { token = ERROR; break; } yy6: - yych = *++p; - if (yych == '\n') goto yy65; - goto yy5; -yy7: ++p; -yy8: { token = NEWLINE; break; } +yy8: + yych = *++p; + if (yych == '\n') goto yy28; + goto yy5; yy9: - ++p; - if ((yych = *p) == 'u') goto yy60; - goto yy27; -yy10: - { token = IDENT; break; } + yyaccept = 0; + q = ++p; + yych = *p; + if (yybm[0+yych] & 32) { + goto yy9; + } + if (yych <= '\f') { + if (yych == '\n') goto yy6; + } else { + if (yych <= '\r') goto yy30; + if (yych == '#') goto yy32; + } yy11: - yych = *++p; - if (yych == 'o') goto yy56; - goto yy27; + { token = INDENT; break; } yy12: - yych = *++p; - if (yych == 'u') goto yy52; - goto yy27; + yyaccept = 1; + yych = *(q = ++p); + if (yych <= 0x00) goto yy5; + goto yy33; yy13: - yych = *++p; - if (yych == 'e') goto yy45; - goto yy27; -yy14: ++p; - { token = EQUALS; break; } + yych = *p; +yy14: + if (yybm[0+yych] & 64) { + goto yy13; + } + { token = IDENT; break; } yy16: ++p; { token = COLON; break; } yy18: ++p; - if ((yych = *p) == '|') goto yy43; - { token = PIPE; break; } + { token = EQUALS; break; } yy20: yych = *++p; - if (yych == 'n') goto yy36; - goto yy27; + if (yych == 'u') goto yy36; + goto yy14; yy21: yych = *++p; - if (yych == 'u') goto yy28; - goto yy27; + if (yych == 'e') goto yy37; + goto yy14; yy22: yych = *++p; - goto yy27; + if (yych == 'n') goto yy38; + goto yy14; yy23: - ++p; - { token = TEOF; break; } + yych = *++p; + if (yych == 'o') goto yy39; + goto yy14; +yy24: + yych = *++p; + if (yych == 'u') goto yy40; + goto yy14; yy25: yych = *++p; - goto yy5; + if (yych == 'u') goto yy41; + goto yy14; yy26: ++p; - yych = *p; -yy27: - if (yybm[0+yych] & 32) { - goto yy26; - } - goto yy10; + if ((yych = *p) == '|') goto yy42; + { token = PIPE; break; } yy28: + ++p; + { token = NEWLINE; break; } +yy30: yych = *++p; - if (yych != 'b') goto yy27; - yych = *++p; - if (yych != 'n') goto yy27; - yych = *++p; - if (yych != 'i') goto yy27; - yych = *++p; - if (yych != 'n') goto yy27; - yych = *++p; - if (yych != 'j') goto yy27; - yych = *++p; - if (yych != 'a') goto yy27; + if (yych == '\n') goto yy28; +yy31: + p = q; + if (yyaccept == 0) { + goto yy11; + } else { + goto yy5; + } +yy32: ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; + yych = *p; +yy33: + if (yybm[0+yych] & 128) { + goto yy32; } - { token = SUBNINJA; break; } + if (yych <= 0x00) goto yy31; + ++p; + { continue; } yy36: yych = *++p; - if (yych != 'c') goto yy27; + if (yych == 'i') goto yy44; + goto yy14; +yy37: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'f') goto yy45; + goto yy14; +yy38: yych = *++p; - if (yych != 'u') goto yy27; + if (yych == 'c') goto yy46; + goto yy14; +yy39: yych = *++p; - if (yych != 'd') goto yy27; + if (yych == 'o') goto yy47; + goto yy14; +yy40: yych = *++p; - if (yych != 'e') goto yy27; - ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; - } - { token = INCLUDE; break; } -yy43: + if (yych == 'l') goto yy48; + goto yy14; +yy41: + yych = *++p; + if (yych == 'b') goto yy49; + goto yy14; +yy42: ++p; { token = PIPE2; break; } +yy44: + yych = *++p; + if (yych == 'l') goto yy50; + goto yy14; yy45: yych = *++p; - if (yych != 'f') goto yy27; + if (yych == 'a') goto yy51; + goto yy14; +yy46: yych = *++p; - if (yych != 'a') goto yy27; + if (yych == 'l') goto yy52; + goto yy14; +yy47: yych = *++p; - if (yych != 'u') goto yy27; + if (yych == 'l') goto yy53; + goto yy14; +yy48: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'e') goto yy55; + goto yy14; +yy49: yych = *++p; - if (yych != 't') goto yy27; - ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; - } - { token = DEFAULT; break; } -yy52: + if (yych == 'n') goto yy57; + goto yy14; +yy50: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'd') goto yy58; + goto yy14; +yy51: yych = *++p; - if (yych != 'e') goto yy27; + if (yych == 'u') goto yy60; + goto yy14; +yy52: + yych = *++p; + if (yych == 'u') goto yy61; + goto yy14; +yy53: ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; + } + { token = POOL; break; } +yy55: + ++p; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } { token = RULE; break; } -yy56: +yy57: yych = *++p; - if (yych != 'o') goto yy27; - yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'i') goto yy62; + goto yy14; +yy58: ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } - { token = POOL; break; } + { token = BUILD; break; } yy60: yych = *++p; - if (yych != 'i') goto yy27; + if (yych == 'l') goto yy63; + goto yy14; +yy61: yych = *++p; - if (yych != 'l') goto yy27; + if (yych == 'd') goto yy64; + goto yy14; +yy62: yych = *++p; - if (yych != 'd') goto yy27; - ++p; - if (yybm[0+(yych = *p)] & 32) { - goto yy26; - } - { token = BUILD; break; } + if (yych == 'n') goto yy65; + goto yy14; +yy63: + yych = *++p; + if (yych == 't') goto yy66; + goto yy14; +yy64: + yych = *++p; + if (yych == 'e') goto yy68; + goto yy14; yy65: + yych = *++p; + if (yych == 'j') goto yy70; + goto yy14; +yy66: ++p; - { token = NEWLINE; break; } -yy67: - ++p; - yych = *p; -yy68: - if (yybm[0+yych] & 64) { - goto yy67; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } - if (yych >= 0x01) goto yy70; -yy69: - p = q; - if (yyaccept <= 0) { - goto yy3; - } else { - goto yy5; - } -yy70: + { token = DEFAULT; break; } +yy68: ++p; - { continue; } -yy72: - yyaccept = 0; - q = ++p; - yych = *p; -yy73: - if (yybm[0+yych] & 128) { - goto yy72; - } - if (yych <= '\f') { - if (yych != '\n') goto yy3; - } else { - if (yych <= '\r') goto yy75; - if (yych == '#') goto yy67; - goto yy3; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; } + { token = INCLUDE; break; } +yy70: yych = *++p; - goto yy8; -yy75: + if (yych != 'a') goto yy14; ++p; - if ((yych = *p) == '\n') goto yy65; - goto yy69; + if (yybm[0+(yych = *p)] & 64) { + goto yy13; + } + { token = SUBNINJA; break; } } } @@ -487,49 +507,42 @@ void Lexer::EatWhitespace() { 0, 0, 0, 0, 0, 0, 0, 0, }; yych = *p; - if (yych <= ' ') { - if (yych <= 0x00) goto yy82; - if (yych <= 0x1F) goto yy84; - } else { - if (yych == '$') goto yy80; - goto yy84; + if (yybm[0+yych] & 128) { + goto yy79; } + if (yych <= 0x00) goto yy75; + if (yych == '$') goto yy82; + goto yy77; +yy75: ++p; - yych = *p; - goto yy92; -yy79: - { continue; } -yy80: - yych = *(q = ++p); - if (yych == '\n') goto yy85; - if (yych == '\r') goto yy87; -yy81: { break; } -yy82: +yy77: ++p; +yy78: { break; } -yy84: - yych = *++p; - goto yy81; -yy85: +yy79: ++p; + yych = *p; + if (yybm[0+yych] & 128) { + goto yy79; + } { continue; } -yy87: +yy82: + yych = *(q = ++p); + if (yych == '\n') goto yy83; + if (yych == '\r') goto yy85; + goto yy78; +yy83: + ++p; + { continue; } +yy85: yych = *++p; - if (yych == '\n') goto yy89; + if (yych == '\n') goto yy87; p = q; - goto yy81; -yy89: + goto yy78; +yy87: ++p; { continue; } -yy91: - ++p; - yych = *p; -yy92: - if (yybm[0+yych] & 128) { - goto yy91; - } - goto yy79; } } @@ -537,8 +550,9 @@ void Lexer::EatWhitespace() { bool Lexer::ReadIdent(string* out) { const char* p = ofs_; + const char* start; for (;;) { - const char* start = p; + start = p; { unsigned char yych; @@ -577,45 +591,28 @@ bool Lexer::ReadIdent(string* out) { 0, 0, 0, 0, 0, 0, 0, 0, }; yych = *p; - if (yych <= '@') { - if (yych <= '.') { - if (yych <= ',') goto yy97; - } else { - if (yych <= '/') goto yy97; - if (yych >= ':') goto yy97; - } - } else { - if (yych <= '_') { - if (yych <= 'Z') goto yy95; - if (yych <= '^') goto yy97; - } else { - if (yych <= '`') goto yy97; - if (yych >= '{') goto yy97; - } + if (yybm[0+yych] & 128) { + goto yy93; } -yy95: ++p; - yych = *p; - goto yy100; -yy96: { - out->assign(start, p - start); - break; + last_token_ = start; + return false; } -yy97: - ++p; - { return false; } -yy99: +yy93: ++p; yych = *p; -yy100: if (yybm[0+yych] & 128) { - goto yy99; + goto yy93; } - goto yy96; + { + out->assign(start, p - start); + break; + } } } + last_token_ = start; ofs_ = p; EatWhitespace(); return true; @@ -631,72 +628,69 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { { unsigned char yych; static const unsigned char yybm[] = { - 0, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 0, 128, 128, 0, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 16, 128, 128, 128, 0, 128, 128, 128, - 128, 128, 128, 128, 128, 224, 160, 128, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 0, 128, 128, 128, 128, 128, - 128, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 128, 128, 128, 128, 224, - 128, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 224, 224, 224, 224, 224, - 224, 224, 224, 128, 0, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, + 0, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 0, 16, 16, 0, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 32, 16, 16, 16, 0, 16, 16, 16, + 16, 16, 16, 16, 16, 208, 144, 16, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 0, 16, 16, 16, 16, 16, + 16, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 16, 16, 16, 16, 208, + 16, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 208, 208, 208, 208, 208, + 208, 208, 208, 16, 0, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, }; yych = *p; - if (yych <= ' ') { - if (yych <= '\n') { - if (yych <= 0x00) goto yy110; - if (yych >= '\n') goto yy107; - } else { - if (yych == '\r') goto yy105; - if (yych >= ' ') goto yy107; - } + if (yybm[0+yych] & 16) { + goto yy100; + } + if (yych <= '\r') { + if (yych <= 0x00) goto yy98; + if (yych <= '\n') goto yy103; + goto yy105; } else { - if (yych <= '9') { - if (yych == '$') goto yy109; - } else { - if (yych <= ':') goto yy107; - if (yych == '|') goto yy107; - } + if (yych <= ' ') goto yy103; + if (yych <= '$') goto yy107; + goto yy103; } +yy98: ++p; - yych = *p; - goto yy140; -yy104: { - eval->AddText(StringPiece(start, p - start)); - continue; + last_token_ = start; + return Error("unexpected EOF", err); } -yy105: +yy100: ++p; - if ((yych = *p) == '\n') goto yy137; + yych = *p; + if (yybm[0+yych] & 16) { + goto yy100; + } { - last_token_ = start; - return Error(DescribeLastError(), err); + eval->AddText(StringPiece(start, p - start)); + continue; } -yy107: +yy103: ++p; { if (path) { @@ -709,152 +703,121 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { continue; } } -yy109: +yy105: + ++p; + if ((yych = *p) == '\n') goto yy108; + { + last_token_ = start; + return Error(DescribeLastError(), err); + } +yy107: yych = *++p; - if (yych <= '-') { - if (yych <= 0x1F) { - if (yych <= '\n') { - if (yych <= '\t') goto yy112; - goto yy124; - } else { - if (yych == '\r') goto yy114; - goto yy112; - } + if (yybm[0+yych] & 64) { + goto yy120; + } + if (yych <= ' ') { + if (yych <= '\f') { + if (yych == '\n') goto yy112; + goto yy110; } else { - if (yych <= '#') { - if (yych <= ' ') goto yy115; - goto yy112; - } else { - if (yych <= '$') goto yy117; - if (yych <= ',') goto yy112; - goto yy119; - } + if (yych <= '\r') goto yy115; + if (yych <= 0x1F) goto yy110; + goto yy116; } } else { - if (yych <= 'Z') { - if (yych <= '9') { - if (yych <= '/') goto yy112; - goto yy119; - } else { - if (yych <= ':') goto yy121; - if (yych <= '@') goto yy112; - goto yy119; - } + if (yych <= '/') { + if (yych == '$') goto yy118; + goto yy110; } else { - if (yych <= '`') { - if (yych == '_') goto yy119; - goto yy112; - } else { - if (yych <= 'z') goto yy119; - if (yych <= '{') goto yy123; - goto yy112; - } + if (yych <= ':') goto yy123; + if (yych <= '`') goto yy110; + if (yych <= '{') goto yy125; + goto yy110; } } +yy108: + ++p; + { + if (path) + p = start; + break; + } yy110: ++p; +yy111: { last_token_ = start; - return Error("unexpected EOF", err); + return Error("bad $-escape (literal $ must be written as $$)", err); } yy112: ++p; -yy113: + yych = *p; + if (yybm[0+yych] & 32) { + goto yy112; + } { - last_token_ = start; - return Error("bad $-escape (literal $ must be written as $$)", err); + continue; } -yy114: - yych = *++p; - if (yych == '\n') goto yy134; - goto yy113; yy115: + yych = *++p; + if (yych == '\n') goto yy126; + goto yy111; +yy116: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy117: +yy118: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy119: +yy120: ++p; yych = *p; - goto yy133; -yy120: + if (yybm[0+yych] & 64) { + goto yy120; + } { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy121: +yy123: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy123: +yy125: yych = *(q = ++p); - if (yybm[0+yych] & 32) { - goto yy127; + if (yybm[0+yych] & 128) { + goto yy129; } - goto yy113; -yy124: + goto yy111; +yy126: ++p; yych = *p; - if (yybm[0+yych] & 16) { - goto yy124; - } + if (yych == ' ') goto yy126; { continue; } -yy127: +yy129: ++p; yych = *p; - if (yybm[0+yych] & 32) { - goto yy127; + if (yybm[0+yych] & 128) { + goto yy129; } - if (yych == '}') goto yy130; + if (yych == '}') goto yy132; p = q; - goto yy113; -yy130: - ++p; - { - eval->AddSpecial(StringPiece(start + 2, p - start - 3)); - continue; - } + goto yy111; yy132: ++p; - yych = *p; -yy133: - if (yybm[0+yych] & 64) { - goto yy132; - } - goto yy120; -yy134: - ++p; - yych = *p; - if (yych == ' ') goto yy134; { + eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy137: - ++p; - { - if (path) - p = start; - break; - } -yy139: - ++p; - yych = *p; -yy140: - if (yybm[0+yych] & 128) { - goto yy139; - } - goto yy104; } } diff --git a/src/lexer.in.cc b/src/lexer.in.cc index f861239089..c1fb8227cc 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -22,14 +22,14 @@ bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; - const char* context = input_.str_; + const char* line_start = input_.str_; for (const char* p = input_.str_; p < last_token_; ++p) { if (*p == '\n') { ++line; - context = p + 1; + line_start = p + 1; } } - int col = last_token_ ? (int)(last_token_ - context) : 0; + int col = last_token_ ? (int)(last_token_ - line_start) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); @@ -42,12 +42,12 @@ bool Lexer::Error(const string& message, string* err) { int len; bool truncated = true; for (len = 0; len < kTruncateColumn; ++len) { - if (context[len] == 0 || context[len] == '\n') { + if (line_start[len] == 0 || line_start[len] == '\n') { truncated = false; break; } } - *err += string(context, len); + *err += string(line_start, len); if (truncated) *err += "..."; *err += "\n"; @@ -182,16 +182,21 @@ void Lexer::EatWhitespace() { bool Lexer::ReadIdent(string* out) { const char* p = ofs_; + const char* start; for (;;) { - const char* start = p; + start = p; /*!re2c varname { out->assign(start, p - start); break; } - [^] { return false; } + [^] { + last_token_ = start; + return false; + } */ } + last_token_ = start; ofs_ = p; EatWhitespace(); return true; diff --git a/src/line_printer.cc b/src/line_printer.cc index 2cd3e174c4..953982af8c 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -18,6 +18,9 @@ #include #ifdef _WIN32 #include +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x4 +#endif #else #include #include @@ -40,6 +43,20 @@ LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { console_ = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO csbi; smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi); +#endif + supports_color_ = smart_terminal_; + if (!supports_color_) { + const char* clicolor_force = getenv("CLICOLOR_FORCE"); + supports_color_ = clicolor_force && string(clicolor_force) != "0"; + } +#ifdef _WIN32 + // Try enabling ANSI escape sequence support on Windows 10 terminals. + if (supports_color_) { + DWORD mode; + if (GetConsoleMode(console_, &mode)) { + SetConsoleMode(console_, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); + } + } #endif } @@ -82,7 +99,7 @@ void LinePrinter::Print(string to_print, LineType type) { // Limit output to width of the terminal if provided so we don't cause // line-wrapping. winsize size; - if ((ioctl(0, TIOCGWINSZ, &size) == 0) && size.ws_col) { + if ((ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0) && size.ws_col) { to_print = ElideMiddle(to_print, size.ws_col); } printf("%s", to_print.c_str()); diff --git a/src/line_printer.h b/src/line_printer.h index 55225e5211..92d4dc4480 100644 --- a/src/line_printer.h +++ b/src/line_printer.h @@ -27,6 +27,8 @@ struct LinePrinter { bool is_smart_terminal() const { return smart_terminal_; } void set_smart_terminal(bool smart) { smart_terminal_ = smart; } + bool supports_color() const { return supports_color_; } + enum LineType { FULL, ELIDE @@ -46,6 +48,9 @@ struct LinePrinter { /// Whether we can do fancy terminal control codes. bool smart_terminal_; + /// Whether we can use ISO 6429 (ANSI) color sequences. + bool supports_color_; + /// Whether the caret is at the beginning of a blank line. bool have_blank_line_; diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 39ed810658..c91d8d156c 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -523,7 +523,7 @@ TEST_F(ParserTest, Errors) { EXPECT_FALSE(parser.ParseTest("build x: y z\n", &err)); EXPECT_EQ("input:1: unknown build rule 'y'\n" "build x: y z\n" - " ^ near here" + " ^ near here" , err); } @@ -534,7 +534,7 @@ TEST_F(ParserTest, Errors) { EXPECT_FALSE(parser.ParseTest("build x:: y z\n", &err)); EXPECT_EQ("input:1: expected build command name\n" "build x:: y z\n" - " ^ near here" + " ^ near here" , err); } @@ -636,7 +636,10 @@ TEST_F(ParserTest, Errors) { string err; EXPECT_FALSE(parser.ParseTest("rule %foo\n", &err)); - EXPECT_EQ("input:1: expected rule name\n", err); + EXPECT_EQ("input:1: expected rule name\n" + "rule %foo\n" + " ^ near here", + err); } { @@ -672,7 +675,10 @@ TEST_F(ParserTest, Errors) { string err; EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n && bar", &err)); - EXPECT_EQ("input:3: expected variable name\n", err); + EXPECT_EQ("input:3: expected variable name\n" + " && bar\n" + " ^ near here", + err); } { @@ -767,7 +773,9 @@ TEST_F(ParserTest, Errors) { ManifestParser parser(&local_state, NULL); string err; EXPECT_FALSE(parser.ParseTest("pool\n", &err)); - EXPECT_EQ("input:1: expected pool name\n", err); + EXPECT_EQ("input:1: expected pool name\n" + "pool\n" + " ^ near here", err); } { diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc index 1efb085a9c..ca936387bd 100644 --- a/src/minidump-win32.cc +++ b/src/minidump-win32.cc @@ -32,17 +32,17 @@ typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( /// Creates a windows minidump in temp folder. void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep) { char temp_path[MAX_PATH]; - GetTempPath(sizeof(temp_path), temp_path); + GetTempPathA(sizeof(temp_path), temp_path); char temp_file[MAX_PATH]; sprintf(temp_file, "%s\\ninja_crash_dump_%lu.dmp", temp_path, GetCurrentProcessId()); // Delete any previous minidump of the same name. - DeleteFile(temp_file); + DeleteFileA(temp_file); // Load DbgHelp.dll dynamically, as library is not present on all // Windows versions. - HMODULE dbghelp = LoadLibrary("dbghelp.dll"); + HMODULE dbghelp = LoadLibraryA("dbghelp.dll"); if (dbghelp == NULL) { Error("failed to create minidump: LoadLibrary('dbghelp.dll'): %s", GetLastErrorString().c_str()); diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index e37a26ea60..de6147a5e5 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -43,10 +43,10 @@ int CLWrapper::Run(const string& command, string* output) { security_attributes.bInheritHandle = TRUE; // Must be inheritable so subprocesses can dup to children. - HANDLE nul = CreateFile("NUL", GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | - FILE_SHARE_DELETE, - &security_attributes, OPEN_EXISTING, 0, NULL); + HANDLE nul = + CreateFileA("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); if (nul == INVALID_HANDLE_VALUE) Fatal("couldn't open nul"); @@ -58,8 +58,8 @@ int CLWrapper::Run(const string& command, string* output) { Win32Fatal("SetHandleInformation"); PROCESS_INFORMATION process_info = {}; - STARTUPINFO startup_info = {}; - startup_info.cb = sizeof(STARTUPINFO); + STARTUPINFOA startup_info = {}; + startup_info.cb = sizeof(STARTUPINFOA); startup_info.hStdInput = nul; startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); startup_info.hStdOutput = stdout_write; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index e419cd7742..644b2a2e24 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -113,7 +113,7 @@ int MSVCHelperMain(int argc, char** argv) { PushPathIntoEnvironment(env); } - char* command = GetCommandLine(); + char* command = GetCommandLineA(); command = strstr(command, " -- "); if (!command) { Fatal("expected command line to end with \" -- command args\""); diff --git a/src/ninja.cc b/src/ninja.cc index ed004ac8f1..b608426ba9 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -73,6 +73,10 @@ struct Options { /// Whether phony cycles should warn or print an error. bool phony_cycle_should_err; + + /// Whether a depfile with multiple targets on separate lines should + /// warn or print an error. + bool depfile_distinct_target_lines_should_err; }; /// The Ninja main() loads up a series of data structures; various tools need @@ -154,7 +158,7 @@ struct NinjaMain : public BuildLogUser { // Just checking n isn't enough: If an old output is both in the build log // and in the deps log, it will have a Node object in state_. (It will also // have an in edge if one of its inputs is another output that's in the deps - // log, but having a deps edge product an output thats input to another deps + // log, but having a deps edge product an output that's input to another deps // edge is rare, and the first recompaction will delete all old outputs from // the deps log, and then a second recompaction will clear the build log, // which seems good enough for this corner case.) @@ -201,21 +205,21 @@ void Usage(const BuildConfig& config) { "if targets are unspecified, builds the 'default' target (see manual).\n" "\n" "options:\n" -" --version print ninja version (\"%s\")\n" +" --version print ninja version (\"%s\")\n" +" -v, --verbose show all command lines while building\n" "\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" "\n" -" -j N run N jobs in parallel [default=%d, derived from CPUs available]\n" -" -k N keep going until N jobs fail [default=1]\n" +" -j N run N jobs in parallel (0 means infinity) [default=%d on this system]\n" +" -k N keep going until N jobs fail (0 means infinity) [default=1]\n" " -l N do not start new jobs if the load average is greater than N\n" " -n dry run (don't run commands but act like they succeeded)\n" -" -v show all command lines while building\n" "\n" -" -d MODE enable debugging (use -d list to list modes)\n" -" -t TOOL run a subtool (use -t list to list subtools)\n" +" -d MODE enable debugging (use '-d list' to list modes)\n" +" -t TOOL run a subtool (use '-t list' to list subtools)\n" " terminates toplevel options; further flags are passed to the tool\n" -" -w FLAG adjust warnings (use -w list to list warnings)\n", +" -w FLAG adjust warnings (use '-w list' to list warnings)\n", kNinjaVersion, config.parallelism); } @@ -387,7 +391,12 @@ int NinjaMain::ToolBrowse(const Options* options, int argc, char* argv[]) { // If we get here, the browse failed. return 1; } -#endif // _WIN32 +#else +int NinjaMain::ToolBrowse(const Options*, int, char**) { + Fatal("browse tool not supported on this platform"); + return 1; +} +#endif #if defined(_MSC_VER) int NinjaMain::ToolMSVC(const Options* options, int argc, char* argv[]) { @@ -494,7 +503,7 @@ int NinjaMain::ToolDeps(const Options* options, int argc, char** argv) { TimeStamp mtime = disk_interface.Stat((*it)->path(), &err); if (mtime == -1) Error("%s", err.c_str()); // Log and ignore Stat() errors; - printf("%s: #deps %d, deps mtime %d (%s)\n", + printf("%s: #deps %d, deps mtime %" PRId64 " (%s)\n", (*it)->path().c_str(), deps->node_count, deps->mtime, (!mtime || mtime > deps->mtime ? "STALE":"VALID")); for (int i = 0; i < deps->node_count; ++i) @@ -662,7 +671,65 @@ void EncodeJSONString(const char *str) { } } -int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, char* argv[]) { +enum EvaluateCommandMode { + ECM_NORMAL, + ECM_EXPAND_RSPFILE +}; +string EvaluateCommandWithRspfile(Edge* edge, EvaluateCommandMode mode) { + string command = edge->EvaluateCommand(); + if (mode == ECM_NORMAL) + return command; + + string rspfile = edge->GetUnescapedRspfile(); + if (rspfile.empty()) + return command; + + size_t index = command.find(rspfile); + if (index == 0 || index == string::npos || command[index - 1] != '@') + return command; + + string rspfile_content = edge->GetBinding("rspfile_content"); + size_t newline_index = 0; + while ((newline_index = rspfile_content.find('\n', newline_index)) != + string::npos) { + rspfile_content.replace(newline_index, 1, 1, ' '); + ++newline_index; + } + command.replace(index - 1, rspfile.length() + 1, rspfile_content); + return command; +} + +int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, + char* argv[]) { + // The compdb tool uses getopt, and expects argv[0] to contain the name of + // the tool, i.e. "compdb". + argc++; + argv--; + + EvaluateCommandMode eval_mode = ECM_NORMAL; + + optind = 1; + int opt; + while ((opt = getopt(argc, argv, const_cast("hx"))) != -1) { + switch(opt) { + case 'x': + eval_mode = ECM_EXPAND_RSPFILE; + break; + + case 'h': + default: + printf( + "usage: ninja -t compdb [options] [rules]\n" + "\n" + "options:\n" + " -x expand @rspfile style response file invocations\n" + ); + return 1; + } + } + argv += optind; + argc -= optind; + bool first = true; vector cwd; @@ -688,9 +755,11 @@ int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, char* a printf("\n {\n \"directory\": \""); EncodeJSONString(&cwd[0]); printf("\",\n \"command\": \""); - EncodeJSONString((*e)->EvaluateCommand().c_str()); + EncodeJSONString(EvaluateCommandWithRspfile(*e, eval_mode).c_str()); printf("\",\n \"file\": \""); EncodeJSONString((*e)->inputs_[0]->path().c_str()); + printf("\",\n \"output\": \""); + EncodeJSONString((*e)->outputs_[0]->path().c_str()); printf("\"\n }"); first = false; @@ -743,10 +812,8 @@ int NinjaMain::ToolUrtle(const Options* options, int argc, char** argv) { /// Returns a Tool, or NULL if Ninja should exit. const Tool* ChooseTool(const string& tool_name) { static const Tool kTools[] = { -#if defined(NINJA_HAVE_BROWSE) { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolBrowse }, -#endif #if defined(_MSC_VER) { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)", Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolMSVC }, @@ -849,7 +916,9 @@ bool WarningEnable(const string& name, Options* options) { if (name == "list") { printf("warning flags:\n" " dupbuild={err,warn} multiple build lines for one target\n" -" phonycycle={err,warn} phony build statement references itself\n"); +" phonycycle={err,warn} phony build statement references itself\n" +" depfilemulti={err,warn} depfile has multiple output paths on separate lines\n" + ); return false; } else if (name == "dupbuild=err") { options->dupe_edges_should_err = true; @@ -863,6 +932,12 @@ bool WarningEnable(const string& name, Options* options) { } else if (name == "phonycycle=warn") { options->phony_cycle_should_err = false; return true; + } else if (name == "depfilemulti=err") { + options->depfile_distinct_target_lines_should_err = true; + return true; + } else if (name == "depfilemulti=warn") { + options->depfile_distinct_target_lines_should_err = false; + return true; } else { const char* suggestion = SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn", @@ -1042,6 +1117,7 @@ int ReadFlags(int* argc, char*** argv, const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, OPT_VERSION }, + { "verbose", no_argument, NULL, 'v' }, { NULL, 0, NULL, 0 } }; @@ -1060,9 +1136,12 @@ int ReadFlags(int* argc, char*** argv, case 'j': { char* end; int value = strtol(optarg, &end, 10); - if (*end != 0 || value <= 0) + if (*end != 0 || value < 0) Fatal("invalid -j parameter"); - config->parallelism = value; + + // We want to run N jobs in parallel. For N = 0, INT_MAX + // is close enough to infinite for most sane builds. + config->parallelism = value > 0 ? value : INT_MAX; break; } case 'k': { @@ -1118,17 +1197,25 @@ int ReadFlags(int* argc, char*** argv, return -1; } -int real_main(int argc, char** argv) { +NORETURN void real_main(int argc, char** argv) { + // Use exit() instead of return in this function to avoid potentially + // expensive cleanup when destructing NinjaMain. BuildConfig config; Options options = {}; options.input_file = "build.ninja"; + options.dupe_edges_should_err = true; setvbuf(stdout, NULL, _IOLBF, BUFSIZ); const char* ninja_command = argv[0]; int exit_code = ReadFlags(&argc, &argv, &options, &config); if (exit_code >= 0) - return exit_code; + exit(exit_code); + + if (options.depfile_distinct_target_lines_should_err) { + config.depfile_parser_options.depfile_distinct_target_lines_action_ = + kDepfileDistinctTargetLinesActionError; + } if (options.working_dir) { // The formatting of this string, complete with funny quotes, is @@ -1147,7 +1234,7 @@ int real_main(int argc, char** argv) { // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed // by other tools. NinjaMain ninja(ninja_command, config); - return (ninja.*options.tool->func)(&options, argc, argv); + exit((ninja.*options.tool->func)(&options, argc, argv)); } // Limit number of rebuilds, to prevent infinite loops. @@ -1166,43 +1253,43 @@ int real_main(int argc, char** argv) { string err; if (!parser.Load(options.input_file, &err)) { Error("%s", err.c_str()); - return 1; + exit(1); } if (options.tool && options.tool->when == Tool::RUN_AFTER_LOAD) - return (ninja.*options.tool->func)(&options, argc, argv); + exit((ninja.*options.tool->func)(&options, argc, argv)); if (!ninja.EnsureBuildDirExists()) - return 1; + exit(1); if (!ninja.OpenBuildLog() || !ninja.OpenDepsLog()) - return 1; + exit(1); if (options.tool && options.tool->when == Tool::RUN_AFTER_LOGS) - return (ninja.*options.tool->func)(&options, argc, argv); + exit((ninja.*options.tool->func)(&options, argc, argv)); // Attempt to rebuild the manifest before building anything else if (ninja.RebuildManifest(options.input_file, &err)) { // In dry_run mode the regeneration will succeed without changing the // manifest forever. Better to return immediately. if (config.dry_run) - return 0; + exit(0); // Start the build over with the new manifest. continue; } else if (!err.empty()) { Error("rebuilding '%s': %s", options.input_file, err.c_str()); - return 1; + exit(1); } int result = ninja.RunBuild(argc, argv); if (g_metrics) ninja.DumpMetrics(); - return result; + exit(result); } Error("manifest '%s' still dirty after %d tries\n", options.input_file, kCycleLimit); - return 1; + exit(1); } } // anonymous namespace @@ -1215,7 +1302,7 @@ int main(int argc, char** argv) { __try { // Running inside __try ... __except suppresses any Windows error // dialogs for errors such as bad_alloc. - return real_main(argc, argv); + real_main(argc, argv); } __except(ExceptionFilter(GetExceptionCode(), GetExceptionInformation())) { // Common error situations return exitCode=1. 2 was chosen to @@ -1223,6 +1310,6 @@ int main(int argc, char** argv) { return 2; } #else - return real_main(argc, argv); + real_main(argc, argv); #endif } diff --git a/src/state.h b/src/state.h index 54e9dc54a7..6fe886c1b2 100644 --- a/src/state.h +++ b/src/state.h @@ -33,7 +33,7 @@ struct Rule; /// Pools are scoped to a State. Edges within a State will share Pools. A Pool /// will keep a count of the total 'weight' of the currently scheduled edges. If /// a Plan attempts to schedule an Edge which would cause the total weight to -/// exceed the depth of the Pool, the Pool will enque the Edge instead of +/// exceed the depth of the Pool, the Pool will enqueue the Edge instead of /// allowing the Plan to schedule it. The Pool will relinquish queued Edges when /// the total scheduled weight diminishes enough (i.e. when a scheduled edge /// completes). diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index 1de22c38f7..fc5543e85f 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -14,6 +14,7 @@ #include "subprocess.h" +#include #include #include #include @@ -54,21 +55,25 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { SetCloseOnExec(fd_); posix_spawn_file_actions_t action; - if (posix_spawn_file_actions_init(&action) != 0) - Fatal("posix_spawn_file_actions_init: %s", strerror(errno)); + int err = posix_spawn_file_actions_init(&action); + if (err != 0) + Fatal("posix_spawn_file_actions_init: %s", strerror(err)); - if (posix_spawn_file_actions_addclose(&action, output_pipe[0]) != 0) - Fatal("posix_spawn_file_actions_addclose: %s", strerror(errno)); + err = posix_spawn_file_actions_addclose(&action, output_pipe[0]); + if (err != 0) + Fatal("posix_spawn_file_actions_addclose: %s", strerror(err)); posix_spawnattr_t attr; - if (posix_spawnattr_init(&attr) != 0) - Fatal("posix_spawnattr_init: %s", strerror(errno)); + err = posix_spawnattr_init(&attr); + if (err != 0) + Fatal("posix_spawnattr_init: %s", strerror(err)); short flags = 0; flags |= POSIX_SPAWN_SETSIGMASK; - if (posix_spawnattr_setsigmask(&attr, &set->old_mask_) != 0) - Fatal("posix_spawnattr_setsigmask: %s", strerror(errno)); + err = posix_spawnattr_setsigmask(&attr, &set->old_mask_); + if (err != 0) + Fatal("posix_spawnattr_setsigmask: %s", strerror(err)); // Signals which are set to be caught in the calling process image are set to // default action in the new process image, so no explicit // POSIX_SPAWN_SETSIGDEF parameter is needed. @@ -79,17 +84,21 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { // No need to posix_spawnattr_setpgroup(&attr, 0), it's the default. // Open /dev/null over stdin. - if (posix_spawn_file_actions_addopen(&action, 0, "/dev/null", O_RDONLY, - 0) != 0) { - Fatal("posix_spawn_file_actions_addopen: %s", strerror(errno)); + err = posix_spawn_file_actions_addopen(&action, 0, "/dev/null", O_RDONLY, + 0); + if (err != 0) { + Fatal("posix_spawn_file_actions_addopen: %s", strerror(err)); } - if (posix_spawn_file_actions_adddup2(&action, output_pipe[1], 1) != 0) - Fatal("posix_spawn_file_actions_adddup2: %s", strerror(errno)); - if (posix_spawn_file_actions_adddup2(&action, output_pipe[1], 2) != 0) - Fatal("posix_spawn_file_actions_adddup2: %s", strerror(errno)); - if (posix_spawn_file_actions_addclose(&action, output_pipe[1]) != 0) - Fatal("posix_spawn_file_actions_addclose: %s", strerror(errno)); + err = posix_spawn_file_actions_adddup2(&action, output_pipe[1], 1); + if (err != 0) + Fatal("posix_spawn_file_actions_adddup2: %s", strerror(err)); + err = posix_spawn_file_actions_adddup2(&action, output_pipe[1], 2); + if (err != 0) + Fatal("posix_spawn_file_actions_adddup2: %s", strerror(err)); + err = posix_spawn_file_actions_addclose(&action, output_pipe[1]); + if (err != 0) + Fatal("posix_spawn_file_actions_addclose: %s", strerror(err)); // In the console case, output_pipe is still inherited by the child and // closed when the subprocess finishes, which then notifies ninja. } @@ -97,18 +106,22 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { flags |= POSIX_SPAWN_USEVFORK; #endif - if (posix_spawnattr_setflags(&attr, flags) != 0) - Fatal("posix_spawnattr_setflags: %s", strerror(errno)); + err = posix_spawnattr_setflags(&attr, flags); + if (err != 0) + Fatal("posix_spawnattr_setflags: %s", strerror(err)); const char* spawned_args[] = { "/bin/sh", "-c", command.c_str(), NULL }; - if (posix_spawn(&pid_, "/bin/sh", &action, &attr, - const_cast(spawned_args), environ) != 0) - Fatal("posix_spawn: %s", strerror(errno)); - - if (posix_spawnattr_destroy(&attr) != 0) - Fatal("posix_spawnattr_destroy: %s", strerror(errno)); - if (posix_spawn_file_actions_destroy(&action) != 0) - Fatal("posix_spawn_file_actions_destroy: %s", strerror(errno)); + err = posix_spawn(&pid_, "/bin/sh", &action, &attr, + const_cast(spawned_args), environ); + if (err != 0) + Fatal("posix_spawn: %s", strerror(err)); + + err = posix_spawnattr_destroy(&attr); + if (err != 0) + Fatal("posix_spawnattr_destroy: %s", strerror(err)); + err = posix_spawn_file_actions_destroy(&action); + if (err != 0) + Fatal("posix_spawn_file_actions_destroy: %s", strerror(err)); close(output_pipe[1]); return true; diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 4bab71939d..a4a7669524 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -59,8 +59,8 @@ HANDLE Subprocess::SetupPipe(HANDLE ioport) { } // Get the write end of the pipe as a handle inheritable across processes. - HANDLE output_write_handle = CreateFile(pipe_name, GENERIC_WRITE, 0, - NULL, OPEN_EXISTING, 0, NULL); + HANDLE output_write_handle = + CreateFileA(pipe_name, GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL); HANDLE output_write_child; if (!DuplicateHandle(GetCurrentProcess(), output_write_handle, GetCurrentProcess(), &output_write_child, @@ -80,9 +80,10 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); security_attributes.bInheritHandle = TRUE; // Must be inheritable so subprocesses can dup to children. - HANDLE nul = CreateFile("NUL", GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - &security_attributes, OPEN_EXISTING, 0, NULL); + HANDLE nul = + CreateFileA("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); if (nul == INVALID_HANDLE_VALUE) Fatal("couldn't open nul"); @@ -123,6 +124,10 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { buf_ = "CreateProcess failed: The system cannot find the file " "specified.\n"; return true; + } else if (error == ERROR_INVALID_PARAMETER) { + // This generally means that the command line was too long. Give extra + // context for this case. + Win32Fatal("CreateProcess", "is the command line too long?"); } else { Win32Fatal("CreateProcess"); // pass all other errors to Win32Fatal } diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 0a8c2061b7..6e487dbde8 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -182,7 +182,7 @@ TEST_F(SubprocessTest, SetWithMulti) { "cmd /c echo hi", "cmd /c time /t", #else - "whoami", + "id -u", "pwd", #endif }; diff --git a/src/test.h b/src/test.h index 3bce8f75ae..6af17b3f90 100644 --- a/src/test.h +++ b/src/test.h @@ -104,7 +104,7 @@ extern testing::Test* g_current_test; } \ } -// Support utilites for tests. +// Support utilities for tests. struct Node; diff --git a/src/timestamp.h b/src/timestamp.h index cee7ba8f21..6a7ccd0b06 100644 --- a/src/timestamp.h +++ b/src/timestamp.h @@ -15,10 +15,19 @@ #ifndef NINJA_TIMESTAMP_H_ #define NINJA_TIMESTAMP_H_ +#ifdef _WIN32 +#include "win32port.h" +#else +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#include +#endif + // When considering file modification times we only care to compare // them against one another -- we never convert them to an absolute -// real time. On POSIX we use time_t (seconds since epoch) and on -// Windows we use a different value. Both fit in an int. -typedef int TimeStamp; +// real time. On POSIX we use timespec (seconds&nanoseconds since epoch) +// and on Windows we use a different value. Both fit in an int64. +typedef int64_t TimeStamp; #endif // NINJA_TIMESTAMP_H_ diff --git a/src/util.cc b/src/util.cc index ae94d346bc..47a5de2ffb 100644 --- a/src/util.cc +++ b/src/util.cc @@ -197,7 +197,7 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, case '\\': bits |= bits_mask; *c = '/'; - // Intentional fallthrough. + NINJA_FALLTHROUGH; case '/': bits_mask <<= 1; } @@ -318,13 +318,8 @@ int ReadFile(const string& path, string* contents, string* err) { // This makes a ninja run on a set of 1500 manifest files about 4% faster // than using the generic fopen code below. err->clear(); - HANDLE f = ::CreateFile(path.c_str(), - GENERIC_READ, - FILE_SHARE_READ, - NULL, - OPEN_EXISTING, - FILE_FLAG_SEQUENTIAL_SCAN, - NULL); + HANDLE f = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); if (f == INVALID_HANDLE_VALUE) { err->assign(GetLastErrorString()); return -ENOENT; @@ -351,9 +346,19 @@ int ReadFile(const string& path, string* contents, string* err) { return -errno; } + struct stat st; + if (fstat(fileno(f), &st) < 0) { + err->assign(strerror(errno)); + fclose(f); + return -errno; + } + + // +1 is for the resize in ManifestParser::Load + contents->reserve(st.st_size + 1); + char buf[64 << 10]; size_t len; - while ((len = fread(buf, 1, sizeof(buf), f)) > 0) { + while (!feof(f) && (len = fread(buf, 1, sizeof(buf), f)) > 0) { contents->append(buf, len); } if (ferror(f)) { @@ -437,8 +442,12 @@ string GetLastErrorString() { return msg; } -void Win32Fatal(const char* function) { - Fatal("%s: %s", function, GetLastErrorString().c_str()); +void Win32Fatal(const char* function, const char* hint) { + if (hint) { + Fatal("%s: %s (%s)", function, GetLastErrorString().c_str(), hint); + } else { + Fatal("%s: %s", function, GetLastErrorString().c_str()); + } } #endif @@ -578,7 +587,7 @@ double GetLoadAverage() { string ElideMiddle(const string& str, size_t width) { const int kMargin = 3; // Space for "...". string result = str; - if (result.size() + kMargin > width) { + if (result.size() > width) { size_t elide_size = (width - kMargin) / 2; result = result.substr(0, elide_size) + "..." diff --git a/src/util.h b/src/util.h index 4ee41a500a..6a4a7a9f84 100644 --- a/src/util.h +++ b/src/util.h @@ -34,6 +34,20 @@ using namespace std; /// Log a fatal message and exit. NORETURN void Fatal(const char* msg, ...); +// Have a generic fall-through for different versions of C/C++. +#if defined(__cplusplus) && __cplusplus >= 201703L +#define NINJA_FALLTHROUGH [[fallthrough]] +#elif defined(__cplusplus) && __cplusplus >= 201103L && defined(__clang__) +#define NINJA_FALLTHROUGH [[clang::fallthrough]] +#elif defined(__cplusplus) && __cplusplus >= 201103L && defined(__GNUC__) && \ + __GNUC__ >= 7 +#define NINJA_FALLTHROUGH [[gnu::fallthrough]] +#elif defined(__GNUC__) && __GNUC__ >= 7 // gcc 7 +#define NINJA_FALLTHROUGH __attribute__ ((fallthrough)) +#else // C++11 on gcc 6, and all other cases +#define NINJA_FALLTHROUGH +#endif + /// Log a warning message. void Warning(const char* msg, ...); @@ -105,7 +119,7 @@ bool Truncate(const string& path, size_t size, string* err); string GetLastErrorString(); /// Calls Fatal() with a function name and GetLastErrorString. -NORETURN void Win32Fatal(const char* function); +NORETURN void Win32Fatal(const char* function, const char* hint = NULL); #endif #endif // NINJA_UTIL_H_ diff --git a/src/util_test.cc b/src/util_test.cc index b4b75169d6..d97b48ccc2 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -419,10 +419,12 @@ TEST(StripAnsiEscapeCodes, StripColors) { TEST(ElideMiddle, NothingToElide) { string input = "Nothing to elide in this short string."; EXPECT_EQ(input, ElideMiddle(input, 80)); + EXPECT_EQ(input, ElideMiddle(input, 38)); } TEST(ElideMiddle, ElideInTheMiddle) { string input = "01234567890123456789"; string elided = ElideMiddle(input, 10); EXPECT_EQ("012...789", elided); + EXPECT_EQ("01234567...23456789", ElideMiddle(input, 19)); } diff --git a/src/version.cc b/src/version.cc index 3a20205cd8..bda25be66e 100644 --- a/src/version.cc +++ b/src/version.cc @@ -18,7 +18,7 @@ #include "util.h" -const char* kNinjaVersion = "1.8.2"; +const char* kNinjaVersion = "1.9.0"; void ParseVersion(const string& version, int* major, int* minor) { size_t end = version.find('.'); diff --git a/src/win32port.h b/src/win32port.h index ce3c9498e5..e542536cc7 100644 --- a/src/win32port.h +++ b/src/win32port.h @@ -15,6 +15,13 @@ #ifndef NINJA_WIN32PORT_H_ #define NINJA_WIN32PORT_H_ +#if defined(__MINGW32__) || defined(__MINGW64__) +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#include +#endif + typedef signed short int16_t; typedef unsigned short uint16_t; /// A 64-bit integer type @@ -23,6 +30,7 @@ typedef unsigned long long uint64_t; // printf format specifier for uint64_t, from C99. #ifndef PRIu64 +#define PRId64 "I64d" #define PRIu64 "I64u" #define PRIx64 "I64x" #endif