Index: CMakeLists.txt
===================================================================
--- CMakeLists.txt (nonexistent)
+++ CMakeLists.txt (revision 5)
@@ -0,0 +1,219 @@
+cmake_minimum_required(VERSION 3.15)
+
+include(CheckIncludeFileCXX)
+include(CheckIPOSupported)
+
+project(ninja)
+
+# --- optional link-time optimization
+check_ipo_supported(RESULT lto_supported OUTPUT error)
+
+if(lto_supported)
+ message(STATUS "IPO / LTO enabled")
+ set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE)
+else()
+ message(STATUS "IPO / LTO not supported: <${error}>")
+endif()
+
+# --- compiler flags
+if(MSVC)
+ set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+ string(APPEND CMAKE_CXX_FLAGS " /W4 /GR- /Zc:__cplusplus")
+else()
+ include(CheckCXXCompilerFlag)
+ check_cxx_compiler_flag(-Wno-deprecated flag_no_deprecated)
+ if(flag_no_deprecated)
+ string(APPEND CMAKE_CXX_FLAGS " -Wno-deprecated")
+ endif()
+ check_cxx_compiler_flag(-fdiagnostics-color flag_color_diag)
+ if(flag_color_diag)
+ string(APPEND CMAKE_CXX_FLAGS " -fdiagnostics-color")
+ endif()
+endif()
+
+# --- optional re2c
+find_program(RE2C re2c)
+if(RE2C)
+ # the depfile parser and ninja lexers are generated using re2c.
+ function(re2c IN OUT)
+ add_custom_command(DEPENDS ${IN} OUTPUT ${OUT}
+ COMMAND ${RE2C} -b -i --no-generation-date -o ${OUT} ${IN}
+ )
+ endfunction()
+ re2c(${PROJECT_SOURCE_DIR}/src/depfile_parser.in.cc ${PROJECT_BINARY_DIR}/depfile_parser.cc)
+ re2c(${PROJECT_SOURCE_DIR}/src/lexer.in.cc ${PROJECT_BINARY_DIR}/lexer.cc)
+ add_library(libninja-re2c OBJECT ${PROJECT_BINARY_DIR}/depfile_parser.cc ${PROJECT_BINARY_DIR}/lexer.cc)
+else()
+ message(WARNING "re2c was not found; changes to src/*.in.cc will not affect your build.")
+ add_library(libninja-re2c OBJECT src/depfile_parser.cc src/lexer.cc)
+endif()
+target_include_directories(libninja-re2c PRIVATE src)
+
+# --- Check for 'browse' mode support
+function(check_platform_supports_browse_mode RESULT)
+ # Make sure the inline.sh script works on this platform.
+ # It uses the shell commands such as 'od', which may not be available.
+ execute_process(
+ COMMAND sh -c "echo 'TEST' | src/inline.sh var"
+ WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+ RESULT_VARIABLE inline_result
+ OUTPUT_QUIET
+ ERROR_QUIET
+ )
+ if(NOT inline_result EQUAL "0")
+ # The inline script failed, so browse mode is not supported.
+ set(${RESULT} "0" PARENT_SCOPE)
+ return()
+ endif()
+
+ # Now check availability of the unistd header
+ check_include_file_cxx(unistd.h PLATFORM_HAS_UNISTD_HEADER)
+ set(${RESULT} "${PLATFORM_HAS_UNISTD_HEADER}" PARENT_SCOPE)
+endfunction()
+
+check_platform_supports_browse_mode(platform_supports_ninja_browse)
+
+# Core source files all build into ninja library.
+add_library(libninja OBJECT
+ src/build_log.cc
+ src/build.cc
+ src/clean.cc
+ src/clparser.cc
+ src/dyndep.cc
+ src/dyndep_parser.cc
+ src/debug_flags.cc
+ src/deps_log.cc
+ src/disk_interface.cc
+ src/edit_distance.cc
+ src/eval_env.cc
+ src/graph.cc
+ src/graphviz.cc
+ src/line_printer.cc
+ src/manifest_parser.cc
+ src/metrics.cc
+ src/parser.cc
+ src/state.cc
+ src/string_piece_util.cc
+ src/tokenpool-gnu-make.cc
+ src/util.cc
+ src/version.cc
+)
+if(WIN32)
+ target_sources(libninja PRIVATE
+ src/subprocess-win32.cc
+ src/includes_normalize-win32.cc
+ src/msvc_helper-win32.cc
+ src/msvc_helper_main-win32.cc
+ src/getopt.c
+ src/tokenpool-gnu-make-win32.cc
+ )
+ if(MSVC)
+ target_sources(libninja PRIVATE src/minidump-win32.cc)
+ endif()
+else()
+ target_sources(libninja PRIVATE
+ src/subprocess-posix.cc
+ src/tokenpool-gnu-make-posix.cc
+ )
+ if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
+ target_sources(libninja PRIVATE src/getopt.c)
+ endif()
+
+ # Needed for perfstat_cpu_total
+ if(CMAKE_SYSTEM_NAME STREQUAL "AIX")
+ target_link_libraries(libninja PUBLIC "-lperfstat")
+ endif()
+endif()
+
+#Fixes GetActiveProcessorCount on MinGW
+if(MINGW)
+target_compile_definitions(libninja PRIVATE _WIN32_WINNT=0x0601 __USE_MINGW_ANSI_STDIO=1)
+endif()
+
+# On IBM i (identified as "OS400" for compatibility reasons) and AIX, this fixes missing
+# PRId64 (and others) at compile time in C++ sources
+if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
+ string(APPEND CMAKE_CXX_FLAGS " -D__STDC_FORMAT_MACROS")
+endif()
+
+# Main executable is library plus main() function.
+add_executable(ninja src/ninja.cc)
+target_link_libraries(ninja PRIVATE libninja libninja-re2c)
+
+# Adds browse mode into the ninja binary if it's supported by the host platform.
+if(platform_supports_ninja_browse)
+ # Inlines src/browse.py into the browse_py.h header, so that it can be included
+ # by src/browse.cc
+ add_custom_command(
+ OUTPUT build/browse_py.h
+ MAIN_DEPENDENCY src/browse.py
+ DEPENDS src/inline.sh
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/build
+ COMMAND src/inline.sh kBrowsePy
+ < src/browse.py
+ > ${CMAKE_BINARY_DIR}/build/browse_py.h
+ WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+ VERBATIM
+ )
+
+ target_compile_definitions(ninja PRIVATE NINJA_HAVE_BROWSE)
+ target_sources(ninja PRIVATE src/browse.cc)
+ set_source_files_properties(src/browse.cc
+ PROPERTIES
+ OBJECT_DEPENDS "${CMAKE_BINARY_DIR}/build/browse_py.h"
+ INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}"
+ COMPILE_DEFINITIONS NINJA_PYTHON="python"
+ )
+endif()
+
+include(CTest)
+if(BUILD_TESTING)
+ # Tests all build into ninja_test executable.
+ add_executable(ninja_test
+ src/build_log_test.cc
+ src/build_test.cc
+ src/clean_test.cc
+ src/clparser_test.cc
+ src/depfile_parser_test.cc
+ src/deps_log_test.cc
+ src/disk_interface_test.cc
+ src/dyndep_parser_test.cc
+ src/edit_distance_test.cc
+ src/graph_test.cc
+ src/lexer_test.cc
+ src/manifest_parser_test.cc
+ src/ninja_test.cc
+ src/state_test.cc
+ src/string_piece_util_test.cc
+ src/subprocess_test.cc
+ src/test.cc
+ src/tokenpool_test.cc
+ src/util_test.cc
+ )
+ if(WIN32)
+ target_sources(ninja_test PRIVATE src/includes_normalize_test.cc src/msvc_helper_test.cc)
+ endif()
+ target_link_libraries(ninja_test PRIVATE libninja libninja-re2c)
+
+ foreach(perftest
+ build_log_perftest
+ canon_perftest
+ clparser_perftest
+ depfile_parser_perftest
+ hash_collision_bench
+ manifest_parser_perftest
+ )
+ add_executable(${perftest} src/${perftest}.cc)
+ target_link_libraries(${perftest} PRIVATE libninja libninja-re2c)
+ endforeach()
+
+ if(CMAKE_SYSTEM_NAME STREQUAL "AIX" AND CMAKE_SIZEOF_VOID_P EQUAL 4)
+ # These tests require more memory than will fit in the standard AIX shared stack/heap (256M)
+ target_link_libraries(hash_collision_bench PRIVATE "-Wl,-bmaxdata:0x80000000")
+ target_link_libraries(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000")
+ endif()
+
+ add_test(NinjaTest ninja_test)
+endif()
+
+install(TARGETS ninja DESTINATION bin)
Index: README.md
===================================================================
--- README.md (nonexistent)
+++ README.md (revision 5)
@@ -0,0 +1,74 @@
+
+Kitware maintains this branch of Ninja in order to provide features
+that have not yet been integrated upstream:
+
+* make-style jobserver support
+
+This branch may be *rebased* without notice for maintenance on top of
+the upstream `master` branch. It will be removed once upstream has
+integrated the features.
+
+Parts of this branch are under upstream consideration:
+
+* https://github.com/ninja-build/ninja/pull/1140
+
+As each PR is accepted additional parts of this branch will be submitted
+incrementally.
+
+Binaries built from versions of this branch are available here:
+
+* https://github.com/Kitware/ninja/releases
+
+-----------------------------------------------------------------------------
+
+# Ninja
+
+Ninja is a small build system with a focus on speed.
+https://ninja-build.org/
+
+See [the manual](https://ninja-build.org/manual.html) or
+`doc/manual.asciidoc` included in the distribution for background
+and more details.
+
+Binaries for Linux, Mac, and Windows are available at
+ [GitHub](https://github.com/ninja-build/ninja/releases).
+Run `./ninja -h` for Ninja help.
+
+Installation is not necessary because the only required file is the
+resulting ninja binary. However, to enable features like Bash
+completion and Emacs and Vim editing modes, some files in misc/ must be
+copied to appropriate locations.
+
+If you're interested in making changes to Ninja, read
+[CONTRIBUTING.md](CONTRIBUTING.md) first.
+
+## Building Ninja itself
+
+You can either build Ninja via the custom generator script written in Python or
+via CMake. For more details see
+[the wiki](https://github.com/ninja-build/ninja/wiki).
+
+### Python
+
+```
+./configure.py --bootstrap
+```
+
+This will generate the `ninja` binary and a `build.ninja` file you can now use
+to build Ninja with itself.
+
+### CMake
+
+```
+cmake -Bbuild-cmake -H.
+cmake --build build-cmake
+```
+
+The `ninja` binary will now be inside the `build-cmake` directory (you can
+choose any other name you like).
+
+To run the unit tests:
+
+```
+./build-cmake/ninja_test
+```
Index: configure.py
===================================================================
--- configure.py (nonexistent)
+++ configure.py (revision 5)
@@ -0,0 +1,719 @@
+#!/usr/bin/env python
+#
+# Copyright 2001 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script that generates the build.ninja for ninja itself.
+
+Projects that use ninja themselves should either write a similar script
+or use a meta-build system that supports Ninja output."""
+
+from __future__ import print_function
+
+from optparse import OptionParser
+import os
+import pipes
+import string
+import subprocess
+import sys
+
+sourcedir = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(sourcedir, 'misc'))
+import ninja_syntax
+
+
+class Platform(object):
+ """Represents a host/target platform and its specific build attributes."""
+ def __init__(self, platform):
+ self._platform = platform
+ if self._platform is not None:
+ return
+ self._platform = sys.platform
+ if self._platform.startswith('linux'):
+ self._platform = 'linux'
+ elif self._platform.startswith('freebsd'):
+ self._platform = 'freebsd'
+ elif self._platform.startswith('gnukfreebsd'):
+ self._platform = 'freebsd'
+ elif self._platform.startswith('openbsd'):
+ self._platform = 'openbsd'
+ elif self._platform.startswith('solaris') or self._platform == 'sunos5':
+ self._platform = 'solaris'
+ elif self._platform.startswith('mingw'):
+ self._platform = 'mingw'
+ elif self._platform.startswith('win'):
+ self._platform = 'msvc'
+ elif self._platform.startswith('bitrig'):
+ self._platform = 'bitrig'
+ elif self._platform.startswith('netbsd'):
+ self._platform = 'netbsd'
+ elif self._platform.startswith('aix'):
+ self._platform = 'aix'
+ elif self._platform.startswith('os400'):
+ self._platform = 'os400'
+ elif self._platform.startswith('dragonfly'):
+ self._platform = 'dragonfly'
+
+ @staticmethod
+ def known_platforms():
+ return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5',
+ 'mingw', 'msvc', 'gnukfreebsd', 'bitrig', 'netbsd', 'aix',
+ 'dragonfly']
+
+ def platform(self):
+ return self._platform
+
+ def is_linux(self):
+ return self._platform == 'linux'
+
+ def is_mingw(self):
+ return self._platform == 'mingw'
+
+ def is_msvc(self):
+ return self._platform == 'msvc'
+
+ def msvc_needs_fs(self):
+ popen = subprocess.Popen(['cl', '/nologo', '/?'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = popen.communicate()
+ return b'/FS' in out
+
+ def is_windows(self):
+ return self.is_mingw() or self.is_msvc()
+
+ def is_solaris(self):
+ return self._platform == 'solaris'
+
+ def is_aix(self):
+ return self._platform == 'aix'
+
+ def is_os400_pase(self):
+ return self._platform == 'os400' or os.uname().sysname.startswith('OS400')
+
+ def uses_usr_local(self):
+ return self._platform in ('freebsd', 'openbsd', 'bitrig', 'dragonfly', 'netbsd')
+
+ def supports_ppoll(self):
+ return self._platform in ('freebsd', 'linux', 'openbsd', 'bitrig',
+ 'dragonfly')
+
+ def supports_ninja_browse(self):
+ return (not self.is_windows()
+ and not self.is_solaris()
+ and not self.is_aix())
+
+ def can_rebuild_in_place(self):
+ return not (self.is_windows() or self.is_aix())
+
+class Bootstrap:
+ """API shim for ninja_syntax.Writer that instead runs the commands.
+
+ Used to bootstrap Ninja from scratch. In --bootstrap mode this
+ class is used to execute all the commands to build an executable.
+ It also proxies all calls to an underlying ninja_syntax.Writer, to
+ behave like non-bootstrap mode.
+ """
+ def __init__(self, writer, verbose=False):
+ self.writer = writer
+ self.verbose = verbose
+ # Map of variable name => expanded variable value.
+ self.vars = {}
+ # Map of rule name => dict of rule attributes.
+ self.rules = {
+ 'phony': {}
+ }
+
+ def comment(self, text):
+ return self.writer.comment(text)
+
+ def newline(self):
+ return self.writer.newline()
+
+ def variable(self, key, val):
+ # In bootstrap mode, we have no ninja process to catch /showIncludes
+ # output.
+ self.vars[key] = self._expand(val).replace('/showIncludes', '')
+ return self.writer.variable(key, val)
+
+ def rule(self, name, **kwargs):
+ self.rules[name] = kwargs
+ return self.writer.rule(name, **kwargs)
+
+ def build(self, outputs, rule, inputs=None, **kwargs):
+ ruleattr = self.rules[rule]
+ cmd = ruleattr.get('command')
+ if cmd is None: # A phony rule, for example.
+ return
+
+ # Implement just enough of Ninja variable expansion etc. to
+ # make the bootstrap build work.
+ local_vars = {
+ 'in': self._expand_paths(inputs),
+ 'out': self._expand_paths(outputs)
+ }
+ for key, val in kwargs.get('variables', []):
+ local_vars[key] = ' '.join(ninja_syntax.as_list(val))
+
+ self._run_command(self._expand(cmd, local_vars))
+
+ return self.writer.build(outputs, rule, inputs, **kwargs)
+
+ def default(self, paths):
+ return self.writer.default(paths)
+
+ def _expand_paths(self, paths):
+ """Expand $vars in an array of paths, e.g. from a 'build' block."""
+ paths = ninja_syntax.as_list(paths)
+ return ' '.join(map(self._shell_escape, (map(self._expand, paths))))
+
+ def _expand(self, str, local_vars={}):
+ """Expand $vars in a string."""
+ return ninja_syntax.expand(str, self.vars, local_vars)
+
+ def _shell_escape(self, path):
+ """Quote paths containing spaces."""
+ return '"%s"' % path if ' ' in path else path
+
+ def _run_command(self, cmdline):
+ """Run a subcommand, quietly. Prints the full command on error."""
+ try:
+ if self.verbose:
+ print(cmdline)
+ subprocess.check_call(cmdline, shell=True)
+ except subprocess.CalledProcessError:
+ print('when running: ', cmdline)
+ raise
+
+
+parser = OptionParser()
+profilers = ['gmon', 'pprof']
+parser.add_option('--bootstrap', action='store_true',
+ help='bootstrap a ninja binary from nothing')
+parser.add_option('--verbose', action='store_true',
+ help='enable verbose build')
+parser.add_option('--platform',
+ help='target platform (' +
+ '/'.join(Platform.known_platforms()) + ')',
+ choices=Platform.known_platforms())
+parser.add_option('--host',
+ help='host platform (' +
+ '/'.join(Platform.known_platforms()) + ')',
+ choices=Platform.known_platforms())
+parser.add_option('--debug', action='store_true',
+ help='enable debugging extras',)
+parser.add_option('--profile', metavar='TYPE',
+ choices=profilers,
+ help='enable profiling (' + '/'.join(profilers) + ')',)
+parser.add_option('--with-gtest', metavar='PATH', help='ignored')
+parser.add_option('--with-python', metavar='EXE',
+ help='use EXE as the Python interpreter',
+ default=os.path.basename(sys.executable))
+parser.add_option('--force-pselect', action='store_true',
+ help='ppoll() is used by default where available, '
+ 'but some platforms may need to use pselect instead',)
+(options, args) = parser.parse_args()
+if args:
+ print('ERROR: extra unparsed command-line arguments:', args)
+ sys.exit(1)
+
+platform = Platform(options.platform)
+if options.host:
+ host = Platform(options.host)
+else:
+ host = platform
+
+BUILD_FILENAME = 'build.ninja'
+ninja_writer = ninja_syntax.Writer(open(BUILD_FILENAME, 'w'))
+n = ninja_writer
+
+if options.bootstrap:
+ # Make the build directory.
+ try:
+ os.mkdir('build')
+ except OSError:
+ pass
+ # Wrap ninja_writer with the Bootstrapper, which also executes the
+ # commands.
+ print('bootstrapping ninja...')
+ n = Bootstrap(n, verbose=options.verbose)
+
+n.comment('This file is used to build ninja itself.')
+n.comment('It is generated by ' + os.path.basename(__file__) + '.')
+n.newline()
+
+n.variable('ninja_required_version', '1.3')
+n.newline()
+
+n.comment('The arguments passed to configure.py, for rerunning it.')
+configure_args = sys.argv[1:]
+if '--bootstrap' in configure_args:
+ configure_args.remove('--bootstrap')
+n.variable('configure_args', ' '.join(configure_args))
+env_keys = set(['CXX', 'AR', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS'])
+configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys)
+if configure_env:
+ config_str = ' '.join([k + '=' + pipes.quote(configure_env[k])
+ for k in configure_env])
+ n.variable('configure_env', config_str + '$ ')
+n.newline()
+
+CXX = configure_env.get('CXX', 'c++')
+objext = '.o'
+if platform.is_msvc():
+ CXX = 'cl'
+ objext = '.obj'
+
+def src(filename):
+ return os.path.join('$root', 'src', filename)
+def built(filename):
+ return os.path.join('$builddir', filename)
+def doc(filename):
+ return os.path.join('$root', 'doc', filename)
+def cc(name, **kwargs):
+ return n.build(built(name + objext), 'cxx', src(name + '.c'), **kwargs)
+def cxx(name, **kwargs):
+ return n.build(built(name + objext), 'cxx', src(name + '.cc'), **kwargs)
+def binary(name):
+ if platform.is_windows():
+ exe = name + '.exe'
+ n.build(name, 'phony', exe)
+ return exe
+ return name
+
+root = sourcedir
+if root == os.getcwd():
+ # In the common case where we're building directly in the source
+ # tree, simplify all the paths to just be cwd-relative.
+ root = '.'
+n.variable('root', root)
+n.variable('builddir', 'build')
+n.variable('cxx', CXX)
+if platform.is_msvc():
+ n.variable('ar', 'link')
+else:
+ n.variable('ar', configure_env.get('AR', 'ar'))
+
+if platform.is_msvc():
+ cflags = ['/showIncludes',
+ '/nologo', # Don't print startup banner.
+ '/Zi', # Create pdb with debug info.
+ '/W4', # Highest warning level.
+ '/WX', # Warnings as errors.
+ '/wd4530', '/wd4100', '/wd4706', '/wd4244',
+ '/wd4512', '/wd4800', '/wd4702', '/wd4819',
+ # Disable warnings about constant conditional expressions.
+ '/wd4127',
+ # Disable warnings about passing "this" during initialization.
+ '/wd4355',
+ # Disable warnings about ignored typedef in DbgHelp.h
+ '/wd4091',
+ '/GR-', # Disable RTTI.
+ # Disable size_t -> int truncation warning.
+ # We never have strings or arrays larger than 2**31.
+ '/wd4267',
+ '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS',
+ '/D_HAS_EXCEPTIONS=0',
+ '/DNINJA_PYTHON="%s"' % options.with_python]
+ if platform.msvc_needs_fs():
+ cflags.append('/FS')
+ ldflags = ['/DEBUG', '/libpath:$builddir']
+ if not options.debug:
+ cflags += ['/Ox', '/DNDEBUG', '/GL']
+ ldflags += ['/LTCG', '/OPT:REF', '/OPT:ICF']
+else:
+ cflags = ['-g', '-Wall', '-Wextra',
+ '-Wno-deprecated',
+ '-Wno-missing-field-initializers',
+ '-Wno-unused-parameter',
+ '-fno-rtti',
+ '-fno-exceptions',
+ '-fvisibility=hidden', '-pipe',
+ '-DNINJA_PYTHON="%s"' % options.with_python]
+ if options.debug:
+ cflags += ['-D_GLIBCXX_DEBUG', '-D_GLIBCXX_DEBUG_PEDANTIC']
+ cflags.remove('-fno-rtti') # Needed for above pedanticness.
+ else:
+ cflags += ['-O2', '-DNDEBUG']
+ try:
+ proc = subprocess.Popen(
+ [CXX, '-fdiagnostics-color', '-c', '-x', 'c++', '/dev/null',
+ '-o', '/dev/null'],
+ stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
+ if proc.wait() == 0:
+ cflags += ['-fdiagnostics-color']
+ except:
+ pass
+ if platform.is_mingw():
+ cflags += ['-D_WIN32_WINNT=0x0601', '-D__USE_MINGW_ANSI_STDIO=1']
+ ldflags = ['-L$builddir']
+ if platform.uses_usr_local():
+ cflags.append('-I/usr/local/include')
+ ldflags.append('-L/usr/local/lib')
+ if platform.is_aix():
+ # printf formats for int64_t, uint64_t; large file support
+ cflags.append('-D__STDC_FORMAT_MACROS')
+ cflags.append('-D_LARGE_FILES')
+
+
+libs = []
+
+if platform.is_mingw():
+ cflags.remove('-fvisibility=hidden');
+ ldflags.append('-static')
+elif platform.is_solaris():
+ cflags.remove('-fvisibility=hidden')
+elif platform.is_aix():
+ cflags.remove('-fvisibility=hidden')
+elif platform.is_msvc():
+ pass
+else:
+ if options.profile == 'gmon':
+ cflags.append('-pg')
+ ldflags.append('-pg')
+ elif options.profile == 'pprof':
+ cflags.append('-fno-omit-frame-pointer')
+ libs.extend(['-Wl,--no-as-needed', '-lprofiler'])
+
+if platform.supports_ppoll() and not options.force_pselect:
+ cflags.append('-DUSE_PPOLL')
+if platform.supports_ninja_browse():
+ cflags.append('-DNINJA_HAVE_BROWSE')
+
+# Search for generated headers relative to build dir.
+cflags.append('-I.')
+
+def shell_escape(str):
+ """Escape str such that it's interpreted as a single argument by
+ the shell."""
+
+ # This isn't complete, but it's just enough to make NINJA_PYTHON work.
+ if platform.is_windows():
+ return str
+ if '"' in str:
+ return "'%s'" % str.replace("'", "\\'")
+ return str
+
+if 'CFLAGS' in configure_env:
+ cflags.append(configure_env['CFLAGS'])
+ ldflags.append(configure_env['CFLAGS'])
+if 'CXXFLAGS' in configure_env:
+ cflags.append(configure_env['CXXFLAGS'])
+ ldflags.append(configure_env['CXXFLAGS'])
+n.variable('cflags', ' '.join(shell_escape(flag) for flag in cflags))
+if 'LDFLAGS' in configure_env:
+ ldflags.append(configure_env['LDFLAGS'])
+n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags))
+n.newline()
+
+if platform.is_msvc():
+ n.rule('cxx',
+ command='$cxx $cflags -c $in /Fo$out /Fd' + built('$pdb'),
+ description='CXX $out',
+ deps='msvc' # /showIncludes is included in $cflags.
+ )
+else:
+ n.rule('cxx',
+ command='$cxx -MMD -MT $out -MF $out.d $cflags -c $in -o $out',
+ depfile='$out.d',
+ deps='gcc',
+ description='CXX $out')
+n.newline()
+
+if host.is_msvc():
+ n.rule('ar',
+ command='lib /nologo /ltcg /out:$out $in',
+ description='LIB $out')
+elif host.is_mingw():
+ n.rule('ar',
+ command='$ar crs $out $in',
+ description='AR $out')
+else:
+ n.rule('ar',
+ command='rm -f $out && $ar crs $out $in',
+ description='AR $out')
+n.newline()
+
+if platform.is_msvc():
+ n.rule('link',
+ command='$cxx $in $libs /nologo /link $ldflags /out:$out',
+ description='LINK $out')
+else:
+ n.rule('link',
+ command='$cxx $ldflags -o $out $in $libs',
+ description='LINK $out')
+n.newline()
+
+objs = []
+
+if platform.supports_ninja_browse():
+ n.comment('browse_py.h is used to inline browse.py.')
+ n.rule('inline',
+ command='"%s"' % src('inline.sh') + ' $varname < $in > $out',
+ description='INLINE $out')
+ n.build(built('browse_py.h'), 'inline', src('browse.py'),
+ implicit=src('inline.sh'),
+ variables=[('varname', 'kBrowsePy')])
+ n.newline()
+
+ objs += cxx('browse', order_only=built('browse_py.h'))
+ n.newline()
+
+n.comment('the depfile parser and ninja lexers are generated using re2c.')
+def has_re2c():
+ try:
+ proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE)
+ return int(proc.communicate()[0], 10) >= 1103
+ except OSError:
+ return False
+if has_re2c():
+ n.rule('re2c',
+ command='re2c -b -i --no-generation-date -o $out $in',
+ description='RE2C $out')
+ # Generate the .cc files in the source directory so we can check them in.
+ n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
+ n.build(src('lexer.cc'), 're2c', src('lexer.in.cc'))
+else:
+ print("warning: A compatible version of re2c (>= 0.11.3) was not found; "
+ "changes to src/*.in.cc will not affect your build.")
+n.newline()
+
+n.comment('Core source files all build into ninja library.')
+cxxvariables = []
+if platform.is_msvc():
+ cxxvariables = [('pdb', 'ninja.pdb')]
+for name in ['build',
+ 'build_log',
+ 'clean',
+ 'clparser',
+ 'debug_flags',
+ 'depfile_parser',
+ 'deps_log',
+ 'disk_interface',
+ 'dyndep',
+ 'dyndep_parser',
+ 'edit_distance',
+ 'eval_env',
+ 'graph',
+ 'graphviz',
+ 'lexer',
+ 'line_printer',
+ 'manifest_parser',
+ 'metrics',
+ 'parser',
+ 'state',
+ 'string_piece_util',
+ 'tokenpool-gnu-make',
+ 'util',
+ 'version']:
+ objs += cxx(name, variables=cxxvariables)
+if platform.is_windows():
+ for name in ['subprocess-win32',
+ 'tokenpool-gnu-make-win32',
+ 'includes_normalize-win32',
+ 'msvc_helper-win32',
+ 'msvc_helper_main-win32']:
+ objs += cxx(name, variables=cxxvariables)
+ if platform.is_msvc():
+ objs += cxx('minidump-win32', variables=cxxvariables)
+ objs += cc('getopt')
+else:
+ for name in ['subprocess-posix',
+ 'tokenpool-gnu-make-posix']:
+ objs += cxx(name)
+if platform.is_aix():
+ objs += cc('getopt')
+if platform.is_msvc():
+ ninja_lib = n.build(built('ninja.lib'), 'ar', objs)
+else:
+ ninja_lib = n.build(built('libninja.a'), 'ar', objs)
+n.newline()
+
+if platform.is_msvc():
+ libs.append('ninja.lib')
+else:
+ libs.append('-lninja')
+
+if platform.is_aix() and not platform.is_os400_pase():
+ libs.append('-lperfstat')
+
+all_targets = []
+
+n.comment('Main executable is library plus main() function.')
+objs = cxx('ninja', variables=cxxvariables)
+ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib,
+ variables=[('libs', libs)])
+n.newline()
+all_targets += ninja
+
+if options.bootstrap:
+ # We've built the ninja binary. Don't run any more commands
+ # through the bootstrap executor, but continue writing the
+ # build.ninja file.
+ n = ninja_writer
+
+n.comment('Tests all build into ninja_test executable.')
+
+objs = []
+if platform.is_msvc():
+ cxxvariables = [('pdb', 'ninja_test.pdb')]
+
+for name in ['build_log_test',
+ 'build_test',
+ 'clean_test',
+ 'clparser_test',
+ 'depfile_parser_test',
+ 'deps_log_test',
+ 'dyndep_parser_test',
+ 'disk_interface_test',
+ 'edit_distance_test',
+ 'graph_test',
+ 'lexer_test',
+ 'manifest_parser_test',
+ 'ninja_test',
+ 'state_test',
+ 'string_piece_util_test',
+ 'subprocess_test',
+ 'test',
+ 'tokenpool_test',
+ 'util_test']:
+ objs += cxx(name, variables=cxxvariables)
+if platform.is_windows():
+ for name in ['includes_normalize_test', 'msvc_helper_test']:
+ objs += cxx(name, variables=cxxvariables)
+
+ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib,
+ variables=[('libs', libs)])
+n.newline()
+all_targets += ninja_test
+
+
+n.comment('Ancillary executables.')
+
+if platform.is_aix() and '-maix64' not in ldflags:
+ # Both hash_collision_bench and manifest_parser_perftest require more
+ # memory than will fit in the standard 32-bit AIX shared stack/heap (256M)
+ libs.append('-Wl,-bmaxdata:0x80000000')
+
+for name in ['build_log_perftest',
+ 'canon_perftest',
+ 'depfile_parser_perftest',
+ 'hash_collision_bench',
+ 'manifest_parser_perftest',
+ 'clparser_perftest']:
+ if platform.is_msvc():
+ cxxvariables = [('pdb', name + '.pdb')]
+ objs = cxx(name, variables=cxxvariables)
+ all_targets += n.build(binary(name), 'link', objs,
+ implicit=ninja_lib, variables=[('libs', libs)])
+
+n.newline()
+
+n.comment('Generate a graph using the "graph" tool.')
+n.rule('gendot',
+ command='./ninja -t graph all > $out')
+n.rule('gengraph',
+ command='dot -Tpng $in > $out')
+dot = n.build(built('graph.dot'), 'gendot', ['ninja', 'build.ninja'])
+n.build('graph.png', 'gengraph', dot)
+n.newline()
+
+n.comment('Generate the manual using asciidoc.')
+n.rule('asciidoc',
+ command='asciidoc -b docbook -d book -o $out $in',
+ description='ASCIIDOC $out')
+n.rule('xsltproc',
+ command='xsltproc --nonet doc/docbook.xsl $in > $out',
+ description='XSLTPROC $out')
+docbookxml = n.build(built('manual.xml'), 'asciidoc', doc('manual.asciidoc'))
+manual = n.build(doc('manual.html'), 'xsltproc', docbookxml,
+ implicit=[doc('style.css'), doc('docbook.xsl')])
+n.build('manual', 'phony',
+ order_only=manual)
+n.newline()
+
+n.rule('dblatex',
+ command='dblatex -q -o $out -p doc/dblatex.xsl $in',
+ description='DBLATEX $out')
+n.build(doc('manual.pdf'), 'dblatex', docbookxml,
+ implicit=[doc('dblatex.xsl')])
+
+n.comment('Generate Doxygen.')
+n.rule('doxygen',
+ command='doxygen $in',
+ description='DOXYGEN $in')
+n.variable('doxygen_mainpage_generator',
+ src('gen_doxygen_mainpage.sh'))
+n.rule('doxygen_mainpage',
+ command='$doxygen_mainpage_generator $in > $out',
+ description='DOXYGEN_MAINPAGE $out')
+mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage',
+ ['README.md', 'COPYING'],
+ implicit=['$doxygen_mainpage_generator'])
+n.build('doxygen', 'doxygen', doc('doxygen.config'),
+ implicit=mainpage)
+n.newline()
+
+if not host.is_mingw():
+ n.comment('Regenerate build files if build script changes.')
+ n.rule('configure',
+ command='${configure_env}%s $root/configure.py $configure_args' %
+ options.with_python,
+ generator=True)
+ n.build('build.ninja', 'configure',
+ implicit=['$root/configure.py',
+ os.path.normpath('$root/misc/ninja_syntax.py')])
+ n.newline()
+
+n.default(ninja)
+n.newline()
+
+if host.is_linux():
+ n.comment('Packaging')
+ n.rule('rpmbuild',
+ command="misc/packaging/rpmbuild.sh",
+ description='Building rpms..')
+ n.build('rpm', 'rpmbuild')
+ n.newline()
+
+n.build('all', 'phony', all_targets)
+
+n.close()
+print('wrote %s.' % BUILD_FILENAME)
+
+if options.bootstrap:
+ print('bootstrap complete. rebuilding...')
+
+ rebuild_args = []
+
+ if platform.can_rebuild_in_place():
+ rebuild_args.append('./ninja')
+ else:
+ if platform.is_windows():
+ bootstrap_exe = 'ninja.bootstrap.exe'
+ final_exe = 'ninja.exe'
+ else:
+ bootstrap_exe = './ninja.bootstrap'
+ final_exe = './ninja'
+
+ if os.path.exists(bootstrap_exe):
+ os.unlink(bootstrap_exe)
+ os.rename(final_exe, bootstrap_exe)
+
+ rebuild_args.append(bootstrap_exe)
+
+ if options.verbose:
+ rebuild_args.append('-v')
+
+ subprocess.check_call(rebuild_args)
Property changes on: configure.py
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
Index: src/build.cc
===================================================================
--- src/build.cc (nonexistent)
+++ src/build.cc (revision 5)
@@ -0,0 +1,1196 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "build.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <functional>
+
+#ifdef _WIN32
+#include <fcntl.h>
+#include <io.h>
+#endif
+
+#if defined(__SVR4) && defined(__sun)
+#include <sys/termios.h>
+#endif
+
+#include "build_log.h"
+#include "clparser.h"
+#include "debug_flags.h"
+#include "depfile_parser.h"
+#include "deps_log.h"
+#include "disk_interface.h"
+#include "graph.h"
+#include "state.h"
+#include "subprocess.h"
+#include "tokenpool.h"
+#include "util.h"
+
+using namespace std;
+
+namespace {
+
+/// A CommandRunner that doesn't actually run the commands.
+struct DryRunCommandRunner : public CommandRunner {
+ virtual ~DryRunCommandRunner() {}
+
+ // Overridden from CommandRunner:
+ virtual bool CanRunMore() const;
+ virtual bool AcquireToken();
+ virtual bool StartCommand(Edge* edge);
+ virtual bool WaitForCommand(Result* result, bool more_ready);
+
+ private:
+ queue<Edge*> finished_;
+};
+
+bool DryRunCommandRunner::CanRunMore() const {
+ return true;
+}
+
+bool DryRunCommandRunner::AcquireToken() {
+ return true;
+}
+
+bool DryRunCommandRunner::StartCommand(Edge* edge) {
+ finished_.push(edge);
+ return true;
+}
+
+bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) {
+ if (finished_.empty())
+ return false;
+
+ result->status = ExitSuccess;
+ result->edge = finished_.front();
+ finished_.pop();
+ return true;
+}
+
+} // namespace
+
+BuildStatus::BuildStatus(const BuildConfig& config)
+ : config_(config), start_time_millis_(GetTimeMillis()), started_edges_(0),
+ finished_edges_(0), total_edges_(0), progress_status_format_(NULL),
+ current_rate_(config.parallelism) {
+ // Don't do anything fancy in verbose mode.
+ if (config_.verbosity != BuildConfig::NORMAL)
+ printer_.set_smart_terminal(false);
+
+ progress_status_format_ = getenv("NINJA_STATUS");
+ if (!progress_status_format_)
+ progress_status_format_ = "[%f/%t] ";
+}
+
+void BuildStatus::PlanHasTotalEdges(int total) {
+ total_edges_ = total;
+}
+
+void BuildStatus::BuildEdgeStarted(const Edge* edge) {
+ assert(running_edges_.find(edge) == running_edges_.end());
+ int start_time = (int)(GetTimeMillis() - start_time_millis_);
+ running_edges_.insert(make_pair(edge, start_time));
+ ++started_edges_;
+
+ if (edge->use_console() || printer_.is_smart_terminal())
+ PrintStatus(edge, kEdgeStarted);
+
+ if (edge->use_console())
+ printer_.SetConsoleLocked(true);
+}
+
+void BuildStatus::BuildEdgeFinished(Edge* edge,
+ bool success,
+ const string& output,
+ int* start_time,
+ int* end_time) {
+ int64_t now = GetTimeMillis();
+
+ ++finished_edges_;
+
+ RunningEdgeMap::iterator i = running_edges_.find(edge);
+ *start_time = i->second;
+ *end_time = (int)(now - start_time_millis_);
+ running_edges_.erase(i);
+
+ if (edge->use_console())
+ printer_.SetConsoleLocked(false);
+
+ if (config_.verbosity == BuildConfig::QUIET)
+ return;
+
+ if (!edge->use_console())
+ PrintStatus(edge, kEdgeFinished);
+
+ // Print the command that is spewing before printing its output.
+ if (!success) {
+ string outputs;
+ for (vector<Node*>::const_iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o)
+ outputs += (*o)->path() + " ";
+
+ if (printer_.supports_color()) {
+ printer_.PrintOnNewLine("\x1B[31m" "FAILED: " "\x1B[0m" + outputs + "\n");
+ } else {
+ printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
+ }
+ printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n");
+ }
+
+ if (!output.empty()) {
+ // ninja sets stdout and stderr of subprocesses to a pipe, to be able to
+ // check if the output is empty. Some compilers, e.g. clang, check
+ // isatty(stderr) to decide if they should print colored output.
+ // To make it possible to use colored output with ninja, subprocesses should
+ // be run with a flag that forces them to always print color escape codes.
+ // To make sure these escape codes don't show up in a file if ninja's output
+ // is piped to a file, ninja strips ansi escape codes again if it's not
+ // writing to a |smart_terminal_|.
+ // (Launching subprocesses in pseudo ttys doesn't work because there are
+ // only a few hundred available on some systems, and ninja can launch
+ // thousands of parallel compile commands.)
+ string final_output;
+ if (!printer_.supports_color())
+ final_output = StripAnsiEscapeCodes(output);
+ else
+ final_output = output;
+
+#ifdef _WIN32
+ // Fix extra CR being added on Windows, writing out CR CR LF (#773)
+ _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix
+#endif
+
+ printer_.PrintOnNewLine(final_output);
+
+#ifdef _WIN32
+ _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix
+#endif
+ }
+}
+
+void BuildStatus::BuildLoadDyndeps() {
+ // The DependencyScan calls EXPLAIN() to print lines explaining why
+ // it considers a portion of the graph to be out of date. Normally
+ // this is done before the build starts, but our caller is about to
+ // load a dyndep file during the build. Doing so may generate more
+ // explanation lines (via fprintf directly to stderr), but in an
+ // interactive console the cursor is currently at the end of a status
+ // line. Start a new line so that the first explanation does not
+ // append to the status line. After the explanations are done a
+ // new build status line will appear.
+ if (g_explaining)
+ printer_.PrintOnNewLine("");
+}
+
+void BuildStatus::BuildStarted() {
+ overall_rate_.Restart();
+ current_rate_.Restart();
+}
+
+void BuildStatus::BuildFinished() {
+ printer_.SetConsoleLocked(false);
+ printer_.PrintOnNewLine("");
+}
+
+string BuildStatus::FormatProgressStatus(
+ const char* progress_status_format, EdgeStatus status) const {
+ string out;
+ char buf[32];
+ int percent;
+ for (const char* s = progress_status_format; *s != '\0'; ++s) {
+ if (*s == '%') {
+ ++s;
+ switch (*s) {
+ case '%':
+ out.push_back('%');
+ break;
+
+ // Started edges.
+ case 's':
+ snprintf(buf, sizeof(buf), "%d", started_edges_);
+ out += buf;
+ break;
+
+ // Total edges.
+ case 't':
+ snprintf(buf, sizeof(buf), "%d", total_edges_);
+ out += buf;
+ break;
+
+ // Running edges.
+ case 'r': {
+ int running_edges = started_edges_ - finished_edges_;
+ // count the edge that just finished as a running edge
+ if (status == kEdgeFinished)
+ running_edges++;
+ snprintf(buf, sizeof(buf), "%d", running_edges);
+ out += buf;
+ break;
+ }
+
+ // Unstarted edges.
+ case 'u':
+ snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_);
+ out += buf;
+ break;
+
+ // Finished edges.
+ case 'f':
+ snprintf(buf, sizeof(buf), "%d", finished_edges_);
+ out += buf;
+ break;
+
+ // Overall finished edges per second.
+ case 'o':
+ overall_rate_.UpdateRate(finished_edges_);
+ SnprintfRate(overall_rate_.rate(), buf, "%.1f");
+ out += buf;
+ break;
+
+ // Current rate, average over the last '-j' jobs.
+ case 'c':
+ current_rate_.UpdateRate(finished_edges_);
+ SnprintfRate(current_rate_.rate(), buf, "%.1f");
+ out += buf;
+ break;
+
+ // Percentage
+ case 'p':
+ percent = (100 * finished_edges_) / total_edges_;
+ snprintf(buf, sizeof(buf), "%3i%%", percent);
+ out += buf;
+ break;
+
+ case 'e': {
+ double elapsed = overall_rate_.Elapsed();
+ snprintf(buf, sizeof(buf), "%.3f", elapsed);
+ out += buf;
+ break;
+ }
+
+ default:
+ Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s);
+ return "";
+ }
+ } else {
+ out.push_back(*s);
+ }
+ }
+
+ return out;
+}
+
+void BuildStatus::PrintStatus(const Edge* edge, EdgeStatus status) {
+ if (config_.verbosity == BuildConfig::QUIET)
+ return;
+
+ bool force_full_command = config_.verbosity == BuildConfig::VERBOSE;
+
+ string to_print = edge->GetBinding("description");
+ if (to_print.empty() || force_full_command)
+ to_print = edge->GetBinding("command");
+
+ to_print = FormatProgressStatus(progress_status_format_, status) + to_print;
+
+ printer_.Print(to_print,
+ force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
+}
+
+Plan::Plan(Builder* builder)
+ : builder_(builder)
+ , command_edges_(0)
+ , wanted_edges_(0)
+{}
+
+void Plan::Reset() {
+ command_edges_ = 0;
+ wanted_edges_ = 0;
+ ready_.clear();
+ want_.clear();
+}
+
+bool Plan::AddTarget(const Node* node, string* err) {
+ return AddSubTarget(node, NULL, err, NULL);
+}
+
+bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err,
+ set<Edge*>* dyndep_walk) {
+ Edge* edge = node->in_edge();
+ if (!edge) { // Leaf node.
+ if (node->dirty()) {
+ string referenced;
+ if (dependent)
+ referenced = ", needed by '" + dependent->path() + "',";
+ *err = "'" + node->path() + "'" + referenced + " missing "
+ "and no known rule to make it";
+ }
+ return false;
+ }
+
+ if (edge->outputs_ready())
+ return false; // Don't need to do anything.
+
+ // If an entry in want_ does not already exist for edge, create an entry which
+ // maps to kWantNothing, indicating that we do not want to build this entry itself.
+ pair<map<Edge*, Want>::iterator, bool> want_ins =
+ want_.insert(make_pair(edge, kWantNothing));
+ Want& want = want_ins.first->second;
+
+ if (dyndep_walk && want == kWantToFinish)
+ return false; // Don't need to do anything with already-scheduled edge.
+
+ // If we do need to build edge and we haven't already marked it as wanted,
+ // mark it now.
+ if (node->dirty() && want == kWantNothing) {
+ want = kWantToStart;
+ EdgeWanted(edge);
+ if (!dyndep_walk && edge->AllInputsReady())
+ ScheduleWork(want_ins.first);
+ }
+
+ if (dyndep_walk)
+ dyndep_walk->insert(edge);
+
+ if (!want_ins.second)
+ return true; // We've already processed the inputs.
+
+ for (vector<Node*>::iterator i = edge->inputs_.begin();
+ i != edge->inputs_.end(); ++i) {
+ if (!AddSubTarget(*i, node, err, dyndep_walk) && !err->empty())
+ return false;
+ }
+
+ return true;
+}
+
+void Plan::EdgeWanted(const Edge* edge) {
+ ++wanted_edges_;
+ if (!edge->is_phony())
+ ++command_edges_;
+}
+
+Edge* Plan::FindWork() {
+ if (!more_ready())
+ return NULL;
+ set<Edge*>::iterator e = ready_.begin();
+ Edge* edge = *e;
+ ready_.erase(e);
+ return edge;
+}
+
+void Plan::ScheduleWork(map<Edge*, Want>::iterator want_e) {
+ if (want_e->second == kWantToFinish) {
+ // This edge has already been scheduled. We can get here again if an edge
+ // and one of its dependencies share an order-only input, or if a node
+ // duplicates an out edge (see https://github.com/ninja-build/ninja/pull/519).
+ // Avoid scheduling the work again.
+ return;
+ }
+ assert(want_e->second == kWantToStart);
+ want_e->second = kWantToFinish;
+
+ Edge* edge = want_e->first;
+ Pool* pool = edge->pool();
+ if (pool->ShouldDelayEdge()) {
+ pool->DelayEdge(edge);
+ pool->RetrieveReadyEdges(&ready_);
+ } else {
+ pool->EdgeScheduled(*edge);
+ ready_.insert(edge);
+ }
+}
+
+bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) {
+ map<Edge*, Want>::iterator e = want_.find(edge);
+ assert(e != want_.end());
+ bool directly_wanted = e->second != kWantNothing;
+
+ // See if this job frees up any delayed jobs.
+ if (directly_wanted)
+ edge->pool()->EdgeFinished(*edge);
+ edge->pool()->RetrieveReadyEdges(&ready_);
+
+ // The rest of this function only applies to successful commands.
+ if (result != kEdgeSucceeded)
+ return true;
+
+ if (directly_wanted)
+ --wanted_edges_;
+ want_.erase(e);
+ edge->outputs_ready_ = true;
+
+ // Check off any nodes we were waiting for with this edge.
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ if (!NodeFinished(*o, err))
+ return false;
+ }
+ return true;
+}
+
+bool Plan::NodeFinished(Node* node, string* err) {
+ // If this node provides dyndep info, load it now.
+ if (node->dyndep_pending()) {
+ assert(builder_ && "dyndep requires Plan to have a Builder");
+ // Load the now-clean dyndep file. This will also update the
+ // build plan and schedule any new work that is ready.
+ return builder_->LoadDyndeps(node, err);
+ }
+
+ // See if we we want any edges from this node.
+ for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
+ oe != node->out_edges().end(); ++oe) {
+ map<Edge*, Want>::iterator want_e = want_.find(*oe);
+ if (want_e == want_.end())
+ continue;
+
+ // See if the edge is now ready.
+ if (!EdgeMaybeReady(want_e, err))
+ return false;
+ }
+ return true;
+}
+
+bool Plan::EdgeMaybeReady(map<Edge*, Want>::iterator want_e, string* err) {
+ Edge* edge = want_e->first;
+ if (edge->AllInputsReady()) {
+ if (want_e->second != kWantNothing) {
+ ScheduleWork(want_e);
+ } else {
+ // We do not need to build this edge, but we might need to build one of
+ // its dependents.
+ if (!EdgeFinished(edge, kEdgeSucceeded, err))
+ return false;
+ }
+ }
+ return true;
+}
+
+bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
+ node->set_dirty(false);
+
+ for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
+ oe != node->out_edges().end(); ++oe) {
+ // Don't process edges that we don't actually want.
+ map<Edge*, Want>::iterator want_e = want_.find(*oe);
+ if (want_e == want_.end() || want_e->second == kWantNothing)
+ continue;
+
+ // Don't attempt to clean an edge if it failed to load deps.
+ if ((*oe)->deps_missing_)
+ continue;
+
+ // If all non-order-only inputs for this edge are now clean,
+ // we might have changed the dirty state of the outputs.
+ vector<Node*>::iterator
+ begin = (*oe)->inputs_.begin(),
+ end = (*oe)->inputs_.end() - (*oe)->order_only_deps_;
+#if __cplusplus < 201703L
+#define MEM_FN mem_fun
+#else
+#define MEM_FN mem_fn // mem_fun was removed in C++17.
+#endif
+ if (find_if(begin, end, MEM_FN(&Node::dirty)) == end) {
+ // Recompute most_recent_input.
+ Node* most_recent_input = NULL;
+ for (vector<Node*>::iterator i = begin; i != end; ++i) {
+ if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime())
+ most_recent_input = *i;
+ }
+
+ // Now, this edge is dirty if any of the outputs are dirty.
+ // If the edge isn't dirty, clean the outputs and mark the edge as not
+ // wanted.
+ bool outputs_dirty = false;
+ if (!scan->RecomputeOutputsDirty(*oe, most_recent_input,
+ &outputs_dirty, err)) {
+ return false;
+ }
+ if (!outputs_dirty) {
+ for (vector<Node*>::iterator o = (*oe)->outputs_.begin();
+ o != (*oe)->outputs_.end(); ++o) {
+ if (!CleanNode(scan, *o, err))
+ return false;
+ }
+
+ want_e->second = kWantNothing;
+ --wanted_edges_;
+ if (!(*oe)->is_phony())
+ --command_edges_;
+ }
+ }
+ }
+ return true;
+}
+
+bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node,
+ const DyndepFile& ddf, string* err) {
+ // Recompute the dirty state of all our direct and indirect dependents now
+ // that our dyndep information has been loaded.
+ if (!RefreshDyndepDependents(scan, node, err))
+ return false;
+
+ // We loaded dyndep information for those out_edges of the dyndep node that
+ // specify the node in a dyndep binding, but they may not be in the plan.
+ // Starting with those already in the plan, walk newly-reachable portion
+ // of the graph through the dyndep-discovered dependencies.
+
+ // Find edges in the the build plan for which we have new dyndep info.
+ std::vector<DyndepFile::const_iterator> dyndep_roots;
+ for (DyndepFile::const_iterator oe = ddf.begin(); oe != ddf.end(); ++oe) {
+ Edge* edge = oe->first;
+
+ // If the edge outputs are ready we do not need to consider it here.
+ if (edge->outputs_ready())
+ continue;
+
+ map<Edge*, Want>::iterator want_e = want_.find(edge);
+
+ // If the edge has not been encountered before then nothing already in the
+ // plan depends on it so we do not need to consider the edge yet either.
+ if (want_e == want_.end())
+ continue;
+
+ // This edge is already in the plan so queue it for the walk.
+ dyndep_roots.push_back(oe);
+ }
+
+ // Walk dyndep-discovered portion of the graph to add it to the build plan.
+ std::set<Edge*> dyndep_walk;
+ for (std::vector<DyndepFile::const_iterator>::iterator
+ oei = dyndep_roots.begin(); oei != dyndep_roots.end(); ++oei) {
+ DyndepFile::const_iterator oe = *oei;
+ for (vector<Node*>::const_iterator i = oe->second.implicit_inputs_.begin();
+ i != oe->second.implicit_inputs_.end(); ++i) {
+ if (!AddSubTarget(*i, oe->first->outputs_[0], err, &dyndep_walk) &&
+ !err->empty())
+ return false;
+ }
+ }
+
+ // Add out edges from this node that are in the plan (just as
+ // Plan::NodeFinished would have without taking the dyndep code path).
+ for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
+ oe != node->out_edges().end(); ++oe) {
+ map<Edge*, Want>::iterator want_e = want_.find(*oe);
+ if (want_e == want_.end())
+ continue;
+ dyndep_walk.insert(want_e->first);
+ }
+
+ // See if any encountered edges are now ready.
+ for (set<Edge*>::iterator wi = dyndep_walk.begin();
+ wi != dyndep_walk.end(); ++wi) {
+ map<Edge*, Want>::iterator want_e = want_.find(*wi);
+ if (want_e == want_.end())
+ continue;
+ if (!EdgeMaybeReady(want_e, err))
+ return false;
+ }
+
+ return true;
+}
+
+bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node,
+ string* err) {
+ // Collect the transitive closure of dependents and mark their edges
+ // as not yet visited by RecomputeDirty.
+ set<Node*> dependents;
+ UnmarkDependents(node, &dependents);
+
+ // Update the dirty state of all dependents and check if their edges
+ // have become wanted.
+ for (set<Node*>::iterator i = dependents.begin();
+ i != dependents.end(); ++i) {
+ Node* n = *i;
+
+ // Check if this dependent node is now dirty. Also checks for new cycles.
+ if (!scan->RecomputeDirty(n, err))
+ return false;
+ if (!n->dirty())
+ continue;
+
+ // This edge was encountered before. However, we may not have wanted to
+ // build it if the outputs were not known to be dirty. With dyndep
+ // information an output is now known to be dirty, so we want the edge.
+ Edge* edge = n->in_edge();
+ assert(edge && !edge->outputs_ready());
+ map<Edge*, Want>::iterator want_e = want_.find(edge);
+ assert(want_e != want_.end());
+ if (want_e->second == kWantNothing) {
+ want_e->second = kWantToStart;
+ EdgeWanted(edge);
+ }
+ }
+ return true;
+}
+
+void Plan::UnmarkDependents(const Node* node, set<Node*>* dependents) {
+ for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
+ oe != node->out_edges().end(); ++oe) {
+ Edge* edge = *oe;
+
+ map<Edge*, Want>::iterator want_e = want_.find(edge);
+ if (want_e == want_.end())
+ continue;
+
+ if (edge->mark_ != Edge::VisitNone) {
+ edge->mark_ = Edge::VisitNone;
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ if (dependents->insert(*o).second)
+ UnmarkDependents(*o, dependents);
+ }
+ }
+ }
+}
+
+void Plan::Dump() const {
+ printf("pending: %d\n", (int)want_.size());
+ for (map<Edge*, Want>::const_iterator e = want_.begin(); e != want_.end(); ++e) {
+ if (e->second != kWantNothing)
+ printf("want ");
+ e->first->Dump();
+ }
+ printf("ready: %d\n", (int)ready_.size());
+}
+
+struct RealCommandRunner : public CommandRunner {
+ explicit RealCommandRunner(const BuildConfig& config);
+ virtual ~RealCommandRunner();
+ virtual bool CanRunMore() const;
+ virtual bool AcquireToken();
+ virtual bool StartCommand(Edge* edge);
+ virtual bool WaitForCommand(Result* result, bool more_ready);
+ virtual vector<Edge*> GetActiveEdges();
+ virtual void Abort();
+
+ const BuildConfig& config_;
+ // copy of config_.max_load_average; can be modified by TokenPool setup
+ double max_load_average_;
+ SubprocessSet subprocs_;
+ TokenPool* tokens_;
+ map<const Subprocess*, Edge*> subproc_to_edge_;
+};
+
+RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
+ max_load_average_ = config.max_load_average;
+ if ((tokens_ = TokenPool::Get()) != NULL) {
+ if (!tokens_->Setup(config_.parallelism_from_cmdline,
+ config_.verbosity == BuildConfig::VERBOSE,
+ max_load_average_)) {
+ delete tokens_;
+ tokens_ = NULL;
+ }
+ }
+}
+
+RealCommandRunner::~RealCommandRunner() {
+ delete tokens_;
+}
+
+vector<Edge*> RealCommandRunner::GetActiveEdges() {
+ vector<Edge*> edges;
+ for (map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin();
+ e != subproc_to_edge_.end(); ++e)
+ edges.push_back(e->second);
+ return edges;
+}
+
+void RealCommandRunner::Abort() {
+ subprocs_.Clear();
+ if (tokens_)
+ tokens_->Clear();
+}
+
+bool RealCommandRunner::CanRunMore() const {
+ bool parallelism_limit_not_reached =
+ tokens_ || // ignore config_.parallelism
+ ((int) (subprocs_.running_.size() +
+ subprocs_.finished_.size()) < config_.parallelism);
+ return parallelism_limit_not_reached
+ && (subprocs_.running_.empty() ||
+ (max_load_average_ <= 0.0f ||
+ GetLoadAverage() < max_load_average_));
+}
+
+bool RealCommandRunner::AcquireToken() {
+ return (!tokens_ || tokens_->Acquire());
+}
+
+bool RealCommandRunner::StartCommand(Edge* edge) {
+ string command = edge->EvaluateCommand();
+ Subprocess* subproc = subprocs_.Add(command, edge->use_console());
+ if (!subproc)
+ return false;
+ if (tokens_)
+ tokens_->Reserve();
+ subproc_to_edge_.insert(make_pair(subproc, edge));
+
+ return true;
+}
+
+bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) {
+ Subprocess* subproc;
+ subprocs_.ResetTokenAvailable();
+ while (((subproc = subprocs_.NextFinished()) == NULL) &&
+ !subprocs_.IsTokenAvailable()) {
+ bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL);
+ if (interrupted)
+ return false;
+ }
+
+ // token became available
+ if (subproc == NULL) {
+ result->status = ExitTokenAvailable;
+ return true;
+ }
+
+ // command completed
+ if (tokens_)
+ tokens_->Release();
+
+ result->status = subproc->Finish();
+ result->output = subproc->GetOutput();
+
+ map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.find(subproc);
+ result->edge = e->second;
+ subproc_to_edge_.erase(e);
+
+ delete subproc;
+ return true;
+}
+
+Builder::Builder(State* state, const BuildConfig& config,
+ BuildLog* build_log, DepsLog* deps_log,
+ DiskInterface* disk_interface)
+ : state_(state), config_(config),
+ plan_(this), disk_interface_(disk_interface),
+ scan_(state, build_log, deps_log, disk_interface,
+ &config_.depfile_parser_options) {
+ status_ = new BuildStatus(config);
+}
+
+Builder::~Builder() {
+ Cleanup();
+}
+
+void Builder::Cleanup() {
+ if (command_runner_.get()) {
+ vector<Edge*> active_edges = command_runner_->GetActiveEdges();
+ command_runner_->Abort();
+
+ for (vector<Edge*>::iterator e = active_edges.begin();
+ e != active_edges.end(); ++e) {
+ string depfile = (*e)->GetUnescapedDepfile();
+ for (vector<Node*>::iterator o = (*e)->outputs_.begin();
+ o != (*e)->outputs_.end(); ++o) {
+ // Only delete this output if it was actually modified. This is
+ // important for things like the generator where we don't want to
+ // delete the manifest file if we can avoid it. But if the rule
+ // uses a depfile, always delete. (Consider the case where we
+ // need to rebuild an output because of a modified header file
+ // mentioned in a depfile, and the command touches its depfile
+ // but is interrupted before it touches its output file.)
+ string err;
+ TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err);
+ if (new_mtime == -1) // Log and ignore Stat() errors.
+ Error("%s", err.c_str());
+ if (!depfile.empty() || (*o)->mtime() != new_mtime)
+ disk_interface_->RemoveFile((*o)->path());
+ }
+ if (!depfile.empty())
+ disk_interface_->RemoveFile(depfile);
+ }
+ }
+}
+
+Node* Builder::AddTarget(const string& name, string* err) {
+ Node* node = state_->LookupNode(name);
+ if (!node) {
+ *err = "unknown target: '" + name + "'";
+ return NULL;
+ }
+ if (!AddTarget(node, err))
+ return NULL;
+ return node;
+}
+
+bool Builder::AddTarget(Node* node, string* err) {
+ if (!scan_.RecomputeDirty(node, err))
+ return false;
+
+ if (Edge* in_edge = node->in_edge()) {
+ if (in_edge->outputs_ready())
+ return true; // Nothing to do.
+ }
+
+ if (!plan_.AddTarget(node, err))
+ return false;
+
+ return true;
+}
+
+bool Builder::AlreadyUpToDate() const {
+ return !plan_.more_to_do();
+}
+
+bool Builder::Build(string* err) {
+ assert(!AlreadyUpToDate());
+
+ status_->PlanHasTotalEdges(plan_.command_edge_count());
+ int pending_commands = 0;
+ int failures_allowed = config_.failures_allowed;
+
+ // Set up the command runner if we haven't done so already.
+ if (!command_runner_.get()) {
+ if (config_.dry_run)
+ command_runner_.reset(new DryRunCommandRunner);
+ else
+ command_runner_.reset(new RealCommandRunner(config_));
+ }
+
+ // We are about to start the build process.
+ status_->BuildStarted();
+
+ // This main loop runs the entire build process.
+ // It is structured like this:
+ // First, we attempt to start as many commands as allowed by the
+ // command runner.
+ // Second, we attempt to wait for / reap the next finished command.
+ while (plan_.more_to_do()) {
+ // See if we can start any more commands...
+ bool can_run_more =
+ failures_allowed &&
+ plan_.more_ready() &&
+ command_runner_->CanRunMore();
+
+ // ... but we also need a token to do that.
+ if (can_run_more && command_runner_->AcquireToken()) {
+ Edge* edge = plan_.FindWork();
+ if (edge->GetBindingBool("generator")) {
+ scan_.build_log()->Close();
+ }
+
+ if (!StartEdge(edge, err)) {
+ Cleanup();
+ status_->BuildFinished();
+ return false;
+ }
+
+ if (edge->is_phony()) {
+ if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) {
+ Cleanup();
+ status_->BuildFinished();
+ return false;
+ }
+ } else {
+ ++pending_commands;
+ }
+
+ // We made some progress; go back to the main loop.
+ continue;
+ }
+
+ // See if we can reap any finished commands.
+ if (pending_commands) {
+ CommandRunner::Result result;
+ if (!command_runner_->WaitForCommand(&result, can_run_more) ||
+ result.status == ExitInterrupted) {
+ Cleanup();
+ status_->BuildFinished();
+ *err = "interrupted by user";
+ return false;
+ }
+
+ // We might be able to start another command; start the main loop over.
+ if (result.status == ExitTokenAvailable)
+ continue;
+
+ --pending_commands;
+ if (!FinishCommand(&result, err)) {
+ Cleanup();
+ status_->BuildFinished();
+ return false;
+ }
+
+ if (!result.success()) {
+ if (failures_allowed)
+ failures_allowed--;
+ }
+
+ // We made some progress; start the main loop over.
+ continue;
+ }
+
+ // If we get here, we cannot make any more progress.
+ status_->BuildFinished();
+ if (failures_allowed == 0) {
+ if (config_.failures_allowed > 1)
+ *err = "subcommands failed";
+ else
+ *err = "subcommand failed";
+ } else if (failures_allowed < config_.failures_allowed)
+ *err = "cannot make progress due to previous errors";
+ else
+ *err = "stuck [this is a bug]";
+
+ return false;
+ }
+
+ status_->BuildFinished();
+ return true;
+}
+
+bool Builder::StartEdge(Edge* edge, string* err) {
+ METRIC_RECORD("StartEdge");
+ if (edge->is_phony())
+ return true;
+
+ status_->BuildEdgeStarted(edge);
+
+ // Create directories necessary for outputs.
+ // XXX: this will block; do we care?
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ if (!disk_interface_->MakeDirs((*o)->path()))
+ return false;
+ }
+
+ // Create response file, if needed
+ // XXX: this may also block; do we care?
+ string rspfile = edge->GetUnescapedRspfile();
+ if (!rspfile.empty()) {
+ string content = edge->GetBinding("rspfile_content");
+ if (!disk_interface_->WriteFile(rspfile, content))
+ return false;
+ }
+
+ // start command computing and run it
+ if (!command_runner_->StartCommand(edge)) {
+ err->assign("command '" + edge->EvaluateCommand() + "' failed.");
+ return false;
+ }
+
+ return true;
+}
+
+bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
+ METRIC_RECORD("FinishCommand");
+
+ Edge* edge = result->edge;
+
+ // First try to extract dependencies from the result, if any.
+ // This must happen first as it filters the command output (we want
+ // to filter /showIncludes output, even on compile failure) and
+ // extraction itself can fail, which makes the command fail from a
+ // build perspective.
+ vector<Node*> deps_nodes;
+ string deps_type = edge->GetBinding("deps");
+ const string deps_prefix = edge->GetBinding("msvc_deps_prefix");
+ if (!deps_type.empty()) {
+ string extract_err;
+ if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
+ &extract_err) &&
+ result->success()) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append(extract_err);
+ result->status = ExitFailure;
+ }
+ }
+
+ int start_time, end_time;
+ status_->BuildEdgeFinished(edge, result->success(), result->output,
+ &start_time, &end_time);
+
+ // The rest of this function only applies to successful commands.
+ if (!result->success()) {
+ return plan_.EdgeFinished(edge, Plan::kEdgeFailed, err);
+ }
+
+ // Restat the edge outputs
+ TimeStamp output_mtime = 0;
+ bool restat = edge->GetBindingBool("restat");
+ if (!config_.dry_run) {
+ bool node_cleaned = false;
+
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err);
+ if (new_mtime == -1)
+ return false;
+ if (new_mtime > output_mtime)
+ output_mtime = new_mtime;
+ if ((*o)->mtime() == new_mtime && restat) {
+ // The rule command did not change the output. Propagate the clean
+ // state through the build graph.
+ // Note that this also applies to nonexistent outputs (mtime == 0).
+ if (!plan_.CleanNode(&scan_, *o, err))
+ return false;
+ node_cleaned = true;
+ }
+ }
+
+ if (node_cleaned) {
+ TimeStamp restat_mtime = 0;
+ // If any output was cleaned, find the most recent mtime of any
+ // (existing) non-order-only input or the depfile.
+ for (vector<Node*>::iterator i = edge->inputs_.begin();
+ i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
+ TimeStamp input_mtime = disk_interface_->Stat((*i)->path(), err);
+ if (input_mtime == -1)
+ return false;
+ if (input_mtime > restat_mtime)
+ restat_mtime = input_mtime;
+ }
+
+ string depfile = edge->GetUnescapedDepfile();
+ if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) {
+ TimeStamp depfile_mtime = disk_interface_->Stat(depfile, err);
+ if (depfile_mtime == -1)
+ return false;
+ if (depfile_mtime > restat_mtime)
+ restat_mtime = depfile_mtime;
+ }
+
+ // The total number of edges in the plan may have changed as a result
+ // of a restat.
+ status_->PlanHasTotalEdges(plan_.command_edge_count());
+
+ output_mtime = restat_mtime;
+ }
+ }
+
+ if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err))
+ return false;
+
+ // Delete any left over response file.
+ string rspfile = edge->GetUnescapedRspfile();
+ if (!rspfile.empty() && !g_keep_rsp)
+ disk_interface_->RemoveFile(rspfile);
+
+ if (scan_.build_log()) {
+ if (!scan_.build_log()->RecordCommand(edge, start_time, end_time,
+ output_mtime)) {
+ *err = string("Error writing to build log: ") + strerror(errno);
+ return false;
+ }
+ }
+
+ if (!deps_type.empty() && !config_.dry_run) {
+ assert(!edge->outputs_.empty() && "should have been rejected by parser");
+ for (std::vector<Node*>::const_iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err);
+ if (deps_mtime == -1)
+ return false;
+ if (!scan_.deps_log()->RecordDeps(*o, deps_mtime, deps_nodes)) {
+ *err = std::string("Error writing to deps log: ") + strerror(errno);
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+bool Builder::ExtractDeps(CommandRunner::Result* result,
+ const string& deps_type,
+ const string& deps_prefix,
+ vector<Node*>* deps_nodes,
+ string* err) {
+ if (deps_type == "msvc") {
+ CLParser parser;
+ string output;
+ if (!parser.Parse(result->output, deps_prefix, &output, err))
+ return false;
+ result->output = output;
+ for (set<string>::iterator i = parser.includes_.begin();
+ i != parser.includes_.end(); ++i) {
+ // ~0 is assuming that with MSVC-parsed headers, it's ok to always make
+ // all backslashes (as some of the slashes will certainly be backslashes
+ // anyway). This could be fixed if necessary with some additional
+ // complexity in IncludesNormalize::Relativize.
+ deps_nodes->push_back(state_->GetNode(*i, ~0u));
+ }
+ } else if (deps_type == "gcc") {
+ string depfile = result->edge->GetUnescapedDepfile();
+ if (depfile.empty()) {
+ *err = string("edge with deps=gcc but no depfile makes no sense");
+ return false;
+ }
+
+ // Read depfile content. Treat a missing depfile as empty.
+ string content;
+ switch (disk_interface_->ReadFile(depfile, &content, err)) {
+ case DiskInterface::Okay:
+ break;
+ case DiskInterface::NotFound:
+ err->clear();
+ break;
+ case DiskInterface::OtherError:
+ return false;
+ }
+ if (content.empty())
+ return true;
+
+ DepfileParser deps(config_.depfile_parser_options);
+ if (!deps.Parse(&content, err))
+ return false;
+
+ // XXX check depfile matches expected output.
+ deps_nodes->reserve(deps.ins_.size());
+ for (vector<StringPiece>::iterator i = deps.ins_.begin();
+ i != deps.ins_.end(); ++i) {
+ uint64_t slash_bits;
+ if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits,
+ err))
+ return false;
+ deps_nodes->push_back(state_->GetNode(*i, slash_bits));
+ }
+
+ if (!g_keep_depfile) {
+ if (disk_interface_->RemoveFile(depfile) < 0) {
+ *err = string("deleting depfile: ") + strerror(errno) + string("\n");
+ return false;
+ }
+ }
+ } else {
+ Fatal("unknown deps type '%s'", deps_type.c_str());
+ }
+
+ return true;
+}
+
+bool Builder::LoadDyndeps(Node* node, string* err) {
+ status_->BuildLoadDyndeps();
+
+ // Load the dyndep information provided by this node.
+ DyndepFile ddf;
+ if (!scan_.LoadDyndeps(node, &ddf, err))
+ return false;
+
+ // Update the build plan to account for dyndep modifications to the graph.
+ if (!plan_.DyndepsLoaded(&scan_, node, ddf, err))
+ return false;
+
+ // New command edges may have been added to the plan.
+ status_->PlanHasTotalEdges(plan_.command_edge_count());
+
+ return true;
+}
Index: src/build.h
===================================================================
--- src/build.h (nonexistent)
+++ src/build.h (revision 5)
@@ -0,0 +1,346 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_BUILD_H_
+#define NINJA_BUILD_H_
+
+#include <cstdio>
+#include <map>
+#include <memory>
+#include <queue>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "depfile_parser.h"
+#include "graph.h" // XXX needed for DependencyScan; should rearrange.
+#include "exit_status.h"
+#include "line_printer.h"
+#include "metrics.h"
+#include "util.h" // int64_t
+
+struct BuildLog;
+struct BuildStatus;
+struct Builder;
+struct DiskInterface;
+struct Edge;
+struct Node;
+struct State;
+
+/// Plan stores the state of a build plan: what we intend to build,
+/// which steps we're ready to execute.
+struct Plan {
+ Plan(Builder* builder = NULL);
+
+ /// Add a target to our plan (including all its dependencies).
+ /// Returns false if we don't need to build this target; may
+ /// fill in |err| with an error message if there's a problem.
+ bool AddTarget(const Node* node, std::string* err);
+
+ // Pop a ready edge off the queue of edges to build.
+ // Returns NULL if there's no work to do.
+ Edge* FindWork();
+
+ /// Returns true if there's more work to be done.
+ bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; }
+
+ /// Returns true if there's more edges ready to start
+ bool more_ready() const { return !ready_.empty(); }
+
+ /// Dumps the current state of the plan.
+ void Dump() const;
+
+ enum EdgeResult {
+ kEdgeFailed,
+ kEdgeSucceeded
+ };
+
+ /// Mark an edge as done building (whether it succeeded or failed).
+ /// If any of the edge's outputs are dyndep bindings of their dependents,
+ /// this loads dynamic dependencies from the nodes' paths.
+ /// Returns 'false' if loading dyndep info fails and 'true' otherwise.
+ bool EdgeFinished(Edge* edge, EdgeResult result, std::string* err);
+
+ /// Clean the given node during the build.
+ /// Return false on error.
+ bool CleanNode(DependencyScan* scan, Node* node, std::string* err);
+
+ /// Number of edges with commands to run.
+ int command_edge_count() const { return command_edges_; }
+
+ /// Reset state. Clears want and ready sets.
+ void Reset();
+
+ /// Update the build plan to account for modifications made to the graph
+ /// by information loaded from a dyndep file.
+ bool DyndepsLoaded(DependencyScan* scan, const Node* node,
+ const DyndepFile& ddf, std::string* err);
+private:
+ bool RefreshDyndepDependents(DependencyScan* scan, const Node* node, std::string* err);
+ void UnmarkDependents(const Node* node, std::set<Node*>* dependents);
+ bool AddSubTarget(const Node* node, const Node* dependent, std::string* err,
+ std::set<Edge*>* dyndep_walk);
+
+ /// Update plan with knowledge that the given node is up to date.
+ /// If the node is a dyndep binding on any of its dependents, this
+ /// loads dynamic dependencies from the node's path.
+ /// Returns 'false' if loading dyndep info fails and 'true' otherwise.
+ bool NodeFinished(Node* node, std::string* err);
+
+ /// Enumerate possible steps we want for an edge.
+ enum Want
+ {
+ /// We do not want to build the edge, but we might want to build one of
+ /// its dependents.
+ kWantNothing,
+ /// We want to build the edge, but have not yet scheduled it.
+ kWantToStart,
+ /// We want to build the edge, have scheduled it, and are waiting
+ /// for it to complete.
+ kWantToFinish
+ };
+
+ void EdgeWanted(const Edge* edge);
+ bool EdgeMaybeReady(std::map<Edge*, Want>::iterator want_e, std::string* err);
+
+ /// Submits a ready edge as a candidate for execution.
+ /// The edge may be delayed from running, for example if it's a member of a
+ /// currently-full pool.
+ void ScheduleWork(std::map<Edge*, Want>::iterator want_e);
+
+ /// Keep track of which edges we want to build in this plan. If this map does
+ /// not contain an entry for an edge, we do not want to build the entry or its
+ /// dependents. If it does contain an entry, the enumeration indicates what
+ /// we want for the edge.
+ std::map<Edge*, Want> want_;
+
+ std::set<Edge*> ready_;
+
+ Builder* builder_;
+
+ /// Total number of edges that have commands (not phony).
+ int command_edges_;
+
+ /// Total remaining number of wanted edges.
+ int wanted_edges_;
+};
+
+/// CommandRunner is an interface that wraps running the build
+/// subcommands. This allows tests to abstract out running commands.
+/// RealCommandRunner is an implementation that actually runs commands.
+struct CommandRunner {
+ virtual ~CommandRunner() {}
+ virtual bool CanRunMore() const = 0;
+ virtual bool AcquireToken() = 0;
+ virtual bool StartCommand(Edge* edge) = 0;
+
+ /// The result of waiting for a command.
+ struct Result {
+ Result() : edge(NULL) {}
+ Edge* edge;
+ ExitStatus status;
+ std::string output;
+ bool success() const { return status == ExitSuccess; }
+ };
+ /// Wait for a command to complete, or return false if interrupted.
+ /// If more_ready is true then the optional TokenPool is monitored too
+ /// and we return when a token becomes available.
+ virtual bool WaitForCommand(Result* result, bool more_ready) = 0;
+
+ virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
+ virtual void Abort() {}
+};
+
+/// Options (e.g. verbosity, parallelism) passed to a build.
+struct BuildConfig {
+ BuildConfig() : verbosity(NORMAL), dry_run(false),
+ parallelism(1), parallelism_from_cmdline(false),
+ failures_allowed(1), max_load_average(-0.0f) {}
+
+ enum Verbosity {
+ NORMAL,
+ QUIET, // No output -- used when testing.
+ VERBOSE
+ };
+ Verbosity verbosity;
+ bool dry_run;
+ int parallelism;
+ bool parallelism_from_cmdline;
+ int failures_allowed;
+ /// The maximum load average we must not exceed. A negative value
+ /// means that we do not have any limit.
+ double max_load_average;
+ DepfileParserOptions depfile_parser_options;
+};
+
+/// Builder wraps the build process: starting commands, updating status.
+struct Builder {
+ Builder(State* state, const BuildConfig& config,
+ BuildLog* build_log, DepsLog* deps_log,
+ DiskInterface* disk_interface);
+ ~Builder();
+
+ /// Clean up after interrupted commands by deleting output files.
+ void Cleanup();
+
+ Node* AddTarget(const std::string& name, std::string* err);
+
+ /// Add a target to the build, scanning dependencies.
+ /// @return false on error.
+ bool AddTarget(Node* target, std::string* err);
+
+ /// Returns true if the build targets are already up to date.
+ bool AlreadyUpToDate() const;
+
+ /// Run the build. Returns false on error.
+ /// It is an error to call this function when AlreadyUpToDate() is true.
+ bool Build(std::string* err);
+
+ bool StartEdge(Edge* edge, std::string* err);
+
+ /// Update status ninja logs following a command termination.
+ /// @return false if the build can not proceed further due to a fatal error.
+ bool FinishCommand(CommandRunner::Result* result, std::string* err);
+
+ /// Used for tests.
+ void SetBuildLog(BuildLog* log) {
+ scan_.set_build_log(log);
+ }
+
+ /// Load the dyndep information provided by the given node.
+ bool LoadDyndeps(Node* node, std::string* err);
+
+ State* state_;
+ const BuildConfig& config_;
+ Plan plan_;
+#if __cplusplus < 201703L
+ std::auto_ptr<CommandRunner> command_runner_;
+#else
+ std::unique_ptr<CommandRunner> command_runner_; // auto_ptr was removed in C++17.
+#endif
+ BuildStatus* status_;
+
+ private:
+ bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type,
+ const std::string& deps_prefix,
+ std::vector<Node*>* deps_nodes, std::string* err);
+
+ DiskInterface* disk_interface_;
+ DependencyScan scan_;
+
+ // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr.
+ Builder(const Builder &other); // DO NOT IMPLEMENT
+ void operator=(const Builder &other); // DO NOT IMPLEMENT
+};
+
+/// Tracks the status of a build: completion fraction, printing updates.
+struct BuildStatus {
+ explicit BuildStatus(const BuildConfig& config);
+ void PlanHasTotalEdges(int total);
+ void BuildEdgeStarted(const Edge* edge);
+ void BuildEdgeFinished(Edge* edge, bool success, const std::string& output,
+ int* start_time, int* end_time);
+ void BuildLoadDyndeps();
+ void BuildStarted();
+ void BuildFinished();
+
+ enum EdgeStatus {
+ kEdgeStarted,
+ kEdgeFinished,
+ };
+
+ /// Format the progress status string by replacing the placeholders.
+ /// See the user manual for more information about the available
+ /// placeholders.
+ /// @param progress_status_format The format of the progress status.
+ /// @param status The status of the edge.
+ std::string FormatProgressStatus(const char* progress_status_format,
+ EdgeStatus status) const;
+
+ private:
+ void PrintStatus(const Edge* edge, EdgeStatus status);
+
+ const BuildConfig& config_;
+
+ /// Time the build started.
+ int64_t start_time_millis_;
+
+ int started_edges_, finished_edges_, total_edges_;
+
+ /// Map of running edge to time the edge started running.
+ typedef std::map<const Edge*, int> RunningEdgeMap;
+ RunningEdgeMap running_edges_;
+
+ /// Prints progress output.
+ LinePrinter printer_;
+
+ /// The custom progress status format to use.
+ const char* progress_status_format_;
+
+ template<size_t S>
+ void SnprintfRate(double rate, char(&buf)[S], const char* format) const {
+ if (rate == -1)
+ snprintf(buf, S, "?");
+ else
+ snprintf(buf, S, format, rate);
+ }
+
+ struct RateInfo {
+ RateInfo() : rate_(-1) {}
+
+ void Restart() { stopwatch_.Restart(); }
+ double Elapsed() const { return stopwatch_.Elapsed(); }
+ double rate() { return rate_; }
+
+ void UpdateRate(int edges) {
+ if (edges && stopwatch_.Elapsed())
+ rate_ = edges / stopwatch_.Elapsed();
+ }
+
+ private:
+ double rate_;
+ Stopwatch stopwatch_;
+ };
+
+ struct SlidingRateInfo {
+ SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {}
+
+ void Restart() { stopwatch_.Restart(); }
+ double rate() { return rate_; }
+
+ void UpdateRate(int update_hint) {
+ if (update_hint == last_update_)
+ return;
+ last_update_ = update_hint;
+
+ if (times_.size() == N)
+ times_.pop();
+ times_.push(stopwatch_.Elapsed());
+ if (times_.back() != times_.front())
+ rate_ = times_.size() / (times_.back() - times_.front());
+ }
+
+ private:
+ double rate_;
+ Stopwatch stopwatch_;
+ const size_t N;
+ std::queue<double> times_;
+ int last_update_;
+ };
+
+ mutable RateInfo overall_rate_;
+ mutable SlidingRateInfo current_rate_;
+};
+
+#endif // NINJA_BUILD_H_
Index: src/build_test.cc
===================================================================
--- src/build_test.cc (nonexistent)
+++ src/build_test.cc (revision 5)
@@ -0,0 +1,3663 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "build.h"
+
+#include <assert.h>
+#include <stdarg.h>
+
+#include "build_log.h"
+#include "deps_log.h"
+#include "graph.h"
+#include "test.h"
+
+using namespace std;
+
+struct CompareEdgesByOutput {
+ static bool cmp(const Edge* a, const Edge* b) {
+ return a->outputs_[0]->path() < b->outputs_[0]->path();
+ }
+};
+
+/// Fixture for tests involving Plan.
+// Though Plan doesn't use State, it's useful to have one around
+// to create Nodes and Edges.
+struct PlanTest : public StateTestWithBuiltinRules {
+ Plan plan_;
+
+ /// Because FindWork does not return Edges in any sort of predictable order,
+ // provide a means to get available Edges in order and in a format which is
+ // easy to write tests around.
+ void FindWorkSorted(deque<Edge*>* ret, int count) {
+ for (int i = 0; i < count; ++i) {
+ ASSERT_TRUE(plan_.more_to_do());
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ret->push_back(edge);
+ }
+ ASSERT_FALSE(plan_.FindWork());
+ sort(ret->begin(), ret->end(), CompareEdgesByOutput::cmp);
+ }
+
+ void TestPoolWithDepthOne(const char *test_case);
+};
+
+TEST_F(PlanTest, Basic) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat mid\n"
+"build mid: cat in\n"));
+ GetNode("mid")->MarkDirty();
+ GetNode("out")->MarkDirty();
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("mid", edge->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("mid", edge->inputs_[0]->path());
+ ASSERT_EQ("out", edge->outputs_[0]->path());
+
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ ASSERT_FALSE(plan_.more_to_do());
+ edge = plan_.FindWork();
+ ASSERT_EQ(0, edge);
+}
+
+// Test that two outputs from one rule can be handled as inputs to the next.
+TEST_F(PlanTest, DoubleOutputDirect) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat mid1 mid2\n"
+"build mid1 mid2: cat in\n"));
+ GetNode("mid1")->MarkDirty();
+ GetNode("mid2")->MarkDirty();
+ GetNode("out")->MarkDirty();
+
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge;
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat in
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat mid1 mid2
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_FALSE(edge); // done
+}
+
+// Test that two outputs from one rule can eventually be routed to another.
+TEST_F(PlanTest, DoubleOutputIndirect) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat b1 b2\n"
+"build b1: cat a1\n"
+"build b2: cat a2\n"
+"build a1 a2: cat in\n"));
+ GetNode("a1")->MarkDirty();
+ GetNode("a2")->MarkDirty();
+ GetNode("b1")->MarkDirty();
+ GetNode("b2")->MarkDirty();
+ GetNode("out")->MarkDirty();
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge;
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat in
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat a1
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat a2
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat b1 b2
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_FALSE(edge); // done
+}
+
+// Test that two edges from one output can both execute.
+TEST_F(PlanTest, DoubleDependent) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat a1 a2\n"
+"build a1: cat mid\n"
+"build a2: cat mid\n"
+"build mid: cat in\n"));
+ GetNode("mid")->MarkDirty();
+ GetNode("a1")->MarkDirty();
+ GetNode("a2")->MarkDirty();
+ GetNode("out")->MarkDirty();
+
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge;
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat in
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat mid
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat mid
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge); // cat a1 a2
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_FALSE(edge); // done
+}
+
+void PlanTest::TestPoolWithDepthOne(const char* test_case) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, test_case));
+ GetNode("out1")->MarkDirty();
+ GetNode("out2")->MarkDirty();
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out1"), &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out2"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out1", edge->outputs_[0]->path());
+
+ // This will be false since poolcat is serialized
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out2", edge->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ ASSERT_FALSE(plan_.more_to_do());
+ edge = plan_.FindWork();
+ ASSERT_EQ(0, edge);
+}
+
+TEST_F(PlanTest, PoolWithDepthOne) {
+ TestPoolWithDepthOne(
+"pool foobar\n"
+" depth = 1\n"
+"rule poolcat\n"
+" command = cat $in > $out\n"
+" pool = foobar\n"
+"build out1: poolcat in\n"
+"build out2: poolcat in\n");
+}
+
+TEST_F(PlanTest, ConsolePool) {
+ TestPoolWithDepthOne(
+"rule poolcat\n"
+" command = cat $in > $out\n"
+" pool = console\n"
+"build out1: poolcat in\n"
+"build out2: poolcat in\n");
+}
+
+TEST_F(PlanTest, PoolsWithDepthTwo) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"pool foobar\n"
+" depth = 2\n"
+"pool bazbin\n"
+" depth = 2\n"
+"rule foocat\n"
+" command = cat $in > $out\n"
+" pool = foobar\n"
+"rule bazcat\n"
+" command = cat $in > $out\n"
+" pool = bazbin\n"
+"build out1: foocat in\n"
+"build out2: foocat in\n"
+"build out3: foocat in\n"
+"build outb1: bazcat in\n"
+"build outb2: bazcat in\n"
+"build outb3: bazcat in\n"
+" pool =\n"
+"build allTheThings: cat out1 out2 out3 outb1 outb2 outb3\n"
+));
+ // Mark all the out* nodes dirty
+ for (int i = 0; i < 3; ++i) {
+ GetNode("out" + string(1, '1' + static_cast<char>(i)))->MarkDirty();
+ GetNode("outb" + string(1, '1' + static_cast<char>(i)))->MarkDirty();
+ }
+ GetNode("allTheThings")->MarkDirty();
+
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("allTheThings"), &err));
+ ASSERT_EQ("", err);
+
+ deque<Edge*> edges;
+ FindWorkSorted(&edges, 5);
+
+ for (int i = 0; i < 4; ++i) {
+ Edge *edge = edges[i];
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ string base_name(i < 2 ? "out" : "outb");
+ ASSERT_EQ(base_name + string(1, '1' + (i % 2)), edge->outputs_[0]->path());
+ }
+
+ // outb3 is exempt because it has an empty pool
+ Edge* edge = edges[4];
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("outb3", edge->outputs_[0]->path());
+
+ // finish out1
+ plan_.EdgeFinished(edges.front(), Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+ edges.pop_front();
+
+ // out3 should be available
+ Edge* out3 = plan_.FindWork();
+ ASSERT_TRUE(out3);
+ ASSERT_EQ("in", out3->inputs_[0]->path());
+ ASSERT_EQ("out3", out3->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(out3, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ for (deque<Edge*>::iterator it = edges.begin(); it != edges.end(); ++it) {
+ plan_.EdgeFinished(*it, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+ }
+
+ Edge* last = plan_.FindWork();
+ ASSERT_TRUE(last);
+ ASSERT_EQ("allTheThings", last->outputs_[0]->path());
+
+ plan_.EdgeFinished(last, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ ASSERT_FALSE(plan_.more_to_do());
+ ASSERT_FALSE(plan_.FindWork());
+}
+
+TEST_F(PlanTest, PoolWithRedundantEdges) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "pool compile\n"
+ " depth = 1\n"
+ "rule gen_foo\n"
+ " command = touch foo.cpp\n"
+ "rule gen_bar\n"
+ " command = touch bar.cpp\n"
+ "rule echo\n"
+ " command = echo $out > $out\n"
+ "build foo.cpp.obj: echo foo.cpp || foo.cpp\n"
+ " pool = compile\n"
+ "build bar.cpp.obj: echo bar.cpp || bar.cpp\n"
+ " pool = compile\n"
+ "build libfoo.a: echo foo.cpp.obj bar.cpp.obj\n"
+ "build foo.cpp: gen_foo\n"
+ "build bar.cpp: gen_bar\n"
+ "build all: phony libfoo.a\n"));
+ GetNode("foo.cpp")->MarkDirty();
+ GetNode("foo.cpp.obj")->MarkDirty();
+ GetNode("bar.cpp")->MarkDirty();
+ GetNode("bar.cpp.obj")->MarkDirty();
+ GetNode("libfoo.a")->MarkDirty();
+ GetNode("all")->MarkDirty();
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("all"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge = NULL;
+
+ deque<Edge*> initial_edges;
+ FindWorkSorted(&initial_edges, 2);
+
+ edge = initial_edges[1]; // Foo first
+ ASSERT_EQ("foo.cpp", edge->outputs_[0]->path());
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
+ ASSERT_EQ("foo.cpp", edge->inputs_[0]->path());
+ ASSERT_EQ("foo.cpp", edge->inputs_[1]->path());
+ ASSERT_EQ("foo.cpp.obj", edge->outputs_[0]->path());
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = initial_edges[0]; // Now for bar
+ ASSERT_EQ("bar.cpp", edge->outputs_[0]->path());
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
+ ASSERT_EQ("bar.cpp", edge->inputs_[0]->path());
+ ASSERT_EQ("bar.cpp", edge->inputs_[1]->path());
+ ASSERT_EQ("bar.cpp.obj", edge->outputs_[0]->path());
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
+ ASSERT_EQ("foo.cpp.obj", edge->inputs_[0]->path());
+ ASSERT_EQ("bar.cpp.obj", edge->inputs_[1]->path());
+ ASSERT_EQ("libfoo.a", edge->outputs_[0]->path());
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_FALSE(plan_.FindWork());
+ ASSERT_EQ("libfoo.a", edge->inputs_[0]->path());
+ ASSERT_EQ("all", edge->outputs_[0]->path());
+ plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_FALSE(edge);
+ ASSERT_FALSE(plan_.more_to_do());
+}
+
+TEST_F(PlanTest, PoolWithFailingEdge) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "pool foobar\n"
+ " depth = 1\n"
+ "rule poolcat\n"
+ " command = cat $in > $out\n"
+ " pool = foobar\n"
+ "build out1: poolcat in\n"
+ "build out2: poolcat in\n"));
+ GetNode("out1")->MarkDirty();
+ GetNode("out2")->MarkDirty();
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out1"), &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out2"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out1", edge->outputs_[0]->path());
+
+ // This will be false since poolcat is serialized
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge, Plan::kEdgeFailed, &err);
+ ASSERT_EQ("", err);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out2", edge->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge, Plan::kEdgeFailed, &err);
+ ASSERT_EQ("", err);
+
+ ASSERT_TRUE(plan_.more_to_do()); // Jobs have failed
+ edge = plan_.FindWork();
+ ASSERT_EQ(0, edge);
+}
+
+/// Fake implementation of CommandRunner, useful for tests.
+struct FakeCommandRunner : public CommandRunner {
+ explicit FakeCommandRunner(VirtualFileSystem* fs) :
+ max_active_edges_(1), fs_(fs) {}
+
+ // CommandRunner impl
+ virtual bool CanRunMore() const;
+ virtual bool AcquireToken();
+ virtual bool StartCommand(Edge* edge);
+ virtual bool WaitForCommand(Result* result, bool more_ready);
+ virtual vector<Edge*> GetActiveEdges();
+ virtual void Abort();
+
+ vector<string> commands_ran_;
+ vector<Edge*> active_edges_;
+ size_t max_active_edges_;
+ VirtualFileSystem* fs_;
+};
+
+struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser {
+ BuildTest() : config_(MakeConfig()), command_runner_(&fs_),
+ builder_(&state_, config_, NULL, NULL, &fs_),
+ status_(config_) {
+ }
+
+ BuildTest(DepsLog* log) : config_(MakeConfig()), command_runner_(&fs_),
+ builder_(&state_, config_, NULL, log, &fs_),
+ status_(config_) {
+ }
+
+ virtual void SetUp() {
+ StateTestWithBuiltinRules::SetUp();
+
+ builder_.command_runner_.reset(&command_runner_);
+ AssertParse(&state_,
+"build cat1: cat in1\n"
+"build cat2: cat in1 in2\n"
+"build cat12: cat cat1 cat2\n");
+
+ fs_.Create("in1", "");
+ fs_.Create("in2", "");
+ }
+
+ ~BuildTest() {
+ builder_.command_runner_.release();
+ }
+
+ virtual bool IsPathDead(StringPiece s) const { return false; }
+
+ /// Rebuild target in the 'working tree' (fs_).
+ /// State of command_runner_ and logs contents (if specified) ARE MODIFIED.
+ /// Handy to check for NOOP builds, and higher-level rebuild tests.
+ void RebuildTarget(const string& target, const char* manifest,
+ const char* log_path = NULL, const char* deps_path = NULL,
+ State* state = NULL);
+
+ // Mark a path dirty.
+ void Dirty(const string& path);
+
+ BuildConfig MakeConfig() {
+ BuildConfig config;
+ config.verbosity = BuildConfig::QUIET;
+ return config;
+ }
+
+ BuildConfig config_;
+ FakeCommandRunner command_runner_;
+ VirtualFileSystem fs_;
+ Builder builder_;
+
+ BuildStatus status_;
+};
+
+void BuildTest::RebuildTarget(const string& target, const char* manifest,
+ const char* log_path, const char* deps_path,
+ State* state) {
+ State local_state, *pstate = &local_state;
+ if (state)
+ pstate = state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(pstate));
+ AssertParse(pstate, manifest);
+
+ string err;
+ BuildLog build_log, *pbuild_log = NULL;
+ if (log_path) {
+ ASSERT_TRUE(build_log.Load(log_path, &err));
+ ASSERT_TRUE(build_log.OpenForWrite(log_path, *this, &err));
+ ASSERT_EQ("", err);
+ pbuild_log = &build_log;
+ }
+
+ DepsLog deps_log, *pdeps_log = NULL;
+ if (deps_path) {
+ ASSERT_TRUE(deps_log.Load(deps_path, pstate, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite(deps_path, &err));
+ ASSERT_EQ("", err);
+ pdeps_log = &deps_log;
+ }
+
+ Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_);
+ EXPECT_TRUE(builder.AddTarget(target, &err));
+
+ command_runner_.commands_ran_.clear();
+ builder.command_runner_.reset(&command_runner_);
+ if (!builder.AlreadyUpToDate()) {
+ bool build_res = builder.Build(&err);
+ EXPECT_TRUE(build_res);
+ }
+ builder.command_runner_.release();
+}
+
+bool FakeCommandRunner::CanRunMore() const {
+ return active_edges_.size() < max_active_edges_;
+}
+
+bool FakeCommandRunner::AcquireToken() {
+ return true;
+}
+
+bool FakeCommandRunner::StartCommand(Edge* edge) {
+ assert(active_edges_.size() < max_active_edges_);
+ assert(find(active_edges_.begin(), active_edges_.end(), edge)
+ == active_edges_.end());
+ commands_ran_.push_back(edge->EvaluateCommand());
+ if (edge->rule().name() == "cat" ||
+ edge->rule().name() == "cat_rsp" ||
+ edge->rule().name() == "cat_rsp_out" ||
+ edge->rule().name() == "cc" ||
+ edge->rule().name() == "cp_multi_msvc" ||
+ edge->rule().name() == "cp_multi_gcc" ||
+ edge->rule().name() == "touch" ||
+ edge->rule().name() == "touch-interrupt" ||
+ edge->rule().name() == "touch-fail-tick2") {
+ for (vector<Node*>::iterator out = edge->outputs_.begin();
+ out != edge->outputs_.end(); ++out) {
+ fs_->Create((*out)->path(), "");
+ }
+ } else if (edge->rule().name() == "true" ||
+ edge->rule().name() == "fail" ||
+ edge->rule().name() == "interrupt" ||
+ edge->rule().name() == "console") {
+ // Don't do anything.
+ } else if (edge->rule().name() == "cp") {
+ assert(!edge->inputs_.empty());
+ assert(edge->outputs_.size() == 1);
+ string content;
+ string err;
+ if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) ==
+ DiskInterface::Okay)
+ fs_->WriteFile(edge->outputs_[0]->path(), content);
+ } else {
+ printf("unknown command\n");
+ return false;
+ }
+
+ active_edges_.push_back(edge);
+
+ // Allow tests to control the order by the name of the first output.
+ sort(active_edges_.begin(), active_edges_.end(),
+ CompareEdgesByOutput::cmp);
+
+ return true;
+}
+
+bool FakeCommandRunner::WaitForCommand(Result* result, bool more_ready) {
+ if (active_edges_.empty())
+ return false;
+
+ // All active edges were already completed immediately when started,
+ // so we can pick any edge here. Pick the last edge. Tests can
+ // control the order of edges by the name of the first output.
+ vector<Edge*>::iterator edge_iter = active_edges_.end() - 1;
+
+ Edge* edge = *edge_iter;
+ result->edge = edge;
+
+ if (edge->rule().name() == "interrupt" ||
+ edge->rule().name() == "touch-interrupt") {
+ result->status = ExitInterrupted;
+ return true;
+ }
+
+ if (edge->rule().name() == "console") {
+ if (edge->use_console())
+ result->status = ExitSuccess;
+ else
+ result->status = ExitFailure;
+ active_edges_.erase(edge_iter);
+ return true;
+ }
+
+ if (edge->rule().name() == "cp_multi_msvc") {
+ const std::string prefix = edge->GetBinding("msvc_deps_prefix");
+ for (std::vector<Node*>::iterator in = edge->inputs_.begin();
+ in != edge->inputs_.end(); ++in) {
+ result->output += prefix + (*in)->path() + '\n';
+ }
+ }
+
+ if (edge->rule().name() == "fail" ||
+ (edge->rule().name() == "touch-fail-tick2" && fs_->now_ == 2))
+ result->status = ExitFailure;
+ else
+ result->status = ExitSuccess;
+
+ // Provide a way for test cases to verify when an edge finishes that
+ // some other edge is still active. This is useful for test cases
+ // covering behavior involving multiple active edges.
+ const string& verify_active_edge = edge->GetBinding("verify_active_edge");
+ if (!verify_active_edge.empty()) {
+ bool verify_active_edge_found = false;
+ for (vector<Edge*>::iterator i = active_edges_.begin();
+ i != active_edges_.end(); ++i) {
+ if (!(*i)->outputs_.empty() &&
+ (*i)->outputs_[0]->path() == verify_active_edge) {
+ verify_active_edge_found = true;
+ }
+ }
+ EXPECT_TRUE(verify_active_edge_found);
+ }
+
+ active_edges_.erase(edge_iter);
+ return true;
+}
+
+vector<Edge*> FakeCommandRunner::GetActiveEdges() {
+ return active_edges_;
+}
+
+void FakeCommandRunner::Abort() {
+ active_edges_.clear();
+}
+
+void BuildTest::Dirty(const string& path) {
+ Node* node = GetNode(path);
+ node->MarkDirty();
+
+ // If it's an input file, mark that we've already stat()ed it and
+ // it's missing.
+ if (!node->in_edge())
+ node->MarkMissing();
+}
+
+TEST_F(BuildTest, NoWork) {
+ string err;
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+}
+
+TEST_F(BuildTest, OneStep) {
+ // Given a dirty target with one ready input,
+ // we should rebuild the target.
+ Dirty("cat1");
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat1", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cat in1 > cat1", command_runner_.commands_ran_[0]);
+}
+
+TEST_F(BuildTest, OneStep2) {
+ // Given a target with one dirty input,
+ // we should rebuild the target.
+ Dirty("cat1");
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat1", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cat in1 > cat1", command_runner_.commands_ran_[0]);
+}
+
+TEST_F(BuildTest, TwoStep) {
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat12", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ // Depending on how the pointers work out, we could've ran
+ // the first two commands in either order.
+ EXPECT_TRUE((command_runner_.commands_ran_[0] == "cat in1 > cat1" &&
+ command_runner_.commands_ran_[1] == "cat in1 in2 > cat2") ||
+ (command_runner_.commands_ran_[1] == "cat in1 > cat1" &&
+ command_runner_.commands_ran_[0] == "cat in1 in2 > cat2"));
+
+ EXPECT_EQ("cat cat1 cat2 > cat12", command_runner_.commands_ran_[2]);
+
+ fs_.Tick();
+
+ // Modifying in2 requires rebuilding one intermediate file
+ // and the final file.
+ fs_.Create("in2", "");
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("cat12", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(5u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cat in1 in2 > cat2", command_runner_.commands_ran_[3]);
+ EXPECT_EQ("cat cat1 cat2 > cat12", command_runner_.commands_ran_[4]);
+}
+
+TEST_F(BuildTest, TwoOutputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"build out1 out2: touch in.txt\n"));
+
+ fs_.Create("in.txt", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("touch out1 out2", command_runner_.commands_ran_[0]);
+}
+
+TEST_F(BuildTest, ImplicitOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"build out | out.imp: touch in.txt\n"));
+ fs_.Create("in.txt", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out.imp", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[0]);
+}
+
+// Test case from
+// https://github.com/ninja-build/ninja/issues/148
+TEST_F(BuildTest, MultiOutIn) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"build in1 otherfile: touch in\n"
+"build out: touch in | in1\n"));
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("in1", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+}
+
+TEST_F(BuildTest, Chain) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build c2: cat c1\n"
+"build c3: cat c2\n"
+"build c4: cat c3\n"
+"build c5: cat c4\n"));
+
+ fs_.Create("c1", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("c5", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(4u, command_runner_.commands_ran_.size());
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("c5", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+
+ fs_.Tick();
+
+ fs_.Create("c3", "");
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("c5", &err));
+ ASSERT_EQ("", err);
+ EXPECT_FALSE(builder_.AlreadyUpToDate());
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // 3->4, 4->5
+}
+
+TEST_F(BuildTest, MissingInput) {
+ // Input is referenced by build file, but no rule for it.
+ string err;
+ Dirty("in1");
+ EXPECT_FALSE(builder_.AddTarget("cat1", &err));
+ EXPECT_EQ("'in1', needed by 'cat1', missing and no known rule to make it",
+ err);
+}
+
+TEST_F(BuildTest, MissingTarget) {
+ // Target is not referenced by build file.
+ string err;
+ EXPECT_FALSE(builder_.AddTarget("meow", &err));
+ EXPECT_EQ("unknown target: 'meow'", err);
+}
+
+TEST_F(BuildTest, MakeDirs) {
+ string err;
+
+#ifdef _WIN32
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build subdir\\dir2\\file: cat in1\n"));
+#else
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build subdir/dir2/file: cat in1\n"));
+#endif
+ EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err));
+
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(2u, fs_.directories_made_.size());
+ EXPECT_EQ("subdir", fs_.directories_made_[0]);
+ EXPECT_EQ("subdir/dir2", fs_.directories_made_[1]);
+}
+
+TEST_F(BuildTest, DepFileMissing) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n depfile = $out.d\n"
+"build fo$ o.o: cc foo.c\n"));
+ fs_.Create("foo.c", "");
+
+ EXPECT_TRUE(builder_.AddTarget("fo o.o", &err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, fs_.files_read_.size());
+ EXPECT_EQ("fo o.o.d", fs_.files_read_[0]);
+}
+
+TEST_F(BuildTest, DepFileOK) {
+ string err;
+ int orig_edges = state_.edges_.size();
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n depfile = $out.d\n"
+"build foo.o: cc foo.c\n"));
+ Edge* edge = state_.edges_.back();
+
+ fs_.Create("foo.c", "");
+ GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, fs_.files_read_.size());
+ EXPECT_EQ("foo.o.d", fs_.files_read_[0]);
+
+ // Expect three new edges: one generating foo.o, and two more from
+ // loading the depfile.
+ ASSERT_EQ(orig_edges + 3, (int)state_.edges_.size());
+ // Expect our edge to now have three inputs: foo.c and two headers.
+ ASSERT_EQ(3u, edge->inputs_.size());
+
+ // Expect the command line we generate to only use the original input.
+ ASSERT_EQ("cc foo.c", edge->EvaluateCommand());
+}
+
+TEST_F(BuildTest, DepFileParseError) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n depfile = $out.d\n"
+"build foo.o: cc foo.c\n"));
+ fs_.Create("foo.c", "");
+ fs_.Create("foo.o.d", "randomtext\n");
+ EXPECT_FALSE(builder_.AddTarget("foo.o", &err));
+ EXPECT_EQ("foo.o.d: expected ':' in depfile", err);
+}
+
+TEST_F(BuildTest, EncounterReadyTwice) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"build c: touch\n"
+"build b: touch || c\n"
+"build a: touch | b || c\n"));
+
+ vector<Edge*> c_out = GetNode("c")->out_edges();
+ ASSERT_EQ(2u, c_out.size());
+ EXPECT_EQ("b", c_out[0]->outputs_[0]->path());
+ EXPECT_EQ("a", c_out[1]->outputs_[0]->path());
+
+ fs_.Create("b", "");
+ EXPECT_TRUE(builder_.AddTarget("a", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, OrderOnlyDeps) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n depfile = $out.d\n"
+"build foo.o: cc foo.c || otherfile\n"));
+ Edge* edge = state_.edges_.back();
+
+ fs_.Create("foo.c", "");
+ fs_.Create("otherfile", "");
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ ASSERT_EQ("", err);
+
+ // One explicit, two implicit, one order only.
+ ASSERT_EQ(4u, edge->inputs_.size());
+ EXPECT_EQ(2, edge->implicit_deps_);
+ EXPECT_EQ(1, edge->order_only_deps_);
+ // Verify the inputs are in the order we expect
+ // (explicit then implicit then orderonly).
+ EXPECT_EQ("foo.c", edge->inputs_[0]->path());
+ EXPECT_EQ("blah.h", edge->inputs_[1]->path());
+ EXPECT_EQ("bar.h", edge->inputs_[2]->path());
+ EXPECT_EQ("otherfile", edge->inputs_[3]->path());
+
+ // Expect the command line we generate to only use the original input.
+ ASSERT_EQ("cc foo.c", edge->EvaluateCommand());
+
+ // explicit dep dirty, expect a rebuild.
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+
+ fs_.Tick();
+
+ // Recreate the depfile, as it should have been deleted by the build.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+
+ // implicit dep dirty, expect a rebuild.
+ fs_.Create("blah.h", "");
+ fs_.Create("bar.h", "");
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+
+ fs_.Tick();
+
+ // Recreate the depfile, as it should have been deleted by the build.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+
+ // order only dep dirty, no rebuild.
+ fs_.Create("otherfile", "");
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+
+ // implicit dep missing, expect rebuild.
+ fs_.RemoveFile("bar.h");
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, RebuildOrderOnlyDeps) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n"
+"rule true\n command = true\n"
+"build oo.h: cc oo.h.in\n"
+"build foo.o: cc foo.c || oo.h\n"));
+
+ fs_.Create("foo.c", "");
+ fs_.Create("oo.h.in", "");
+
+ // foo.o and order-only dep dirty, build both.
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // all clean, no rebuild.
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+
+ // order-only dep missing, build it only.
+ fs_.RemoveFile("oo.h");
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]);
+
+ fs_.Tick();
+
+ // order-only dep dirty, build it only.
+ fs_.Create("oo.h.in", "");
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]);
+}
+
+#ifdef _WIN32
+TEST_F(BuildTest, DepFileCanonicalize) {
+ string err;
+ int orig_edges = state_.edges_.size();
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n depfile = $out.d\n"
+"build gen/stuff\\things/foo.o: cc x\\y/z\\foo.c\n"));
+ Edge* edge = state_.edges_.back();
+
+ fs_.Create("x/y/z/foo.c", "");
+ GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing.
+ // Note, different slashes from manifest.
+ fs_.Create("gen/stuff\\things/foo.o.d",
+ "gen\\stuff\\things\\foo.o: blah.h bar.h\n");
+ EXPECT_TRUE(builder_.AddTarget("gen/stuff/things/foo.o", &err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, fs_.files_read_.size());
+ // The depfile path does not get Canonicalize as it seems unnecessary.
+ EXPECT_EQ("gen/stuff\\things/foo.o.d", fs_.files_read_[0]);
+
+ // Expect three new edges: one generating foo.o, and two more from
+ // loading the depfile.
+ ASSERT_EQ(orig_edges + 3, (int)state_.edges_.size());
+ // Expect our edge to now have three inputs: foo.c and two headers.
+ ASSERT_EQ(3u, edge->inputs_.size());
+
+ // Expect the command line we generate to only use the original input, and
+ // using the slashes from the manifest.
+ ASSERT_EQ("cc x\\y/z\\foo.c", edge->EvaluateCommand());
+}
+#endif
+
+TEST_F(BuildTest, Phony) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat bar.cc\n"
+"build all: phony out\n"));
+ fs_.Create("bar.cc", "");
+
+ EXPECT_TRUE(builder_.AddTarget("all", &err));
+ ASSERT_EQ("", err);
+
+ // Only one command to run, because phony runs no command.
+ EXPECT_FALSE(builder_.AlreadyUpToDate());
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, PhonyNoWork) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat bar.cc\n"
+"build all: phony out\n"));
+ fs_.Create("bar.cc", "");
+ fs_.Create("out", "");
+
+ EXPECT_TRUE(builder_.AddTarget("all", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+}
+
+// Test a self-referencing phony. Ideally this should not work, but
+// ninja 1.7 and below tolerated and CMake 2.8.12.x and 3.0.x both
+// incorrectly produce it. We tolerate it for compatibility.
+TEST_F(BuildTest, PhonySelfReference) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build a: phony a\n"));
+
+ EXPECT_TRUE(builder_.AddTarget("a", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+}
+
+TEST_F(BuildTest, Fail) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule fail\n"
+" command = fail\n"
+"build out1: fail\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("subcommand failed", err);
+}
+
+TEST_F(BuildTest, SwallowFailures) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule fail\n"
+" command = fail\n"
+"build out1: fail\n"
+"build out2: fail\n"
+"build out3: fail\n"
+"build all: phony out1 out2 out3\n"));
+
+ // Swallow two failures, die on the third.
+ config_.failures_allowed = 3;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("all", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("subcommands failed", err);
+}
+
+TEST_F(BuildTest, SwallowFailuresLimit) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule fail\n"
+" command = fail\n"
+"build out1: fail\n"
+"build out2: fail\n"
+"build out3: fail\n"
+"build final: cat out1 out2 out3\n"));
+
+ // Swallow ten failures; we should stop before building final.
+ config_.failures_allowed = 11;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("final", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("cannot make progress due to previous errors", err);
+}
+
+TEST_F(BuildTest, SwallowFailuresPool) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"pool failpool\n"
+" depth = 1\n"
+"rule fail\n"
+" command = fail\n"
+" pool = failpool\n"
+"build out1: fail\n"
+"build out2: fail\n"
+"build out3: fail\n"
+"build final: cat out1 out2 out3\n"));
+
+ // Swallow ten failures; we should stop before building final.
+ config_.failures_allowed = 11;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("final", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ ASSERT_EQ("cannot make progress due to previous errors", err);
+}
+
+TEST_F(BuildTest, PoolEdgesReadyButNotWanted) {
+ fs_.Create("x", "");
+
+ const char* manifest =
+ "pool some_pool\n"
+ " depth = 4\n"
+ "rule touch\n"
+ " command = touch $out\n"
+ " pool = some_pool\n"
+ "rule cc\n"
+ " command = touch grit\n"
+ "\n"
+ "build B.d.stamp: cc | x\n"
+ "build C.stamp: touch B.d.stamp\n"
+ "build final.stamp: touch || C.stamp\n";
+
+ RebuildTarget("final.stamp", manifest);
+
+ fs_.RemoveFile("B.d.stamp");
+
+ State save_state;
+ RebuildTarget("final.stamp", manifest, NULL, NULL, &save_state);
+ EXPECT_GE(save_state.LookupPool("some_pool")->current_use(), 0);
+}
+
+struct BuildWithLogTest : public BuildTest {
+ BuildWithLogTest() {
+ builder_.SetBuildLog(&build_log_);
+ }
+
+ BuildLog build_log_;
+};
+
+TEST_F(BuildWithLogTest, NotInLogButOnDisk) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n"
+" command = cc\n"
+"build out1: cc in\n"));
+
+ // Create input/output that would be considered up to date when
+ // not considering the command line hash.
+ fs_.Create("in", "");
+ fs_.Create("out1", "");
+ string err;
+
+ // Because it's not in the log, it should not be up-to-date until
+ // we build again.
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_FALSE(builder_.AlreadyUpToDate());
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+}
+
+TEST_F(BuildWithLogTest, RebuildAfterFailure) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch-fail-tick2\n"
+" command = touch-fail-tick2\n"
+"build out1: touch-fail-tick2 in\n"));
+
+ string err;
+
+ fs_.Create("in", "");
+
+ // Run once successfully to get out1 in the log
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ builder_.Cleanup();
+ builder_.plan_.Reset();
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ // Run again with a failure that updates the output file timestamp
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ builder_.Cleanup();
+ builder_.plan_.Reset();
+
+ fs_.Tick();
+
+ // Run again, should rerun even though the output file is up to date on disk
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_FALSE(builder_.AlreadyUpToDate());
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("", err);
+}
+
+TEST_F(BuildWithLogTest, RebuildWithNoInputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch\n"
+"build out1: touch\n"
+"build out2: touch in\n"));
+
+ string err;
+
+ fs_.Create("in", "");
+
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+
+ fs_.Tick();
+
+ fs_.Create("in", "");
+
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildWithLogTest, RestatTest) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n"
+" command = true\n"
+" restat = 1\n"
+"rule cc\n"
+" command = cc\n"
+" restat = 1\n"
+"build out1: cc in\n"
+"build out2: true out1\n"
+"build out3: cat out2\n"));
+
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("out3", "");
+
+ fs_.Tick();
+
+ fs_.Create("in", "");
+
+ // Do a pre-build so that there's commands in the log for the outputs,
+ // otherwise, the lack of an entry in the build log will cause out3 to rebuild
+ // regardless of restat.
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out3", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]",
+ BuildStatus::kEdgeStarted));
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+
+ fs_.Tick();
+
+ fs_.Create("in", "");
+ // "cc" touches out1, so we should build out2. But because "true" does not
+ // touch out2, we should cancel the build of out3.
+ EXPECT_TRUE(builder_.AddTarget("out3", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // If we run again, it should be a no-op, because the build log has recorded
+ // that we've already built out2 with an input timestamp of 2 (from out1).
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out3", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+
+ fs_.Tick();
+
+ fs_.Create("in", "");
+
+ // The build log entry should not, however, prevent us from rebuilding out2
+ // if out1 changes.
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out3", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildWithLogTest, RestatMissingFile) {
+ // If a restat rule doesn't create its output, and the output didn't
+ // exist before the rule was run, consider that behavior equivalent
+ // to a rule that doesn't modify its existent output file.
+
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n"
+" command = true\n"
+" restat = 1\n"
+"rule cc\n"
+" command = cc\n"
+"build out1: true in\n"
+"build out2: cc out1\n"));
+
+ fs_.Create("in", "");
+ fs_.Create("out2", "");
+
+ // Do a pre-build so that there's commands in the log for the outputs,
+ // otherwise, the lack of an entry in the build log will cause out2 to rebuild
+ // regardless of restat.
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+
+ fs_.Tick();
+ fs_.Create("in", "");
+ fs_.Create("out2", "");
+
+ // Run a build, expect only the first command to run.
+ // It doesn't touch its output (due to being the "true" command), so
+ // we shouldn't run the dependent build.
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildWithLogTest, RestatSingleDependentOutputDirty) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "rule true\n"
+ " command = true\n"
+ " restat = 1\n"
+ "rule touch\n"
+ " command = touch\n"
+ "build out1: true in\n"
+ "build out2 out3: touch out1\n"
+ "build out4: touch out2\n"
+ ));
+
+ // Create the necessary files
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out4", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+
+ fs_.Tick();
+ fs_.Create("in", "");
+ fs_.RemoveFile("out3");
+
+ // Since "in" is missing, out1 will be built. Since "out3" is missing,
+ // out2 and out3 will be built even though "in" is not touched when built.
+ // Then, since out2 is rebuilt, out4 should be rebuilt -- the restat on the
+ // "true" rule should not lead to the "touch" edge writing out2 and out3 being
+ // cleard.
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out4", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+}
+
+// Test scenario, in which an input file is removed, but output isn't changed
+// https://github.com/ninja-build/ninja/issues/295
+TEST_F(BuildWithLogTest, RestatMissingInput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "rule true\n"
+ " command = true\n"
+ " depfile = $out.d\n"
+ " restat = 1\n"
+ "rule cc\n"
+ " command = cc\n"
+ "build out1: true in\n"
+ "build out2: cc out1\n"));
+
+ // Create all necessary files
+ fs_.Create("in", "");
+
+ // The implicit dependencies and the depfile itself
+ // are newer than the output
+ TimeStamp restat_mtime = fs_.Tick();
+ fs_.Create("out1.d", "out1: will.be.deleted restat.file\n");
+ fs_.Create("will.be.deleted", "");
+ fs_.Create("restat.file", "");
+
+ // Run the build, out1 and out2 get built
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // See that an entry in the logfile is created, capturing
+ // the right mtime
+ BuildLog::LogEntry* log_entry = build_log_.LookupByOutput("out1");
+ ASSERT_TRUE(NULL != log_entry);
+ ASSERT_EQ(restat_mtime, log_entry->mtime);
+
+ // Now remove a file, referenced from depfile, so that target becomes
+ // dirty, but the output does not change
+ fs_.RemoveFile("will.be.deleted");
+
+ // Trigger the build again - only out1 gets built
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+
+ // Check that the logfile entry remains correctly set
+ log_entry = build_log_.LookupByOutput("out1");
+ ASSERT_TRUE(NULL != log_entry);
+ ASSERT_EQ(restat_mtime, log_entry->mtime);
+}
+
+struct BuildDryRun : public BuildWithLogTest {
+ BuildDryRun() {
+ config_.dry_run = true;
+ }
+};
+
+TEST_F(BuildDryRun, AllCommandsShown) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n"
+" command = true\n"
+" restat = 1\n"
+"rule cc\n"
+" command = cc\n"
+" restat = 1\n"
+"build out1: cc in\n"
+"build out2: true out1\n"
+"build out3: cat out2\n"));
+
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("out3", "");
+
+ fs_.Tick();
+
+ fs_.Create("in", "");
+
+ // "cc" touches out1, so we should build out2. But because "true" does not
+ // touch out2, we should cancel the build of out3.
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out3", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+}
+
+// Test that RSP files are created when & where appropriate and deleted after
+// successful execution.
+TEST_F(BuildTest, RspFileSuccess)
+{
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "rule cat_rsp\n"
+ " command = cat $rspfile > $out\n"
+ " rspfile = $rspfile\n"
+ " rspfile_content = $long_command\n"
+ "rule cat_rsp_out\n"
+ " command = cat $rspfile > $out\n"
+ " rspfile = $out.rsp\n"
+ " rspfile_content = $long_command\n"
+ "build out1: cat in\n"
+ "build out2: cat_rsp in\n"
+ " rspfile = out 2.rsp\n"
+ " long_command = Some very long command\n"
+ "build out$ 3: cat_rsp_out in\n"
+ " long_command = Some very long command\n"));
+
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("out 3", "");
+
+ fs_.Tick();
+
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AddTarget("out 3", &err));
+ ASSERT_EQ("", err);
+
+ size_t files_created = fs_.files_created_.size();
+ size_t files_removed = fs_.files_removed_.size();
+
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+
+ // The RSP files were created
+ ASSERT_EQ(files_created + 2, fs_.files_created_.size());
+ ASSERT_EQ(1u, fs_.files_created_.count("out 2.rsp"));
+ ASSERT_EQ(1u, fs_.files_created_.count("out 3.rsp"));
+
+ // The RSP files were removed
+ ASSERT_EQ(files_removed + 2, fs_.files_removed_.size());
+ ASSERT_EQ(1u, fs_.files_removed_.count("out 2.rsp"));
+ ASSERT_EQ(1u, fs_.files_removed_.count("out 3.rsp"));
+}
+
+// Test that RSP file is created but not removed for commands, which fail
+TEST_F(BuildTest, RspFileFailure) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "rule fail\n"
+ " command = fail\n"
+ " rspfile = $rspfile\n"
+ " rspfile_content = $long_command\n"
+ "build out: fail in\n"
+ " rspfile = out.rsp\n"
+ " long_command = Another very long command\n"));
+
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ size_t files_created = fs_.files_created_.size();
+ size_t files_removed = fs_.files_removed_.size();
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ("subcommand failed", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+
+ // The RSP file was created
+ ASSERT_EQ(files_created + 1, fs_.files_created_.size());
+ ASSERT_EQ(1u, fs_.files_created_.count("out.rsp"));
+
+ // The RSP file was NOT removed
+ ASSERT_EQ(files_removed, fs_.files_removed_.size());
+ ASSERT_EQ(0u, fs_.files_removed_.count("out.rsp"));
+
+ // The RSP file contains what it should
+ ASSERT_EQ("Another very long command", fs_.files_["out.rsp"].contents);
+}
+
+// Test that contents of the RSP file behaves like a regular part of
+// command line, i.e. triggers a rebuild if changed
+TEST_F(BuildWithLogTest, RspFileCmdLineChange) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "rule cat_rsp\n"
+ " command = cat $rspfile > $out\n"
+ " rspfile = $rspfile\n"
+ " rspfile_content = $long_command\n"
+ "build out: cat_rsp in\n"
+ " rspfile = out.rsp\n"
+ " long_command = Original very long command\n"));
+
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ // 1. Build for the 1st time (-> populate log)
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+
+ // 2. Build again (no change)
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ ASSERT_TRUE(builder_.AlreadyUpToDate());
+
+ // 3. Alter the entry in the logfile
+ // (to simulate a change in the command line between 2 builds)
+ BuildLog::LogEntry* log_entry = build_log_.LookupByOutput("out");
+ ASSERT_TRUE(NULL != log_entry);
+ ASSERT_NO_FATAL_FAILURE(AssertHash(
+ "cat out.rsp > out;rspfile=Original very long command",
+ log_entry->command_hash));
+ log_entry->command_hash++; // Change the command hash to something else.
+ // Now expect the target to be rebuilt
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, InterruptCleanup) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule interrupt\n"
+" command = interrupt\n"
+"rule touch-interrupt\n"
+" command = touch-interrupt\n"
+"build out1: interrupt in1\n"
+"build out2: touch-interrupt in2\n"));
+
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Tick();
+ fs_.Create("in1", "");
+ fs_.Create("in2", "");
+
+ // An untouched output of an interrupted command should be retained.
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("interrupted by user", err);
+ builder_.Cleanup();
+ EXPECT_GT(fs_.Stat("out1", &err), 0);
+ err = "";
+
+ // A touched output of an interrupted command should be deleted.
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("interrupted by user", err);
+ builder_.Cleanup();
+ EXPECT_EQ(0, fs_.Stat("out2", &err));
+}
+
+TEST_F(BuildTest, StatFailureAbortsBuild) {
+ const string kTooLongToStat(400, 'i');
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+("build " + kTooLongToStat + ": cat in\n").c_str()));
+ fs_.Create("in", "");
+
+ // This simulates a stat failure:
+ fs_.files_[kTooLongToStat].mtime = -1;
+ fs_.files_[kTooLongToStat].stat_error = "stat failed";
+
+ string err;
+ EXPECT_FALSE(builder_.AddTarget(kTooLongToStat, &err));
+ EXPECT_EQ("stat failed", err);
+}
+
+TEST_F(BuildTest, PhonyWithNoInputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build nonexistent: phony\n"
+"build out1: cat || nonexistent\n"
+"build out2: cat nonexistent\n"));
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+
+ // out1 should be up to date even though its input is dirty, because its
+ // order-only dependency has nothing to do.
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.AlreadyUpToDate());
+
+ // out2 should still be out of date though, because its input is dirty.
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, DepsGccWithEmptyDepfileErrorsOut) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n"
+" command = cc\n"
+" deps = gcc\n"
+"build out: cc\n"));
+ Dirty("out");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_FALSE(builder_.AlreadyUpToDate());
+
+ EXPECT_FALSE(builder_.Build(&err));
+ ASSERT_EQ("subcommand failed", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, StatusFormatElapsed) {
+ status_.BuildStarted();
+ // Before any task is done, the elapsed time must be zero.
+ EXPECT_EQ("[%/e0.000]",
+ status_.FormatProgressStatus("[%%/e%e]",
+ BuildStatus::kEdgeStarted));
+}
+
+TEST_F(BuildTest, StatusFormatReplacePlaceholder) {
+ EXPECT_EQ("[%/s0/t0/r0/u0/f0]",
+ status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]",
+ BuildStatus::kEdgeStarted));
+}
+
+TEST_F(BuildTest, FailedDepsParse) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build bad_deps.o: cat in1\n"
+" deps = gcc\n"
+" depfile = in1.d\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("bad_deps.o", &err));
+ ASSERT_EQ("", err);
+
+ // These deps will fail to parse, as they should only have one
+ // path to the left of the colon.
+ fs_.Create("in1.d", "AAA BBB");
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+}
+
+struct BuildWithQueryDepsLogTest : public BuildTest {
+ BuildWithQueryDepsLogTest() : BuildTest(&log_) {
+ }
+
+ ~BuildWithQueryDepsLogTest() {
+ log_.Close();
+ }
+
+ virtual void SetUp() {
+ BuildTest::SetUp();
+
+ temp_dir_.CreateAndEnter("BuildWithQueryDepsLogTest");
+
+ std::string err;
+ ASSERT_TRUE(log_.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+ }
+
+ ScopedTempDir temp_dir_;
+
+ DepsLog log_;
+};
+
+/// Test a MSVC-style deps log with multiple outputs.
+TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileMSVC) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cp_multi_msvc\n"
+" command = echo 'using $in' && for file in $out; do cp $in $$file; done\n"
+" deps = msvc\n"
+" msvc_deps_prefix = using \n"
+"build out1 out2: cp_multi_msvc in1\n"));
+
+ std::string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("echo 'using in1' && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]);
+
+ Node* out1_node = state_.LookupNode("out1");
+ DepsLog::Deps* out1_deps = log_.GetDeps(out1_node);
+ EXPECT_EQ(1, out1_deps->node_count);
+ EXPECT_EQ("in1", out1_deps->nodes[0]->path());
+
+ Node* out2_node = state_.LookupNode("out2");
+ DepsLog::Deps* out2_deps = log_.GetDeps(out2_node);
+ EXPECT_EQ(1, out2_deps->node_count);
+ EXPECT_EQ("in1", out2_deps->nodes[0]->path());
+}
+
+/// Test a GCC-style deps log with multiple outputs.
+TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOneLine) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cp_multi_gcc\n"
+" command = echo '$out: $in' > in.d && for file in $out; do cp in1 $$file; done\n"
+" deps = gcc\n"
+" depfile = in.d\n"
+"build out1 out2: cp_multi_gcc in1 in2\n"));
+
+ std::string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in.d", "out1 out2: in1 in2");
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("echo 'out1 out2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]);
+
+ Node* out1_node = state_.LookupNode("out1");
+ DepsLog::Deps* out1_deps = log_.GetDeps(out1_node);
+ EXPECT_EQ(2, out1_deps->node_count);
+ EXPECT_EQ("in1", out1_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out1_deps->nodes[1]->path());
+
+ Node* out2_node = state_.LookupNode("out2");
+ DepsLog::Deps* out2_deps = log_.GetDeps(out2_node);
+ EXPECT_EQ(2, out2_deps->node_count);
+ EXPECT_EQ("in1", out2_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out2_deps->nodes[1]->path());
+}
+
+/// Test a GCC-style deps log with multiple outputs using a line per input.
+TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCMultiLineInput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cp_multi_gcc\n"
+" command = echo '$out: in1\\n$out: in2' > in.d && for file in $out; do cp in1 $$file; done\n"
+" deps = gcc\n"
+" depfile = in.d\n"
+"build out1 out2: cp_multi_gcc in1 in2\n"));
+
+ std::string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in.d", "out1 out2: in1\nout1 out2: in2");
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("echo 'out1 out2: in1\\nout1 out2: in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]);
+
+ Node* out1_node = state_.LookupNode("out1");
+ DepsLog::Deps* out1_deps = log_.GetDeps(out1_node);
+ EXPECT_EQ(2, out1_deps->node_count);
+ EXPECT_EQ("in1", out1_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out1_deps->nodes[1]->path());
+
+ Node* out2_node = state_.LookupNode("out2");
+ DepsLog::Deps* out2_deps = log_.GetDeps(out2_node);
+ EXPECT_EQ(2, out2_deps->node_count);
+ EXPECT_EQ("in1", out2_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out2_deps->nodes[1]->path());
+}
+
+/// Test a GCC-style deps log with multiple outputs using a line per output.
+TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCMultiLineOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cp_multi_gcc\n"
+" command = echo 'out1: $in\\nout2: $in' > in.d && for file in $out; do cp in1 $$file; done\n"
+" deps = gcc\n"
+" depfile = in.d\n"
+"build out1 out2: cp_multi_gcc in1 in2\n"));
+
+ std::string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in.d", "out1: in1 in2\nout2: in1 in2");
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("echo 'out1: in1 in2\\nout2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]);
+
+ Node* out1_node = state_.LookupNode("out1");
+ DepsLog::Deps* out1_deps = log_.GetDeps(out1_node);
+ EXPECT_EQ(2, out1_deps->node_count);
+ EXPECT_EQ("in1", out1_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out1_deps->nodes[1]->path());
+
+ Node* out2_node = state_.LookupNode("out2");
+ DepsLog::Deps* out2_deps = log_.GetDeps(out2_node);
+ EXPECT_EQ(2, out2_deps->node_count);
+ EXPECT_EQ("in1", out2_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out2_deps->nodes[1]->path());
+}
+
+/// Test a GCC-style deps log with multiple outputs mentioning only the main output.
+TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOnlyMainOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cp_multi_gcc\n"
+" command = echo 'out1: $in' > in.d && for file in $out; do cp in1 $$file; done\n"
+" deps = gcc\n"
+" depfile = in.d\n"
+"build out1 out2: cp_multi_gcc in1 in2\n"));
+
+ std::string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in.d", "out1: in1 in2");
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("echo 'out1: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]);
+
+ Node* out1_node = state_.LookupNode("out1");
+ DepsLog::Deps* out1_deps = log_.GetDeps(out1_node);
+ EXPECT_EQ(2, out1_deps->node_count);
+ EXPECT_EQ("in1", out1_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out1_deps->nodes[1]->path());
+
+ Node* out2_node = state_.LookupNode("out2");
+ DepsLog::Deps* out2_deps = log_.GetDeps(out2_node);
+ EXPECT_EQ(2, out2_deps->node_count);
+ EXPECT_EQ("in1", out2_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out2_deps->nodes[1]->path());
+}
+
+/// Test a GCC-style deps log with multiple outputs mentioning only the secondary output.
+TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOnlySecondaryOutput) {
+ // Note: This ends up short-circuiting the node creation due to the primary
+ // output not being present, but it should still work.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cp_multi_gcc\n"
+" command = echo 'out2: $in' > in.d && for file in $out; do cp in1 $$file; done\n"
+" deps = gcc\n"
+" depfile = in.d\n"
+"build out1 out2: cp_multi_gcc in1 in2\n"));
+
+ std::string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in.d", "out2: in1 in2");
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("echo 'out2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]);
+
+ Node* out1_node = state_.LookupNode("out1");
+ DepsLog::Deps* out1_deps = log_.GetDeps(out1_node);
+ EXPECT_EQ(2, out1_deps->node_count);
+ EXPECT_EQ("in1", out1_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out1_deps->nodes[1]->path());
+
+ Node* out2_node = state_.LookupNode("out2");
+ DepsLog::Deps* out2_deps = log_.GetDeps(out2_node);
+ EXPECT_EQ(2, out2_deps->node_count);
+ EXPECT_EQ("in1", out2_deps->nodes[0]->path());
+ EXPECT_EQ("in2", out2_deps->nodes[1]->path());
+}
+
+/// Tests of builds involving deps logs necessarily must span
+/// multiple builds. We reuse methods on BuildTest but not the
+/// builder_ it sets up, because we want pristine objects for
+/// each build.
+struct BuildWithDepsLogTest : public BuildTest {
+ BuildWithDepsLogTest() {}
+
+ virtual void SetUp() {
+ BuildTest::SetUp();
+
+ temp_dir_.CreateAndEnter("BuildWithDepsLogTest");
+ }
+
+ virtual void TearDown() {
+ temp_dir_.Cleanup();
+ }
+
+ ScopedTempDir temp_dir_;
+
+ /// Shadow parent class builder_ so we don't accidentally use it.
+ void* builder_;
+};
+
+/// Run a straightforwad build where the deps log is used.
+TEST_F(BuildWithDepsLogTest, Straightforward) {
+ string err;
+ // Note: in1 was created by the superclass SetUp().
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in1.d", "out: in2");
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // The deps file should have been removed.
+ EXPECT_EQ(0, fs_.Stat("in1.d", &err));
+ // Recreate it for the next step.
+ fs_.Create("in1.d", "out: in2");
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Touch the file only mentioned in the deps.
+ fs_.Tick();
+ fs_.Create("in2", "");
+
+ // Run the build again.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // We should have rebuilt the output due to in2 being
+ // out of date.
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+ }
+}
+
+/// Verify that obsolete dependency info causes a rebuild.
+/// 1) Run a successful build where everything has time t, record deps.
+/// 2) Move input/output to time t+1 -- despite files in alignment,
+/// should still need to rebuild due to deps at older time.
+TEST_F(BuildWithDepsLogTest, ObsoleteDeps) {
+ string err;
+ // Note: in1 was created by the superclass SetUp().
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+ {
+ // Run an ordinary build that gathers dependencies.
+ fs_.Create("in1", "");
+ fs_.Create("in1.d", "out: ");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ // Push all files one tick forward so that only the deps are out
+ // of date.
+ fs_.Tick();
+ fs_.Create("in1", "");
+ fs_.Create("out", "");
+
+ // The deps file should have been removed, so no need to timestamp it.
+ EXPECT_EQ(0, fs_.Stat("in1.d", &err));
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ // Recreate the deps file here because the build expects them to exist.
+ fs_.Create("in1.d", "out: ");
+
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // We should have rebuilt the output due to the deps being
+ // out of date.
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+ }
+}
+
+TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) {
+ const char* manifest =
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in1", "");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // The deps log is NULL in dry runs.
+ config_.dry_run = true;
+ Builder builder(&state, config_, NULL, NULL, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+
+ string err;
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+}
+
+/// Check that a restat rule generating a header cancels compilations correctly.
+TEST_F(BuildTest, RestatDepfileDependency) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n"
+" command = true\n" // Would be "write if out-of-date" in reality.
+" restat = 1\n"
+"build header.h: true header.in\n"
+"build out: cat in1\n"
+" depfile = in1.d\n"));
+
+ fs_.Create("header.h", "");
+ fs_.Create("in1.d", "out: header.h");
+ fs_.Tick();
+ fs_.Create("header.in", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+}
+
+/// Check that a restat rule generating a header cancels compilations correctly,
+/// depslog case.
+TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) {
+ string err;
+ // Note: in1 was created by the superclass SetUp().
+ const char* manifest =
+ "rule true\n"
+ " command = true\n" // Would be "write if out-of-date" in reality.
+ " restat = 1\n"
+ "build header.h: true header.in\n"
+ "build out: cat in1\n"
+ " deps = gcc\n"
+ " depfile = in1.d\n";
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("in1.d", "out: header.h");
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Touch the input of the restat rule.
+ fs_.Tick();
+ fs_.Create("header.in", "");
+
+ // Run the build again.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ command_runner_.commands_ran_.clear();
+ EXPECT_TRUE(builder.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // Rule "true" should have run again, but the build of "out" should have
+ // been cancelled due to restat propagating through the depfile header.
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ builder.command_runner_.release();
+ }
+}
+
+TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) {
+ string err;
+ const char* manifest =
+ "rule cc\n command = cc $in\n depfile = $out.d\n deps = gcc\n"
+ "build fo$ o.o: cc foo.c\n";
+
+ fs_.Create("foo.c", "");
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("fo o.o", &err));
+ ASSERT_EQ("", err);
+ fs_.Create("fo o.o.d", "fo\\ o.o: blah.h bar.h\n");
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+
+ Edge* edge = state.edges_.back();
+
+ state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing.
+ EXPECT_TRUE(builder.AddTarget("fo o.o", &err));
+ ASSERT_EQ("", err);
+
+ // Expect three new edges: one generating fo o.o, and two more from
+ // loading the depfile.
+ ASSERT_EQ(3u, state.edges_.size());
+ // Expect our edge to now have three inputs: foo.c and two headers.
+ ASSERT_EQ(3u, edge->inputs_.size());
+
+ // Expect the command line we generate to only use the original input.
+ ASSERT_EQ("cc foo.c", edge->EvaluateCommand());
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+}
+
+#ifdef _WIN32
+TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) {
+ string err;
+ const char* manifest =
+ "rule cc\n command = cc $in\n depfile = $out.d\n deps = gcc\n"
+ "build a/b\\c\\d/e/fo$ o.o: cc x\\y/z\\foo.c\n";
+
+ fs_.Create("x/y/z/foo.c", "");
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ // Run the build once, everything should be ok.
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+ EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err));
+ ASSERT_EQ("", err);
+ // Note, different slashes from manifest.
+ fs_.Create("a/b\\c\\d/e/fo o.o.d",
+ "a\\b\\c\\d\\e\\fo\\ o.o: blah.h bar.h\n");
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ {
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ builder.command_runner_.reset(&command_runner_);
+
+ Edge* edge = state.edges_.back();
+
+ state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing.
+ EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err));
+ ASSERT_EQ("", err);
+
+ // Expect three new edges: one generating fo o.o, and two more from
+ // loading the depfile.
+ ASSERT_EQ(3u, state.edges_.size());
+ // Expect our edge to now have three inputs: foo.c and two headers.
+ ASSERT_EQ(3u, edge->inputs_.size());
+
+ // Expect the command line we generate to only use the original input.
+ // Note, slashes from manifest, not .d.
+ ASSERT_EQ("cc x\\y/z\\foo.c", edge->EvaluateCommand());
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+}
+#endif
+
+/// Check that a restat rule doesn't clear an edge if the depfile is missing.
+/// Follows from: https://github.com/ninja-build/ninja/issues/603
+TEST_F(BuildTest, RestatMissingDepfile) {
+const char* manifest =
+"rule true\n"
+" command = true\n" // Would be "write if out-of-date" in reality.
+" restat = 1\n"
+"build header.h: true header.in\n"
+"build out: cat header.h\n"
+" depfile = out.d\n";
+
+ fs_.Create("header.h", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+ fs_.Create("header.in", "");
+
+ // Normally, only 'header.h' would be rebuilt, as
+ // its rule doesn't touch the output and has 'restat=1' set.
+ // But we are also missing the depfile for 'out',
+ // which should force its command to run anyway!
+ RebuildTarget("out", manifest);
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+}
+
+/// Check that a restat rule doesn't clear an edge if the deps are missing.
+/// https://github.com/ninja-build/ninja/issues/603
+TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) {
+ string err;
+ const char* manifest =
+"rule true\n"
+" command = true\n" // Would be "write if out-of-date" in reality.
+" restat = 1\n"
+"build header.h: true header.in\n"
+"build out: cat header.h\n"
+" deps = gcc\n"
+" depfile = out.d\n";
+
+ // Build once to populate ninja deps logs from out.d
+ fs_.Create("header.in", "");
+ fs_.Create("out.d", "out: header.h");
+ fs_.Create("header.h", "");
+
+ RebuildTarget("out", manifest, "build_log", "ninja_deps");
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Sanity: this rebuild should be NOOP
+ RebuildTarget("out", manifest, "build_log", "ninja_deps");
+ ASSERT_EQ(0u, command_runner_.commands_ran_.size());
+
+ // Touch 'header.in', blank dependencies log (create a different one).
+ // Building header.h triggers 'restat' outputs cleanup.
+ // Validate that out is rebuilt netherless, as deps are missing.
+ fs_.Tick();
+ fs_.Create("header.in", "");
+
+ // (switch to a new blank deps_log "ninja_deps2")
+ RebuildTarget("out", manifest, "build_log", "ninja_deps2");
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Sanity: this build should be NOOP
+ RebuildTarget("out", manifest, "build_log", "ninja_deps2");
+ ASSERT_EQ(0u, command_runner_.commands_ran_.size());
+
+ // Check that invalidating deps by target timestamp also works here
+ // Repeat the test but touch target instead of blanking the log.
+ fs_.Tick();
+ fs_.Create("header.in", "");
+ fs_.Create("out", "");
+ RebuildTarget("out", manifest, "build_log", "ninja_deps2");
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // And this build should be NOOP again
+ RebuildTarget("out", manifest, "build_log", "ninja_deps2");
+ ASSERT_EQ(0u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, WrongOutputInDepfileCausesRebuild) {
+ string err;
+ const char* manifest =
+"rule cc\n"
+" command = cc $in\n"
+" depfile = $out.d\n"
+"build foo.o: cc foo.c\n";
+
+ fs_.Create("foo.c", "");
+ fs_.Create("foo.o", "");
+ fs_.Create("header.h", "");
+ fs_.Create("foo.o.d", "bar.o.d: header.h\n");
+
+ RebuildTarget("foo.o", manifest, "build_log", "ninja_deps");
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, Console) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule console\n"
+" command = console\n"
+" pool = console\n"
+"build cons: console in.txt\n"));
+
+ fs_.Create("in.txt", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cons", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, DyndepMissingAndNoRule) {
+ // Verify that we can diagnose when a dyndep file is missing and
+ // has no rule to build it.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+));
+
+ string err;
+ EXPECT_FALSE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("loading 'dd': No such file or directory", err);
+}
+
+TEST_F(BuildTest, DyndepReadyImplicitConnection) {
+ // Verify that a dyndep file can be loaded immediately to discover
+ // that one edge has an implicit output that is also an implicit
+ // input of another edge.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"build tmp: touch || dd\n"
+" dyndep = dd\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+));
+ fs_.Create("dd",
+"ninja_dyndep_version = 1\n"
+"build out | out.imp: dyndep | tmp.imp\n"
+"build tmp | tmp.imp: dyndep\n"
+);
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[1]);
+}
+
+TEST_F(BuildTest, DyndepReadySyntaxError) {
+ // Verify that a dyndep file can be loaded immediately to discover
+ // and reject a syntax error in it.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+));
+ fs_.Create("dd",
+"build out: dyndep\n"
+);
+
+ string err;
+ EXPECT_FALSE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("dd:1: expected 'ninja_dyndep_version = ...'\n", err);
+}
+
+TEST_F(BuildTest, DyndepReadyCircular) {
+ // Verify that a dyndep file can be loaded immediately to discover
+ // and reject a circular dependency.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule r\n"
+" command = unused\n"
+"build out: r in || dd\n"
+" dyndep = dd\n"
+"build in: r circ\n"
+ ));
+ fs_.Create("dd",
+"ninja_dyndep_version = 1\n"
+"build out | circ: dyndep\n"
+ );
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_FALSE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("dependency cycle: circ -> in -> circ", err);
+}
+
+TEST_F(BuildTest, DyndepBuild) {
+ // Verify that a dyndep file can be built and loaded to discover nothing.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+));
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep\n"
+);
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ size_t files_created = fs_.files_created_.size();
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch out", command_runner_.commands_ran_[1]);
+ ASSERT_EQ(2u, fs_.files_read_.size());
+ EXPECT_EQ("dd-in", fs_.files_read_[0]);
+ EXPECT_EQ("dd", fs_.files_read_[1]);
+ ASSERT_EQ(2u + files_created, fs_.files_created_.size());
+ EXPECT_EQ(1u, fs_.files_created_.count("dd"));
+ EXPECT_EQ(1u, fs_.files_created_.count("out"));
+}
+
+TEST_F(BuildTest, DyndepBuildSyntaxError) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // and reject a syntax error in it.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+));
+ fs_.Create("dd-in",
+"build out: dyndep\n"
+);
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("dd:1: expected 'ninja_dyndep_version = ...'\n", err);
+}
+
+TEST_F(BuildTest, DyndepBuildUnrelatedOutput) {
+ // Verify that a dyndep file can have dependents that do not specify
+ // it as their dyndep binding.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build unrelated: touch || dd\n"
+"build out: touch unrelated || dd\n"
+" dyndep = dd\n"
+ ));
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep\n"
+);
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch unrelated", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverNewOutput) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // a new output of an edge.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build out: touch in || dd\n"
+" dyndep = dd\n"
+ ));
+ fs_.Create("in", "");
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out | out.imp: dyndep\n"
+);
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(2u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[1]);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverNewOutputWithMultipleRules1) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // a new output of an edge that is already the output of another edge.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build out1 | out-twice.imp: touch in\n"
+"build out2: touch in || dd\n"
+" dyndep = dd\n"
+ ));
+ fs_.Create("in", "");
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out2 | out-twice.imp: dyndep\n"
+);
+ fs_.Tick();
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("multiple rules generate out-twice.imp", err);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverNewOutputWithMultipleRules2) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // a new output of an edge that is already the output of another
+ // edge also discovered by dyndep.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd1: cp dd1-in\n"
+"build out1: touch || dd1\n"
+" dyndep = dd1\n"
+"build dd2: cp dd2-in || dd1\n" // make order predictable for test
+"build out2: touch || dd2\n"
+" dyndep = dd2\n"
+));
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("dd1-in",
+"ninja_dyndep_version = 1\n"
+"build out1 | out-twice.imp: dyndep\n"
+);
+ fs_.Create("dd2-in", "");
+ fs_.Create("dd2",
+"ninja_dyndep_version = 1\n"
+"build out2 | out-twice.imp: dyndep\n"
+);
+ fs_.Tick();
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("multiple rules generate out-twice.imp", err);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverNewInput) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // a new input to an edge.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build in: touch\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+ ));
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep | in\n"
+);
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch in", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // that one edge has an implicit output that is also an implicit
+ // input of another edge.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build tmp: touch || dd\n"
+" dyndep = dd\n"
+"build out: touch || dd\n"
+" dyndep = dd\n"
+));
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out | out.imp: dyndep | tmp.imp\n"
+"build tmp | tmp.imp: dyndep\n"
+);
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdge) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // that an edge is actually wanted due to a missing implicit output.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build tmp: touch || dd\n"
+" dyndep = dd\n"
+"build out: touch tmp || dd\n"
+" dyndep = dd\n"
+));
+ fs_.Create("tmp", "");
+ fs_.Create("out", "");
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep\n"
+"build tmp | tmp.imp: dyndep\n"
+);
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdgeAndDependent) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // that an edge and a dependent are actually wanted.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build tmp: touch || dd\n"
+" dyndep = dd\n"
+"build out: touch tmp\n"
+));
+ fs_.Create("tmp", "");
+ fs_.Create("out", "");
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build tmp | tmp.imp: dyndep\n"
+);
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverCircular) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // and reject a circular dependency.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule r\n"
+" command = unused\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build out: r in || dd\n"
+" depfile = out.d\n"
+" dyndep = dd\n"
+"build in: r || dd\n"
+" dyndep = dd\n"
+ ));
+ fs_.Create("out.d", "out: inimp\n");
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out | circ: dyndep\n"
+"build in: dyndep | circ\n"
+ );
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ // Depending on how the pointers in Plan::ready_ work out, we could have
+ // discovered the cycle from either starting point.
+ EXPECT_TRUE(err == "dependency cycle: circ -> in -> circ" ||
+ err == "dependency cycle: in -> circ -> in");
+}
+
+TEST_F(BuildWithLogTest, DyndepBuildDiscoverRestat) {
+ // Verify that a dyndep file can be built and loaded to discover
+ // that an edge has a restat binding.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n"
+" command = true\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd: cp dd-in\n"
+"build out1: true in || dd\n"
+" dyndep = dd\n"
+"build out2: cat out1\n"));
+
+ fs_.Create("out1", "");
+ fs_.Create("out2", "");
+ fs_.Create("dd-in",
+"ninja_dyndep_version = 1\n"
+"build out1: dyndep\n"
+" restat = 1\n"
+);
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ // Do a pre-build so that there's commands in the log for the outputs,
+ // otherwise, the lack of an entry in the build log will cause "out2" to
+ // rebuild regardless of restat.
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("true", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("cat out1 > out2", command_runner_.commands_ran_[2]);
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ // We touched "in", so we should build "out1". But because "true" does not
+ // touch "out1", we should cancel the build of "out2".
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ ASSERT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("true", command_runner_.commands_ran_[0]);
+}
+
+TEST_F(BuildTest, DyndepBuildDiscoverScheduledEdge) {
+ // Verify that a dyndep file can be built and loaded to discover a
+ // new input that itself is an output from an edge that has already
+ // been scheduled but not finished. We should not re-schedule it.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build out1 | out1.imp: touch\n"
+"build zdd: cp zdd-in\n"
+" verify_active_edge = out1\n" // verify out1 is active when zdd is finished
+"build out2: cp out1 || zdd\n"
+" dyndep = zdd\n"
+));
+ fs_.Create("zdd-in",
+"ninja_dyndep_version = 1\n"
+"build out2: dyndep | out1.imp\n"
+);
+
+ // Enable concurrent builds so that we can load the dyndep file
+ // while another edge is still active.
+ command_runner_.max_active_edges_ = 2;
+
+ // During the build "out1" and "zdd" should be built concurrently.
+ // The fake command runner will finish these in reverse order
+ // of the names of the first outputs, so "zdd" will finish first
+ // and we will load the dyndep file while the edge for "out1" is
+ // still active. This will add a new dependency on "out1.imp",
+ // also produced by the active edge. The builder should not
+ // re-schedule the already-active edge.
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out1", &err));
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ // Depending on how the pointers in Plan::ready_ work out, the first
+ // two commands may have run in either order.
+ EXPECT_TRUE((command_runner_.commands_ran_[0] == "touch out1 out1.imp" &&
+ command_runner_.commands_ran_[1] == "cp zdd-in zdd") ||
+ (command_runner_.commands_ran_[1] == "touch out1 out1.imp" &&
+ command_runner_.commands_ran_[0] == "cp zdd-in zdd"));
+ EXPECT_EQ("cp out1 out2", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepTwoLevelDirect) {
+ // Verify that a clean dyndep file can depend on a dirty dyndep file
+ // and be loaded properly after the dirty one is built and loaded.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd1: cp dd1-in\n"
+"build out1 | out1.imp: touch || dd1\n"
+" dyndep = dd1\n"
+"build dd2: cp dd2-in || dd1\n" // direct order-only dep on dd1
+"build out2: touch || dd2\n"
+" dyndep = dd2\n"
+));
+ fs_.Create("out1.imp", "");
+ fs_.Create("out2", "");
+ fs_.Create("out2.imp", "");
+ fs_.Create("dd1-in",
+"ninja_dyndep_version = 1\n"
+"build out1: dyndep\n"
+);
+ fs_.Create("dd2-in", "");
+ fs_.Create("dd2",
+"ninja_dyndep_version = 1\n"
+"build out2 | out2.imp: dyndep | out1.imp\n"
+);
+
+ // During the build dd1 should be built and loaded. The RecomputeDirty
+ // called as a result of loading dd1 should not cause dd2 to be loaded
+ // because the builder will never get a chance to update the build plan
+ // to account for dd2. Instead dd2 should only be later loaded once the
+ // builder recognizes that it is now ready (as its order-only dependency
+ // on dd1 has been satisfied). This test case verifies that each dyndep
+ // file is loaded to update the build graph independently.
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch out1 out1.imp", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out2 out2.imp", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepTwoLevelIndirect) {
+ // Verify that dyndep files can add to an edge new implicit inputs that
+ // correspond to implicit outputs added to other edges by other dyndep
+ // files on which they (order-only) depend.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out $out.imp\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd1: cp dd1-in\n"
+"build out1: touch || dd1\n"
+" dyndep = dd1\n"
+"build dd2: cp dd2-in || out1\n" // indirect order-only dep on dd1
+"build out2: touch || dd2\n"
+" dyndep = dd2\n"
+));
+ fs_.Create("out1.imp", "");
+ fs_.Create("out2", "");
+ fs_.Create("out2.imp", "");
+ fs_.Create("dd1-in",
+"ninja_dyndep_version = 1\n"
+"build out1 | out1.imp: dyndep\n"
+);
+ fs_.Create("dd2-in", "");
+ fs_.Create("dd2",
+"ninja_dyndep_version = 1\n"
+"build out2 | out2.imp: dyndep | out1.imp\n"
+);
+
+ // During the build dd1 should be built and loaded. Then dd2 should
+ // be built and loaded. Loading dd2 should cause the builder to
+ // recognize that out2 needs to be built even though it was originally
+ // clean without dyndep info.
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch out1 out1.imp", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch out2 out2.imp", command_runner_.commands_ran_[2]);
+}
+
+TEST_F(BuildTest, DyndepTwoLevelDiscoveredReady) {
+ // Verify that a dyndep file can discover a new input whose
+ // edge also has a dyndep file that is ready to load immediately.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd0: cp dd0-in\n"
+"build dd1: cp dd1-in\n"
+"build in: touch\n"
+"build tmp: touch || dd0\n"
+" dyndep = dd0\n"
+"build out: touch || dd1\n"
+" dyndep = dd1\n"
+ ));
+ fs_.Create("dd1-in",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep | tmp\n"
+);
+ fs_.Create("dd0-in", "");
+ fs_.Create("dd0",
+"ninja_dyndep_version = 1\n"
+"build tmp: dyndep | in\n"
+);
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(4u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("touch in", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch tmp", command_runner_.commands_ran_[2]);
+ EXPECT_EQ("touch out", command_runner_.commands_ran_[3]);
+}
+
+TEST_F(BuildTest, DyndepTwoLevelDiscoveredDirty) {
+ // Verify that a dyndep file can discover a new input whose
+ // edge also has a dyndep file that needs to be built.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n"
+" command = touch $out\n"
+"rule cp\n"
+" command = cp $in $out\n"
+"build dd0: cp dd0-in\n"
+"build dd1: cp dd1-in\n"
+"build in: touch\n"
+"build tmp: touch || dd0\n"
+" dyndep = dd0\n"
+"build out: touch || dd1\n"
+" dyndep = dd1\n"
+ ));
+ fs_.Create("dd1-in",
+"ninja_dyndep_version = 1\n"
+"build out: dyndep | tmp\n"
+);
+ fs_.Create("dd0-in",
+"ninja_dyndep_version = 1\n"
+"build tmp: dyndep | in\n"
+);
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ ASSERT_EQ(5u, command_runner_.commands_ran_.size());
+ EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]);
+ EXPECT_EQ("cp dd0-in dd0", command_runner_.commands_ran_[1]);
+ EXPECT_EQ("touch in", command_runner_.commands_ran_[2]);
+ EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]);
+ EXPECT_EQ("touch out", command_runner_.commands_ran_[4]);
+}
+
+/// The token tests are concerned with the main loop functionality when
+// the CommandRunner has an active TokenPool. It is therefore intentional
+// that the plan doesn't complete and that builder_.Build() returns false!
+
+/// Fake implementation of CommandRunner that simulates a TokenPool
+struct FakeTokenCommandRunner : public CommandRunner {
+ explicit FakeTokenCommandRunner() {}
+
+ // CommandRunner impl
+ virtual bool CanRunMore() const;
+ virtual bool AcquireToken();
+ virtual bool StartCommand(Edge* edge);
+ virtual bool WaitForCommand(Result* result, bool more_ready);
+ virtual vector<Edge*> GetActiveEdges();
+ virtual void Abort();
+
+ vector<string> commands_ran_;
+ vector<Edge *> edges_;
+
+ vector<bool> acquire_token_;
+ vector<bool> can_run_more_;
+ vector<bool> wait_for_command_;
+};
+
+bool FakeTokenCommandRunner::CanRunMore() const {
+ if (can_run_more_.size() == 0) {
+ EXPECT_FALSE("unexpected call to CommandRunner::CanRunMore()");
+ return false;
+ }
+
+ bool result = can_run_more_[0];
+
+ // Unfortunately CanRunMore() isn't "const" for tests
+ const_cast<FakeTokenCommandRunner*>(this)->can_run_more_.erase(
+ const_cast<FakeTokenCommandRunner*>(this)->can_run_more_.begin()
+ );
+
+ return result;
+}
+
+bool FakeTokenCommandRunner::AcquireToken() {
+ if (acquire_token_.size() == 0) {
+ EXPECT_FALSE("unexpected call to CommandRunner::AcquireToken()");
+ return false;
+ }
+
+ bool result = acquire_token_[0];
+ acquire_token_.erase(acquire_token_.begin());
+ return result;
+}
+
+bool FakeTokenCommandRunner::StartCommand(Edge* edge) {
+ commands_ran_.push_back(edge->EvaluateCommand());
+ edges_.push_back(edge);
+ return true;
+}
+
+bool FakeTokenCommandRunner::WaitForCommand(Result* result, bool more_ready) {
+ if (wait_for_command_.size() == 0) {
+ EXPECT_FALSE("unexpected call to CommandRunner::WaitForCommand()");
+ return false;
+ }
+
+ bool expected = wait_for_command_[0];
+ if (expected != more_ready) {
+ EXPECT_EQ(expected, more_ready);
+ return false;
+ }
+ wait_for_command_.erase(wait_for_command_.begin());
+
+ if (edges_.size() == 0)
+ return false;
+
+ Edge* edge = edges_[0];
+ result->edge = edge;
+
+ if (more_ready &&
+ (edge->rule().name() == "token-available")) {
+ result->status = ExitTokenAvailable;
+ } else {
+ edges_.erase(edges_.begin());
+ result->status = ExitSuccess;
+ }
+
+ return true;
+}
+
+vector<Edge*> FakeTokenCommandRunner::GetActiveEdges() {
+ return edges_;
+}
+
+void FakeTokenCommandRunner::Abort() {
+ edges_.clear();
+}
+
+struct BuildTokenTest : public BuildTest {
+ virtual void SetUp();
+ virtual void TearDown();
+
+ FakeTokenCommandRunner token_command_runner_;
+
+ void ExpectAcquireToken(int count, ...);
+ void ExpectCanRunMore(int count, ...);
+ void ExpectWaitForCommand(int count, ...);
+
+private:
+ void EnqueueBooleans(vector<bool>& booleans, int count, va_list ao);
+};
+
+void BuildTokenTest::SetUp() {
+ BuildTest::SetUp();
+
+ // replace FakeCommandRunner with FakeTokenCommandRunner
+ builder_.command_runner_.release();
+ builder_.command_runner_.reset(&token_command_runner_);
+}
+void BuildTokenTest::TearDown() {
+ EXPECT_EQ(0u, token_command_runner_.acquire_token_.size());
+ EXPECT_EQ(0u, token_command_runner_.can_run_more_.size());
+ EXPECT_EQ(0u, token_command_runner_.wait_for_command_.size());
+
+ BuildTest::TearDown();
+}
+
+void BuildTokenTest::ExpectAcquireToken(int count, ...) {
+ va_list ap;
+ va_start(ap, count);
+ EnqueueBooleans(token_command_runner_.acquire_token_, count, ap);
+ va_end(ap);
+}
+
+void BuildTokenTest::ExpectCanRunMore(int count, ...) {
+ va_list ap;
+ va_start(ap, count);
+ EnqueueBooleans(token_command_runner_.can_run_more_, count, ap);
+ va_end(ap);
+}
+
+void BuildTokenTest::ExpectWaitForCommand(int count, ...) {
+ va_list ap;
+ va_start(ap, count);
+ EnqueueBooleans(token_command_runner_.wait_for_command_, count, ap);
+ va_end(ap);
+}
+
+void BuildTokenTest::EnqueueBooleans(vector<bool>& booleans, int count, va_list ap) {
+ while (count--) {
+ int value = va_arg(ap, int);
+ booleans.push_back(!!value); // force bool
+ }
+}
+
+TEST_F(BuildTokenTest, DoNotAquireToken) {
+ // plan should execute one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat1", &err));
+ ASSERT_EQ("", err);
+
+ // pretend we can't run anything
+ ExpectCanRunMore(1, false);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("stuck [this is a bug]", err);
+
+ EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTokenTest, DoNotStartWithoutToken) {
+ // plan should execute one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat1", &err));
+ ASSERT_EQ("", err);
+
+ // we could run a command but do not have a token for it
+ ExpectCanRunMore(1, true);
+ ExpectAcquireToken(1, false);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("stuck [this is a bug]", err);
+
+ EXPECT_EQ(0u, token_command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTokenTest, CompleteOneStep) {
+ // plan should execute one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat1", &err));
+ ASSERT_EQ("", err);
+
+ // allow running of one command
+ ExpectCanRunMore(1, true);
+ ExpectAcquireToken(1, true);
+ // block and wait for command to finalize
+ ExpectWaitForCommand(1, false);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
+ EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1");
+}
+
+TEST_F(BuildTokenTest, AcquireOneToken) {
+ // plan should execute more than one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat12", &err));
+ ASSERT_EQ("", err);
+
+ // allow running of one command
+ ExpectCanRunMore(3, true, false, false);
+ ExpectAcquireToken(1, true);
+ // block and wait for command to finalize
+ ExpectWaitForCommand(1, false);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("stuck [this is a bug]", err);
+
+ EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
+ // any of the two dependencies could have been executed
+ EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1" ||
+ token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2");
+}
+
+TEST_F(BuildTokenTest, WantTwoTokens) {
+ // plan should execute more than one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat12", &err));
+ ASSERT_EQ("", err);
+
+ // allow running of one command
+ ExpectCanRunMore(3, true, true, false);
+ ExpectAcquireToken(2, true, false);
+ // wait for command to finalize or token to become available
+ ExpectWaitForCommand(1, true);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("stuck [this is a bug]", err);
+
+ EXPECT_EQ(1u, token_command_runner_.commands_ran_.size());
+ // any of the two dependencies could have been executed
+ EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > cat1" ||
+ token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2");
+}
+
+TEST_F(BuildTokenTest, CompleteTwoSteps) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out1: cat in1\n"
+"build out2: cat out1\n"));
+
+ // plan should execute more than one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+
+ // allow running of two commands
+ ExpectCanRunMore(2, true, true);
+ ExpectAcquireToken(2, true, true);
+ // wait for commands to finalize
+ ExpectWaitForCommand(2, false, false);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, token_command_runner_.commands_ran_.size());
+ EXPECT_TRUE(token_command_runner_.commands_ran_[0] == "cat in1 > out1");
+ EXPECT_TRUE(token_command_runner_.commands_ran_[1] == "cat out1 > out2");
+}
+
+TEST_F(BuildTokenTest, TwoCommandsInParallel) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule token-available\n"
+" command = cat $in > $out\n"
+"build out1: token-available in1\n"
+"build out2: token-available in2\n"
+"build out12: cat out1 out2\n"));
+
+ // plan should execute more than one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out12", &err));
+ ASSERT_EQ("", err);
+
+ // 1st command: token available -> allow running
+ // 2nd command: no token available but becomes available later
+ ExpectCanRunMore(4, true, true, true, false);
+ ExpectAcquireToken(3, true, false, true);
+ // 1st call waits for command to finalize or token to become available
+ // 2nd call waits for command to finalize
+ // 3rd call waits for command to finalize
+ ExpectWaitForCommand(3, true, false, false);
+
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("stuck [this is a bug]", err);
+
+ EXPECT_EQ(2u, token_command_runner_.commands_ran_.size());
+ EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > out1" &&
+ token_command_runner_.commands_ran_[1] == "cat in2 > out2") ||
+ (token_command_runner_.commands_ran_[0] == "cat in2 > out2" &&
+ token_command_runner_.commands_ran_[1] == "cat in1 > out1"));
+}
+
+TEST_F(BuildTokenTest, CompleteThreeStepsSerial) {
+ // plan should execute more than one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("cat12", &err));
+ ASSERT_EQ("", err);
+
+ // allow running of all commands
+ ExpectCanRunMore(4, true, true, true, true);
+ ExpectAcquireToken(4, true, false, true, true);
+ // wait for commands to finalize
+ ExpectWaitForCommand(3, true, false, false);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(3u, token_command_runner_.commands_ran_.size());
+ EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > cat1" &&
+ token_command_runner_.commands_ran_[1] == "cat in1 in2 > cat2") ||
+ (token_command_runner_.commands_ran_[0] == "cat in1 in2 > cat2" &&
+ token_command_runner_.commands_ran_[1] == "cat in1 > cat1" ));
+ EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat cat1 cat2 > cat12");
+}
+
+TEST_F(BuildTokenTest, CompleteThreeStepsParallel) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule token-available\n"
+" command = cat $in > $out\n"
+"build out1: token-available in1\n"
+"build out2: token-available in2\n"
+"build out12: cat out1 out2\n"));
+
+ // plan should execute more than one command
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out12", &err));
+ ASSERT_EQ("", err);
+
+ // allow running of all commands
+ ExpectCanRunMore(4, true, true, true, true);
+ ExpectAcquireToken(4, true, false, true, true);
+ // wait for commands to finalize
+ ExpectWaitForCommand(4, true, false, false, false);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(3u, token_command_runner_.commands_ran_.size());
+ EXPECT_TRUE((token_command_runner_.commands_ran_[0] == "cat in1 > out1" &&
+ token_command_runner_.commands_ran_[1] == "cat in2 > out2") ||
+ (token_command_runner_.commands_ran_[0] == "cat in2 > out2" &&
+ token_command_runner_.commands_ran_[1] == "cat in1 > out1"));
+ EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat out1 out2 > out12");
+}
Index: src/exit_status.h
===================================================================
--- src/exit_status.h (nonexistent)
+++ src/exit_status.h (revision 5)
@@ -0,0 +1,25 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_EXIT_STATUS_H_
+#define NINJA_EXIT_STATUS_H_
+
+enum ExitStatus {
+ ExitSuccess,
+ ExitFailure,
+ ExitTokenAvailable,
+ ExitInterrupted,
+};
+
+#endif // NINJA_EXIT_STATUS_H_
Index: src/ninja.cc
===================================================================
--- src/ninja.cc (nonexistent)
+++ src/ninja.cc (revision 5)
@@ -0,0 +1,1458 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <cstdlib>
+
+#ifdef _WIN32
+#include "getopt.h"
+#include <direct.h>
+#include <windows.h>
+#elif defined(_AIX)
+#include "getopt.h"
+#include <unistd.h>
+#else
+#include <getopt.h>
+#include <unistd.h>
+#endif
+
+#include "browse.h"
+#include "build.h"
+#include "build_log.h"
+#include "deps_log.h"
+#include "clean.h"
+#include "debug_flags.h"
+#include "disk_interface.h"
+#include "graph.h"
+#include "graphviz.h"
+#include "manifest_parser.h"
+#include "metrics.h"
+#include "state.h"
+#include "util.h"
+#include "version.h"
+
+using namespace std;
+
+#ifdef _MSC_VER
+// Defined in msvc_helper_main-win32.cc.
+int MSVCHelperMain(int argc, char** argv);
+
+// Defined in minidump-win32.cc.
+void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep);
+#endif
+
+namespace {
+
+struct Tool;
+
+/// Command-line options.
+struct Options {
+ /// Build file to load.
+ const char* input_file;
+
+ /// Directory to change into before running.
+ const char* working_dir;
+
+ /// Tool to run rather than building.
+ const Tool* tool;
+
+ /// Whether duplicate rules for one target should warn or print an error.
+ bool dupe_edges_should_err;
+
+ /// Whether phony cycles should warn or print an error.
+ bool phony_cycle_should_err;
+};
+
+/// The Ninja main() loads up a series of data structures; various tools need
+/// to poke into these, so store them as fields on an object.
+struct NinjaMain : public BuildLogUser {
+ NinjaMain(const char* ninja_command, const BuildConfig& config) :
+ ninja_command_(ninja_command), config_(config) {}
+
+ /// Command line used to run Ninja.
+ const char* ninja_command_;
+
+ /// Build configuration set from flags (e.g. parallelism).
+ const BuildConfig& config_;
+
+ /// Loaded state (rules, nodes).
+ State state_;
+
+ /// Functions for accessing the disk.
+ RealDiskInterface disk_interface_;
+
+ /// The build directory, used for storing the build log etc.
+ string build_dir_;
+
+ BuildLog build_log_;
+ DepsLog deps_log_;
+
+ /// The type of functions that are the entry points to tools (subcommands).
+ typedef int (NinjaMain::*ToolFunc)(const Options*, int, char**);
+
+ /// Get the Node for a given command-line path, handling features like
+ /// spell correction.
+ Node* CollectTarget(const char* cpath, string* err);
+
+ /// CollectTarget for all command-line arguments, filling in \a targets.
+ bool CollectTargetsFromArgs(int argc, char* argv[],
+ vector<Node*>* targets, string* err);
+
+ // The various subcommands, run via "-t XXX".
+ int ToolGraph(const Options* options, int argc, char* argv[]);
+ int ToolQuery(const Options* options, int argc, char* argv[]);
+ int ToolDeps(const Options* options, int argc, char* argv[]);
+ int ToolBrowse(const Options* options, int argc, char* argv[]);
+ int ToolMSVC(const Options* options, int argc, char* argv[]);
+ int ToolTargets(const Options* options, int argc, char* argv[]);
+ int ToolCommands(const Options* options, int argc, char* argv[]);
+ int ToolClean(const Options* options, int argc, char* argv[]);
+ int ToolCleanDead(const Options* options, int argc, char* argv[]);
+ int ToolCompilationDatabase(const Options* options, int argc, char* argv[]);
+ int ToolRecompact(const Options* options, int argc, char* argv[]);
+ int ToolRestat(const Options* options, int argc, char* argv[]);
+ int ToolUrtle(const Options* options, int argc, char** argv);
+ int ToolRules(const Options* options, int argc, char* argv[]);
+
+ /// Open the build log.
+ /// @return LOAD_ERROR on error.
+ bool OpenBuildLog(bool recompact_only = false);
+
+ /// Open the deps log: load it, then open for writing.
+ /// @return LOAD_ERROR on error.
+ bool OpenDepsLog(bool recompact_only = false);
+
+ /// Ensure the build directory exists, creating it if necessary.
+ /// @return false on error.
+ bool EnsureBuildDirExists();
+
+ /// Rebuild the manifest, if necessary.
+ /// Fills in \a err on error.
+ /// @return true if the manifest was rebuilt.
+ bool RebuildManifest(const char* input_file, string* err);
+
+ /// Build the targets listed on the command line.
+ /// @return an exit code.
+ int RunBuild(int argc, char** argv);
+
+ /// Dump the output requested by '-d stats'.
+ void DumpMetrics();
+
+ virtual bool IsPathDead(StringPiece s) const {
+ Node* n = state_.LookupNode(s);
+ if (n && n->in_edge())
+ return false;
+ // Just checking n isn't enough: If an old output is both in the build log
+ // and in the deps log, it will have a Node object in state_. (It will also
+ // have an in edge if one of its inputs is another output that's in the deps
+ // log, but having a deps edge product an output that's input to another deps
+ // edge is rare, and the first recompaction will delete all old outputs from
+ // the deps log, and then a second recompaction will clear the build log,
+ // which seems good enough for this corner case.)
+ // Do keep entries around for files which still exist on disk, for
+ // generators that want to use this information.
+ string err;
+ TimeStamp mtime = disk_interface_.Stat(s.AsString(), &err);
+ if (mtime == -1)
+ Error("%s", err.c_str()); // Log and ignore Stat() errors.
+ return mtime == 0;
+ }
+};
+
+/// Subtools, accessible via "-t foo".
+struct Tool {
+ /// Short name of the tool.
+ const char* name;
+
+ /// Description (shown in "-t list").
+ const char* desc;
+
+ /// When to run the tool.
+ enum {
+ /// Run after parsing the command-line flags and potentially changing
+ /// the current working directory (as early as possible).
+ RUN_AFTER_FLAGS,
+
+ /// Run after loading build.ninja.
+ RUN_AFTER_LOAD,
+
+ /// Run after loading the build/deps logs.
+ RUN_AFTER_LOGS,
+ } when;
+
+ /// Implementation of the tool.
+ NinjaMain::ToolFunc func;
+};
+
+/// Print usage information.
+void Usage(const BuildConfig& config) {
+ fprintf(stderr,
+"usage: ninja [options] [targets...]\n"
+"\n"
+"if targets are unspecified, builds the 'default' target (see manual).\n"
+"\n"
+"options:\n"
+" --version print ninja version (\"%s\")\n"
+" -v, --verbose show all command lines while building\n"
+"\n"
+" -C DIR change to DIR before doing anything else\n"
+" -f FILE specify input build file [default=build.ninja]\n"
+"\n"
+" -j N run N jobs in parallel (0 means infinity) [default=%d on this system]\n"
+" -k N keep going until N jobs fail (0 means infinity) [default=1]\n"
+" -l N do not start new jobs if the load average is greater than N\n"
+" -n dry run (don't run commands but act like they succeeded)\n"
+"\n"
+" -d MODE enable debugging (use '-d list' to list modes)\n"
+" -t TOOL run a subtool (use '-t list' to list subtools)\n"
+" terminates toplevel options; further flags are passed to the tool\n"
+" -w FLAG adjust warnings (use '-w list' to list warnings)\n",
+ kNinjaVersion, config.parallelism);
+}
+
+/// Choose a default value for the -j (parallelism) flag.
+int GuessParallelism() {
+ switch (int processors = GetProcessorCount()) {
+ case 0:
+ case 1:
+ return 2;
+ case 2:
+ return 3;
+ default:
+ return processors + 2;
+ }
+}
+
+/// Rebuild the build manifest, if necessary.
+/// Returns true if the manifest was rebuilt.
+bool NinjaMain::RebuildManifest(const char* input_file, string* err) {
+ string path = input_file;
+ uint64_t slash_bits; // Unused because this path is only used for lookup.
+ if (!CanonicalizePath(&path, &slash_bits, err))
+ return false;
+ Node* node = state_.LookupNode(path);
+ if (!node)
+ return false;
+
+ Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
+ if (!builder.AddTarget(node, err))
+ return false;
+
+ if (builder.AlreadyUpToDate())
+ return false; // Not an error, but we didn't rebuild.
+
+ if (!builder.Build(err))
+ return false;
+
+ // The manifest was only rebuilt if it is now dirty (it may have been cleaned
+ // by a restat).
+ if (!node->dirty()) {
+ // Reset the state to prevent problems like
+ // https://github.com/ninja-build/ninja/issues/874
+ state_.Reset();
+ return false;
+ }
+
+ return true;
+}
+
+Node* NinjaMain::CollectTarget(const char* cpath, string* err) {
+ string path = cpath;
+ uint64_t slash_bits;
+ if (!CanonicalizePath(&path, &slash_bits, err))
+ return NULL;
+
+ // Special syntax: "foo.cc^" means "the first output of foo.cc".
+ bool first_dependent = false;
+ if (!path.empty() && path[path.size() - 1] == '^') {
+ path.resize(path.size() - 1);
+ first_dependent = true;
+ }
+
+ Node* node = state_.LookupNode(path);
+ if (node) {
+ if (first_dependent) {
+ if (node->out_edges().empty()) {
+ *err = "'" + path + "' has no out edge";
+ return NULL;
+ }
+ Edge* edge = node->out_edges()[0];
+ if (edge->outputs_.empty()) {
+ edge->Dump();
+ Fatal("edge has no outputs");
+ }
+ node = edge->outputs_[0];
+ }
+ return node;
+ } else {
+ *err =
+ "unknown target '" + Node::PathDecanonicalized(path, slash_bits) + "'";
+ if (path == "clean") {
+ *err += ", did you mean 'ninja -t clean'?";
+ } else if (path == "help") {
+ *err += ", did you mean 'ninja -h'?";
+ } else {
+ Node* suggestion = state_.SpellcheckNode(path);
+ if (suggestion) {
+ *err += ", did you mean '" + suggestion->path() + "'?";
+ }
+ }
+ return NULL;
+ }
+}
+
+bool NinjaMain::CollectTargetsFromArgs(int argc, char* argv[],
+ vector<Node*>* targets, string* err) {
+ if (argc == 0) {
+ *targets = state_.DefaultNodes(err);
+ return err->empty();
+ }
+
+ for (int i = 0; i < argc; ++i) {
+ Node* node = CollectTarget(argv[i], err);
+ if (node == NULL)
+ return false;
+ targets->push_back(node);
+ }
+ return true;
+}
+
+int NinjaMain::ToolGraph(const Options* options, int argc, char* argv[]) {
+ vector<Node*> nodes;
+ string err;
+ if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ GraphViz graph(&state_, &disk_interface_);
+ graph.Start();
+ for (vector<Node*>::const_iterator n = nodes.begin(); n != nodes.end(); ++n)
+ graph.AddTarget(*n);
+ graph.Finish();
+
+ return 0;
+}
+
+int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) {
+ if (argc == 0) {
+ Error("expected a target to query");
+ return 1;
+ }
+
+ DyndepLoader dyndep_loader(&state_, &disk_interface_);
+
+ for (int i = 0; i < argc; ++i) {
+ string err;
+ Node* node = CollectTarget(argv[i], &err);
+ if (!node) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ printf("%s:\n", node->path().c_str());
+ if (Edge* edge = node->in_edge()) {
+ if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) {
+ if (!dyndep_loader.LoadDyndeps(edge->dyndep_, &err)) {
+ Warning("%s\n", err.c_str());
+ }
+ }
+ printf(" input: %s\n", edge->rule_->name().c_str());
+ for (int in = 0; in < (int)edge->inputs_.size(); in++) {
+ const char* label = "";
+ if (edge->is_implicit(in))
+ label = "| ";
+ else if (edge->is_order_only(in))
+ label = "|| ";
+ printf(" %s%s\n", label, edge->inputs_[in]->path().c_str());
+ }
+ }
+ printf(" outputs:\n");
+ for (vector<Edge*>::const_iterator edge = node->out_edges().begin();
+ edge != node->out_edges().end(); ++edge) {
+ for (vector<Node*>::iterator out = (*edge)->outputs_.begin();
+ out != (*edge)->outputs_.end(); ++out) {
+ printf(" %s\n", (*out)->path().c_str());
+ }
+ }
+ }
+ return 0;
+}
+
+#if defined(NINJA_HAVE_BROWSE)
+int NinjaMain::ToolBrowse(const Options* options, int argc, char* argv[]) {
+ RunBrowsePython(&state_, ninja_command_, options->input_file, argc, argv);
+ // If we get here, the browse failed.
+ return 1;
+}
+#else
+int NinjaMain::ToolBrowse(const Options*, int, char**) {
+ Fatal("browse tool not supported on this platform");
+ return 1;
+}
+#endif
+
+#if defined(_MSC_VER)
+int NinjaMain::ToolMSVC(const Options* options, int argc, char* argv[]) {
+ // Reset getopt: push one argument onto the front of argv, reset optind.
+ argc++;
+ argv--;
+ optind = 0;
+ return MSVCHelperMain(argc, argv);
+}
+#endif
+
+int ToolTargetsList(const vector<Node*>& nodes, int depth, int indent) {
+ for (vector<Node*>::const_iterator n = nodes.begin();
+ n != nodes.end();
+ ++n) {
+ for (int i = 0; i < indent; ++i)
+ printf(" ");
+ const char* target = (*n)->path().c_str();
+ if ((*n)->in_edge()) {
+ printf("%s: %s\n", target, (*n)->in_edge()->rule_->name().c_str());
+ if (depth > 1 || depth <= 0)
+ ToolTargetsList((*n)->in_edge()->inputs_, depth - 1, indent + 1);
+ } else {
+ printf("%s\n", target);
+ }
+ }
+ return 0;
+}
+
+int ToolTargetsSourceList(State* state) {
+ for (vector<Edge*>::iterator e = state->edges_.begin();
+ e != state->edges_.end(); ++e) {
+ for (vector<Node*>::iterator inps = (*e)->inputs_.begin();
+ inps != (*e)->inputs_.end(); ++inps) {
+ if (!(*inps)->in_edge())
+ printf("%s\n", (*inps)->path().c_str());
+ }
+ }
+ return 0;
+}
+
+int ToolTargetsList(State* state, const string& rule_name) {
+ set<string> rules;
+
+ // Gather the outputs.
+ for (vector<Edge*>::iterator e = state->edges_.begin();
+ e != state->edges_.end(); ++e) {
+ if ((*e)->rule_->name() == rule_name) {
+ for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
+ out_node != (*e)->outputs_.end(); ++out_node) {
+ rules.insert((*out_node)->path());
+ }
+ }
+ }
+
+ // Print them.
+ for (set<string>::const_iterator i = rules.begin();
+ i != rules.end(); ++i) {
+ printf("%s\n", (*i).c_str());
+ }
+
+ return 0;
+}
+
+int ToolTargetsList(State* state) {
+ for (vector<Edge*>::iterator e = state->edges_.begin();
+ e != state->edges_.end(); ++e) {
+ for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
+ out_node != (*e)->outputs_.end(); ++out_node) {
+ printf("%s: %s\n",
+ (*out_node)->path().c_str(),
+ (*e)->rule_->name().c_str());
+ }
+ }
+ return 0;
+}
+
+int NinjaMain::ToolDeps(const Options* options, int argc, char** argv) {
+ vector<Node*> nodes;
+ if (argc == 0) {
+ for (vector<Node*>::const_iterator ni = deps_log_.nodes().begin();
+ ni != deps_log_.nodes().end(); ++ni) {
+ if (deps_log_.IsDepsEntryLiveFor(*ni))
+ nodes.push_back(*ni);
+ }
+ } else {
+ string err;
+ if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+ }
+
+ RealDiskInterface disk_interface;
+ for (vector<Node*>::iterator it = nodes.begin(), end = nodes.end();
+ it != end; ++it) {
+ DepsLog::Deps* deps = deps_log_.GetDeps(*it);
+ if (!deps) {
+ printf("%s: deps not found\n", (*it)->path().c_str());
+ continue;
+ }
+
+ string err;
+ TimeStamp mtime = disk_interface.Stat((*it)->path(), &err);
+ if (mtime == -1)
+ Error("%s", err.c_str()); // Log and ignore Stat() errors;
+ printf("%s: #deps %d, deps mtime %" PRId64 " (%s)\n",
+ (*it)->path().c_str(), deps->node_count, deps->mtime,
+ (!mtime || mtime > deps->mtime ? "STALE":"VALID"));
+ for (int i = 0; i < deps->node_count; ++i)
+ printf(" %s\n", deps->nodes[i]->path().c_str());
+ printf("\n");
+ }
+
+ return 0;
+}
+
+int NinjaMain::ToolTargets(const Options* options, int argc, char* argv[]) {
+ int depth = 1;
+ if (argc >= 1) {
+ string mode = argv[0];
+ if (mode == "rule") {
+ string rule;
+ if (argc > 1)
+ rule = argv[1];
+ if (rule.empty())
+ return ToolTargetsSourceList(&state_);
+ else
+ return ToolTargetsList(&state_, rule);
+ } else if (mode == "depth") {
+ if (argc > 1)
+ depth = atoi(argv[1]);
+ } else if (mode == "all") {
+ return ToolTargetsList(&state_);
+ } else {
+ const char* suggestion =
+ SpellcheckString(mode.c_str(), "rule", "depth", "all", NULL);
+ if (suggestion) {
+ Error("unknown target tool mode '%s', did you mean '%s'?",
+ mode.c_str(), suggestion);
+ } else {
+ Error("unknown target tool mode '%s'", mode.c_str());
+ }
+ return 1;
+ }
+ }
+
+ string err;
+ vector<Node*> root_nodes = state_.RootNodes(&err);
+ if (err.empty()) {
+ return ToolTargetsList(root_nodes, depth, 0);
+ } else {
+ Error("%s", err.c_str());
+ return 1;
+ }
+}
+
+int NinjaMain::ToolRules(const Options* options, int argc, char* argv[]) {
+ // Parse options.
+
+ // The rules tool uses getopt, and expects argv[0] to contain the name of
+ // the tool, i.e. "rules".
+ argc++;
+ argv--;
+
+ bool print_description = false;
+
+ optind = 1;
+ int opt;
+ while ((opt = getopt(argc, argv, const_cast<char*>("hd"))) != -1) {
+ switch (opt) {
+ case 'd':
+ print_description = true;
+ break;
+ case 'h':
+ default:
+ printf("usage: ninja -t rules [options]\n"
+ "\n"
+ "options:\n"
+ " -d also print the description of the rule\n"
+ " -h print this message\n"
+ );
+ return 1;
+ }
+ }
+ argv += optind;
+ argc -= optind;
+
+ // Print rules
+
+ typedef map<string, const Rule*> Rules;
+ const Rules& rules = state_.bindings_.GetRules();
+ for (Rules::const_iterator i = rules.begin(); i != rules.end(); ++i) {
+ printf("%s", i->first.c_str());
+ if (print_description) {
+ const Rule* rule = i->second;
+ const EvalString* description = rule->GetBinding("description");
+ if (description != NULL) {
+ printf(": %s", description->Unparse().c_str());
+ }
+ }
+ printf("\n");
+ }
+ return 0;
+}
+
+enum PrintCommandMode { PCM_Single, PCM_All };
+void PrintCommands(Edge* edge, set<Edge*>* seen, PrintCommandMode mode) {
+ if (!edge)
+ return;
+ if (!seen->insert(edge).second)
+ return;
+
+ if (mode == PCM_All) {
+ for (vector<Node*>::iterator in = edge->inputs_.begin();
+ in != edge->inputs_.end(); ++in)
+ PrintCommands((*in)->in_edge(), seen, mode);
+ }
+
+ if (!edge->is_phony())
+ puts(edge->EvaluateCommand().c_str());
+}
+
+int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) {
+ // The clean tool uses getopt, and expects argv[0] to contain the name of
+ // the tool, i.e. "commands".
+ ++argc;
+ --argv;
+
+ PrintCommandMode mode = PCM_All;
+
+ optind = 1;
+ int opt;
+ while ((opt = getopt(argc, argv, const_cast<char*>("hs"))) != -1) {
+ switch (opt) {
+ case 's':
+ mode = PCM_Single;
+ break;
+ case 'h':
+ default:
+ printf("usage: ninja -t commands [options] [targets]\n"
+"\n"
+"options:\n"
+" -s only print the final command to build [target], not the whole chain\n"
+ );
+ return 1;
+ }
+ }
+ argv += optind;
+ argc -= optind;
+
+ vector<Node*> nodes;
+ string err;
+ if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ set<Edge*> seen;
+ for (vector<Node*>::iterator in = nodes.begin(); in != nodes.end(); ++in)
+ PrintCommands((*in)->in_edge(), &seen, mode);
+
+ return 0;
+}
+
+int NinjaMain::ToolClean(const Options* options, int argc, char* argv[]) {
+ // The clean tool uses getopt, and expects argv[0] to contain the name of
+ // the tool, i.e. "clean".
+ argc++;
+ argv--;
+
+ bool generator = false;
+ bool clean_rules = false;
+
+ optind = 1;
+ int opt;
+ while ((opt = getopt(argc, argv, const_cast<char*>("hgr"))) != -1) {
+ switch (opt) {
+ case 'g':
+ generator = true;
+ break;
+ case 'r':
+ clean_rules = true;
+ break;
+ case 'h':
+ default:
+ printf("usage: ninja -t clean [options] [targets]\n"
+"\n"
+"options:\n"
+" -g also clean files marked as ninja generator output\n"
+" -r interpret targets as a list of rules to clean instead\n"
+ );
+ return 1;
+ }
+ }
+ argv += optind;
+ argc -= optind;
+
+ if (clean_rules && argc == 0) {
+ Error("expected a rule to clean");
+ return 1;
+ }
+
+ Cleaner cleaner(&state_, config_, &disk_interface_);
+ if (argc >= 1) {
+ if (clean_rules)
+ return cleaner.CleanRules(argc, argv);
+ else
+ return cleaner.CleanTargets(argc, argv);
+ } else {
+ return cleaner.CleanAll(generator);
+ }
+}
+
+int NinjaMain::ToolCleanDead(const Options* options, int argc, char* argv[]) {
+ Cleaner cleaner(&state_, config_, &disk_interface_);
+ return cleaner.CleanDead(build_log_.entries());
+}
+
+void EncodeJSONString(const char *str) {
+ while (*str) {
+ if (*str == '"' || *str == '\\')
+ putchar('\\');
+ putchar(*str);
+ str++;
+ }
+}
+
+enum EvaluateCommandMode {
+ ECM_NORMAL,
+ ECM_EXPAND_RSPFILE
+};
+std::string EvaluateCommandWithRspfile(const Edge* edge,
+ const EvaluateCommandMode mode) {
+ string command = edge->EvaluateCommand();
+ if (mode == ECM_NORMAL)
+ return command;
+
+ string rspfile = edge->GetUnescapedRspfile();
+ if (rspfile.empty())
+ return command;
+
+ size_t index = command.find(rspfile);
+ if (index == 0 || index == string::npos || command[index - 1] != '@')
+ return command;
+
+ string rspfile_content = edge->GetBinding("rspfile_content");
+ size_t newline_index = 0;
+ while ((newline_index = rspfile_content.find('\n', newline_index)) !=
+ string::npos) {
+ rspfile_content.replace(newline_index, 1, 1, ' ');
+ ++newline_index;
+ }
+ command.replace(index - 1, rspfile.length() + 1, rspfile_content);
+ return command;
+}
+
+void printCompdb(const char* const directory, const Edge* const edge,
+ const EvaluateCommandMode eval_mode) {
+ printf("\n {\n \"directory\": \"");
+ EncodeJSONString(directory);
+ printf("\",\n \"command\": \"");
+ EncodeJSONString(EvaluateCommandWithRspfile(edge, eval_mode).c_str());
+ printf("\",\n \"file\": \"");
+ EncodeJSONString(edge->inputs_[0]->path().c_str());
+ printf("\",\n \"output\": \"");
+ EncodeJSONString(edge->outputs_[0]->path().c_str());
+ printf("\"\n }");
+}
+
+int NinjaMain::ToolCompilationDatabase(const Options* options, int argc,
+ char* argv[]) {
+ // The compdb tool uses getopt, and expects argv[0] to contain the name of
+ // the tool, i.e. "compdb".
+ argc++;
+ argv--;
+
+ EvaluateCommandMode eval_mode = ECM_NORMAL;
+
+ optind = 1;
+ int opt;
+ while ((opt = getopt(argc, argv, const_cast<char*>("hx"))) != -1) {
+ switch(opt) {
+ case 'x':
+ eval_mode = ECM_EXPAND_RSPFILE;
+ break;
+
+ case 'h':
+ default:
+ printf(
+ "usage: ninja -t compdb [options] [rules]\n"
+ "\n"
+ "options:\n"
+ " -x expand @rspfile style response file invocations\n"
+ );
+ return 1;
+ }
+ }
+ argv += optind;
+ argc -= optind;
+
+ bool first = true;
+ vector<char> cwd;
+ char* success = NULL;
+
+ do {
+ cwd.resize(cwd.size() + 1024);
+ errno = 0;
+ success = getcwd(&cwd[0], cwd.size());
+ } while (!success && errno == ERANGE);
+ if (!success) {
+ Error("cannot determine working directory: %s", strerror(errno));
+ return 1;
+ }
+
+ putchar('[');
+ for (vector<Edge*>::iterator e = state_.edges_.begin();
+ e != state_.edges_.end(); ++e) {
+ if ((*e)->inputs_.empty())
+ continue;
+ if (argc == 0) {
+ if (!first) {
+ putchar(',');
+ }
+ printCompdb(&cwd[0], *e, eval_mode);
+ first = false;
+ } else {
+ for (int i = 0; i != argc; ++i) {
+ if ((*e)->rule_->name() == argv[i]) {
+ if (!first) {
+ putchar(',');
+ }
+ printCompdb(&cwd[0], *e, eval_mode);
+ first = false;
+ }
+ }
+ }
+ }
+
+ puts("\n]");
+ return 0;
+}
+
+int NinjaMain::ToolRecompact(const Options* options, int argc, char* argv[]) {
+ if (!EnsureBuildDirExists())
+ return 1;
+
+ if (OpenBuildLog(/*recompact_only=*/true) == LOAD_ERROR ||
+ OpenDepsLog(/*recompact_only=*/true) == LOAD_ERROR)
+ return 1;
+
+ return 0;
+}
+
+int NinjaMain::ToolRestat(const Options* options, int argc, char* argv[]) {
+ // The restat tool uses getopt, and expects argv[0] to contain the name of the
+ // tool, i.e. "restat"
+ argc++;
+ argv--;
+
+ optind = 1;
+ int opt;
+ while ((opt = getopt(argc, argv, const_cast<char*>("h"))) != -1) {
+ switch (opt) {
+ case 'h':
+ default:
+ printf("usage: ninja -t restat [outputs]\n");
+ return 1;
+ }
+ }
+ argv += optind;
+ argc -= optind;
+
+ if (!EnsureBuildDirExists())
+ return 1;
+
+ string log_path = ".ninja_log";
+ if (!build_dir_.empty())
+ log_path = build_dir_ + "/" + log_path;
+
+ string err;
+ const LoadStatus status = build_log_.Load(log_path, &err);
+ if (status == LOAD_ERROR) {
+ Error("loading build log %s: %s", log_path.c_str(), err.c_str());
+ return EXIT_FAILURE;
+ }
+ if (status == LOAD_NOT_FOUND) {
+ // Nothing to restat, ignore this
+ return EXIT_SUCCESS;
+ }
+ if (!err.empty()) {
+ // Hack: Load() can return a warning via err by returning LOAD_SUCCESS.
+ Warning("%s", err.c_str());
+ err.clear();
+ }
+
+ bool success = build_log_.Restat(log_path, disk_interface_, argc, argv, &err);
+ if (!success) {
+ Error("failed recompaction: %s", err.c_str());
+ return EXIT_FAILURE;
+ }
+
+ if (!config_.dry_run) {
+ if (!build_log_.OpenForWrite(log_path, *this, &err)) {
+ Error("opening build log: %s", err.c_str());
+ return EXIT_FAILURE;
+ }
+ }
+
+ return EXIT_SUCCESS;
+}
+
+int NinjaMain::ToolUrtle(const Options* options, int argc, char** argv) {
+ // RLE encoded.
+ const char* urtle =
+" 13 ,3;2!2;\n8 ,;<11!;\n5 `'<10!(2`'2!\n11 ,6;, `\\. `\\9 .,c13$ec,.\n6 "
+",2;11!>; `. ,;!2> .e8$2\".2 \"?7$e.\n <:<8!'` 2.3,.2` ,3!' ;,(?7\";2!2'<"
+"; `?6$PF ,;,\n2 `'4!8;<!3'`2 3! ;,`'2`2'3!;4!`2.`!;2 3,2 .<!2'`).\n5 3`5"
+"'2`9 `!2 `4!><3;5! J2$b,`!>;2!:2!`,d?b`!>\n26 `'-;,(<9!> $F3 )3.:!.2 d\""
+"2 ) !>\n30 7`2'<3!- \"=-='5 .2 `2-=\",!>\n25 .ze9$er2 .,cd16$bc.'\n22 .e"
+"14$,26$.\n21 z45$c .\n20 J50$c\n20 14$P\"`?34$b\n20 14$ dbc `2\"?22$?7$c"
+"\n20 ?18$c.6 4\"8?4\" c8$P\n9 .2,.8 \"20$c.3 ._14 J9$\n .2,2c9$bec,.2 `?"
+"21$c.3`4%,3%,3 c8$P\"\n22$c2 2\"?21$bc2,.2` .2,c7$P2\",cb\n23$b bc,.2\"2"
+"?14$2F2\"5?2\",J5$P\" ,zd3$\n24$ ?$3?%3 `2\"2?12$bcucd3$P3\"2 2=7$\n23$P"
+"\" ,3;<5!>2;,. `4\"6?2\"2 ,9;, `\"?2$\n";
+ int count = 0;
+ for (const char* p = urtle; *p; p++) {
+ if ('0' <= *p && *p <= '9') {
+ count = count*10 + *p - '0';
+ } else {
+ for (int i = 0; i < max(count, 1); ++i)
+ printf("%c", *p);
+ count = 0;
+ }
+ }
+ return 0;
+}
+
+/// Find the function to execute for \a tool_name and return it via \a func.
+/// Returns a Tool, or NULL if Ninja should exit.
+const Tool* ChooseTool(const string& tool_name) {
+ static const Tool kTools[] = {
+ { "browse", "browse dependency graph in a web browser",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolBrowse },
+#if defined(_MSC_VER)
+ { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)",
+ Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolMSVC },
+#endif
+ { "clean", "clean built files",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolClean },
+ { "commands", "list all commands required to rebuild given targets",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCommands },
+ { "deps", "show dependencies stored in the deps log",
+ Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps },
+ { "graph", "output graphviz dot file for targets",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolGraph },
+ { "query", "show inputs/outputs for a path",
+ Tool::RUN_AFTER_LOGS, &NinjaMain::ToolQuery },
+ { "targets", "list targets by their rule or depth in the DAG",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolTargets },
+ { "compdb", "dump JSON compilation database to stdout",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabase },
+ { "recompact", "recompacts ninja-internal data structures",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRecompact },
+ { "restat", "restats all outputs in the build log",
+ Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolRestat },
+ { "rules", "list all rules",
+ Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRules },
+ { "cleandead", "clean built files that are no longer produced by the manifest",
+ Tool::RUN_AFTER_LOGS, &NinjaMain::ToolCleanDead },
+ { "urtle", NULL,
+ Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolUrtle },
+ { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL }
+ };
+
+ if (tool_name == "list") {
+ printf("ninja subtools:\n");
+ for (const Tool* tool = &kTools[0]; tool->name; ++tool) {
+ if (tool->desc)
+ printf("%10s %s\n", tool->name, tool->desc);
+ }
+ return NULL;
+ }
+
+ for (const Tool* tool = &kTools[0]; tool->name; ++tool) {
+ if (tool->name == tool_name)
+ return tool;
+ }
+
+ vector<const char*> words;
+ for (const Tool* tool = &kTools[0]; tool->name; ++tool)
+ words.push_back(tool->name);
+ const char* suggestion = SpellcheckStringV(tool_name, words);
+ if (suggestion) {
+ Fatal("unknown tool '%s', did you mean '%s'?",
+ tool_name.c_str(), suggestion);
+ } else {
+ Fatal("unknown tool '%s'", tool_name.c_str());
+ }
+ return NULL; // Not reached.
+}
+
+/// Enable a debugging mode. Returns false if Ninja should exit instead
+/// of continuing.
+bool DebugEnable(const string& name) {
+ if (name == "list") {
+ printf("debugging modes:\n"
+" stats print operation counts/timing info\n"
+" explain explain what caused a command to execute\n"
+" keepdepfile don't delete depfiles after they're read by ninja\n"
+" keeprsp don't delete @response files on success\n"
+#ifdef _WIN32
+" nostatcache don't batch stat() calls per directory and cache them\n"
+#endif
+"multiple modes can be enabled via -d FOO -d BAR\n");
+ return false;
+ } else if (name == "stats") {
+ g_metrics = new Metrics;
+ return true;
+ } else if (name == "explain") {
+ g_explaining = true;
+ return true;
+ } else if (name == "keepdepfile") {
+ g_keep_depfile = true;
+ return true;
+ } else if (name == "keeprsp") {
+ g_keep_rsp = true;
+ return true;
+ } else if (name == "nostatcache") {
+ g_experimental_statcache = false;
+ return true;
+ } else {
+ const char* suggestion =
+ SpellcheckString(name.c_str(),
+ "stats", "explain", "keepdepfile", "keeprsp",
+ "nostatcache", NULL);
+ if (suggestion) {
+ Error("unknown debug setting '%s', did you mean '%s'?",
+ name.c_str(), suggestion);
+ } else {
+ Error("unknown debug setting '%s'", name.c_str());
+ }
+ return false;
+ }
+}
+
+/// Set a warning flag. Returns false if Ninja should exit instead of
+/// continuing.
+bool WarningEnable(const string& name, Options* options) {
+ if (name == "list") {
+ printf("warning flags:\n"
+" dupbuild={err,warn} multiple build lines for one target\n"
+" phonycycle={err,warn} phony build statement references itself\n"
+ );
+ return false;
+ } else if (name == "dupbuild=err") {
+ options->dupe_edges_should_err = true;
+ return true;
+ } else if (name == "dupbuild=warn") {
+ options->dupe_edges_should_err = false;
+ return true;
+ } else if (name == "phonycycle=err") {
+ options->phony_cycle_should_err = true;
+ return true;
+ } else if (name == "phonycycle=warn") {
+ options->phony_cycle_should_err = false;
+ return true;
+ } else if (name == "depfilemulti=err" ||
+ name == "depfilemulti=warn") {
+ Warning("deprecated warning 'depfilemulti'");
+ return true;
+ } else {
+ const char* suggestion =
+ SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn",
+ "phonycycle=err", "phonycycle=warn", NULL);
+ if (suggestion) {
+ Error("unknown warning flag '%s', did you mean '%s'?",
+ name.c_str(), suggestion);
+ } else {
+ Error("unknown warning flag '%s'", name.c_str());
+ }
+ return false;
+ }
+}
+
+bool NinjaMain::OpenBuildLog(bool recompact_only) {
+ string log_path = ".ninja_log";
+ if (!build_dir_.empty())
+ log_path = build_dir_ + "/" + log_path;
+
+ string err;
+ const LoadStatus status = build_log_.Load(log_path, &err);
+ if (status == LOAD_ERROR) {
+ Error("loading build log %s: %s", log_path.c_str(), err.c_str());
+ return false;
+ }
+ if (!err.empty()) {
+ // Hack: Load() can return a warning via err by returning LOAD_SUCCESS.
+ Warning("%s", err.c_str());
+ err.clear();
+ }
+
+ if (recompact_only) {
+ if (status == LOAD_NOT_FOUND) {
+ return true;
+ }
+ bool success = build_log_.Recompact(log_path, *this, &err);
+ if (!success)
+ Error("failed recompaction: %s", err.c_str());
+ return success;
+ }
+
+ if (!config_.dry_run) {
+ if (!build_log_.OpenForWrite(log_path, *this, &err)) {
+ Error("opening build log: %s", err.c_str());
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/// Open the deps log: load it, then open for writing.
+/// @return false on error.
+bool NinjaMain::OpenDepsLog(bool recompact_only) {
+ string path = ".ninja_deps";
+ if (!build_dir_.empty())
+ path = build_dir_ + "/" + path;
+
+ string err;
+ const LoadStatus status = deps_log_.Load(path, &state_, &err);
+ if (status == LOAD_ERROR) {
+ Error("loading deps log %s: %s", path.c_str(), err.c_str());
+ return false;
+ }
+ if (!err.empty()) {
+ // Hack: Load() can return a warning via err by returning LOAD_SUCCESS.
+ Warning("%s", err.c_str());
+ err.clear();
+ }
+
+ if (recompact_only) {
+ if (status == LOAD_NOT_FOUND) {
+ return true;
+ }
+ bool success = deps_log_.Recompact(path, &err);
+ if (!success)
+ Error("failed recompaction: %s", err.c_str());
+ return success;
+ }
+
+ if (!config_.dry_run) {
+ if (!deps_log_.OpenForWrite(path, &err)) {
+ Error("opening deps log: %s", err.c_str());
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void NinjaMain::DumpMetrics() {
+ g_metrics->Report();
+
+ printf("\n");
+ int count = (int)state_.paths_.size();
+ int buckets = (int)state_.paths_.bucket_count();
+ printf("path->node hash load %.2f (%d entries / %d buckets)\n",
+ count / (double) buckets, count, buckets);
+}
+
+bool NinjaMain::EnsureBuildDirExists() {
+ build_dir_ = state_.bindings_.LookupVariable("builddir");
+ if (!build_dir_.empty() && !config_.dry_run) {
+ if (!disk_interface_.MakeDirs(build_dir_ + "/.") && errno != EEXIST) {
+ Error("creating build directory %s: %s",
+ build_dir_.c_str(), strerror(errno));
+ return false;
+ }
+ }
+ return true;
+}
+
+int NinjaMain::RunBuild(int argc, char** argv) {
+ string err;
+ vector<Node*> targets;
+ if (!CollectTargetsFromArgs(argc, argv, &targets, &err)) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ disk_interface_.AllowStatCache(g_experimental_statcache);
+
+ Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
+ for (size_t i = 0; i < targets.size(); ++i) {
+ if (!builder.AddTarget(targets[i], &err)) {
+ if (!err.empty()) {
+ Error("%s", err.c_str());
+ return 1;
+ } else {
+ // Added a target that is already up-to-date; not really
+ // an error.
+ }
+ }
+ }
+
+ // Make sure restat rules do not see stale timestamps.
+ disk_interface_.AllowStatCache(false);
+
+ if (builder.AlreadyUpToDate()) {
+ printf("ninja: no work to do.\n");
+ return 0;
+ }
+
+ if (!builder.Build(&err)) {
+ printf("ninja: build stopped: %s.\n", err.c_str());
+ if (err.find("interrupted by user") != string::npos) {
+ return 2;
+ }
+ return 1;
+ }
+
+ return 0;
+}
+
+#ifdef _MSC_VER
+
+/// This handler processes fatal crashes that you can't catch
+/// Test example: C++ exception in a stack-unwind-block
+/// Real-world example: ninja launched a compiler to process a tricky
+/// C++ input file. The compiler got itself into a state where it
+/// generated 3 GB of output and caused ninja to crash.
+void TerminateHandler() {
+ CreateWin32MiniDump(NULL);
+ Fatal("terminate handler called");
+}
+
+/// On Windows, we want to prevent error dialogs in case of exceptions.
+/// This function handles the exception, and writes a minidump.
+int ExceptionFilter(unsigned int code, struct _EXCEPTION_POINTERS *ep) {
+ Error("exception: 0x%X", code); // e.g. EXCEPTION_ACCESS_VIOLATION
+ fflush(stderr);
+ CreateWin32MiniDump(ep);
+ return EXCEPTION_EXECUTE_HANDLER;
+}
+
+#endif // _MSC_VER
+
+/// Parse argv for command-line options.
+/// Returns an exit code, or -1 if Ninja should continue.
+int ReadFlags(int* argc, char*** argv,
+ Options* options, BuildConfig* config) {
+ config->parallelism = GuessParallelism();
+
+ enum { OPT_VERSION = 1 };
+ const option kLongOptions[] = {
+ { "help", no_argument, NULL, 'h' },
+ { "version", no_argument, NULL, OPT_VERSION },
+ { "verbose", no_argument, NULL, 'v' },
+ { NULL, 0, NULL, 0 }
+ };
+
+ int opt;
+ while (!options->tool &&
+ (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vw:C:h", kLongOptions,
+ NULL)) != -1) {
+ switch (opt) {
+ case 'd':
+ if (!DebugEnable(optarg))
+ return 1;
+ break;
+ case 'f':
+ options->input_file = optarg;
+ break;
+ case 'j': {
+ char* end;
+ int value = strtol(optarg, &end, 10);
+ if (*end != 0 || value < 0)
+ Fatal("invalid -j parameter");
+
+ // We want to run N jobs in parallel. For N = 0, INT_MAX
+ // is close enough to infinite for most sane builds.
+ config->parallelism = value > 0 ? value : INT_MAX;
+ config->parallelism_from_cmdline = true;
+ break;
+ }
+ case 'k': {
+ char* end;
+ int value = strtol(optarg, &end, 10);
+ if (*end != 0)
+ Fatal("-k parameter not numeric; did you mean -k 0?");
+
+ // We want to go until N jobs fail, which means we should allow
+ // N failures and then stop. For N <= 0, INT_MAX is close enough
+ // to infinite for most sane builds.
+ config->failures_allowed = value > 0 ? value : INT_MAX;
+ break;
+ }
+ case 'l': {
+ char* end;
+ double value = strtod(optarg, &end);
+ if (end == optarg)
+ Fatal("-l parameter not numeric: did you mean -l 0.0?");
+ config->max_load_average = value;
+ break;
+ }
+ case 'n':
+ config->dry_run = true;
+ break;
+ case 't':
+ options->tool = ChooseTool(optarg);
+ if (!options->tool)
+ return 0;
+ break;
+ case 'v':
+ config->verbosity = BuildConfig::VERBOSE;
+ break;
+ case 'w':
+ if (!WarningEnable(optarg, options))
+ return 1;
+ break;
+ case 'C':
+ options->working_dir = optarg;
+ break;
+ case OPT_VERSION:
+ printf("%s\n", kNinjaVersion);
+ return 0;
+ case 'h':
+ default:
+ Usage(*config);
+ return 1;
+ }
+ }
+ *argv += optind;
+ *argc -= optind;
+
+ return -1;
+}
+
+NORETURN void real_main(int argc, char** argv) {
+ // Use exit() instead of return in this function to avoid potentially
+ // expensive cleanup when destructing NinjaMain.
+ BuildConfig config;
+ Options options = {};
+ options.input_file = "build.ninja";
+ options.dupe_edges_should_err = true;
+
+ setvbuf(stdout, NULL, _IOLBF, BUFSIZ);
+ const char* ninja_command = argv[0];
+
+ int exit_code = ReadFlags(&argc, &argv, &options, &config);
+ if (exit_code >= 0)
+ exit(exit_code);
+
+ if (options.working_dir) {
+ // The formatting of this string, complete with funny quotes, is
+ // so Emacs can properly identify that the cwd has changed for
+ // subsequent commands.
+ // Don't print this if a tool is being used, so that tool output
+ // can be piped into a file without this string showing up.
+ if (!options.tool)
+ printf("ninja: Entering directory `%s'\n", options.working_dir);
+ if (chdir(options.working_dir) < 0) {
+ Fatal("chdir to '%s' - %s", options.working_dir, strerror(errno));
+ }
+ }
+
+ if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) {
+ // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed
+ // by other tools.
+ NinjaMain ninja(ninja_command, config);
+ exit((ninja.*options.tool->func)(&options, argc, argv));
+ }
+
+ // Limit number of rebuilds, to prevent infinite loops.
+ const int kCycleLimit = 100;
+ for (int cycle = 1; cycle <= kCycleLimit; ++cycle) {
+ NinjaMain ninja(ninja_command, config);
+
+ ManifestParserOptions parser_opts;
+ if (options.dupe_edges_should_err) {
+ parser_opts.dupe_edge_action_ = kDupeEdgeActionError;
+ }
+ if (options.phony_cycle_should_err) {
+ parser_opts.phony_cycle_action_ = kPhonyCycleActionError;
+ }
+ ManifestParser parser(&ninja.state_, &ninja.disk_interface_, parser_opts);
+ string err;
+ if (!parser.Load(options.input_file, &err)) {
+ Error("%s", err.c_str());
+ exit(1);
+ }
+
+ if (options.tool && options.tool->when == Tool::RUN_AFTER_LOAD)
+ exit((ninja.*options.tool->func)(&options, argc, argv));
+
+ if (!ninja.EnsureBuildDirExists())
+ exit(1);
+
+ if (!ninja.OpenBuildLog() || !ninja.OpenDepsLog())
+ exit(1);
+
+ if (options.tool && options.tool->when == Tool::RUN_AFTER_LOGS)
+ exit((ninja.*options.tool->func)(&options, argc, argv));
+
+ // Attempt to rebuild the manifest before building anything else
+ if (ninja.RebuildManifest(options.input_file, &err)) {
+ // In dry_run mode the regeneration will succeed without changing the
+ // manifest forever. Better to return immediately.
+ if (config.dry_run)
+ exit(0);
+ // Start the build over with the new manifest.
+ continue;
+ } else if (!err.empty()) {
+ Error("rebuilding '%s': %s", options.input_file, err.c_str());
+ exit(1);
+ }
+
+ int result = ninja.RunBuild(argc, argv);
+ if (g_metrics)
+ ninja.DumpMetrics();
+ exit(result);
+ }
+
+ Error("manifest '%s' still dirty after %d tries\n",
+ options.input_file, kCycleLimit);
+ exit(1);
+}
+
+} // anonymous namespace
+
+int main(int argc, char** argv) {
+#if defined(_MSC_VER)
+ // Set a handler to catch crashes not caught by the __try..__except
+ // block (e.g. an exception in a stack-unwind-block).
+ std::set_terminate(TerminateHandler);
+ __try {
+ // Running inside __try ... __except suppresses any Windows error
+ // dialogs for errors such as bad_alloc.
+ real_main(argc, argv);
+ }
+ __except(ExceptionFilter(GetExceptionCode(), GetExceptionInformation())) {
+ // Common error situations return exitCode=1. 2 was chosen to
+ // indicate a more serious problem.
+ return 2;
+ }
+#else
+ real_main(argc, argv);
+#endif
+}
Index: src/subprocess-posix.cc
===================================================================
--- src/subprocess-posix.cc (nonexistent)
+++ src/subprocess-posix.cc (revision 5)
@@ -0,0 +1,397 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "subprocess.h"
+#include "tokenpool.h"
+
+#include <sys/select.h>
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/wait.h>
+#include <spawn.h>
+
+#if defined(USE_PPOLL)
+#include <poll.h>
+#else
+#include <sys/select.h>
+#endif
+
+extern char** environ;
+
+#include "util.h"
+
+using namespace std;
+
+Subprocess::Subprocess(bool use_console) : fd_(-1), pid_(-1),
+ use_console_(use_console) {
+}
+
+Subprocess::~Subprocess() {
+ if (fd_ >= 0)
+ close(fd_);
+ // Reap child if forgotten.
+ if (pid_ != -1)
+ Finish();
+}
+
+bool Subprocess::Start(SubprocessSet* set, const string& command) {
+ int output_pipe[2];
+ if (pipe(output_pipe) < 0)
+ Fatal("pipe: %s", strerror(errno));
+ fd_ = output_pipe[0];
+#if !defined(USE_PPOLL)
+ // If available, we use ppoll in DoWork(); otherwise we use pselect
+ // and so must avoid overly-large FDs.
+ if (fd_ >= static_cast<int>(FD_SETSIZE))
+ Fatal("pipe: %s", strerror(EMFILE));
+#endif // !USE_PPOLL
+ SetCloseOnExec(fd_);
+
+ posix_spawn_file_actions_t action;
+ int err = posix_spawn_file_actions_init(&action);
+ if (err != 0)
+ Fatal("posix_spawn_file_actions_init: %s", strerror(err));
+
+ err = posix_spawn_file_actions_addclose(&action, output_pipe[0]);
+ if (err != 0)
+ Fatal("posix_spawn_file_actions_addclose: %s", strerror(err));
+
+ posix_spawnattr_t attr;
+ err = posix_spawnattr_init(&attr);
+ if (err != 0)
+ Fatal("posix_spawnattr_init: %s", strerror(err));
+
+ short flags = 0;
+
+ flags |= POSIX_SPAWN_SETSIGMASK;
+ err = posix_spawnattr_setsigmask(&attr, &set->old_mask_);
+ if (err != 0)
+ Fatal("posix_spawnattr_setsigmask: %s", strerror(err));
+ // Signals which are set to be caught in the calling process image are set to
+ // default action in the new process image, so no explicit
+ // POSIX_SPAWN_SETSIGDEF parameter is needed.
+
+ if (!use_console_) {
+ // Put the child in its own process group, so ctrl-c won't reach it.
+ flags |= POSIX_SPAWN_SETPGROUP;
+ // No need to posix_spawnattr_setpgroup(&attr, 0), it's the default.
+
+ // Open /dev/null over stdin.
+ err = posix_spawn_file_actions_addopen(&action, 0, "/dev/null", O_RDONLY,
+ 0);
+ if (err != 0) {
+ Fatal("posix_spawn_file_actions_addopen: %s", strerror(err));
+ }
+
+ err = posix_spawn_file_actions_adddup2(&action, output_pipe[1], 1);
+ if (err != 0)
+ Fatal("posix_spawn_file_actions_adddup2: %s", strerror(err));
+ err = posix_spawn_file_actions_adddup2(&action, output_pipe[1], 2);
+ if (err != 0)
+ Fatal("posix_spawn_file_actions_adddup2: %s", strerror(err));
+ err = posix_spawn_file_actions_addclose(&action, output_pipe[1]);
+ if (err != 0)
+ Fatal("posix_spawn_file_actions_addclose: %s", strerror(err));
+ // In the console case, output_pipe is still inherited by the child and
+ // closed when the subprocess finishes, which then notifies ninja.
+ }
+#ifdef POSIX_SPAWN_USEVFORK
+ flags |= POSIX_SPAWN_USEVFORK;
+#endif
+
+ err = posix_spawnattr_setflags(&attr, flags);
+ if (err != 0)
+ Fatal("posix_spawnattr_setflags: %s", strerror(err));
+
+ const char* spawned_args[] = { "/bin/sh", "-c", command.c_str(), NULL };
+ err = posix_spawn(&pid_, "/bin/sh", &action, &attr,
+ const_cast<char**>(spawned_args), environ);
+ if (err != 0)
+ Fatal("posix_spawn: %s", strerror(err));
+
+ err = posix_spawnattr_destroy(&attr);
+ if (err != 0)
+ Fatal("posix_spawnattr_destroy: %s", strerror(err));
+ err = posix_spawn_file_actions_destroy(&action);
+ if (err != 0)
+ Fatal("posix_spawn_file_actions_destroy: %s", strerror(err));
+
+ close(output_pipe[1]);
+ return true;
+}
+
+void Subprocess::OnPipeReady() {
+ char buf[4 << 10];
+ ssize_t len = read(fd_, buf, sizeof(buf));
+ if (len > 0) {
+ buf_.append(buf, len);
+ } else {
+ if (len < 0)
+ Fatal("read: %s", strerror(errno));
+ close(fd_);
+ fd_ = -1;
+ }
+}
+
+ExitStatus Subprocess::Finish() {
+ assert(pid_ != -1);
+ int status;
+ if (waitpid(pid_, &status, 0) < 0)
+ Fatal("waitpid(%d): %s", pid_, strerror(errno));
+ pid_ = -1;
+
+#ifdef _AIX
+ if (WIFEXITED(status) && WEXITSTATUS(status) & 0x80) {
+ // Map the shell's exit code used for signal failure (128 + signal) to the
+ // status code expected by AIX WIFSIGNALED and WTERMSIG macros which, unlike
+ // other systems, uses a different bit layout.
+ int signal = WEXITSTATUS(status) & 0x7f;
+ status = (signal << 16) | signal;
+ }
+#endif
+
+ if (WIFEXITED(status)) {
+ int exit = WEXITSTATUS(status);
+ if (exit == 0)
+ return ExitSuccess;
+ } else if (WIFSIGNALED(status)) {
+ if (WTERMSIG(status) == SIGINT || WTERMSIG(status) == SIGTERM
+ || WTERMSIG(status) == SIGHUP)
+ return ExitInterrupted;
+ }
+ return ExitFailure;
+}
+
+bool Subprocess::Done() const {
+ return fd_ == -1;
+}
+
+const string& Subprocess::GetOutput() const {
+ return buf_;
+}
+
+int SubprocessSet::interrupted_;
+
+void SubprocessSet::SetInterruptedFlag(int signum) {
+ interrupted_ = signum;
+}
+
+void SubprocessSet::HandlePendingInterruption() {
+ sigset_t pending;
+ sigemptyset(&pending);
+ if (sigpending(&pending) == -1) {
+ perror("ninja: sigpending");
+ return;
+ }
+ if (sigismember(&pending, SIGINT))
+ interrupted_ = SIGINT;
+ else if (sigismember(&pending, SIGTERM))
+ interrupted_ = SIGTERM;
+ else if (sigismember(&pending, SIGHUP))
+ interrupted_ = SIGHUP;
+}
+
+SubprocessSet::SubprocessSet() {
+ sigset_t set;
+ sigemptyset(&set);
+ sigaddset(&set, SIGINT);
+ sigaddset(&set, SIGTERM);
+ sigaddset(&set, SIGHUP);
+ if (sigprocmask(SIG_BLOCK, &set, &old_mask_) < 0)
+ Fatal("sigprocmask: %s", strerror(errno));
+
+ struct sigaction act;
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = SetInterruptedFlag;
+ if (sigaction(SIGINT, &act, &old_int_act_) < 0)
+ Fatal("sigaction: %s", strerror(errno));
+ if (sigaction(SIGTERM, &act, &old_term_act_) < 0)
+ Fatal("sigaction: %s", strerror(errno));
+ if (sigaction(SIGHUP, &act, &old_hup_act_) < 0)
+ Fatal("sigaction: %s", strerror(errno));
+}
+
+SubprocessSet::~SubprocessSet() {
+ Clear();
+
+ if (sigaction(SIGINT, &old_int_act_, 0) < 0)
+ Fatal("sigaction: %s", strerror(errno));
+ if (sigaction(SIGTERM, &old_term_act_, 0) < 0)
+ Fatal("sigaction: %s", strerror(errno));
+ if (sigaction(SIGHUP, &old_hup_act_, 0) < 0)
+ Fatal("sigaction: %s", strerror(errno));
+ if (sigprocmask(SIG_SETMASK, &old_mask_, 0) < 0)
+ Fatal("sigprocmask: %s", strerror(errno));
+}
+
+Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
+ Subprocess *subprocess = new Subprocess(use_console);
+ if (!subprocess->Start(this, command)) {
+ delete subprocess;
+ return 0;
+ }
+ running_.push_back(subprocess);
+ return subprocess;
+}
+
+#ifdef USE_PPOLL
+bool SubprocessSet::DoWork(TokenPool* tokens) {
+ vector<pollfd> fds;
+ nfds_t nfds = 0;
+
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ++i) {
+ int fd = (*i)->fd_;
+ if (fd < 0)
+ continue;
+ pollfd pfd = { fd, POLLIN | POLLPRI, 0 };
+ fds.push_back(pfd);
+ ++nfds;
+ }
+
+ if (tokens) {
+ pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 };
+ fds.push_back(pfd);
+ ++nfds;
+ }
+
+ interrupted_ = 0;
+ int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_);
+ if (ret == -1) {
+ if (errno != EINTR) {
+ perror("ninja: ppoll");
+ return false;
+ }
+ return IsInterrupted();
+ }
+
+ HandlePendingInterruption();
+ if (IsInterrupted())
+ return true;
+
+ nfds_t cur_nfd = 0;
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ) {
+ int fd = (*i)->fd_;
+ if (fd < 0)
+ continue;
+ assert(fd == fds[cur_nfd].fd);
+ if (fds[cur_nfd++].revents) {
+ (*i)->OnPipeReady();
+ if ((*i)->Done()) {
+ finished_.push(*i);
+ i = running_.erase(i);
+ continue;
+ }
+ }
+ ++i;
+ }
+
+ if (tokens) {
+ pollfd *pfd = &fds[nfds - 1];
+ if (pfd->fd >= 0) {
+ assert(pfd->fd == tokens->GetMonitorFd());
+ if (pfd->revents != 0)
+ token_available_ = true;
+ }
+ }
+
+ return IsInterrupted();
+}
+
+#else // !defined(USE_PPOLL)
+bool SubprocessSet::DoWork(TokenPool* tokens) {
+ fd_set set;
+ int nfds = 0;
+ FD_ZERO(&set);
+
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ++i) {
+ int fd = (*i)->fd_;
+ if (fd >= 0) {
+ FD_SET(fd, &set);
+ if (nfds < fd+1)
+ nfds = fd+1;
+ }
+ }
+
+ if (tokens) {
+ int fd = tokens->GetMonitorFd();
+ FD_SET(fd, &set);
+ if (nfds < fd+1)
+ nfds = fd+1;
+ }
+
+ interrupted_ = 0;
+ int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_);
+ if (ret == -1) {
+ if (errno != EINTR) {
+ perror("ninja: pselect");
+ return false;
+ }
+ return IsInterrupted();
+ }
+
+ HandlePendingInterruption();
+ if (IsInterrupted())
+ return true;
+
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ) {
+ int fd = (*i)->fd_;
+ if (fd >= 0 && FD_ISSET(fd, &set)) {
+ (*i)->OnPipeReady();
+ if ((*i)->Done()) {
+ finished_.push(*i);
+ i = running_.erase(i);
+ continue;
+ }
+ }
+ ++i;
+ }
+
+ if (tokens) {
+ int fd = tokens->GetMonitorFd();
+ if ((fd >= 0) && FD_ISSET(fd, &set))
+ token_available_ = true;
+ }
+
+ return IsInterrupted();
+}
+#endif // !defined(USE_PPOLL)
+
+Subprocess* SubprocessSet::NextFinished() {
+ if (finished_.empty())
+ return NULL;
+ Subprocess* subproc = finished_.front();
+ finished_.pop();
+ return subproc;
+}
+
+void SubprocessSet::Clear() {
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ++i)
+ // Since the foreground process is in our process group, it will receive
+ // the interruption signal (i.e. SIGINT or SIGTERM) at the same time as us.
+ if (!(*i)->use_console_)
+ kill(-(*i)->pid_, interrupted_);
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ++i)
+ delete *i;
+ running_.clear();
+}
Index: src/subprocess-win32.cc
===================================================================
--- src/subprocess-win32.cc (nonexistent)
+++ src/subprocess-win32.cc (revision 5)
@@ -0,0 +1,316 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "subprocess.h"
+#include "tokenpool.h"
+
+#include <assert.h>
+#include <stdio.h>
+
+#include <algorithm>
+
+#include "util.h"
+
+using namespace std;
+
+Subprocess::Subprocess(bool use_console) : child_(NULL) , overlapped_(),
+ is_reading_(false),
+ use_console_(use_console) {
+}
+
+Subprocess::~Subprocess() {
+ if (pipe_) {
+ if (!CloseHandle(pipe_))
+ Win32Fatal("CloseHandle");
+ }
+ // Reap child if forgotten.
+ if (child_)
+ Finish();
+}
+
+HANDLE Subprocess::SetupPipe(HANDLE ioport) {
+ char pipe_name[100];
+ snprintf(pipe_name, sizeof(pipe_name),
+ "\\\\.\\pipe\\ninja_pid%lu_sp%p", GetCurrentProcessId(), this);
+
+ pipe_ = ::CreateNamedPipeA(pipe_name,
+ PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED,
+ PIPE_TYPE_BYTE,
+ PIPE_UNLIMITED_INSTANCES,
+ 0, 0, INFINITE, NULL);
+ if (pipe_ == INVALID_HANDLE_VALUE)
+ Win32Fatal("CreateNamedPipe");
+
+ if (!CreateIoCompletionPort(pipe_, ioport, (ULONG_PTR)this, 0))
+ Win32Fatal("CreateIoCompletionPort");
+
+ memset(&overlapped_, 0, sizeof(overlapped_));
+ if (!ConnectNamedPipe(pipe_, &overlapped_) &&
+ GetLastError() != ERROR_IO_PENDING) {
+ Win32Fatal("ConnectNamedPipe");
+ }
+
+ // Get the write end of the pipe as a handle inheritable across processes.
+ HANDLE output_write_handle =
+ CreateFileA(pipe_name, GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL);
+ HANDLE output_write_child;
+ if (!DuplicateHandle(GetCurrentProcess(), output_write_handle,
+ GetCurrentProcess(), &output_write_child,
+ 0, TRUE, DUPLICATE_SAME_ACCESS)) {
+ Win32Fatal("DuplicateHandle");
+ }
+ CloseHandle(output_write_handle);
+
+ return output_write_child;
+}
+
+bool Subprocess::Start(SubprocessSet* set, const string& command) {
+ HANDLE child_pipe = SetupPipe(set->ioport_);
+
+ SECURITY_ATTRIBUTES security_attributes;
+ memset(&security_attributes, 0, sizeof(SECURITY_ATTRIBUTES));
+ security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES);
+ security_attributes.bInheritHandle = TRUE;
+ // Must be inheritable so subprocesses can dup to children.
+ HANDLE nul =
+ CreateFileA("NUL", GENERIC_READ,
+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+ &security_attributes, OPEN_EXISTING, 0, NULL);
+ if (nul == INVALID_HANDLE_VALUE)
+ Fatal("couldn't open nul");
+
+ STARTUPINFOA startup_info;
+ memset(&startup_info, 0, sizeof(startup_info));
+ startup_info.cb = sizeof(STARTUPINFO);
+ if (!use_console_) {
+ startup_info.dwFlags = STARTF_USESTDHANDLES;
+ startup_info.hStdInput = nul;
+ startup_info.hStdOutput = child_pipe;
+ startup_info.hStdError = child_pipe;
+ }
+ // In the console case, child_pipe is still inherited by the child and closed
+ // when the subprocess finishes, which then notifies ninja.
+
+ PROCESS_INFORMATION process_info;
+ memset(&process_info, 0, sizeof(process_info));
+
+ // Ninja handles ctrl-c, except for subprocesses in console pools.
+ DWORD process_flags = use_console_ ? 0 : CREATE_NEW_PROCESS_GROUP;
+
+ // Do not prepend 'cmd /c' on Windows, this breaks command
+ // lines greater than 8,191 chars.
+ if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL,
+ /* inherit handles */ TRUE, process_flags,
+ NULL, NULL,
+ &startup_info, &process_info)) {
+ DWORD error = GetLastError();
+ if (error == ERROR_FILE_NOT_FOUND) {
+ // File (program) not found error is treated as a normal build
+ // action failure.
+ if (child_pipe)
+ CloseHandle(child_pipe);
+ CloseHandle(pipe_);
+ CloseHandle(nul);
+ pipe_ = NULL;
+ // child_ is already NULL;
+ buf_ = "CreateProcess failed: The system cannot find the file "
+ "specified.\n";
+ return true;
+ } else {
+ fprintf(stderr, "\nCreateProcess failed. Command attempted:\n\"%s\"\n",
+ command.c_str());
+ const char* hint = NULL;
+ // ERROR_INVALID_PARAMETER means the command line was formatted
+ // incorrectly. This can be caused by a command line being too long or
+ // leading whitespace in the command. Give extra context for this case.
+ if (error == ERROR_INVALID_PARAMETER) {
+ if (command.length() > 0 && (command[0] == ' ' || command[0] == '\t'))
+ hint = "command contains leading whitespace";
+ else
+ hint = "is the command line too long?";
+ }
+ Win32Fatal("CreateProcess", hint);
+ }
+ }
+
+ // Close pipe channel only used by the child.
+ if (child_pipe)
+ CloseHandle(child_pipe);
+ CloseHandle(nul);
+
+ CloseHandle(process_info.hThread);
+ child_ = process_info.hProcess;
+
+ return true;
+}
+
+void Subprocess::OnPipeReady() {
+ DWORD bytes;
+ if (!GetOverlappedResult(pipe_, &overlapped_, &bytes, TRUE)) {
+ if (GetLastError() == ERROR_BROKEN_PIPE) {
+ CloseHandle(pipe_);
+ pipe_ = NULL;
+ return;
+ }
+ Win32Fatal("GetOverlappedResult");
+ }
+
+ if (is_reading_ && bytes)
+ buf_.append(overlapped_buf_, bytes);
+
+ memset(&overlapped_, 0, sizeof(overlapped_));
+ is_reading_ = true;
+ if (!::ReadFile(pipe_, overlapped_buf_, sizeof(overlapped_buf_),
+ &bytes, &overlapped_)) {
+ if (GetLastError() == ERROR_BROKEN_PIPE) {
+ CloseHandle(pipe_);
+ pipe_ = NULL;
+ return;
+ }
+ if (GetLastError() != ERROR_IO_PENDING)
+ Win32Fatal("ReadFile");
+ }
+
+ // Even if we read any bytes in the readfile call, we'll enter this
+ // function again later and get them at that point.
+}
+
+ExitStatus Subprocess::Finish() {
+ if (!child_)
+ return ExitFailure;
+
+ // TODO: add error handling for all of these.
+ WaitForSingleObject(child_, INFINITE);
+
+ DWORD exit_code = 0;
+ GetExitCodeProcess(child_, &exit_code);
+
+ CloseHandle(child_);
+ child_ = NULL;
+
+ return exit_code == 0 ? ExitSuccess :
+ exit_code == CONTROL_C_EXIT ? ExitInterrupted :
+ ExitFailure;
+}
+
+bool Subprocess::Done() const {
+ return pipe_ == NULL;
+}
+
+const string& Subprocess::GetOutput() const {
+ return buf_;
+}
+
+HANDLE SubprocessSet::ioport_;
+
+SubprocessSet::SubprocessSet() {
+ ioport_ = ::CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 1);
+ if (!ioport_)
+ Win32Fatal("CreateIoCompletionPort");
+ if (!SetConsoleCtrlHandler(NotifyInterrupted, TRUE))
+ Win32Fatal("SetConsoleCtrlHandler");
+}
+
+SubprocessSet::~SubprocessSet() {
+ Clear();
+
+ SetConsoleCtrlHandler(NotifyInterrupted, FALSE);
+ CloseHandle(ioport_);
+}
+
+BOOL WINAPI SubprocessSet::NotifyInterrupted(DWORD dwCtrlType) {
+ if (dwCtrlType == CTRL_C_EVENT || dwCtrlType == CTRL_BREAK_EVENT) {
+ if (!PostQueuedCompletionStatus(ioport_, 0, 0, NULL))
+ Win32Fatal("PostQueuedCompletionStatus");
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+Subprocess *SubprocessSet::Add(const string& command, bool use_console) {
+ Subprocess *subprocess = new Subprocess(use_console);
+ if (!subprocess->Start(this, command)) {
+ delete subprocess;
+ return 0;
+ }
+ if (subprocess->child_)
+ running_.push_back(subprocess);
+ else
+ finished_.push(subprocess);
+ return subprocess;
+}
+
+bool SubprocessSet::DoWork(TokenPool* tokens) {
+ DWORD bytes_read;
+ Subprocess* subproc;
+ OVERLAPPED* overlapped;
+
+ if (tokens)
+ tokens->WaitForTokenAvailability(ioport_);
+
+ if (!GetQueuedCompletionStatus(ioport_, &bytes_read, (PULONG_PTR)&subproc,
+ &overlapped, INFINITE)) {
+ if (GetLastError() != ERROR_BROKEN_PIPE)
+ Win32Fatal("GetQueuedCompletionStatus");
+ }
+
+ if (!subproc) // A NULL subproc indicates that we were interrupted and is
+ // delivered by NotifyInterrupted above.
+ return true;
+
+ if (tokens && tokens->TokenIsAvailable((ULONG_PTR)subproc)) {
+ token_available_ = true;
+ return false;
+ }
+
+ subproc->OnPipeReady();
+
+ if (subproc->Done()) {
+ vector<Subprocess*>::iterator end =
+ remove(running_.begin(), running_.end(), subproc);
+ if (running_.end() != end) {
+ finished_.push(subproc);
+ running_.resize(end - running_.begin());
+ }
+ }
+
+ return false;
+}
+
+Subprocess* SubprocessSet::NextFinished() {
+ if (finished_.empty())
+ return NULL;
+ Subprocess* subproc = finished_.front();
+ finished_.pop();
+ return subproc;
+}
+
+void SubprocessSet::Clear() {
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ++i) {
+ // Since the foreground process is in our process group, it will receive a
+ // CTRL_C_EVENT or CTRL_BREAK_EVENT at the same time as us.
+ if ((*i)->child_ && !(*i)->use_console_) {
+ if (!GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT,
+ GetProcessId((*i)->child_))) {
+ Win32Fatal("GenerateConsoleCtrlEvent");
+ }
+ }
+ }
+ for (vector<Subprocess*>::iterator i = running_.begin();
+ i != running_.end(); ++i)
+ delete *i;
+ running_.clear();
+}
Index: src/subprocess.h
===================================================================
--- src/subprocess.h (nonexistent)
+++ src/subprocess.h (revision 5)
@@ -0,0 +1,119 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_SUBPROCESS_H_
+#define NINJA_SUBPROCESS_H_
+
+#include <string>
+#include <vector>
+#include <queue>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <signal.h>
+#endif
+
+// ppoll() exists on FreeBSD, but only on newer versions.
+#ifdef __FreeBSD__
+# include <sys/param.h>
+# if defined USE_PPOLL && __FreeBSD_version < 1002000
+# undef USE_PPOLL
+# endif
+#endif
+
+#include "exit_status.h"
+
+/// Subprocess wraps a single async subprocess. It is entirely
+/// passive: it expects the caller to notify it when its fds are ready
+/// for reading, as well as call Finish() to reap the child once done()
+/// is true.
+struct Subprocess {
+ ~Subprocess();
+
+ /// Returns ExitSuccess on successful process exit, ExitInterrupted if
+ /// the process was interrupted, ExitFailure if it otherwise failed.
+ ExitStatus Finish();
+
+ bool Done() const;
+
+ const std::string& GetOutput() const;
+
+ private:
+ Subprocess(bool use_console);
+ bool Start(struct SubprocessSet* set, const std::string& command);
+ void OnPipeReady();
+
+ std::string buf_;
+
+#ifdef _WIN32
+ /// Set up pipe_ as the parent-side pipe of the subprocess; return the
+ /// other end of the pipe, usable in the child process.
+ HANDLE SetupPipe(HANDLE ioport);
+
+ HANDLE child_;
+ HANDLE pipe_;
+ OVERLAPPED overlapped_;
+ char overlapped_buf_[4 << 10];
+ bool is_reading_;
+#else
+ int fd_;
+ pid_t pid_;
+#endif
+ bool use_console_;
+
+ friend struct SubprocessSet;
+};
+
+struct TokenPool;
+
+/// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses.
+/// DoWork() waits for any state change in subprocesses; finished_
+/// is a queue of subprocesses as they finish.
+struct SubprocessSet {
+ SubprocessSet();
+ ~SubprocessSet();
+
+ Subprocess* Add(const std::string& command, bool use_console = false);
+ bool DoWork(TokenPool* tokens);
+ Subprocess* NextFinished();
+ void Clear();
+
+ std::vector<Subprocess*> running_;
+ std::queue<Subprocess*> finished_;
+
+ bool token_available_;
+ bool IsTokenAvailable() { return token_available_; }
+ void ResetTokenAvailable() { token_available_ = false; }
+
+#ifdef _WIN32
+ static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType);
+ static HANDLE ioport_;
+#else
+ static void SetInterruptedFlag(int signum);
+ static void HandlePendingInterruption();
+ /// Store the signal number that causes the interruption.
+ /// 0 if not interruption.
+ static int interrupted_;
+
+ static bool IsInterrupted() { return interrupted_ != 0; }
+
+ struct sigaction old_int_act_;
+ struct sigaction old_term_act_;
+ struct sigaction old_hup_act_;
+ sigset_t old_mask_;
+#endif
+};
+
+#endif // NINJA_SUBPROCESS_H_
Index: src/subprocess_test.cc
===================================================================
--- src/subprocess_test.cc (nonexistent)
+++ src/subprocess_test.cc (revision 5)
@@ -0,0 +1,384 @@
+// Copyright 2012 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "subprocess.h"
+#include "tokenpool.h"
+
+#include "test.h"
+
+#ifndef _WIN32
+// SetWithLots need setrlimit.
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+#endif
+
+using namespace std;
+
+namespace {
+
+#ifdef _WIN32
+const char* kSimpleCommand = "cmd /c dir \\";
+#else
+const char* kSimpleCommand = "ls /";
+#endif
+
+struct TestTokenPool : public TokenPool {
+ bool Acquire() { return false; }
+ void Reserve() {}
+ void Release() {}
+ void Clear() {}
+ bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; }
+
+#ifdef _WIN32
+ bool _token_available;
+ void WaitForTokenAvailability(HANDLE ioport) {
+ if (_token_available)
+ // unblock GetQueuedCompletionStatus()
+ PostQueuedCompletionStatus(ioport, 0, (ULONG_PTR) this, NULL);
+ }
+ bool TokenIsAvailable(ULONG_PTR key) { return key == (ULONG_PTR) this; }
+#else
+ int _fd;
+ int GetMonitorFd() { return _fd; }
+#endif
+};
+
+struct SubprocessTest : public testing::Test {
+ SubprocessSet subprocs_;
+ TestTokenPool tokens_;
+};
+
+} // anonymous namespace
+
+// Run a command that fails and emits to stderr.
+TEST_F(SubprocessTest, BadCommandStderr) {
+ Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ // Pretend we discovered that stderr was ready for writing.
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+
+ EXPECT_EQ(ExitFailure, subproc->Finish());
+ EXPECT_NE("", subproc->GetOutput());
+}
+
+// Run a command that does not exist
+TEST_F(SubprocessTest, NoSuchCommand) {
+ Subprocess* subproc = subprocs_.Add("ninja_no_such_command");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ // Pretend we discovered that stderr was ready for writing.
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+
+ EXPECT_EQ(ExitFailure, subproc->Finish());
+ EXPECT_NE("", subproc->GetOutput());
+#ifdef _WIN32
+ ASSERT_EQ("CreateProcess failed: The system cannot find the file "
+ "specified.\n", subproc->GetOutput());
+#endif
+}
+
+#ifndef _WIN32
+
+TEST_F(SubprocessTest, InterruptChild) {
+ Subprocess* subproc = subprocs_.Add("kill -INT $$");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+
+ EXPECT_EQ(ExitInterrupted, subproc->Finish());
+}
+
+TEST_F(SubprocessTest, InterruptParent) {
+ Subprocess* subproc = subprocs_.Add("kill -INT $PPID ; sleep 1");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ while (!subproc->Done()) {
+ bool interrupted = subprocs_.DoWork(NULL);
+ if (interrupted)
+ return;
+ }
+
+ ASSERT_FALSE("We should have been interrupted");
+}
+
+TEST_F(SubprocessTest, InterruptChildWithSigTerm) {
+ Subprocess* subproc = subprocs_.Add("kill -TERM $$");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+
+ EXPECT_EQ(ExitInterrupted, subproc->Finish());
+}
+
+TEST_F(SubprocessTest, InterruptParentWithSigTerm) {
+ Subprocess* subproc = subprocs_.Add("kill -TERM $PPID ; sleep 1");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ while (!subproc->Done()) {
+ bool interrupted = subprocs_.DoWork(NULL);
+ if (interrupted)
+ return;
+ }
+
+ ASSERT_FALSE("We should have been interrupted");
+}
+
+TEST_F(SubprocessTest, InterruptChildWithSigHup) {
+ Subprocess* subproc = subprocs_.Add("kill -HUP $$");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+
+ EXPECT_EQ(ExitInterrupted, subproc->Finish());
+}
+
+TEST_F(SubprocessTest, InterruptParentWithSigHup) {
+ Subprocess* subproc = subprocs_.Add("kill -HUP $PPID ; sleep 1");
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ while (!subproc->Done()) {
+ bool interrupted = subprocs_.DoWork(NULL);
+ if (interrupted)
+ return;
+ }
+
+ ASSERT_FALSE("We should have been interrupted");
+}
+
+TEST_F(SubprocessTest, Console) {
+ // Skip test if we don't have the console ourselves.
+ if (isatty(0) && isatty(1) && isatty(2)) {
+ Subprocess* subproc =
+ subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true);
+ ASSERT_NE((Subprocess*)0, subproc);
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+
+ EXPECT_EQ(ExitSuccess, subproc->Finish());
+ }
+}
+
+#endif
+
+TEST_F(SubprocessTest, SetWithSingle) {
+ Subprocess* subproc = subprocs_.Add(kSimpleCommand);
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+ ASSERT_EQ(ExitSuccess, subproc->Finish());
+ ASSERT_NE("", subproc->GetOutput());
+
+ ASSERT_EQ(1u, subprocs_.finished_.size());
+}
+
+TEST_F(SubprocessTest, SetWithMulti) {
+ Subprocess* processes[3];
+ const char* kCommands[3] = {
+ kSimpleCommand,
+#ifdef _WIN32
+ "cmd /c echo hi",
+ "cmd /c time /t",
+#else
+ "id -u",
+ "pwd",
+#endif
+ };
+
+ for (int i = 0; i < 3; ++i) {
+ processes[i] = subprocs_.Add(kCommands[i]);
+ ASSERT_NE((Subprocess *) 0, processes[i]);
+ }
+
+ ASSERT_EQ(3u, subprocs_.running_.size());
+ for (int i = 0; i < 3; ++i) {
+ ASSERT_FALSE(processes[i]->Done());
+ ASSERT_EQ("", processes[i]->GetOutput());
+ }
+
+ subprocs_.ResetTokenAvailable();
+ while (!processes[0]->Done() || !processes[1]->Done() ||
+ !processes[2]->Done()) {
+ ASSERT_GT(subprocs_.running_.size(), 0u);
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+ ASSERT_EQ(0u, subprocs_.running_.size());
+ ASSERT_EQ(3u, subprocs_.finished_.size());
+
+ for (int i = 0; i < 3; ++i) {
+ ASSERT_EQ(ExitSuccess, processes[i]->Finish());
+ ASSERT_NE("", processes[i]->GetOutput());
+ delete processes[i];
+ }
+}
+
+#if defined(USE_PPOLL)
+TEST_F(SubprocessTest, SetWithLots) {
+ // Arbitrary big number; needs to be over 1024 to confirm we're no longer
+ // hostage to pselect.
+ const unsigned kNumProcs = 1025;
+
+ // Make sure [ulimit -n] isn't going to stop us from working.
+ rlimit rlim;
+ ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlim));
+ if (rlim.rlim_cur < kNumProcs) {
+ printf("Raise [ulimit -n] above %u (currently %lu) to make this test go\n",
+ kNumProcs, rlim.rlim_cur);
+ return;
+ }
+
+ vector<Subprocess*> procs;
+ for (size_t i = 0; i < kNumProcs; ++i) {
+ Subprocess* subproc = subprocs_.Add("/bin/echo");
+ ASSERT_NE((Subprocess *) 0, subproc);
+ procs.push_back(subproc);
+ }
+ subprocs_.ResetTokenAvailable();
+ while (!subprocs_.running_.empty())
+ subprocs_.DoWork(NULL);
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+ for (size_t i = 0; i < procs.size(); ++i) {
+ ASSERT_EQ(ExitSuccess, procs[i]->Finish());
+ ASSERT_NE("", procs[i]->GetOutput());
+ }
+ ASSERT_EQ(kNumProcs, subprocs_.finished_.size());
+}
+#endif // !__APPLE__ && !_WIN32
+
+// TODO: this test could work on Windows, just not sure how to simply
+// read stdin.
+#ifndef _WIN32
+// Verify that a command that attempts to read stdin correctly thinks
+// that stdin is closed.
+TEST_F(SubprocessTest, ReadStdin) {
+ Subprocess* subproc = subprocs_.Add("cat -");
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ subprocs_.DoWork(NULL);
+ }
+ ASSERT_FALSE(subprocs_.IsTokenAvailable());
+ ASSERT_EQ(ExitSuccess, subproc->Finish());
+ ASSERT_EQ(1u, subprocs_.finished_.size());
+}
+#endif // _WIN32
+
+TEST_F(SubprocessTest, TokenAvailable) {
+ Subprocess* subproc = subprocs_.Add(kSimpleCommand);
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ // simulate GNUmake jobserver pipe with 1 token
+#ifdef _WIN32
+ tokens_._token_available = true;
+#else
+ int fds[2];
+ ASSERT_EQ(0u, pipe(fds));
+ tokens_._fd = fds[0];
+ ASSERT_EQ(1u, write(fds[1], "T", 1));
+#endif
+
+ subprocs_.ResetTokenAvailable();
+ subprocs_.DoWork(&tokens_);
+#ifdef _WIN32
+ tokens_._token_available = false;
+ // we need to loop here as we have no conrol where the token
+ // I/O completion post ends up in the queue
+ while (!subproc->Done() && !subprocs_.IsTokenAvailable()) {
+ subprocs_.DoWork(&tokens_);
+ }
+#endif
+
+ EXPECT_TRUE(subprocs_.IsTokenAvailable());
+ EXPECT_EQ(0u, subprocs_.finished_.size());
+
+ // remove token to let DoWork() wait for command again
+#ifndef _WIN32
+ char token;
+ ASSERT_EQ(1u, read(fds[0], &token, 1));
+#endif
+
+ while (!subproc->Done()) {
+ subprocs_.DoWork(&tokens_);
+ }
+
+#ifndef _WIN32
+ close(fds[1]);
+ close(fds[0]);
+#endif
+
+ EXPECT_EQ(ExitSuccess, subproc->Finish());
+ EXPECT_NE("", subproc->GetOutput());
+
+ EXPECT_EQ(1u, subprocs_.finished_.size());
+}
+
+TEST_F(SubprocessTest, TokenNotAvailable) {
+ Subprocess* subproc = subprocs_.Add(kSimpleCommand);
+ ASSERT_NE((Subprocess *) 0, subproc);
+
+ // simulate GNUmake jobserver pipe with 0 tokens
+#ifdef _WIN32
+ tokens_._token_available = false;
+#else
+ int fds[2];
+ ASSERT_EQ(0u, pipe(fds));
+ tokens_._fd = fds[0];
+#endif
+
+ subprocs_.ResetTokenAvailable();
+ while (!subproc->Done()) {
+ subprocs_.DoWork(&tokens_);
+ }
+
+#ifndef _WIN32
+ close(fds[1]);
+ close(fds[0]);
+#endif
+
+ EXPECT_FALSE(subprocs_.IsTokenAvailable());
+ EXPECT_EQ(ExitSuccess, subproc->Finish());
+ EXPECT_NE("", subproc->GetOutput());
+
+ EXPECT_EQ(1u, subprocs_.finished_.size());
+}
Index: src/tokenpool-gnu-make-posix.cc
===================================================================
--- src/tokenpool-gnu-make-posix.cc (nonexistent)
+++ src/tokenpool-gnu-make-posix.cc (revision 5)
@@ -0,0 +1,202 @@
+// Copyright 2016-2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenpool-gnu-make.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <unistd.h>
+#include <signal.h>
+#include <sys/time.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+// TokenPool implementation for GNU make jobserver - POSIX implementation
+// (http://make.mad-scientist.net/papers/jobserver-implementation/)
+struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool {
+ GNUmakeTokenPoolPosix();
+ virtual ~GNUmakeTokenPoolPosix();
+
+ virtual int GetMonitorFd();
+
+ virtual const char* GetEnv(const char* name) { return getenv(name); };
+ virtual bool ParseAuth(const char* jobserver);
+ virtual bool AcquireToken();
+ virtual bool ReturnToken();
+
+ private:
+ int rfd_;
+ int wfd_;
+
+ struct sigaction old_act_;
+ bool restore_;
+
+ static int dup_rfd_;
+ static void CloseDupRfd(int signum);
+
+ bool CheckFd(int fd);
+ bool SetAlarmHandler();
+};
+
+GNUmakeTokenPoolPosix::GNUmakeTokenPoolPosix() : rfd_(-1), wfd_(-1), restore_(false) {
+}
+
+GNUmakeTokenPoolPosix::~GNUmakeTokenPoolPosix() {
+ Clear();
+ if (restore_)
+ sigaction(SIGALRM, &old_act_, NULL);
+}
+
+bool GNUmakeTokenPoolPosix::CheckFd(int fd) {
+ if (fd < 0)
+ return false;
+ int ret = fcntl(fd, F_GETFD);
+ if (ret < 0)
+ return false;
+ return true;
+}
+
+int GNUmakeTokenPoolPosix::dup_rfd_ = -1;
+
+void GNUmakeTokenPoolPosix::CloseDupRfd(int signum) {
+ close(dup_rfd_);
+ dup_rfd_ = -1;
+}
+
+bool GNUmakeTokenPoolPosix::SetAlarmHandler() {
+ struct sigaction act;
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = CloseDupRfd;
+ if (sigaction(SIGALRM, &act, &old_act_) < 0) {
+ perror("sigaction:");
+ return false;
+ }
+ restore_ = true;
+ return true;
+}
+
+bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) {
+ int rfd = -1;
+ int wfd = -1;
+ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
+ CheckFd(rfd) &&
+ CheckFd(wfd) &&
+ SetAlarmHandler()) {
+ rfd_ = rfd;
+ wfd_ = wfd;
+ return true;
+ }
+
+ return false;
+}
+
+bool GNUmakeTokenPoolPosix::AcquireToken() {
+ // Please read
+ //
+ // http://make.mad-scientist.net/papers/jobserver-implementation/
+ //
+ // for the reasoning behind the following code.
+ //
+ // Try to read one character from the pipe. Returns true on success.
+ //
+ // First check if read() would succeed without blocking.
+#ifdef USE_PPOLL
+ pollfd pollfds[] = {{rfd_, POLLIN, 0}};
+ int ret = poll(pollfds, 1, 0);
+#else
+ fd_set set;
+ struct timeval timeout = { 0, 0 };
+ FD_ZERO(&set);
+ FD_SET(rfd_, &set);
+ int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout);
+#endif
+ if (ret > 0) {
+ // Handle potential race condition:
+ // - the above check succeeded, i.e. read() should not block
+ // - the character disappears before we call read()
+ //
+ // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_
+ // can safely be closed by signal handlers without affecting rfd_.
+ dup_rfd_ = dup(rfd_);
+
+ if (dup_rfd_ != -1) {
+ struct sigaction act, old_act;
+ int ret = 0;
+
+ // Temporarily replace SIGCHLD handler with our own
+ memset(&act, 0, sizeof(act));
+ act.sa_handler = CloseDupRfd;
+ if (sigaction(SIGCHLD, &act, &old_act) == 0) {
+ struct itimerval timeout;
+
+ // install a 100ms timeout that generates SIGALARM on expiration
+ memset(&timeout, 0, sizeof(timeout));
+ timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec]
+ if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) {
+ char buf;
+
+ // Now try to read() from dup_rfd_. Return values from read():
+ //
+ // 1. token read -> 1
+ // 2. pipe closed -> 0
+ // 3. alarm expires -> -1 (EINTR)
+ // 4. child exits -> -1 (EINTR)
+ // 5. alarm expired before entering read() -> -1 (EBADF)
+ // 6. child exited before entering read() -> -1 (EBADF)
+ // 7. child exited before handler is installed -> go to 1 - 3
+ ret = read(dup_rfd_, &buf, 1);
+
+ // disarm timer
+ memset(&timeout, 0, sizeof(timeout));
+ setitimer(ITIMER_REAL, &timeout, NULL);
+ }
+
+ sigaction(SIGCHLD, &old_act, NULL);
+ }
+
+ CloseDupRfd(0);
+
+ // Case 1 from above list
+ if (ret > 0)
+ return true;
+ }
+ }
+
+ // read() would block, i.e. no token available,
+ // cases 2-6 from above list or
+ // select() / poll() / dup() / sigaction() / setitimer() failed
+ return false;
+}
+
+bool GNUmakeTokenPoolPosix::ReturnToken() {
+ const char buf = '+';
+ while (1) {
+ int ret = write(wfd_, &buf, 1);
+ if (ret > 0)
+ return true;
+ if ((ret != -1) || (errno != EINTR))
+ return false;
+ // write got interrupted - retry
+ }
+}
+
+int GNUmakeTokenPoolPosix::GetMonitorFd() {
+ return rfd_;
+}
+
+TokenPool* TokenPool::Get() {
+ return new GNUmakeTokenPoolPosix;
+}
Index: src/tokenpool-gnu-make-win32.cc
===================================================================
--- src/tokenpool-gnu-make-win32.cc (nonexistent)
+++ src/tokenpool-gnu-make-win32.cc (revision 5)
@@ -0,0 +1,239 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenpool-gnu-make.h"
+
+// Always include this first.
+// Otherwise the other system headers don't work correctly under Win32
+#include <windows.h>
+
+#include <ctype.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "util.h"
+
+// TokenPool implementation for GNU make jobserver - Win32 implementation
+// (https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html)
+struct GNUmakeTokenPoolWin32 : public GNUmakeTokenPool {
+ GNUmakeTokenPoolWin32();
+ virtual ~GNUmakeTokenPoolWin32();
+
+ virtual void WaitForTokenAvailability(HANDLE ioport);
+ virtual bool TokenIsAvailable(ULONG_PTR key);
+
+ virtual const char* GetEnv(const char* name);
+ virtual bool ParseAuth(const char* jobserver);
+ virtual bool AcquireToken();
+ virtual bool ReturnToken();
+
+ private:
+ // Semaphore for GNU make jobserver protocol
+ HANDLE semaphore_jobserver_;
+ // Semaphore Child -> Parent
+ // - child releases it before entering wait on jobserver semaphore
+ // - parent blocks on it to know when child enters wait
+ HANDLE semaphore_enter_wait_;
+ // Semaphore Parent -> Child
+ // - parent releases it to allow child to restart loop
+ // - child blocks on it to know when to restart loop
+ HANDLE semaphore_restart_;
+ // set to false if child should exit loop and terminate thread
+ bool running_;
+ // child thread
+ HANDLE child_;
+ // I/O completion port from SubprocessSet
+ HANDLE ioport_;
+
+
+ DWORD SemaphoreThread();
+ void ReleaseSemaphore(HANDLE semaphore);
+ void WaitForObject(HANDLE object);
+ static DWORD WINAPI SemaphoreThreadWrapper(LPVOID param);
+ static void NoopAPCFunc(ULONG_PTR param);
+};
+
+GNUmakeTokenPoolWin32::GNUmakeTokenPoolWin32() : semaphore_jobserver_(NULL),
+ semaphore_enter_wait_(NULL),
+ semaphore_restart_(NULL),
+ running_(false),
+ child_(NULL),
+ ioport_(NULL) {
+}
+
+GNUmakeTokenPoolWin32::~GNUmakeTokenPoolWin32() {
+ Clear();
+ CloseHandle(semaphore_jobserver_);
+ semaphore_jobserver_ = NULL;
+
+ if (child_) {
+ // tell child thread to exit
+ running_ = false;
+ ReleaseSemaphore(semaphore_restart_);
+
+ // wait for child thread to exit
+ WaitForObject(child_);
+ CloseHandle(child_);
+ child_ = NULL;
+ }
+
+ if (semaphore_restart_) {
+ CloseHandle(semaphore_restart_);
+ semaphore_restart_ = NULL;
+ }
+
+ if (semaphore_enter_wait_) {
+ CloseHandle(semaphore_enter_wait_);
+ semaphore_enter_wait_ = NULL;
+ }
+}
+
+const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) {
+ // getenv() does not work correctly together with tokenpool_tests.cc
+ static char buffer[MAX_PATH + 1];
+ if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0)
+ return NULL;
+ return buffer;
+}
+
+bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) {
+ // match "--jobserver-auth=gmake_semaphore_<INTEGER>..."
+ const char* start = strchr(jobserver, '=');
+ if (start) {
+ const char* end = start;
+ unsigned int len;
+ char c, *auth;
+
+ while ((c = *++end) != '\0')
+ if (!(isalnum(c) || (c == '_')))
+ break;
+ len = end - start; // includes string terminator in count
+
+ if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) {
+ strncpy(auth, start + 1, len - 1);
+ auth[len - 1] = '\0';
+
+ if ((semaphore_jobserver_ =
+ OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */
+ FALSE, /* Child processes DON'T inherit */
+ auth /* Semaphore name */
+ )) != NULL) {
+ free(auth);
+ return true;
+ }
+
+ free(auth);
+ }
+ }
+
+ return false;
+}
+
+bool GNUmakeTokenPoolWin32::AcquireToken() {
+ return WaitForSingleObject(semaphore_jobserver_, 0) == WAIT_OBJECT_0;
+}
+
+bool GNUmakeTokenPoolWin32::ReturnToken() {
+ ReleaseSemaphore(semaphore_jobserver_);
+ return true;
+}
+
+DWORD GNUmakeTokenPoolWin32::SemaphoreThread() {
+ while (running_) {
+ // indicate to parent that we are entering wait
+ ReleaseSemaphore(semaphore_enter_wait_);
+
+ // alertable wait forever on token semaphore
+ if (WaitForSingleObjectEx(semaphore_jobserver_, INFINITE, TRUE) == WAIT_OBJECT_0) {
+ // release token again for AcquireToken()
+ ReleaseSemaphore(semaphore_jobserver_);
+
+ // indicate to parent on ioport that a token might be available
+ if (!PostQueuedCompletionStatus(ioport_, 0, (ULONG_PTR) this, NULL))
+ Win32Fatal("PostQueuedCompletionStatus");
+ }
+
+ // wait for parent to allow loop restart
+ WaitForObject(semaphore_restart_);
+ // semaphore is now in nonsignaled state again for next run...
+ }
+
+ return 0;
+}
+
+DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) {
+ GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param;
+ return This->SemaphoreThread();
+}
+
+void GNUmakeTokenPoolWin32::NoopAPCFunc(ULONG_PTR param) {
+}
+
+void GNUmakeTokenPoolWin32::WaitForTokenAvailability(HANDLE ioport) {
+ if (child_ == NULL) {
+ // first invocation
+ //
+ // subprocess-win32.cc uses I/O completion port (IOCP) which can't be
+ // used as a waitable object. Therefore we can't use WaitMultipleObjects()
+ // to wait on the IOCP and the token semaphore at the same time. Create
+ // a child thread that waits on the semaphore and posts an I/O completion
+ ioport_ = ioport;
+
+ // create both semaphores in nonsignaled state
+ if ((semaphore_enter_wait_ = CreateSemaphore(NULL, 0, 1, NULL))
+ == NULL)
+ Win32Fatal("CreateSemaphore/enter_wait");
+ if ((semaphore_restart_ = CreateSemaphore(NULL, 0, 1, NULL))
+ == NULL)
+ Win32Fatal("CreateSemaphore/restart");
+
+ // start child thread
+ running_ = true;
+ if ((child_ = CreateThread(NULL, 0, &SemaphoreThreadWrapper, this, 0, NULL))
+ == NULL)
+ Win32Fatal("CreateThread");
+
+ } else {
+ // all further invocations - allow child thread to loop
+ ReleaseSemaphore(semaphore_restart_);
+ }
+
+ // wait for child thread to enter wait
+ WaitForObject(semaphore_enter_wait_);
+ // semaphore is now in nonsignaled state again for next run...
+
+ // now SubprocessSet::DoWork() can enter GetQueuedCompletionStatus()...
+}
+
+bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) {
+ // alert child thread to break wait on token semaphore
+ QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL);
+
+ // return true when GetQueuedCompletionStatus() returned our key
+ return key == (ULONG_PTR) this;
+}
+
+void GNUmakeTokenPoolWin32::ReleaseSemaphore(HANDLE semaphore) {
+ if (!::ReleaseSemaphore(semaphore, 1, NULL))
+ Win32Fatal("ReleaseSemaphore");
+}
+
+void GNUmakeTokenPoolWin32::WaitForObject(HANDLE object) {
+ if (WaitForSingleObject(object, INFINITE) != WAIT_OBJECT_0)
+ Win32Fatal("WaitForSingleObject");
+}
+
+TokenPool* TokenPool::Get() {
+ return new GNUmakeTokenPoolWin32;
+}
Index: src/tokenpool-gnu-make.cc
===================================================================
--- src/tokenpool-gnu-make.cc (nonexistent)
+++ src/tokenpool-gnu-make.cc (revision 5)
@@ -0,0 +1,108 @@
+// Copyright 2016-2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenpool-gnu-make.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "line_printer.h"
+
+// TokenPool implementation for GNU make jobserver - common bits
+// every instance owns an implicit token -> available_ == 1
+GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) {
+}
+
+GNUmakeTokenPool::~GNUmakeTokenPool() {
+}
+
+bool GNUmakeTokenPool::Setup(bool ignore,
+ bool verbose,
+ double& max_load_average) {
+ const char* value = GetEnv("MAKEFLAGS");
+ if (!value)
+ return false;
+
+ // GNU make <= 4.1
+ const char* jobserver = strstr(value, "--jobserver-fds=");
+ if (!jobserver)
+ // GNU make => 4.2
+ jobserver = strstr(value, "--jobserver-auth=");
+ if (jobserver) {
+ LinePrinter printer;
+
+ if (ignore) {
+ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n");
+ } else {
+ if (ParseAuth(jobserver)) {
+ const char* l_arg = strstr(value, " -l");
+ int load_limit = -1;
+
+ if (verbose) {
+ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n");
+ }
+
+ // translate GNU make -lN to ninja -lN
+ if (l_arg &&
+ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) &&
+ (load_limit > 0)) {
+ max_load_average = load_limit;
+ }
+
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+bool GNUmakeTokenPool::Acquire() {
+ if (available_ > 0)
+ return true;
+
+ if (AcquireToken()) {
+ // token acquired
+ available_++;
+ return true;
+ }
+
+ // no token available
+ return false;
+}
+
+void GNUmakeTokenPool::Reserve() {
+ available_--;
+ used_++;
+}
+
+void GNUmakeTokenPool::Return() {
+ if (ReturnToken())
+ available_--;
+}
+
+void GNUmakeTokenPool::Release() {
+ available_++;
+ used_--;
+ if (available_ > 1)
+ Return();
+}
+
+void GNUmakeTokenPool::Clear() {
+ while (used_ > 0)
+ Release();
+ while (available_ > 1)
+ Return();
+}
Index: src/tokenpool-gnu-make.h
===================================================================
--- src/tokenpool-gnu-make.h (nonexistent)
+++ src/tokenpool-gnu-make.h (revision 5)
@@ -0,0 +1,40 @@
+// Copyright 2016-2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenpool.h"
+
+// interface to GNU make token pool
+struct GNUmakeTokenPool : public TokenPool {
+ GNUmakeTokenPool();
+ ~GNUmakeTokenPool();
+
+ // token pool implementation
+ virtual bool Acquire();
+ virtual void Reserve();
+ virtual void Release();
+ virtual void Clear();
+ virtual bool Setup(bool ignore, bool verbose, double& max_load_average);
+
+ // platform specific implementation
+ virtual const char* GetEnv(const char* name) = 0;
+ virtual bool ParseAuth(const char* jobserver) = 0;
+ virtual bool AcquireToken() = 0;
+ virtual bool ReturnToken() = 0;
+
+ private:
+ int available_;
+ int used_;
+
+ void Return();
+};
Index: src/tokenpool.h
===================================================================
--- src/tokenpool.h (nonexistent)
+++ src/tokenpool.h (revision 5)
@@ -0,0 +1,42 @@
+// Copyright 2016-2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+// interface to token pool
+struct TokenPool {
+ virtual ~TokenPool() {}
+
+ virtual bool Acquire() = 0;
+ virtual void Reserve() = 0;
+ virtual void Release() = 0;
+ virtual void Clear() = 0;
+
+ // returns false if token pool setup failed
+ virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0;
+
+#ifdef _WIN32
+ virtual void WaitForTokenAvailability(HANDLE ioport) = 0;
+ // returns true if a token has become available
+ // key is result from GetQueuedCompletionStatus()
+ virtual bool TokenIsAvailable(ULONG_PTR key) = 0;
+#else
+ virtual int GetMonitorFd() = 0;
+#endif
+
+ // returns NULL if token pool is not available
+ static TokenPool* Get();
+};
Index: src/tokenpool_test.cc
===================================================================
--- src/tokenpool_test.cc (nonexistent)
+++ src/tokenpool_test.cc (revision 5)
@@ -0,0 +1,269 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tokenpool.h"
+
+#include "test.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+// should contain all valid characters
+#define SEMAPHORE_NAME "abcdefghijklmnopqrstwxyz01234567890_"
+#define AUTH_FORMAT(tmpl) "foo " tmpl "=%s bar"
+#define ENVIRONMENT_CLEAR() SetEnvironmentVariable("MAKEFLAGS", NULL)
+#define ENVIRONMENT_INIT(v) SetEnvironmentVariable("MAKEFLAGS", v)
+#else
+#define AUTH_FORMAT(tmpl) "foo " tmpl "=%d,%d bar"
+#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS")
+#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true)
+#endif
+
+namespace {
+
+const double kLoadAverageDefault = -1.23456789;
+
+struct TokenPoolTest : public testing::Test {
+ double load_avg_;
+ TokenPool* tokens_;
+ char buf_[1024];
+#ifdef _WIN32
+ const char* semaphore_name_;
+ HANDLE semaphore_;
+#else
+ int fds_[2];
+#endif
+
+ virtual void SetUp() {
+ load_avg_ = kLoadAverageDefault;
+ tokens_ = NULL;
+ ENVIRONMENT_CLEAR();
+#ifdef _WIN32
+ semaphore_name_ = SEMAPHORE_NAME;
+ if ((semaphore_ = CreateSemaphore(0, 0, 2, SEMAPHORE_NAME)) == NULL)
+#else
+ if (pipe(fds_) < 0)
+#endif
+ ASSERT_TRUE(false);
+ }
+
+ void CreatePool(const char* format, bool ignore_jobserver = false) {
+ if (format) {
+ sprintf(buf_, format,
+#ifdef _WIN32
+ semaphore_name_
+#else
+ fds_[0], fds_[1]
+#endif
+ );
+ ENVIRONMENT_INIT(buf_);
+ }
+ if ((tokens_ = TokenPool::Get()) != NULL) {
+ if (!tokens_->Setup(ignore_jobserver, false, load_avg_)) {
+ delete tokens_;
+ tokens_ = NULL;
+ }
+ }
+ }
+
+ void CreateDefaultPool() {
+ CreatePool(AUTH_FORMAT("--jobserver-auth"));
+ }
+
+ virtual void TearDown() {
+ if (tokens_)
+ delete tokens_;
+#ifdef _WIN32
+ CloseHandle(semaphore_);
+#else
+ close(fds_[0]);
+ close(fds_[1]);
+#endif
+ ENVIRONMENT_CLEAR();
+ }
+};
+
+} // anonymous namespace
+
+// verifies none implementation
+TEST_F(TokenPoolTest, NoTokenPool) {
+ CreatePool(NULL, false);
+
+ EXPECT_EQ(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+}
+
+TEST_F(TokenPoolTest, SuccessfulOldSetup) {
+ // GNUmake <= 4.1
+ CreatePool(AUTH_FORMAT("--jobserver-fds"));
+
+ EXPECT_NE(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+}
+
+TEST_F(TokenPoolTest, SuccessfulNewSetup) {
+ // GNUmake => 4.2
+ CreateDefaultPool();
+
+ EXPECT_NE(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+}
+
+TEST_F(TokenPoolTest, IgnoreWithJN) {
+ CreatePool(AUTH_FORMAT("--jobserver-auth"), true);
+
+ EXPECT_EQ(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+}
+
+TEST_F(TokenPoolTest, HonorLN) {
+ CreatePool(AUTH_FORMAT("-l9 --jobserver-auth"));
+
+ EXPECT_NE(NULL, tokens_);
+ EXPECT_EQ(9.0, load_avg_);
+}
+
+#ifdef _WIN32
+TEST_F(TokenPoolTest, SemaphoreNotFound) {
+ semaphore_name_ = SEMAPHORE_NAME "_foobar";
+ CreateDefaultPool();
+
+ EXPECT_EQ(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+}
+
+TEST_F(TokenPoolTest, TokenIsAvailable) {
+ CreateDefaultPool();
+
+ ASSERT_NE(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+
+ EXPECT_TRUE(tokens_->TokenIsAvailable((ULONG_PTR)tokens_));
+}
+#else
+TEST_F(TokenPoolTest, MonitorFD) {
+ CreateDefaultPool();
+
+ ASSERT_NE(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+
+ EXPECT_EQ(fds_[0], tokens_->GetMonitorFd());
+}
+#endif
+
+TEST_F(TokenPoolTest, ImplicitToken) {
+ CreateDefaultPool();
+
+ ASSERT_NE(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+
+ EXPECT_TRUE(tokens_->Acquire());
+ tokens_->Reserve();
+ EXPECT_FALSE(tokens_->Acquire());
+ tokens_->Release();
+ EXPECT_TRUE(tokens_->Acquire());
+}
+
+TEST_F(TokenPoolTest, TwoTokens) {
+ CreateDefaultPool();
+
+ ASSERT_NE(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+
+ // implicit token
+ EXPECT_TRUE(tokens_->Acquire());
+ tokens_->Reserve();
+ EXPECT_FALSE(tokens_->Acquire());
+
+ // jobserver offers 2nd token
+#ifdef _WIN32
+ LONG previous;
+ ASSERT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous));
+ ASSERT_EQ(0, previous);
+#else
+ ASSERT_EQ(1u, write(fds_[1], "T", 1));
+#endif
+ EXPECT_TRUE(tokens_->Acquire());
+ tokens_->Reserve();
+ EXPECT_FALSE(tokens_->Acquire());
+
+ // release 2nd token
+ tokens_->Release();
+ EXPECT_TRUE(tokens_->Acquire());
+
+ // release implict token - must return 2nd token back to jobserver
+ tokens_->Release();
+ EXPECT_TRUE(tokens_->Acquire());
+
+ // there must be one token available
+#ifdef _WIN32
+ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0));
+ EXPECT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous));
+ EXPECT_EQ(0, previous);
+#else
+ EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_)));
+#endif
+
+ // implicit token
+ EXPECT_TRUE(tokens_->Acquire());
+}
+
+TEST_F(TokenPoolTest, Clear) {
+ CreateDefaultPool();
+
+ ASSERT_NE(NULL, tokens_);
+ EXPECT_EQ(kLoadAverageDefault, load_avg_);
+
+ // implicit token
+ EXPECT_TRUE(tokens_->Acquire());
+ tokens_->Reserve();
+ EXPECT_FALSE(tokens_->Acquire());
+
+ // jobserver offers 2nd & 3rd token
+#ifdef _WIN32
+ LONG previous;
+ ASSERT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous));
+ ASSERT_EQ(0, previous);
+#else
+ ASSERT_EQ(2u, write(fds_[1], "TT", 2));
+#endif
+ EXPECT_TRUE(tokens_->Acquire());
+ tokens_->Reserve();
+ EXPECT_TRUE(tokens_->Acquire());
+ tokens_->Reserve();
+ EXPECT_FALSE(tokens_->Acquire());
+
+ tokens_->Clear();
+ EXPECT_TRUE(tokens_->Acquire());
+
+ // there must be two tokens available
+#ifdef _WIN32
+ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0));
+ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0));
+ EXPECT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous));
+ EXPECT_EQ(0, previous);
+#else
+ EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_)));
+#endif
+
+ // implicit token
+ EXPECT_TRUE(tokens_->Acquire());
+}
Index: src
===================================================================
--- src (nonexistent)
+++ src (revision 5)
Property changes on: src
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,73 ##
+
+# install dir
+dist
+
+# Target build dirs
+.a1x-newlib
+.a2x-newlib
+.at91sam7s-newlib
+
+.build-machine
+
+.a1x-glibc
+.a2x-glibc
+.h3-glibc
+.h5-glibc
+.i586-glibc
+.i686-glibc
+.imx6-glibc
+.jz47xx-glibc
+.makefile
+.am335x-glibc
+.omap543x-glibc
+.p5600-glibc
+.power8-glibc
+.power8le-glibc
+.power9-glibc
+.power9le-glibc
+.m1000-glibc
+.riscv64-glibc
+.rk328x-glibc
+.rk33xx-glibc
+.rk339x-glibc
+.s8xx-glibc
+.s9xx-glibc
+.x86_64-glibc
+
+# Hidden files (each file)
+.makefile
+.dist
+.rootfs
+
+# src & hw requires
+.src_requires
+.src_requires_depend
+.requires
+.requires_depend
+
+# Tarballs
+*.gz
+*.bz2
+*.lz
+*.xz
+*.tgz
+*.txz
+
+# Signatures
+*.asc
+*.sig
+*.sign
+*.sha1sum
+
+# Patches
+*.patch
+
+# Descriptions
+*.dsc
+*.txt
+
+# Default linux config files
+*.defconfig
+
+# backup copies
+*~
Index: .
===================================================================
--- . (nonexistent)
+++ . (revision 5)
Property changes on: .
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,73 ##
+
+# install dir
+dist
+
+# Target build dirs
+.a1x-newlib
+.a2x-newlib
+.at91sam7s-newlib
+
+.build-machine
+
+.a1x-glibc
+.a2x-glibc
+.h3-glibc
+.h5-glibc
+.i586-glibc
+.i686-glibc
+.imx6-glibc
+.jz47xx-glibc
+.makefile
+.am335x-glibc
+.omap543x-glibc
+.p5600-glibc
+.power8-glibc
+.power8le-glibc
+.power9-glibc
+.power9le-glibc
+.m1000-glibc
+.riscv64-glibc
+.rk328x-glibc
+.rk33xx-glibc
+.rk339x-glibc
+.s8xx-glibc
+.s9xx-glibc
+.x86_64-glibc
+
+# Hidden files (each file)
+.makefile
+.dist
+.rootfs
+
+# src & hw requires
+.src_requires
+.src_requires_depend
+.requires
+.requires_depend
+
+# Tarballs
+*.gz
+*.bz2
+*.lz
+*.xz
+*.tgz
+*.txz
+
+# Signatures
+*.asc
+*.sig
+*.sign
+*.sha1sum
+
+# Patches
+*.patch
+
+# Descriptions
+*.dsc
+*.txt
+
+# Default linux config files
+*.defconfig
+
+# backup copies
+*~
Radix cross Linux
The main Radix cross Linux repository contains the build scripts of packages, which have the most complete and common functionality for desktop machines
452 Commits
2 Branches
1 Tag