[CI] Add IWYU pass

cmake/static_analysis.cmake

+# Ancillary function retrieving direct sources absolute file path for direct
+# library dependencies (no recursion) for the given target
+function(get_source_files target_name out_files)
+  set(found_files "")
+  if(target_name)
+    get_target_property(target_source_dir ${target_name} SOURCE_DIR)
+    get_target_property(target_sources ${target_name} SOURCES)
+    foreach(target_source IN ITEMS ${target_sources})
+      cmake_path(SET source_path ${target_source})
+      cmake_path(GET source_path EXTENSION source_extension)
+      cmake_path(
+        ABSOLUTE_PATH
+        source_path
+        BASE_DIRECTORY
+        ${target_source_dir}
+        NORMALIZE
+        OUTPUT_VARIABLE
+        absolute_path)
+      list(APPEND found_files "${absolute_path}")
+    endforeach()
+  endif()
+  set(${out_files}
+      "${found_files}"
+      PARENT_SCOPE)
+endfunction()
+
 function(create_analysis_target target_name)
   add_custom_target(${target_name}_analysis DEPENDS ${target_name}_clang_tidy)
 
@@ -12,4 +38,27 @@ function(create_analysis_target target_name)
       "-header-filter=.*sandbox(?!.*_deps|.*cpm_cache).*"
       "-source-filter=.*sandbox(?!.*_deps|.*cpm_cache).*"
     VERBATIM USES_TERMINAL)
+
+  # Include what you use
+  find_program(IWYU_COMMAND NAMES iwyu)
+  # We feed it the compilation database, however we filter it to direct sources
+  # for direct library dependencies (no recursion) for every target
+  list(APPEND ${target_name}_sources)
+  get_target_property(${target_name}_libs ${target_name}
+                      INTERFACE_LINK_LIBRARIES)
+  foreach(library IN ITEMS ${${target_name}_libs})
+    set(library_sources "")
+    get_source_files(${library} library_sources)
+    list(APPEND ${target_name}_sources ${library_sources})
+  endforeach()
+  get_source_files(${target_name} target_sources)
+  list(APPEND ${target_name}_sources ${target_sources})
+
+  # We emit clang "errors" so they can be caught easily in the output log
+  # (as other passes will rather emit warnings)
+  add_custom_target(
+    ${target_name}_iwyu
+    COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/iwyu_tool.py
+            -p${CMAKE_BINARY_DIR} ${${target_name}_sources} -o clang-warning
+    VERBATIM USES_TERMINAL)
 endfunction()

scripts/analyse_log.sh

@@ -8,7 +8,7 @@ set -e
 
 echo "Analysing log"
 
-grep -e "warning:" -e "should add" -e "should remove" -i $1 | tee $2
+grep -e "warning:" -e "error:" -i $1 | tee $2
 
 if [ -s $2 ]; then
     warnings_count=$(cat $2 | wc -l)

scripts/iwyu_tool.py

+#!/usr/bin/env python3
+
+##===--- iwyu_tool.py -----------------------------------------------------===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+""" Driver to consume a Clang compilation database and invoke IWYU.
+
+Example usage with CMake:
+
+  # Unix systems
+  $ mkdir build && cd build
+  $ CC="clang" CXX="clang++" cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ...
+  $ iwyu_tool.py -p .
+
+  # Windows systems
+  $ mkdir build && cd build
+  $ cmake -DCMAKE_CXX_COMPILER="%VCINSTALLDIR%/bin/cl.exe" \
+    -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+    -G Ninja ...
+  $ python3 iwyu_tool.py -p .
+
+See iwyu_tool.py -h for more details on command-line arguments.
+"""
+from __future__ import print_function
+import os
+import re
+import sys
+import json
+import time
+import shlex
+import shutil
+import argparse
+import tempfile
+import subprocess
+
+
+CORRECT_RE = re.compile(r'^\((.*?) has correct #includes/fwd-decls\)$')
+SHOULD_ADD_RE = re.compile(r'^(.*?) should add these lines:$')
+ADD_RE = re.compile('^(.*?) +// (.*)$')
+SHOULD_REMOVE_RE = re.compile(r'^(.*?) should remove these lines:$')
+FULL_LIST_RE = re.compile(r'The full include-list for (.*?):$')
+END_RE = re.compile(r'^---$')
+LINES_RE = re.compile(r'^- (.*?)  // lines ([0-9]+)-[0-9]+$')
+
+
+GENERAL, ADD, REMOVE, LIST = range(4)
+
+
+def clang_formatter(output, style):
+    """ Process iwyu's output into something clang-like. """
+    formatted = []
+
+    state = (GENERAL, None)
+    for line in output.splitlines():
+        match = CORRECT_RE.match(line)
+        if match:
+            formatted.append('%s:1:1: note: #includes/fwd-decls are correct' %
+                             match.groups(1))
+            continue
+        match = SHOULD_ADD_RE.match(line)
+        if match:
+            state = (ADD, match.group(1))
+            continue
+        match = SHOULD_REMOVE_RE.match(line)
+        if match:
+            state = (REMOVE, match.group(1))
+            continue
+        match = FULL_LIST_RE.match(line)
+        if match:
+            state = (LIST, match.group(1))
+        elif END_RE.match(line):
+            state = (GENERAL, None)
+        elif not line.strip():
+            continue
+        elif state[0] == GENERAL:
+            formatted.append(line)
+        elif state[0] == ADD:
+            match = ADD_RE.match(line)
+            if match:
+                formatted.append("%s:1:1: %s: add '%s' (%s)" %
+                                 (state[1],
+                                  style,
+                                  match.group(1),
+                                  match.group(2)))
+            else:
+                formatted.append("%s:1:1: %s: add '%s'" %
+                                 (state[1], style, line))
+        elif state[0] == REMOVE:
+            match = LINES_RE.match(line)
+            line_no = match.group(2) if match else '1'
+            formatted.append("%s:%s:1: %s: superfluous '%s'" %
+                             (state[1], line_no, style, match.group(1)))
+
+    return os.linesep.join(formatted)
+
+
+DEFAULT_FORMAT = 'iwyu'
+FORMATTERS = {
+    'iwyu': lambda output: output,
+    'clang': lambda output: clang_formatter(output, style="error"),
+    'clang-warning': lambda output: clang_formatter(output, style="warning"),
+}
+
+
+if sys.platform.startswith('win'):
+    # Case-insensitive match on Windows
+    def normcase(s):
+        return s.lower()
+else:
+    def normcase(s):
+        return s
+
+
+def is_subpath_of(path, parent):
+    """ Return True if path is equal to or fully contained within parent.
+
+    Assumes both paths are canonicalized with os.path.realpath.
+    """
+    parent = normcase(parent)
+    path = normcase(path)
+
+    if path == parent:
+        return True
+
+    if not path.startswith(parent):
+        return False
+
+    # Now we know parent is a prefix of path, but they only share lineage if the
+    # difference between them starts with a path separator, e.g. /a/b/c/file
+    # is not a parent of /a/b/c/file.cpp, but /a/b/c and /a/b/c/ are.
+    parent = parent.rstrip(os.path.sep)
+    suffix = path[len(parent):]
+    return suffix.startswith(os.path.sep)
+
+
+def is_msvc_driver(compile_command):
+    """ Return True if compile_command matches an MSVC CL-style driver. """
+    compile_command = normcase(compile_command)
+
+    if compile_command.endswith('cl.exe'):
+        # Native MSVC compiler or clang-cl.exe
+        return True
+
+    if compile_command.endswith('clang-cl'):
+        # Cross clang-cl on non-Windows
+        return True
+
+    return False
+
+
+def win_split(cmdline):
+    """ Minimal implementation of shlex.split for Windows following
+    https://msdn.microsoft.com/en-us/library/windows/desktop/17w5ykft.aspx.
+    """
+    def split_iter(cmdline):
+        in_quotes = False
+        backslashes = 0
+        arg = ''
+        for c in cmdline:
+            if c == '\\':
+                # MSDN: Backslashes are interpreted literally, unless they
+                # immediately precede a double quotation mark.
+                # Buffer them until we know what comes next.
+                backslashes += 1
+            elif c == '"':
+                # Quotes can either be an escaped quote or the start of a quoted
+                # string. Paraphrasing MSDN:
+                # Before quotes, place one backslash in the arg for every pair
+                # of leading backslashes. If the number of backslashes is odd,
+                # retain the double quotation mark, otherwise interpret it as a
+                # string delimiter and switch state.
+                arg += '\\' * (backslashes // 2)
+                if backslashes % 2 == 1:
+                    arg += c
+                else:
+                    in_quotes = not in_quotes
+                backslashes = 0
+            elif c in (' ', '\t') and not in_quotes:
+                # MSDN: Arguments are delimited by white space, which is either
+                # a space or a tab [but only outside of a string].
+                # Flush any buffered backslashes and yield arg, unless empty.
+                arg += '\\' * backslashes
+                if arg:
+                    yield arg
+                    arg = ''
+                backslashes = 0
+            else:
+                # Flush buffered backslashes and append.
+                arg += '\\' * backslashes
+                arg += c
+                backslashes = 0
+
+        if arg:
+            arg += '\\' * backslashes
+            yield arg
+
+    return list(split_iter(cmdline))
+
+
+def split_command(cmdstr):
+    """ Split a command string into a list, respecting shell quoting. """
+    if sys.platform.startswith('win'):
+        # shlex.split does not work for Windows command-lines, so special-case
+        # to our own implementation.
+        cmd = win_split(cmdstr)
+    else:
+        cmd = shlex.split(cmdstr)
+
+    return cmd
+
+
+def find_include_what_you_use():
+    """ Find IWYU executable and return its full pathname. """
+    env_iwyu_path = os.environ.get('IWYU_BINARY')
+    if env_iwyu_path:
+        return os.path.realpath(env_iwyu_path)
+
+    # Search in same dir as this script.
+    iwyu_path = shutil.which('include-what-you-use',
+                             path=os.path.dirname(__file__))
+    if iwyu_path:
+        return os.path.realpath(iwyu_path)
+
+    # Search the system PATH.
+    iwyu_path = shutil.which('include-what-you-use')
+    if iwyu_path:
+        return os.path.realpath(iwyu_path)
+
+    return None
+
+
+IWYU_EXECUTABLE = find_include_what_you_use()
+
+
+class Process(object):
+    """ Manages an IWYU process in flight """
+    def __init__(self, proc, outfile):
+        self.proc = proc
+        self.outfile = outfile
+        self.output = None
+
+    def poll(self):
+        """ Return the exit code if the process has completed, None otherwise.
+        """
+        return self.proc.poll()
+
+    @property
+    def returncode(self):
+        return self.proc.returncode
+
+    def get_output(self):
+        """ Return stdout+stderr output of the process.
+
+        This call blocks until the process is complete, then returns the output.
+        """
+        if not self.output:
+            self.proc.wait()
+            self.outfile.seek(0)
+            self.output = self.outfile.read().decode("utf-8")
+            self.outfile.close()
+
+        return self.output
+
+    @classmethod
+    def start(cls, invocation):
+        """ Start a Process for the invocation and capture stdout+stderr. """
+        outfile = tempfile.TemporaryFile(prefix='iwyu')
+        process = subprocess.Popen(
+            invocation.command,
+            cwd=invocation.cwd,
+            stdout=outfile,
+            stderr=subprocess.STDOUT)
+        return cls(process, outfile)
+
+
+KNOWN_COMPILER_WRAPPERS=frozenset([
+    "ccache"
+])
+
+
+class Invocation(object):
+    """ Holds arguments of an IWYU invocation. """
+    def __init__(self, command, cwd):
+        self.command = command
+        self.cwd = cwd
+
+    def __str__(self):
+        return ' '.join(self.command)
+
+    @classmethod
+    def from_compile_command(cls, entry, extra_args):
+        """ Parse a JSON compilation database entry into new Invocation. """
+        if 'arguments' in entry:
+            # arguments is a command-line in list form.
+            command = entry['arguments']
+        elif 'command' in entry:
+            # command is a command-line in string form, split to list.
+            command = split_command(entry['command'])
+        else:
+            raise ValueError('Invalid compilation database entry: %s' % entry)
+
+        if command[0] in KNOWN_COMPILER_WRAPPERS:
+            # Remove the compiler wrapper from the command.
+            command = command[1:]
+
+        # Rewrite the compile command for IWYU
+        compile_command, compile_args = command[0], command[1:]
+        if is_msvc_driver(compile_command):
+            # If the compiler is cl-compatible, let IWYU be cl-compatible.
+            extra_args = ['--driver-mode=cl'] + extra_args
+
+        command = [IWYU_EXECUTABLE] + extra_args + compile_args
+        return cls(command, entry['directory'])
+
+    def start(self, verbose):
+        """ Run invocation and collect output. """
+        if verbose:
+            print('# %s' % self, file=sys.stderr)
+
+        return Process.start(self)
+
+
+def fixup_compilation_db(compilation_db):
+    """ Canonicalize paths in JSON compilation database. """
+    for entry in compilation_db:
+        # Convert relative paths to absolute ones if possible, based on the entry's directory.
+        if 'directory' in entry and not os.path.isabs(entry['file']):
+            entry['file'] = os.path.join(entry['directory'], entry['file'])
+
+        # Expand relative paths and symlinks
+        entry['file'] = os.path.realpath(entry['file'])
+
+    return compilation_db
+
+
+def slice_compilation_db(compilation_db, selection):
+    """ Return a new compilation database reduced to the paths in selection. """
+    if not selection:
+        return compilation_db
+
+    # Canonicalize selection paths to match compilation database.
+    selection = [os.path.realpath(p) for p in selection]
+
+    new_db = []
+    for path in selection:
+        if not os.path.exists(path):
+            print('warning: \'%s\' not found on disk.' % path, file=sys.stderr)
+            continue
+
+        found = [e for e in compilation_db if is_subpath_of(e['file'], path)]
+        if not found:
+            print('warning: \'%s\' not found in compilation database.' % path,
+                  file=sys.stderr)
+            continue
+
+        new_db.extend(found)
+
+    return new_db
+
+
+def worst_exit_code(worst, cur):
+    """Return the most extreme exit code of two.
+
+    Negative exit codes occur if the program exits due to a signal (Unix) or
+    structured exception (Windows). If we've seen a negative one before, keep
+    it, as it usually indicates a critical error.
+
+    Otherwise return the biggest positive exit code.
+    """
+    if cur < 0:
+        # Negative results take precedence, return the minimum
+        return min(worst, cur)
+    elif worst < 0:
+        # We know cur is non-negative, negative worst must be minimum
+        return worst
+    else:
+        # We know neither are negative, return the maximum
+        return max(worst, cur)
+
+
+def execute(invocations, verbose, formatter, jobs, max_load_average=0):
+    """ Launch processes described by invocations. """
+    exit_code = 0
+    if jobs == 1:
+        for invocation in invocations:
+            proc = invocation.start(verbose)
+            print(formatter(proc.get_output()))
+            exit_code = worst_exit_code(exit_code, proc.returncode)
+        return exit_code
+
+    pending = []
+    while invocations or pending:
+        # Collect completed IWYU processes and print results.
+        complete = [proc for proc in pending if proc.poll() is not None]
+        for proc in complete:
+            pending.remove(proc)
+            print(formatter(proc.get_output()))
+            exit_code = worst_exit_code(exit_code, proc.returncode)
+
+        # Schedule new processes if there's room.
+        capacity = jobs - len(pending)
+
+        if max_load_average > 0:
+            one_min_load_average, _, _ = os.getloadavg()
+            load_capacity = max_load_average - one_min_load_average
+            if load_capacity < 0:
+                load_capacity = 0
+            if load_capacity < capacity:
+                capacity = int(load_capacity)
+                if not capacity and not pending:
+                    # Ensure there is at least one job running.
+                    capacity = 1
+
+        pending.extend(i.start(verbose) for i in invocations[:capacity])
+        invocations = invocations[capacity:]
+
+        # Yield CPU.
+        time.sleep(0.0001)
+    return exit_code
+
+
+def main(compilation_db_path, source_files, verbose, formatter, jobs,
+         max_load_average, extra_args):
+    """ Entry point. """
+
+    if not IWYU_EXECUTABLE:
+        print('error: include-what-you-use executable not found',
+              file=sys.stderr)
+        return 1
+
+    try:
+        if os.path.isdir(compilation_db_path):
+            compilation_db_path = os.path.join(compilation_db_path,
+                                               'compile_commands.json')
+
+        # Read compilation db from disk.
+        compilation_db_path = os.path.realpath(compilation_db_path)
+        with open(compilation_db_path, 'r') as fileobj:
+            compilation_db = json.load(fileobj)
+    except IOError as why:
+        print('error: failed to parse compilation database: %s' % why,
+              file=sys.stderr)
+        return 1
+
+    compilation_db = fixup_compilation_db(compilation_db)
+    compilation_db = slice_compilation_db(compilation_db, source_files)
+
+    # Transform compilation db entries into a list of IWYU invocations.
+    invocations = [
+        Invocation.from_compile_command(e, extra_args) for e in compilation_db
+    ]
+
+    return execute(invocations, verbose, formatter, jobs, max_load_average)
+
+
+def _bootstrap(sys_argv):
+    """ Parse arguments and dispatch to main(). """
+
+    # This hackery is necessary to add the forwarded IWYU args to the
+    # usage and help strings.
+    def customize_usage(parser):
+        """ Rewrite the parser's format_usage. """
+        original_format_usage = parser.format_usage
+        parser.format_usage = lambda: original_format_usage().rstrip() + \
+                              ' -- [<IWYU args>]' + os.linesep
+
+    def customize_help(parser):
+        """ Rewrite the parser's format_help. """
+        original_format_help = parser.format_help
+
+        def custom_help():
+            """ Customized help string, calls the adjusted format_usage. """
+            helpmsg = original_format_help()
+            helplines = helpmsg.splitlines()
+            helplines[0] = parser.format_usage().rstrip()
+            return os.linesep.join(helplines) + os.linesep
+
+        parser.format_help = custom_help
+
+    # Parse arguments.
+    parser = argparse.ArgumentParser(
+        description='Include-what-you-use compilation database driver.',
+        epilog='Assumes include-what-you-use is available on the PATH.')
+    customize_usage(parser)
+    customize_help(parser)
+
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Print IWYU commands')
+    parser.add_argument('-o', '--output-format', type=str,
+                        choices=FORMATTERS.keys(), default=DEFAULT_FORMAT,
+                        help='Output format (default: %s)' % DEFAULT_FORMAT)
+    parser.add_argument('-j', '--jobs', type=int, default=1,
+                        nargs='?', const=0,
+                        help=('Number of concurrent subprocesses. If zero, '
+                              'will try to match the logical cores of the '
+                              'system.'))
+    parser.add_argument('-l', '--load', type=float, default=0,
+                        help=('Do not start new jobs if the 1min load average '
+                              'is greater than the provided value'))
+    parser.add_argument('-p', metavar='<build-path>', required=True,
+                        help='Compilation database path', dest='dbpath')
+    parser.add_argument('source', nargs='*',
+                        help=('Zero or more source files (or directories) to '
+                              'run IWYU on. Defaults to all in compilation '
+                              'database.'))
+
+    def partition_args(argv):
+        """ Split around '--' into driver args and IWYU args. """
+        try:
+            double_dash = argv.index('--')
+            return argv[:double_dash], argv[double_dash+1:]
+        except ValueError:
+            return argv, []
+    argv, extra_args = partition_args(sys_argv[1:])
+    args = parser.parse_args(argv)
+
+    jobs = args.jobs
+    if jobs == 0:
+        jobs = os.cpu_count() or 1
+
+    return main(args.dbpath, args.source, args.verbose,
+                FORMATTERS[args.output_format], jobs, args.load, extra_args)
+
+
+if __name__ == '__main__':
+    sys.exit(_bootstrap(sys.argv))