Merge pull request myint#110 from esquires/infer-system-includes

Infer system includes

.gitignore

@@ -11,3 +11,4 @@ htmlcov/
 *.egg*/
 .coverage
 .travis-solo/
+tags

cpp/ast.py

@@ -19,6 +19,8 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import os
+
 from . import keywords
 from . import tokenize
 
@@ -613,11 +615,14 @@ def get_template_indices(self, names):
 
 class ASTBuilder(object):
 
-    def __init__(self, token_stream, filename, in_class=None,
+    def __init__(self, token_stream, filename, system_includes=tuple(),
+                 nonsystem_includes=tuple(), in_class=None,
                  namespace_stack=None, quiet=False):
         if namespace_stack is None:
             namespace_stack = []
 
+        self.system_includes = system_includes
+        self.nonsystem_includes = nonsystem_includes
         self.tokens = token_stream
         self.filename = filename
         self.token_queue = []
@@ -742,13 +747,24 @@ def _generate_one(self, token):
                 if name.startswith('\\'):
                     name = name[1:].strip()
 
-                system = True
-                filename = name
-                if name[0] in '<"':
-                    assert_parse(name[-1] in '>"', token)
+                filename = name.strip('<>"')
+
+                def _is_file(prefix):
+                    return os.path.isfile(os.path.join(prefix, filename))
+
+                if any([d for d in self.system_includes if _is_file(d)]):
+                    system = True
+                elif any([d for d in self.nonsystem_includes if _is_file(d)]):
+                    system = False
+                else:
+                    system = True
+                    filename = name
+
+                    if name[0] in '<"':
+                        assert_parse(name[-1] in '>"', token)
+                        system = name[0] == '<'
+                        filename = name[1:-1]
 
-                    system = name[0] == '<'
-                    filename = name[1:-1]
                 return Include(token.start, token.end, filename, system)
             if name.startswith('define'):
                 # Remove "define".
@@ -1481,7 +1497,9 @@ def _get_class(self, class_type, templated_types):
         body = None
         if token.token_type == tokenize.SYNTAX and token.name == '{':
             name = class_name or '__unamed__'
-            ast = ASTBuilder(self.get_scope(), self.filename, name,
+            ast = ASTBuilder(self.get_scope(), self.filename,
+                             self.system_includes, self.nonsystem_includes,
+                             name,
                              self.namespace_stack,
                              quiet=self.quiet)
             body = list(ast.generate())
@@ -1548,7 +1566,8 @@ def handle_operator(self):
         pass
 
 
-def builder_from_source(source, filename, quiet=False):
+def builder_from_source(source, filename, system_includes,
+                        nonsystem_includes, quiet=False):
     """Utility method that returns an ASTBuilder from source code.
 
     Args:
@@ -1561,6 +1580,8 @@ def builder_from_source(source, filename, quiet=False):
     """
     return ASTBuilder(tokenize.get_tokens(source),
                       filename,
+                      system_includes,
+                      nonsystem_includes,
                       quiet=quiet)
 
 

cpp/find_warnings.py

@@ -89,11 +89,15 @@ class WarningHunter(object):
     # Cache filename: ast_list
     _module_cache = {}
 
-    def __init__(self, filename, source, ast_list, include_paths, quiet=False):
+    def __init__(self, filename, source, ast_list, include_paths,
+                 system_include_paths, nonsystem_include_paths,
+                 quiet=False):
         self.filename = filename
         self.source = source
         self.ast_list = ast_list
         self.include_paths = include_paths[:]
+        self.system_include_paths = system_include_paths
+        self.nonsystem_include_paths = nonsystem_include_paths
         self.quiet = quiet
         self.symbol_table = symbols.SymbolTable()
 
@@ -147,6 +151,8 @@ def _get_module(self, node):
             ast_list = None
             try:
                 builder = ast.builder_from_source(source, filename,
+                                                  self.system_include_paths,
+                                                  self.nonsystem_include_paths,
                                                   quiet=self.quiet)
                 ast_list = [_f for _f in builder.generate() if _f]
             except tokenize.TokenError:
@@ -587,9 +593,12 @@ def get_correct_include_filename(filename, candidate_filenames):
     return None
 
 
-def run(filename, source, entire_ast, include_paths, quiet):
+def run(filename, source, entire_ast, include_paths,
+        system_include_paths, nonsystem_include_paths, quiet):
     hunter = WarningHunter(filename, source, entire_ast,
                            include_paths=include_paths,
+                           system_include_paths=system_include_paths,
+                           nonsystem_include_paths=nonsystem_include_paths,
                            quiet=quiet)
     hunter.find_warnings()
     hunter.show_warnings()

cpp/nonvirtual_dtors.py

@@ -54,5 +54,6 @@ def _find_warnings(filename, source, ast_list):
     return count
 
 
-def run(filename, source, entire_ast, include_paths, quiet):
+def run(filename, source, entire_ast, include_paths,
+        system_include_paths, nonsystem_include_paths, quiet):
     return _find_warnings(filename, source, entire_ast)

cpp/static_data.py

@@ -110,7 +110,8 @@ def find_variables_use(body):
     return count
 
 
-def run(filename, source, entire_ast, include_paths, quiet):
+def run(filename, source, entire_ast, include_paths,
+        system_include_paths, nonsystem_include_paths, quiet):
     lines = metrics.Metrics(source)
 
     return (

cppclean

@@ -88,6 +88,19 @@ def main():
                         help='add a header include path; '
                              'specify this multiple times for multiple '
                              'include paths')
+    parser.add_argument('--include-path-system', '-S', action='append',
+                        dest='include_system_paths', default=[],
+                        metavar='sys_path',
+                        help=('same as --include-path but explicitly '
+                              'designates all header files found in these '
+                              'directories as "system" includes'))
+    parser.add_argument('--include-path-non-system', '-N',
+                        action='append', dest='include_nonsystem_paths',
+                        metavar='nonsys_path',
+                        default=[],
+                        help=('same as --include-path but explicitly '
+                              'designates all header files found in these '
+                              'directories as "non-system" includes'))
     parser.add_argument('--verbose', action='store_true',
                         help='print verbose messages')
     parser.add_argument('--version', action='version',
@@ -101,6 +114,10 @@ def main():
                   if hasattr(filename, 'decode') else filename
                   for filename in args.files]
 
+    all_includes = list(set(
+        args.include_paths + args.include_system_paths +
+        args.include_nonsystem_paths))
+
     status = 0
     for filename in (
         sorted(find_files(args.files,
@@ -116,6 +133,8 @@ def main():
 
             builder = ast.builder_from_source(source,
                                               filename,
+                                              args.include_system_paths,
+                                              args.include_nonsystem_paths,
                                               quiet=args.quiet)
             entire_ast = list([_f for _f in builder.generate() if _f])
         except tokenize.TokenError as exception:
@@ -134,7 +153,9 @@ def main():
                        nonvirtual_dtors,
                        static_data]:
             if module.run(filename, source, entire_ast,
-                          include_paths=args.include_paths,
+                          include_paths=all_includes,
+                          system_include_paths=args.include_system_paths,
+                          nonsystem_include_paths=args.include_nonsystem_paths,
                           quiet=args.quiet):
                 status = 1
 

test_ast.py

@@ -20,6 +20,7 @@
 from __future__ import absolute_import
 
 import unittest
+import os
 
 from cpp import ast
 from cpp import tokenize
@@ -1060,6 +1061,35 @@ def test_system_include(self):
         self.assertEqual(1, len(nodes), repr(nodes))
         self.assertEqual(Include('vector', system=True), nodes[0])
 
+    def test_include_path_overrides(self):
+        paths = [os.path.dirname(os.path.realpath(__file__))]
+        fname = "test/include.h"
+
+        def _tokens():
+            tokens_quotes = get_tokens('#include "test/include.h"')
+            tokens_brackets = get_tokens('#include <test/include.h>')
+            return [tokens_quotes, tokens_brackets]
+
+        def _do_test(tokens, system_includes, nonsystem_includes, is_system):
+            nodes = list(ast.ASTBuilder(
+                tokens, '<test>', system_includes=system_includes,
+                nonsystem_includes=nonsystem_includes).generate())
+            self.assertEqual(1, len(nodes), repr(nodes))
+            self.assertEqual(Include(fname, system=is_system), nodes[0])
+
+        # forcing system
+        for tokens in _tokens():
+            _do_test(tokens, paths, [], True)
+
+        # forcing nonsystem
+        for tokens in _tokens():
+            _do_test(tokens, [], paths, False)
+
+        # let the system infer
+        tokens_quotes, tokens_brackets = _tokens()
+        _do_test(tokens_quotes, [], [], False)
+        _do_test(tokens_brackets, [], [], True)
+
     def test_operator_new_bracket(self):
         nodes = list(
             MakeBuilder('void* operator new[](std::size_t size);').generate())
@@ -1143,5 +1173,6 @@ def test_inline_function(self):
             nodes[0])
 
 
+
 if __name__ == '__main__':
     unittest.main()