From a617018b66e492b6f5cf230d9709e1e2eb441d5c Mon Sep 17 00:00:00 2001 From: Spicuzza Date: Sun, 8 Dec 2024 01:23:17 -0500 Subject: [PATCH] Add support for generating depfiles for pcpp/gcc --- cxxheaderparser/dump.py | 42 ++++++++++++++++++++----- cxxheaderparser/preprocessor.py | 56 +++++++++++++++++++++++++++++++-- tests/test_preprocessor.py | 44 ++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 9 deletions(-) diff --git a/cxxheaderparser/dump.py b/cxxheaderparser/dump.py index 18bcd91..70b89c3 100644 --- a/cxxheaderparser/dump.py +++ b/cxxheaderparser/dump.py @@ -1,6 +1,7 @@ import argparse import dataclasses import json +import pathlib import pprint import subprocess import sys @@ -28,23 +29,50 @@ def dumpmain() -> None: parser.add_argument( "--pcpp", default=False, action="store_true", help="Use pcpp preprocessor" ) + parser.add_argument( + "--gcc", default=False, action="store_true", help="Use GCC as preprocessor" + ) + parser.add_argument( + "--depfile", + default=None, + type=pathlib.Path, + help="Generate a depfile (requires preprocessor)", + ) + parser.add_argument( + "--deptarget", default=[], action="append", help="depfile target" + ) parser.add_argument( "--encoding", default=None, help="Use this encoding to open the file" ) args = parser.parse_args() + pp_kwargs = dict(encoding=args.encoding) + + if args.depfile: + if not (args.pcpp or args.gcc): + parser.error("--depfile requires either --pcpp or --gcc") + + pp_kwargs["depfile"] = args.depfile + pp_kwargs["deptarget"] = args.deptarget + preprocessor = None - if args.pcpp or args.mode == "pponly": + if args.gcc: + from .preprocessor import make_gcc_preprocessor + + preprocessor = make_gcc_preprocessor(**pp_kwargs) + + if args.pcpp or (args.mode == "pponly" and preprocessor is None): from .preprocessor import make_pcpp_preprocessor - preprocessor = make_pcpp_preprocessor(encoding=args.encoding) + preprocessor = make_pcpp_preprocessor(**pp_kwargs) - if args.mode == "pponly": - with open(args.header, "r", encoding=args.encoding) as fp: - pp_content = preprocessor(args.header, fp.read()) - sys.stdout.write(pp_content) - sys.exit(0) + if args.mode == "pponly": + assert preprocessor is not None + with open(args.header, "r", encoding=args.encoding) as fp: + pp_content = preprocessor(args.header, fp.read()) + sys.stdout.write(pp_content) + sys.exit(0) options = ParserOptions(verbose=args.verbose, preprocessor=preprocessor) data = parse_file(args.header, encoding=args.encoding, options=options) diff --git a/cxxheaderparser/preprocessor.py b/cxxheaderparser/preprocessor.py index f3f001f..aa37dd7 100644 --- a/cxxheaderparser/preprocessor.py +++ b/cxxheaderparser/preprocessor.py @@ -3,8 +3,10 @@ """ import io +import pathlib import re import os +import os.path import subprocess import sys import tempfile @@ -48,6 +50,8 @@ def make_gcc_preprocessor( encoding: typing.Optional[str] = None, gcc_args: typing.List[str] = ["g++"], print_cmd: bool = True, + depfile: typing.Optional[pathlib.Path] = None, + deptarget: typing.Optional[typing.List[str]] = None, ) -> PreprocessorFunction: """ Creates a preprocessor function that uses g++ to preprocess the input text. @@ -62,6 +66,9 @@ def make_gcc_preprocessor( :param encoding: If specified any include files are opened with this encoding :param gcc_args: This is the path to G++ and any extra args you might want :param print_cmd: Prints the gcc command as its executed + :param depfile: If specified, will generate a preprocessor depfile that contains + a list of include files that were parsed. Must also specify deptarget. + :param deptarget: List of targets to put in the depfile .. code-block:: python @@ -93,6 +100,16 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str: else: cmd.append(filename) + if depfile is not None: + if deptarget is None: + raise PreprocessorError( + "must specify deptarget if depfile is specified" + ) + cmd.append("-MD") + for target in deptarget: + cmd += ["-MQ", target] + cmd += ["-MF", str(depfile)] + if print_cmd: print("+", " ".join(cmd), file=sys.stderr) @@ -242,7 +259,9 @@ def on_comment(self, *ignored): pcpp = None -def _pcpp_filter(fname: str, fp: typing.TextIO) -> str: +def _pcpp_filter( + fname: str, fp: typing.TextIO, deps: typing.Optional[typing.Dict[str, bool]] +) -> str: # the output of pcpp includes the contents of all the included files, which # isn't what a typical user of cxxheaderparser would want, so we strip out # the line directives and any content that isn't in our original file @@ -255,6 +274,9 @@ def _pcpp_filter(fname: str, fp: typing.TextIO) -> str: for line in fp: if line.startswith("#line"): keep = line.endswith(line_ending) + if deps is not None: + start = line.find('"') + deps[line[start + 1 : -2]] = True if keep: new_output.write(line) @@ -270,6 +292,8 @@ def make_pcpp_preprocessor( retain_all_content: bool = False, encoding: typing.Optional[str] = None, passthru_includes: typing.Optional["re.Pattern"] = None, + depfile: typing.Optional[pathlib.Path] = None, + deptarget: typing.Optional[typing.List[str]] = None, ) -> PreprocessorFunction: """ Creates a preprocessor function that uses pcpp (which must be installed @@ -285,6 +309,10 @@ def make_pcpp_preprocessor( :param encoding: If specified any include files are opened with this encoding :param passthru_includes: If specified any #include directives that match the compiled regex pattern will be part of the output. + :param depfile: If specified, will generate a preprocessor depfile that contains + a list of include files that were parsed. Must also specify deptarget. + Not compatible with retain_all_content + :param deptarget: List of targets to put in the depfile .. code-block:: python @@ -309,6 +337,8 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str: if not retain_all_content: pp.line_directive = "#line" + elif depfile: + raise PreprocessorError("retain_all_content and depfile not compatible") if content is None: with open(filename, "r", encoding=encoding) as fp: @@ -327,6 +357,16 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str: if retain_all_content: return fp.read() else: + deps: typing.Optional[typing.Dict[str, bool]] = None + target = None + if depfile: + deps = {} + if not deptarget: + base, _ = os.path.splitext(filename) + target = f"{base}.o" + else: + target = " ".join(deptarget) + # pcpp emits the #line directive using the filename you pass in # but will rewrite it if it's on the include path it uses. This # is copied from pcpp: @@ -339,6 +379,18 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str: filename = filename.replace(os.sep, "/") break - return _pcpp_filter(filename, fp) + filtered = _pcpp_filter(filename, fp, deps) + + if depfile is not None: + assert deps is not None + with open(depfile, "w") as fp: + fp.write(f"{target}:") + for dep in reversed(list(deps.keys())): + dep = dep.replace("\\", "\\\\") + dep = dep.replace(" ", "\\ ") + fp.write(f" \\\n {dep}") + fp.write("\n") + + return filtered return _preprocess_file diff --git a/tests/test_preprocessor.py b/tests/test_preprocessor.py index bff60c1..83e6251 100644 --- a/tests/test_preprocessor.py +++ b/tests/test_preprocessor.py @@ -202,3 +202,47 @@ def test_preprocessor_passthru_includes(tmp_path: pathlib.Path) -> None: assert data == ParsedData( namespace=NamespaceScope(), includes=[Include(filename='"t2.h"')] ) + + +def test_preprocessor_depfile( + make_pp: typing.Callable[..., PreprocessorFunction], + tmp_path: pathlib.Path, +) -> None: + + tmp_path = tmp_path / "hard path" + tmp_path.mkdir(parents=True, exist_ok=True) + + # not supported + if make_pp is preprocessor.make_msvc_preprocessor: + return + + h_content = '#include "t2.h"' "\n" "int x = X;\n" + h2_content = '#include "t3.h"\n' "#define X 2\n" "int omitted = 1;\n" + h3_content = "int h3;" + + with open(tmp_path / "t1.h", "w") as fp: + fp.write(h_content) + + with open(tmp_path / "t2.h", "w") as fp: + fp.write(h2_content) + + with open(tmp_path / "t3.h", "w") as fp: + fp.write(h3_content) + + depfile = tmp_path / "t1.d" + deptarget = ["tgt"] + + options = ParserOptions(preprocessor=make_pp(depfile=depfile, deptarget=deptarget)) + parse_file(tmp_path / "t1.h", options=options) + + with open(depfile) as fp: + depcontent = fp.read() + + assert depcontent.startswith("tgt:") + deps = [d.strip() for d in depcontent[4:].strip().split("\\\n")] + deps = [d.replace("\\ ", " ").replace("\\\\", "\\") for d in deps if d] + + # gcc will insert extra paths of predefined stuff, so just make sure this is sane + assert str(tmp_path / "t1.h") in deps + assert str(tmp_path / "t2.h") in deps + assert str(tmp_path / "t3.h") in deps