diff --git a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/CxxPreprocessor.java b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/CxxPreprocessor.java index a09938f78d..738831dd48 100644 --- a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/CxxPreprocessor.java +++ b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/CxxPreprocessor.java @@ -76,7 +76,6 @@ public class CxxPreprocessor extends Preprocessor { - private static final String CPLUSPLUS = "__cplusplus"; private static final String EVALUATED_TO_FALSE = "[{}:{}]: '{}' evaluated to false, skipping tokens that follow"; private static final String MISSING_INCLUDE_MSG = "Preprocessor: {} include directive error(s). This is only relevant if parser creates syntax errors." @@ -90,8 +89,55 @@ public class CxxPreprocessor extends Preprocessor { private File currentContextFile; private final Parser pplineParser; + /** + * Contains a standard set of pre-compiled macros, which are defined for each + * compilation unit. This map is used if there is no specific compilation unit + * settings. The map contains + *
    + *
  1. {@link Macro#STANDARD_MACROS}
    2. + *
  2. {@link CxxConfiguration#getDefines()} and
    3. + *
  3. forced includes (see + * {@link CxxConfiguration#getForceIncludeFiles()})
    4. + *
+ * All hi-prio macros are pre-parsed while construction of + * {@link CxxPreprocessor} + */ private final MapChain fixedMacros = new MapChain<>(); + /** + * If CxxConfiguration contains any compilation unit settings, this map is + * filled with a set of pre-compiled macros. Those macros must be defined for + * each compilation unit: + *
    + *
  1. {@link Macro#UNIT_MACROS}
    2. + *
  2. {@link CxxConfiguration#getDefines()} and
    3. + *
  3. forced includes (see + * {@link CxxConfiguration#getForceIncludeFiles()})
    4. + *
+ * The map is immutable; macros are pre-parsed while construction of + * {@link CxxPreprocessor} + */ + private final Map predefinedUnitMacros; + /** + * If current processed file has some specific configuration settings, this + * map will be filled with relevant macros and defines: + *
    + *
  1. predefined compilation unit macros will be added (see + * {@link CxxPreprocessor#predefinedUnitMacros}
    2. + *
  2. specific unit settings will be parsed and added (see + * {@link CxxConfiguration#getCompilationUnitSettings(String)}
    3. + *
+ * Map is recalculated each time {@link CxxPreprocessor} is about to analyze a + * new file (see {@link CxxPreprocessor#init()}. + * + * If current processed file has no specific configuration settings - + * {@link CxxPreprocessor#fixedMacros} will be used. + */ private MapChain unitMacros; + /** + * Pre-parsed defines from the global compilation unit settings, see + * {@link CxxConfiguration#getGlobalCompilationUnitSettings()}. + */ + private final Map globalUnitMacros; private final Set analysedFiles = new HashSet<>(); private final SourceCodeProvider codeProvider; private SourceCodeProvider unitCodeProvider; @@ -127,36 +173,82 @@ public CxxPreprocessor(SquidAstVisitorContext context, pplineParser = CppParser.create(conf); + final Map configuredMacros = parseConfiguredMacros(); + fillFixedMacros(configuredMacros); + predefinedUnitMacros = parsePredefinedUnitMacros(configuredMacros); + globalUnitMacros = parseGlobalUnitMacros(); + + ctorInProgress = false; + } + + private Map parseConfiguredMacros() { + final List configuredDefines = conf.getDefines(); + if (configuredDefines.isEmpty()) { + return Collections.emptyMap(); + } + LOG.debug("parsing configured defines"); + return parseMacroDefinitions(configuredDefines); + } + + private void fillFixedMacros(Map configuredMacros) { + if (!ctorInProgress || (getMacros() != fixedMacros) || !fixedMacros.getHighPrioMap().isEmpty()) { + throw new IllegalStateException("Preconditions for initial fill-out of fixedMacros were violated"); + } + try { getMacros().setHighPrio(true); + getMacros().putAll(Macro.STANDARD_MACROS); + getMacros().putAll(configuredMacros); + parseForcedIncludes(); + } finally { + getMacros().setHighPrio(false); + } + } - // parse the configured defines and store into the macro library - for (String define : conf.getDefines()) { - LOG.debug("parsing external macro: '{}'", define); - if (!"".equals(define)) { - Macro macro = parseMacroDefinition("#define " + define); - LOG.debug("storing external macro: '{}'", macro); - getMacros().put(macro.name, macro); - } - } + /** + * Create temporary unitMacros map; This map will be used as an active macros' + * storage while parsing of forced includes. After parsing was over extract + * resulting macros and destroy the unitMacros. fixedMacros will be set as + * active macros again. + */ + private Map parsePredefinedUnitMacros(Map configuredMacros) { + if (!ctorInProgress || (unitMacros != null)) { + throw new IllegalStateException("Preconditions for initial fill-out of predefinedUnitMacros were violated"); + } - // set standard macros - getMacros().putAll(Macro.STANDARD_MACROS); + if (conf.getCompilationUnitSourceFiles().isEmpty() && (conf.getGlobalCompilationUnitSettings() == null)) { + // configuration doesn't contain any settings for compilation units. + // CxxPreprocessor will use fixedMacros only + return Collections.emptyMap(); + } - // parse the configured force includes and store into the macro library - for (String include : conf.getForceIncludeFiles()) { - LOG.debug("parsing force include: '{}'", include); - if (!"".equals(include)) { - parseIncludeLine("#include \"" + include + "\"", "sonar." + this.language.getPropertiesKey() - + ".forceIncludes", conf.getEncoding()); - } - } + unitMacros = new MapChain<>(); + if (getMacros() != unitMacros) { + throw new IllegalStateException("expected unitMacros as active macros map"); + } + + try { + getMacros().setHighPrio(true); + getMacros().putAll(Macro.UNIT_MACROS); + getMacros().putAll(configuredMacros); + parseForcedIncludes(); + final HashMap result = new HashMap<>(unitMacros.getHighPrioMap()); + return result; } finally { - getMacros().setHighPrio(false); - ctorInProgress = false; + getMacros().setHighPrio(false); // just for the symmetry + unitMacros = null; // remove unitMacros, switch getMacros() to fixedMacros } } + private Map parseGlobalUnitMacros() { + final CxxCompilationUnitSettings globalCUSettings = conf.getGlobalCompilationUnitSettings(); + if (globalCUSettings == null) { + return Collections.emptyMap(); + } + LOG.debug("parsing global compilation unit defines"); + return parseMacroDefinitions(globalCUSettings.getDefines()); + } + public static void finalReport() { if (missingIncludeFilesCounter != 0) { LOG.warn(MISSING_INCLUDE_MSG, missingIncludeFilesCounter); @@ -457,14 +549,14 @@ public void init() { Objects.requireNonNull(context.getFile(), "SquidAstVisitorContext::getFile() must be non-null"); compilationUnitSettings = conf.getCompilationUnitSettings(currentContextFile.getAbsolutePath()); + boolean useGlobalCUSettings = false; + if (compilationUnitSettings != null) { LOG.debug("compilation unit settings for: '{}'", currentContextFile); - } else { + } else if (conf.getGlobalCompilationUnitSettings() != null) { compilationUnitSettings = conf.getGlobalCompilationUnitSettings(); - - if (compilationUnitSettings != null) { - LOG.debug("global compilation unit settings for: '{}'", currentContextFile); - } + useGlobalCUSettings = true; + LOG.debug("global compilation unit settings for: '{}'", currentContextFile); } if (compilationUnitSettings != null) { @@ -478,54 +570,32 @@ public void init() { // Treat all global defines as high prio getMacros().setHighPrio(true); - // parse the configured defines and store into the macro library - for (String define : conf.getDefines()) { - LOG.debug("parsing external macro to unit: '{}'", define); - if (!"".equals(define)) { - Macro macro = parseMacroDefinition("#define " + define); - if (macro != null) { - LOG.debug("storing external macro to unit: '{}'", macro); - getMacros().put(macro.name, macro); - } - } - } - - // set standard macros - // using smaller set of defines as rest is provides by compilation unit settings - getMacros().putAll(Macro.UNIT_MACROS); - - // parse the configured force includes and store into the macro library - for (String include : conf.getForceIncludeFiles()) { - LOG.debug("parsing force include to unit: '{}'", include); - if (!"".equals(include)) { - // TODO -> this needs to come from language - parseIncludeLine("#include \"" + include + "\"", "sonar.cxx.forceIncludes", conf.getEncoding()); - } - } + // add macros which are predefined for each compilation unit + getMacros().putAll(predefinedUnitMacros); // rest of defines comes from compilation unit settings - for (Map.Entry entry : compilationUnitSettings.getDefines().entrySet()) { - final String name = entry.getKey(); - final String body = entry.getValue(); - getMacros().put(name, new Macro(name, body)); + if (useGlobalCUSettings) { + getMacros().putAll(globalUnitMacros); + } else { + getMacros().putAll(parseMacroDefinitions(compilationUnitSettings.getDefines())); } } finally { getMacros().setHighPrio(false); } - if (getMacro(CPLUSPLUS) == null) { + if (getMacro(Macro.CPLUSPLUS) == null) { //Create macros to replace C++ keywords when parsing C files getMacros().putAll(Macro.COMPATIBILITY_MACROS); } } else { // Use global settings LOG.debug("global settings for: '{}'", currentContextFile); - if (isCFile(currentContextFile.getAbsolutePath())) { + if (isCFile(currentContextFile.getName())) { //Create macros to replace C++ keywords when parsing C files getMacros().putAll(Macro.COMPATIBILITY_MACROS); - fixedMacros.disable(CPLUSPLUS); + getMacros().disable(Macro.CPLUSPLUS); } else { - fixedMacros.enable(CPLUSPLUS); + getMacros().enable(Macro.CPLUSPLUS); } } } @@ -710,6 +780,21 @@ private int expandFunctionLikeMacro(String macroName, List restTokens, Li return tokensConsumedMatchingArgs; } + /** + * Parse the configured forced includes and store into the macro library. + * Current macro library depends on the return value of + * CxxPreprocessor#getMacros() + */ + private void parseForcedIncludes() { + for (String include : conf.getForceIncludeFiles()) { + if (!include.isEmpty()) { + LOG.debug("parsing force include: '{}'", include); + parseIncludeLine("#include \"" + include + "\"", "sonar." + this.language.getPropertiesKey() + ".forceIncludes", + conf.getEncoding()); + } + } + } + private List expandMacro(String macroName, String macroExpression) { // C++ standard 16.3.4/2 Macro Replacement - Rescanning and further replacement List tokens = null; @@ -855,6 +940,42 @@ private Macro parseMacroDefinition(String macroDef) { .getFirstDescendant(CppGrammar.defineLine)); } + /** + * Parse defines spited into key-value format + * (sonar.cxx.jsonCompilationDatabase) + */ + private Map parseMacroDefinitions(Map defines) { + final List margedDefines = defines.entrySet().stream().map(e -> e.getKey() + " " + e.getValue()) + .collect(Collectors.toList()); + return parseMacroDefinitions(margedDefines); + } + + /** + * Parse defines, which are merged into one string (see sonar.cxx.defines) + */ + private Map parseMacroDefinitions(List defines) { + final Map result = new HashMap<>(); + + for (String define : defines) { + if (define.isEmpty()) { + continue; + } + + final String defineString = "#define " + define; + + LOG.debug("parsing external macro: '{}'", defineString); + final Macro macro = parseMacroDefinition(defineString); + + if (macro != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("storing external macro: '{}'", macro); + } + result.put(macro.name, macro); + } + } + return result; + } + private File findIncludedFile(AstNode ast, Token token, String currFileName) { String includedFileName = null; boolean quoted = false; diff --git a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/Macro.java b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/Macro.java index 610a6aaff3..a6ea5ddc4d 100644 --- a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/Macro.java +++ b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/Macro.java @@ -61,7 +61,7 @@ public Macro(String name, @Nullable List params, @Nullable List bo * @param name * @param body */ - public Macro(String name, String body) { + private Macro(String name, String body) { this.name = name; this.params = null; this.body = Collections.singletonList(Token.builder() @@ -96,6 +96,8 @@ private static void add(Map map, String name, String body) { map.put(name, new Macro(name, body)); } + public static final String CPLUSPLUS = "__cplusplus"; + /** * This is a collection of standard macros according to * http://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html @@ -111,7 +113,7 @@ private static void add(Map map, String name, String body) { add(STANDARD_MACROS_IMPL, "__TIME__", "\"??:??:??\""); add(STANDARD_MACROS_IMPL, "__STDC__", "1"); add(STANDARD_MACROS_IMPL, "__STDC_HOSTED__", "1"); - add(STANDARD_MACROS_IMPL, "__cplusplus", "201103L"); + add(STANDARD_MACROS_IMPL, CPLUSPLUS, "201103L"); // __has_include support (C++17) add(STANDARD_MACROS_IMPL, "__has_include", "1"); } diff --git a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/MapChain.java b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/MapChain.java index 9c4c0b1fca..aa4de8c11c 100644 --- a/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/MapChain.java +++ b/cxx-squid/src/main/java/org/sonar/cxx/preprocessor/MapChain.java @@ -19,6 +19,7 @@ */ package org.sonar.cxx.preprocessor; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -120,4 +121,8 @@ private void move(K key, Map from, Map to) { } } + public Map getHighPrioMap() { + return Collections.unmodifiableMap(highPrioMap); + } + } diff --git a/cxx-squid/src/test/java/org/sonar/cxx/lexer/CxxLexerWithPreprocessingTest.java b/cxx-squid/src/test/java/org/sonar/cxx/lexer/CxxLexerWithPreprocessingTest.java index 3c931f44dc..973b6e64a8 100644 --- a/cxx-squid/src/test/java/org/sonar/cxx/lexer/CxxLexerWithPreprocessingTest.java +++ b/cxx-squid/src/test/java/org/sonar/cxx/lexer/CxxLexerWithPreprocessingTest.java @@ -27,10 +27,12 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import org.assertj.core.api.SoftAssertions; import org.junit.Test; +import org.mockito.Mockito; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -630,6 +632,89 @@ public void externalMacrosCannotBeOverriden() { softly.assertAll(); } + /** + * Test the expansion of default macros. Document the reference value of + * __LINE__ == 1 + */ + @Test + public void defaultMacros() { + CxxConfiguration conf = mock(CxxConfiguration.class); + when(conf.getDefines()).thenReturn(Arrays.asList()); + CxxPreprocessor cxxpp = new CxxPreprocessor(context, conf, language); + + final Lexer l = CxxLexer.create(conf, cxxpp); + final List tokens = l.lex("__LINE__"); + + SoftAssertions softly = new SoftAssertions(); + softly.assertThat(tokens).hasSize(2); // __LINE__ + EOF + softly.assertThat(tokens).anySatisfy(token -> assertThat(token).isValue("1").hasType(CxxTokenType.NUMBER)); + softly.assertAll(); + } + + /** + * Configured defines override default macros. This is equivalent to the + * standard preprocessor behavior:
+ * + * main.cpp: printf("%d", __LINE__); + * g++ -D__LINE__=123 main.cpp && ./a.out + * + * Expected Output: 123 + * + */ + @Test + public void configuredDefinesOverrideDefaultMacros() { + CxxConfiguration conf = mock(CxxConfiguration.class); + when(conf.getDefines()).thenReturn(Arrays.asList("__LINE__ 123")); + CxxPreprocessor cxxpp = new CxxPreprocessor(context, conf, language); + + final Lexer l = CxxLexer.create(conf, cxxpp); + final List tokens = l.lex("__LINE__"); + + SoftAssertions softly = new SoftAssertions(); + softly.assertThat(tokens).hasSize(2); // __LINE__ + EOF + softly.assertThat(tokens).anySatisfy(token -> assertThat(token).isValue("123").hasType(CxxTokenType.NUMBER)); + softly.assertAll(); + } + + /** + * Forced includes override configured defines and default macros (similar to + * the fact that [included] #define directives override configured defines). + * This is equivalent to the standard preprocessor behavior:
+ * + * main.cpp: #define __LINE__ 345 + * printf("%d", __LINE__); + * g++ -D__LINE__=123 main.cpp && ./a.out + * + * Expected Output: 345 + * + */ + @Test + public void forcedIncludesOverrideConfiguredDefines() throws IOException { + final String forceIncludePath = "/home/user/force.h"; + final File forceIncludeFile = new File(forceIncludePath); + + final CxxConfiguration conf = new CxxConfiguration(); + conf.setForceIncludeFiles(Collections.singletonList(forceIncludePath)); + conf.setDefines(new String[] { "__LINE__ 123" }); + conf.setErrorRecoveryEnabled(false); + + final SourceCodeProvider provider = mock(SourceCodeProvider.class); + when(provider.getSourceCodeFile(Mockito.eq(forceIncludePath), Mockito.any(String.class), Mockito.anyBoolean())) + .thenReturn(forceIncludeFile); + when(provider.getSourceCode(Mockito.eq(forceIncludeFile), Mockito.any(Charset.class))) + .thenReturn("#define __LINE__ 345"); + + final CxxPreprocessor cxxpp = new CxxPreprocessor(context, conf, provider, language); + final Lexer l = CxxLexer.create(conf, cxxpp); + + final List tokens = l.lex("__LINE__\n"); + + SoftAssertions softly = new SoftAssertions(); + softly.assertThat(tokens).hasSize(2); // __LINE__ + EOF + softly.assertThat(tokens).anySatisfy(token -> assertThat(token).isValue("345").hasType(CxxTokenType.NUMBER)); + softly.assertAll(); + } + @Test public void elif_expression() { List tokens = lexer.lex("#if 0\n"