From cf9a060bdf5937ecbf1a01ae7c6400013230af62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Thu, 18 Jan 2024 10:37:12 +0100 Subject: [PATCH] Support enhancing the ECJ logs with API problems --- .../eclipse/tycho/apitools/ApiAnalysis.java | 16 +- .../tycho/apitools/ApiAnalysisMojo.java | 104 ++++++++--- .../tycho/apitools/ApiAnalysisResult.java | 9 + .../tycho/apitools/LogFileEnhancer.java | 163 ++++++++++++++++++ 4 files changed, 262 insertions(+), 30 deletions(-) create mode 100644 tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/LogFileEnhancer.java diff --git a/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysis.java b/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysis.java index 93ca134ea6..2471295c32 100644 --- a/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysis.java +++ b/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysis.java @@ -116,7 +116,7 @@ public class ApiAnalysis implements Serializable, Callable { @Override public ApiAnalysisResult call() throws Exception { - ApiAnalysisResult result = new ApiAnalysisResult(); + Platform.addLogListener((status, plugin) -> debug(status.toString())); IJobManager jobManager = Job.getJobManager(); jobManager.addJobChangeListener(new IJobChangeListener() { @@ -161,6 +161,7 @@ public void aboutToRun(IJobChangeEvent event) { BundleComponent projectComponent = getApiComponent(project, projectPath); IApiBaseline baseline = createBaseline(baselineBundles, baselineName + " - baseline"); ResolverError[] resolverErrors = projectComponent.getErrors(); + ApiAnalysisResult result = new ApiAnalysisResult(getVersion()); if (resolverErrors != null && resolverErrors.length > 0) { for (ResolverError error : resolverErrors) { result.addResolverError(error); @@ -187,6 +188,14 @@ public void aboutToRun(IJobChangeEvent event) { return result; } + private String getVersion() { + Bundle apiToolsBundle = FrameworkUtil.getBundle(ApiModelFactory.class); + if (apiToolsBundle != null) { + return apiToolsBundle.getVersion().toString(); + } + return "n/a"; + } + private void disableJVMDiscovery() { IEclipsePreferences instanceNode = InstanceScope.INSTANCE .getNode(LaunchingPlugin.getDefault().getBundle().getSymbolicName()); @@ -424,10 +433,7 @@ private Properties getPreferences() throws IOException { } private void printVersion() { - Bundle apiToolsBundle = FrameworkUtil.getBundle(ApiModelFactory.class); - if (apiToolsBundle != null) { - debug("API Tools version: " + apiToolsBundle.getVersion()); - } + debug("API Tools version: " + getVersion()); } private IApiBaseline createBaseline(Collection bundles, String name) throws CoreException { diff --git a/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysisMojo.java b/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysisMojo.java index 1edad48400..bf67f52071 100644 --- a/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysisMojo.java +++ b/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/ApiAnalysisMojo.java @@ -13,6 +13,7 @@ package org.eclipse.tycho.apitools; import java.io.File; +import java.io.IOException; import java.net.URI; import java.nio.file.Path; import java.util.Collection; @@ -102,6 +103,31 @@ public class ApiAnalysisMojo extends AbstractMojo { @Parameter(defaultValue = "${project.basedir}/.settings/org.eclipse.pde.api.tools.prefs") private File apiPreferences; + /** + * If given a folder, enhances the ECJ compiler logs with API errors so it can + * be analyzed by tools understanding that format + */ + @Parameter(defaultValue = "${project.build.directory}/compile-logs") + private File logDirectory; + + @Parameter(defaultValue = "true") + private boolean printProblems; + + @Parameter(defaultValue = "true") + private boolean printSummary; + + @Parameter(defaultValue = "true") + private boolean failOnError; + + @Parameter(defaultValue = "false") + private boolean failOnWarning; + + @Parameter(defaultValue = "false") + private boolean parallel; + + @Parameter(defaultValue = "false") + private boolean enhanceLogs; + @Component private EclipseWorkspaceManager workspaceManager; @@ -120,8 +146,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { return; } Optional eclipseProject = projectManager.getEclipseProject(project); - if (eclipseProject.isEmpty() - || !eclipseProject.get().hasNature(ApiPlugin.NATURE_ID)) { + if (eclipseProject.isEmpty() || !eclipseProject.get().hasNature(ApiPlugin.NATURE_ID)) { return; } @@ -154,21 +179,15 @@ public void execute() throws MojoExecutionException, MojoFailureException { throw new MojoFailureException("Start Framework failed!", e); } ApiAnalysisResult analysisResult; - synchronized (ApiAnalysisMojo.class) { - // due to - // https://gitlab.eclipse.org/eclipsefdn/helpdesk/-/issues/3885#note_1266412 we - // can not execute more than one analysis without excessive memory consumption - // unless this is fixed it is safer to only run one analysis at a time - try { - ApiAnalysis analysis = new ApiAnalysis(baselineBundles, dependencyBundles, project.getName(), - fileToPath(apiFilter), fileToPath(apiPreferences), fileToPath(project.getBasedir()), debug, - fileToPath(project.getArtifact().getFile()), - stringToPath(project.getBuild().getOutputDirectory())); - analysisResult = eclipseFramework.execute(analysis); - } catch (Exception e) { - throw new MojoExecutionException("Execute ApiApplication failed", e); - } finally { - eclipseFramework.close(); + if (parallel) { + analysisResult = performAnalysis(baselineBundles, dependencyBundles, eclipseFramework); + } else { + synchronized (ApiAnalysisMojo.class) { + // due to + // https://gitlab.eclipse.org/eclipsefdn/helpdesk/-/issues/3885#note_1266412 we + // can not execute more than one analysis without excessive memory consumption + // unless this is fixed it is safer to only run one analysis at a time + analysisResult = performAnalysis(baselineBundles, dependencyBundles, eclipseFramework); } } log.info("API Analysis finished in " + time(start) + "."); @@ -178,15 +197,26 @@ public void execute() throws MojoExecutionException, MojoFailureException { .collect(Collectors.groupingBy(IApiProblem::getSeverity)); List errors = problems.getOrDefault(ApiPlugin.SEVERITY_ERROR, List.of()); List warnings = problems.getOrDefault(ApiPlugin.SEVERITY_WARNING, List.of()); - log.info(errors.size() + " API ERRORS"); - log.info(warnings.size() + " API warnings"); - for (IApiProblem problem : errors) { - printProblem(problem, "API ERROR", log::error); + if (printSummary) { + log.info(errors.size() + " API ERRORS"); + log.info(warnings.size() + " API warnings"); } - for (IApiProblem problem : warnings) { - printProblem(problem, "API WARNING", log::warn); + if (printProblems) { + for (IApiProblem problem : errors) { + printProblem(problem, "API ERROR", log::error); + } + for (IApiProblem problem : warnings) { + printProblem(problem, "API WARNING", log::warn); + } } - if (errors.size() > 0) { + if (enhanceLogs && logDirectory != null && logDirectory.isDirectory()) { + try { + LogFileEnhancer.enhanceXml(logDirectory, analysisResult); + } catch (IOException e) { + log.warn("Can't enhance logs in directory " + logDirectory); + } + } + if (errors.size() > 0 && failOnError) { String msg = errors.stream().map(problem -> { if (problem.getResourcePath() == null) { return problem.getMessage(); @@ -195,6 +225,30 @@ public void execute() throws MojoExecutionException, MojoFailureException { }).collect(Collectors.joining(System.lineSeparator())); throw new MojoFailureException("There are API errors:" + System.lineSeparator() + msg); } + if (warnings.size() > 0 && failOnWarning) { + String msg = warnings.stream().map(problem -> { + if (problem.getResourcePath() == null) { + return problem.getMessage(); + } + return problem.getResourcePath() + ":" + problem.getLineNumber() + " " + problem.getMessage(); + }).collect(Collectors.joining(System.lineSeparator())); + throw new MojoFailureException("There are API warnings:" + System.lineSeparator() + msg); + } + } + } + + private ApiAnalysisResult performAnalysis(Collection baselineBundles, Collection dependencyBundles, + EclipseFramework eclipseFramework) throws MojoExecutionException { + try { + ApiAnalysis analysis = new ApiAnalysis(baselineBundles, dependencyBundles, project.getName(), + fileToPath(apiFilter), fileToPath(apiPreferences), fileToPath(project.getBasedir()), debug, + fileToPath(project.getArtifact().getFile()), + stringToPath(project.getBuild().getOutputDirectory())); + return eclipseFramework.execute(analysis); + } catch (Exception e) { + throw new MojoExecutionException("Execute ApiApplication failed", e); + } finally { + eclipseFramework.close(); } } @@ -211,7 +265,7 @@ private void printProblem(IApiProblem problem, String type, Consumer problems = new ArrayList<>(); private List resolveError = new ArrayList<>(); + private String version; + + public ApiAnalysisResult(String version) { + this.version = version; + } public Stream problems() { return problems.stream(); @@ -41,4 +46,8 @@ public void addProblem(IApiProblem problem, IProject project) { public void addResolverError(ResolverError error) { resolveError.add(new ResolverErrorDTO(error)); } + + public String getApiToolsVersion() { + return version; + } } diff --git a/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/LogFileEnhancer.java b/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/LogFileEnhancer.java new file mode 100644 index 0000000000..c425750e8f --- /dev/null +++ b/tycho-apitools-plugin/src/main/java/org/eclipse/tycho/apitools/LogFileEnhancer.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * Copyright (c) 2024 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.apitools; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; + +import org.eclipse.pde.api.tools.internal.provisional.ApiPlugin; +import org.eclipse.pde.api.tools.internal.provisional.problems.IApiProblem; + +import de.pdark.decentxml.Document; +import de.pdark.decentxml.Element; +import de.pdark.decentxml.XMLIOSource; +import de.pdark.decentxml.XMLParser; +import de.pdark.decentxml.XMLWriter; + +public class LogFileEnhancer { + + private static final String SEVERITY_ERROR = "ERROR"; + private static final String SEVERITY_WARNING = "WARNING"; + private static final String ATTRIBUTES_WARNINGS = "warnings"; + private static final String ELEMENT_PROBLEMS = "problems"; + private static final String ATTRIBUTES_PROBLEMS = "problems"; + private static final String ATTRIBUTES_INFOS = "infos"; + private static final String ATTRIBUTES_ERRORS = "errors"; + + public static void enhanceXml(File logDirectory, ApiAnalysisResult analysisResult) throws IOException { + Map> problems = analysisResult.problems() + .collect(Collectors.groupingBy(IApiProblem::getResourcePath)); + if (problems.isEmpty()) { + return; + } + Set needsUpdate = new HashSet<>(); + Map documents = readDocuments(logDirectory); + for (Entry> problemEntry : problems.entrySet()) { + String path = problemEntry.getKey(); + for (Entry documentEntry : documents.entrySet()) { + Document document = documentEntry.getValue(); + Element statsElement = getStatsElement(document); + for (Element sources : document.getRootElement().getChildren("sources")) { + for (Element source : sources.getChildren("source")) { + String pathAttribute = source.getAttributeValue("path"); + if (pathAttribute != null && !pathAttribute.isEmpty() && pathAttribute.endsWith(path)) { + needsUpdate.add(documentEntry.getKey()); + Element problemsElement = getProblemsElement(source); + List list = problemEntry.getValue(); + Map> problemsBySeverity = list.stream() + .collect(Collectors.groupingBy(IApiProblem::getSeverity)); + List errors = problemsBySeverity.getOrDefault(ApiPlugin.SEVERITY_ERROR, + List.of()); + List warnings = problemsBySeverity.getOrDefault(ApiPlugin.SEVERITY_WARNING, + List.of()); + incrementAttribute(problemsElement, ATTRIBUTES_PROBLEMS, list.size()); + incrementAttribute(problemsElement, ATTRIBUTES_WARNINGS, warnings.size()); + incrementAttribute(problemsElement, ATTRIBUTES_ERRORS, errors.size()); + if (statsElement != null) { + incrementAttribute(statsElement, ATTRIBUTES_PROBLEMS, list.size()); + incrementAttribute(statsElement, ATTRIBUTES_WARNINGS, warnings.size()); + incrementAttribute(statsElement, ATTRIBUTES_ERRORS, errors.size()); + } + for (IApiProblem problem : warnings) { + addProblem(problemsElement, problem, SEVERITY_WARNING); + } + for (IApiProblem problem : errors) { + addProblem(problemsElement, problem, SEVERITY_ERROR); + } + } + } + } + } + + } + writeDocuments(needsUpdate, documents); + } + + private static Element getStatsElement(Document document) { + for (Element stats : document.getRootElement().getChildren("stats")) { + for (Element problem_summary : stats.getChildren("problem_summary")) { + return problem_summary; + } + } + return null; + } + + private static void addProblem(Element problemsElement, IApiProblem problem, String severity) { + Element element = new Element("problem"); + element.setAttribute("line", Integer.toString(problem.getLineNumber())); + element.setAttribute("severity", severity); + element.setAttribute("charStart", Integer.toString(problem.getCharStart())); + element.setAttribute("charEnd", Integer.toString(problem.getCharEnd())); + element.setAttribute("categoryID", Integer.toString(problem.getCategory())); + element.setAttribute("problemID", Integer.toString(problem.getId())); + Element messageElement = new Element("message"); + messageElement.setAttribute("value", problem.getMessage()); + element.addNode(messageElement); + problemsElement.addNode(element); + } + + private static void incrementAttribute(Element element, String attribute, int increment) { + if (increment > 0) { + int current = Integer.parseInt(element.getAttributeValue(attribute)); + element.setAttribute(attribute, Integer.toString(current + increment)); + } + } + + private static void writeDocuments(Set needsUpdate, Map documents) + throws IOException, FileNotFoundException { + for (File file : needsUpdate) { + Document document = documents.get(file); + try (Writer w = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8); + XMLWriter xw = new XMLWriter(w)) { + document.toXML(xw); + } + } + } + + private static Map readDocuments(File logDirectory) throws IOException { + XMLParser parser = new XMLParser(); + Map documents = new HashMap<>(); + for (File child : logDirectory.listFiles()) { + if (child.getName().toLowerCase().endsWith(".xml")) { + documents.put(child, parser.parse(new XMLIOSource(child))); + } + } + return documents; + } + + private static Element getProblemsElement(Element source) { + Element element = source.getChild(ELEMENT_PROBLEMS); + if (element == null) { + element = new Element(ELEMENT_PROBLEMS); + element.setAttribute(ATTRIBUTES_ERRORS, "0"); + element.setAttribute(ATTRIBUTES_INFOS, "0"); + element.setAttribute(ATTRIBUTES_PROBLEMS, "0"); + element.setAttribute(ATTRIBUTES_WARNINGS, "0"); + source.addNode(0, element); + } + return element; + } + +}