From f40328375bfa289242f942fb3d992032ab662e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3zala?= <377355+jozala@users.noreply.github.com> Date: Mon, 25 Nov 2024 08:22:03 +0100 Subject: [PATCH] [7.17] [ci] Add debian-12 to matrix in packaging and platform jobs (#116172) (#117336) * [ci] Add debian-12 to matrix in packaging and platform jobs (#116172) Lintian test has been changed to parse the result instead of using exit code. This was required, because now `mismatched-override` is non-erasable tag which cannot be ignored for exit code. Lintian introduced non-backward-compatible format change for overrides file. Because of that, some overrides are now duplicated in a format for older versions. Additionally, Lintian overrides file has been cleaned up to remove the tags which are no longer failing. (cherry picked from commit 2ac267de3a0f14f62d426ae47b4dc1adc1fe9161) # Conflicts: # distribution/packages/src/deb/lintian/elasticsearch # qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java # qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java * Spotless fix --- .../pipelines/periodic-packaging.template.yml | 1 + .buildkite/pipelines/periodic-packaging.yml | 1 + .../pipelines/periodic-platform-support.yml | 1 + .../pull-request/packaging-tests-unix.yml | 3 + distribution/packages/build.gradle | 1 - .../packages/src/deb/lintian/elasticsearch | 58 ++++++--- .../packaging/test/DebMetadataTests.java | 43 ++++++- .../packaging/util/LintianResultParser.java | 110 ++++++++++++++++++ 8 files changed, 195 insertions(+), 23 deletions(-) create mode 100644 qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index 96fa433752450..c0d6a9d485c39 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -8,6 +8,7 @@ steps: setup: image: - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 669a78c0abc94..5723c911c57d7 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -9,6 +9,7 @@ steps: setup: image: - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index e86def9a465dd..601c6bb215d99 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -8,6 +8,7 @@ steps: setup: image: - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index e94baac8d9448..11cfed5902564 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -11,6 +11,7 @@ steps: setup: image: - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 @@ -38,6 +39,7 @@ steps: setup: image: - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 @@ -65,6 +67,7 @@ steps: setup: image: - debian-11 + - debian-12 - opensuse-leap-15 - oraclelinux-7 - oraclelinux-8 diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 389333197d4b9..b54bf6fe7180e 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -359,7 +359,6 @@ Closure commonDebConfig(boolean oss, boolean jdk, String architecture) { // versions found on oldest supported distro, centos-6 requires('bash', '4.1', GREATER | EQUAL) - requires('lsb-base', '4', GREATER | EQUAL) requires 'libc6' requires 'adduser' diff --git a/distribution/packages/src/deb/lintian/elasticsearch b/distribution/packages/src/deb/lintian/elasticsearch index 05c25a40ba544..c312f629c4eba 100644 --- a/distribution/packages/src/deb/lintian/elasticsearch +++ b/distribution/packages/src/deb/lintian/elasticsearch @@ -1,10 +1,10 @@ # we don't have a changelog, but we put our copyright file # under /usr/share/doc/elasticsearch, which triggers this warning +# Note that this is renamed to `no-changelog` in newer versions of +# lintian, but we still support Debian 8+, so we can't change this. changelog-file-missing-in-native-package # we intentionally copy our copyright file for all deb packages -copyright-file-contains-full-apache-2-license -copyright-should-refer-to-common-license-file-for-apache-2 copyright-without-copyright-notice # we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages @@ -13,6 +13,43 @@ arch-dependent-file-in-usr-share # we have a bundled jdk, so don't use jarwrapper missing-dep-on-jarwrapper +# we prefer to not make our config and log files world readable +non-standard-file-perm 0660 != 0644 [etc/default/elasticsearch] +non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/] +non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/jvm.options.d/] +non-standard-file-perm 0660 != 0644 [etc/elasticsearch/*] +non-standard-dir-perm 2750 != 0755 [var/lib/elasticsearch/] +non-standard-dir-perm 2750 != 0755 [var/log/elasticsearch/] +non-standard-executable-perm 0750 != 0755 [etc/init.d/elasticsearch] +executable-is-not-world-readable 0750 [etc/init.d/elasticsearch] +non-standard-file-permissions-for-etc-init.d-script 0750 != 0755 [etc/init.d/elasticsearch] + +# the package scripts handle systemd directly and don't need to use deb helpers +maintainer-script-calls-systemctl +prerm-calls-updaterc.d elasticsearch [prerm:94] + +# we do not automatically enable the service in init.d or systemd +script-in-etc-init.d-not-registered-via-update-rc.d [etc/init.d/elasticsearch] + +# bundled JDK +embedded-library +unstripped-binary-or-object [usr/share/elasticsearch/jdk/*] + +# the system java version that lintian assumes is far behind what elasticsearch uses +unknown-java-class-version + +# There's no `License` field in Debian control files, but earlier versions +# of `lintian` were more permissive. Override this warning so that we can +# run `lintian` on different releases of Debian. The format of this override +# varies between `lintian` versions. +unknown-field elasticsearch-*.deb License +unknown-field License + + +# Below is the copy of some of the above rules in format for Lintian versions <= 2.104 (Debian 11) +# Override syntax changes between Lintian versions in a non-backwards compatible way, so we handle it with +# duplication and ignoring some issues in the test code. + # we prefer to not make our config and log files world readable non-standard-file-perm etc/default/elasticsearch 0660 != 0644 non-standard-dir-perm etc/elasticsearch/ 2750 != 0755 @@ -20,30 +57,15 @@ non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755 non-standard-file-perm etc/elasticsearch/* non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755 non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755 +non-standard-executable-perm etc/init.d/elasticsearch 0750 != 0755 executable-is-not-world-readable etc/init.d/elasticsearch 0750 non-standard-file-permissions-for-etc-init.d-script etc/init.d/elasticsearch 0750 != 0755 -# this lintian tag is simply wrong; contrary to the explanation, debian systemd -# does actually look at /usr/lib/systemd/system -systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service - # we do not automatically enable the service in init.d or systemd script-in-etc-init.d-not-registered-via-update-rc.d etc/init.d/elasticsearch # the package scripts handle init.d/systemd directly and don't need to use deb helpers -maintainer-script-calls-systemctl prerm-calls-updaterc.d elasticsearch # bundled JDK -embedded-library -arch-dependent-file-in-usr-share usr/share/elasticsearch/jdk/* unstripped-binary-or-object usr/share/elasticsearch/jdk/* -extra-license-file usr/share/elasticsearch/jdk/legal/* -hardening-no-pie usr/share/elasticsearch/jdk/bin/* -hardening-no-pie usr/share/elasticsearch/jdk/lib/* - -# the system java version that lintian assumes is far behind what elasticsearch uses -unknown-java-class-version - -# elastic licensed modules contain elastic license -extra-license-file usr/share/elasticsearch/modules/* diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java index 37277cecbbdb2..44520a15c28be 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java @@ -11,17 +11,32 @@ import junit.framework.TestCase; import org.elasticsearch.packaging.util.Distribution; -import org.elasticsearch.packaging.util.FileUtils; +import org.elasticsearch.packaging.util.LintianResultParser; +import org.elasticsearch.packaging.util.LintianResultParser.Issue; +import org.elasticsearch.packaging.util.LintianResultParser.Result; import org.elasticsearch.packaging.util.Shell; import org.junit.BeforeClass; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile; import static org.junit.Assume.assumeTrue; public class DebMetadataTests extends PackagingTestCase { + private final LintianResultParser lintianParser = new LintianResultParser(); + private static final List IGNORED_TAGS = Arrays.asList( + // Override syntax changes between lintian versions in a non-backwards compatible way, so we have to tolerate these. + // Tag mismatched-override is a non-erasable tag which cannot be ignored with overrides, so we handle it here. + "mismatched-override", + // systemd-service-file-outside-lib has been incorrect and removed in the newer version on Lintian + "systemd-service-file-outside-lib" + ); + @BeforeClass public static void filterDistros() { assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB); @@ -29,10 +44,30 @@ public static void filterDistros() { public void test05CheckLintian() { String extraArgs = ""; - if (sh.run("lintian --help").stdout.contains("fail-on-warnings")) { - extraArgs = "--fail-on-warnings "; + final String helpText = sh.run("lintian --help").stdout; + if (helpText.contains("--fail-on-warnings")) { + extraArgs = "--fail-on-warnings"; + } else if (helpText.contains("--fail-on error")) { + extraArgs = "--fail-on error,warning"; + } + Shell.Result result = sh.runIgnoreExitCode( + String.format(Locale.ROOT, "lintian %s %s", extraArgs, getDistributionFile(distribution())) + ); + Result lintianResult = lintianParser.parse(result.stdout); + // Unfortunately Lintian overrides syntax changes between Lintian versions in a non-backwards compatible + // way, so we have to manage some exclusions outside the overrides file. + if (lintianResult.isSuccess() == false) { + List importantIssues = lintianResult.issues() + .stream() + .filter(issue -> IGNORED_TAGS.contains(issue.tag()) == false) + .collect(Collectors.toList()); + if (importantIssues.isEmpty() == false) { + fail( + "Issues for DEB package found by Lintian:\n" + + importantIssues.stream().map(Issue::toString).collect(Collectors.joining("\n")) + ); + } } - sh.run("lintian " + extraArgs + FileUtils.getDistributionFile(distribution())); } public void test06Dependencies() { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java new file mode 100644 index 0000000000000..8ec0492a77083 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.packaging.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class LintianResultParser { + + private static final Logger logger = LogManager.getLogger(LintianResultParser.class); + private static final Pattern RESULT_PATTERN = Pattern.compile("(?[EW]): (?\\S+): (?\\S+) (?.+)"); + + public Result parse(String output) { + String[] lines = output.split("\n"); + List issues = Arrays.stream(lines).map(line -> { + Matcher matcher = RESULT_PATTERN.matcher(line); + if (matcher.matches() == false) { + logger.info("Lintian output not matching expected pattern: {}", line); + return null; + } + Severity severity; + switch (matcher.group("severity")) { + case "E": + severity = Severity.ERROR; + break; + case "W": + severity = Severity.WARNING; + break; + default: + severity = Severity.UNKNOWN; + break; + } + return new Issue(severity, matcher.group("tag"), matcher.group("message")); + }).filter(Objects::nonNull).collect(Collectors.toList()); + + return new Result(issues.stream().noneMatch(it -> it.severity == Severity.ERROR || it.severity == Severity.WARNING), issues); + } + + public static final class Result { + private final boolean isSuccess; + private final List issues; + + public Result(boolean isSuccess, List issues) { + this.isSuccess = isSuccess; + this.issues = issues; + } + + public boolean isSuccess() { + return isSuccess; + } + + public List issues() { + return issues; + } + + @Override + public String toString() { + return "Result{" + "isSuccess=" + isSuccess + ", issues=" + issues + '}'; + } + } + + public static final class Issue { + private final Severity severity; + private final String tag; + private final String message; + + public Issue(Severity severity, String tag, String message) { + this.severity = severity; + this.tag = tag; + this.message = message; + } + + public Severity severity() { + return severity; + } + + public String tag() { + return tag; + } + + public String message() { + return message; + } + + @Override + public String toString() { + return "Issue{" + "severity=" + severity + ", tag='" + tag + '\'' + ", message='" + message + '\'' + '}'; + } + } + + public enum Severity { + ERROR, + WARNING, + UNKNOWN + } +}