Skip to content

Commit

Permalink
[7.17] [ci] Add debian-12 to matrix in packaging and platform jobs (#…
Browse files Browse the repository at this point in the history
…116172) (#117336)

* [ci] Add debian-12 to matrix in packaging and platform jobs (#116172)

Lintian test has been changed to parse the result instead of using exit
code. This was required, because now `mismatched-override` is
non-erasable tag which cannot be ignored for exit code.
Lintian introduced non-backward-compatible format change for overrides
file. Because of that, some overrides are now duplicated in a format for
 older versions.
Additionally, Lintian overrides file has been cleaned up to remove the
tags which are no longer failing.

(cherry picked from commit 2ac267d)

# Conflicts:
#	distribution/packages/src/deb/lintian/elasticsearch
#	qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java
#	qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java

* Spotless fix
  • Loading branch information
jozala authored Nov 25, 2024
1 parent db8aecf commit f403283
Show file tree
Hide file tree
Showing 8 changed files with 195 additions and 23 deletions.
1 change: 1 addition & 0 deletions .buildkite/pipelines/periodic-packaging.template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
1 change: 1 addition & 0 deletions .buildkite/pipelines/periodic-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
1 change: 1 addition & 0 deletions .buildkite/pipelines/periodic-platform-support.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
3 changes: 3 additions & 0 deletions .buildkite/pipelines/pull-request/packaging-tests-unix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down Expand Up @@ -38,6 +39,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down Expand Up @@ -65,6 +67,7 @@ steps:
setup:
image:
- debian-11
- debian-12
- opensuse-leap-15
- oraclelinux-7
- oraclelinux-8
Expand Down
1 change: 0 additions & 1 deletion distribution/packages/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,6 @@ Closure commonDebConfig(boolean oss, boolean jdk, String architecture) {

// versions found on oldest supported distro, centos-6
requires('bash', '4.1', GREATER | EQUAL)
requires('lsb-base', '4', GREATER | EQUAL)
requires 'libc6'
requires 'adduser'

Expand Down
58 changes: 40 additions & 18 deletions distribution/packages/src/deb/lintian/elasticsearch
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# we don't have a changelog, but we put our copyright file
# under /usr/share/doc/elasticsearch, which triggers this warning
# Note that this is renamed to `no-changelog` in newer versions of
# lintian, but we still support Debian 8+, so we can't change this.
changelog-file-missing-in-native-package

# we intentionally copy our copyright file for all deb packages
copyright-file-contains-full-apache-2-license
copyright-should-refer-to-common-license-file-for-apache-2
copyright-without-copyright-notice

# we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages
Expand All @@ -13,37 +13,59 @@ arch-dependent-file-in-usr-share
# we have a bundled jdk, so don't use jarwrapper
missing-dep-on-jarwrapper

# we prefer to not make our config and log files world readable
non-standard-file-perm 0660 != 0644 [etc/default/elasticsearch]
non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/]
non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/jvm.options.d/]
non-standard-file-perm 0660 != 0644 [etc/elasticsearch/*]
non-standard-dir-perm 2750 != 0755 [var/lib/elasticsearch/]
non-standard-dir-perm 2750 != 0755 [var/log/elasticsearch/]
non-standard-executable-perm 0750 != 0755 [etc/init.d/elasticsearch]
executable-is-not-world-readable 0750 [etc/init.d/elasticsearch]
non-standard-file-permissions-for-etc-init.d-script 0750 != 0755 [etc/init.d/elasticsearch]

# the package scripts handle systemd directly and don't need to use deb helpers
maintainer-script-calls-systemctl
prerm-calls-updaterc.d elasticsearch [prerm:94]

# we do not automatically enable the service in init.d or systemd
script-in-etc-init.d-not-registered-via-update-rc.d [etc/init.d/elasticsearch]

# bundled JDK
embedded-library
unstripped-binary-or-object [usr/share/elasticsearch/jdk/*]

# the system java version that lintian assumes is far behind what elasticsearch uses
unknown-java-class-version

# There's no `License` field in Debian control files, but earlier versions
# of `lintian` were more permissive. Override this warning so that we can
# run `lintian` on different releases of Debian. The format of this override
# varies between `lintian` versions.
unknown-field elasticsearch-*.deb License
unknown-field License


# Below is the copy of some of the above rules in format for Lintian versions <= 2.104 (Debian 11)
# Override syntax changes between Lintian versions in a non-backwards compatible way, so we handle it with
# duplication and ignoring some issues in the test code.

# we prefer to not make our config and log files world readable
non-standard-file-perm etc/default/elasticsearch 0660 != 0644
non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
non-standard-file-perm etc/elasticsearch/*
non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
non-standard-executable-perm etc/init.d/elasticsearch 0750 != 0755
executable-is-not-world-readable etc/init.d/elasticsearch 0750
non-standard-file-permissions-for-etc-init.d-script etc/init.d/elasticsearch 0750 != 0755

# this lintian tag is simply wrong; contrary to the explanation, debian systemd
# does actually look at /usr/lib/systemd/system
systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service

# we do not automatically enable the service in init.d or systemd
script-in-etc-init.d-not-registered-via-update-rc.d etc/init.d/elasticsearch

# the package scripts handle init.d/systemd directly and don't need to use deb helpers
maintainer-script-calls-systemctl
prerm-calls-updaterc.d elasticsearch

# bundled JDK
embedded-library
arch-dependent-file-in-usr-share usr/share/elasticsearch/jdk/*
unstripped-binary-or-object usr/share/elasticsearch/jdk/*
extra-license-file usr/share/elasticsearch/jdk/legal/*
hardening-no-pie usr/share/elasticsearch/jdk/bin/*
hardening-no-pie usr/share/elasticsearch/jdk/lib/*

# the system java version that lintian assumes is far behind what elasticsearch uses
unknown-java-class-version

# elastic licensed modules contain elastic license
extra-license-file usr/share/elasticsearch/modules/*
Original file line number Diff line number Diff line change
Expand Up @@ -11,28 +11,63 @@
import junit.framework.TestCase;

import org.elasticsearch.packaging.util.Distribution;
import org.elasticsearch.packaging.util.FileUtils;
import org.elasticsearch.packaging.util.LintianResultParser;
import org.elasticsearch.packaging.util.LintianResultParser.Issue;
import org.elasticsearch.packaging.util.LintianResultParser.Result;
import org.elasticsearch.packaging.util.Shell;
import org.junit.BeforeClass;

import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
import static org.junit.Assume.assumeTrue;

public class DebMetadataTests extends PackagingTestCase {

private final LintianResultParser lintianParser = new LintianResultParser();
private static final List<String> IGNORED_TAGS = Arrays.asList(
// Override syntax changes between lintian versions in a non-backwards compatible way, so we have to tolerate these.
// Tag mismatched-override is a non-erasable tag which cannot be ignored with overrides, so we handle it here.
"mismatched-override",
// systemd-service-file-outside-lib has been incorrect and removed in the newer version on Lintian
"systemd-service-file-outside-lib"
);

@BeforeClass
public static void filterDistros() {
assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB);
}

public void test05CheckLintian() {
String extraArgs = "";
if (sh.run("lintian --help").stdout.contains("fail-on-warnings")) {
extraArgs = "--fail-on-warnings ";
final String helpText = sh.run("lintian --help").stdout;
if (helpText.contains("--fail-on-warnings")) {
extraArgs = "--fail-on-warnings";
} else if (helpText.contains("--fail-on error")) {
extraArgs = "--fail-on error,warning";
}
Shell.Result result = sh.runIgnoreExitCode(
String.format(Locale.ROOT, "lintian %s %s", extraArgs, getDistributionFile(distribution()))
);
Result lintianResult = lintianParser.parse(result.stdout);
// Unfortunately Lintian overrides syntax changes between Lintian versions in a non-backwards compatible
// way, so we have to manage some exclusions outside the overrides file.
if (lintianResult.isSuccess() == false) {
List<Issue> importantIssues = lintianResult.issues()
.stream()
.filter(issue -> IGNORED_TAGS.contains(issue.tag()) == false)
.collect(Collectors.toList());
if (importantIssues.isEmpty() == false) {
fail(
"Issues for DEB package found by Lintian:\n"
+ importantIssues.stream().map(Issue::toString).collect(Collectors.joining("\n"))
);
}
}
sh.run("lintian " + extraArgs + FileUtils.getDistributionFile(distribution()));
}

public void test06Dependencies() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.packaging.util;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class LintianResultParser {

private static final Logger logger = LogManager.getLogger(LintianResultParser.class);
private static final Pattern RESULT_PATTERN = Pattern.compile("(?<severity>[EW]): (?<package>\\S+): (?<tag>\\S+) (?<message>.+)");

public Result parse(String output) {
String[] lines = output.split("\n");
List<Issue> issues = Arrays.stream(lines).map(line -> {
Matcher matcher = RESULT_PATTERN.matcher(line);
if (matcher.matches() == false) {
logger.info("Lintian output not matching expected pattern: {}", line);
return null;
}
Severity severity;
switch (matcher.group("severity")) {
case "E":
severity = Severity.ERROR;
break;
case "W":
severity = Severity.WARNING;
break;
default:
severity = Severity.UNKNOWN;
break;
}
return new Issue(severity, matcher.group("tag"), matcher.group("message"));
}).filter(Objects::nonNull).collect(Collectors.toList());

return new Result(issues.stream().noneMatch(it -> it.severity == Severity.ERROR || it.severity == Severity.WARNING), issues);
}

public static final class Result {
private final boolean isSuccess;
private final List<Issue> issues;

public Result(boolean isSuccess, List<Issue> issues) {
this.isSuccess = isSuccess;
this.issues = issues;
}

public boolean isSuccess() {
return isSuccess;
}

public List<Issue> issues() {
return issues;
}

@Override
public String toString() {
return "Result{" + "isSuccess=" + isSuccess + ", issues=" + issues + '}';
}
}

public static final class Issue {
private final Severity severity;
private final String tag;
private final String message;

public Issue(Severity severity, String tag, String message) {
this.severity = severity;
this.tag = tag;
this.message = message;
}

public Severity severity() {
return severity;
}

public String tag() {
return tag;
}

public String message() {
return message;
}

@Override
public String toString() {
return "Issue{" + "severity=" + severity + ", tag='" + tag + '\'' + ", message='" + message + '\'' + '}';
}
}

public enum Severity {
ERROR,
WARNING,
UNKNOWN
}
}

0 comments on commit f403283

Please sign in to comment.