Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow configuring port of http probes #30566

Merged
merged 4 commits into from
Feb 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import io.quarkus.kubernetes.spi.KubernetesJobBuildItem;
import io.quarkus.kubernetes.spi.KubernetesLabelBuildItem;
import io.quarkus.kubernetes.spi.KubernetesPortBuildItem;
import io.quarkus.kubernetes.spi.KubernetesProbePortNameBuildItem;
import io.quarkus.kubernetes.spi.KubernetesResourceMetadataBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBindingBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBuildItem;
Expand Down Expand Up @@ -104,6 +105,7 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
Optional<ContainerImageInfoBuildItem> image,
Optional<KubernetesCommandBuildItem> command,
List<KubernetesPortBuildItem> ports,
Optional<KubernetesProbePortNameBuildItem> portName,
Optional<KubernetesHealthLivenessPathBuildItem> livenessPath,
Optional<KubernetesHealthReadinessPathBuildItem> readinessPath,
List<KubernetesRoleBuildItem> roles,
Expand All @@ -112,6 +114,7 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic

return DevClusterHelper.createDecorators(KIND, applicationInfo, outputTarget, config, packageConfig,
metricsConfiguration, initContainers, jobs, annotations, labels, envs, baseImage, image, command, ports,
portName,
livenessPath,
readinessPath,
roles, roleBindings, customProjectRoot);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import io.quarkus.kubernetes.spi.KubernetesJobBuildItem;
import io.quarkus.kubernetes.spi.KubernetesLabelBuildItem;
import io.quarkus.kubernetes.spi.KubernetesPortBuildItem;
import io.quarkus.kubernetes.spi.KubernetesProbePortNameBuildItem;
import io.quarkus.kubernetes.spi.KubernetesResourceMetadataBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBindingBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBuildItem;
Expand Down Expand Up @@ -100,6 +101,7 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
Optional<ContainerImageInfoBuildItem> image,
Optional<KubernetesCommandBuildItem> command,
List<KubernetesPortBuildItem> ports,
Optional<KubernetesProbePortNameBuildItem> portName,
Optional<KubernetesHealthLivenessPathBuildItem> livenessPath,
Optional<KubernetesHealthReadinessPathBuildItem> readinessPath,
List<KubernetesRoleBuildItem> roles,
Expand All @@ -108,6 +110,7 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic

return DevClusterHelper.createDecorators(MINIKUBE, applicationInfo, outputTarget, config, packageConfig,
metricsConfiguration, initContainers, jobs, annotations, labels, envs, baseImage, image, command, ports,
portName,
livenessPath,
readinessPath,
roles, roleBindings, customProjectRoot);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package io.quarkus.kubernetes.spi;

import io.quarkus.builder.item.SimpleBuildItem;

/**
* A build item for selecting which port to use for probes using an {@literal HTTP get} action.
*/
public class KubernetesProbePortNameBuildItem extends SimpleBuildItem {

private final String name;

public KubernetesProbePortNameBuildItem(String name) {
this.name = name;
}

public String getName() {
return name;
}
}
Original file line number Diff line number Diff line change
@@ -1,33 +1,89 @@

package io.quarkus.kubernetes.deployment;

import static io.dekorate.utils.Metadata.getMetadata;

import java.util.List;
import java.util.Map;
import java.util.Optional;

import io.dekorate.kubernetes.decorator.AddLivenessProbeDecorator;
import io.dekorate.kubernetes.decorator.AddReadinessProbeDecorator;
import io.dekorate.kubernetes.decorator.AddSidecarDecorator;
import io.dekorate.kubernetes.decorator.ApplicationContainerDecorator;
import io.dekorate.kubernetes.decorator.Decorator;
import io.dekorate.kubernetes.decorator.ResourceProvidingDecorator;
import io.fabric8.kubernetes.api.builder.Builder;
import io.fabric8.kubernetes.api.model.ContainerBuilder;
import io.fabric8.kubernetes.api.model.HTTPGetActionFluent;

public class ApplyHttpGetActionPortDecorator extends ApplicationContainerDecorator<HTTPGetActionFluent<?>> {
public class ApplyHttpGetActionPortDecorator extends Decorator<HTTPGetActionFluent<?>> {

private final String deployment;
private final String container;
private final Integer port;
private final String probeKind;

public ApplyHttpGetActionPortDecorator(Integer port) {
this(ANY, ANY, port);
this(ANY, ANY, port, ANY);
}

public ApplyHttpGetActionPortDecorator(Integer port, String probeKind) {
this(ANY, ANY, port, probeKind);
}

public ApplyHttpGetActionPortDecorator(String deployment, Integer port) {
this(deployment, ANY, port);
this(deployment, ANY, port, ANY);
}

public ApplyHttpGetActionPortDecorator(String deployment, Integer port, String probeKind) {
this(deployment, ANY, port, probeKind);
}

public ApplyHttpGetActionPortDecorator(String deployment, String container, Integer port) {
super(deployment, container);
this(deployment, container, port, ANY);
}

public ApplyHttpGetActionPortDecorator(String deployment, String container, Integer port, String probeKind) {
this.deployment = deployment;
this.container = container;
this.port = port;
this.probeKind = probeKind;
}

@Override
public void visit(List<Map.Entry<String, Object>> path, HTTPGetActionFluent<?> action) {
boolean inMatchingProbe = probeKind == ANY || path.stream().map(e -> e.getKey()).anyMatch(i -> i.equals(probeKind));
if (!inMatchingProbe) {
return;
}

boolean inMatchingContainer = container == ANY || path.stream()
.map(e -> e.getValue())
.filter(v -> v instanceof ContainerBuilder)
.map(v -> (ContainerBuilder) v)
.anyMatch(c -> c.getName() != null && c.getName().equals(container));

if (!inMatchingContainer) {
return;
}

boolean inMatchingResource = deployment == ANY || path.stream()
.map(e -> e.getValue())
.filter(v -> v instanceof Builder)
.map(v -> (Builder) v)
.map(b -> getMetadata(b))
.filter(m -> m.isPresent())
.map(Optional::get)
.anyMatch(m -> m.getName() != null && m.getName().equals(deployment));

if (!inMatchingResource) {
return;
}

visit(action);
}

@Override
public void andThenVisit(HTTPGetActionFluent<?> action) {
public void visit(HTTPGetActionFluent<?> action) {
if (port == null) {
// workaround to make sure we don't get a NPE
action.withNewPort((String) null);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ public ChangeDeploymentTriggerDecorator(String containerName, String imageStream
public Class<? extends Decorator>[] after() {
return new Class[] { ApplyDeploymentTriggerDecorator.class, RemoveDeploymentTriggerDecorator.class };
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need this change?

Copy link
Contributor Author

@iocanel iocanel Feb 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same goes for change decorators.
Now, things may work even without those changes but it will be circumstantial.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same. I am wrong, let me completely wipe them.


}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@

package io.quarkus.kubernetes.deployment;

import static io.quarkus.kubernetes.deployment.Constants.DEFAULT_HTTP_PORT;
import static io.quarkus.kubernetes.deployment.Constants.KUBERNETES;
import static io.quarkus.kubernetes.deployment.Constants.MAX_NODE_PORT_VALUE;
import static io.quarkus.kubernetes.deployment.Constants.MAX_PORT_NUMBER;
Expand Down Expand Up @@ -42,6 +41,7 @@
import io.quarkus.kubernetes.spi.KubernetesJobBuildItem;
import io.quarkus.kubernetes.spi.KubernetesLabelBuildItem;
import io.quarkus.kubernetes.spi.KubernetesPortBuildItem;
import io.quarkus.kubernetes.spi.KubernetesProbePortNameBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBindingBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBuildItem;

Expand All @@ -64,6 +64,7 @@ public static List<DecoratorBuildItem> createDecorators(String clusterKind,
Optional<ContainerImageInfoBuildItem> image,
Optional<KubernetesCommandBuildItem> command,
List<KubernetesPortBuildItem> ports,
Optional<KubernetesProbePortNameBuildItem> portName,
Optional<KubernetesHealthLivenessPathBuildItem> livenessPath,
Optional<KubernetesHealthReadinessPathBuildItem> readinessPath,
List<KubernetesRoleBuildItem> roles,
Expand Down Expand Up @@ -118,8 +119,14 @@ public static List<DecoratorBuildItem> createDecorators(String clusterKind,
}

//Probe port handling
Integer portNumber = port.map(Port::getContainerPort).orElse(DEFAULT_HTTP_PORT);
result.add(new DecoratorBuildItem(clusterKind, new ApplyHttpGetActionPortDecorator(name, name, portNumber)));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, clusterKind, "livenessProbe", config.livenessProbe,
portName,
ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, clusterKind, "readinessProbe", config.readinessProbe,
portName,
ports));

// Handle init Containers
result.addAll(KubernetesCommonHelper.createInitContainerDecorators(clusterKind, name, initContainers, result));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
result.add(new DecoratorBuildItem(KNATIVE, new ApplyServiceTypeDecorator(name, config.getServiceType().name())));

//In Knative its expected that all http ports in probe are omitted (so we set them to null).
result.add(new DecoratorBuildItem(KNATIVE, new ApplyHttpGetActionPortDecorator(name, null)));
result.add(new DecoratorBuildItem(KNATIVE, new ApplyHttpGetActionPortDecorator(name, (Integer) null)));

//Traffic Splitting
config.revisionName.ifPresent(r -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
package io.quarkus.kubernetes.deployment;

import static io.dekorate.kubernetes.decorator.AddServiceResourceDecorator.distinct;
import static io.quarkus.kubernetes.deployment.Constants.DEFAULT_HTTP_PORT;
import static io.quarkus.kubernetes.deployment.Constants.HTTP_PORT;
import static io.quarkus.kubernetes.deployment.Constants.QUARKUS_ANNOTATIONS_BUILD_TIMESTAMP;
import static io.quarkus.kubernetes.deployment.Constants.QUARKUS_ANNOTATIONS_COMMIT_ID;
import static io.quarkus.kubernetes.deployment.Constants.QUARKUS_ANNOTATIONS_VCS_URL;
Expand Down Expand Up @@ -81,6 +83,7 @@
import io.quarkus.kubernetes.spi.KubernetesJobBuildItem;
import io.quarkus.kubernetes.spi.KubernetesLabelBuildItem;
import io.quarkus.kubernetes.spi.KubernetesPortBuildItem;
import io.quarkus.kubernetes.spi.KubernetesProbePortNameBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBindingBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBuildItem;

Expand Down Expand Up @@ -687,6 +690,46 @@ private static List<DecoratorBuildItem> createAnnotationDecorators(Optional<Proj
return result;
}

/**
* Create a decorator that sets the port to the http probe.
* The rules for setting the probe are the following:
* 1. if 'http-action-port' is set, use that.
* 2. if 'http-action-port-name' is set, use that to lookup the port value.
* 3. if a `KubernetesPorbePortBuild` is set, then use that to lookup the port.
* 4. if we still haven't found a port fallback to 8080.
*
* @param name The name of the deployment / container.
* @param target The deployment target
* @param the probe kind (e.g. readinessProbe, livenessProbe etc)
* @param portName the probe port name build item
* @paramt ports a list of kubernetes port build items
* @return a decorator for configures the port of the http action of the probe.
*/
public static DecoratorBuildItem createProbeHttpPortDecorator(String name, String target, String probeKind,
ProbeConfig probeConfig,
Optional<KubernetesProbePortNameBuildItem> portName,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's awesome! I just have to emit that build item if the management interface is used.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's exactly the idea.

List<KubernetesPortBuildItem> ports) {

//1. check if `httpActionPort` is defined
//2. lookup port by `httpPortName`
//3. fallback to DEFAULT_HTTP_PORT
String httpPortName = probeConfig.httpActionPortName
.or(() -> portName.map(KubernetesProbePortNameBuildItem::getName))
.orElse(HTTP_PORT);

Integer port = probeConfig.httpActionPort
.orElse(ports.stream().filter(p -> httpPortName.equals(p.getName()))
.map(KubernetesPortBuildItem::getPort).findFirst().orElse(DEFAULT_HTTP_PORT));
return new DecoratorBuildItem(target, new ApplyHttpGetActionPortDecorator(name, name, port, probeKind));
}

/**
* Create the decorators needed for setting up probes.
* The method will not create decorators related to ports, as they are not supported by all targets (e.g. knative)
* Port related decorators are created by `applyProbePort` instead.
*
* @return a list of decorators that configure the probes
*/
private static List<DecoratorBuildItem> createProbeDecorators(String name, String target, ProbeConfig livenessProbe,
ProbeConfig readinessProbe,
Optional<KubernetesHealthLivenessPathBuildItem> livenessPath,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@

package io.quarkus.kubernetes.deployment;

import static io.quarkus.kubernetes.deployment.Constants.DEFAULT_HTTP_PORT;
import static io.quarkus.kubernetes.deployment.Constants.DEFAULT_S2I_IMAGE_NAME;
import static io.quarkus.kubernetes.deployment.Constants.OPENSHIFT;
import static io.quarkus.kubernetes.deployment.Constants.OPENSHIFT_APP_RUNTIME;
Expand Down Expand Up @@ -61,6 +60,7 @@
import io.quarkus.kubernetes.spi.KubernetesJobBuildItem;
import io.quarkus.kubernetes.spi.KubernetesLabelBuildItem;
import io.quarkus.kubernetes.spi.KubernetesPortBuildItem;
import io.quarkus.kubernetes.spi.KubernetesProbePortNameBuildItem;
import io.quarkus.kubernetes.spi.KubernetesResourceMetadataBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBindingBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBuildItem;
Expand Down Expand Up @@ -179,6 +179,7 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
Optional<BaseImageInfoBuildItem> baseImage,
Optional<ContainerImageInfoBuildItem> image,
Optional<KubernetesCommandBuildItem> command,
Optional<KubernetesProbePortNameBuildItem> portName,
List<KubernetesPortBuildItem> ports,
Optional<KubernetesHealthLivenessPathBuildItem> livenessPath,
Optional<KubernetesHealthReadinessPathBuildItem> readinessPath,
Expand Down Expand Up @@ -301,8 +302,13 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
}

// Probe port handling
Integer portNumber = port.map(Port::getContainerPort).orElse(DEFAULT_HTTP_PORT);
result.add(new DecoratorBuildItem(OPENSHIFT, new ApplyHttpGetActionPortDecorator(name, name, portNumber)));
result.add(KubernetesCommonHelper.createProbeHttpPortDecorator(name, OPENSHIFT, "livenssProbe", config.livenessProbe,
portName,
ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, OPENSHIFT, "readinessProbe", config.readinessProbe,
portName,
ports));

// Handle non-openshift builds
if (deploymentKind == DeploymentResourceKind.DeploymentConfig
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,19 @@
@ConfigGroup
public class ProbeConfig {

/**
* The port number to use when configuring the {@literal http get} action.
* If not configured, the port corresponding to the {@code httpActionPortName} will be used.
*/
@ConfigItem
Optional<Integer> httpActionPort;

/**
* The port name for selecting the port of the {@literal HTTP get} action.
*/
@ConfigItem
Optional<String> httpActionPortName;

/**
* The http path to use for the probe For this to work, the container port also
* needs to be set.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@

package io.quarkus.kubernetes.deployment;

import static io.quarkus.kubernetes.deployment.Constants.DEFAULT_HTTP_PORT;
import static io.quarkus.kubernetes.deployment.Constants.DEPLOYMENT_GROUP;
import static io.quarkus.kubernetes.deployment.Constants.DEPLOYMENT_VERSION;
import static io.quarkus.kubernetes.deployment.Constants.INGRESS;
Expand Down Expand Up @@ -52,6 +51,7 @@
import io.quarkus.kubernetes.spi.KubernetesJobBuildItem;
import io.quarkus.kubernetes.spi.KubernetesLabelBuildItem;
import io.quarkus.kubernetes.spi.KubernetesPortBuildItem;
import io.quarkus.kubernetes.spi.KubernetesProbePortNameBuildItem;
import io.quarkus.kubernetes.spi.KubernetesResourceMetadataBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBindingBuildItem;
import io.quarkus.kubernetes.spi.KubernetesRoleBuildItem;
Expand Down Expand Up @@ -132,6 +132,7 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
List<KubernetesAnnotationBuildItem> annotations,
List<KubernetesLabelBuildItem> labels, List<KubernetesEnvBuildItem> envs,
Optional<ContainerImageInfoBuildItem> image, Optional<KubernetesCommandBuildItem> command,
Optional<KubernetesProbePortNameBuildItem> portName,
List<KubernetesPortBuildItem> ports, Optional<KubernetesHealthLivenessPathBuildItem> livenessPath,
Optional<KubernetesHealthReadinessPathBuildItem> readinessPath, List<KubernetesRoleBuildItem> roles,
List<KubernetesRoleBindingBuildItem> roleBindings, Optional<CustomProjectRootBuildItem> customProjectRoot,
Expand Down Expand Up @@ -241,8 +242,14 @@ public List<DecoratorBuildItem> createDecorators(ApplicationInfoBuildItem applic
}

// Probe port handling
Integer portNumber = port.map(Port::getContainerPort).orElse(DEFAULT_HTTP_PORT);
result.add(new DecoratorBuildItem(KUBERNETES, new ApplyHttpGetActionPortDecorator(name, name, portNumber)));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, KUBERNETES, "livenessProbe", config.livenessProbe,
portName,
ports));
result.add(
KubernetesCommonHelper.createProbeHttpPortDecorator(name, KUBERNETES, "readinessProbe", config.readinessProbe,
portName,
ports));

// Handle remote debug configuration
if (config.remoteDebug.enabled) {
Expand Down Expand Up @@ -274,5 +281,4 @@ void externalizeInitTasks(
decorators);
}
}

}
Loading