feat: Write pod cost to CRD (#1134)

* alternative to adding the Kubernetes annotation directly to the pod resource
  in case allowing PATCH verb is a security concern
* a separate Kubernetes operator with more privilages would watch the CR and
  update the pod resource with the annotation

* separate ci job for the CR integration tests

.github/workflows/integration-tests-rollingupdate-cr.yml

@@ -0,0 +1,68 @@
+name: Integration test for Rolling Update CR Kubernetes
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - release-*
+    tags-ignore: [ v.* ]
+  schedule:
+    - cron: '0 2 * * *'  # every day 2am
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+   integration-test:
+    name: Integration Tests for Rolling Update CR Kubernetes
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.1.0
+        with:
+          fetch-depth: 0
+
+      - name: Checkout GitHub merge
+        if: github.event.pull_request
+        run: |-
+          git fetch origin pull/${{ github.event.pull_request.number }}/merge:scratch
+          git checkout scratch
+
+      - name: Cache Coursier cache
+        uses: coursier/cache-action@v6.4.0
+
+      - name: Set up JDK 11
+        uses: coursier/setup-action@v1.3.0
+        with:
+          jvm: temurin:1.11.0
+
+      - name: Setup Minikube
+        # https://github.com/manusa/actions-setup-minikube/releases
+        # v2.7.1
+        uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
+        with:
+          minikube version: 'v1.21.0'
+          kubernetes version: 'v1.22.0'
+          driver: docker
+          start args: '--addons=ingress'
+
+      - name: Run Integration Tests
+        timeout-minutes: 15
+        run: |-
+          echo 'Creating namespace'
+          kubectl create namespace rolling
+          echo 'Creating resources'
+          kubectl apply -f ./rolling-update-kubernetes/pod-cost.yml
+          echo 'Adding proxy port'
+          kubectl proxy --port=8080 &
+          echo 'Running tests'
+          sbt "rolling-update-kubernetes/IntegrationTest/test"
+          ./integration-test/rollingupdate-kubernetes/test-cr.sh
+
+      - name: Print logs on failure
+        if: ${{ failure() }}
+        run: find . -name "*.log" -exec ./scripts/cat-log.sh {} \;

build.sbt

@@ -167,6 +167,10 @@ lazy val `rolling-update-kubernetes` = project
     libraryDependencies := Dependencies.RollingUpdateKubernetes,
     mimaPreviousArtifacts := Set.empty
   )
+  .settings(
+    Defaults.itSettings
+  )
+  .configs(IntegrationTest)
   .dependsOn(`akka-management-pki`)
 
 lazy val `lease-kubernetes` = project

docs/src/main/paradox/rolling-updates.md

@@ -2,7 +2,7 @@
 
 Rolling updates allow you to update an application by gradually replacing old nodes with new ones. This ensures that the application remains available throughout the update process, with minimal disruption to clients.
 
-#### Graceful shutdown
+## Graceful shutdown
 
 Akka Cluster can handle hard failures using a downing provider such as Lightbend's @extref:[Split Brain Resolver](akka:split-brain-resolver.html).
 However, this should not be relied upon for regular rolling redeploys. Features such as `ClusterSingleton`s and `ClusterSharding`
@@ -19,12 +19,12 @@ Upon receiving a `SIGTERM` Coordinated Shutdown will:
   `ClusterSingleton`s to be migrated if this was the oldest node. Finally, the node is removed from the Akka Cluster membership.
 
 
-#### Number of nodes to redeploy at once
+## Number of nodes to redeploy at once
 
 Akka bootstrap requires a `stable-period` where service discovery returns a stable set of contact points. When doing rolling
 updates it is best to wait for a node (or group of nodes) to finish joining the cluster before adding and removing other nodes.
 
-#### Cluster Singletons
+## Cluster Singletons
 
 `ClusterSingleton`s run on the oldest node in the cluster. To avoid singletons moving during every node deployment it is advised
 to start a rolling redeploy starting at the newest node. Then `ClusterSingleton`s only move once. Cluster Sharding uses a singleton internally so this is important even if not using singletons directly.
@@ -89,11 +89,11 @@ Additionally, the pod annotator needs to know which namespace the pod belongs to
 from the service account secret, in `/var/run/secrets/kubernetes.io/serviceaccount/namespace`, but can be overridden by
 setting `akka.rollingupdate.kubernetes.namespace` or by providing `KUBERNETES_NAMESPACE` environment variable.
 
-##### Role based access control
+#### Role based access control
 
 @@@ warning
 
-This extension uses the Kubernetes API to set the `pod-deletion-cost` annotation on its own pod. To be able to do that, it requires special permission to be able to `patch` the pod configuration. Each pod only needs access to the namespace they are in.
+This extension uses the Kubernetes API to set the `pod-deletion-cost` annotation on its own pod. To be able to do that, it requires special permission to be able to `patch` the pod configuration. Each pod only needs access to the namespace they are in. If this is a security concern in your environment you may instead use @ref:[Alternative with Custom Resource Definition](#alternative-with-custom-resource-definition).
 
 @@@
 
@@ -130,5 +130,54 @@ This RBAC example covers only the permissions needed for this `PodDeletionCost`
 
 @@@
 
+#### Alternative with Custom Resource Definition
+
+If it's a security concern in your environment to allow "patch" in RBAC as described above, you can instead use an
+intermediate Custom Resource Definition (CRD). Instead of updating the `controller.kubernetes.io/pod-deletion-cost`
+annotation directly it will update a `PodCost` custom resource and then you would have an operator that reconciles
+that and updates the pod-deletion-cost annotation of the pod resource. 
+
+@@@ note
+
+You would have to write the Kubernetes operator that watches the `PodCost` resource and updates the
+`controller.kubernetes.io/pod-deletion-cost` annotation of the corresponding pod resource. This operator
+is not provided by Akka.
+
+@@@
+
+Enable updates of custom resource with configuration:
+
+```
+akka.rollingupdate.kubernetes.custom-resource.enabled = true
+```
+
+The `PodCost` CRD:
+
+@@snip [pod-cost.yml](/rolling-update-kubernetes/pod-cost.yml) {}
+
+The RBAC for the application to update the `PodCost` CR, instead of "patch" of the "pods" resources:
+
+```
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: podcost-access
+rules:
+  - apiGroups: ["akka.io"]
+    resources: ["podcosts"]
+    verbs: ["get", "create", "update", "delete", "list"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: podcost-access
+subjects:
+  - kind: User
+    name: system:serviceaccount:<YOUR NAMESPACE>:default
+roleRef:
+  kind: Role
+  name: podcost-access
+  apiGroup: rbac.authorization.k8s.io
+```
 
 

integration-test/rollingupdate-kubernetes/build.sbt

@@ -1,20 +1,8 @@
-import com.typesafe.sbt.packager.docker._
 
-enablePlugins(JavaServerAppPackaging)
+enablePlugins(JavaAppPackaging, DockerPlugin)
 
 version := "1.3.3.7" // we hard-code the version here, it could be anything really
 
-dockerCommands :=
-  dockerCommands.value.flatMap {
-    case ExecCmd("ENTRYPOINT", args @ _*) => Seq(Cmd("ENTRYPOINT", args.mkString(" ")))
-    case v                                => Seq(v)
-  }
-
 dockerExposedPorts := Seq(8080, 8558, 2552)
-dockerBaseImage := "openjdk:8-jre-alpine"
-
-dockerCommands ++= Seq(
-  Cmd("USER", "root"),
-  Cmd("RUN", "/sbin/apk", "add", "--no-cache", "bash", "bind-tools", "busybox-extras", "curl", "strace"),
-  Cmd("RUN", "chgrp -R 0 . && chmod -R g=u .")
-)
+dockerBaseImage := "docker.io/library/adoptopenjdk:11-jre-hotspot"
+dockerUpdateLatest := true

integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml

@@ -0,0 +1,126 @@
+#deployment
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: akka-rollingupdate-demo
+  name: akka-rollingupdate-demo
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+     app: akka-rollingupdate-demo
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+    type: RollingUpdate
+
+  template:
+    metadata:
+      labels:
+        app: akka-rollingupdate-demo
+        actorSystemName: akka-rollingupdate-demo
+    spec:
+      containers:
+      - name: akka-rollingupdate-demo
+        image: integration-test-rollingupdate-kubernetes:1.3.3.7
+        # Remove for a real project, the image is picked up locally for the integration test
+        imagePullPolicy: Never
+        resources:
+          limits:
+            memory: "256Mi"
+          requests:
+            memory: "256Mi"
+            cpu: "300m"
+        #health
+        livenessProbe:
+          httpGet:
+            path: /alive
+            port: management
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: management
+        #health
+        ports:
+        # akka-management bootstrap
+        - name: management
+          containerPort: 8558
+          protocol: TCP
+        - name: http
+          containerPort: 8080
+          protocol: TCP
+        env:
+        - name: KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        # The pod deletion cost will use this var to identity the pod to be annotated (in case that applies)
+        - name: KUBERNETES_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: REQUIRED_CONTACT_POINT_NR
+          value: "3"
+        - name: JAVA_TOOL_OPTIONS
+          value: "-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -Dakka.rollingupdate.kubernetes.custom-resource.enabled=on"
+#deployment
+---
+#rbac-reader
+#
+# Create a role, `pod-annotator`, that can list pods and
+# bind the default service account in the namespace
+# that the binding is deployed to to that role.
+#
+
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-reader
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"] # requires "patch" to annotate the pod
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pod-reader
+subjects:
+  # Uses the default service account.
+  # Consider creating a dedicated service account to run your
+  # Akka Cluster services and binding the role to that one.
+- kind: ServiceAccount
+  name: default
+roleRef:
+  kind: Role
+  name: pod-reader
+  apiGroup: rbac.authorization.k8s.io
+#rbac-reader
+---
+#rbac-podcost-cr
+#
+# Create a role, `podcost-access`, that can update the PodCost CR
+#
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: podcost-access
+rules:
+  - apiGroups: ["akka.io"]
+    resources: ["podcosts"]
+    verbs: ["get", "create", "update", "delete", "list"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: podcost-access
+subjects:
+  - kind: User
+    name: system:serviceaccount:akka-rollingupdate-demo-cr-ns:default
+roleRef:
+  kind: Role
+  name: podcost-access
+  apiGroup: rbac.authorization.k8s.io
+#rbac-podcost-cr

integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster.yml

@@ -57,6 +57,8 @@ spec:
               fieldPath: metadata.name
         - name: REQUIRED_CONTACT_POINT_NR
           value: "3"
+        - name: JAVA_TOOL_OPTIONS
+          value: "-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75"
 #deployment
 ---
 #rbac-reader

integration-test/rollingupdate-kubernetes/src/main/resources/logback.xml

@@ -8,10 +8,12 @@
         </encoder>
     </appender>
 
-    <root level="INFO">
+    <root level="DEBUG">
         <appender-ref ref="CONSOLE"/>
     </root>
 
     <logger name="akka.rollingupdate" level="DEBUG"/>
+    <logger name="akka.http" level="INFO"/>
+    <logger name="akka.io" level="INFO"/>
 
-</configuration>
+</configuration>

integration-test/rollingupdate-kubernetes/test-cr.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -exu
+
+export NAMESPACE=akka-rollingupdate-demo-cr-ns
+export APP_NAME=akka-rollingupdate-demo
+export PROJECT_NAME=integration-test-rollingupdate-kubernetes
+export CRD=rolling-update-kubernetes/pod-cost.yml
+export DEPLOYMENT=integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml
+
+integration-test/scripts/rollingupdate-kubernetes-cr-test.sh
+