Merge branch 'main' into bump-deps

.circleci/config.yml

@@ -2,37 +2,6 @@ version: 2.1
 
 jobs:
 
-  test-multi-arch:
-    parameters:
-      platform:
-        type: string
-    environment:
-      _JAVA_OPTIONS: "-Xms512m -Xmx1g"
-      DOCKER_CLI_EXPERIMENTAL: "enabled"
-    working_directory: ~/workspace
-    machine:
-      image: ubuntu-2004:202107-02
-    steps:
-      - checkout
-      - run: |
-         # install required qemu libraries
-         docker run --rm --privileged tonistiigi/binfmt:latest --install all
-         # run docker container with qemu emulation
-         docker run --rm \
-           --platform << parameters.platform >> \
-           --name qemu-cross-<< parameters.platform >> \
-           --mount type=bind,source=${PWD},target=/github_workspace \
-           --workdir /github_workspace \
-           << parameters.platform >>/eclipse-temurin:11-jdk-focal uname -a; ./gradlew --no-daemon -PmaxParallelForks=1 build
-      - run:
-          command: mkdir ~/test-results
-      - run:
-          command: find ~/workspace -type f -regex ".*/test-results/.*xml" -exec ln {} ~/test-results/ \;
-      - store_test_results:
-          path: ~/test-results
-      - store_artifacts:
-          path: ~/test-results
-
   build:
     environment:
       _JAVA_OPTIONS: "-Xms512m -Xmx1g"
@@ -106,21 +75,13 @@ workflows:
   version: 2.1
   build-and-publish:
     jobs:
-      - test-multi-arch:
-          matrix:
-            parameters:
-              platform: ["s390x"]
-          filters:
-            tags:
-              only: /.*/
       - build:
           filters:
             tags:
               only: /.*/
       - integration-test:
           requires:
             - build
-            - test-multi-arch
           filters:
             branches:
               ignore: /.*/

.github/workflows/artifactory.yaml

@@ -0,0 +1,25 @@
+name: Artifactory
+
+on:
+  workflow_dispatch: # manual trigger
+  #release:
+  #  types: [published]
+
+jobs:
+  publish:
+    # if: startsWith(github.event.ref, 'release/')
+    name: publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # so gradle doesn't fail traversing the history
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: microsoft
+          cache: gradle
+      - uses: gradle/actions/setup-gradle@v4 # v4.0.0
+      - name: publish
+        run: |
+          ./gradlew :artifactoryPublish :cruise-control:artifactoryPublish :cruise-control-core:artifactoryPublish :cruise-control-metrics-reporter:artifactoryPublish

.github/workflows/ci.yaml

@@ -0,0 +1,80 @@
+name: CI
+
+on:
+  push:
+    branches: ['main']
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+jobs:
+  test:
+    name: "test with JDK=${{matrix.java-dist}}:${{matrix.java-ver}}"
+    runs-on: [ubuntu-latest]
+    strategy:
+      fail-fast: false
+      matrix:
+        java-ver: [11]
+        java-dist: ['microsoft', 'temurin']
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # so gradle doesn't fail traversing the history
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-ver }}
+          distribution: ${{ matrix.java-dist }}
+          cache: gradle
+      # see: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
+      - uses: gradle/actions/setup-gradle@v4 # v4.0.0
+      - name: gradle build
+        run: ./gradlew --no-daemon -PmaxParallelForks=1 build
+
+  integration-test:
+    name: "integration-test with JDK=${{matrix.java-dist}}:${{matrix.java-ver}}"
+    runs-on: [ubuntu-latest]
+    strategy:
+      fail-fast: false
+      matrix:
+        java-ver: [11]
+        java-dist: ['microsoft', 'temurin']
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # so gradle doesn't fail traversing the history
+      - uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-ver }}
+          distribution: ${{ matrix.java-dist }}
+          cache: gradle
+      # see: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
+      - uses: gradle/actions/setup-gradle@v4 # v4.0.0
+      - name: gradle integration test
+        run: ./gradlew --no-daemon -PmaxParallelForks=1 clean integrationTest
+
+  build-platform:
+    name: platform build with JDK=${{matrix.java-dist}}:${{matrix.java-ver}} on ${{matrix.hw_platform}}
+    strategy:
+      fail-fast: false
+      matrix:
+        java-ver: [11]
+        java-dist: ['temurin']
+        hw_platform: ['s390x']
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # so gradle doesn't fail traversing the history
+      - run: |
+          # install required qemu libraries
+          docker run --rm --privileged tonistiigi/binfmt:latest --install all
+          # run docker container with qemu emulation
+          docker run --rm \
+            --platform ${{ matrix.hw_platform }} \
+            --name qemu-cross-${{ matrix.hw_platform }} \
+            --mount type=bind,source=${PWD},target=/workspace \
+            --workdir /workspace \
+            ${{matrix.hw_platform}}/eclipse-temurin:11-jdk-focal uname -a; ./gradlew --no-daemon -PmaxParallelForks=1 build

README.md

@@ -1,14 +1,16 @@
 Cruise Control for Apache Kafka
 ===================
 
+[![CI](https://github.com/linkedin/cruise-control/actions/workflows/ci.yaml/badge.svg)](https://github.com/linkedin/cruise-control/actions/workflows/ci.yaml)
 [![CircleCI](https://circleci.com/gh/linkedin/cruise-control.svg?style=svg)](https://circleci.com/gh/linkedin/cruise-control)
 
+
 ### Introduction ###
   Cruise Control is a product that helps run Apache Kafka clusters at large scale. Due to the popularity of 
-  Apache Kafka, many companies have bigger and bigger Kafka clusters. At LinkedIn, we have ~7K+ Kafka brokers, 
+  Apache Kafka, many companies have increasingly large Kafka clusters with hundreds of brokers. At LinkedIn, we have 10K+ Kafka brokers, 
   which means broker deaths are an almost daily occurrence and balancing the workload of Kafka also becomes a big overhead. 
 
-  Kafka Cruise Control is designed to address this operation scalability issue.
+  Kafka Cruise Control is designed to address this operational scalability issue.
 
 ### Features ###
   Kafka Cruise Control provides the following features out of the box:

build.gradle

@@ -289,7 +289,7 @@ project(':cruise-control') {
     implementation 'com.google.code.gson:gson:2.9.0'
     implementation "org.eclipse.jetty:jetty-server:${jettyVersion}"
     implementation 'io.dropwizard.metrics:metrics-jmx:4.2.9'
-    implementation 'com.nimbusds:nimbus-jose-jwt:9.37.3'
+    implementation 'com.nimbusds:nimbus-jose-jwt:9.45'
     implementation 'io.swagger.parser.v3:swagger-parser-v3:2.1.16'
     implementation 'io.github.classgraph:classgraph:4.8.141'
     implementation 'com.google.code.findbugs:jsr305:3.0.2'
@@ -315,7 +315,7 @@ project(':cruise-control') {
     testImplementation 'commons-io:commons-io:2.11.0'
     testImplementation 'org.apache.httpcomponents:httpclient:4.5.13:tests'
     testImplementation 'org.bouncycastle:bcpkix-jdk15on:1.70'
-    testImplementation 'org.apache.kerby:kerb-simplekdc:2.0.1'
+    testImplementation 'org.apache.kerby:kerb-simplekdc:2.1.0'
     testImplementation 'com.jayway.jsonpath:json-path:2.7.0'
     testImplementation 'org.powermock:powermock-module-junit4:2.0.9'
     testImplementation 'org.powermock:powermock-api-easymock:2.0.9'

cruise-control-core/src/main/java/com/linkedin/cruisecontrol/common/Generationed.java

@@ -27,7 +27,7 @@ public interface Generationed<G> {
    * have a different comparing method along with the generation comparison.
    *
    * @param other another generationed object to compare the generation with.
-   * @return -1 if the the generation of this object is earlier than the other's. 0 when the generations are the same.
+   * @return -1 if the generation of this object is earlier than the other's. 0 when the generations are the same.
    * 1 when the generation of this object is later than the other's.
    */
   int compareGeneration(Generationed<G> other);
@@ -36,7 +36,7 @@ public interface Generationed<G> {
    * Compare the generation of this object with the given generation.
    *
    * @param generation the given generation.
-   * @return -1 if the the generation of this object is earlier than the given generation.
+   * @return -1 if the generation of this object is earlier than the given generation.
    * 0 when the generations are the same.
    * 1 when the generation of this object is later than the given one.
    */

cruise-control-core/src/main/java/com/linkedin/cruisecontrol/monitor/sampling/aggregator/MetricSampleAggregatorState.java

@@ -124,7 +124,7 @@ synchronized void resetWindowIndices(long startingWindowIndex, int numWindowIndi
 
   /**
    * Get the list of window indices that need to be updated based on the current generation.
-   * This method also removes the windows that are older than the oldestWindowIndex from the the internal state
+   * This method also removes the windows that are older than the oldestWindowIndex from the internal state
    * of this class.
    *
    * @param oldestWindowIndex the index of the oldest window in the MetricSampleAggregator.

cruise-control-metrics-reporter/src/main/java/com/linkedin/kafka/cruisecontrol/metricsreporter/metric/ContainerMetricUtils.java

@@ -43,7 +43,7 @@ private static double getCpuQuota() throws IOException {
   }
 
   /**
-   * Gets the the number of logical cores available to the node.
+   * Gets the number of logical cores available to the node.
    * <p>
    * We can get this value while running in a container by using the "nproc" command.
    * Using other methods like OperatingSystemMXBean.getAvailableProcessors() and

cruise-control-metrics-reporter/src/test/java/com/linkedin/kafka/cruisecontrol/metricsreporter/utils/CCEmbeddedZookeeper.java

@@ -5,7 +5,6 @@
 package com.linkedin.kafka.cruisecontrol.metricsreporter.utils;
 
 import java.io.File;
-import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.concurrent.CountDownLatch;
 import org.apache.zookeeper.server.NIOServerCnxnFactory;
@@ -14,7 +13,7 @@
 
 
 public class CCEmbeddedZookeeper implements AutoCloseable {
-  private final String _hostAddress;
+  private final String _hostAddress = "127.0.0.1";
   private final int _port;
   private final ZooKeeperServer _zk;
   private final ServerCnxnFactory _cnxnFactory;
@@ -27,9 +26,7 @@ public CCEmbeddedZookeeper() {
       File logDir = CCKafkaTestUtils.newTempDir();
       _zk = new ZooKeeperServer(snapshotDir, logDir, tickTime);
       _cnxnFactory = new NIOServerCnxnFactory();
-      InetAddress localHost = InetAddress.getLocalHost();
-      _hostAddress = localHost.getHostAddress();
-      InetSocketAddress bindAddress = new InetSocketAddress(localHost, 0);
+      InetSocketAddress bindAddress = new InetSocketAddress(_hostAddress, 0);
       _cnxnFactory.configure(bindAddress, 0);
       _cnxnFactory.startup(_zk);
       _port = _zk.getClientPort();

cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/analyzer/goals/ReplicaCapacityGoal.java

@@ -264,7 +264,7 @@ protected void rebalanceForBroker(Broker broker,
    *
    * A alive destination broker is eligible for a given replica if
    * (1) the broker contains less than allowed maximum number of replicas, or
-   * (2) If the the self healing mode is true.
+   * (2) If the self healing mode is true.
    *
    * Returned brokers are sorted by number of replicas on them in ascending order.
    *

cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/async/progress/OperationProgress.java

@@ -100,6 +100,26 @@ public synchronized String toString() {
     return sb.toString();
   }
 
+  /**
+   * @return the up-to-date total execution time on the progress
+   */
+  public synchronized long getCurrentTotalExecutionTimeMs() {
+    if (_startTimes.isEmpty()) {
+      return 0;
+    } else {
+      long currentTime = System.currentTimeMillis();
+      long startTime = _startTimes.get(0);
+      return currentTime - startTime;
+    }
+  }
+
+  /**
+   * @return the name of the operation
+   */
+  public String getOperation() {
+    return _operation;
+  }
+
   /**
    * @return The map describing the progress of the operation.
    */

cruise-control/src/main/java/com/linkedin/kafka/cruisecontrol/executor/ExecutionTaskPlanner.java

@@ -507,7 +507,7 @@ private void removeInterBrokerReplicaActionForExecution(ExecutionTask task) {
   /**
    * The comparing order:
    * <ul>
-   *   <li>Priority of the the first task of each broker</li>
+   *   <li>Priority of the first task of each broker</li>
    *   <li>The task set size of each broker. Prioritize broker with the larger size</li>
    *   <li>Broker ID integer. Prioritize broker with the smaller ID</li>
    * </ul>