Merge branch 'master' into fix-duplicate-messages-seek

apache · Jul 7, 2022 · 79a9fae · 79a9fae
2 parents e6a6010 + 76646cf
commit 79a9fae
Show file tree

Hide file tree

Showing 1,274 changed files with 124,500 additions and 3,448 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -49,6 +49,7 @@ github:
         # Contexts are the names of checks that must pass.
         # See ./github/workflows/README.md for more documentation on this list.
         contexts:
+          - label
           - build
           - cpp-tests
           - Changed files check
@@ -77,8 +78,7 @@ github:
           - CI - System - Schema
           - CI - System - Tiered FileSystem
           - CI - System - Tiered JCloud
-# Sql integration tests are disabled until https://github.com/apache/pulsar/issues/14951 has been resolved
-#          - CI - System - Sql
+          - CI - System - Sql
 
       required_pull_request_reviews:
         dismiss_stale_reviews: false

diff --git a/.github/workflows/ci-cancel-duplicate-workflows.yaml b/.github/workflows/ci-cancel-duplicate-workflows.yaml
@@ -61,3 +61,9 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           cancelMode: allDuplicates
           workflowFileName: pulsar-ci.yaml
+      - name: cancel duplicate ci-documentbot.yml
+        uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          cancelMode: allDuplicates
+          workflowFileName: ci-documentbot.yml
diff --git a/.gitignore b/.gitignore
@@ -25,7 +25,7 @@ pulsar-functions/worker/src/test/resources/
 .recommenders/
 .factorypath
 
-# Intellij
+# IntelliJ
 .idea/
 *.iml
 *.iws

diff --git a/README.md b/README.md
@@ -21,31 +21,39 @@
 
 ![logo](site2/website/static/img/pulsar.svg)
 
+[![unit test](https://img.shields.io/github/workflow/status/apache/pulsar/CI%20-%20Unit?label=unit%20test)](https://github.com/apache/pulsar/actions/workflows/ci-unit.yaml)
+[![docker build](https://img.shields.io/github/workflow/status/apache/pulsar/CI%20-%20Docker%20Build?label=docker%20build)](https://hub.docker.com/r/apachepulsar/pulsar)
+[![contributors](https://img.shields.io/github/contributors-anon/apache/pulsar)](https://github.com/apache/pulsar/graphs/contributors)
+[![last commit](https://img.shields.io/github/last-commit/apache/pulsar)](https://github.com/apache/pulsar/commits/master)
+[![release](https://img.shields.io/github/v/release/apache/pulsar)](https://github.com/apache/pulsar/releases)
+[![release date](https://img.shields.io/github/release-date/apache/pulsar)](https://github.com/apache/pulsar/releases)
+[![downloads](https://img.shields.io/github/downloads/apache/pulsar/total)](https://pulsar.apache.org/download/)
+
 Pulsar is a distributed pub-sub messaging platform with a very
 flexible messaging model and an intuitive client API.
 
 Learn more about Pulsar at https://pulsar.apache.org
 
 ## Main features
 
-* Horizontally scalable (Millions of independent topics and millions
+- Horizontally scalable (Millions of independent topics and millions
   of messages published per second)
-* Strong ordering and consistency guarantees
-* Low latency durable storage
-* Topic and queue semantics
-* Load balancer
-* Designed for being deployed as a hosted service:
-  * Multi-tenant
-  * Authentication
-  * Authorization
-  * Quotas
-  * Support mixing very different workloads
-  * Optional hardware isolation
-* Keeps track of consumer cursor position
-* REST API for provisioning, admin and stats
-* Geo replication
-* Transparent handling of partitioned topics
-* Transparent batching of messages
+- Strong ordering and consistency guarantees
+- Low latency durable storage
+- Topic and queue semantics
+- Load balancer
+- Designed for being deployed as a hosted service:
+  - Multi-tenant
+  - Authentication
+  - Authorization
+  - Quotas
+  - Support mixing very different workloads
+  - Optional hardware isolation
+- Keeps track of consumer cursor position
+- REST API for provisioning, admin and stats
+- Geo replication
+- Transparent handling of partitioned topics
+- Transparent batching of messages
 
 ## Repositories
 
@@ -88,34 +96,34 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la
 - pulsar master branch
 
 | Pulsar Components | Java Version  |
-| ----------------- |:-------------:|
-| Broker            | 17            |
-| Functions / IO    | 17            |
-| CLI               | 17            |
+| ----------------- | :-----------: |
+| Broker            |      17       |
+| Functions / IO    |      17       |
+| CLI               |      17       |
 | Java Client       | 8 or 11 or 17 |
 
 - 2.8 <= pulsar ver <= 2.10
 
-| Pulsar Components | Java Version  |
-| ----------------- |:-------------:|
-| Broker            | 11            |
-| Functions / IO    | 11            |
-| CLI               | 8 or 11       |
-| Java Client       | 8 or 11       |
+| Pulsar Components | Java Version |
+| ----------------- | :----------: |
+| Broker            |      11      |
+| Functions / IO    |      11      |
+| CLI               |   8 or 11    |
+| Java Client       |   8 or 11    |
 
 - pulsar ver 2.8 <
 
-| Pulsar Components | Java Version  |
-| ----------------- |:-------------:|
-| All               | 8 or 11       |
-
+| Pulsar Components | Java Version |
+| ----------------- | :----------: |
+| All               |   8 or 11    |
 
 ## Build Pulsar
 
 Requirements:
- * Java [JDK 17](https://adoptium.net/?variant=openjdk17)
- * Maven 3.6.1+
- * zip
+
+- Java [JDK 17](https://adoptium.net/?variant=openjdk17)
+- Maven 3.6.1+
+- zip
 
 > Note: this project includes a [Maven Wrapper](https://maven.apache.org/wrapper/) that can be used instead of a system installed Maven.
 > Use it by replacing `mvn` by `./mvnw` on Linux and `mvnw.cmd` on Windows in the commands below.
@@ -127,11 +135,13 @@ $ mvn install -DskipTests
 ```
 
 Compile and install individual module
+
 ```bash
 $ mvn -pl module-name (e.g: pulsar-broker) install -DskipTests
 ```
 
 ## Minimal build (This skips most of external connectors and tiered storage handlers)
+
 ```
 mvn install -Pcore-modules,-main -DskipTests
 ```
@@ -176,8 +186,8 @@ mvn clean install -DskipTests
 mvn package -Pdocker,-main -am -pl docker/pulsar-all -DskipTests
 ```
 
-After the images are built, they can be tagged and pushed to your custom repository. 
-Here's an example of a bash script that tags the docker images with the current version and git revision and 
+After the images are built, they can be tagged and pushed to your custom repository.
+Here's an example of a bash script that tags the docker images with the current version and git revision and
 pushes them to `localhost:32000/apachepulsar`.
 
 ```bash
@@ -197,28 +207,27 @@ docker push ${image_repo_and_project}/pulsar:$tag
 Apache Pulsar is using [lombok](https://projectlombok.org/) so you have to ensure your IDE setup with
 required plugins.
 
-### Intellij
+### IntelliJ
 
 #### Configure Project JDK to Java 17 JDK
 
-1. Open **Project Settings**. 
+1. Open **Project Settings**.
+
+   Click **File** -> **Project Structure** -> **Project Settings** -> **Project**.
 
-    Click **File** -> **Project Structure** -> **Project Settings** -> **Project**.
-
 2. Select the JDK version.
-    
-    From the JDK version drop-down list, select **Download JDK...** or choose an existing recent Java 17 JDK version.
+
+   From the JDK version drop-down list, select **Download JDK...** or choose an existing recent Java 17 JDK version.
 
 3. In the download dialog, select version **17**. You can pick a version from many vendors. Unless you have a specific preference, choose **Eclipse Temurin (AdoptOpenJDK (Hotspot))**.
-
 
 #### Configure Java version for Maven in IntelliJ
 
-1. Open Maven Importing Settings dialog by going to 
+1. Open Maven Importing Settings dialog by going to
    **Settings** -> **Build, Execution, Deployment** -> **Build Tools** -> **Maven** -> **Importing**.
 
-2. Choose **Use Project JDK** for **JDK for Importer** setting. This uses the Java 17 JDK for running Maven 
-   when importing the project to IntelliJ. Some of the configuration in the Maven build is conditional based on 
+2. Choose **Use Project JDK** for **JDK for Importer** setting. This uses the Java 17 JDK for running Maven
+   when importing the project to IntelliJ. Some of the configuration in the Maven build is conditional based on
    the JDK version. Incorrect configuration gets chosen when the "JDK for Importer" isn't the same as the "Project JDK".
 
 3. Validate that the JRE setting in **Maven** -> **Runner** dialog is set to **Use Project JDK**.
@@ -229,23 +238,25 @@ required plugins.
    **Settings** -> **Build, Execution, Deployment** -> **Compiler** -> **Annotation Processors**.
 
 2. Select the following buttons:
+
    1. **Enable annotation processing**
    2. **Obtain processors from project classpath**
    3. Store generated sources relative to: **Module content root**
 
 3. Set the generated source directories to be equal to the Maven directories:
+
    1. Set "Production sources directory:" to "target/generated-sources/annotations".
    2. Set "Test sources directory:" to "target/generated-test-sources/test-annotations".
 
 4. Click **OK**.
 
-5. Install the lombok plugin in intellij.
+5. Install the lombok plugin in intelliJ.
 
 #### Configure code style
 
 1. Open Code Style Settings dialog box by going to **Settings** -> **Editor** -> **Code Style**.
 
-2. Click on the :gear: symbol -> **Import scheme** -> **Intellij IDEA code style XML**
+2. Click on the :gear: symbol -> **Import scheme** -> **IntelliJ IDEA code style XML**
 
 3. Pick the file `${pulsar_dir}/src/idea-code-style.xml`
 
@@ -264,6 +275,7 @@ required plugins.
 4. Set **Scan scope** to **Only Java sources (including tests)**.
 
 5. Click **+** button in the **Configuration** section to open a dialog to choose the checkfile file.
+
    1. Enter a **Description**. For example, Pulsar.
    2. Select **Use a local checkstyle file**.
    3. Set **File** to **buildtools/src/main/resources/pulsar/checkstyle.xml**.
@@ -274,33 +286,32 @@ required plugins.
 
 7. Click **OK**.
 
-#### Further configuration in IntelliJ 
+#### Further configuration in IntelliJ
 
-* When working on the Pulsar core modules in IntelliJ, reduce the number of active projects in IntelliJ to speed up IDE actions and reduce unrelated IDE warnings.
-  * In IntelliJ's Maven UI's tree view under "Profiles"
-    * Activate "core-modules" Maven profile
-    * De-activate "main" Maven profile
-    * Run the "Reload All Maven Projects" action from the Maven UI toolbar. You can also find the action by the name in the IntelliJ "Search Everywhere" window that gets activated by pressing the **Shift** key twice.
+- When working on the Pulsar core modules in IntelliJ, reduce the number of active projects in IntelliJ to speed up IDE actions and reduce unrelated IDE warnings.
 
-* Run the "Generate Sources and Update Folders For All Projects" action from the Maven UI toolbar. You can also find the action by the name in the IntelliJ "Search Everywhere" window that gets activated by pressing the **Shift** key twice. Running the action takes about 10 minutes for all projects. This is faster when the "core-modules" profile is the only active profile.
+  - In IntelliJ's Maven UI's tree view under "Profiles"
+    - Activate "core-modules" Maven profile
+    - De-activate "main" Maven profile
+    - Run the "Reload All Maven Projects" action from the Maven UI toolbar. You can also find the action by the name in the IntelliJ "Search Everywhere" window that gets activated by pressing the **Shift** key twice.
 
+- Run the "Generate Sources and Update Folders For All Projects" action from the Maven UI toolbar. You can also find the action by the name in the IntelliJ "Search Everywhere" window that gets activated by pressing the **Shift** key twice. Running the action takes about 10 minutes for all projects. This is faster when the "core-modules" profile is the only active profile.
 
 #### IntelliJ usage tips
 
-* In the case of compilation errors with missing Protobuf classes, ensure to run the "Generate Sources and Update Folders For All Projects" action.
+- In the case of compilation errors with missing Protobuf classes, ensure to run the "Generate Sources and Update Folders For All Projects" action.
 
-* All of the Pulsar source code doesn't compile properly in IntelliJ and there are compilation errors.
-  * Use the "core-modules" profile if working on the Pulsar core modules since the source code for those modules can be compiled in IntelliJ.
-  * Sometimes it might help to mark a specific project ignored in IntelliJ Maven UI by right-clicking the project name and select **Ignore Projects** from the menu.
-  * Currently, it is not always possible to run unit tests directly from the IDE because of the compilation issues. As a workaround, individual test classes can be run by using the `mvn test -Dtest=TestClassName` command.
-
-* The above steps have all been performed, but a test still won't run.
-  * In this case, try the following steps:
+- All of the Pulsar source code doesn't compile properly in IntelliJ and there are compilation errors.
+  - Use the "core-modules" profile if working on the Pulsar core modules since the source code for those modules can be compiled in IntelliJ.
+  - Sometimes it might help to mark a specific project ignored in IntelliJ Maven UI by right-clicking the project name and select **Ignore Projects** from the menu.
+  - Currently, it is not always possible to run unit tests directly from the IDE because of the compilation issues. As a workaround, individual test classes can be run by using the `mvn test -Dtest=TestClassName` command.
+- The above steps have all been performed, but a test still won't run.
+  - In this case, try the following steps:
     1. Close IntelliJ.
     2. Run `mvn clean install -DskipTests` on the command line.
     3. Reopen IntelliJ.
-  * If that still doesn't work:
-    1. Verify Maven is using a supported version. Currently, the supported version of Maven is specified in the 
+  - If that still doesn't work:
+    1. Verify Maven is using a supported version. Currently, the supported version of Maven is specified in the
        <requireMavenVersion> section of the main pom.xml file.
     2. Try "restart and clear caches" in IntelliJ and repeat the above steps to reload projects and generate sources.
 
@@ -312,15 +323,15 @@ to configure your Eclipse setup.
 ## Documentation
 
 > **Tip**
-> 
+>
 > For how to make contributions to Pulsar documentation, see [Pulsar Documentation Contribution Guide](https://docs.google.com/document/d/11DTnNPpvcPrebLkMAFcDEIFlD8ARD-k6F-LXoIwdD9Y/edit#).
 
 ## Contact
 
 ##### Mailing lists
 
-| Name                                                                          | Scope                           |                                                                 |                                                                     |                                                                              |
-|:------------------------------------------------------------------------------|:--------------------------------|:----------------------------------------------------------------|:--------------------------------------------------------------------|:-----------------------------------------------------------------------------|
+| Name                                                      | Scope                           |                                                       |                                                           |                                                                    |
+| :-------------------------------------------------------- | :------------------------------ | :---------------------------------------------------- | :-------------------------------------------------------- | :----------------------------------------------------------------- |
 | [users@pulsar.apache.org](mailto:users@pulsar.apache.org) | User-related discussions        | [Subscribe](mailto:users-subscribe@pulsar.apache.org) | [Unsubscribe](mailto:users-unsubscribe@pulsar.apache.org) | [Archives](http://mail-archives.apache.org/mod_mbox/pulsar-users/) |
 | [dev@pulsar.apache.org](mailto:dev@pulsar.apache.org)     | Development-related discussions | [Subscribe](mailto:dev-subscribe@pulsar.apache.org)   | [Unsubscribe](mailto:dev-unsubscribe@pulsar.apache.org)   | [Archives](http://mail-archives.apache.org/mod_mbox/pulsar-dev/)   |
 

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
@@ -98,11 +98,6 @@
       <artifactId>testng</artifactId>
       <version>${testng.version}</version>
     </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <version>4.13.1</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>

diff --git a/conf/broker.conf b/conf/broker.conf
@@ -845,6 +845,14 @@ bookkeeperClientTimeoutInSeconds=30
 # Default is Runtime.getRuntime().availableProcessors()
 bookkeeperClientNumWorkerThreads=
 
+# Number of BookKeeper client IO threads
+# Default is Runtime.getRuntime().availableProcessors() * 2
+bookkeeperClientNumIoThreads=
+
+# Use separated IO threads for BookKeeper client
+# Default is false, which will use Pulsar IO threads
+bookkeeperClientSeparatedIoThreadsEnabled=false
+
 # Speculative reads are initiated if a read request doesn't complete within a certain time
 # Using a value of 0, is disabling the speculative reads
 bookkeeperClientSpeculativeReadTimeoutInMillis=0
@@ -1432,6 +1440,9 @@ transactionBufferSnapshotMinTimeInMillis=5000
 # The max concurrent requests for transaction buffer client, default is 1000
 transactionBufferClientMaxConcurrentRequests=1000
 
+# The max active transactions per transaction coordinator, default value 0 indicates no limit.
+maxActiveTransactionsPerCoordinator=0
+
 # MLPendingAckStore maintains a ConcurrentSkipListMap pendingAckLogIndex,
 # It stores the position in pendingAckStore as its value and saves a position used to determine
 # whether the previous data can be cleaned up as a key.

diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
@@ -159,7 +159,7 @@ functionRuntimeFactoryConfigs:
 #### Kubernetes Runtime ####
 # Pulsar function are deployed to Kubernetes
 
-# Upload the builtin sources/sinks to BookKeeper.
+# Upload the builtin sources/sinks/functions to BookKeeper.
 # True by default.
 # uploadBuiltinSinksSources: true
 #functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory

diff --git a/conf/presto/config.properties b/conf/presto/config.properties
@@ -37,6 +37,4 @@ scheduler.http-client.idle-timeout=1m
 query.client.timeout=5m
 query.min-expire-age=30m
 
-presto.version=testversion
-
 node-scheduler.include-coordinator=true
diff --git a/conf/presto/jvm.config b/conf/presto/jvm.config
@@ -26,4 +26,3 @@
 -XX:+ExitOnOutOfMemoryError
 -Dpresto-temporarily-allow-java8=true
 -Djdk.attach.allowAttachSelf=true
--javaagent:java-version-trim-agent.jar
diff --git a/conf/standalone.conf b/conf/standalone.conf
@@ -546,6 +546,14 @@ bookkeeperClientTimeoutInSeconds=30
 # Default is Runtime.getRuntime().availableProcessors()
 bookkeeperClientNumWorkerThreads=
 
+# Number of BookKeeper client IO threads
+# Default is Runtime.getRuntime().availableProcessors() * 2
+bookkeeperClientNumIoThreads=
+
+# Use separated IO threads for BookKeeper client
+# Default is false, which will use Pulsar IO threads
+bookkeeperClientSeparatedIoThreadsEnabled=false
+
 # Speculative reads are initiated if a read request doesn't complete within a certain time
 # Using a value of 0, is disabling the speculative reads
 bookkeeperClientSpeculativeReadTimeoutInMillis=0