From 35ab0dca05f5598b15892a26697f0e708478f6af Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Mon, 25 Mar 2024 17:56:03 -0700 Subject: [PATCH 1/9] First draft of Helidon 4 threading example --- examples/webserver/threads/README.md | 23 ++ examples/webserver/threads/pom.xml | 68 ++++++ .../examples/webserver/threads/Main.java | 59 ++++++ .../webserver/threads/ThreadService.java | 199 ++++++++++++++++++ .../webserver/threads/package-info.java | 1 + .../src/main/resources/application.yaml | 14 ++ .../src/main/resources/logging.properties | 6 + .../examples/webserver/threads/MainTest.java | 9 + .../src/test/resources/application-test.yaml | 0 9 files changed, 379 insertions(+) create mode 100644 examples/webserver/threads/README.md create mode 100644 examples/webserver/threads/pom.xml create mode 100644 examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java create mode 100644 examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java create mode 100644 examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java create mode 100644 examples/webserver/threads/src/main/resources/application.yaml create mode 100644 examples/webserver/threads/src/main/resources/logging.properties create mode 100644 examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java create mode 100644 examples/webserver/threads/src/test/resources/application-test.yaml diff --git a/examples/webserver/threads/README.md b/examples/webserver/threads/README.md new file mode 100644 index 00000000000..f3bd07ac110 --- /dev/null +++ b/examples/webserver/threads/README.md @@ -0,0 +1,23 @@ +# helidon-examples-webserver-threads + + +TODO XXXXXXXXXXXXX + +## Build and run + + +With JDK21 +```bash +mvn package +java -jar target/helidon-examples-webserver-threads.jar +``` + +## Exercise the application + +Basic: +``` +curl -X GET http://localhost:8080/thread/compute/5 +Hello World! +``` + + diff --git a/examples/webserver/threads/pom.xml b/examples/webserver/threads/pom.xml new file mode 100644 index 00000000000..1ed136989f1 --- /dev/null +++ b/examples/webserver/threads/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + io.helidon.applications + helidon-se + 4.0.0-SNAPSHOT + ../../../applications/se/pom.xml + + io.helidon.examples.webserver + helidon-examples-webserver-threads + 4.0.0-SNAPSHOT + + + io.helidon.examples.webserver.threads.Main + + + + + io.helidon.webserver + helidon-webserver + + + io.helidon.webclient + helidon-webclient + + + io.helidon.config + helidon-config-yaml + + + io.helidon.logging + helidon-logging-jul + runtime + + + org.junit.jupiter + junit-jupiter-api + test + + + org.hamcrest + hamcrest-all + test + + + io.helidon.webserver.testing.junit5 + helidon-webserver-testing-junit5 + test + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-libs + + + + + + diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java new file mode 100644 index 00000000000..e31fb326c67 --- /dev/null +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java @@ -0,0 +1,59 @@ + +package io.helidon.examples.webserver.threads; + + +import io.helidon.logging.common.LogConfig; +import io.helidon.config.Config; +import io.helidon.webserver.WebServer; +import io.helidon.webserver.http.HttpRouting; + + + + +/** + * The application main class. + */ +public class Main { + + + /** + * Cannot be instantiated. + */ + private Main() { + } + + + /** + * Application main entry point. + * @param args command line arguments. + */ + public static void main(String[] args) { + + // load logging configuration + LogConfig.configureRuntime(); + + // initialize global config from default configuration + Config config = Config.create(); + Config.global(config); + + + WebServer server = WebServer.builder() + .config(config.get("server")) + .routing(Main::routing) + .build() + .start(); + + + System.out.println("WEB server is up! http://localhost:" + server.port() + "/thread"); + + } + + + /** + * Updates HTTP Routing. + */ + static void routing(HttpRouting.Builder routing) { + routing + .register("/thread", new ThreadService()); + } +} \ No newline at end of file diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java new file mode 100644 index 00000000000..7f07dffb5da --- /dev/null +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java @@ -0,0 +1,199 @@ +package io.helidon.examples.webserver.threads; + +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.lang.System.Logger.Level; +import java.util.concurrent.RejectedExecutionException; + +import io.helidon.common.configurable.ThreadPoolSupplier; +import io.helidon.config.Config; +import io.helidon.http.Status; +import io.helidon.webclient.api.HttpClientResponse; +import io.helidon.webclient.api.WebClient; +import io.helidon.webserver.http.HttpRules; +import io.helidon.webserver.http.HttpService; +import io.helidon.webserver.http.ServerRequest; +import io.helidon.webserver.http.ServerResponse; + +class ThreadService implements HttpService { + + private static final System.Logger LOGGER = System.getLogger(ThreadService.class.getName()); + private static final Random rand = new Random(System.currentTimeMillis()); + + // ThreadPool of platform threads. + private static ExecutorService platformExecutorService; + // Executor of virtual threads. + private static ExecutorService virtualExecutorService; + + WebClient client = WebClient.builder() + .baseUri("http://localhost:8080/thread") + .build(); + + /** + * The config value for the key {@code greeting}. + */ + + ThreadService() { + this(Config.global().get("app")); + } + + ThreadService(Config appConfig) { + /* + * We create two executor services. One is a thread pool of platform threads. + * The second is a virtual thread executor service. + * See `application.yaml` for configuration of each of these. + */ + ThreadPoolSupplier platformThreadPoolSupplier = ThreadPoolSupplier.builder() + .config(appConfig.get("application-platform-executor")) + .build(); + platformExecutorService = platformThreadPoolSupplier.get(); + + ThreadPoolSupplier virtualThreadPoolSupplier = ThreadPoolSupplier.builder() + .config(appConfig.get("application-virtual-executor")) + .build(); + virtualExecutorService = virtualThreadPoolSupplier.get(); + } + + /** + * A service registers itself by updating the routing rules. + * + * @param rules the routing rules. + */ + @Override + public void routing(HttpRules rules) { + rules + .get("/compute", this::computeHandler) + .get("/compute/{iterations}", this::computeHandler) + .get("/fanout", this::fanOutHandler) + .get("/fanout/{count}", this::fanOutHandler) + .get("/sleep", this::sleepHandler) + .get("/sleep/{seconds}", this::sleepHandler); + } + + /** + * Perform a CPU intensive operation. + * The optional path parameter controls the number of iterations of the computation. The more + * iterations the longer it will take. + * + * @param request server request + * @param response server response + */ + private void computeHandler(ServerRequest request, ServerResponse response) { + String iterations = request.path().pathParameters().first("iterations").orElse("1"); + try { + // We execute the computation on a platform thread. This prevents pining of the virtual + // thread, plus provides us the ability to limit the number of concurrent computation requests + // we handle by limiting the thread pool work queue length (as defined in application.yaml) + Future future = platformExecutorService.submit(() -> compute(Integer.parseInt(iterations))); + response.send(future.get().toString()); + } catch (RejectedExecutionException e) { + // Work queue is full! We reject the request + LOGGER.log(Level.WARNING, e); + response.status(Status.SERVICE_UNAVAILABLE_503).send("Server busy"); + } catch (ExecutionException | InterruptedException e) { + LOGGER.log(Level.ERROR, e); + response.status(Status.INTERNAL_SERVER_ERROR_500).send(); + } + } + + /** + * Sleep for a specified number of secons. + * The optional path parameter controls the number of seconds to sleep. Defaults to 1 + * + * @param request server request + * @param response server response + */ + private void sleepHandler(ServerRequest request, ServerResponse response) { + String seconds = request.path().pathParameters().first("seconds").orElse("1"); + response.send(Integer.toString(sleep(Integer.parseInt(seconds)))); + } + + /** + * Fan out a number of remote requests in parallel. + * The optional path parameter controls the number of parallel requests to make. + * + * @param request server request + * @param response server response + */ + private void fanOutHandler(ServerRequest request, ServerResponse response) { + int count = Integer.parseInt(request.path().pathParameters().first("count").orElse("1")); + LOGGER.log(Level.INFO, "Fanning out " + count + " parallel requests"); + // We simulate multiple client requests running in parallel by calling our sleep endpoint. + try { + // For this we use our virtual thread based executor. We submit the work and save the Futures + var futures = new ArrayList>(); + for (int i = 0; i < count; i++) { + futures.add(virtualExecutorService.submit(() -> callRemote(rand.nextInt(5)))); + } + + // After work has been submitted we loop through the future and block getting the results. + // We aggregate the results in a list of Strings + var responses = new ArrayList(); + for (var future : futures) { + try { + responses.add(future.get()); + } catch (InterruptedException e) { + responses.add(e.getMessage()); + } + } + + // All parallel calls are complete! + response.send(String.join(":", responses)); + } catch (ExecutionException e) { + LOGGER.log(Level.ERROR, e); + response.status(Status.INTERNAL_SERVER_ERROR_500).send(); + } + } + + /** + * Simulate a remote client call be calling the sleep endpoint on ourself. + * + * @param seconds number of seconds the endpoint should sleep. + * @return + */ + private String callRemote(int seconds) { + LOGGER.log(Level.INFO, Thread.currentThread() + ": Calling remote sleep for " + seconds + "s"); + try (HttpClientResponse response = client.get("/sleep/" + seconds).request()) { + if (response.status().equals(Status.OK_200)) { + return response.as(String.class); + } else { + return (response.status().toString()); + } + } + } + + /** + * Sleep current thread + * @param seconds number of seconds to sleep + * @return + */ + private int sleep(int seconds) { + try { + Thread.sleep(seconds * 1_000L); + } catch (InterruptedException e) { + LOGGER.log(Level.WARNING, e); + } + return seconds; + } + + /** + * Perform a CPU intensive computation + * @param iterations: number of times to perform computation + */ + private double compute(int iterations) { + LOGGER.log(Level.INFO, Thread.currentThread() + ": Computing with " + iterations + " iterations"); + double d = 123456789.123456789 * rand.nextInt(100); + for (int i=0; i < iterations; i++) { + for (int n=0; n < 1_000_000; n++) { + for (int j = 0; j < 5; j++) { + d = Math.tan(d); + d = Math.atan(d); + } + } + } + return d; + } +} diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java new file mode 100644 index 00000000000..17d1693e0ef --- /dev/null +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java @@ -0,0 +1 @@ +package io.helidon.examples.webserver.threads; diff --git a/examples/webserver/threads/src/main/resources/application.yaml b/examples/webserver/threads/src/main/resources/application.yaml new file mode 100644 index 00000000000..fbda3d443a1 --- /dev/null +++ b/examples/webserver/threads/src/main/resources/application.yaml @@ -0,0 +1,14 @@ +server: + port: 8080 + host: 0.0.0.0 + +app: + greeting: "Hello" + application-platform-executor: + thread-name-prefix: "application-platform-executor-" + core-pool-size: 1 + max-pool-size: 2 + queue-capacity: 10 + application-virtual-executor: + thread-name-prefix: "application-virtual-executor-" + virtual-threads: true diff --git a/examples/webserver/threads/src/main/resources/logging.properties b/examples/webserver/threads/src/main/resources/logging.properties new file mode 100644 index 00000000000..0f674e20e0a --- /dev/null +++ b/examples/webserver/threads/src/main/resources/logging.properties @@ -0,0 +1,6 @@ +handlers=io.helidon.logging.jul.HelidonConsoleHandler +java.util.logging.SimpleFormatter.format=%1$tY.%1$tm.%1$td %1$tH:%1$tM:%1$tS.%1$tL %5$s%6$s%n +# Global logging level. Can be overridden by specific loggers +.level=INFO + +io.helidon.common.configurable.ThreadPool.level=ALL diff --git a/examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java b/examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java new file mode 100644 index 00000000000..274eeb2557a --- /dev/null +++ b/examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java @@ -0,0 +1,9 @@ +package io.helidon.examples.webserver.threads; + +import io.helidon.webserver.testing.junit5.RoutingTest; + +@RoutingTest +class MainTest { + MainTest() { + } +} \ No newline at end of file diff --git a/examples/webserver/threads/src/test/resources/application-test.yaml b/examples/webserver/threads/src/test/resources/application-test.yaml new file mode 100644 index 00000000000..e69de29bb2d From c75d26822989fb81a05cf89f3d99442a59e761f8 Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Tue, 26 Mar 2024 12:13:14 -0700 Subject: [PATCH 2/9] Update README --- examples/webserver/threads/README.md | 87 ++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/examples/webserver/threads/README.md b/examples/webserver/threads/README.md index f3bd07ac110..8a5d1c073a0 100644 --- a/examples/webserver/threads/README.md +++ b/examples/webserver/threads/README.md @@ -1,23 +1,98 @@ -# helidon-examples-webserver-threads +# Helidon SE Threading Example +Helidon's adoption of virtual threads has eliminated a lot of the headaches +of thread pools and thread pool tuning. But there are still cases where using +application specific executors is desirable. This example illustrates two +such cases: -TODO XXXXXXXXXXXXX +1. You want to execute multiple tasks in parallel. +2. You want to execute a CPU intensive, long running operation. -## Build and run +Let's look at these two use cases in a bit more detail. + +## Use Case 1: Executing Tasks in Parallel + +In this case you have an endpoint that wants to execute multiple tasks in parallel +that perform some blocking operations. +Examples of this might be making multiple calls to a database or to some other service. +Virtual threads are a good fit for this because they are lightweight and do not consume +platform threads when performing blocking operations (like network I/O). + +The `fanout` endpoint in this example demonstrates this use case. You pass the endpoint +the number of parallel tasks to execute and it simulates remote client calls by using +the Helidon WebClient to call the `sleep` endpoint on the server. + +## Use Case 2: Executing a CPU Intensive Task + +In this case you have an endpoint that performs an in-memory, CPU intensive task. +This is not a good fit for virtual threads because the virtual thread would be pinned to +a platform thread -- potentially causing unbounded consumption of platform threads. Instead, +the example uses a small, bounded pool of platform threads. Bounded meaning that the number +of threads and the size of the work queue are both limited and will reject work when they fill up. +This lets you have tight control over the resources you allocate to these CPU +intensive tasks. +The `compute` endpoint in this example demonstrates this use case. You pass the endpoint +the number of times you want to make the computation, and it uses a small bounded pool +of platform threads to execute the task. + +## Use of Helidon's ThreadPoolSupplier and Configuration + +The example uses `io.helidon.common.configurable.ThreadPoolSupplier` to create the +two executors used in the example. This provides a couple of benefits: + +1. ThreadPoolSupplier supports a number of tuning parameters that enable us to configure a small, bounded threadpool. +2. You can drive the thread pool configuration via Helidon config -- see this example's `application.yaml` +3. You get propagation of Helidon's Context which supports Helidon's features as well as direct use by the application. + +## Logging + +In `logging.properties` we increase the log level for `io.helidon.common.configurable.ThreadPool` +so that you can see the values used to configure the platform thread pool. + +## Build and run -With JDK21 ```bash mvn package java -jar target/helidon-examples-webserver-threads.jar ``` +You will see a line like the following: +``` +ThreadPool 'application-platform-executor-thread-pool-1' {corePoolSize=1, maxPoolSize=2, + queueCapacity=10, growthThreshold=1000, growthRate=0%, averageQueueSize=0.00, peakQueueSize=0, averageActiveThreads=0.00, peakPoolSize=0, currentPoolSize=0, completedTasks=0, failedTasks=0, rejectedTasks=0} +``` +This reflects the configuration of the platform thread pool created by the application +and used by the `compute` endpoint. At most the thread pool will consume two platform +threads for computations. The work queue is limited to 10 entries to allow for small +bursts of requests. + ## Exercise the application -Basic: +__Compute:__ ``` curl -X GET http://localhost:8080/thread/compute/5 -Hello World! ``` +Depending on the speed of your machine this should take a few seconds to complete. You +can increase the number to force the computation to take longer. + +The request returns the results of the computation (not important!). +__Fanout:__ +``` +curl -X GET http://localhost:8080/thread/fanout/5 +``` +This will simulate a fanout of five remote calls. Each call will sleep anywhere from +0 to 4 seconds. Since the requests are executed in parallel the request should not +take longer than 4 seconds total. + +The request returns a list of numbers showing the sleep value of each remote client call. + +__Sleep:__ +``` +curl -X GET http://localhost:8080/thread/sleep/4 +``` +This is a simple endpoint that just sleeps for the specified number of seconds. It is +used by the `fanout` endpoint. +The request returns the number of seconds requested to sleep. From a9ae9d766d3c4bbc47c16fe0e3cdbc5fdeb20b14 Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Tue, 26 Mar 2024 12:13:33 -0700 Subject: [PATCH 3/9] Fix copyrights. Add tests. Change how web client is initialized --- .../examples/webserver/threads/Main.java | 32 +++++++---- .../webserver/threads/ThreadService.java | 30 ++++++++--- .../webserver/threads/package-info.java | 16 ++++++ .../src/main/resources/application.yaml | 33 ++++++++---- .../src/main/resources/logging.properties | 16 ++++++ .../examples/webserver/threads/MainTest.java | 54 +++++++++++++++++-- 6 files changed, 152 insertions(+), 29 deletions(-) diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java index e31fb326c67..6549bb3ca66 100644 --- a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java @@ -1,20 +1,34 @@ +/* + * Copyright (c) 2024 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.helidon.examples.webserver.threads; - import io.helidon.logging.common.LogConfig; import io.helidon.config.Config; +import io.helidon.webclient.api.WebClient; import io.helidon.webserver.WebServer; import io.helidon.webserver.http.HttpRouting; - - - /** * The application main class. */ public class Main { + static WebServer webserver; + static WebClient webclient; /** * Cannot be instantiated. @@ -22,7 +36,6 @@ public class Main { private Main() { } - /** * Application main entry point. * @param args command line arguments. @@ -36,16 +49,17 @@ public static void main(String[] args) { Config config = Config.create(); Config.global(config); - - WebServer server = WebServer.builder() + webserver = WebServer.builder() .config(config.get("server")) .routing(Main::routing) .build() .start(); + webclient = WebClient.builder() + .baseUri("http://localhost:" + webserver.port() + "/thread") + .build(); - System.out.println("WEB server is up! http://localhost:" + server.port() + "/thread"); - + System.out.println("WEB server is up! http://localhost:" + webserver.port() + "/thread"); } diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java index 7f07dffb5da..0323c0e2b26 100644 --- a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.helidon.examples.webserver.threads; import java.util.ArrayList; @@ -28,10 +44,6 @@ class ThreadService implements HttpService { // Executor of virtual threads. private static ExecutorService virtualExecutorService; - WebClient client = WebClient.builder() - .baseUri("http://localhost:8080/thread") - .build(); - /** * The config value for the key {@code greeting}. */ @@ -100,7 +112,7 @@ private void computeHandler(ServerRequest request, ServerResponse response) { } /** - * Sleep for a specified number of secons. + * Sleep for a specified number of seconds. * The optional path parameter controls the number of seconds to sleep. Defaults to 1 * * @param request server request @@ -149,13 +161,14 @@ private void fanOutHandler(ServerRequest request, ServerResponse response) { } /** - * Simulate a remote client call be calling the sleep endpoint on ourself. + * Simulate a remote client call be calling this server's sleep endpoint * * @param seconds number of seconds the endpoint should sleep. - * @return + * @return string response from client */ private String callRemote(int seconds) { LOGGER.log(Level.INFO, Thread.currentThread() + ": Calling remote sleep for " + seconds + "s"); + WebClient client = Main.webclient; try (HttpClientResponse response = client.get("/sleep/" + seconds).request()) { if (response.status().equals(Status.OK_200)) { return response.as(String.class); @@ -168,7 +181,7 @@ private String callRemote(int seconds) { /** * Sleep current thread * @param seconds number of seconds to sleep - * @return + * @return number of seconds requested to sleep */ private int sleep(int seconds) { try { @@ -182,6 +195,7 @@ private int sleep(int seconds) { /** * Perform a CPU intensive computation * @param iterations: number of times to perform computation + * @return result of computation */ private double compute(int iterations) { LOGGER.log(Level.INFO, Thread.currentThread() + ": Computing with " + iterations + " iterations"); diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java index 17d1693e0ef..f7dfc7e8a4e 100644 --- a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/package-info.java @@ -1 +1,17 @@ +/* + * Copyright (c) 2024 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.helidon.examples.webserver.threads; diff --git a/examples/webserver/threads/src/main/resources/application.yaml b/examples/webserver/threads/src/main/resources/application.yaml index fbda3d443a1..05ee0f283e5 100644 --- a/examples/webserver/threads/src/main/resources/application.yaml +++ b/examples/webserver/threads/src/main/resources/application.yaml @@ -1,14 +1,29 @@ +# +# Copyright (c) 2024 Oracle and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + server: port: 8080 host: 0.0.0.0 app: - greeting: "Hello" - application-platform-executor: - thread-name-prefix: "application-platform-executor-" - core-pool-size: 1 - max-pool-size: 2 - queue-capacity: 10 - application-virtual-executor: - thread-name-prefix: "application-virtual-executor-" - virtual-threads: true + application-platform-executor: + thread-name-prefix: "application-platform-executor-" + core-pool-size: 1 + max-pool-size: 2 + queue-capacity: 10 + application-virtual-executor: + thread-name-prefix: "application-virtual-executor-" + virtual-threads: true diff --git a/examples/webserver/threads/src/main/resources/logging.properties b/examples/webserver/threads/src/main/resources/logging.properties index 0f674e20e0a..5fddb7f3d50 100644 --- a/examples/webserver/threads/src/main/resources/logging.properties +++ b/examples/webserver/threads/src/main/resources/logging.properties @@ -1,3 +1,19 @@ +# +# Copyright (c) 2024 Oracle and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + handlers=io.helidon.logging.jul.HelidonConsoleHandler java.util.logging.SimpleFormatter.format=%1$tY.%1$tm.%1$td %1$tH:%1$tM:%1$tS.%1$tL %5$s%6$s%n # Global logging level. Can be overridden by specific loggers diff --git a/examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java b/examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java index 274eeb2557a..bbe78c10f9a 100644 --- a/examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java +++ b/examples/webserver/threads/src/test/java/io/helidon/examples/webserver/threads/MainTest.java @@ -1,9 +1,57 @@ +/* + * Copyright (c) 2024 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.helidon.examples.webserver.threads; -import io.helidon.webserver.testing.junit5.RoutingTest; +import io.helidon.http.Status; +import io.helidon.webclient.api.HttpClientResponse; +import io.helidon.webclient.api.WebClient; +import io.helidon.webserver.http.HttpRouting; +import io.helidon.webserver.testing.junit5.ServerTest; +import io.helidon.webserver.testing.junit5.SetUpRoute; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; -@RoutingTest +@ServerTest class MainTest { - MainTest() { + private final WebClient client; + + protected MainTest(WebClient client) { + this.client = client; + Main.webclient = this.client; // Needed for ThreadService to make calls + } + + @SetUpRoute + static void routing(HttpRouting.Builder builder) { + Main.routing(builder); + } + + @Test + void testFanOut() { + try (HttpClientResponse response = client.get("/thread/fanout/2").request()) { + assertThat(response.status(), is(Status.OK_200)); + } + } + + @Test + void testCompute() { + try (HttpClientResponse response = client.get("/thread/compute").request()) { + assertThat(response.status(), is(Status.OK_200)); + } } } \ No newline at end of file From 33795bab132045567b38305a7c1ead7f7476d693 Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Tue, 26 Mar 2024 14:40:00 -0700 Subject: [PATCH 4/9] Fix pom.xml copyright --- examples/webserver/threads/pom.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/webserver/threads/pom.xml b/examples/webserver/threads/pom.xml index 1ed136989f1..3fc7bbf87e5 100644 --- a/examples/webserver/threads/pom.xml +++ b/examples/webserver/threads/pom.xml @@ -1,4 +1,21 @@ + From 8ce7b9f9d88bf161d9ea03703792cd3e179cd18f Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Wed, 27 Mar 2024 13:40:48 -0700 Subject: [PATCH 5/9] More README updates. Put details at end. --- examples/webserver/threads/README.md | 118 +++++++++++++-------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/examples/webserver/threads/README.md b/examples/webserver/threads/README.md index 8a5d1c073a0..6b301484238 100644 --- a/examples/webserver/threads/README.md +++ b/examples/webserver/threads/README.md @@ -5,50 +5,8 @@ of thread pools and thread pool tuning. But there are still cases where using application specific executors is desirable. This example illustrates two such cases: -1. You want to execute multiple tasks in parallel. -2. You want to execute a CPU intensive, long running operation. - -Let's look at these two use cases in a bit more detail. - -## Use Case 1: Executing Tasks in Parallel - -In this case you have an endpoint that wants to execute multiple tasks in parallel -that perform some blocking operations. -Examples of this might be making multiple calls to a database or to some other service. -Virtual threads are a good fit for this because they are lightweight and do not consume -platform threads when performing blocking operations (like network I/O). - -The `fanout` endpoint in this example demonstrates this use case. You pass the endpoint -the number of parallel tasks to execute and it simulates remote client calls by using -the Helidon WebClient to call the `sleep` endpoint on the server. - -## Use Case 2: Executing a CPU Intensive Task - -In this case you have an endpoint that performs an in-memory, CPU intensive task. -This is not a good fit for virtual threads because the virtual thread would be pinned to -a platform thread -- potentially causing unbounded consumption of platform threads. Instead, -the example uses a small, bounded pool of platform threads. Bounded meaning that the number -of threads and the size of the work queue are both limited and will reject work when they fill up. -This lets you have tight control over the resources you allocate to these CPU -intensive tasks. - -The `compute` endpoint in this example demonstrates this use case. You pass the endpoint -the number of times you want to make the computation, and it uses a small bounded pool -of platform threads to execute the task. - -## Use of Helidon's ThreadPoolSupplier and Configuration - -The example uses `io.helidon.common.configurable.ThreadPoolSupplier` to create the -two executors used in the example. This provides a couple of benefits: - -1. ThreadPoolSupplier supports a number of tuning parameters that enable us to configure a small, bounded threadpool. -2. You can drive the thread pool configuration via Helidon config -- see this example's `application.yaml` -3. You get propagation of Helidon's Context which supports Helidon's features as well as direct use by the application. - -## Logging - -In `logging.properties` we increase the log level for `io.helidon.common.configurable.ThreadPool` -so that you can see the values used to configure the platform thread pool. +1. Using a virtual thread executor to execute multiple tasks in parallel. +2. Using a platform thread executor to execute long-running CPU intensive operations using platform threads. ## Build and run @@ -57,24 +15,14 @@ mvn package java -jar target/helidon-examples-webserver-threads.jar ``` -You will see a line like the following: -``` -ThreadPool 'application-platform-executor-thread-pool-1' {corePoolSize=1, maxPoolSize=2, - queueCapacity=10, growthThreshold=1000, growthRate=0%, averageQueueSize=0.00, peakQueueSize=0, averageActiveThreads=0.00, peakPoolSize=0, currentPoolSize=0, completedTasks=0, failedTasks=0, rejectedTasks=0} -``` -This reflects the configuration of the platform thread pool created by the application -and used by the `compute` endpoint. At most the thread pool will consume two platform -threads for computations. The work queue is limited to 10 entries to allow for small -bursts of requests. - ## Exercise the application __Compute:__ ``` curl -X GET http://localhost:8080/thread/compute/5 ``` -Depending on the speed of your machine this should take a few seconds to complete. You -can increase the number to force the computation to take longer. +The `compute` endpoint runs a costly floating point computation using a platform thread. +Increases the number to make the computation more costly (and take longer). The request returns the results of the computation (not important!). @@ -82,9 +30,11 @@ __Fanout:__ ``` curl -X GET http://localhost:8080/thread/fanout/5 ``` -This will simulate a fanout of five remote calls. Each call will sleep anywhere from -0 to 4 seconds. Since the requests are executed in parallel the request should not -take longer than 4 seconds total. +The `fanout` endpoint simulates a fanout of remote calls that are run in parallel using +virtual threads. Each call invokes the server's `sleep` endpoint sleeping anywhere from +0 to 4 seconds. Since the remote requests are executed in parallel the request should not +take longer than 4 seconds to return. Increase the number to have more remote calls made +in parallel. The request returns a list of numbers showing the sleep value of each remote client call. @@ -96,3 +46,53 @@ This is a simple endpoint that just sleeps for the specified number of seconds. used by the `fanout` endpoint. The request returns the number of seconds requested to sleep. + +## Further Discussion + +### Use Case 1: Virtual Threads: Executing Tasks in Parallel + +Sometimes an endpoint needs to perform multiple blocking operations in parallel: +querying a database, calling another service, etc. Virtual threads are a +good fit for this because they are lightweight and do not consume platform +threads when performing blocking operations (like network I/O). + +The `fanout` endpoint in this example demonstrates this use case. You pass the endpoint +the number of parallel tasks to execute and it simulates remote client calls by using +the Helidon WebClient to call the `sleep` endpoint on the server. + +### Use Case 2: Platform Threads: Executing a CPU Intensive Task + +If you have an endpoint that performs an in-memory, CPU intensive task, then +platform threads might be a better match. This is because a virtual thread would be pinned to +a platform thread throughout the computation -- potentially causing unbounded consumption +of platform threads. Instead, the example uses a small, bounded pool of platform +threads to perform computations. Bounded meaning that the number of threads and the +size of the work queue are both limited and will reject work when they fill up. +This gives the application tight control over the resources allocated to these CPU intensive tasks. + +The `compute` endpoint in this example demonstrates this use case. You pass the endpoint +the number of times you want to make the computation, and it uses a small bounded pool +of platform threads to execute the task. + +### Use of Helidon's ThreadPoolSupplier and Configuration + +This example uses `io.helidon.common.configurable.ThreadPoolSupplier` to create the +two executors used in the example. This provides a couple of benefits: + +1. ThreadPoolSupplier supports a number of tuning parameters that enable us to configure a small, bounded threadpool. +2. You can drive the thread pool configuration via Helidon config -- see this example's `application.yaml` +3. You get propagation of Helidon's Context which supports Helidon's features as well as direct use by the application. + +### Logging + +In `logging.properties` the log level for `io.helidon.common.configurable.ThreadPool` +is increased so that you can see the values used to configure the platform thread pool. +When you start the application you will see a line like +``` +ThreadPool 'application-platform-executor-thread-pool-1' {corePoolSize=1, maxPoolSize=2, + queueCapacity=10, growthThreshold=1000, growthRate=0%, averageQueueSize=0.00, peakQueueSize=0, averageActiveThreads=0.00, peakPoolSize=0, currentPoolSize=0, completedTasks=0, failedTasks=0, rejectedTasks=0} +``` +This reflects the configuration of the platform thread pool created by the application +and used by the `compute` endpoint. At most the thread pool will consume two platform +threads for computations. The work queue is limited to 10 entries to allow for small +bursts of requests. From 44e7eb9bd54c3b1803ab1cf03cabb8340ca2b516 Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Wed, 27 Mar 2024 15:08:08 -0700 Subject: [PATCH 6/9] Code cleanup --- .../examples/webserver/threads/Main.java | 4 +- .../webserver/threads/ThreadService.java | 43 ++++++++----------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java index 6549bb3ca66..41e008633c1 100644 --- a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/Main.java @@ -27,7 +27,6 @@ */ public class Main { - static WebServer webserver; static WebClient webclient; /** @@ -49,12 +48,13 @@ public static void main(String[] args) { Config config = Config.create(); Config.global(config); - webserver = WebServer.builder() + WebServer webserver = WebServer.builder() .config(config.get("server")) .routing(Main::routing) .build() .start(); + // Construct webclient here using port of running server webclient = WebClient.builder() .baseUri("http://localhost:" + webserver.port() + "/thread") .build(); diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java index 0323c0e2b26..b02fb8e8ed1 100644 --- a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java @@ -16,18 +16,18 @@ package io.helidon.examples.webserver.threads; +import java.lang.System.Logger.Level; import java.util.ArrayList; import java.util.Random; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.lang.System.Logger.Level; import java.util.concurrent.RejectedExecutionException; import io.helidon.common.configurable.ThreadPoolSupplier; import io.helidon.config.Config; import io.helidon.http.Status; -import io.helidon.webclient.api.HttpClientResponse; +import io.helidon.webclient.api.ClientResponseTyped; import io.helidon.webclient.api.WebClient; import io.helidon.webserver.http.HttpRules; import io.helidon.webserver.http.HttpService; @@ -69,11 +69,6 @@ class ThreadService implements HttpService { virtualExecutorService = virtualThreadPoolSupplier.get(); } - /** - * A service registers itself by updating the routing rules. - * - * @param rules the routing rules. - */ @Override public void routing(HttpRules rules) { rules @@ -90,16 +85,16 @@ public void routing(HttpRules rules) { * The optional path parameter controls the number of iterations of the computation. The more * iterations the longer it will take. * - * @param request server request + * @param request server request * @param response server response */ private void computeHandler(ServerRequest request, ServerResponse response) { - String iterations = request.path().pathParameters().first("iterations").orElse("1"); + int iterations = request.path().pathParameters().first("iterations").asInt().orElse(1); try { // We execute the computation on a platform thread. This prevents pining of the virtual // thread, plus provides us the ability to limit the number of concurrent computation requests // we handle by limiting the thread pool work queue length (as defined in application.yaml) - Future future = platformExecutorService.submit(() -> compute(Integer.parseInt(iterations))); + Future future = platformExecutorService.submit(() -> compute(iterations)); response.send(future.get().toString()); } catch (RejectedExecutionException e) { // Work queue is full! We reject the request @@ -115,23 +110,23 @@ private void computeHandler(ServerRequest request, ServerResponse response) { * Sleep for a specified number of seconds. * The optional path parameter controls the number of seconds to sleep. Defaults to 1 * - * @param request server request + * @param request server request * @param response server response */ private void sleepHandler(ServerRequest request, ServerResponse response) { - String seconds = request.path().pathParameters().first("seconds").orElse("1"); - response.send(Integer.toString(sleep(Integer.parseInt(seconds)))); + int seconds = request.path().pathParameters().first("seconds").asInt().orElse(1); + response.send(String.valueOf(sleep(seconds))); } /** * Fan out a number of remote requests in parallel. * The optional path parameter controls the number of parallel requests to make. * - * @param request server request + * @param request server request * @param response server response */ private void fanOutHandler(ServerRequest request, ServerResponse response) { - int count = Integer.parseInt(request.path().pathParameters().first("count").orElse("1")); + int count = request.path().pathParameters().first("count").asInt().orElse(1); LOGGER.log(Level.INFO, "Fanning out " + count + " parallel requests"); // We simulate multiple client requests running in parallel by calling our sleep endpoint. try { @@ -161,7 +156,7 @@ private void fanOutHandler(ServerRequest request, ServerResponse response) { } /** - * Simulate a remote client call be calling this server's sleep endpoint + * Simulate a remote client call by calling this server's sleep endpoint * * @param seconds number of seconds the endpoint should sleep. * @return string response from client @@ -169,17 +164,16 @@ private void fanOutHandler(ServerRequest request, ServerResponse response) { private String callRemote(int seconds) { LOGGER.log(Level.INFO, Thread.currentThread() + ": Calling remote sleep for " + seconds + "s"); WebClient client = Main.webclient; - try (HttpClientResponse response = client.get("/sleep/" + seconds).request()) { - if (response.status().equals(Status.OK_200)) { - return response.as(String.class); - } else { - return (response.status().toString()); - } + ClientResponseTyped response = client.get("/sleep/" + seconds).request(String.class); + if (response.status().equals(Status.OK_200)) { + return response.entity(); } + return response.status().toString(); } /** * Sleep current thread + * * @param seconds number of seconds to sleep * @return number of seconds requested to sleep */ @@ -194,14 +188,15 @@ private int sleep(int seconds) { /** * Perform a CPU intensive computation + * * @param iterations: number of times to perform computation * @return result of computation */ private double compute(int iterations) { LOGGER.log(Level.INFO, Thread.currentThread() + ": Computing with " + iterations + " iterations"); double d = 123456789.123456789 * rand.nextInt(100); - for (int i=0; i < iterations; i++) { - for (int n=0; n < 1_000_000; n++) { + for (int i = 0; i < iterations; i++) { + for (int n = 0; n < 1_000_000; n++) { for (int j = 0; j < 5; j++) { d = Math.tan(d); d = Math.atan(d); From 0b5e1e406de283979e52e3988d85588365737ef0 Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Wed, 27 Mar 2024 15:44:45 -0700 Subject: [PATCH 7/9] Minor update to README --- examples/webserver/threads/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/webserver/threads/README.md b/examples/webserver/threads/README.md index 6b301484238..7c3b5355c93 100644 --- a/examples/webserver/threads/README.md +++ b/examples/webserver/threads/README.md @@ -6,7 +6,7 @@ application specific executors is desirable. This example illustrates two such cases: 1. Using a virtual thread executor to execute multiple tasks in parallel. -2. Using a platform thread executor to execute long-running CPU intensive operations using platform threads. +2. Using a platform thread executor to execute long-running CPU intensive operations. ## Build and run @@ -22,7 +22,7 @@ __Compute:__ curl -X GET http://localhost:8080/thread/compute/5 ``` The `compute` endpoint runs a costly floating point computation using a platform thread. -Increases the number to make the computation more costly (and take longer). +Increase the number to make the computation more costly (and take longer). The request returns the results of the computation (not important!). @@ -31,8 +31,8 @@ __Fanout:__ curl -X GET http://localhost:8080/thread/fanout/5 ``` The `fanout` endpoint simulates a fanout of remote calls that are run in parallel using -virtual threads. Each call invokes the server's `sleep` endpoint sleeping anywhere from -0 to 4 seconds. Since the remote requests are executed in parallel the request should not +virtual threads. Each remote call invokes the server's `sleep` endpoint sleeping anywhere from +0 to 4 seconds. Since the remote requests are executed in parallel the curl request should not take longer than 4 seconds to return. Increase the number to have more remote calls made in parallel. @@ -77,7 +77,7 @@ of platform threads to execute the task. ### Use of Helidon's ThreadPoolSupplier and Configuration This example uses `io.helidon.common.configurable.ThreadPoolSupplier` to create the -two executors used in the example. This provides a couple of benefits: +two executors used in the example. This provides a few benefits: 1. ThreadPoolSupplier supports a number of tuning parameters that enable us to configure a small, bounded threadpool. 2. You can drive the thread pool configuration via Helidon config -- see this example's `application.yaml` From 42ebc8bb4c0576674204cba9a72bc818885859b8 Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Thu, 28 Mar 2024 08:06:01 -0700 Subject: [PATCH 8/9] Remove empty file --- .../webserver/threads/src/test/resources/application-test.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 examples/webserver/threads/src/test/resources/application-test.yaml diff --git a/examples/webserver/threads/src/test/resources/application-test.yaml b/examples/webserver/threads/src/test/resources/application-test.yaml deleted file mode 100644 index e69de29bb2d..00000000000 From b4d9f3ead113d885a14402007697526881f3fc03 Mon Sep 17 00:00:00 2001 From: Joe Di Pol Date: Mon, 1 Apr 2024 10:05:06 -0700 Subject: [PATCH 9/9] Rename XxxThreadPoolSupplier to XxxThreadSupplier --- .../examples/webserver/threads/ThreadService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java index b02fb8e8ed1..60ece4b13b3 100644 --- a/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java +++ b/examples/webserver/threads/src/main/java/io/helidon/examples/webserver/threads/ThreadService.java @@ -58,15 +58,15 @@ class ThreadService implements HttpService { * The second is a virtual thread executor service. * See `application.yaml` for configuration of each of these. */ - ThreadPoolSupplier platformThreadPoolSupplier = ThreadPoolSupplier.builder() + ThreadPoolSupplier platformThreadSupplier = ThreadPoolSupplier.builder() .config(appConfig.get("application-platform-executor")) .build(); - platformExecutorService = platformThreadPoolSupplier.get(); + platformExecutorService = platformThreadSupplier.get(); - ThreadPoolSupplier virtualThreadPoolSupplier = ThreadPoolSupplier.builder() + ThreadPoolSupplier virtualThreadSupplier = ThreadPoolSupplier.builder() .config(appConfig.get("application-virtual-executor")) .build(); - virtualExecutorService = virtualThreadPoolSupplier.get(); + virtualExecutorService = virtualThreadSupplier.get(); } @Override @@ -91,8 +91,8 @@ public void routing(HttpRules rules) { private void computeHandler(ServerRequest request, ServerResponse response) { int iterations = request.path().pathParameters().first("iterations").asInt().orElse(1); try { - // We execute the computation on a platform thread. This prevents pining of the virtual - // thread, plus provides us the ability to limit the number of concurrent computation requests + // We execute the computation on a platform thread. This prevents unbounded obstruction of virtual + // threads, plus provides us the ability to limit the number of concurrent computation requests // we handle by limiting the thread pool work queue length (as defined in application.yaml) Future future = platformExecutorService.submit(() -> compute(iterations)); response.send(future.get().toString());