4.x: Media Context and streaming (#7396)

* Improved error messages in Http token validation
* Fix intermittent failure when the server took time to shut down
* Media writers now correctly handle streaming where needed, without buffering in memory.
* Configurable max-in-memory-entity for server

Signed-off-by: Tomas Langer <tomas.langer@oracle.com>

examples/webserver/streaming/src/main/java/io/helidon/examples/webserver/streaming/StreamingService.java

@@ -22,6 +22,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.Objects;
 import java.util.logging.Logger;
 
@@ -57,7 +58,8 @@ private void upload(ServerRequest request, ServerResponse response) {
         LOGGER.info("Entering upload ... " + Thread.currentThread());
         try {
             Path tempFilePath = Files.createTempFile("large-file", ".tmp");
-            Files.copy(request.content().inputStream(), tempFilePath);
+            Files.copy(request.content().inputStream(), tempFilePath, StandardCopyOption.REPLACE_EXISTING);
+            response.send("File was stored as " + tempFilePath);
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }

examples/webserver/tutorial/src/test/java/io/helidon/examples/webserver/tutorial/MainTest.java

@@ -17,12 +17,12 @@
 package io.helidon.examples.webserver.tutorial;
 
 import io.helidon.http.Http;
-import io.helidon.webserver.testing.junit5.ServerTest;
-import io.helidon.webserver.testing.junit5.SetUpServer;
 import io.helidon.webclient.http1.Http1Client;
 import io.helidon.webclient.http1.Http1ClientResponse;
 import io.helidon.webserver.WebServer;
 import io.helidon.webserver.WebServerConfig;
+import io.helidon.webserver.testing.junit5.ServerTest;
+import io.helidon.webserver.testing.junit5.SetUpServer;
 
 import org.junit.jupiter.api.Test;
 
@@ -50,10 +50,19 @@ static void setup(WebServerConfig.Builder server) {
     }
 
     @Test
-    public void testShutDown() {
+    public void testShutDown() throws InterruptedException {
         try (Http1ClientResponse response = client.post("/mgmt/shutdown").request()) {
             assertThat(response.status(), is(Http.Status.OK_200));
-            assertThat(server.isRunning(), is(false));
         }
+        // there may be some delay between the request being completed, and the server shutting down
+        // let's give it a second to shut down, then fail
+        for (int i = 0; i < 10; i++) {
+            if (server.isRunning()) {
+                Thread.sleep(100);
+            } else {
+                break;
+            }
+        }
+        assertThat(server.isRunning(), is(false));
     }
 }

http/http/src/main/java/io/helidon/http/HttpToken.java

@@ -16,6 +16,10 @@
 
 package io.helidon.http;
 
+import java.nio.charset.StandardCharsets;
+
+import io.helidon.common.buffers.BufferData;
+
 /**
  * HTTP Token utility.
  * Token is defined by the HTTP specification and must not contain a set of characters.
@@ -32,25 +36,46 @@ private HttpToken() {
      */
     public static void validate(String token) throws IllegalArgumentException {
         char[] chars = token.toCharArray();
-        for (char aChar : chars) {
+        for (int i = 0; i < chars.length; i++) {
+            char aChar = chars[i];
             if (aChar > 254) {
-                throw new IllegalArgumentException("Token contains non-ASCII character");
+                throw new IllegalArgumentException("Token contains non-ASCII character at position "
+                                                           + hex(i)
+                                                           + " \n"
+                                                           + debugToken(token));
             }
             if (Character.isISOControl(aChar)) {
-                throw new IllegalArgumentException("Token contains control character");
+                throw new IllegalArgumentException("Token contains control character at position "
+                                                           + hex(i)
+                                                           + "\n"
+                                                           + debugToken(token));
             }
             if (Character.isWhitespace(aChar)) {
-                throw new IllegalArgumentException("Token contains whitespace character");
+                throw new IllegalArgumentException("Token contains whitespace character at position "
+                                                           + hex(i)
+                                                           + "\n"
+                                                           + debugToken(token));
             }
             switch (aChar) {
             case '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}' -> {
-                throw new IllegalArgumentException(
-                        "Token contains illegal character: " + aChar);
+                throw new IllegalArgumentException("Token contains illegal character at position "
+                                                           + hex(i)
+                                                           + "\n"
+                                                           + debugToken(token));
             }
             default -> {
                 // this is a valid character
             }
             }
         }
     }
+
+    private static String hex(int i) {
+        return Integer.toHexString(i);
+    }
+
+    private static String debugToken(String token) {
+        return BufferData.create(token.getBytes(StandardCharsets.US_ASCII))
+                .debugDataHex();
+    }
 }

http/media/media/src/main/java/io/helidon/http/media/EntityWriter.java

@@ -28,6 +28,46 @@
  * @param <T> type of entity
  */
 public interface EntityWriter<T> {
+    /**
+     * Whether this entity writer can provide more information about each entity instance, such as content length.
+     *
+     * @return whether {@code instanceWriter} methods are supported;
+     *         If not one of the {@code write} methods would be called instead
+     */
+    default boolean supportsInstanceWriter() {
+        return false;
+    }
+
+    /**
+     * Client request entity instance writer.
+     *
+     * @param type            type of entity
+     * @param object          object to write
+     * @param requestHeaders  request headers
+     * @return instance writer ready to write the provided entity
+     */
+    default InstanceWriter instanceWriter(GenericType<T> type,
+                                          T object,
+                                          WritableHeaders<?> requestHeaders) {
+        throw new UnsupportedOperationException("This writer does not support instance writers: " + getClass().getName());
+    }
+
+    /**
+     * Server response entity instance writer.
+     *
+     * @param type            type of entity
+     * @param object          object to write
+     * @param requestHeaders  request headers
+     * @param responseHeaders response headers
+     * @return instance writer ready to write the provided entity
+     */
+    default InstanceWriter instanceWriter(GenericType<T> type,
+                                          T object,
+                                          Headers requestHeaders,
+                                          WritableHeaders<?> responseHeaders) {
+        throw new UnsupportedOperationException("This writer does not support instance writers: " + getClass().getName());
+    }
+
     /**
      * Write server response entity and close the stream.
      *

http/media/media/src/main/java/io/helidon/http/media/InstanceWriter.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2023 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.helidon.http.media;
+
+import java.io.OutputStream;
+import java.util.OptionalLong;
+
+/**
+ * A writer dedicated to a specific instance. Method {@link #write(java.io.OutputStream)} will write the instance
+ * to the output stream, method {@link #instanceBytes()} will provide all the bytes of entity.
+ * The caller decided which method to call depending on results of {@link #alwaysInMemory()} and {@link #contentLength()}.
+ */
+public interface InstanceWriter {
+    /**
+     * If we can determine the number of bytes to be written to the stream, provide the information here.
+     * The returned number must be a valid content length (content-length >= 0)
+     *
+     * @return number of bytes or empty if not possible (or too expensive) to find out
+     */
+    OptionalLong contentLength();
+
+    /**
+     * Whether the byte array is always available. If true {@link #instanceBytes()} will ALWAYS be called.
+     *
+     * @return whether the bytes will always be materialized in memory
+     */
+    boolean alwaysInMemory();
+
+    /**
+     * Write the instance to the output stream. This method is NEVER called if {@link #alwaysInMemory()} is {@code true},
+     * otherwise this method is ALWAYS called if {@link #contentLength()} returns empty.
+     * This method MAY be called if {@link #contentLength()} returns a value.
+     *
+     * @param stream to write to
+     */
+    void write(OutputStream stream);
+
+    /**
+     * Get the instance as byte array. This method is always called if {@link #alwaysInMemory()} returns {@code true}.
+     * This method is NEVER called if {@link #contentLength()} returns empty.
+     * This method MAY be called if {@link #contentLength()} returns a value.
+     *
+     * @return bytes of the instance
+     */
+    byte[] instanceBytes();
+}

http/media/media/src/main/java/io/helidon/http/media/MediaContextImpl.java

@@ -175,7 +175,7 @@ public void write(GenericType type,
                 LOGGER.log(System.Logger.Level.WARNING, "There is no media writer configured for " + type);
             }
 
-            throw new IllegalArgumentException("No server response media writer for " + type + " configured");
+            throw new UnsupportedTypeException("No server response media writer for " + type + " configured");
         }
 
         @Override
@@ -188,7 +188,7 @@ public void write(GenericType type,
                 LOGGER.log(System.Logger.Level.WARNING, "There is no media writer configured for " + type);
             }
 
-            throw new IllegalArgumentException("No client request media writer for " + type + " configured");
+            throw new UnsupportedTypeException("No client request media writer for " + type + " configured");
         }
     }
 
@@ -205,7 +205,7 @@ public Object read(GenericType type, InputStream stream, Headers headers) {
             if (LOGGED_READERS.computeIfAbsent(type, it -> new AtomicBoolean()).compareAndSet(false, true)) {
                 LOGGER.log(System.Logger.Level.WARNING, "There is no media reader configured for " + type);
             }
-            throw new IllegalArgumentException("No server request media support for " + type + " configured");
+            throw new UnsupportedTypeException("No server request media support for " + type + " configured");
         }
 
         @Override
@@ -216,7 +216,7 @@ public Object read(GenericType type,
             if (LOGGED_READERS.computeIfAbsent(type, it -> new AtomicBoolean()).compareAndSet(false, true)) {
                 LOGGER.log(System.Logger.Level.WARNING, "There is no media reader configured for " + type);
             }
-            throw new IllegalArgumentException("No client response media support for " + type + " configured");
+            throw new UnsupportedTypeException("No client response media support for " + type + " configured");
         }
     }
 
@@ -258,6 +258,24 @@ private static final class CloseStreamWriter implements EntityWriter {
             this.delegate = delegate;
         }
 
+        @Override
+        public boolean supportsInstanceWriter() {
+            return delegate.supportsInstanceWriter();
+        }
+
+        @Override
+        public InstanceWriter instanceWriter(GenericType type, Object object, WritableHeaders requestHeaders) {
+            return delegate.instanceWriter(type, object, requestHeaders);
+        }
+
+        @Override
+        public InstanceWriter instanceWriter(GenericType type,
+                                             Object object,
+                                             Headers requestHeaders,
+                                             WritableHeaders responseHeaders) {
+            return delegate.instanceWriter(type, object, requestHeaders, responseHeaders);
+        }
+
         @Override
         public void write(GenericType type,
                           Object object,