Issue #5019 - hot-reload SSL certificates if keystore file changed

jetty-documentation/src/main/asciidoc/configuring/connectors/configuring-ssl.adoc

@@ -989,3 +989,15 @@ As a reminder, when configuring your includes/excludes, *excludes always win*.
 
 Dumps can be configured as part of the `jetty.xml` configuration for your server.
 Please see the documentation on the link:#jetty-dump-tool[Jetty Dump Tool] for more information.
+
+
+==== SslContextFactory KeyStore Reload
+
+Jetty can be configured to monitor the directory of the KeyStore file specified in the SslContextFactory, and reload the
+SslContextFactory if any changes are detected to the KeyStore file.
+
+If changes need to be done to other file such as the TrustStore file, this must be done before the change to the Keystore
+file which will then trigger the `SslContextFactory` reload.
+
+With the Jetty distribution this feature can be used by simply activating the `ssl-reload` startup module.
+For embedded usage the `KeyStoreScanner` should be created given the `SslContextFactory` and added as a bean on the Server.

jetty-server/src/main/config/etc/jetty-ssl-context-reload.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0"?><!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_9_3.dtd">
+
+<Configure id="Server" class="org.eclipse.jetty.server.Server">
+  <Call name="addBean">
+    <Arg>
+      <New id="keyStoreScanner" class="org.eclipse.jetty.util.ssl.KeyStoreScanner">
+        <Arg><Ref refid="sslContextFactory"/></Arg>
+        <Set name="scanInterval"><Property name="jetty.sslContext.reload.scanInterval" default="1"/></Set>
+      </New>
+    </Arg>
+  </Call>
+</Configure>

jetty-server/src/main/config/modules/ssl-reload.mod

@@ -0,0 +1,18 @@
+# DO NOT EDIT - See: https://www.eclipse.org/jetty/documentation/current/startup-modules.html
+
+[description]
+Enables the SSL keystore to be reloaded after any changes are detected on the file system.
+
+[tags]
+connector
+ssl
+
+[depend]
+ssl
+
+[xml]
+etc/jetty-ssl-context-reload.xml
+
+[ini-template]
+# Monitored directory scan period (seconds)
+# jetty.sslContext.reload.scanInterval=1

jetty-util/src/main/java/org/eclipse/jetty/util/ssl/KeyStoreScanner.java

@@ -0,0 +1,133 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.util.ssl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.function.Consumer;
+
+import org.eclipse.jetty.util.Scanner;
+import org.eclipse.jetty.util.annotation.ManagedAttribute;
+import org.eclipse.jetty.util.annotation.ManagedOperation;
+import org.eclipse.jetty.util.component.ContainerLifeCycle;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+
+/**
+ * <p>The {@link KeyStoreScanner} is used to monitor the KeyStore file used by the {@link SslContextFactory}.
+ * It will reload the {@link SslContextFactory} if it detects that the KeyStore file has been modified.</p>
+ * <p>If the TrustStore file needs to be changed, then this should be done before touching the KeyStore file,
+ * the {@link SslContextFactory#reload(Consumer)} will only occur after the KeyStore file has been modified.</p>
+ */
+public class KeyStoreScanner extends ContainerLifeCycle implements Scanner.DiscreteListener
+{
+    private static final Logger LOG = Log.getLogger(KeyStoreScanner.class);
+
+    private final SslContextFactory sslContextFactory;
+    private final File keystoreFile;
+    private final Scanner _scanner;
+
+    public KeyStoreScanner(SslContextFactory sslContextFactory)
+    {
+        this.sslContextFactory = sslContextFactory;
+        try
+        {
+            keystoreFile = sslContextFactory.getKeyStoreResource().getFile();
+            if (keystoreFile == null || !keystoreFile.exists())
+                throw new IllegalArgumentException("keystore file does not exist");
+            if (keystoreFile.isDirectory())
+                throw new IllegalArgumentException("expected keystore file not directory");
+        }
+        catch (IOException e)
+        {
+            throw new IllegalArgumentException("could not obtain keystore file", e);
+        }
+
+        File parentFile = keystoreFile.getParentFile();
+        if (!parentFile.exists() || !parentFile.isDirectory())
+            throw new IllegalArgumentException("error obtaining keystore dir");
+
+        _scanner = new Scanner();
+        _scanner.setScanDirs(Collections.singletonList(parentFile));
+        _scanner.setScanInterval(1);
+        _scanner.setReportDirs(false);
+        _scanner.setReportExistingFilesOnStartup(false);
+        _scanner.setScanDepth(1);
+        _scanner.addListener(this);
+        addBean(_scanner);
+    }
+
+    @Override
+    public void fileAdded(String filename)
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("added {}", filename);
+
+        if (keystoreFile.toPath().toString().equals(filename))
+            reload();
+    }
+
+    @Override
+    public void fileChanged(String filename)
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("changed {}", filename);
+
+        if (keystoreFile.toPath().toString().equals(filename))
+            reload();
+    }
+
+    @Override
+    public void fileRemoved(String filename)
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("removed {}", filename);
+
+        if (keystoreFile.toPath().toString().equals(filename))
+            reload();
+    }
+
+    @ManagedOperation(value = "Reload the SSL Keystore", impact = "ACTION")
+    public void reload()
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("reloading keystore file {}", keystoreFile);
+
+        try
+        {
+            sslContextFactory.reload(scf -> {});
+        }
+        catch (Throwable t)
+        {
+            LOG.warn("Keystore Reload Failed", t);
+        }
+    }
+
+    @ManagedAttribute("scanning interval to detect changes which need reloaded")
+    public int getScanInterval()
+    {
+        return _scanner.getScanInterval();
+    }
+
+    public void setScanInterval(int scanInterval)
+    {
+        _scanner.setScanInterval(scanInterval);
+    }
+}

jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java

@@ -1131,6 +1131,9 @@ public SSLContext getSslContext()
 
         synchronized (this)
         {
+            if (_factory == null)
+                throw new IllegalStateException("SslContextFactory reload failed");
+
             return _factory._context;
         }
     }
@@ -1532,6 +1535,9 @@ public KeyStore getKeyStore()
 
         synchronized (this)
         {
+            if (_factory == null)
+                throw new IllegalStateException("SslContextFactory reload failed");
+
             return _factory._keyStore;
         }
     }
@@ -1553,6 +1559,9 @@ public KeyStore getTrustStore()
 
         synchronized (this)
         {
+            if (_factory == null)
+                throw new IllegalStateException("SslContextFactory reload failed");
+
             return _factory._trustStore;
         }
     }