Support for mutable file based MP config sources. (#3666) (#3730)

Signed-off-by: Tomas Langer <tomas.langer@oracle.com>
(cherry picked from commit 5ee165b)

config/config-mp/src/main/java/io/helidon/config/mp/MpConfigSources.java

@@ -22,6 +22,7 @@
 import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -34,6 +35,7 @@
 
 import io.helidon.config.Config;
 import io.helidon.config.ConfigException;
+import io.helidon.config.MutabilitySupport;
 
 import org.eclipse.microprofile.config.spi.ConfigSource;
 
@@ -158,6 +160,19 @@ public static ConfigSource create(String name, Path path) {
             throw new ConfigException("Failed to read properties from " + path.toAbsolutePath());
         }
 
+        if ("true".equals(props.getProperty("helidon.config.polling.enabled"))) {
+            String durationString = props.getProperty("helidon.config.polling.duration");
+            Duration duration;
+            if (durationString == null) {
+                duration = Duration.ofSeconds(10);
+            } else {
+                duration = Duration.parse(durationString);
+            }
+            MutabilitySupport.poll(path, duration, changed -> update(path, props), changed -> props.clear());
+        } else if ("true".equals(props.getProperty("helidon.config.watcher.enabled"))) {
+            MutabilitySupport.watch(path, changed -> update(path, props), changed -> props.clear());
+        }
+
         return create(name, props);
     }
 
@@ -184,12 +199,10 @@ public static ConfigSource create(Properties properties) {
      * @param properties serving as configuration data
      * @return a new config source
      */
+    @SuppressWarnings("rawtypes")
     public static ConfigSource create(String name, Properties properties) {
-        Map<String, String> result = new HashMap<>();
-        for (String key : properties.stringPropertyNames()) {
-            result.put(key, properties.getProperty(key));
-        }
-        return new MpMapSource(name, result);
+        Map map = properties;
+        return new MpMapSource(name, map);
     }
 
     /**
@@ -411,4 +424,17 @@ private static String toProfileResource(String resource, String profile) {
         }
         return resource + "-" + profile;
     }
+
+    private static void update(Path path, Properties originalProperties) {
+        Properties props = new Properties();
+
+        try (InputStream in = Files.newInputStream(path)) {
+            props.load(in);
+        } catch (IOException e) {
+            throw new ConfigException("Failed to read properties from " + path.toAbsolutePath());
+        }
+
+        originalProperties.keySet().removeIf(it -> !props.containsKey(it));
+        originalProperties.putAll(props);
+    }
 }

config/config/src/main/java/io/helidon/config/MutabilitySupport.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2021 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.helidon.config;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.function.Consumer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import io.helidon.common.LazyValue;
+import io.helidon.common.NativeImageHelper;
+import io.helidon.config.spi.ChangeEventType;
+import io.helidon.config.spi.PollingStrategy;
+
+/**
+ * Mutability support for file based sources.
+ * <p>
+ * Provides support for polling based strategy
+ * ({@link #poll(java.nio.file.Path, java.time.Duration, java.util.function.Consumer, java.util.function.Consumer)}) and
+ * for file watching ({@link #watch(java.nio.file.Path, java.util.function.Consumer, java.util.function.Consumer)}).
+ */
+public final class MutabilitySupport {
+    private static final Logger LOGGER = Logger.getLogger(MutabilitySupport.class.getName());
+    private static final LazyValue<ScheduledExecutorService> EXECUTOR
+            = LazyValue.create(Executors::newSingleThreadScheduledExecutor);
+
+    private MutabilitySupport() {
+    }
+
+    /**
+     * Start polling for changes.
+     *
+     * @param path path to watch
+     * @param duration duration of polling
+     * @param updater consumer that reads the file content and updates properties (in case file is changed)
+     * @param cleaner runnable to clean the properties (in case file is deleted)
+     * @return runnable to stop the file watcher
+     */
+    public static Runnable poll(Path path, Duration duration, Consumer<Path> updater, Consumer<Path> cleaner) {
+        if (NativeImageHelper.isBuildTime()) {
+            LOGGER.info("File polling is not enabled in native image build time. Path: " + path);
+        }
+
+        PollingStrategy strategy = PollingStrategies.regular(duration)
+                .executor(EXECUTOR.get())
+                .build();
+
+        strategy.start(new PathPolled(path, updater, cleaner));
+        return strategy::stop;
+    }
+
+    /**
+     * Start watching a file for changes.
+     *
+     * @param path path to watch
+     * @param updater consumer that reads the file content and updates properties
+     * @param cleaner runnable to clean the properties (in case file is deleted)
+     * @return runnable to stop the file watcher
+     */
+    public static Runnable watch(Path path, Consumer<Path> updater, Consumer<Path> cleaner) {
+        if (NativeImageHelper.isBuildTime()) {
+            LOGGER.info("File watching is not enabled in native image build time. Path: " + path);
+        }
+        FileSystemWatcher watcher = FileSystemWatcher.builder()
+                .executor(EXECUTOR.get())
+                .build();
+
+        watcher.start(path, event -> {
+            try {
+                if (event.type() == ChangeEventType.DELETED) {
+                    cleaner.accept(event.target());
+                } else {
+                    updater.accept(event.target());
+                }
+            } catch (Exception e) {
+                LOGGER.log(Level.WARNING, "Failed to process change watcher event " + event
+                        + " for file " + path.toAbsolutePath(), e);
+            }
+        });
+        return watcher::stop;
+    }
+
+    private static class PathPolled implements PollingStrategy.Polled {
+        private final Path path;
+        private final Consumer<Path> updater;
+        private final Consumer<Path> cleaner;
+
+        private boolean exists;
+        private Instant lastChange;
+
+        private PathPolled(Path path,
+                           Consumer<Path> updater,
+                           Consumer<Path> cleaner) {
+
+            this.path = path;
+            this.updater = updater;
+            this.cleaner = cleaner;
+            this.exists = Files.exists(path);
+            if (exists) {
+                try {
+                    this.lastChange = Files.getLastModifiedTime(path).toInstant();
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        }
+
+        @Override
+        public ChangeEventType poll(Instant when) {
+            try {
+                return doPoll();
+            } catch (Exception e) {
+                LOGGER.log(Level.WARNING, "Failed to poll for changes at " + when, e);
+                return ChangeEventType.CHANGED;
+            }
+        }
+
+        private ChangeEventType doPoll() {
+            if (Files.exists(path)) {
+                ChangeEventType response;
+                if (exists) {
+                    // existed and exists now, let's see if modified
+                    Instant instant = Instant.now();
+                    try {
+                        instant = Files.getLastModifiedTime(path).toInstant();
+                    } catch (IOException e) {
+                        LOGGER.log(Level.WARNING, "Failed to get last modified for " + path.toAbsolutePath(), e);
+                    }
+                    if (instant.isAfter(this.lastChange)) {
+                        this.lastChange = instant;
+                        response = ChangeEventType.CHANGED;
+                        updater.accept(path);
+                    } else {
+                        response = ChangeEventType.UNCHANGED;
+                    }
+                } else {
+                    response = ChangeEventType.CREATED;
+                    updater.accept(path);
+                }
+                exists = true;
+                return response;
+            } else {
+                ChangeEventType response;
+                if (exists) {
+                    response = ChangeEventType.DELETED;
+                    cleaner.accept(path);
+                } else {
+                    response = ChangeEventType.UNCHANGED;
+                }
+                exists = false;
+                return response;
+            }
+        }
+    }
+}

config/yaml-mp/src/main/java/io/helidon/config/yaml/mp/YamlMpConfigSource.java

@@ -16,13 +16,15 @@
 
 package io.helidon.config.yaml.mp;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashMap;
@@ -34,6 +36,7 @@
 import java.util.Set;
 
 import io.helidon.config.ConfigException;
+import io.helidon.config.MutabilitySupport;
 
 import org.eclipse.microprofile.config.spi.ConfigSource;
 import org.yaml.snakeyaml.Yaml;
@@ -92,13 +95,47 @@ private YamlMpConfigSource(String name, Map<String, String> properties) {
      * @see #create(java.net.URL)
      */
     public static ConfigSource create(Path path) {
-        try {
-            return create(path.toUri().toURL());
-        } catch (MalformedURLException e) {
+        String name = path.toAbsolutePath().toString();
+
+        try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
+            Map yamlMap = toMap(reader);
+            // this is a mutable HashMap that we can use
+            Map<String, String> props = fromMap(yamlMap == null ? Map.of() : yamlMap);
+
+            if ("true".equals(props.get("helidon.config.polling.enabled"))) {
+                String durationString = props.get("helidon.config.polling.duration");
+                Duration duration;
+                if (durationString == null) {
+                    duration = Duration.ofSeconds(10);
+                } else {
+                    duration = Duration.parse(durationString);
+                }
+                MutabilitySupport.poll(path, duration, changed -> update(path, props), changed -> props.clear());
+            } else if ("true".equals(props.get("helidon.config.watcher.enabled"))) {
+                MutabilitySupport.watch(path, changed -> update(path, props), changed -> props.clear());
+            }
+
+
+            return new YamlMpConfigSource(name, props);
+        } catch (IOException e) {
             throw new ConfigException("Failed to load YAML config source from path: " + path.toAbsolutePath(), e);
         }
     }
 
+    private static void update(Path path, Map<String, String> originalProps) {
+        try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
+            Map yamlMap = toMap(reader);
+            // this is a mutable HashMap that we can use
+            Map<String, String> props = fromMap(yamlMap == null ? Map.of() : yamlMap);
+
+            // first delete those that no longer exist
+            originalProps.keySet().removeIf(it -> !props.containsKey(it));
+            originalProps.putAll(props);
+        } catch (IOException e) {
+            throw new ConfigException("Failed to load updated YAML config source from path: " + path.toAbsolutePath(), e);
+        }
+    }
+
     /**
      * Load a YAML config source from URL.
      * The URL may be any URL which is support by the used JVM.

docs/mp/config/01_introduction.adoc

@@ -117,6 +117,25 @@ service-1: "${uri}/service1"
 service-2: "${uri}/service2"
 ----
 
+* *Change support* +
+Polling (or change watching) for file based config sources (not classpath based).
+
+To enable polling for a config source created using meta configuration (see below), or using
+`MpConfigSources.create(Path)`, or `YamlMpConfigSource.create(Path)`, use the following properties:
+
+[cols="3,5"]
+|===
+|Property |Description
+
+|`helidon.config.polling.enabled`   |To enable polling file for changes, uses timestamp to identify a change.
+
+|`helidon.config.polling.duration`   |Polling period duration, defaults to 10 seconds ('PT10S`) +
+See https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/time/Duration.html#parse(java.lang.CharSequence)
+
+|`helidon.config.watcher.enabled`  |To enable watching file for changes using the Java `WatchService`. +
+See https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/WatchService.html
+
+|===
 
 * *Encryption* +
 You can encrypt secrets using a master password and store them in a configuration file.