From 5448e0217b613b5f9f610494f497916657018411 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Fri, 29 Mar 2024 12:42:13 -0400 Subject: [PATCH] Support an idiomatic environment source Resolves https://github.com/sksamuel/hoplite/issues/410. --- .../hoplite/ConfigLoaderBuilderExtensions.kt | 22 ++- .../com/sksamuel/hoplite/PropertySource.kt | 27 +++- .../EnvironmentVariablesPropertySource.kt | 2 + .../hoplite/CascadingNormalizationTest.kt | 1 + .../EnvironmentVariablesPropertySourceTest.kt | 43 +++++- .../hoplite/transformer/PathNormalizerTest.kt | 1 + .../hoplite/json/AmbiguousPropertyNameTest.kt | 59 +++++++++ .../EnvPropertySourceNormalizationTest.kt | 125 ++++++++++++++++++ ...tySourceSingleUnderscoreAsSeparatorTest.kt | 35 +++++ .../json/EnvPropertySourceUppercaseTest.kt | 4 + 10 files changed, 313 insertions(+), 6 deletions(-) create mode 100644 hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/AmbiguousPropertyNameTest.kt create mode 100644 hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceNormalizationTest.kt create mode 100644 hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceSingleUnderscoreAsSeparatorTest.kt diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilderExtensions.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilderExtensions.kt index 396bee9f..787e4da8 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilderExtensions.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilderExtensions.kt @@ -124,12 +124,32 @@ fun ConfigLoaderBuilder.addCommandLineSource( * * @param useUnderscoresAsSeparator if true, use double underscore instead of period to separate keys in nested config * @param allowUppercaseNames if true, allow uppercase-only names + * @param useSingleUnderscoresAsSeparator if true, allows single underscores as separators, to conform with + * idiomatic environment variable names */ fun ConfigLoaderBuilder.addEnvironmentSource( useUnderscoresAsSeparator: Boolean = true, allowUppercaseNames: Boolean = true, + useSingleUnderscoresAsSeparator: Boolean = false, ) = addPropertySource( - EnvironmentVariablesPropertySource(useUnderscoresAsSeparator, allowUppercaseNames) + EnvironmentVariablesPropertySource(useUnderscoresAsSeparator, useSingleUnderscoresAsSeparator, allowUppercaseNames) +) + +/** + * Adds a [PropertySource] that will read the environment settings. + * + * With this source, environment variables are expected to be idiomatic i.e. uppercase, with underscores as + * separators for path elements. Dashes are removed. + * + * Generally a [PathNormalizer] should be added to the [ConfigLoaderBuilder] to normalize paths when this source + * is used. + */ +fun ConfigLoaderBuilder.addIdiomaticEnvironmentSource() = addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + useSingleUnderscoresAsSeparator = true, + allowUppercaseNames = false + ) ) /** diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/PropertySource.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/PropertySource.kt index e6acf1f5..28a7c7f5 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/PropertySource.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/PropertySource.kt @@ -93,11 +93,32 @@ interface PropertySource { CommandLinePropertySource(arguments, prefix, delimiter) /** - * Returns a [PropertySource] that will read the environment settings. + * Returns a [PropertySource] that will read the environment settings, by default with the classic + * parsing mechanism using double-underscore as a path separator, and converting uppercase names with + * underscores to camel case. */ - fun environment(useUnderscoresAsSeparator: Boolean = true, allowUppercaseNames: Boolean = true) = - EnvironmentVariablesPropertySource(useUnderscoresAsSeparator, allowUppercaseNames) + fun environment( + useUnderscoresAsSeparator: Boolean = true, + allowUppercaseNames: Boolean = true, + useSingleUnderscoresAsSeparator: Boolean = false, + ) = + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator, + useSingleUnderscoresAsSeparator, + allowUppercaseNames + ) + /** + * Returns a [PropertySource] that will read the environment settings, supporting idiomatic environment + * names. Underscores are used as path separators, and "-" are removed/ignored. We recommend this be used + * along with a [PathNormalizer]. + */ + fun idiomaticEnvironment() = + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + useSingleUnderscoresAsSeparator = true, + allowUppercaseNames = false + ) /** * Returns a [PropertySource] that will read from the specified string. diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt index 2dad1ce1..f762581a 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt @@ -9,6 +9,7 @@ import com.sksamuel.hoplite.parsers.toNode class EnvironmentVariablesPropertySource( private val useUnderscoresAsSeparator: Boolean, + private val useSingleUnderscoresAsSeparator: Boolean, private val allowUppercaseNames: Boolean, private val environmentVariableMap: () -> Map = { System.getenv() }, private val prefix: String? = null, // optional prefix to strip from the vars @@ -24,6 +25,7 @@ class EnvironmentVariablesPropertySource( key .let { if (prefix == null) it else it.removePrefix(prefix) } .let { if (useUnderscoresAsSeparator) it.replace("__", ".") else it } + .let { if (useSingleUnderscoresAsSeparator) it.replace("_", ".") else it } .let { if (allowUppercaseNames && Character.isUpperCase(it.codePointAt(0))) { it.split(".").joinToString(separator = ".") { value -> diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/CascadingNormalizationTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/CascadingNormalizationTest.kt index 5a1fe66b..75a6503e 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/CascadingNormalizationTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/CascadingNormalizationTest.kt @@ -22,6 +22,7 @@ class CascadingNormalizationTest : FunSpec() { EnvironmentVariablesPropertySource( useUnderscoresAsSeparator = false, allowUppercaseNames = false, + useSingleUnderscoresAsSeparator = false, environmentVariableMap = { mapOf("section.subSection.someValue" to "3") } ) ) diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt index fd9bfb5d..30d30070 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt @@ -7,11 +7,43 @@ import io.kotest.matchers.shouldBe class EnvironmentVariablesPropertySourceTest : FunSpec({ - test("build env source should include paths") { + test("build env source with the classic env var mapping should include paths") { + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, + allowUppercaseNames = false, + environmentVariableMap = { mapOf("a" to "foo", "a.b" to "bar", "c" to "baz", "d__e" to "gaz") } + ).node( + PropertySourceContext.empty + ).getUnsafe() shouldBe MapNode( + mapOf( + "a" to MapNode( + value = StringNode("foo", Pos.env, DotPath("a"), sourceKey = "a"), + map = mapOf("b" to StringNode("bar", Pos.env, DotPath("a", "b"), sourceKey = "a.b")), + pos = Pos.SourcePos("env"), + path = DotPath("a"), + sourceKey = "a" + ), + "c" to StringNode("baz", Pos.env, DotPath("c"), sourceKey = "c"), + "d" to MapNode( + value = Undefined, + map = mapOf("e" to StringNode("gaz", Pos.env, DotPath("d", "e"), sourceKey = "d__e")), + pos = Pos.SourcePos("env"), + path = DotPath("d"), + sourceKey = null + ), + ), + pos = Pos.env, + DotPath.root + ) + } + + test("build env source with the idiomatic env var mapping should include paths") { EnvironmentVariablesPropertySource( useUnderscoresAsSeparator = false, + useSingleUnderscoresAsSeparator = true, allowUppercaseNames = false, - environmentVariableMap = { mapOf("a" to "foo", "a.b" to "bar", "c" to "baz") } + environmentVariableMap = { mapOf("a" to "foo", "a.b" to "bar", "c" to "baz", "d_e" to "gaz") } ).node( PropertySourceContext.empty ).getUnsafe() shouldBe MapNode( @@ -24,6 +56,13 @@ class EnvironmentVariablesPropertySourceTest : FunSpec({ sourceKey = "a" ), "c" to StringNode("baz", Pos.env, DotPath("c"), sourceKey = "c"), + "d" to MapNode( + value = Undefined, + map = mapOf("e" to StringNode("gaz", Pos.env, DotPath("d", "e"), sourceKey = "d_e")), + pos = Pos.SourcePos("env"), + path = DotPath("d"), + sourceKey = null + ), ), pos = Pos.env, DotPath.root diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt index fc2c10cf..59b8f03c 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt @@ -14,6 +14,7 @@ class PathNormalizerTest : FunSpec({ val node = EnvironmentVariablesPropertySource( useUnderscoresAsSeparator = false, allowUppercaseNames = false, + useSingleUnderscoresAsSeparator = false, environmentVariableMap = { mapOf("A" to "a", "A.B" to "ab", "A.B.CD" to "abcd") }, ).node(PropertySourceContext.empty).getUnsafe() diff --git a/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/AmbiguousPropertyNameTest.kt b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/AmbiguousPropertyNameTest.kt new file mode 100644 index 00000000..db878854 --- /dev/null +++ b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/AmbiguousPropertyNameTest.kt @@ -0,0 +1,59 @@ +package com.sksamuel.hoplite.json + +import com.sksamuel.hoplite.ConfigLoader +import com.sksamuel.hoplite.sources.EnvironmentVariablesPropertySource +import com.sksamuel.hoplite.transformer.PathNormalizer +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.shouldBe + +class AmbiguousPropertyNameTest : DescribeSpec({ + + data class Ambiguous(val someCamelSetting: String, val somecamelsetting: String) + + describe("loading property differing in case from envs") { + it("with path normalizer cannot disambiguate") { + run { + ConfigLoader { + withReport() + addNodeTransformer(PathNormalizer) + addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, + allowUppercaseNames = false, + environmentVariableMap = { + mapOf( + "someCamelSetting" to "a", + "somecamelsetting" to "b", + "SOMECAMELSETTING" to "c", + ) + } + ) + ) + }.loadConfigOrThrow() + } shouldBe Ambiguous("c", "c") + } + + it("without path normalizer") { + run { + ConfigLoader { + withReport() + addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, + allowUppercaseNames = false, + environmentVariableMap = { + mapOf( + "someCamelSetting" to "a", + "somecamelsetting" to "b", + "SOMECAMELSETTING" to "c", + ) + } + ) + ) + }.loadConfigOrThrow() + } shouldBe Ambiguous("a", "b") + } + } +}) diff --git a/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceNormalizationTest.kt b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceNormalizationTest.kt new file mode 100644 index 00000000..50f453f2 --- /dev/null +++ b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceNormalizationTest.kt @@ -0,0 +1,125 @@ +package com.sksamuel.hoplite.json + +import com.sksamuel.hoplite.ConfigLoader +import com.sksamuel.hoplite.sources.EnvironmentVariablesPropertySource +import com.sksamuel.hoplite.transformer.PathNormalizer +import io.kotest.core.spec.style.DescribeSpec +import io.kotest.matchers.shouldBe + +class EnvPropertySourceNormalizationTest : DescribeSpec({ + + data class Creds(val username: String, val password: String) + data class Config(val creds: Creds, val someCamelSetting: String) + + describe("loading from envs") { + it("with path normalizer") { + run { + ConfigLoader { + addNodeTransformer(PathNormalizer) + addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, + allowUppercaseNames = false, + environmentVariableMap = { + mapOf( + "CREDS__USERNAME" to "a", + "CREDS__PASSWORD" to "c", + "SOMECAMELSETTING" to "c" + ) + } + ) + ) + }.loadConfigOrThrow() + } shouldBe Config(Creds("a", "c"), "c") + } + + it("with path normalizer and kebab case underscore separator") { + run { + ConfigLoader { + addNodeTransformer(PathNormalizer) + addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, + allowUppercaseNames = false, + environmentVariableMap = { + mapOf( + "CREDS__USERNAME" to "a", + "CREDS__PASSWORD" to "c", + "SOME_CAMEL_SETTING" to "c" + ) + } + ) + ) + }.loadConfigOrThrow() + } shouldBe Config(Creds("a", "c"), "c") + } + + it("with path normalizer and underscore separator") { + run { + ConfigLoader { + addNodeTransformer(PathNormalizer) + addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + useSingleUnderscoresAsSeparator = true, + allowUppercaseNames = true, + environmentVariableMap = { + mapOf( + "CREDS_USERNAME" to "a", + "CREDS_PASSWORD" to "b", + "SOMECAMELSETTING" to "c" + ) + } + ) + ) + }.loadConfigOrThrow() + } shouldBe Config(Creds("a", "b"), "c") + } + + it("with path normalizer and lowercase names") { + run { + ConfigLoader { + addNodeTransformer(PathNormalizer) + addPropertySource(EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + useSingleUnderscoresAsSeparator = true, + allowUppercaseNames = false, + environmentVariableMap = { + mapOf( + "creds_username" to "a", + "creds_password" to "d", + "somecamelsetting" to "e" + ) + } + )) + }.loadConfigOrThrow() + } shouldBe Config(Creds("a", "d"), "e") + } + + it("with path normalizer and prefix") { + run { + ConfigLoader { + addNodeTransformer(PathNormalizer) + addPropertySource( + EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + useSingleUnderscoresAsSeparator = true, + allowUppercaseNames = false, + environmentVariableMap = { + mapOf( + "WIBBLE_CREDS_USERNAME" to "a", + "WIBBLE_CREDS_PASSWORD" to "c", + "WIBBLE_SOMECAMELSETTING" to "c" + ) + }, + prefix = "WIBBLE_" + ) + ) + }.loadConfigOrThrow() + } shouldBe Config(Creds("a", "c"), "c") + } + } + +}) diff --git a/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceSingleUnderscoreAsSeparatorTest.kt b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceSingleUnderscoreAsSeparatorTest.kt new file mode 100644 index 00000000..e5fdebbc --- /dev/null +++ b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceSingleUnderscoreAsSeparatorTest.kt @@ -0,0 +1,35 @@ +package com.sksamuel.hoplite.json + +import com.sksamuel.hoplite.ConfigLoader +import com.sksamuel.hoplite.addIdiomaticEnvironmentSource +import com.sksamuel.hoplite.transformer.PathNormalizer +import io.kotest.core.spec.style.FunSpec +import io.kotest.extensions.system.withEnvironment +import io.kotest.matchers.shouldBe + +class EnvPropertySourceSingleUnderscoreAsSeparatorTest : FunSpec({ + + data class Creds(val username: String, val password: String) + data class Config(val creds: Creds, val someCamelSetting: String) + + test("loading from envs") { + withEnvironment(mapOf("creds_username" to "a", "creds_password" to "b", "someCamelSetting" to "c")) { + ConfigLoader + .builder() + .addIdiomaticEnvironmentSource() + .build() + .loadConfigOrThrow() shouldBe Config(Creds("a", "b"), "c") + } + } + + test("loading from envs with a path normalizer") { + withEnvironment(mapOf("CREDS_USERNAME" to "a", "CREDS_PASSWORD" to "b", "SOMECAMELSETTING" to "c")) { + ConfigLoader + .builder() + .addNodeTransformer(PathNormalizer) + .addIdiomaticEnvironmentSource() + .build() + .loadConfigOrThrow() shouldBe Config(Creds("a", "b"), "c") + } + } +}) diff --git a/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceUppercaseTest.kt b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceUppercaseTest.kt index 9084dd32..592a43ac 100644 --- a/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceUppercaseTest.kt +++ b/hoplite-json/src/test/kotlin/com/sksamuel/hoplite/json/EnvPropertySourceUppercaseTest.kt @@ -17,6 +17,7 @@ class EnvPropertySourceUppercaseTest : DescribeSpec({ addPropertySource( EnvironmentVariablesPropertySource( useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, allowUppercaseNames = true, environmentVariableMap = { mapOf( @@ -37,6 +38,7 @@ class EnvPropertySourceUppercaseTest : DescribeSpec({ addPropertySource( EnvironmentVariablesPropertySource( useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, allowUppercaseNames = true, environmentVariableMap = { mapOf( @@ -56,6 +58,7 @@ class EnvPropertySourceUppercaseTest : DescribeSpec({ ConfigLoader { addPropertySource(EnvironmentVariablesPropertySource( useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, allowUppercaseNames = true, environmentVariableMap = { mapOf( @@ -75,6 +78,7 @@ class EnvPropertySourceUppercaseTest : DescribeSpec({ addPropertySource( EnvironmentVariablesPropertySource( useUnderscoresAsSeparator = true, + useSingleUnderscoresAsSeparator = false, allowUppercaseNames = true, environmentVariableMap = { mapOf(