Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixes #2102 support decrypt or not for values.yml and env injection #2103

Merged
merged 1 commit into from
Feb 2, 2024
Merged

fixes #2102 support decrypt or not for values.yml and env injection #2103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions config/src/main/java/com/networknt/config/CentralizedManagement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,16 @@
*/
public class CentralizedManagement {
// Merge map config with values generated by ConfigInjection.class and return map
public static void mergeMap(Map<String, Object> config) {
merge(config);
public static void mergeMap(boolean decrypt, Map<String, Object> config) {
merge(decrypt, config);
}
// Merge map config with values generated by ConfigInjection.class and return mapping object
public static Object mergeObject(Object config, Class clazz) {
merge(config);
public static Object mergeObject(boolean decrypt, Object config, Class clazz) {
merge(decrypt, config);
return convertMapToObj((Map<String, Object>) config, clazz);
}
// Search the config map recursively, expand List and Map level by level util no further expand
private static void merge(Object m1) {
private static void merge(boolean decrypt, Object m1) {
if (m1 instanceof Map) {
Iterator<Object> fieldNames = ((Map<Object, Object>) m1).keySet().iterator();
String fieldName = null;
Expand All @@ -55,21 +55,21 @@ private static void merge(Object m1) {
Object field1 = ((Map<String, Object>) m1).get(fieldName);
if (field1 != null) {
if (field1 instanceof Map || field1 instanceof List) {
merge(field1);
merge(decrypt, field1);
// Overwrite previous value when the field1 can not be expanded further
} else if (field1 instanceof String) {
// Retrieve values from ConfigInjection.class
Object injectValue = ConfigInjection.getInjectValue((String) field1);
Object injectValue = ConfigInjection.getInjectValue((String) field1, decrypt);
((Map<String, Object>) m1).put(fieldName, injectValue);
}
}
// post order, in case the key of configuration can also be injected.
Object injectedFieldName = ConfigInjection.getInjectValue(fieldName);
Object injectedFieldName = ConfigInjection.getInjectValue(fieldName, decrypt);
// only inject when key has been changed
if (!fieldName.equals(injectedFieldName)) {
validateInjectedFieldName(fieldName, injectedFieldName);
// the map is unmodifiable during iterator, so put in another map and put it back after iteration.
mapWithInjectedKey.put((String)ConfigInjection.getInjectValue(fieldName), field1);
mapWithInjectedKey.put((String)ConfigInjection.getInjectValue(fieldName, decrypt), field1);
fieldNames.remove();
}
}
Expand All @@ -79,11 +79,11 @@ private static void merge(Object m1) {
for (int i = 0; i < ((List<Object>) m1).size(); i++) {
Object field1 = ((List<Object>) m1).get(i);
if (field1 instanceof Map || field1 instanceof List) {
merge(field1);
merge(decrypt, field1);
// Overwrite previous value when the field1 can not be expanded further
} else if (field1 instanceof String) {
// Retrieve values from ConfigInjection.class
Object injectValue = ConfigInjection.getInjectValue((String) field1);
Object injectValue = ConfigInjection.getInjectValue((String) field1, decrypt);
((List<Object>) m1).set(i, injectValue);
}
}
Expand Down
28 changes: 20 additions & 8 deletions config/src/main/java/com/networknt/config/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;

/**
* A injectable singleton config that has default implementation
* An injectable singleton config that has default implementation
* based on FileSystem json files. It can be extended to
* other sources (database, distributed cache etc.) by providing
* another jar in the classpath to replace the default implementation.
Expand Down Expand Up @@ -104,6 +104,8 @@ protected Config() {

public abstract Yaml getYaml();

public abstract boolean isDecrypt();

public abstract void clear();

public abstract void setClassLoader(ClassLoader urlClassLoader);
Expand All @@ -117,7 +119,7 @@ public static Config getNoneDecryptedInstance() {
return NoneDecryptedConfigImpl.NONE_DECRYPTED;
}

public abstract String getDecryptorClassPublic();
// public abstract String getDecryptorClassPublic();

private static abstract class AbstractConfigImpl extends Config {
static final String CONFIG_NAME = "config";
Expand All @@ -129,7 +131,7 @@ private static abstract class AbstractConfigImpl extends Config {
public final String[] EXTERNALIZED_PROPERTY_DIR = System.getProperty(LIGHT_4J_CONFIG_DIR, "").split(File.pathSeparator);
private ConfigLoader configLoader;
private ClassLoader classLoader;
private String configLoaderClass;
private final String configLoaderClass;

// Memory cache of all the configuration object. Each config will be loaded on the first time it is accessed.
final Map<String, Object> configCache = new ConcurrentHashMap<>(10, 0.9f, 1);
Expand Down Expand Up @@ -427,7 +429,7 @@ private <T> Object loadSpecificConfigFileAsObject(String configName, String file
} else {
// Parse into map first, since map is easier to be manipulated in merging process
Map<String, Object> configMap = getYaml().load(inStream);
config = CentralizedManagement.mergeObject(configMap, clazz);
config = CentralizedManagement.mergeObject(isDecrypt(), configMap, clazz);
}
}
} catch (Exception e) {
Expand Down Expand Up @@ -460,7 +462,7 @@ private Map<String, Object> loadSpecificConfigFileAsMap(String configName, Strin
if (inStream != null) {
config = getYaml().load(inStream);
if (!ConfigInjection.isExclusionConfigFile(configName)) {
CentralizedManagement.mergeMap(config); // mutates the config map in place.
CentralizedManagement.mergeMap(isDecrypt(), config); // mutates the config map in place.
}
}
} catch (Exception e) {
Expand Down Expand Up @@ -545,7 +547,7 @@ private String getAbsolutePath(String path, int index) {
if (path.startsWith("/")) {
return path;
} else {
return path.equals("") ? EXTERNALIZED_PROPERTY_DIR[index].trim() : EXTERNALIZED_PROPERTY_DIR[index].trim() + "/" + path;
return path.isEmpty() ? EXTERNALIZED_PROPERTY_DIR[index].trim() : EXTERNALIZED_PROPERTY_DIR[index].trim() + "/" + path;
}
}

Expand All @@ -563,7 +565,7 @@ protected String getDecryptorClass() {
return DecryptConstructor.DEFAULT_DECRYPTOR_CLASS;
}

public String getDecryptorClassPublic() { return getDecryptorClass(); }
// public String getDecryptorClassPublic() { return getDecryptorClass(); }

private String getConfigLoaderClass() {
Map<String, Object> config = loadModuleConfig();
Expand Down Expand Up @@ -647,7 +649,7 @@ private Object loadJsonObjectConfigWithSpecificConfigLoader(String configName, C
}
if (this.configLoader != null) {
logger.trace("Trying to load {} with extension yaml, yml or json by using ConfigLoader: {}.", configName, configLoader.getClass().getName());
if (path == null || path.equals("")) {
if (path == null || path.isEmpty()) {
config = configLoader.loadObjectConfig(configName, clazz);
} else {
config = configLoader.loadObjectConfig(configName, clazz, path);
Expand Down Expand Up @@ -687,6 +689,11 @@ private static Config initialize() {
public Yaml getYaml() {
return yaml;
}

@Override
public boolean isDecrypt() {
return false;
}
}

private static final class FileConfigImpl extends AbstractConfigImpl {
Expand Down Expand Up @@ -719,6 +726,11 @@ private static Config initialize() {
public Yaml getYaml() {
return yaml;
}

@Override
public boolean isDecrypt() {
return true;
}
}

static String convertStreamToString(java.io.InputStream is) {
Expand Down
63 changes: 39 additions & 24 deletions config/src/main/java/com/networknt/config/ConfigInjection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,29 +56,29 @@ public class ConfigInjection {
private static final Map<String, Object> exclusionMap = Config.getInstance().getJsonMapConfig(SCALABLE_CONFIG);

// Define the injection pattern which represents the injection points
private static Pattern pattern = Pattern.compile("\\$\\{(.*?)\\}");
private static final Pattern pattern = Pattern.compile("\\$\\{(.*?)\\}");

private static String[] trueArray = {"y", "Y", "yes", "Yes", "YES", "true", "True", "TRUE", "on", "On", "ON"};
private static String[] falseArray = {"n", "N", "no", "No", "NO", "false", "False", "FALSE", "off", "Off", "OFF"};
private static Decryptor decryptor = DecryptConstructor.getInstance().getDecryptor();
private static final String[] trueArray = {"y", "Y", "yes", "Yes", "YES", "true", "True", "TRUE", "on", "On", "ON"};
private static final String[] falseArray = {"n", "N", "no", "No", "NO", "false", "False", "FALSE", "off", "Off", "OFF"};
private static final Decryptor decryptor = DecryptConstructor.getInstance().getDecryptor();

// Method used to generate the values from environment variables or "values.yaml"
public static Object getInjectValue(String string) {
public static Object getInjectValue(String string, boolean decrypt) {
Matcher m = pattern.matcher(string);
StringBuffer sb = new StringBuffer();
// Parse the content inside pattern "${}" when this pattern is found
while (m.find()) {
// Get parsing result
Object value = getValue(m.group(1));
// Return directly when the parsing result don't need to be casted to String
Object value = getValue(m.group(1), decrypt);
// Return directly when the parsing result don't need to be cast to String
if (!(value instanceof String)) {
return value;
}
String valueStr = (String)value;
if(valueStr.contains("\\")) {
m.appendReplacement(sb, (String)value);
} else {
m.appendReplacement(sb, m.quoteReplacement((String)value));
m.appendReplacement(sb, Matcher.quoteReplacement((String)value));
}
}
return m.appendTail(sb).toString();
Expand All @@ -99,7 +99,7 @@ public static Decryptor getDecryptor() {

static String convertEnvVars(String input){
// check for any non-alphanumeric chars and convert to underscore
// convert to uppcase
// convert to upper case
if (input == null) {
return null;
}
Expand All @@ -118,36 +118,51 @@ public static Object decryptEnvValue(Decryptor decryptor, String envVal) {


// Method used to parse the content inside pattern "${}"
private static Object getValue(String content) {
private static Object getValue(String content, boolean decrypt) {
InjectionPattern injectionPattern = getInjectionPattern(content);
Object value = null;
if (injectionPattern != null) {
// Flag to validate whether the environment or values.yml contains the corresponding field
Boolean containsField = false;
boolean containsField = false;
// Use key of injectionPattern to get value from both environment variables and "values.yaml"
String envValString = System.getenv(convertEnvVars(injectionPattern.getKey()));
Object envValue = decryptEnvValue(decryptor, envValString);
// change to no cache method to support config-reload.
Map<String, Object> valueMap = Config.getInstance().getDefaultJsonMapConfigNoCache(CENTRALIZED_MANAGEMENT);
Object fileValue = (valueMap != null) ? valueMap.get(injectionPattern.getKey()) : null;

Object envValue;
Object fileValue;
// the reason we don't cache the valueMap is to support the config reload. This needs to be revisited.
// TODO We should cache it and reload it when the file is changed.
if(decrypt) {
envValue = decryptEnvValue(decryptor, envValString);
Map<String, Object> valueMap = Config.getInstance().getDefaultJsonMapConfigNoCache(CENTRALIZED_MANAGEMENT);
fileValue = (valueMap != null) ? valueMap.get(injectionPattern.getKey()) : null;
// Skip none validation to inject null or empty string directly when the corresponding field is presented in value.yml or environment
if ((valueMap != null && valueMap.containsKey(injectionPattern.getKey())) ||
(System.getenv() != null && System.getenv().containsKey(injectionPattern.getKey()))) {
containsField = true;
}
} else {
envValue = envValString;
Map<String, Object> valueMap = Config.getNoneDecryptedInstance().getDefaultJsonMapConfigNoCache(CENTRALIZED_MANAGEMENT);
fileValue = (valueMap != null) ? valueMap.get(injectionPattern.getKey()) : null;
// Skip none validation to inject null or empty string directly when the corresponding field is presented in value.yml or environment
if ((valueMap != null && valueMap.containsKey(injectionPattern.getKey())) ||
(System.getenv() != null && System.getenv().containsKey(injectionPattern.getKey()))) {
containsField = true;
}
}
// Return different value from different sources based on injection order defined before
if ((INJECTION_ORDER_CODE.equals("2") && envValue != null) || (INJECTION_ORDER_CODE.equals("1") && fileValue == null)) {
value = envValue;
} else {
value = fileValue;
}
// Skip none validation to inject null or empty string directly when the corresponding field is presented in value.yml or environment
if ((valueMap != null && valueMap.containsKey(injectionPattern.getKey())) ||
(System.getenv() != null && System.getenv().containsKey(injectionPattern.getKey()))) {
containsField = true;
}
// Return default value when no matched value found from environment variables and "values.yaml"
if (value == null && !containsField) {
value = typeCast(injectionPattern.getDefaultValue());
// Throw exception when error text provided
if (value == null || value.equals("")) {
String error_text = injectionPattern.getErrorText();
if (error_text != null && !error_text.equals("")) {
if (error_text != null && !error_text.isEmpty()) {
throw new ConfigException(error_text);
}
// Throw exception when no parsing result found
Expand All @@ -162,15 +177,15 @@ private static Object getValue(String content) {

// Get instance of InjectionPattern based on the contents inside pattern "${}"
private static InjectionPattern getInjectionPattern(String contents) {
if (contents == null || contents.trim().equals("")) {
if (contents == null || contents.trim().isEmpty()) {
return null;
}
InjectionPattern injectionPattern = new InjectionPattern();
contents = contents.trim();
// Retrieve key, default value and error text
String[] array = contents.split(":", 2);
array[0] = array[0].trim();
if ("".equals(array[0])) {
if (array[0].isEmpty()) {
return null;
}
// Set key of the injectionPattern
Expand Down Expand Up @@ -234,7 +249,7 @@ public void setKey(String key) {

// Method used to cast string into int, double or boolean
private static Object typeCast(String str) {
if (str == null || str.equals("")) {
if (str == null || str.isEmpty()) {
return null;
}
// Try to cast to boolean true
Expand Down
4 changes: 2 additions & 2 deletions config/src/test/java/com/networknt/config/ConfigEscapeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ public class ConfigEscapeTest {
@Test
public void testGetInjectValueWithDollar() {
String s1 = "${password:abc$defg}";
Object obj = getInjectValue(s1);
Object obj = getInjectValue(s1, true);
System.out.println(obj);

}

@Test
public void testGetInjectValueWithBackSlash() {
String s2 = "${password:abc\\$defg}";
Object obj = getInjectValue(s2);
Object obj = getInjectValue(s2, true);
System.out.println(obj);
}

Expand Down
4 changes: 2 additions & 2 deletions config/src/test/java/com/networknt/config/ConfigInjectionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public void testGetInjectValueIssue744() {

Object oldConfigValue = null;
try {
oldConfigValue = ConfigInjection.getInjectValue(value);
oldConfigValue = ConfigInjection.getInjectValue(value, true);
} catch (Exception ce) {
// expected exception since no valuemap defined yet.
assertTrue(ce instanceof ConfigException);
Expand All @@ -39,7 +39,7 @@ public void testGetInjectValueIssue744() {
newValueMap.put(configKey, configValue);
Config.getInstance().putInConfigCache(valueMapKey, newValueMap);

Object newConfigValue = ConfigInjection.getInjectValue(value);
Object newConfigValue = ConfigInjection.getInjectValue(value, true);

assertNotNull(newConfigValue);
assertEquals(configValue, newConfigValue);
Expand Down
Loading