diff --git a/SingularityService/pom.xml b/SingularityService/pom.xml index f6425cabc2..ac60592570 100644 --- a/SingularityService/pom.xml +++ b/SingularityService/pom.xml @@ -177,6 +177,11 @@ dropwizard-core + + io.dropwizard + dropwizard-configuration + + io.dropwizard dropwizard-db @@ -253,6 +258,11 @@ jackson-databind + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + com.fasterxml.jackson.datatype jackson-datatype-guava diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java index 99c127cf51..624125a0a7 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java @@ -4,12 +4,15 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.datatype.guava.GuavaModule; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.inject.Module; import com.hubspot.dropwizard.guicier.GuiceBundle; import com.hubspot.jackson.datatype.protobuf.ProtobufModule; import com.hubspot.singularity.bundles.CorsBundle; +import com.hubspot.singularity.config.MergingSourceProvider; import com.hubspot.singularity.config.SingularityConfiguration; import io.dropwizard.Application; @@ -23,11 +26,15 @@ import io.dropwizard.views.ViewBundle; public class SingularityService extends Application { + private static final String SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY = "singularityDefaultConfiguration"; public static final String API_BASE_PATH = "/api"; @Override public void initialize(final Bootstrap bootstrap) { + if (!Strings.isNullOrEmpty(System.getProperty(SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY))) { + bootstrap.setConfigurationSourceProvider(new MergingSourceProvider(bootstrap.getConfigurationSourceProvider(), System.getProperty(SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY), bootstrap.getObjectMapper(), new YAMLFactory())); + } final Iterable additionalModules = checkNotNull(getGuiceModules(bootstrap), "getGuiceModules() returned null"); final Iterable additionalBundles = checkNotNull(getDropwizardBundles(bootstrap), "getDropwizardBundles() returned null"); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/MergingSourceProvider.java b/SingularityService/src/main/java/com/hubspot/singularity/config/MergingSourceProvider.java new file mode 100644 index 0000000000..da1ddbcddc --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/MergingSourceProvider.java @@ -0,0 +1,67 @@ +package com.hubspot.singularity.config; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import io.dropwizard.configuration.ConfigurationSourceProvider; + +public class MergingSourceProvider implements ConfigurationSourceProvider { + private final ConfigurationSourceProvider delegate; + private final String defaultConfigurationPath; + private final ObjectMapper objectMapper; + private final YAMLFactory yamlFactory; + + public MergingSourceProvider(ConfigurationSourceProvider delegate, String defaultConfigurationPath, ObjectMapper objectMapper, YAMLFactory yamlFactory) { + this.delegate = delegate; + this.defaultConfigurationPath = defaultConfigurationPath; + this.objectMapper = objectMapper; + this.yamlFactory = yamlFactory; + } + + @Override + public InputStream open(String path) throws IOException { + final JsonNode originalNode = objectMapper.readTree(yamlFactory.createParser(delegate.open(defaultConfigurationPath))); + final JsonNode overrideNode = objectMapper.readTree(yamlFactory.createParser(delegate.open(path))); + + if (!(originalNode instanceof ObjectNode && overrideNode instanceof ObjectNode)) { + throw new SingularityConfigurationMergeException(String.format("Both %s and %s need to be YAML objects", defaultConfigurationPath, path)); + } + + merge((ObjectNode)originalNode, (ObjectNode)overrideNode); + + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + objectMapper.writeTree(yamlFactory.createGenerator(baos), originalNode); + + return new ByteArrayInputStream(baos.toByteArray()); + } + + private static void merge(ObjectNode to, ObjectNode from) { + Iterator newFieldNames = from.fieldNames(); + + while (newFieldNames.hasNext()) { + String newFieldName = newFieldNames.next(); + JsonNode oldVal = to.get(newFieldName); + JsonNode newVal = from.get(newFieldName); + + if (oldVal == null || oldVal.isNull()) { + to.put(newFieldName, newVal); + } else if (oldVal.isArray() && newVal.isArray()) { + ((ArrayNode) oldVal).removeAll(); + ((ArrayNode) oldVal).addAll((ArrayNode) newVal); + } else if (oldVal.isObject() && newVal.isObject()) { + merge((ObjectNode) oldVal, (ObjectNode) newVal); + } else if (!(newVal == null || newVal.isNull())) { + to.put(newFieldName, newVal); + } + } + } +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfigurationMergeException.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfigurationMergeException.java new file mode 100644 index 0000000000..6ad82f09cd --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfigurationMergeException.java @@ -0,0 +1,7 @@ +package com.hubspot.singularity.config; + +public class SingularityConfigurationMergeException extends RuntimeException { + public SingularityConfigurationMergeException(String message) { + super(message); + } +} diff --git a/SingularityService/src/test/java/com/hubspot/singularity/config/MergingSourceProviderTest.java b/SingularityService/src/test/java/com/hubspot/singularity/config/MergingSourceProviderTest.java new file mode 100644 index 0000000000..3fc5bff5ad --- /dev/null +++ b/SingularityService/src/test/java/com/hubspot/singularity/config/MergingSourceProviderTest.java @@ -0,0 +1,66 @@ +package com.hubspot.singularity.config; + +import static junit.framework.TestCase.assertEquals; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; + +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.inject.Inject; +import com.hubspot.singularity.SingularityTestBaseNoDb; + +import io.dropwizard.configuration.ConfigurationSourceProvider; + +public class MergingSourceProviderTest extends SingularityTestBaseNoDb { + private static final String DEFAULT_PATH = "/configs/default.yaml"; + private static final String OVERRIDE_PATH = "/configs/override.yaml"; + private static final String JUST_A_STRING_PATH = "/configs/just_a_string.yaml"; + private static final String DOESNT_EXIST_PATH = "/configs/doesnt_exist.yaml"; + + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + + @Inject + private ObjectMapper objectMapper; + + private ConfigurationSourceProvider buildConfigurationSourceProvider(String baseFilename) { + final Class klass = getClass(); + + return new MergingSourceProvider(new ConfigurationSourceProvider() { + @Override + public InputStream open(String path) throws IOException { + final InputStream stream = klass.getResourceAsStream(path); + if (stream == null) { + throw new FileNotFoundException("File " + path + " not found in test resources directory"); + } + return stream; + } + }, baseFilename, objectMapper, YAML_FACTORY); + } + + @Test + public void testMergedConfigs() throws Exception { + final InputStream mergedConfigStream = buildConfigurationSourceProvider(DEFAULT_PATH).open(OVERRIDE_PATH); + final SingularityConfiguration mergedConfig = objectMapper.readValue(YAML_FACTORY.createParser(mergedConfigStream), SingularityConfiguration.class); + + assertEquals(10000, mergedConfig.getCacheTasksMaxSize()); + assertEquals(500, mergedConfig.getCacheTasksInitialSize()); + assertEquals(100, mergedConfig.getCheckDeploysEverySeconds()); + assertEquals("baseuser", mergedConfig.getDatabaseConfiguration().get().getUser()); + assertEquals("overridepassword", mergedConfig.getDatabaseConfiguration().get().getPassword()); + } + + @Test( expected = SingularityConfigurationMergeException.class ) + public void testNonObjectFails() throws Exception { + buildConfigurationSourceProvider(DEFAULT_PATH).open(JUST_A_STRING_PATH); + } + + @Test( expected = FileNotFoundException.class) + public void testFileNoExistFail() throws Exception { + buildConfigurationSourceProvider(DEFAULT_PATH).open(DOESNT_EXIST_PATH); + buildConfigurationSourceProvider(DOESNT_EXIST_PATH).open(OVERRIDE_PATH); + } +} diff --git a/SingularityService/src/test/resources/configs/default.yaml b/SingularityService/src/test/resources/configs/default.yaml new file mode 100644 index 0000000000..2c396608ce --- /dev/null +++ b/SingularityService/src/test/resources/configs/default.yaml @@ -0,0 +1,6 @@ +cacheTasksMaxSize: 10000 # override.yaml will not touch this +cacheTasksInitialSize: 200 # override.yaml will override this to 500 +# override.yaml will override checkDeploysEverySeconds to 100 + +database: # override.yaml will override some of this + user: "baseuser" diff --git a/SingularityService/src/test/resources/configs/just_a_string.yaml b/SingularityService/src/test/resources/configs/just_a_string.yaml new file mode 100644 index 0000000000..ab29415519 --- /dev/null +++ b/SingularityService/src/test/resources/configs/just_a_string.yaml @@ -0,0 +1 @@ +"this is just a string!" diff --git a/SingularityService/src/test/resources/configs/override.yaml b/SingularityService/src/test/resources/configs/override.yaml new file mode 100644 index 0000000000..eedc638a8c --- /dev/null +++ b/SingularityService/src/test/resources/configs/override.yaml @@ -0,0 +1,5 @@ +cacheTasksInitialSize: 500 +checkDeploysEverySeconds: 100 + +database: + password: "overridepassword"