diff --git a/SingularityService/pom.xml b/SingularityService/pom.xml
index f6425cabc2..ac60592570 100644
--- a/SingularityService/pom.xml
+++ b/SingularityService/pom.xml
@@ -177,6 +177,11 @@
dropwizard-core
+
+ io.dropwizard
+ dropwizard-configuration
+
+
io.dropwizard
dropwizard-db
@@ -253,6 +258,11 @@
jackson-databind
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+
+
com.fasterxml.jackson.datatype
jackson-datatype-guava
diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java
index 99c127cf51..624125a0a7 100644
--- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java
+++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java
@@ -4,12 +4,15 @@
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.datatype.guava.GuavaModule;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.inject.Module;
import com.hubspot.dropwizard.guicier.GuiceBundle;
import com.hubspot.jackson.datatype.protobuf.ProtobufModule;
import com.hubspot.singularity.bundles.CorsBundle;
+import com.hubspot.singularity.config.MergingSourceProvider;
import com.hubspot.singularity.config.SingularityConfiguration;
import io.dropwizard.Application;
@@ -23,11 +26,15 @@
import io.dropwizard.views.ViewBundle;
public class SingularityService extends Application {
+ private static final String SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY = "singularityDefaultConfiguration";
public static final String API_BASE_PATH = "/api";
@Override
public void initialize(final Bootstrap bootstrap) {
+ if (!Strings.isNullOrEmpty(System.getProperty(SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY))) {
+ bootstrap.setConfigurationSourceProvider(new MergingSourceProvider(bootstrap.getConfigurationSourceProvider(), System.getProperty(SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY), bootstrap.getObjectMapper(), new YAMLFactory()));
+ }
final Iterable extends Module> additionalModules = checkNotNull(getGuiceModules(bootstrap), "getGuiceModules() returned null");
final Iterable extends Bundle> additionalBundles = checkNotNull(getDropwizardBundles(bootstrap), "getDropwizardBundles() returned null");
diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/MergingSourceProvider.java b/SingularityService/src/main/java/com/hubspot/singularity/config/MergingSourceProvider.java
new file mode 100644
index 0000000000..da1ddbcddc
--- /dev/null
+++ b/SingularityService/src/main/java/com/hubspot/singularity/config/MergingSourceProvider.java
@@ -0,0 +1,67 @@
+package com.hubspot.singularity.config;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+import io.dropwizard.configuration.ConfigurationSourceProvider;
+
+public class MergingSourceProvider implements ConfigurationSourceProvider {
+ private final ConfigurationSourceProvider delegate;
+ private final String defaultConfigurationPath;
+ private final ObjectMapper objectMapper;
+ private final YAMLFactory yamlFactory;
+
+ public MergingSourceProvider(ConfigurationSourceProvider delegate, String defaultConfigurationPath, ObjectMapper objectMapper, YAMLFactory yamlFactory) {
+ this.delegate = delegate;
+ this.defaultConfigurationPath = defaultConfigurationPath;
+ this.objectMapper = objectMapper;
+ this.yamlFactory = yamlFactory;
+ }
+
+ @Override
+ public InputStream open(String path) throws IOException {
+ final JsonNode originalNode = objectMapper.readTree(yamlFactory.createParser(delegate.open(defaultConfigurationPath)));
+ final JsonNode overrideNode = objectMapper.readTree(yamlFactory.createParser(delegate.open(path)));
+
+ if (!(originalNode instanceof ObjectNode && overrideNode instanceof ObjectNode)) {
+ throw new SingularityConfigurationMergeException(String.format("Both %s and %s need to be YAML objects", defaultConfigurationPath, path));
+ }
+
+ merge((ObjectNode)originalNode, (ObjectNode)overrideNode);
+
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ objectMapper.writeTree(yamlFactory.createGenerator(baos), originalNode);
+
+ return new ByteArrayInputStream(baos.toByteArray());
+ }
+
+ private static void merge(ObjectNode to, ObjectNode from) {
+ Iterator newFieldNames = from.fieldNames();
+
+ while (newFieldNames.hasNext()) {
+ String newFieldName = newFieldNames.next();
+ JsonNode oldVal = to.get(newFieldName);
+ JsonNode newVal = from.get(newFieldName);
+
+ if (oldVal == null || oldVal.isNull()) {
+ to.put(newFieldName, newVal);
+ } else if (oldVal.isArray() && newVal.isArray()) {
+ ((ArrayNode) oldVal).removeAll();
+ ((ArrayNode) oldVal).addAll((ArrayNode) newVal);
+ } else if (oldVal.isObject() && newVal.isObject()) {
+ merge((ObjectNode) oldVal, (ObjectNode) newVal);
+ } else if (!(newVal == null || newVal.isNull())) {
+ to.put(newFieldName, newVal);
+ }
+ }
+ }
+}
diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfigurationMergeException.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfigurationMergeException.java
new file mode 100644
index 0000000000..6ad82f09cd
--- /dev/null
+++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfigurationMergeException.java
@@ -0,0 +1,7 @@
+package com.hubspot.singularity.config;
+
+public class SingularityConfigurationMergeException extends RuntimeException {
+ public SingularityConfigurationMergeException(String message) {
+ super(message);
+ }
+}
diff --git a/SingularityService/src/test/java/com/hubspot/singularity/config/MergingSourceProviderTest.java b/SingularityService/src/test/java/com/hubspot/singularity/config/MergingSourceProviderTest.java
new file mode 100644
index 0000000000..3fc5bff5ad
--- /dev/null
+++ b/SingularityService/src/test/java/com/hubspot/singularity/config/MergingSourceProviderTest.java
@@ -0,0 +1,66 @@
+package com.hubspot.singularity.config;
+
+import static junit.framework.TestCase.assertEquals;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.inject.Inject;
+import com.hubspot.singularity.SingularityTestBaseNoDb;
+
+import io.dropwizard.configuration.ConfigurationSourceProvider;
+
+public class MergingSourceProviderTest extends SingularityTestBaseNoDb {
+ private static final String DEFAULT_PATH = "/configs/default.yaml";
+ private static final String OVERRIDE_PATH = "/configs/override.yaml";
+ private static final String JUST_A_STRING_PATH = "/configs/just_a_string.yaml";
+ private static final String DOESNT_EXIST_PATH = "/configs/doesnt_exist.yaml";
+
+ private static final YAMLFactory YAML_FACTORY = new YAMLFactory();
+
+ @Inject
+ private ObjectMapper objectMapper;
+
+ private ConfigurationSourceProvider buildConfigurationSourceProvider(String baseFilename) {
+ final Class> klass = getClass();
+
+ return new MergingSourceProvider(new ConfigurationSourceProvider() {
+ @Override
+ public InputStream open(String path) throws IOException {
+ final InputStream stream = klass.getResourceAsStream(path);
+ if (stream == null) {
+ throw new FileNotFoundException("File " + path + " not found in test resources directory");
+ }
+ return stream;
+ }
+ }, baseFilename, objectMapper, YAML_FACTORY);
+ }
+
+ @Test
+ public void testMergedConfigs() throws Exception {
+ final InputStream mergedConfigStream = buildConfigurationSourceProvider(DEFAULT_PATH).open(OVERRIDE_PATH);
+ final SingularityConfiguration mergedConfig = objectMapper.readValue(YAML_FACTORY.createParser(mergedConfigStream), SingularityConfiguration.class);
+
+ assertEquals(10000, mergedConfig.getCacheTasksMaxSize());
+ assertEquals(500, mergedConfig.getCacheTasksInitialSize());
+ assertEquals(100, mergedConfig.getCheckDeploysEverySeconds());
+ assertEquals("baseuser", mergedConfig.getDatabaseConfiguration().get().getUser());
+ assertEquals("overridepassword", mergedConfig.getDatabaseConfiguration().get().getPassword());
+ }
+
+ @Test( expected = SingularityConfigurationMergeException.class )
+ public void testNonObjectFails() throws Exception {
+ buildConfigurationSourceProvider(DEFAULT_PATH).open(JUST_A_STRING_PATH);
+ }
+
+ @Test( expected = FileNotFoundException.class)
+ public void testFileNoExistFail() throws Exception {
+ buildConfigurationSourceProvider(DEFAULT_PATH).open(DOESNT_EXIST_PATH);
+ buildConfigurationSourceProvider(DOESNT_EXIST_PATH).open(OVERRIDE_PATH);
+ }
+}
diff --git a/SingularityService/src/test/resources/configs/default.yaml b/SingularityService/src/test/resources/configs/default.yaml
new file mode 100644
index 0000000000..2c396608ce
--- /dev/null
+++ b/SingularityService/src/test/resources/configs/default.yaml
@@ -0,0 +1,6 @@
+cacheTasksMaxSize: 10000 # override.yaml will not touch this
+cacheTasksInitialSize: 200 # override.yaml will override this to 500
+# override.yaml will override checkDeploysEverySeconds to 100
+
+database: # override.yaml will override some of this
+ user: "baseuser"
diff --git a/SingularityService/src/test/resources/configs/just_a_string.yaml b/SingularityService/src/test/resources/configs/just_a_string.yaml
new file mode 100644
index 0000000000..ab29415519
--- /dev/null
+++ b/SingularityService/src/test/resources/configs/just_a_string.yaml
@@ -0,0 +1 @@
+"this is just a string!"
diff --git a/SingularityService/src/test/resources/configs/override.yaml b/SingularityService/src/test/resources/configs/override.yaml
new file mode 100644
index 0000000000..eedc638a8c
--- /dev/null
+++ b/SingularityService/src/test/resources/configs/override.yaml
@@ -0,0 +1,5 @@
+cacheTasksInitialSize: 500
+checkDeploysEverySeconds: 100
+
+database:
+ password: "overridepassword"