add support for override defaults file

SingularityService/pom.xml

@@ -177,6 +177,11 @@
       <artifactId>dropwizard-core</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.dropwizard</groupId>
+      <artifactId>dropwizard-configuration</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>io.dropwizard</groupId>
       <artifactId>dropwizard-db</artifactId>
@@ -253,6 +258,11 @@
       <artifactId>jackson-databind</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.fasterxml.jackson.datatype</groupId>
       <artifactId>jackson-datatype-guava</artifactId>

SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java

@@ -4,12 +4,15 @@
 
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.datatype.guava.GuavaModule;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Module;
 import com.hubspot.dropwizard.guicier.GuiceBundle;
 import com.hubspot.jackson.datatype.protobuf.ProtobufModule;
 import com.hubspot.singularity.bundles.CorsBundle;
+import com.hubspot.singularity.config.MergingSourceProvider;
 import com.hubspot.singularity.config.SingularityConfiguration;
 
 import io.dropwizard.Application;
@@ -23,11 +26,15 @@
 import io.dropwizard.views.ViewBundle;
 
 public class SingularityService<T extends SingularityConfiguration> extends Application<T> {
+  private static final String SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY = "singularityDefaultConfiguration";
 
   public static final String API_BASE_PATH = "/api";
 
   @Override
   public void initialize(final Bootstrap<T> bootstrap) {
+    if (!Strings.isNullOrEmpty(System.getProperty(SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY))) {
+      bootstrap.setConfigurationSourceProvider(new MergingSourceProvider(bootstrap.getConfigurationSourceProvider(), System.getProperty(SINGULARITY_DEFAULT_CONFIGURATION_PROPERTY), bootstrap.getObjectMapper(), new YAMLFactory()));
+    }
 
     final Iterable<? extends Module> additionalModules = checkNotNull(getGuiceModules(bootstrap), "getGuiceModules() returned null");
     final Iterable<? extends Bundle> additionalBundles = checkNotNull(getDropwizardBundles(bootstrap), "getDropwizardBundles() returned null");

SingularityService/src/main/java/com/hubspot/singularity/config/MergingSourceProvider.java

@@ -0,0 +1,67 @@
+package com.hubspot.singularity.config;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+import io.dropwizard.configuration.ConfigurationSourceProvider;
+
+public class MergingSourceProvider implements ConfigurationSourceProvider {
+    private final ConfigurationSourceProvider delegate;
+    private final String defaultConfigurationPath;
+    private final ObjectMapper objectMapper;
+    private final YAMLFactory yamlFactory;
+
+    public MergingSourceProvider(ConfigurationSourceProvider delegate, String defaultConfigurationPath, ObjectMapper objectMapper, YAMLFactory yamlFactory) {
+        this.delegate = delegate;
+        this.defaultConfigurationPath = defaultConfigurationPath;
+        this.objectMapper = objectMapper;
+        this.yamlFactory = yamlFactory;
+    }
+
+    @Override
+    public InputStream open(String path) throws IOException {
+        final JsonNode originalNode = objectMapper.readTree(yamlFactory.createParser(delegate.open(defaultConfigurationPath)));
+        final JsonNode overrideNode = objectMapper.readTree(yamlFactory.createParser(delegate.open(path)));
+
+        if (!(originalNode instanceof ObjectNode && overrideNode instanceof ObjectNode)) {
+            throw new SingularityConfigurationMergeException(String.format("Both %s and %s need to be YAML objects", defaultConfigurationPath, path));
+        }
+
+        merge((ObjectNode)originalNode, (ObjectNode)overrideNode);
+
+        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        objectMapper.writeTree(yamlFactory.createGenerator(baos), originalNode);
+
+        return new ByteArrayInputStream(baos.toByteArray());
+    }
+
+    private static void merge(ObjectNode to, ObjectNode from) {
+        Iterator<String> newFieldNames = from.fieldNames();
+
+        while (newFieldNames.hasNext()) {
+            String newFieldName = newFieldNames.next();
+            JsonNode oldVal = to.get(newFieldName);
+            JsonNode newVal = from.get(newFieldName);
+
+            if (oldVal == null || oldVal.isNull()) {
+                to.put(newFieldName, newVal);
+            } else if (oldVal.isArray() && newVal.isArray()) {
+                ((ArrayNode) oldVal).removeAll();
+                ((ArrayNode) oldVal).addAll((ArrayNode) newVal);
+            } else if (oldVal.isObject() && newVal.isObject()) {
+                merge((ObjectNode) oldVal, (ObjectNode) newVal);
+            } else if (!(newVal == null || newVal.isNull())) {
+                to.put(newFieldName, newVal);
+            }
+        }
+    }
+}

SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfigurationMergeException.java

@@ -0,0 +1,7 @@
+package com.hubspot.singularity.config;
+
+public class SingularityConfigurationMergeException extends RuntimeException {
+    public SingularityConfigurationMergeException(String message) {
+        super(message);
+    }
+}

SingularityService/src/test/java/com/hubspot/singularity/config/MergingSourceProviderTest.java

@@ -0,0 +1,66 @@
+package com.hubspot.singularity.config;
+
+import static junit.framework.TestCase.assertEquals;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.inject.Inject;
+import com.hubspot.singularity.SingularityTestBaseNoDb;
+
+import io.dropwizard.configuration.ConfigurationSourceProvider;
+
+public class MergingSourceProviderTest extends SingularityTestBaseNoDb {
+    private static final String DEFAULT_PATH = "/configs/default.yaml";
+    private static final String OVERRIDE_PATH = "/configs/override.yaml";
+    private static final String JUST_A_STRING_PATH = "/configs/just_a_string.yaml";
+    private static final String DOESNT_EXIST_PATH = "/configs/doesnt_exist.yaml";
+
+    private static final YAMLFactory YAML_FACTORY = new YAMLFactory();
+
+    @Inject
+    private ObjectMapper objectMapper;
+
+    private ConfigurationSourceProvider buildConfigurationSourceProvider(String baseFilename) {
+        final Class<?> klass = getClass();
+
+        return new MergingSourceProvider(new ConfigurationSourceProvider() {
+            @Override
+            public InputStream open(String path) throws IOException {
+                final InputStream stream = klass.getResourceAsStream(path);
+                if (stream == null) {
+                    throw new FileNotFoundException("File " + path + " not found in test resources directory");
+                }
+                return stream;
+            }
+        }, baseFilename, objectMapper, YAML_FACTORY);
+    }
+
+    @Test
+    public void testMergedConfigs() throws Exception {
+        final InputStream mergedConfigStream = buildConfigurationSourceProvider(DEFAULT_PATH).open(OVERRIDE_PATH);
+        final SingularityConfiguration mergedConfig = objectMapper.readValue(YAML_FACTORY.createParser(mergedConfigStream), SingularityConfiguration.class);
+
+        assertEquals(10000, mergedConfig.getCacheTasksMaxSize());
+        assertEquals(500, mergedConfig.getCacheTasksInitialSize());
+        assertEquals(100, mergedConfig.getCheckDeploysEverySeconds());
+        assertEquals("baseuser", mergedConfig.getDatabaseConfiguration().get().getUser());
+        assertEquals("overridepassword", mergedConfig.getDatabaseConfiguration().get().getPassword());
+    }
+
+    @Test( expected = SingularityConfigurationMergeException.class )
+    public void testNonObjectFails() throws Exception {
+        buildConfigurationSourceProvider(DEFAULT_PATH).open(JUST_A_STRING_PATH);
+    }
+
+    @Test( expected = FileNotFoundException.class)
+    public void testFileNoExistFail() throws Exception {
+        buildConfigurationSourceProvider(DEFAULT_PATH).open(DOESNT_EXIST_PATH);
+        buildConfigurationSourceProvider(DOESNT_EXIST_PATH).open(OVERRIDE_PATH);
+    }
+}

SingularityService/src/test/resources/configs/default.yaml

@@ -0,0 +1,6 @@
+cacheTasksMaxSize: 10000  # override.yaml will not touch this
+cacheTasksInitialSize: 200  # override.yaml will override this to 500
+# override.yaml will override checkDeploysEverySeconds to 100
+
+database:  # override.yaml will override some of this
+  user: "baseuser"

SingularityService/src/test/resources/configs/just_a_string.yaml

@@ -0,0 +1 @@
+"this is just a string!"

SingularityService/src/test/resources/configs/override.yaml

@@ -0,0 +1,5 @@
+cacheTasksInitialSize: 500
+checkDeploysEverySeconds: 100
+
+database:
+  password: "overridepassword"