Upgrade to mesos 1.6.1 and fork of mesos-rxjava

SingularityExecutor/Dockerfile

@@ -1,4 +1,4 @@
-FROM mesosphere/mesos:1.1.1
+FROM mesosphere/mesos-slave:1.6.1-rc2
 ## mesos + java used to build hubspot/singularitybase
 
 MAINTAINER platform-infrastructure-groups@hubspot.com

SingularityExecutor/src/main/resources/runner.sh.hbs

@@ -4,7 +4,11 @@
 # Task ID: {{{ taskId }}}
 #
 
-export MESOS_TASK_ID={{{bashEscaped taskId}}}  # I can't believe Mesos doesn't expose this.
+export MESOS_TASK_ID=$TASK_ID
+
+# MESOS-8322 changed this directory to 0750, chown owner since we can switch user either in mesos or the executor.
+# Otherwise, when switching user in the executor only, the non-root user can no longer traverse the sandbox
+chown {{{ user }}} $MESOS_SANDBOX
 
 {{#if cfsQuota }}
 # Outputs the path of the deepest cgroup the current process is in.

SingularityS3Uploader/pom.xml

@@ -12,7 +12,6 @@
 
   <properties>
     <basepom.shaded.main-class>com.hubspot.singularity.s3uploader.SingularityS3UploaderRunner</basepom.shaded.main-class>
-    <dep.protobuf.version>3.5.1</dep.protobuf.version>
   </properties>
 
   <dependencies>

SingularityService/pom.xml

@@ -128,17 +128,17 @@
     </dependency>
 
     <dependency>
-      <groupId>com.mesosphere.mesos.rx.java</groupId>
+      <groupId>com.hubspot.mesos.rx.java</groupId>
       <artifactId>mesos-rxjava-client</artifactId>
     </dependency>
 
     <dependency>
-      <groupId>com.mesosphere.mesos.rx.java</groupId>
+      <groupId>com.hubspot.mesos.rx.java</groupId>
       <artifactId>mesos-rxjava-protobuf-client</artifactId>
     </dependency>
 
     <dependency>
-      <groupId>com.mesosphere.mesos.rx.java</groupId>
+      <groupId>com.hubspot.mesos.rx.java</groupId>
       <artifactId>mesos-rxjava-util</artifactId>
     </dependency>
 

SingularityService/src/main/java/com/hubspot/singularity/data/SingularityDataModule.java

@@ -30,6 +30,8 @@ protected void configure() {
     bind(UserManager.class).in(Scopes.SINGLETON);
     bind(UsageManager.class).in(Scopes.SINGLETON);
 
+    bind(NotificationsManager.class).in(Scopes.SINGLETON);
+
     bind(SingularityWebCache.class).in(Scopes.SINGLETON);
 
     bind(ExecutorIdGenerator.class).in(Scopes.SINGLETON);

SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerClient.java

@@ -37,13 +37,13 @@
 import com.hubspot.singularity.config.SingularityConfiguration;
 import com.hubspot.singularity.config.UIConfiguration;
 import com.hubspot.singularity.resources.ui.UiResource;
-import com.mesosphere.mesos.rx.java.AwaitableSubscription;
-import com.mesosphere.mesos.rx.java.MesosClient;
-import com.mesosphere.mesos.rx.java.MesosClientBuilder;
-import com.mesosphere.mesos.rx.java.SinkOperation;
-import com.mesosphere.mesos.rx.java.SinkOperations;
-import com.mesosphere.mesos.rx.java.protobuf.ProtobufMesosClientBuilder;
-import com.mesosphere.mesos.rx.java.util.UserAgentEntries;
+import com.hubspot.mesos.rx.java.AwaitableSubscription;
+import com.hubspot.mesos.rx.java.MesosClient;
+import com.hubspot.mesos.rx.java.MesosClientBuilder;
+import com.hubspot.mesos.rx.java.SinkOperation;
+import com.hubspot.mesos.rx.java.SinkOperations;
+import com.hubspot.mesos.rx.java.protobuf.ProtobufMesosClientBuilder;
+import com.hubspot.mesos.rx.java.util.UserAgentEntries;
 
 import rx.BackpressureOverflow;
 import rx.Observable;
@@ -152,6 +152,7 @@ private void connect(URI mesosMasterURI, FrameworkInfo frameworkInfo, Singularit
     MesosClientBuilder<Call, Event> clientBuilder = ProtobufMesosClientBuilder.schedulerUsingProtos()
         .mesosUri(mesosMasterURI)
         .applicationUserAgentEntry(UserAgentEntries.userAgentEntryForMavenArtifact("com.hubspot.singularity", "SingularityService"))
+        .onSendEventBackpressureBuffer()
         .onBackpressureBuffer(
             scheduler.getEventBufferSize(),
             () -> {

compose-dev.yml

@@ -6,7 +6,7 @@ zk:
     ZK_ID: 1
 
 master:
-  image: mesosphere/mesos-master:1.1.1
+  image: mesosphere/mesos-master:1.6.1-rc2
   net: host
   environment:
     MESOS_ZK: zk://localhost:2181/mesos
@@ -31,6 +31,7 @@ slave:
     MESOS_ATTRIBUTES: "example:value;myNumber:1"
     MESOS_ISOLATION: cgroups/cpu,cgroups/mem
     MESOS_WORK_DIR: /var/lib/mesos/agent
+    MESOS_SYSTEMD_ENABLE_SUPPORT: "false"
   volumes:
     - /var/run/docker.sock:/var/run/docker.sock
     - /sys:/sys

pom.xml

@@ -61,21 +61,22 @@
     <dep.jackson-databind.version>2.9.4</dep.jackson-databind.version>
     <dep.jackson.version>2.9.4</dep.jackson.version>
     <dep.javassist.version>3.21.0-GA</dep.javassist.version>
-    <dep.mesos.rxjava.version>0.2.0</dep.mesos.rxjava.version>
+    <dep.mesos.rxjava.version>0.1.0</dep.mesos.rxjava.version>
+    <dep.rxJava.version>1.3.8</dep.rxJava.version>
     <dep.metrics-guice.version>3.1.3</dep.metrics-guice.version>
     <dep.netty.version>4.1.22.Final</dep.netty.version>
-    <dep.protobuf.version>2.6.1</dep.protobuf.version>
+    <dep.protobuf-java.version>3.5.1</dep.protobuf-java.version>
     <dep.reflections.version>0.9.11</dep.reflections.version>
     <dep.swagger.version>2.0.0</dep.swagger.version>
 
     <dropwizard.guicier.version>1.0.6.0</dropwizard.guicier.version>
     <horizon.version>0.0.25</horizon.version>
     <jukito.version>1.5</jukito.version>
-    <mesos.docker.tag>1.1.1</mesos.docker.tag>
-    <mesos.version>1.1.2</mesos.version>
+    <mesos.docker.tag>1.6.1</mesos.docker.tag>
+    <mesos.version>1.6.1</mesos.version>
     <project.build.targetJdk>1.8</project.build.targetJdk>
     <singularity.jar.name.format>${project.artifactId}-${project.version}</singularity.jar.name.format>
-    <singularitybase.image.revision>1</singularitybase.image.revision>
+    <singularitybase.image.revision>2</singularitybase.image.revision>
   </properties>
 
   <dependencyManagement>
@@ -149,7 +150,7 @@
       <dependency>
         <groupId>com.google.protobuf</groupId>
         <artifactId>protobuf-java</artifactId>
-        <version>${dep.protobuf.version}</version>
+        <version>${dep.protobuf-java.version}</version>
       </dependency>
 
       <dependency>
@@ -244,7 +245,7 @@
       <dependency>
         <groupId>com.hubspot.jackson</groupId>
         <artifactId>jackson-datatype-protobuf</artifactId>
-        <version>0.9.9-jackson2.9-proto2</version>
+        <version>0.9.10-jackson2.9-proto3</version>
       </dependency>
 
       <dependency>
@@ -254,19 +255,19 @@
       </dependency>
 
       <dependency>
-        <groupId>com.mesosphere.mesos.rx.java</groupId>
+        <groupId>com.hubspot.mesos.rx.java</groupId>
         <artifactId>mesos-rxjava-client</artifactId>
         <version>${dep.mesos.rxjava.version}</version>
       </dependency>
 
       <dependency>
-        <groupId>com.mesosphere.mesos.rx.java</groupId>
+        <groupId>com.hubspot.mesos.rx.java</groupId>
         <artifactId>mesos-rxjava-protobuf-client</artifactId>
         <version>${dep.mesos.rxjava.version}</version>
       </dependency>
 
       <dependency>
-        <groupId>com.mesosphere.mesos.rx.java</groupId>
+        <groupId>com.hubspot.mesos.rx.java</groupId>
         <artifactId>mesos-rxjava-util</artifactId>
         <version>${dep.mesos.rxjava.version}</version>
       </dependency>
@@ -308,7 +309,7 @@
       <dependency>
         <groupId>io.reactivex</groupId>
         <artifactId>rxjava</artifactId>
-        <version>1.1.8</version>
+        <version>${dep.rxJava.version}</version>
       </dependency>
 
       <dependency>