prevent pending deploys from being created without corresponding save…

…d deploy
HubSpot · Nov 9, 2021 · 1747910 · 1747910
1 parent 22e0dea
commit 1747910
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 2 deletions.
diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/CuratorAsyncManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/CuratorAsyncManager.java
@@ -122,6 +122,8 @@ public void processResult(CuratorFramework client, CuratorEvent event)
           if (cache.isPresent()) {
             cache.get().set(event.getPath(), object);
           }
+        } catch (Exception e) {
+          LOG.error("Exception processing curator result", e);
         } finally {
           latch.countDown();
         }

diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/DeployManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/DeployManager.java
@@ -276,12 +276,12 @@ public SingularityDeployKey apply(SingularityDeploy input) {
     return deployKeyToDeploy;
   }
 
-  public SingularityCreateResult saveDeploy(
+  public SingularityCreateResult createDeployIfNotExists(
     SingularityRequest request,
     SingularityDeployMarker deployMarker,
     SingularityDeploy deploy
   ) {
-    final SingularityCreateResult deploySaveResult = create(
+    SingularityCreateResult deploySaveResult = create(
       getDeployDataPath(deploy.getRequestId(), deploy.getId()),
       deploy,
       deployTranscoder
@@ -294,7 +294,19 @@ public SingularityCreateResult saveDeploy(
         deployMarker
       );
     }
+    return deploySaveResult;
+  }
 
+  public SingularityCreateResult saveDeploy(
+    SingularityRequest request,
+    SingularityDeployMarker deployMarker,
+    SingularityDeploy deploy
+  ) {
+    final SingularityCreateResult deploySaveResult = createDeployIfNotExists(
+      request,
+      deployMarker,
+      deploy
+    );
     LOG.info("Creating deploy {}", deployMarker);
 
     singularityEventListener.deployHistoryEvent(

diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java
@@ -264,6 +264,11 @@ public SingularityRequestParent deploy(
       // This can cause a conflict if run outside the lock, causing the pending deploy to be checked before deploy data is saved
       schedulerLock.runWithRequestLock(
         () -> {
+          deployManager.createDeployIfNotExists(
+            updatedRequest,
+            deployMarker,
+            validatedDeploy
+          );
           deployAlreadyInProgress.set(
             deployManager.createPendingDeploy(pendingDeployObj) ==
             SingularityCreateResult.EXISTED