#4953: Stop processes when releasing nodes or destroying universes

Summary: Stop the master/tserver on delete universe and release instance, so that if they are brought up again, the processes won't be still running. Test Plan: Destroy onprem universe. Reviewers: bogdan, arnav, sanketh Reviewed By: sanketh Subscribers: kannan, jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D9097
yugabyte · Sep 8, 2020 · 9b11751 · 9b11751
1 parent bb5bd13
commit 9b11751
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 5 deletions.
diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/DestroyUniverse.java
@@ -14,6 +14,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.yugabyte.yw.commissioner.Common.CloudType;
 import com.yugabyte.yw.commissioner.SubTaskGroup;
 import com.yugabyte.yw.commissioner.SubTaskGroupQueue;
 import com.yugabyte.yw.commissioner.UserTaskDetails.SubTaskGroupType;
@@ -62,6 +63,14 @@ public void run() {
                                   primaryCluster.userIntent.universeName)
             .setSubTaskGroupType(SubTaskGroupType.RemovingUnusedServers);
 
+        if (primaryCluster.userIntent.providerType.equals(CloudType.onprem)) {
+          // Stop master and tservers.
+          createStopServerTasks(universe.getNodes(), "master", params().isForceDelete)
+              .setSubTaskGroupType(SubTaskGroupType.StoppingNodeProcesses);
+          createStopServerTasks(universe.getNodes(), "tserver", params().isForceDelete)
+              .setSubTaskGroupType(SubTaskGroupType.StoppingNodeProcesses);
+        }
+
         // Create tasks to destroy the existing nodes.
         createDestroyServerTasks(universe.getNodes(), params().isForceDelete, true)
             .setSubTaskGroupType(SubTaskGroupType.RemovingUnusedServers);

diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReleaseInstanceFromUniverse.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/ReleaseInstanceFromUniverse.java
@@ -86,6 +86,14 @@ public void run() {
           .setSubTaskGroupType(SubTaskGroupType.ReleasingInstance);
 
       if (instanceExists(taskParams())) {
+        if (userIntent.providerType.equals(CloudType.onprem)) {
+          // Stop master and tservers.
+          createStopServerTasks(universe.getNodes(), "master", false)
+              .setSubTaskGroupType(SubTaskGroupType.StoppingNodeProcesses);
+          createStopServerTasks(universe.getNodes(), "tserver", false)
+              .setSubTaskGroupType(SubTaskGroupType.StoppingNodeProcesses);
+        }
+
         // Create tasks to terminate that instance. Force delete and ignore errors.
         createDestroyServerTasks(new HashSet<NodeDetails>(Arrays.asList(currentNode)), true, false)
             .setSubTaskGroupType(SubTaskGroupType.ReleasingInstance);

diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java
@@ -804,6 +804,17 @@ public SubTaskGroup createStartMasterTasks(Collection<NodeDetails> nodes) {
    * @return
    */
   public SubTaskGroup createStopMasterTasks(Collection<NodeDetails> nodes) {
+    return createStopServerTasks(nodes, "master", false);
+  }
+
+  /**
+   * Creates a task list to stop the tservers of the cluster and adds it to the task queue.
+   * @param nodes set of nodes to be stopped as master
+   * @return
+   */
+  public SubTaskGroup createStopServerTasks(Collection<NodeDetails> nodes,
+                                            String serverType,
+                                            boolean isForceDelete) {
     SubTaskGroup subTaskGroup = new SubTaskGroup("AnsibleClusterServerCtl", executor);
     for (NodeDetails node : nodes) {
       AnsibleClusterServerCtl.Params params = new AnsibleClusterServerCtl.Params();
@@ -814,10 +825,11 @@ public SubTaskGroup createStopMasterTasks(Collection<NodeDetails> nodes) {
       // Add the az uuid.
       params.azUuid = node.azUuid;
       // The service and the command we want to run.
-      params.process = "master";
+      params.process = serverType;
       params.command = "stop";
       // Set the InstanceType
       params.instanceType = node.cloudInfo.instance_type;
+      params.isForceDelete = isForceDelete;
       // Create the Ansible task to get the server info.
       AnsibleClusterServerCtl task = new AnsibleClusterServerCtl();
       task.initialize(params);

diff --git a/...ed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/AnsibleClusterServerCtl.java b/...ed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/AnsibleClusterServerCtl.java
@@ -25,6 +25,7 @@ public static class Params extends NodeTaskParams {
     public String process;
     public String command;
     public int sleepAfterCmdMills = 0;
+    public boolean isForceDelete = false;
   }
 
   @Override
@@ -40,10 +41,16 @@ public String getName() {
 
   @Override
   public void run() {
-    // Execute the ansible command.
-    ShellProcessHandler.ShellResponse response = getNodeManager().nodeCommand(
-        NodeManager.NodeCommandType.Control, taskParams());
-    logShellResponse(response);
+    try {
+      // Execute the ansible command.
+      ShellProcessHandler.ShellResponse response = getNodeManager().nodeCommand(
+          NodeManager.NodeCommandType.Control, taskParams());
+      logShellResponse(response);
+    } catch (Exception e) {
+      if (!taskParams().isForceDelete) {
+        throw e;
+      }
+    }
 
     if (taskParams().sleepAfterCmdMills > 0) {
       try {