DAOS-16298 test: improve get_clush_command timeout

src/tests/ftest/recovery/ddb.py

@@ -90,23 +90,21 @@ def copy_remote_to_local(remote_file_path, test_dir, remote):
     # Use clush --rcopy to copy the file from the remote server node to the local test
     # node. clush will append .<server_hostname> to the file when copying.
     args = "--rcopy {} --dest {}".format(remote_file_path, test_dir)
-    clush_command = get_clush_command(hosts=remote, args=args)
+    clush_command = get_clush_command(hosts=remote, args=args, timeout=60)
     try:
-        run_command(command=clush_command)
+        run_command(command=clush_command, timeout=None)
     except DaosTestError as error:
-        print("ERROR: Copying {} from {}: {}".format(remote_file_path, remote, error))
-        raise error
+        raise DaosTestError(
+            f"ERROR: Copying {remote_file_path} from {remote}: {error}") from error
 
     # Remove the appended .<server_hostname> from the copied file.
     current_file_path = "".join([remote_file_path, ".", remote])
     mv_command = "mv {} {}".format(current_file_path, remote_file_path)
     try:
         run_command(command=mv_command)
     except DaosTestError as error:
-        print(
-            "ERROR: Moving {} to {}: {}".format(
-                current_file_path, remote_file_path, error))
-        raise error
+        raise DaosTestError(
+            f"ERROR: Moving {current_file_path} to {remote_file_path}: {error}") from error
 
 
 class DdbTest(RecoveryTestBase):

src/tests/ftest/slurm_setup.py

@@ -145,8 +145,9 @@ def start_munge(self, user):
         non_control = self.nodes.difference(self.control)
         self.log.debug('Copying the munge key to %s', non_control)
         command = get_clush_command(
-            non_control, args=f"-B -S -v --copy {self.MUNGE_KEY} --dest {self.MUNGE_KEY}")
-        result = run_remote(self.log, self.control, command)
+            non_control, args=f"-B -S -v --copy {self.MUNGE_KEY} --dest {self.MUNGE_KEY}",
+            timeout=60)
+        result = run_remote(self.log, self.control, command, timeout=None)
         if not result.passed:
             raise SlurmSetupException(f'Error creating munge key on {result.failed_hosts}')
 

src/tests/ftest/util/run_utils.py

@@ -345,7 +345,8 @@ def log_result_data(log, data):
             log.debug("%s%s", " " * indent, line)
 
 
-def get_clush_command(hosts, args=None, command="", command_env=None, command_sudo=False):
+def get_clush_command(hosts, args=None, command="", command_env=None, command_sudo=False,
+                      timeout=None, fanout=None):
     """Get the clush command with optional sudo arguments.
 
     Args:
@@ -355,14 +356,21 @@ def get_clush_command(hosts, args=None, command="", command_env=None, command_su
         command_env (EnvironmentVariables, optional): environment variables to export with the
             command. Defaults to None.
         sudo (bool, optional): whether to run the command with sudo privileges. Defaults to False.
+        timeout (int, optional): number of seconds to wait for the command to complete.
+            Defaults to None.
+        fanout (int, optional): fanout to use. Default uses the max of the
+            clush default (64) or available cores
 
     Returns:
         str: the clush command
     """
-    cmd_list = ["clush"]
+    if fanout is None:
+        fanout = max(64, len(os.sched_getaffinity(0)))
+    cmd_list = ["clush", "-f", str(fanout), "-w", str(hosts)]
+    if timeout is not None:
+        cmd_list.extend(["-u", str(timeout)])
     if args:
         cmd_list.append(args)
-    cmd_list.extend(["-w", str(hosts)])
     # If ever needed, this is how to disable host key checking:
     # cmd_list.extend(["-o", "-oStrictHostKeyChecking=no"])
     cmd_list.append(command_as_user(command, "root" if command_sudo else "", command_env))