Merge branch 'develop' into fix_archive_timestamp

azurelinuxagent/common/osutil/default.py

@@ -677,7 +677,11 @@ def mount(self, device, mount_point, option="", chk_err=True):
         return retcode, err
 
     def umount(self, mount_point, chk_err=True):
-        return shellutil.run("umount {0}".format(mount_point), chk_err=chk_err)
+        try:
+            shellutil.run_command(["umount", mount_point], log_error=chk_err)
+        except shellutil.CommandError as cmd_err:
+            return cmd_err.returncode
+        return 0
 
     def allow_dhcp_broadcast(self):
         # Open DHCP port if iptables is enabled.
@@ -1112,15 +1116,25 @@ def set_dhcp_hostname(self, hostname):
     def restart_if(self, ifname, retries=3, wait=5):
         retry_limit=retries+1
         for attempt in range(1, retry_limit):
-            return_code=shellutil.run("ifdown {0} && ifup {0}".format(ifname), expected_errors=[1] if attempt < retries else [])
-            if return_code == 0:
+            try:
+                shellutil.run_command(["ifdown", ifname])
+                shellutil.run_command(["ifup", ifname])
                 return
-            logger.warn("failed to restart {0}: return code {1}".format(ifname, return_code))
-            if attempt < retry_limit:
-                logger.info("retrying in {0} seconds".format(wait))
-                time.sleep(wait)
-            else:
-                logger.warn("exceeded restart retries")
+            except shellutil.CommandError as cmd_err:
+
+                msg = "failed to restart {0}: returncode={1}\n[stdout]{2}\n\n[stderr]{3}\n"\
+                    .format(ifname, cmd_err.returncode, cmd_err.stdout, cmd_err.stderr)
+
+                if cmd_err.returncode == 1:
+                    logger.info(msg)
+                else:
+                    logger.warn(msg)
+
+                if attempt < retry_limit:
+                    logger.info("retrying in {0} seconds".format(wait))
+                    time.sleep(wait)
+                else:
+                    logger.warn("exceeded restart retries")
 
     def publish_hostname(self, hostname):
         self.set_dhcp_hostname(hostname)

azurelinuxagent/common/osutil/ubuntu.py

@@ -39,10 +39,18 @@ def start_network(self):
         return shellutil.run("service networking start", chk_err=False)
 
     def stop_agent_service(self):
-        return shellutil.run("service {0} stop".format(self.service_name), chk_err=False)
+        try:
+            shellutil.run_command(["service", self.service_name, "stop"])
+        except shellutil.CommandError as cmd_err:
+            return cmd_err.returncode
+        return 0
 
     def start_agent_service(self):
-        return shellutil.run("service {0} start".format(self.service_name), chk_err=False)
+        try:
+            shellutil.run_command(["service", self.service_name, "start"])
+        except shellutil.CommandError as cmd_err:
+            return cmd_err.returncode
+        return 0
 
     def remove_rules_files(self, rules_files=""):
         pass
@@ -119,15 +127,17 @@ def restart_if(self, ifname, retries=3, wait=5):
         """
         retry_limit=retries+1
         for attempt in range(1, retry_limit):
-            return_code=shellutil.run("ip link set {0} down && ip link set {0} up".format(ifname))
-            if return_code == 0:
-                return
-            logger.warn("failed to restart {0}: return code {1}".format(ifname, return_code))
-            if attempt < retry_limit:
-                logger.info("retrying in {0} seconds".format(wait))
-                time.sleep(wait)
-            else:
-                logger.warn("exceeded restart retries")
+            try:
+                shellutil.run_command(["ip", "link", "set", ifname, "down"])
+                shellutil.run_command(["ip", "link", "set", ifname, "up"])
+
+            except shellutil.CommandError as cmd_err:
+                logger.warn("failed to restart {0}: return code {1}".format(ifname, cmd_err.returncode))
+                if attempt < retry_limit:
+                    logger.info("retrying in {0} seconds".format(wait))
+                    time.sleep(wait)
+                else:
+                    logger.warn("exceeded restart retries")
 
 
 class UbuntuSnappyOSUtil(Ubuntu14OSUtil):

azurelinuxagent/common/utils/cryptutil.py

@@ -41,12 +41,14 @@ def gen_transport_cert(self, prv_file, crt_file):
         """
         Create ssl certificate for https communication with endpoint server.
         """
-        cmd = ("{0} req -x509 -nodes -subj /CN=LinuxTransport -days 730 "
-               "-newkey rsa:2048 -keyout {1} "
-               "-out {2}").format(self.openssl_cmd, prv_file, crt_file)
-        rc = shellutil.run(cmd) # pylint: disable=C0103
-        if rc != 0:
-            logger.error("Failed to create {0} and {1} certificates".format(prv_file, crt_file))
+        cmd = [self.openssl_cmd, "req", "-x509", "-nodes", "-subj", "/CN=LinuxTransport", 
+            "-days", "730", "-newkey", "rsa:2048", "-keyout", prv_file, "-out", crt_file]
+        try:
+            shellutil.run_command(cmd)
+        except shellutil.CommandError as cmd_err:
+            msg = "Failed to create {0} and {1} certificates.\n[stdout]\n{2}\n\n[stderr]\n{3}\n"\
+                .format(prv_file, crt_file, cmd_err.stdout, cmd_err.stderr)
+            logger.error(msg)
 
     def get_pubkey_from_prv(self, file_name):
         if not os.path.exists(file_name): # pylint: disable=R1720
@@ -79,18 +81,44 @@ def decrypt_p7m(self, p7m_file, trans_prv_file, trans_cert_file, pem_file):
         elif not os.path.exists(trans_prv_file):
             raise IOError(errno.ENOENT, "File not found", trans_prv_file)
         else:
-            cmd = ("{0} cms -decrypt -in {1} -inkey {2} -recip {3} "
-                   "| {4} pkcs12 -nodes -password pass: -out {5}"
-                   "").format(self.openssl_cmd, p7m_file, trans_prv_file,
-                              trans_cert_file, self.openssl_cmd, pem_file)
-            shellutil.run(cmd)
-            rc = shellutil.run(cmd) # pylint: disable=C0103
-            if rc != 0:
-                logger.error("Failed to decrypt {0}".format(p7m_file))
+            first_cmd = [self.openssl_cmd, "cms", "-decrypt", "-in", p7m_file, "-inkey",
+                trans_prv_file, "-recip", trans_cert_file]
+            second_cmd = [self.openssl_cmd, "pkcs12", "-nodes", "-password", "pass:", 
+                "-out", pem_file]
+
+            first_proc = subprocess.Popen(first_cmd, stdout=subprocess.PIPE)
+
+            second_proc = subprocess.Popen(second_cmd, stdin=first_proc.stdout, stdout=subprocess.PIPE)
+            first_proc.stdout.close()  # see https://docs.python.org/2/library/subprocess.html#replacing-shell-pipeline
+            stdout, stderr = second_proc.communicate()
+
+            if second_proc.returncode != 0:
+                stdout = ustr(stdout, encoding='utf-8', errors="backslashreplace") if stdout else ""
+                stderr =  ustr(stderr, encoding='utf-8', errors="backslashreplace") if stderr else ""
+
+                msg = "Failed to decrypt {0}\n[stdout]\n{1}\n\n[stderr]\n{2}\n"\
+                    .format(p7m_file, stdout, stderr)
+                logger.error(msg)
+
 
     def crt_to_ssh(self, input_file, output_file):
-        shellutil.run("ssh-keygen -i -m PKCS8 -f {0} >> {1}".format(input_file,
-                                                                    output_file))
+        with open(output_file, "ab") as file_out:
+            cmd = ["ssh-keygen", "-i", "-m", "PKCS8", "-f", input_file]
+
+            try:
+                keygen_proc = subprocess.Popen(cmd, stdout=file_out)
+                keygen_proc.wait()
+
+                if keygen_proc.returncode != 0:
+                    msg = u"Command: [{0}], return code: [{1}]" \
+                        .format(cmd, keygen_proc.returncode)
+                    logger.error(msg)
+
+            except Exception as exception:
+                msg = u"Exception on Command: [{0}]. exception={1}" \
+                    .format(cmd, exception)
+
+
 
     def asn1_to_ssh(self, pubkey):
         lines = pubkey.split("\n")

tests/common/osutil/test_default.py

@@ -56,16 +56,22 @@ def test_restart(self):
         # setup
         retries = 3
         ifname = 'dummy'
-        with patch.object(shellutil, "run") as run_patch:
-            run_patch.return_value = 1
+        with patch.object(shellutil, "run_command") as run_patch:
+            run_patch.side_effect = shellutil.CommandError("ifupdown dummy", 1, "", "")
 
             # execute
             osutil.DefaultOSUtil.restart_if(osutil.DefaultOSUtil(), ifname=ifname, retries=retries, wait=0)
 
             # assert
             self.assertEqual(run_patch.call_count, retries)
-            self.assertEqual(run_patch.call_args_list[0][0][0], 'ifdown {0} && ifup {0}'.format(ifname))
 
+            cmd_queue = list(args[0] for (args, _) in run_patch.call_args_list)            
+            while cmd_queue:
+                self.assertEqual(cmd_queue.pop(0), ["ifdown", ifname])
+                # We don't expect the following command to be called because 'dummy' does
+                # not exist.
+                self.assertNotEqual(cmd_queue[0] if cmd_queue else None, ["ifup", ifname])
+
     def test_get_dvd_device_success(self):
         with patch.object(os, 'listdir', return_value=['cpu', 'cdrom0']):
             osutil.DefaultOSUtil().get_dvd_device()