azdev setup: show error if pip command fails

azdev/help.py

@@ -14,6 +14,7 @@
 
 helps['setup'] = """
     short-summary: Set up your environment for development of Azure CLI command modules and/or extensions.
+    long-summary: Use --verbose to show the commands that are run, --debug to show the command output.
     examples:
         - name: Fully interactive setup.
           text: azdev setup

azdev/operations/setup.py

@@ -15,7 +15,7 @@
     list_extensions, add_extension_repo, remove_extension)
 from azdev.params import Flag
 from azdev.utilities import (
-    display, heading, subheading, pip_cmd, find_file,
+    display, heading, subheading, pip_cmd, CommandError, find_file,
     get_azdev_config_dir, get_azdev_config, require_virtual_env, get_azure_config)
 
 logger = get_logger(__name__)
@@ -76,57 +76,68 @@ def _install_cli(cli_path, deps=None):
         whl_list = " ".join(
             [os.path.join(privates_dir, f) for f in os.listdir(privates_dir)]
         )
-        pip_cmd("install -q {}".format(whl_list), "Installing private whl files...")
+        pip_cmd("install {}".format(whl_list), "Installing private whl files...")
 
     # install general requirements
     pip_cmd(
-        "install -q -r {}/requirements.txt".format(cli_path),
+        "install -r {}".format(os.path.join(cli_path, "requirements.txt")),
         "Installing `requirements.txt`..."
     )
+
+    cli_src = os.path.join(cli_path, 'src')
     if deps == 'setup.py':
         # Resolve dependencies from setup.py files.
         # command modules have dependency on azure-cli-core so install this first
         pip_cmd(
-            "install -q -e {}/src/azure-cli-telemetry".format(cli_path),
+            "install -e {}".format(os.path.join(cli_src, 'azure-cli-telemetry')),
             "Installing `azure-cli-telemetry`..."
         )
         pip_cmd(
-            "install -q -e {}/src/azure-cli-core".format(cli_path),
+            "install -e {}".format(os.path.join(cli_src, 'azure-cli-core')),
             "Installing `azure-cli-core`..."
         )
 
         # azure cli has dependencies on the above packages so install this one last
-        pip_cmd("install -q -e {}/src/azure-cli".format(cli_path), "Installing `azure-cli`...")
         pip_cmd(
-            "install -q -e {}/src/azure-cli-testsdk".format(cli_path),
+            "install -e {}".format(os.path.join(cli_src, 'azure-cli')),
+            "Installing `azure-cli`..."
+        )
+
+        pip_cmd(
+            "install -e {}".format(os.path.join(cli_src, 'azure-cli-testsdk')),
             "Installing `azure-cli-testsdk`..."
         )
     else:
         # First install packages without dependencies,
         # then resolve dependencies from requirements.*.txt file.
         pip_cmd(
-            "install -e {}/src/azure-cli-telemetry --no-deps".format(cli_path),
+            "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli-telemetry')),
             "Installing `azure-cli-telemetry`..."
         )
         pip_cmd(
-            "install -e {}/src/azure-cli-core --no-deps".format(cli_path),
+            "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli-core')),
             "Installing `azure-cli-core`..."
         )
 
-        pip_cmd("install -e {}/src/azure-cli --no-deps".format(cli_path), "Installing `azure-cli`...")
+        pip_cmd(
+            "install -e {} --no-deps".format(os.path.join(cli_src, 'azure-cli')),
+            "Installing `azure-cli`..."
+        )
 
         # The dependencies of testsdk are not in requirements.txt as this package is not needed by the
         # azure-cli package for running commands.
         # Here we need to install with dependencies for azdev test.
         pip_cmd(
-            "install -e {}/src/azure-cli-testsdk".format(cli_path),
+            "install -e {}".format(os.path.join(cli_src, 'azure-cli-testsdk')),
             "Installing `azure-cli-testsdk`..."
         )
         import platform
         system = platform.system()
         req_file = 'requirements.py3.{}.txt'.format(system)
-        pip_cmd("install -r {}/src/azure-cli/{}".format(cli_path, req_file),
-                "Installing `{}`...".format(req_file))
+        pip_cmd(
+            "install -r {}".format(os.path.join(cli_src, 'azure-cli', req_file)),
+            "Installing `{}`...".format(req_file)
+        )
 
 
 def _copy_config_files():
@@ -311,11 +322,15 @@ def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None):
     # install packages
     subheading('Installing packages')
 
-    # upgrade to latest pip
-    pip_cmd('install --upgrade pip -q', 'Upgrading pip...')
+    try:
+        # upgrade to latest pip
+        pip_cmd('install --upgrade pip', 'Upgrading pip...')
+        _install_cli(cli_path, deps=deps)
+        _install_extensions(ext_to_install)
+    except CommandError as err:
+        logger.error(err)
+        return
 
-    _install_cli(cli_path, deps=deps)
-    _install_extensions(ext_to_install)
     _copy_config_files()
 
     end = time.time()

azdev/utilities/__init__.py

@@ -14,7 +14,8 @@
     call,
     cmd,
     py_cmd,
-    pip_cmd
+    pip_cmd,
+    CommandError
 )
 from .const import (
     COMMAND_MODULE_PREFIX,
@@ -67,6 +68,7 @@
     'cmd',
     'py_cmd',
     'pip_cmd',
+    'CommandError',
     'test_cmd',
     'get_env_path',
     'get_azure_config_dir',

azdev/utilities/command.py

@@ -14,6 +14,16 @@
 logger = get_logger(__name__)
 
 
+class CommandError(Exception):
+
+    def __init__(self, output, exit_code, command):
+        message = "Command `{}` failed with exit code {}:\n{}".format(command, exit_code, output)
+        self.exit_code = exit_code
+        self.output = output
+        self.command = command
+        super().__init__(message)
+
+
 def call(command, **kwargs):
     """ Run an arbitrary command but don't buffer the output.
 
@@ -27,12 +37,13 @@ def call(command, **kwargs):
         **kwargs)
 
 
-def cmd(command, message=False, show_stderr=True, **kwargs):
+def cmd(command, message=False, show_stderr=True, raise_error=False, **kwargs):
     """ Run an arbitrary command.
 
     :param command: The entire command line to run.
     :param message: A custom message to display, or True (bool) to use a default.
     :param show_stderr: On error, display the contents of STDERR.
+    :param raise_error: On error, raise CommandError.
     :param kwargs: Any kwargs supported by subprocess.Popen
     :returns: CommandResultItem object.
     """
@@ -45,23 +56,28 @@ def cmd(command, message=False, show_stderr=True, **kwargs):
     if message:
         display(message)
 
+    logger.info("Running: %s", command)
     try:
         output = subprocess.check_output(
             command.split(),
             stderr=subprocess.STDOUT if show_stderr else None,
             shell=IS_WINDOWS,
             **kwargs).decode('utf-8').strip()
+        logger.debug(output)
         return CommandResultItem(output, exit_code=0, error=None)
     except subprocess.CalledProcessError as err:
+        if raise_error:
+            raise CommandError(err.output.decode(), err.returncode, command)
         return CommandResultItem(err.output, exit_code=err.returncode, error=err)
 
 
-def py_cmd(command, message=False, show_stderr=True, is_module=True, **kwargs):
+def py_cmd(command, message=False, show_stderr=True, raise_error=False, is_module=True, **kwargs):
     """ Run a script or command with Python.
 
     :param command: The arguments to run python with.
     :param message: A custom message to display, or True (bool) to use a default.
     :param show_stderr: On error, display the contents of STDERR.
+    :param raise_error: On error, raise CommandError.
     :param is_module: Run a Python module as a script with -m.
     :param kwargs: Any kwargs supported by subprocess.Popen
     :returns: CommandResultItem object.
@@ -74,17 +90,19 @@ def py_cmd(command, message=False, show_stderr=True, is_module=True, **kwargs):
         command = '{} -m {}'.format(python_bin, command)
     else:
         command = '{} {}'.format(python_bin, command)
-    return cmd(command, message, show_stderr, **kwargs)
+    return cmd(command, message, show_stderr, raise_error, **kwargs)
 
 
-def pip_cmd(command, message=False, show_stderr=True, **kwargs):
+def pip_cmd(command, message=False, show_stderr=True, raise_error=True, **kwargs):
     """ Run a pip command.
 
     :param command: The arguments to run pip with.
     :param message: A custom message to display, or True (bool) to use a default.
     :param show_stderr: On error, display the contents of STDERR.
+    :param raise_error: On error, raise CommandError. As pip_cmd is usually called as a control function, instead of
+      a test target, default to True.
     :param kwargs: Any kwargs supported by subprocess.Popen
     :returns: CommandResultItem object.
     """
     command = 'pip {}'.format(command)
-    return py_cmd(command, message, show_stderr, **kwargs)
+    return py_cmd(command, message, show_stderr, raise_error, **kwargs)