From 9718c28d2da86d09b5be487f875e339c37a8a1ea Mon Sep 17 00:00:00 2001 From: jack1902 <39212456+jack1902@users.noreply.github.com> Date: Fri, 21 Feb 2020 22:29:29 +0000 Subject: [PATCH] [cli] Added additional commands to outputs (#1138) * [cli] Added additional commands to outputs 'set' - Set one output, using user_input (pass --update to overwrite existing outputs) 'get' - Get configured outputs for a service (includes creds) optionally pass --decriptors to only pull certain descriptor secrets for the service 'set-from-file' - Set numerous outputs via a json file. Can set multiple services and descriptors (pass --update to overwrite existing outputs) 'generate-skeleton' - Use to create a skeleton json file to be used with 'set-from-file' Signed-off-by: jack1902 <39212456+jack1902@users.noreply.github.com> * [cli] Added 'list' to outputs Also updated inline with PR comments Signed-off-by: jack1902 <39212456+jack1902@users.noreply.github.com> --- streamalert_cli/helpers.py | 62 ++-- streamalert_cli/outputs/handler.py | 475 +++++++++++++++++++++++++++-- streamalert_cli/outputs/helpers.py | 10 +- streamalert_cli/utils.py | 5 +- 4 files changed, 490 insertions(+), 62 deletions(-) diff --git a/streamalert_cli/helpers.py b/streamalert_cli/helpers.py index 3de731f0b..6cc7ef732 100644 --- a/streamalert_cli/helpers.py +++ b/streamalert_cli/helpers.py @@ -176,30 +176,9 @@ def user_input(requested_info, mask, input_restrictions): if not mask: while not response: - response = input(prompt) # nosec - - # Restrict having spaces or colons in items (applies to things like - # descriptors, etc) - if isinstance(input_restrictions, re.Pattern): - valid_response = input_restrictions.match(response) - if not valid_response: - LOGGER.error('The supplied input should match the following ' - 'regular expression: %s', input_restrictions.pattern) - elif callable(input_restrictions): - # Functions can be passed here to perform complex validation of input - # Transform the response with the validating function - response = input_restrictions(response) - valid_response = response is not None and response is not False - if not valid_response: - LOGGER.error('The supplied input failed to pass the validation ' - 'function: %s', input_restrictions.__doc__) - else: - valid_response = not any(x in input_restrictions for x in response) - if not valid_response: - restrictions = ', '.join( - '\'{}\''.format(restriction) for restriction in input_restrictions) - LOGGER.error('The supplied input should not contain any of the following: %s', - restrictions) + response = input(prompt) # nosec + + valid_response = response_is_valid(response, input_restrictions) if not valid_response: return user_input(requested_info, mask, input_restrictions) @@ -210,6 +189,41 @@ def user_input(requested_info, mask, input_restrictions): return response +def response_is_valid(response, input_restrictions): + """Check if the response meets the input_restrictions + + Args: + response (str): Description of the information needed + + Returns: + bool: True if input_restrictions are met else False + """ + valid_response = False + # Restrict having spaces or colons in items (applies to things like + # descriptors, etc) + if isinstance(input_restrictions, re.Pattern): + valid_response = input_restrictions.match(response) + if not valid_response: + LOGGER.error('The supplied input should match the following ' + 'regular expression: %s', input_restrictions.pattern) + elif callable(input_restrictions): + # Functions can be passed here to perform complex validation of input + # Transform the response with the validating function + response = input_restrictions(response) + valid_response = response is not None and response is not False + if not valid_response: + LOGGER.error('The supplied input failed to pass the validation ' + 'function: %s', input_restrictions.__doc__) + else: + valid_response = not any(x in input_restrictions for x in response) + if not valid_response: + restrictions = ', '.join( + '\'{}\''.format(restriction) for restriction in input_restrictions) + LOGGER.error('The supplied input should not contain any of the following: %s', + restrictions) + return valid_response + + def save_parameter(region, name, value, description, force_overwrite=False): """Function to save the designated value to parameter store diff --git a/streamalert_cli/outputs/handler.py b/streamalert_cli/outputs/handler.py index e51a6734d..e3e65bca6 100644 --- a/streamalert_cli/outputs/handler.py +++ b/streamalert_cli/outputs/handler.py @@ -13,43 +13,61 @@ See the License for the specific language governing permissions and limitations under the License. """ +import json + from streamalert.shared.logger import get_logger from streamalert.alert_processor.outputs.output_base import ( StreamAlertOutput, OutputCredentialsProvider ) -from streamalert_cli.helpers import user_input +from streamalert_cli.helpers import user_input, response_is_valid from streamalert_cli.outputs.helpers import output_exists -from streamalert_cli.utils import CLICommand +from streamalert_cli.utils import CLICommand, generate_subparser LOGGER = get_logger(__name__) +OUTPUTS_FILE = 'outputs_to_configure.json' class OutputCommand(CLICommand): - description = 'Create a new StreamAlert output' + description = 'Describe and manage StreamAlert outputs' @classmethod def setup_subparser(cls, subparser): - """Add the output subparser: manage.py output SERVICE""" - outputs = sorted(StreamAlertOutput.get_all_outputs().keys()) + """Add the output subparser: manage.py output""" + output_subparsers = subparser.add_subparsers() - # Output parser arguments - subparser.add_argument( - 'service', - choices=outputs, - metavar='SERVICE', - help='Create a new StreamAlert output for one of the available services: {}'.format( - ', '.join(outputs) - ) - ) + for subcommand in cls._subcommands().values(): + subcommand.setup_subparser(output_subparsers) @classmethod def handler(cls, options, config): - """Configure a new output for this service + subcommands = cls._subcommands() - Args: - options (argparse.Namespace): Basically a namedtuple with the service setting + if options.subcommand in subcommands: + return subcommands[options.subcommand].handler(options, config) + LOGGER.error('Unhandled output subcommand %s', options.subcommand) + + @staticmethod + def _subcommands(): + return { + 'set': OutputSetSubCommand, + 'set-from-file': OutputSetFromFileSubCommand, + 'generate-skeleton': OutputGenerateSkeletonSubCommand, + 'get': OutputGetSubCommand, + 'list': OutputListSubCommand + } + + +class OutputSharedMethods: + @staticmethod + def save_credentials(service, config, properties): + """Save credentials for the provided service + + Args: + service (str): The name of the service the output belongs too + config (StreamAlert.config): The configuration of StreamAlert + properties (OrderedDict): Contains various OutputProperty items Returns: bool: False if errors occurred, True otherwise """ @@ -65,6 +83,93 @@ def handler(cls, options, config): if 'alias/' in kms_key_alias: kms_key_alias = kms_key_alias.split('/')[1] + provider = OutputCredentialsProvider(service, config=config, region=region, prefix=prefix) + result = provider.save_credentials( + properties['descriptor'].value, kms_key_alias, properties + ) + if not result: + LOGGER.error('An error occurred while saving \'%s\' ' + 'output configuration for service \'%s\'', properties['descriptor'].value, + service) + return result + + @staticmethod + def update_config(options, config, properties, output, service): + """Update the local config files + + Args: + options (argparse.Namespace): Basically a namedtuple with the service setting + config (StreamAlert.config): The configuration of StreamAlert + properties (OrderedDict): Contains various OutputProperty items + output (StreamAlert.OutputDispatcher): The output to update + service (str): The name of the service the output belongs too + """ + output_config = config['outputs'] + descriptor = properties['descriptor'].value + + if options.update and output_exists(output_config, properties, service, log_message=False): + # Don't update the config if the output already existed, this will prevent duplicates + LOGGER.debug( + 'Output already exists, don\'t update the config for descriptor %s and service %s', + descriptor, service + ) + else: + updated_config = output.format_output_config(output_config, properties) + output_config[service] = updated_config + config.write() + + LOGGER.debug('Successfully saved \'%s\' output configuration for service \'%s\'', + descriptor, service) + + +class OutputSetSubCommand(CLICommand, OutputSharedMethods): + description = 'Set a single output' + + @classmethod + def setup_subparser(cls, subparser): + """Setup: manage.py output set [options] + + Args: + outputs (list): List of available output services + """ + outputs = sorted(StreamAlertOutput.get_all_outputs().keys()) + + set_parser = generate_subparser( + subparser, + 'set', + description=cls.description, + help=cls.description, + subcommand=True + ) + + # Add the required positional arg of service + set_parser.add_argument( + 'service', + choices=outputs, + metavar='SERVICE', + help='Create a new StreamAlert output for one of the available services: {}'.format( + ', '.join(outputs) + ) + ) + + # Add the optional update flag, which allows existing outputs to be updated + set_parser.add_argument( + '--update', + '-u', + action='store_true', + default=False, + help='If the output already exists, overwrite it' + ) + + @classmethod + def handler(cls, options, config): + """Configure a new output for this service + Args: + options (argparse.Namespace): Basically a namedtuple with the service setting + config (StreamAlert.config): The configuration of StreamAlert + Returns: + bool: False if errors occurred, True otherwise + """ # Retrieve the proper service class to handle dispatching the alerts of this services output = StreamAlertOutput.get_dispatcher(options.service) @@ -74,32 +179,336 @@ def handler(cls, options, config): return False # get dictionary of OutputProperty items to be used for user prompting - props = output.get_user_defined_properties() + properties = output.get_user_defined_properties() - for name, prop in props.items(): + for name, prop in properties.items(): # pylint: disable=protected-access - props[name] = prop._replace( + properties[name] = prop._replace( value=user_input(prop.description, prop.mask_input, prop.input_restrictions)) - output_config = config['outputs'] service = output.__service__ - # If it exists already, ask for user input again for a unique configuration - if output_exists(output_config, props, service): + if not options.update and output_exists(config['outputs'], properties, service): + # If the output already exists and update is not set + # ask for user input again for a unique configuration return cls.handler(options, config) - provider = OutputCredentialsProvider(service, config=config, region=region, prefix=prefix) - result = provider.save_credentials(props['descriptor'].value, kms_key_alias, props) - if not result: - LOGGER.error('An error occurred while saving \'%s\' ' - 'output configuration for service \'%s\'', props['descriptor'].value, - options.service) + if not cls.save_credentials(service, config, properties): + # Error message is already logged so no need to log a new one return False - updated_config = output.format_output_config(output_config, props) - output_config[service] = updated_config - config.write() + cls.update_config(options, config, properties, output, service) LOGGER.info('Successfully saved \'%s\' output configuration for service \'%s\'', - props['descriptor'].value, options.service) + properties['descriptor'].value, service) + return True + + +class OutputSetFromFileSubCommand(CLICommand, OutputSharedMethods): + description = 'Set numerous outputs from a file' + + @classmethod + def setup_subparser(cls, subparser): + """Setup: manage.py output set-from-file [options] + + Args: + outputs (list): List of available output services + """ + set_from_file_parser = generate_subparser( + subparser, + 'set-from-file', + description=cls.description, + help=cls.description, + subcommand=True + ) + + # Add the optional file flag + set_from_file_parser.add_argument( + '--file', + '-f', + default=OUTPUTS_FILE, + help='Path to the json file, relative to the current working directory' + ) + + # Add the optional update flag, which allows existing outputs to be updated + set_from_file_parser.add_argument( + '--update', + '-u', + action='store_true', + default=False, + help='Allow existing outputs to be overwritten' + ) + + @classmethod + def handler(cls, options, config): + """Configure multiple outputs for multiple services + Args: + options (argparse.Namespace): Basically a namedtuple with the service setting + config (StreamAlert.config): The configuration of StreamAlert + Returns: + bool: False if errors occurred, True otherwise + """ + try: + with open(options.file, 'r') as json_file_fp: + file_contents = json.load(json_file_fp) + except Exception: # pylint: disable=broad-except + LOGGER.error("Error opening file %s", options.file) + return False + + if not file_contents: + LOGGER.error('File %s is empty', options.file) + return False + + for service, configurations in file_contents.items(): + LOGGER.debug('Setting outputs for service %s', service) + # Retrieve the proper service class to handle dispatching the alerts of this service + output = StreamAlertOutput.get_dispatcher(service) + + for configuration in configurations: + properties = cls.convert_configuration_to_properties(configuration, output) + if not properties: + # Configuration was not valid + return False + + if not options.update and output_exists(config['outputs'], properties, service): + # If the output already exists and update is not set + # return early + return False + + # For each configuration for this service, save the creds and update the config + if not cls.save_credentials(service, config, properties): + return False + cls.update_config(options, config, properties, output, service) + + LOGGER.info('Saved %s configurations for service: %s', len(configurations), service) + + LOGGER.info('Finished setting all configurations for services: %s', file_contents.keys()) return True + + @staticmethod + def convert_configuration_to_properties(configuration, output): + """Check the configuration meets all input_restrictions + + Args: + configuration (dict): The configuration to check and convert to OutputProperties + output (StreamAlert.OutputDispatcher): The output to map the configuration to + Returns: + OrderedDict: If the configuration is valid for the passed OutputDispatcher else None + """ + properties = output.get_user_defined_properties() + + for name, value in configuration.items(): + if name not in properties: + LOGGER.error('unknown key %s passed for service: %s', name, output.__service__) + break + + prop = properties[name] + if not response_is_valid(value, prop.input_restrictions): + # Error messages are handled by response_is_valid + break + + properties[name] = prop._replace(value=value) + else: + return properties + + +class OutputGenerateSkeletonSubCommand(CLICommand): + description = 'Generate the skeleton file for use with set-from-file' + + @classmethod + def setup_subparser(cls, subparser): + """Add generate-skeleton subparser to the output subparser""" + outputs = sorted(StreamAlertOutput.get_all_outputs().keys()) + + # Create the generate-skeleton parser + generate_skeleton_parser = generate_subparser( + subparser, + 'generate-skeleton', + description=cls.description, + help=cls.description, + subcommand=True + ) + + # Add the optional ability to pass services + generate_skeleton_parser.add_argument( + '--services', + choices=outputs, + nargs='+', + metavar='SERVICE', + default=outputs, + help='Pass the services to generate the skeleton for from services: {}'.format( + ', '.join(outputs) + ) + ) + + # Add the optional file flag + generate_skeleton_parser.add_argument( + '--file', + '-f', + default=OUTPUTS_FILE, + help='File to write to, relative to the current working directory' + ) + + @classmethod + def handler(cls, options, config): + """Generate a skeleton file for use with set-from-file + Args: + options (argparse.Namespace): Basically a namedtuple with the service setting + config (StreamAlert.config): The configuration of StreamAlert + Returns: + bool: False if errors occurred, True otherwise + """ + + skeleton = {} + for service in options.services: + # Retrieve the proper service class to handle dispatching the alerts of this services + # No need to safeguard, as choices are defined on --services + output = StreamAlertOutput.get_dispatcher(service) + + # get dictionary of OutputProperty items to be used for user prompting + properties = output.get_user_defined_properties() + skeleton[service] = [ + { + name: 'desc: {}, restrictions: {}'.format( + prop.description, prop.input_restrictions + ) + for name, prop in properties.items() + } + ] + + try: + with open(options.file, 'w') as json_file_fp: + json.dump(skeleton, json_file_fp, indent=2, sort_keys=True) + except Exception as err: # pylint: disable=broad-except + LOGGER.error(err) + return False + + LOGGER.info( + 'Successfully generated the Skeleton file %s for services: %s', + options.file, + options.services + ) + return True + + +class OutputGetSubCommand(CLICommand): + description = 'Get the existing configuration for outputs' + + @classmethod + def setup_subparser(cls, subparser): + """Add the output get subparser: manage.py output get [options]""" + outputs = sorted(StreamAlertOutput.get_all_outputs().keys()) + + get_parser = generate_subparser( + subparser, + 'get', + description=cls.description, + help=cls.description, + subcommand=True, + ) + + # Add the positional arg of service + get_parser.add_argument( + 'service', + choices=outputs, + metavar='SERVICE', + help='Service to pull configured outputs and their secrets, select from: {}'.format( + ', '.join(outputs) + ) + ) + + # Add the optional ability to pass multiple descriptors + get_parser.add_argument( + '--descriptors', + '-d', + nargs="+", + default=False, + help='Pass descriptor and service to pull back the relevant configuration' + ) + + # Add the optional ability to pass service + + @classmethod + def handler(cls, options, config): + """Fetches the configuration for a service + Args: + options (argparse.Namespace): Basically a namedtuple with the service setting + config (StreamAlert.config): The configuration of StreamAlert + Returns: + bool: False if errors occurred, True otherwise + """ + service = options.service + output = StreamAlertOutput.create_dispatcher(service, config) + + # Get the descriptors for the service. No need to check service + # as this is handled by argparse choices + configured_descriptors = [ + descriptor for descriptor in config["outputs"][service] if 'sample' not in descriptor + ] + + # Set the descriptors to get the secrets for + descriptors = options.descriptors if options.descriptors else configured_descriptors + + LOGGER.debug('Getting secrets for service %s and descriptors %s', service, descriptors) + + credentials = [] + for descriptor in descriptors: + if descriptor not in configured_descriptors: + LOGGER.error('Invalid descriptor %s, it doesn\'t exist', descriptor) + return False + + creds = output._load_creds(descriptor) # pylint: disable=protected-access + creds['descriptor'] = descriptor + credentials.append(creds) + + print('\nService Name:', service) + print(json.dumps(credentials, indent=2, sort_keys=True), '\n') + + +class OutputListSubCommand(CLICommand): + description = 'List the currently configured outputs' + + @classmethod + def setup_subparser(cls, subparser): + """Add the output list subparser: manage.py output list [options]""" + outputs = sorted(StreamAlertOutput.get_all_outputs().keys()) + + list_parser = generate_subparser( + subparser, + 'list', + description=cls.description, + help=cls.description, + subcommand=True, + ) + + # Add the optional arg of service + list_parser.add_argument( + '--service', + '-s', + choices=outputs, + default=outputs, + nargs='*', + metavar='SERVICE', + help='Pass Services to list configured output descriptors, select from: {}'.format( + ', '.join(outputs) + ) + ) + + @classmethod + def handler(cls, options, config): + """List configured outputs + Args: + options (argparse.Namespace): Basically a namedtuple with the service setting + config (StreamAlert.config): The configuration of StreamAlert + Returns: + bool: False if errors occurred, True otherwise + """ + outputs = config["outputs"] + + print("\nConfigured outputs 'service:descriptor':") + for output in options.service: + if output not in outputs: + continue + for descriptor in outputs[output]: + print("\t{}:{}".format(output, descriptor)) + print() # ensure a newline between each service for easier reading diff --git a/streamalert_cli/outputs/helpers.py b/streamalert_cli/outputs/helpers.py index f487203bd..0072c70fc 100644 --- a/streamalert_cli/outputs/helpers.py +++ b/streamalert_cli/outputs/helpers.py @@ -19,20 +19,24 @@ LOGGER = get_logger(__name__) -def output_exists(config, props, service): +def output_exists(config, props, service, log_message=True): """Determine if this service and destination combo has already been created Args: config (dict): The outputs config that has been read from disk props (OrderedDict): Contains various OutputProperty items service (str): The service for which the user is adding a configuration + log_message (bool): Optionally log the error message Returns: [boolean] True if the service/destination exists already """ if service in config and props['descriptor'].value in config[service]: - LOGGER.error('This descriptor is already configured for %s. ' - 'Please select a new and unique descriptor', service) + if log_message: + LOGGER.error('This descriptor %s is already configured for %s. ' + 'Please select a new and unique descriptor', + props['descriptor'].value, service + ) return True return False diff --git a/streamalert_cli/utils.py b/streamalert_cli/utils.py index 3a264e128..7eb14eee8 100644 --- a/streamalert_cli/utils.py +++ b/streamalert_cli/utils.py @@ -202,12 +202,13 @@ def set_parser_epilog(parser, epilog): parser.epilog = textwrap.dedent(epilog) if epilog else None -def generate_subparser(parser, name, description=None, subcommand=False): +def generate_subparser(parser, name, description=None, subcommand=False, **kwargs): """Helper function to return a subparser with the given options""" subparser = parser.add_parser( name, description=description, - formatter_class=RawDescriptionHelpFormatter + formatter_class=RawDescriptionHelpFormatter, + **kwargs ) if subcommand: