diff --git a/README.md b/README.md index a609566..673eb08 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,8 @@ $ gcpmetrics --keyfile ./keyfile.json \ --query --days 2 \ --metric appengine.googleapis.com/system/cpu/usage \ --reduce REDUCE_SUM \ - --align ALIGN_SUM + --align ALIGN_SUM \ + --align-period-seconds 300 ALIGN: ALIGN_SUM seconds: 172800.0 REDUCE: REDUCE_SUM grouping: None @@ -162,6 +163,10 @@ Supported --align parameters: - ALIGN_STDDEV - ALIGN_COUNT_TRUE - ALIGN_FRACTION_TRUE + - ALIGN_PERCENTILE_99 + - ALIGN_PERCENTILE_95 + - ALIGN_PERCENTILE_50 + - ALIGN_PERCENTILE_05 Supported --reduce parameters: @@ -362,32 +367,34 @@ usage: gcpmetrics [-h] [--version] [--init-config DIR] [--config FILE] [--list-resources] [--list-metrics] [--query] [--service ID] [--metric ID] [--infinite] [--days INT] [--hours INT] [--minutes INT] [--resource-filter S] [--metric-filter S] - [--align A] [--reduce R] [--reduce-grouping R] [--iloc00] + [--align A] [--align-period-seconds A] [--reduce R] + [--reduce-grouping R] [--iloc00] Google Cloud Monitoring API Command Line Website: https://github.com/odin-public/gcpmetrics optional arguments: - -h, --help show this help message and exit - --version Print gcpmetics version and exit. - --init-config DIR Location of configuration files. - --config FILE Local configuration *.yaml file to be used. - --keyfile FILE Goolge Cloud Platform service account key file. - --preset ID Preset ID, like http_response_5xx_sum, etc. - --project ID Project ID. - --list-resources List monitored resource descriptors and exit. - --list-metrics List available metric descriptors and exit. - --query Run the time series query. - --service ID Service ID. - --metric ID Metric ID as defined by Google Monitoring API. - --infinite Calculate time delta since the dawn of time. - --days INT Days from now to calculate the query start date. - --hours INT Hours from now to calculate the query start date. - --minutes INT Minutes from now to calculate the query start date. - --resource-filter S Filter of resources in the var:val[,var:val] format. - --metric-filter S Filter of metrics in the var:val[,var:val] format. - --align A Alignment of data ALIGN_NONE, ALIGN_SUM. etc. - --reduce R Reduce of data REDUCE_NONE, REDUCE_SUM, etc. - --reduce-grouping R Reduce grouping in the var1[,var2] format. - --iloc00 Print value from the table index [0:0] only. + -h, --help show this help message and exit + --version Print gcpmetics version and exit. + --init-config DIR Location of configuration files. + --config FILE Local configuration *.yaml file to be used. + --keyfile FILE Goolge Cloud Platform service account key file. + --preset ID Preset ID, like http_response_5xx_sum, etc. + --project ID Project ID. + --list-resources List monitored resource descriptors and exit. + --list-metrics List available metric descriptors and exit. + --query Run the time series query. + --service ID Service ID. + --metric ID Metric ID as defined by Google Monitoring API. + --infinite Calculate time delta since the dawn of time. + --days INT Days from now to calculate the query start date. + --hours INT Hours from now to calculate the query start date. + --minutes INT Minutes from now to calculate the query start date. + --resource-filter S Filter of resources in the var:val[,var:val] format. + --metric-filter S Filter of metrics in the var:val[,var:val] format. + --align A Alignment of data ALIGN_NONE, ALIGN_SUM. etc. + --align-period-seconds A Alignment period in seconds. Default: 300 seconds. + --reduce R Reduce of data REDUCE_NONE, REDUCE_SUM, etc. + --reduce-grouping R Reduce grouping in the var1[,var2] format. + --iloc00 Print value from the table index [0:0] only. ``` diff --git a/gcpmetrics/gcpmetrics.py b/gcpmetrics/gcpmetrics.py index 42fb225..9d32d8d 100755 --- a/gcpmetrics/gcpmetrics.py +++ b/gcpmetrics/gcpmetrics.py @@ -8,49 +8,51 @@ import argparse import datetime import yaml -from gcloud import monitoring +from google.cloud import monitoring_v3 +from google.cloud.monitoring_v3 import query as gcm_v3_query -PARSER = argparse.ArgumentParser( +parser = argparse.ArgumentParser( description='Google Cloud Monitoring API Command Line\nWebsite: https://github.com/odin-public/gcpmetrics', formatter_class=argparse.RawDescriptionHelpFormatter ) -PARSER.add_argument('--version', default=None, action='store_true', help='Print gcpmetics version and exit.') -PARSER.add_argument('--init-config', help='Location of configuration files.', metavar='DIR') -PARSER.add_argument('--config', help='Local configuration *.yaml file to be used.', metavar='FILE') -PARSER.add_argument('--keyfile', help='Goolge Cloud Platform service account key file.', metavar='FILE') -PARSER.add_argument('--preset', help='Preset ID, like http_response_5xx_sum, etc.', metavar='ID') -PARSER.add_argument('--project', help='Project ID.', metavar='ID') -PARSER.add_argument('--list-resources', default=None, action='store_true', help='List monitored resource descriptors and exit.') -PARSER.add_argument('--list-metrics', default=None, action='store_true', help='List available metric descriptors and exit.') -PARSER.add_argument('--query', default=None, action='store_true', help='Run the time series query.') -PARSER.add_argument('--service', help='Service ID.', metavar='ID') -PARSER.add_argument('--metric', help='Metric ID as defined by Google Monitoring API.', metavar='ID') -PARSER.add_argument('--infinite', default=None, action='store_true', help='Calculate time delta since the dawn of time.') -PARSER.add_argument('--days', default=0, help='Days from now to calculate the query start date.', metavar='INT') -PARSER.add_argument('--hours', default=0, help='Hours from now to calculate the query start date.', metavar='INT') -PARSER.add_argument('--minutes', default=0, help='Minutes from now to calculate the query start date.', metavar='INT') -PARSER.add_argument('--resource-filter', default=None, help='Filter of resources in the var:val[,var:val] format.', metavar='S') -PARSER.add_argument('--metric-filter', default=None, help='Filter of metrics in the var:val[,var:val] format.', metavar='S') -PARSER.add_argument('--align', default=None, help='Alignment of data ALIGN_NONE, ALIGN_SUM. etc.', metavar='A') -PARSER.add_argument('--reduce', default=None, help='Reduce of data REDUCE_NONE, REDUCE_SUM, etc.', metavar='R') -PARSER.add_argument('--reduce-grouping', default=None, help='Reduce grouping in the var1[,var2] format.', metavar='R') -PARSER.add_argument('--iloc00', default=None, action='store_true', help='Print value from the table index [0:0] only.') +parser.add_argument('--version', default=None, action='store_true', help='Print gcpmetics version and exit.') +parser.add_argument('--init-config', help='Location of configuration files.', metavar='DIR') +parser.add_argument('--config', help='Local configuration *.yaml file to be used.', metavar='FILE') +parser.add_argument('--keyfile', help='Goolge Cloud Platform service account key file.', metavar='FILE') +parser.add_argument('--preset', help='Preset ID, like http_response_5xx_sum, etc.', metavar='ID') +parser.add_argument('--project', help='Project ID.', metavar='ID') +parser.add_argument('--list-resources', default=None, action='store_true', help='List monitored resource descriptors and exit.') +parser.add_argument('--list-metrics', default=None, action='store_true', help='List available metric descriptors and exit.') +parser.add_argument('--query', default=None, action='store_true', help='Run the time series query.') +parser.add_argument('--service', help='Service ID.', metavar='ID') +parser.add_argument('--metric', help='Metric ID as defined by Google Monitoring API.', metavar='ID') +parser.add_argument('--infinite', default=None, action='store_true', help='Calculate time delta since the dawn of time.') +parser.add_argument('--days', default=0, help='Days from now to calculate the query start date.', metavar='INT') +parser.add_argument('--hours', default=0, help='Hours from now to calculate the query start date.', metavar='INT') +parser.add_argument('--minutes', default=0, help='Minutes from now to calculate the query start date.', metavar='INT') +parser.add_argument('--resource-filter', default=None, help='Filter of resources in the var:val[,var:val] format.', metavar='S') +parser.add_argument('--metric-filter', default=None, help='Filter of metrics in the var:val[,var:val] format.', metavar='S') +parser.add_argument('--align', default=None, help='Alignment of data ALIGN_NONE, ALIGN_SUM. etc.', metavar='A') +parser.add_argument('--align-period-seconds', default=None, help='Alignment period in seconds. Default: 300 seconds.', metavar='A') +parser.add_argument('--reduce', default=None, help='Reduce of data REDUCE_NONE, REDUCE_SUM, etc.', metavar='R') +parser.add_argument('--reduce-grouping', default=None, help='Reduce grouping in the var1[,var2] format.', metavar='R') +parser.add_argument('--iloc00', default=None, action='store_true', help='Print value from the table index [0:0] only.') def error(message): sys.stderr.write('error: {}'.format(message)) print() print() - PARSER.print_help() + parser.print_help() sys.exit(1) -def list_resource_descriptors(client): +def list_monitored_resource_descriptors(client, project_name): print('Monitored resource descriptors:') index = 0 - for descriptor in client.list_resource_descriptors(): + for descriptor in client.list_monitored_resource_descriptors(project_name): index += 1 print('Resource descriptor #{}'.format(index)) print('\tname: {}'.format(descriptor.name)) @@ -65,14 +67,14 @@ def list_resource_descriptors(client): print('\t\t\tkey: {}'.format(label.key)) print('\t\t\tvalue_type: {}'.format(label.value_type)) print('\t\t\tdescription: {}'.format(label.description)) - print + print() -def list_metric_descriptors(client): +def list_metric_descriptors(client, project_name): print('Defined metric descriptors:') index = 0 - for descriptor in client.list_metric_descriptors(): + for descriptor in client.list_metric_descriptors(project_name): index += 1 print('Metric descriptor #{}'.format(index)) print('\tname: {}'.format(descriptor.name)) @@ -85,47 +87,8 @@ def list_metric_descriptors(client): print() -def _build_label_filter(category, *args, **kwargs): - """Construct a filter string to filter on metric or resource labels.""" - terms = list(args) - import six - for key, value in six.iteritems(kwargs): - if value is None: - continue - - suffix = None - ends = ['_prefix', '_suffix', '_greater', '_greaterequal', - '_less', '_lessequal'] - if key.endswith(tuple(ends)): - key, suffix = key.rsplit('_', 1) - - if category == 'resource' and key == 'resource_type': - key = 'resource.type' - else: - key = '.'.join((category, 'label', key)) - - if suffix == 'prefix': - term = '{key} = starts_with("{value}")' - elif suffix == 'suffix': - term = '{key} = ends_with("{value}")' - elif suffix == 'greater': - term = '{key} > {value}' - elif suffix == 'greaterequal': - term = '{key} >= {value}' - elif suffix == 'less': - term = '{key} < {value}' - elif suffix == 'lessequal': - term = '{key} <= {value}' - else: - term = '{key} = "{value}"' - - terms.append(term.format(key=key, value=value)) - - return ' AND '.join(sorted(terms)) - - -def perform_query(client, metric_id, days, hours, minutes, - resource_filter, metric_filter, align, reduce, reduce_grouping, iloc00): +def perform_query(client, project_id, metric_id, days, hours, minutes, resource_filter, metric_filter, + align, align_period_seconds, reduce, reduce_grouping, iloc00): if (days + hours + minutes) == 0: error('No time interval specified. Please use --infinite or --days, --hours, --minutes') @@ -133,11 +96,9 @@ def perform_query(client, metric_id, days, hours, minutes, if not metric_id: error('Metric ID is required for query, please use --metric') - # yes, ugly, but we need to fix this method... - # at least until https://github.com/GoogleCloudPlatform/gcloud-python/pull/2234 is not merged - monitoring.query._build_label_filter = _build_label_filter - - query = client.query( + query = gcm_v3_query.Query( + client=client, + project=project_id, metric_type=metric_id, days=days, hours=hours, @@ -151,11 +112,9 @@ def perform_query(client, metric_id, days, hours, minutes, query = query.select_metrics(**metric_filter) if align: - delta = datetime.timedelta(days=days, hours=hours, minutes=minutes) - seconds = delta.total_seconds() if not iloc00: - print('ALIGN: {} seconds: {}'.format(align, seconds)) - query = query.align(align, seconds=seconds) + print('ALIGN: {} seconds: {}'.format(align, align_period_seconds)) + query = query.align(align, seconds=align_period_seconds) if reduce: if not iloc00: @@ -184,35 +143,40 @@ def perform_query(client, metric_id, days, hours, minutes, else: # print the whole dataset - print(dataframe) + print(dataframe.to_string()) def process(keyfile, config, project_id, list_resources, list_metrics, query, metric_id, days, hours, minutes, - resource_filter, metric_filter, align, reduce, reduce_grouping, iloc00): + resource_filter, metric_filter, align, align_period_seconds, reduce, reduce_grouping, iloc00): + + client = None + project_name = None if not project_id: error('--project not specified') if not keyfile: # --keyfile not specified, use interactive `gcloud auth login` - client = monitoring.Client(project=project_id) + client = monitoring_v3.MetricServiceClient() + project_name = client.project_path(project_id) else: _file = keyfile # file is relative to config (if present) if config: _file = os.path.join(os.path.split(config)[0], keyfile) - client = monitoring.Client.from_service_account_json(_file, project=project_id) + client = monitoring_v3.MetricServiceClient.from_service_account_json(_file) + project_name = client.project_path(project_id) if list_resources: - list_resource_descriptors(client) + list_monitored_resource_descriptors(client, project_name) elif list_metrics: - list_metric_descriptors(client) + list_metric_descriptors(client, project_name) elif query: - perform_query(client, metric_id, days, hours, minutes, - resource_filter, metric_filter, align, reduce, reduce_grouping, iloc00) + perform_query(client, project_id, metric_id, days, hours, minutes, resource_filter, metric_filter, + align, align_period_seconds, reduce, reduce_grouping, iloc00) else: error('No operation specified. Please choose one of --list-resources, --list-metrics, --query') @@ -246,13 +210,13 @@ def apply_configs(args_dict): _path = os.path.split(os.path.abspath(__file__))[0] stream = open(os.path.join(_path, 'global.yaml'), 'r') - global_config = yaml.load(stream) + global_config = yaml.load(stream, Loader=yaml.FullLoader) stream.close() local_config = {} if args_dict['config']: stream = open(args_dict['config'], 'r') - local_config = yaml.load(stream) + local_config = yaml.load(stream, Loader=yaml.FullLoader) stream.close() _ret = args_dict @@ -308,7 +272,7 @@ def version(): def main(): - args_dict = vars(PARSER.parse_args()) + args_dict = vars(parser.parse_args()) if args_dict['version']: print(version()) @@ -350,6 +314,10 @@ def process_filter(_filter): resource_filter = process_filter(args_dict['resource_filter']) metric_filter = process_filter(args_dict['metric_filter']) + if args_dict['align_period_seconds'] is None: + # default to 5 minutes (300 seconds) + args_dict['align_period_seconds'] = 300 + if args_dict['reduce_grouping']: args_dict['reduce_grouping'] = args_dict['reduce_grouping'].split(',') @@ -367,6 +335,7 @@ def process_filter(_filter): resource_filter, metric_filter, args_dict['align'], + int(args_dict['align_period_seconds']), args_dict['reduce'], args_dict['reduce_grouping'], args_dict['iloc00'] diff --git a/requirements.txt b/requirements.txt index 9e3f64e..e6783ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -twine -flake8 -gcloud -pandas -pyyaml +twine==1.13.0 +flake8==3.7.7 +google-cloud-monitoring==0.31.1 +pandas==0.24.2 +PyYAML==5.1