Skip to content

Commit

Permalink
Merge pull request #1 from rbramwell/gcm_pkg_and_align_period_seconds
Browse files Browse the repository at this point in the history 
Updated for google-cloud-monitoring and --align-period-seconds param
  • Loading branch information
@rbramwell
rbramwell authored Jun 4, 2019
2 parents 0c7fc1b + c6f383a commit b5d6e07
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 173 deletions.
51 changes: 27 additions & 24 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ $ gcpmetrics --keyfile ./keyfile.json \
--query --days 2 \
--metric appengine.googleapis.com/system/cpu/usage \
--reduce REDUCE_SUM \
--align ALIGN_SUM
--align ALIGN_SUM \
--align-period-seconds 300
ALIGN: ALIGN_SUM seconds: 172800.0
REDUCE: REDUCE_SUM grouping: None
Expand Down Expand Up @@ -366,32 +367,34 @@ usage: gcpmetrics [-h] [--version] [--init-config DIR] [--config FILE]
[--list-resources] [--list-metrics] [--query] [--service ID]
[--metric ID] [--infinite] [--days INT] [--hours INT]
[--minutes INT] [--resource-filter S] [--metric-filter S]
[--align A] [--reduce R] [--reduce-grouping R] [--iloc00]
[--align A] [--align-period-seconds A] [--reduce R]
[--reduce-grouping R] [--iloc00]
Google Cloud Monitoring API Command Line
Website: https://github.com/odin-public/gcpmetrics
optional arguments:
-h, --help show this help message and exit
--version Print gcpmetics version and exit.
--init-config DIR Location of configuration files.
--config FILE Local configuration *.yaml file to be used.
--keyfile FILE Goolge Cloud Platform service account key file.
--preset ID Preset ID, like http_response_5xx_sum, etc.
--project ID Project ID.
--list-resources List monitored resource descriptors and exit.
--list-metrics List available metric descriptors and exit.
--query Run the time series query.
--service ID Service ID.
--metric ID Metric ID as defined by Google Monitoring API.
--infinite Calculate time delta since the dawn of time.
--days INT Days from now to calculate the query start date.
--hours INT Hours from now to calculate the query start date.
--minutes INT Minutes from now to calculate the query start date.
--resource-filter S Filter of resources in the var:val[,var:val] format.
--metric-filter S Filter of metrics in the var:val[,var:val] format.
--align A Alignment of data ALIGN_NONE, ALIGN_SUM. etc.
--reduce R Reduce of data REDUCE_NONE, REDUCE_SUM, etc.
--reduce-grouping R Reduce grouping in the var1[,var2] format.
--iloc00 Print value from the table index [0:0] only.
-h, --help show this help message and exit
--version Print gcpmetics version and exit.
--init-config DIR Location of configuration files.
--config FILE Local configuration *.yaml file to be used.
--keyfile FILE Goolge Cloud Platform service account key file.
--preset ID Preset ID, like http_response_5xx_sum, etc.
--project ID Project ID.
--list-resources List monitored resource descriptors and exit.
--list-metrics List available metric descriptors and exit.
--query Run the time series query.
--service ID Service ID.
--metric ID Metric ID as defined by Google Monitoring API.
--infinite Calculate time delta since the dawn of time.
--days INT Days from now to calculate the query start date.
--hours INT Hours from now to calculate the query start date.
--minutes INT Minutes from now to calculate the query start date.
--resource-filter S Filter of resources in the var:val[,var:val] format.
--metric-filter S Filter of metrics in the var:val[,var:val] format.
--align A Alignment of data ALIGN_NONE, ALIGN_SUM. etc.
--align-period-seconds A Alignment period in seconds. Default: 300 seconds.
--reduce R Reduce of data REDUCE_NONE, REDUCE_SUM, etc.
--reduce-grouping R Reduce grouping in the var1[,var2] format.
--iloc00 Print value from the table index [0:0] only.
```
147 changes: 58 additions & 89 deletions gcpmetrics/gcpmetrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,49 +8,51 @@
import argparse
import datetime
import yaml
from gcloud import monitoring
from google.cloud import monitoring_v3
from google.cloud.monitoring_v3 import query as gcm_v3_query

PARSER = argparse.ArgumentParser(
parser = argparse.ArgumentParser(
description='Google Cloud Monitoring API Command Line\nWebsite: https://github.com/odin-public/gcpmetrics',
formatter_class=argparse.RawDescriptionHelpFormatter
)

PARSER.add_argument('--version', default=None, action='store_true', help='Print gcpmetics version and exit.')
PARSER.add_argument('--init-config', help='Location of configuration files.', metavar='DIR')
PARSER.add_argument('--config', help='Local configuration *.yaml file to be used.', metavar='FILE')
PARSER.add_argument('--keyfile', help='Goolge Cloud Platform service account key file.', metavar='FILE')
PARSER.add_argument('--preset', help='Preset ID, like http_response_5xx_sum, etc.', metavar='ID')
PARSER.add_argument('--project', help='Project ID.', metavar='ID')
PARSER.add_argument('--list-resources', default=None, action='store_true', help='List monitored resource descriptors and exit.')
PARSER.add_argument('--list-metrics', default=None, action='store_true', help='List available metric descriptors and exit.')
PARSER.add_argument('--query', default=None, action='store_true', help='Run the time series query.')
PARSER.add_argument('--service', help='Service ID.', metavar='ID')
PARSER.add_argument('--metric', help='Metric ID as defined by Google Monitoring API.', metavar='ID')
PARSER.add_argument('--infinite', default=None, action='store_true', help='Calculate time delta since the dawn of time.')
PARSER.add_argument('--days', default=0, help='Days from now to calculate the query start date.', metavar='INT')
PARSER.add_argument('--hours', default=0, help='Hours from now to calculate the query start date.', metavar='INT')
PARSER.add_argument('--minutes', default=0, help='Minutes from now to calculate the query start date.', metavar='INT')
PARSER.add_argument('--resource-filter', default=None, help='Filter of resources in the var:val[,var:val] format.', metavar='S')
PARSER.add_argument('--metric-filter', default=None, help='Filter of metrics in the var:val[,var:val] format.', metavar='S')
PARSER.add_argument('--align', default=None, help='Alignment of data ALIGN_NONE, ALIGN_SUM. etc.', metavar='A')
PARSER.add_argument('--reduce', default=None, help='Reduce of data REDUCE_NONE, REDUCE_SUM, etc.', metavar='R')
PARSER.add_argument('--reduce-grouping', default=None, help='Reduce grouping in the var1[,var2] format.', metavar='R')
PARSER.add_argument('--iloc00', default=None, action='store_true', help='Print value from the table index [0:0] only.')
parser.add_argument('--version', default=None, action='store_true', help='Print gcpmetics version and exit.')
parser.add_argument('--init-config', help='Location of configuration files.', metavar='DIR')
parser.add_argument('--config', help='Local configuration *.yaml file to be used.', metavar='FILE')
parser.add_argument('--keyfile', help='Goolge Cloud Platform service account key file.', metavar='FILE')
parser.add_argument('--preset', help='Preset ID, like http_response_5xx_sum, etc.', metavar='ID')
parser.add_argument('--project', help='Project ID.', metavar='ID')
parser.add_argument('--list-resources', default=None, action='store_true', help='List monitored resource descriptors and exit.')
parser.add_argument('--list-metrics', default=None, action='store_true', help='List available metric descriptors and exit.')
parser.add_argument('--query', default=None, action='store_true', help='Run the time series query.')
parser.add_argument('--service', help='Service ID.', metavar='ID')
parser.add_argument('--metric', help='Metric ID as defined by Google Monitoring API.', metavar='ID')
parser.add_argument('--infinite', default=None, action='store_true', help='Calculate time delta since the dawn of time.')
parser.add_argument('--days', default=0, help='Days from now to calculate the query start date.', metavar='INT')
parser.add_argument('--hours', default=0, help='Hours from now to calculate the query start date.', metavar='INT')
parser.add_argument('--minutes', default=0, help='Minutes from now to calculate the query start date.', metavar='INT')
parser.add_argument('--resource-filter', default=None, help='Filter of resources in the var:val[,var:val] format.', metavar='S')
parser.add_argument('--metric-filter', default=None, help='Filter of metrics in the var:val[,var:val] format.', metavar='S')
parser.add_argument('--align', default=None, help='Alignment of data ALIGN_NONE, ALIGN_SUM. etc.', metavar='A')
parser.add_argument('--align-period-seconds', default=None, help='Alignment period in seconds. Default: 300 seconds.', metavar='A')
parser.add_argument('--reduce', default=None, help='Reduce of data REDUCE_NONE, REDUCE_SUM, etc.', metavar='R')
parser.add_argument('--reduce-grouping', default=None, help='Reduce grouping in the var1[,var2] format.', metavar='R')
parser.add_argument('--iloc00', default=None, action='store_true', help='Print value from the table index [0:0] only.')


def error(message):
sys.stderr.write('error: {}'.format(message))
print()
print()
PARSER.print_help()
parser.print_help()
sys.exit(1)


def list_resource_descriptors(client):
def list_monitored_resource_descriptors(client, project_name):
print('Monitored resource descriptors:')

index = 0
for descriptor in client.list_resource_descriptors():
for descriptor in client.list_monitored_resource_descriptors(project_name):
index += 1
print('Resource descriptor #{}'.format(index))
print('\tname: {}'.format(descriptor.name))
Expand All @@ -65,14 +67,14 @@ def list_resource_descriptors(client):
print('\t\t\tkey: {}'.format(label.key))
print('\t\t\tvalue_type: {}'.format(label.value_type))
print('\t\t\tdescription: {}'.format(label.description))
print
print()


def list_metric_descriptors(client):
def list_metric_descriptors(client, project_name):
print('Defined metric descriptors:')

index = 0
for descriptor in client.list_metric_descriptors():
for descriptor in client.list_metric_descriptors(project_name):
index += 1
print('Metric descriptor #{}'.format(index))
print('\tname: {}'.format(descriptor.name))
Expand All @@ -85,59 +87,18 @@ def list_metric_descriptors(client):
print()


def _build_label_filter(category, *args, **kwargs):
"""Construct a filter string to filter on metric or resource labels."""
terms = list(args)
import six
for key, value in six.iteritems(kwargs):
if value is None:
continue

suffix = None
ends = ['_prefix', '_suffix', '_greater', '_greaterequal',
'_less', '_lessequal']
if key.endswith(tuple(ends)):
key, suffix = key.rsplit('_', 1)

if category == 'resource' and key == 'resource_type':
key = 'resource.type'
else:
key = '.'.join((category, 'label', key))

if suffix == 'prefix':
term = '{key} = starts_with("{value}")'
elif suffix == 'suffix':
term = '{key} = ends_with("{value}")'
elif suffix == 'greater':
term = '{key} > {value}'
elif suffix == 'greaterequal':
term = '{key} >= {value}'
elif suffix == 'less':
term = '{key} < {value}'
elif suffix == 'lessequal':
term = '{key} <= {value}'
else:
term = '{key} = "{value}"'

terms.append(term.format(key=key, value=value))

return ' AND '.join(sorted(terms))


def perform_query(client, metric_id, days, hours, minutes,
resource_filter, metric_filter, align, reduce, reduce_grouping, iloc00):
def perform_query(client, project_id, metric_id, days, hours, minutes, resource_filter, metric_filter,
align, align_period_seconds, reduce, reduce_grouping, iloc00):

if (days + hours + minutes) == 0:
error('No time interval specified. Please use --infinite or --days, --hours, --minutes')

if not metric_id:
error('Metric ID is required for query, please use --metric')

# yes, ugly, but we need to fix this method...
# at least until https://github.com/GoogleCloudPlatform/gcloud-python/pull/2234 is not merged
monitoring.query._build_label_filter = _build_label_filter

query = client.query(
query = gcm_v3_query.Query(
client=client,
project=project_id,
metric_type=metric_id,
days=days,
hours=hours,
Expand All @@ -151,11 +112,9 @@ def perform_query(client, metric_id, days, hours, minutes,
query = query.select_metrics(**metric_filter)

if align:
delta = datetime.timedelta(days=days, hours=hours, minutes=minutes)
seconds = delta.total_seconds()
if not iloc00:
print('ALIGN: {} seconds: {}'.format(align, seconds))
query = query.align(align, seconds=seconds)
print('ALIGN: {} seconds: {}'.format(align, align_period_seconds))
query = query.align(align, seconds=align_period_seconds)

if reduce:
if not iloc00:
Expand Down Expand Up @@ -188,31 +147,36 @@ def perform_query(client, metric_id, days, hours, minutes,


def process(keyfile, config, project_id, list_resources, list_metrics, query, metric_id, days, hours, minutes,
resource_filter, metric_filter, align, reduce, reduce_grouping, iloc00):
resource_filter, metric_filter, align, align_period_seconds, reduce, reduce_grouping, iloc00):

client = None
project_name = None

if not project_id:
error('--project not specified')

if not keyfile:
# --keyfile not specified, use interactive `gcloud auth login`
client = monitoring.Client(project=project_id)
client = monitoring_v3.MetricServiceClient()
project_name = client.project_path(project_id)
else:
_file = keyfile
# file is relative to config (if present)
if config:
_file = os.path.join(os.path.split(config)[0], keyfile)

client = monitoring.Client.from_service_account_json(_file, project=project_id)
client = monitoring_v3.MetricServiceClient.from_service_account_json(_file)
project_name = client.project_path(project_id)

if list_resources:
list_resource_descriptors(client)
list_monitored_resource_descriptors(client, project_name)

elif list_metrics:
list_metric_descriptors(client)
list_metric_descriptors(client, project_name)

elif query:
perform_query(client, metric_id, days, hours, minutes,
resource_filter, metric_filter, align, reduce, reduce_grouping, iloc00)
perform_query(client, project_id, metric_id, days, hours, minutes, resource_filter, metric_filter,
align, align_period_seconds, reduce, reduce_grouping, iloc00)

else:
error('No operation specified. Please choose one of --list-resources, --list-metrics, --query')
Expand Down Expand Up @@ -246,13 +210,13 @@ def apply_configs(args_dict):

_path = os.path.split(os.path.abspath(__file__))[0]
stream = open(os.path.join(_path, 'global.yaml'), 'r')
global_config = yaml.load(stream)
global_config = yaml.load(stream, Loader=yaml.FullLoader)
stream.close()

local_config = {}
if args_dict['config']:
stream = open(args_dict['config'], 'r')
local_config = yaml.load(stream)
local_config = yaml.load(stream, Loader=yaml.FullLoader)
stream.close()

_ret = args_dict
Expand Down Expand Up @@ -308,7 +272,7 @@ def version():


def main():
args_dict = vars(PARSER.parse_args())
args_dict = vars(parser.parse_args())

if args_dict['version']:
print(version())
Expand Down Expand Up @@ -350,6 +314,10 @@ def process_filter(_filter):
resource_filter = process_filter(args_dict['resource_filter'])
metric_filter = process_filter(args_dict['metric_filter'])

if args_dict['align_period_seconds'] is None:
# default to 5 minutes (300 seconds)
args_dict['align_period_seconds'] = 300

if args_dict['reduce_grouping']:
args_dict['reduce_grouping'] = args_dict['reduce_grouping'].split(',')

Expand All @@ -367,6 +335,7 @@ def process_filter(_filter):
resource_filter,
metric_filter,
args_dict['align'],
int(args_dict['align_period_seconds']),
args_dict['reduce'],
args_dict['reduce_grouping'],
args_dict['iloc00']
Expand Down
Loading

0 comments on commit b5d6e07

Please sign in to comment.