From 823f44e112dd6ea4d6913f705a92b0ffe8b98580 Mon Sep 17 00:00:00 2001 From: mbohlool Date: Fri, 21 Jul 2017 14:48:54 -0700 Subject: [PATCH] Add proper GCP config loader and refresher --- config/kube_config.py | 87 ++++++++++--- config/rfc3339.MD | 1 + config/rfc3339.py | 295 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 367 insertions(+), 16 deletions(-) create mode 100644 config/rfc3339.MD create mode 100644 config/rfc3339.py diff --git a/config/kube_config.py b/config/kube_config.py index 04057fb1..d0d3103e 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -14,17 +14,21 @@ import atexit import base64 +import datetime import os import tempfile +import time +import google.auth +import google.auth.transport.requests import urllib3 import yaml -from google.oauth2.credentials import Credentials - from kubernetes.client import ApiClient, ConfigurationObject, configuration from .config_exception import ConfigException +from .rfc3339 import tf_from_timestamp, timestamp_from_tf +EXPIRY_SKEW_PREVENTION_DELAY_S = 600 KUBE_CONFIG_DEFAULT_LOCATION = os.environ.get('KUBECONFIG', '~/.kube/config') _temp_files = {} @@ -54,6 +58,17 @@ def _create_temp_file_with_content(content): return name +def _is_expired(expiry): + tf = tf_from_timestamp(expiry) + n = time.time() + return tf + EXPIRY_SKEW_PREVENTION_DELAY_S <= n + + +def _datetime_to_rfc3339(dt): + tf = (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() + return timestamp_from_tf(tf, time_offset="Z") + + class FileOrData(object): """Utility class to read content of obj[%data_key_name] or file's content of obj[%file_key_name] and represent it as file or data. @@ -110,19 +125,26 @@ class KubeConfigLoader(object): def __init__(self, config_dict, active_context=None, get_google_credentials=None, client_configuration=configuration, - config_base_path=""): + config_base_path="", + config_persister=None): self._config = ConfigNode('kube-config', config_dict) self._current_context = None self._user = None self._cluster = None self.set_active_context(active_context) self._config_base_path = config_base_path + self._config_persister = config_persister + + def _refresh_credentials(): + credentials, project_id = google.auth.default() + request = google.auth.transport.requests.Request() + credentials.refresh(request) + return credentials + if get_google_credentials: self._get_google_credentials = get_google_credentials else: - self._get_google_credentials = lambda: ( - GoogleCredentials.get_application_default() - .get_access_token().access_token) + self._get_google_credentials = _refresh_credentials self._client_configuration = client_configuration def set_active_context(self, context_name=None): @@ -166,16 +188,32 @@ def _load_authentication(self): def _load_gcp_token(self): if 'auth-provider' not in self._user: return - if 'name' not in self._user['auth-provider']: + provider = self._user['auth-provider'] + if 'name' not in provider: return - if self._user['auth-provider']['name'] != 'gcp': + if provider['name'] != 'gcp': return - # Ignore configs in auth-provider and rely on GoogleCredentials - # caching and refresh mechanism. - # TODO: support gcp command based token ("cmd-path" config). - self.token = "Bearer %s" % self._get_google_credentials() + + if (('config' not in provider) or + ('access-token' not in provider['config']) or + ('expiry' in provider['config'] and + _is_expired(provider['config']['expiry']))): + # token is not available or expired, refresh it + self._refresh_gcp_token() + + self.token = "Bearer %s" % provider['config']['access-token'] return self.token + def _refresh_gcp_token(self): + if 'config' not in self._user['auth-provider']: + self._user['auth-provider'].value['config'] = {} + provider = self._user['auth-provider']['config'] + credentials = self._get_google_credentials() + provider.value['access-token'] = credentials.token + provider.value['expiry'] = _datetime_to_rfc3339(credentials.expiry) + if self._config_persister: + self._config_persister(self._config.value) + def _load_user_token(self): token = FileOrData( self._user, 'tokenFile', 'token', @@ -289,6 +327,11 @@ def _get_kube_config_loader_for_yaml_file(filename, **kwargs): **kwargs) +def _save_kube_config(filename, config_map): + with open(filename, 'w') as f: + yaml.safe_dump(config_map, f, default_flow_style=False) + + def list_kube_config_contexts(config_file=None): if config_file is None: @@ -299,7 +342,8 @@ def list_kube_config_contexts(config_file=None): def load_kube_config(config_file=None, context=None, - client_configuration=configuration): + client_configuration=configuration, + persist_config=True): """Loads authentication and cluster information from kube-config file and stores them in kubernetes.client.configuration. @@ -308,21 +352,32 @@ def load_kube_config(config_file=None, context=None, from config file will be used. :param client_configuration: The kubernetes.client.ConfigurationObject to set configs to. + :param persist_config: If True and config changed (e.g. GCP token refresh) + the provided config file will be updated. """ if config_file is None: config_file = os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION) + config_persister = None + if persist_config: + config_persister = lambda config_map, config_file=config_file: ( + _save_kube_config(config_file, config_map)) _get_kube_config_loader_for_yaml_file( config_file, active_context=context, - client_configuration=client_configuration).load_and_set() + client_configuration=client_configuration, + config_persister=config_persister).load_and_set() -def new_client_from_config(config_file=None, context=None): +def new_client_from_config( + config_file=None, + context=None, + persist_config=True): """Loads configuration the same as load_kube_config but returns an ApiClient to be used with any API object. This will allow the caller to concurrently talk with multiple clusters.""" client_config = ConfigurationObject() load_kube_config(config_file=config_file, context=context, - client_configuration=client_config) + client_configuration=client_config, + persist_config=persist_config) return ApiClient(config=client_config) diff --git a/config/rfc3339.MD b/config/rfc3339.MD new file mode 100644 index 00000000..4361af30 --- /dev/null +++ b/config/rfc3339.MD @@ -0,0 +1 @@ +The (rfc3339.py)[rfc3339.py] file is copied from [this site](http://home.blarg.net/~steveha/pyfeed.html) because PyFeed is not available in PyPi. \ No newline at end of file diff --git a/config/rfc3339.py b/config/rfc3339.py new file mode 100644 index 00000000..f0caccb6 --- /dev/null +++ b/config/rfc3339.py @@ -0,0 +1,295 @@ +# feed.date.rfc3339 -- conversion functions for RFC 3339 timestamps + +# This is the BSD license. For more information, see: +# http://www.opensource.org/licenses/bsd-license.php +# +# Copyright (c) 2006, Steve R. Hastings +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# * Neither the name of Steve R. Hastings nor the names +# of any contributors may be used to endorse or promote products +# derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +""" +Conversion functions to handle RFC 3339 timestamp format. + +RFC 3339 format is used in Atom syndication feeds. + +"tf" is short for "time float", a float being used as a time value +(seconds since the epoch). Always store tf values as UTC values, not +local time values. A TF of 0.0 means the epoch in UTC. + + +Please send questions, comments, and bug reports to: pyfeed@langri.com + +""" + +import re +import time +from calendar import timegm + +# _tz_offset_dict +# Look up a time zone offset code and return an offset value. Offset +# represents how many hours offset from UTC. + +_tz_offset_dict = { + "ut": 0, "utc": 0, "gmt": 0, "z": 0, + "et": -5, "est": -5, "edt": -4, + "ct": -6, "cst": -6, "cdt": -5, + "mt": -7, "mst": -7, "mdt": -6, + "pt": -8, "pst": -8, "pdt": -7, + "a": -1, "b": -2, "c": -3, "d": -4, "e": -5, "f": -6, "g": -7, + "h": -8, "i": -9, "k": -10, "l": -11, "m": -12, "n": +1, "o": +2, + "p": +3, "q": +4, "r": +5, "s": +6, "t": +7, "u": +8, "v": +9, + "w": +10, "x": +11, "y": +12} + + +_pat_time_offset = re.compile("([+-])(\d\d):?(\d\d?)?") + + +def parse_time_offset(s): + """ + Given a time offset string, return the offset from UTC, in seconds. + + RSS allows any RFC822-compatible time offset, which includes many + odd codes such as "EST", "PDT", "N", etc. This function understands + them all, plus numeric ones like "-0800". + """ + # Python's time.strptime() function can understand numeric offset, + # or text code, but not either one. + + if s is None: + return 0 + + try: + s = s.lstrip().rstrip().lower() + except AttributeError: + raise TypeError("time offset must be a string") + + if s in _tz_offset_dict: + return _tz_offset_dict[s] * 3600 + + m = _pat_time_offset.search(s) + if not m: + raise ValueError("invalid time offset string") + + sign = m.group(1) + offset_hour = int(m.group(2)) + if m.group(3) is not None: + offset_min = int(m.group(3)) + else: + offset_min = 0 + offset = offset_hour * 3600 + offset_min * 60 + + if sign == "-": + offset *= -1 + + return offset + + +# NOTES ON TIME CONVERSIONS +# +# Most of the time, the tf values will be UTC (aka GMT or Zulu time) +# values. Timestamp strings come complete with time offset specifiers, +# so when you convert a timestamp to a tf, the time offset will cause an +# adjustment to the tf to make it a UTC value. +# +# Then, we use Python's time conversion functions that work on UTC +# values, so we don't get any adjustments for local time. +# +# Finally, when actually formatting the timestamp string for output, we +# calculate the adjustment for the offset value. If you print a +# timestamp value with a "Z" offset value, you get no adjustment; if you +# use "-0800", you get an 8 hour adjustment; and so on. +# +# These two timestamps both represent the same time: +# +# 1969-12-31T16:00:01-08:00 +# 1970-01-01T00:00:01Z +# +# They are both a tf of 1.0. + +def cleanup_time_offset(time_offset): + """ + Given a time offset, return a time offset in a consistent format. + + If the offset is for UTC, always return a "Z". + + Otherwise, return offset in this format: "(+|-)hh:mm" + """ + secs = parse_time_offset(time_offset) + + if secs == 0: + return "Z" + + return s_time_offset_from_secs(secs) + + +_format_RFC3339 = "%Y-%m-%dT%H:%M:%S" + + +def timestamp_from_tf(tf, time_offset="Z"): + """ + Format a time and offset into a string. + + Arguments: + tf + a floating-point time value, seconds since the epoch. + time_offset + a string specifying an offset from UTC. Examples: + z or Z -- offset is 0 ("Zulu" time, UTC, aka GMT) + -08:00 -- 8 hours earlier than UTC (Pacific time zone) + "" -- empty string is technically not legal, but may work + + Notes: + Returned string complies with RFC 3339. + Example: 2003-12-13T18:30:02Z + Example: 2003-12-13T18:30:02+02:00 + """ + + if tf is None: + return "" + + if time_offset is None: + time_offset = s_offset_default + + # converting from tf to timestamp so *add* time offset + tf += parse_time_offset(time_offset) + + try: + s = time.strftime(_format_RFC3339, time.gmtime(tf)) + except ValueError: + return "" % tf + + return s + time_offset + + +# date recognition pattern + +# This is *extremely* permissive as to what it accepts! +# Long form regular expression with lots of comments. + + +_pat_rfc3339 = re.compile(r""" +(\d\d\d\d)\D+(\d\d)\D+(\d\d) # year month day, separated by non-digit +\D+ # non-digit +(\d\d?)\D+(\d\d)\D+(\d\d) # hour minute sec, separated by non-digit +([.,]\d+)? # optional fractional seconds (American decimal or Euro ",") +\s* # optional whitespace +(\w+|[-+]\d\d?\D*\d\d)? # time offset: letter(s), or +/- hours:minutes +""", re.X) + + +def tf_from_timestamp(timestamp): + """ + Take a RFC 3339 timestamp string and return a time float value. + + timestamp example: 2003-12-13T18:30:02Z + timestamp example: 2003-12-13T18:30:02+02:00 + + Leaving off the suffix is technically not legal, but allowed. + """ + + timestamp = timestamp.lstrip().rstrip() + + try: + m = _pat_rfc3339.search(timestamp) + year = int(m.group(1)) + mon = int(m.group(2)) + mday = int(m.group(3)) + hour = int(m.group(4)) + min = int(m.group(5)) + sec = int(m.group(6)) + s_zone_offset = m.group(8) + + tup = (year, mon, mday, hour, min, sec, -1, -1, 0) + + # calendar.timegm() is like time.mktime() but doesn't adjust + # from local to UTC; it just converts to a tf. + tf = timegm(tup) + + # Use time offset from timestamp to adjust from UTC to correct. + # If s_zone_offset is "GMT", "UTC", or "Z", offset is 0. + + # converting from timestamp to tf so *subtract* time offset + tf -= parse_time_offset(s_zone_offset) + except BaseException: + return None + + return float(tf) + + +def s_time_offset_from_secs(secs): + """ + Return a string with offset from UTC in RFC3339 format, from secs. + + """ + + if secs > 0: + sign = "+" + else: + sign = "-" + secs = abs(secs) + + offset_hour = secs // (60 * 60) + offset_min = (secs // 60) % 60 + return "%s%02d:%02d" % (sign, offset_hour, offset_min) + + +def local_time_offset(): + """ + Return a string with local offset from UTC in RFC3339 format. + """ + + # If tf is set to local time in seconds since the epoch, then... + # ...offset is the value you add to tf to get UTC. This is the + # reverse of time.timezone or time.altzone. + + if time.daylight: + secs_offset = -(time.altzone) + else: + secs_offset = -(time.timezone) + + return s_time_offset_from_secs(secs_offset) + + +s_offset_local = local_time_offset() + +offset_default = 0 +s_offset_default = "" + + +def set_default_time_offset(s): + global offset_default + global s_offset_default + offset_default = parse_time_offset(s) + s_offset_default = s + + +set_default_time_offset(s_offset_local)