-
Notifications
You must be signed in to change notification settings - Fork 145
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
add automatic caching for discovery requests, refreshing on a miss #238
Changes from 3 commits
396e4fc
88348e5
2a1773f
c0ed225
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,10 +1,12 @@ | ||
#!/usr/bin/env python | ||
|
||
import os | ||
import sys | ||
import copy | ||
import json | ||
import base64 | ||
from functools import partial | ||
from six import PY2 | ||
from six import PY2, PY3 | ||
|
||
import yaml | ||
from pprint import pformat | ||
|
@@ -32,6 +34,28 @@ | |
'ResourceField', | ||
] | ||
|
||
class CacheEncoder(json.JSONEncoder): | ||
|
||
def default(self, o): | ||
return o.to_dict() | ||
|
||
def cache_decoder(client): | ||
|
||
class CacheDecoder(json.JSONDecoder): | ||
def __init__(self, *args, **kwargs): | ||
json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) | ||
|
||
def object_hook(self, obj): | ||
if '_type' not in obj: | ||
return obj | ||
_type = obj.pop('_type') | ||
if _type == 'Resource': | ||
return Resource(client=client, **obj) | ||
elif _type == 'ResourceList': | ||
return ResourceList(obj['resource']) | ||
return obj | ||
|
||
return CacheDecoder | ||
|
||
def meta_request(func): | ||
""" Handles parsing response structure and translating API Exceptions """ | ||
|
@@ -66,18 +90,44 @@ class DynamicClient(object): | |
the kubernetes API | ||
""" | ||
|
||
def __init__(self, client): | ||
def __init__(self, client, cache_file=None): | ||
self.client = client | ||
self.configuration = client.configuration | ||
default_cache_id = self.configuration.host | ||
if PY3: | ||
default_cache_id = default_cache_id.encode('utf-8') | ||
self.__resources = ResourceContainer({}, client=self) | ||
self.__cache_file = cache_file or '/tmp/osrcp-{0}.json'.format(base64.b64encode(default_cache_id).decode('utf-8')) | ||
self.__init_cache() | ||
|
||
def __init_cache(self, refresh=False): | ||
if refresh or not os.path.exists(self.__cache_file): | ||
self.__cache = {} | ||
refresh = True | ||
else: | ||
with open(self.__cache_file, 'r') as f: | ||
self.__cache = json.load(f, cls=cache_decoder(self)) | ||
self._load_server_info() | ||
self.__resources = ResourceContainer(self.parse_api_groups()) | ||
self.__resources.update(self.parse_api_groups()) | ||
|
||
if refresh: | ||
self.__write_cache() | ||
|
||
def __write_cache(self): | ||
with open(self.__cache_file, 'w') as f: | ||
json.dump(self.__cache, f, cls=CacheEncoder) | ||
|
||
def invalidate_cache(self): | ||
self.__init_cache(refresh=True) | ||
|
||
def _load_server_info(self): | ||
self.__version = {'kubernetes': load_json(self.request('get', '/version'))} | ||
try: | ||
self.__version['openshift'] = load_json(self.request('get', '/version/openshift')) | ||
except ApiException: | ||
pass | ||
if not self.__cache.get('version'): | ||
self.__cache['version'] = {'kubernetes': load_json(self.request('get', '/version'))} | ||
try: | ||
self.__cache['version']['openshift'] = load_json(self.request('get', '/version/openshift')) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I believe There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, I imagine this is usually run against older versions of openshift so I figured I would just leave it in. |
||
except ApiException: | ||
pass | ||
self.__version = self.__cache['version'] | ||
|
||
@property | ||
def resources(self): | ||
|
@@ -102,20 +152,22 @@ def default_groups(self): | |
|
||
def parse_api_groups(self): | ||
""" Discovers all API groups present in the cluster """ | ||
prefix = 'apis' | ||
groups_response = load_json(self.request('GET', '/{}'.format(prefix)))['groups'] | ||
|
||
groups = self.default_groups() | ||
groups[prefix] = {} | ||
|
||
for group in groups_response: | ||
new_group = {} | ||
for version_raw in group['versions']: | ||
version = version_raw['version'] | ||
preferred = version_raw == group['preferredVersion'] | ||
new_group[version] = self.get_resources_for_api_version(prefix, group['name'], version, preferred) | ||
groups[prefix][group['name']] = new_group | ||
return groups | ||
if not self.__cache.get('resources'): | ||
prefix = 'apis' | ||
groups_response = load_json(self.request('GET', '/{}'.format(prefix)))['groups'] | ||
|
||
groups = self.default_groups() | ||
groups[prefix] = {} | ||
|
||
for group in groups_response: | ||
new_group = {} | ||
for version_raw in group['versions']: | ||
version = version_raw['version'] | ||
preferred = version_raw == group['preferredVersion'] | ||
new_group[version] = self.get_resources_for_api_version(prefix, group['name'], version, preferred) | ||
groups[prefix][group['name']] = new_group | ||
self.__cache['resources'] = groups | ||
return self.__cache['resources'] | ||
|
||
def get_resources_for_api_version(self, prefix, group, version, preferred): | ||
""" returns a dictionary of resources associated with provided groupVersion""" | ||
|
@@ -369,6 +421,24 @@ def __init__(self, prefix=None, group=None, api_version=None, kind=None, | |
|
||
self.extra_args = kwargs | ||
|
||
def to_dict(self): | ||
return { | ||
'_type': 'Resource', | ||
'prefix': self.prefix, | ||
'group': self.group, | ||
'api_version': self.api_version, | ||
'kind': self.kind, | ||
'namespaced': self.namespaced, | ||
'verbs': self.verbs, | ||
'name': self.name, | ||
'preferred': self.preferred, | ||
'singular_name': self.singular_name, | ||
'short_names': self.short_names, | ||
'categories': self.categories, | ||
'subresources': {k: sr.to_dict() for k, sr in self.subresources.items()}, | ||
'extra_args': self.extra_args, | ||
} | ||
|
||
@property | ||
def group_version(self): | ||
if self.group: | ||
|
@@ -454,6 +524,12 @@ def patch(self, *args, **kwargs): | |
def __getattr__(self, name): | ||
return getattr(self.resource, name) | ||
|
||
def to_dict(self): | ||
return { | ||
'_type': 'ResourceList', | ||
'resource': self.resource.to_dict(), | ||
} | ||
|
||
|
||
class Subresource(Resource): | ||
""" Represents a subresource of an API resource. This generally includes operations | ||
|
@@ -497,13 +573,27 @@ def urls(self): | |
def __getattr__(self, name): | ||
return partial(getattr(self.parent.client, name), self) | ||
|
||
def to_dict(self): | ||
return { | ||
'kind': self.kind, | ||
'name': self.name, | ||
'subresource': self.subresource, | ||
'namespaced': self.namespaced, | ||
'verbs': self.verbs, | ||
'extra_args': self.extra_args, | ||
} | ||
|
||
|
||
class ResourceContainer(object): | ||
""" A convenient container for storing discovered API resources. Allows | ||
easy searching and retrieval of specific resources | ||
""" | ||
|
||
def __init__(self, resources): | ||
def __init__(self, resources, client=None): | ||
self.__resources = resources | ||
self.__client = client | ||
|
||
def update(self, resources): | ||
self.__resources = resources | ||
|
||
@property | ||
|
@@ -543,7 +633,11 @@ def search(self, **kwargs): | |
|
||
The arbitrary arguments can be any valid attribute for an openshift.dynamic.Resource object | ||
""" | ||
return self.__search(self.__build_search(**kwargs), self.__resources) | ||
results = self.__search(self.__build_search(**kwargs), self.__resources) | ||
if not results: | ||
self.__client.invalidate_cache() | ||
results = self.__search(self.__build_search(**kwargs), self.__resources) | ||
return results | ||
|
||
def __build_search(self, kind=None, api_version=None, prefix=None, **kwargs): | ||
if api_version and '/' in api_version: | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should probably use
$TMPDIR
rather than defaulting to/tmp
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good catch