Skip to content

Commit

Permalink
Merge branch 'master' into solrpy-is-dead-long-live-pysolr
Browse files Browse the repository at this point in the history
  • Loading branch information
@joetsoi
joetsoi committed Sep 10, 2015
2 parents 5f78ea1 + 4671c93 commit e9756fd
Show file tree
Hide file tree
Showing 23 changed files with 201 additions and 913 deletions.
22 changes: 0 additions & 22 deletions ckan/config/routing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,29 +404,7 @@ def make_map():
action='trash', ckan_icon='trash')
map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin')

# Storage routes
with SubMapper(map, controller='ckan.controllers.storage:StorageAPIController') as m:
m.connect('storage_api', '/api/storage', action='index')
m.connect('storage_api_set_metadata', '/api/storage/metadata/{label:.*}',
action='set_metadata', conditions=PUT_POST)
m.connect('storage_api_get_metadata', '/api/storage/metadata/{label:.*}',
action='get_metadata', conditions=GET)
m.connect('storage_api_auth_request',
'/api/storage/auth/request/{label:.*}',
action='auth_request')
m.connect('storage_api_auth_form',
'/api/storage/auth/form/{label:.*}',
action='auth_form')

with SubMapper(map, controller='ckan.controllers.storage:StorageController') as m:
m.connect('storage_upload', '/storage/upload',
action='upload')
m.connect('storage_upload_handle', '/storage/upload_handle',
action='upload_handle')
m.connect('storage_upload_success', '/storage/upload/success',
action='success')
m.connect('storage_upload_success_empty', '/storage/upload/success_empty',
action='success_empty')
m.connect('storage_file', '/storage/f/{label:.*}',
action='file')

Expand Down
334 changes: 9 additions & 325 deletions ckan/controllers/storage.py
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,19 @@
'''
Note: This is the old file store controller for CKAN < 2.2.
If you are looking for how the file uploads work, you should check
`lib/uploader.py` and the `resource_download` method of the package
controller.
'''
import os
import re
import urllib
import uuid
from datetime import datetime
from cgi import FieldStorage

from ofs import get_impl
from pylons import request, response
from pylons.controllers.util import abort, redirect_to
from pylons import config
from paste.fileapp import FileApp
from paste.deploy.converters import asbool

from ckan.lib.base import BaseController, c, request, render, config, h, abort
from ckan.lib.jsonp import jsonpify
import ckan.model as model
import ckan.logic as logic
from ckan.lib.base import BaseController, request, config, h, abort

try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
try:
import json
except:
import simplejson as json

from logging import getLogger
log = getLogger(__name__)
Expand All @@ -36,19 +25,6 @@
_eq_re = re.compile(r"^(.*)(=[0-9]*)$")


def fix_stupid_pylons_encoding(data):
"""
Fix an apparent encoding problem when calling request.body
TODO: Investigate whether this is fixed in later versions?
"""
if data.startswith("%") or data.startswith("+"):
data = urllib.unquote_plus(data)
m = _eq_re.match(data)
if m:
data = m.groups()[0]
return data


def create_pairtree_marker(folder):
""" Creates the pairtree marker for tests if it doesn't exist """
if not folder[:-1] == '/':
Expand Down Expand Up @@ -83,27 +59,6 @@ def get_ofs():
return ofs


def authorize(method, bucket, key, user, ofs):
"""
Check authz for the user with a given bucket/key combo within a
particular ofs implementation.
"""
if not method in ['POST', 'GET', 'PUT', 'DELETE']:
abort(400)
if method != 'GET':
# do not allow overwriting
if ofs.exists(bucket, key):
abort(409)
# now check user stuff
context = {'user': c.user,
'model': model}
try:
logic.check_access('file_upload', context, {})
except logic.NotAuthorized:
h.flash_error('Not authorized to upload files.')
abort(401)


class StorageController(BaseController):
'''Upload to storage backend.
'''
Expand All @@ -115,55 +70,6 @@ def ofs(self):
StorageController._ofs_impl = get_ofs()
return StorageController._ofs_impl

def upload(self):
label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
c.data = {
'action': h.url_for('storage_upload_handle', qualified=False),
'fields': [
{
'name': 'key',
'value': label
}
]
}
return render('storage/index.html')

def upload_handle(self):
bucket_id = BUCKET
params = dict(request.params.items())
stream = params.get('file')
label = params.get('key')
authorize('POST', BUCKET, label, c.userobj, self.ofs)
if not label:
abort(400, "No label")
if not isinstance(stream, FieldStorage):
abort(400, "No file stream.")
del params['file']
params['filename-original'] = stream.filename
#params['_owner'] = c.userobj.name if c.userobj else ""
params['uploaded-by'] = c.userobj.name if c.userobj else ""

self.ofs.put_stream(bucket_id, label, stream.file, params)
success_action_redirect = h.url_for(
'storage_upload_success', qualified=True,
bucket=BUCKET, label=label)
# Do not redirect here as it breaks js file uploads (get infinite loop
# in FF and crash in Chrome)
return self.success(label)

def success(self, label=None):
label = request.params.get('label', label)
h.flash_success('Upload successful')
c.file_url = h.url_for('storage_file',
label=label,
qualified=True)
c.upload_url = h.url_for('storage_upload')
return render('storage/success.html')

def success_empty(self, label=None):
# very simple method that just returns 200 OK
return ''

def file(self, label):
exists = self.ofs.exists(BUCKET, label)
if not exists:
Expand All @@ -188,225 +94,3 @@ def file(self, label):
return fapp(request.environ, self.start_response)
else:
h.redirect_to(file_url.encode('ascii', 'ignore'))


class StorageAPIController(BaseController):
_ofs_impl = None

@property
def ofs(self):
if not StorageAPIController._ofs_impl:
StorageAPIController._ofs_impl = get_ofs()
return StorageAPIController._ofs_impl

@jsonpify
def index(self):
info = {
'metadata/{label}': {
'description': 'Get or set metadata for this '
'item in storage', },
'auth/request/{label}': {
'description': self.auth_request.__doc__, },
'auth/form/{label}': {
'description': self.auth_form.__doc__, }}
return info

def set_metadata(self, label):
bucket = BUCKET
if not label.startswith("/"):
label = "/" + label

try:
data = fix_stupid_pylons_encoding(request.body)
if data:
metadata = json.loads(data)
else:
metadata = {}
except:
abort(400)

try:
b = self.ofs._require_bucket(bucket)
except:
abort(409)

k = self.ofs._get_key(b, label)
if k is None:
k = b.new_key(label)
metadata = metadata.copy()
metadata["_creation_time"] = str(datetime.utcnow())
self.ofs._update_key_metadata(k, metadata)
k.set_contents_from_file(StringIO(''))
elif request.method == "PUT":
old = self.ofs.get_metadata(bucket, label)
to_delete = []
for ok in old.keys():
if ok not in metadata:
to_delete.append(ok)
if to_delete:
self.ofs.del_metadata_keys(bucket, label, to_delete)
self.ofs.update_metadata(bucket, label, metadata)
else:
self.ofs.update_metadata(bucket, label, metadata)

k.make_public()
k.close()

return self.get_metadata(bucket, label)

@jsonpify
def get_metadata(self, label):
bucket = BUCKET
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
if not label.startswith("/"):
label = "/" + label
url = "https://%s%s" % (
self.ofs.conn.calling_format.build_host(
self.ofs.conn.server_name(), bucket), label)
else:
url = h.url_for('storage_file',
label=label,
qualified=False
)
if url.startswith('/'):
url = config.get('ckan.site_url', '').rstrip('/') + url

if not self.ofs.exists(bucket, label):
abort(404)
metadata = self.ofs.get_metadata(bucket, label)
metadata["_location"] = url
return metadata

@jsonpify
def auth_request(self, label):
'''Provide authentication information for a request so a client can
interact with backend storage directly.
:param label: label.
:param kwargs: sent either via query string for GET or json-encoded
dict for POST). Interpreted as http headers for request plus an
(optional) method parameter (being the HTTP method).
Examples of headers are:
Content-Type
Content-Encoding (optional)
Content-Length
Content-MD5
Expect (should be '100-Continue')
:return: is a json hash containing various attributes including a
headers dictionary containing an Authorization field which is good for
15m.
'''
bucket = BUCKET
if request.POST:
try:
data = fix_stupid_pylons_encoding(request.body)
headers = json.loads(data)
except Exception:
from traceback import print_exc
msg = StringIO()
print_exc(msg)
log.error(msg.seek(0).read())
abort(400)
else:
headers = dict(request.params)
if 'method' in headers:
method = headers['method']
del headers['method']
else:
method = 'POST'

authorize(method, bucket, label, c.userobj, self.ofs)

http_request = self.ofs.authenticate_request(method, bucket, label,
headers)
return {
'host': http_request.host,
'method': http_request.method,
'path': http_request.path,
'headers': http_request.headers}

def _get_remote_form_data(self, label):
method = 'POST'
content_length_range = \
int(config.get('ckan.storage.max_content_length', 50000000))
acl = 'public-read'
fields = [{
'name': self.ofs.conn.provider.metadata_prefix + 'uploaded-by',
'value': c.userobj.id}]
conditions = ['{"%s": "%s"}' % (x['name'], x['value']) for x in
fields]
# In FF redirect to this breaks js upload as FF attempts to open file
# (presumably because mimetype = javascript) and this stops js
# success_action_redirect = h.url_for('storage_api_get_metadata',
# qualified=True, label=label)
success_action_redirect = h.url_for('storage_upload_success_empty',
qualified=True,
label=label)
data = self.ofs.conn.build_post_form_args(
BUCKET,
label,
expires_in=72000,
max_content_length=content_length_range,
success_action_redirect=success_action_redirect,
acl=acl,
fields=fields,
conditions=conditions
)
# HACK: fix up some broken stuff from boto
# e.g. should not have content-length-range in list of fields!
storage_backend = config['ofs.impl']
for idx, field in enumerate(data['fields']):
if storage_backend == 'google':
if field['name'] == 'AWSAccessKeyId':
field['name'] = 'GoogleAccessId'
if field['name'] == 'content-length-range':
del data['fields'][idx]
return data

def _get_form_data(self, label):
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
return self._get_remote_form_data(label)
else:
data = {
'action': h.url_for('storage_upload_handle', qualified=False),
'fields': [
{
'name': 'key',
'value': label
}
]
}
return data

@jsonpify
def auth_form(self, label):
'''Provide fields for a form upload to storage including
authentication.
:param label: label.
:return: json-encoded dictionary with action parameter and fields list.
'''
bucket = BUCKET
if request.POST:
try:
data = fix_stupid_pylons_encoding(request.body)
headers = json.loads(data)
except Exception:
from traceback import print_exc
msg = StringIO()
print_exc(msg)
log.error(msg.seek(0).read())
abort(400)
else:
headers = dict(request.params)

method = 'POST'
authorize(method, bucket, label, c.userobj, self.ofs)
data = self._get_form_data(label)
return data
Loading

0 comments on commit e9756fd

Please sign in to comment.