Merge pull request pandas-dev#20 from manahl/bugfix/fix-logging

Tidy up logging

README.md

@@ -125,6 +125,7 @@ It wouldn't be possible without the work of the AHL Data Engineering Team includ
  * Drake Siard
  * [Slavi Marinov](https://github.com/slavi)
  * [Wilfred Hughes](https://github.com/wilfred)
+ * [Edward Easton](https://github.com/eeaston)
  * ... and many others ...
 
 Contributions welcome!

arctic/_compression.py

@@ -1,6 +1,8 @@
 import lz4
 import os
-from .logging import logger
+import logging
+
+logger = logging.getLogger(__name__)
 
 try:
     from . import _compress as clz4

arctic/_util.py

@@ -1,10 +1,10 @@
-from datetime import datetime
 from pandas import DataFrame
 from pandas.util.testing import assert_frame_equal
 from pymongo.errors import OperationFailure
 import string
+import logging
 
-from .logging import logger
+logger = logging.getLogger(__name__)
 
 
 def indent(s, num_spaces):
@@ -45,10 +45,10 @@ def enable_sharding(arctic, library_name, hashed=False):
 def enable_powerof2sizes(arctic, library_name):
     lib = arctic[library_name]._arctic_lib
     collection = lib.get_top_level_collection()
-    lib._db.command({"collMod" : collection.name, 'usePowerOf2Sizes': "true"})
+    lib._db.command({"collMod": collection.name, 'usePowerOf2Sizes': "true"})
     logger.info("usePowerOf2Sizes enabled for %s", collection.name)
 
     for coll in collection.database.collection_names():
         if coll.startswith("%s." % collection.name):
-            lib._db.command({"collMod" : coll, 'usePowerOf2Sizes': "true"})
+            lib._db.command({"collMod": coll, 'usePowerOf2Sizes': "true"})
             logger.info("usePowerOf2Sizes enabled for %s", coll)

arctic/arctic.py

@@ -1,10 +1,10 @@
+import logging
+
 import pymongo
 from pymongo.errors import OperationFailure, AutoReconnect
-from pymongo.read_preferences import ReadPreference
 
 from .auth import authenticate, get_auth
 from .hooks import get_mongodb_uri
-from .logging import logger
 from .decorators import mongo_retry
 from ._util import indent
 
@@ -15,6 +15,8 @@
 
 __all__ = ['Arctic', 'VERSION_STORE', 'TICK_STORE', 'register_library_type']
 
+logger = logging.getLogger(__name__)
+
 # Default Arctic application name: 'arctic'
 APPLICATION_NAME = 'arctic'
 VERSION_STORE = version_store.VERSION_STORE_TYPE
@@ -174,7 +176,7 @@ def initialize_library(self, library, lib_type=VERSION_STORE, **kwargs):
         # Check that we don't create too many namespaces
         if len(self._conn[l.database_name].collection_names()) > 3000:
             raise ArcticException("Too many namespaces %s, not creating: %s" %
-                                        (len(self._conn[l.database_name].collection_names()), library))
+                                  (len(self._conn[l.database_name].collection_names()), library))
         l.set_library_type(lib_type)
         LIBRARY_TYPES[lib_type].initialize_library(l, **kwargs)
         # Add a 10G quota just in case the user is calling this with API.
@@ -224,7 +226,8 @@ def get_library(self, library):
             error = e
 
         if error or not lib_type:
-            raise LibraryNotFoundException("Library %s was not correctly initialized in %s.\nReason: %s" % (library, self, error))
+            raise LibraryNotFoundException("Library %s was not correctly initialized in %s.\nReason: %s" %
+                                           (library, self, error))
         elif lib_type not in LIBRARY_TYPES:
             raise LibraryNotFoundException("Couldn't load LibraryType '%s' for '%s' (has the class been registered?)" %
                                            (lib_type, library))
@@ -249,7 +252,7 @@ def set_quota(self, library, quota):
         ----------
         library : `str`
             The name of the library. e.g. 'library' or 'user.library'
-            
+
         quota : `int`
             Advisory quota for the library - in bytes
         """
@@ -398,9 +401,9 @@ def check_quota(self):
         # Figure out whether the user has exceeded their quota
         library = self.arctic[self.get_name()]
         stats = library.stats()
-        
-        def to_gigabytes(bytes):
-            return bytes / 1024. / 1024. / 1024.
+
+        def to_gigabytes(bytes_):
+            return bytes_ / 1024. / 1024. / 1024.
 
         # Have we exceeded our quota?
         size = stats['totals']['size']

arctic/auth.py

@@ -1,6 +1,8 @@
 from collections import namedtuple
+import logging
+
+logger = logging.getLogger(__name__)
 
-from .logging import logger
 
 def authenticate(db, user, password):
     """
@@ -19,6 +21,7 @@ def authenticate(db, user, password):
 
 Credential = namedtuple("MongoCredentials", ['database', 'user', 'password'])
 
+
 def get_auth(host, app_name, database_name):
     """
     Authentication hook to allow plugging in custom authentication credential providers

arctic/date/_daterange.py

@@ -1,8 +1,5 @@
 import datetime
-from datetime import timedelta
-from dateutil.tz import tzlocal
 
-from ..logging import logger
 from ._generalslice import OPEN_OPEN, CLOSED_CLOSED, OPEN_CLOSED, CLOSED_OPEN, GeneralSlice
 from ._parse import parse
 

arctic/date/_mktz.py

@@ -1,8 +1,6 @@
 import bisect
 import os
 import dateutil
-from decorator import decorator
-import time
 import tzlocal
 
 DEFAULT_TIME_ZONE_NAME = tzlocal.get_localzone().zone  # 'Europe/London'

arctic/date/_util.py

@@ -2,9 +2,8 @@
 import datetime
 from datetime import timedelta
 
-from ..logging import logger
 from ._daterange import DateRange
-from ._generalslice import OPEN_OPEN, CLOSED_CLOSED, OPEN_CLOSED, CLOSED_OPEN, GeneralSlice
+from ._generalslice import OPEN_OPEN, CLOSED_CLOSED, OPEN_CLOSED, CLOSED_OPEN
 from ._parse import parse
 from ._mktz import mktz
 

arctic/decorators.py

@@ -1,13 +1,16 @@
 from datetime import datetime
 from functools import wraps
 import os
-from pymongo.errors import AutoReconnect, OperationFailure, DuplicateKeyError, ServerSelectionTimeoutError
 import sys
 from time import sleep
+import logging
+
+from pymongo.errors import AutoReconnect, OperationFailure, DuplicateKeyError, ServerSelectionTimeoutError
 
-from .logging import logger
 from .hooks import _log_exception_hook as _log_exception
 
+logger = logging.getLogger(__name__)
+
 _MAX_RETRIES = 15
 
 
@@ -67,8 +70,8 @@ def dump_bad_documents(*document):
     """
     Dump bad documents to disk
     """
-    id = str(document[0]['_id'])
-    with open('/tmp/mongo_debug_' + str(os.getpid()) + '_' + id + '_' + str(datetime.now()), 'a') as f:
+    _id = str(document[0]['_id'])
+    with open('/tmp/mongo_debug_' + str(os.getpid()) + '_' + _id + '_' + str(datetime.now()), 'a') as f:
         for d in document:
             f.write(str(d) + '\n')
 

arctic/fixtures/arctic.py

@@ -1,13 +1,15 @@
 import getpass
+import logging
+
 import pytest as pytest
 
 from .. import arctic as m
-from ..logging import logger
-from ..decorators import mongo_retry
 from ..tickstore.tickstore import TICK_STORE_TYPE
 
 from .mongo import mongo_proc, mongodb
 
+logger = logging.getLogger(__name__)
+
 mongo_proc2 = mongo_proc(executable="mongod", port="?",
                          params='--nojournal '
                                 '--noauth '

arctic/fixtures/mongo.py

@@ -124,7 +124,7 @@ def mongodb_factory(request):
         """
         proc_fixture = get_process_fixture(request, process_fixture_name)
 
-        pymongo, config = try_import('pymongo', request)
+        pymongo, _ = try_import('pymongo', request)
 
         mongo_host = proc_fixture.host
         mongo_port = proc_fixture.port

arctic/hosts.py

@@ -1,17 +1,13 @@
 """
 Utilities to resolve a string to Mongo host, or a Arctic library.
 """
-import ConfigParser
-from ConfigParser import NoOptionError, NoSectionError
-import os
 import re
+import logging
 from weakref import WeakValueDictionary
 
-from .logging import logger
-
 __all__ = ['get_arctic_lib', 'get_arctic_for_library']
 
-
+logger = logging.getLogger(__name__)
 
 # Application environment variables
 arctic_cache = WeakValueDictionary()

arctic/scripts/arctic_copy_data.py

@@ -1,15 +1,17 @@
 import argparse
 import os
-import re
+import logging
 from multiprocessing import Pool
 import pwd
 
 from arctic.decorators import _get_host
 from arctic.store.audit import ArcticTransaction
 
-from ..logging import logger
 from ..hosts import get_arctic_lib
 from ..date import DateRange, to_pandas_closed_closed, CLOSED_OPEN, OPEN_CLOSED
+from .utils import setup_logging
+
+logger = logging.getLogger(__name__)
 
 # Use the UID rather than environment variables for auditing
 USER = pwd.getpwuid(os.getuid())[0]
@@ -26,18 +28,22 @@ def _copy_symbol(symbols):
                     elif splice:
                         logger.warn("Symbol: %s already exists in destination, splicing in new data" % symbol)
                     else:
-                        logger.warn("Symbol: {} already exists in {}@{}, use --force to overwrite or --splice to join with existing data".format(symbol,
-                                                                                                                                                 _get_host(dest).get('l'),
-                                                                                                                                                 _get_host(dest).get('mhost')))
+                        logger.warn("Symbol: {} already exists in {}@{}, use --force to overwrite or --splice to join "
+                                    "with existing data".format(symbol, _get_host(dest).get('l'),
+                                                                _get_host(dest).get('mhost')))
                         continue
 
                 version = src.read(symbol)
                 new_data = version.data
 
                 if existing_data and splice:
                     original_data = dest.read(symbol).data
-                    before = original_data.ix[:to_pandas_closed_closed(DateRange(None, new_data.index[0].to_pydatetime(), interval=CLOSED_OPEN)).end]
-                    after = original_data.ix[to_pandas_closed_closed(DateRange(new_data.index[-1].to_pydatetime(), None, interval=OPEN_CLOSED)).start:]
+                    before = original_data.ix[:to_pandas_closed_closed(DateRange(None,
+                                                                                 new_data.index[0].to_pydatetime(),
+                                                                                 interval=CLOSED_OPEN)).end]
+                    after = original_data.ix[to_pandas_closed_closed(DateRange(new_data.index[-1].to_pydatetime(),
+                                                                               None,
+                                                                               interval=OPEN_CLOSED)).start:]
                     new_data = before.append(new_data).append(after)
 
                 mt.write(symbol, new_data, metadata=version.metadata)
@@ -51,6 +57,7 @@ def main():
     Example:
         arctic_copy_data --log "Copying data" --src user.library@host1 --dest user.library@host2 symbol1 symbol2
     """
+    setup_logging()
     p = argparse.ArgumentParser(usage=usage)
     p.add_argument("--src", required=True, help="Source MongoDB like: library@hostname:port")
     p.add_argument("--dest", required=True, help="Destination MongoDB like: library@hostname:port")
@@ -95,6 +102,5 @@ def main():
         copy_symbol(required_symbols)
 
 
-
 if __name__ == '__main__':
     main()

arctic/scripts/arctic_create_user.py

@@ -2,10 +2,10 @@
 import pymongo
 import uuid
 import base64
-import sys
 
 from ..auth import get_auth, authenticate
 from ..hooks import get_mongodb_uri
+from .utils import setup_logging
 
 
 def main():
@@ -14,10 +14,10 @@ def main():
     Create the user's personal Arctic database, and adds them, read-only
     to the central admin database.
     """
-
+    setup_logging()
     parser = optparse.OptionParser(usage=usage)
     parser.add_option("--host", default='localhost', help="Hostname, or clustername. Default: localhost")
-    parser.add_option("--password",dest="password", default=None, help="Password. Default: random")
+    parser.add_option("--password", dest="password", default=None, help="Password. Default: random")
     parser.add_option("--admin-write", dest="admin", action='store_false', default=True,
                       help="Give write access to the admin DB. Default: False")
     parser.add_option("--dryrun", "-n", dest="dryrun", action="store_true", help="Don't really do anything", default=False)
@@ -29,12 +29,10 @@ def main():
     c = pymongo.MongoClient(get_mongodb_uri(opts.host))
     credentials = get_auth(opts.host, 'admin', 'admin')
     if not credentials:
-        print >>sys.stderr, "You have no admin credentials for instance '%s'" % (opts.host)
-        return
+        raise ValueError("You have no admin credentials for instance '%s'" % (opts.host))
 
     if not authenticate(c.admin, credentials.user, credentials.password):
-        print >>sys.stderr, "Failed to authenticate to '%s' as '%s'" % (opts.host, credentials.user)
-        return
+        raise ValueError("Failed to authenticate to '%s' as '%s'" % (opts.host, credentials.user))
 
     for user in args:
 

arctic/scripts/arctic_delete_library.py

@@ -1,20 +1,23 @@
 import optparse
 import pymongo
+import logging
 
-from ..logging import logger
 from ..hooks import get_mongodb_uri
 from ..arctic import Arctic
-from .utils import do_db_auth
+from .utils import do_db_auth, setup_logging
+
+logger = logging.getLogger(__name__)
 
 
 def main():
     usage = """usage: %prog [options]
-    
+
     Deletes the named library from a user's database.
 
     Example:
         %prog --host=hostname --library=arctic_jblackburn.my_library
     """
+    setup_logging()
 
     parser = optparse.OptionParser(usage=usage)
     parser.add_option("--host", default='localhost', help="Hostname, or clustername. Default: localhost")

arctic/scripts/arctic_enable_sharding.py

@@ -6,13 +6,15 @@
 from ..hooks import get_mongodb_uri
 from .._util import enable_sharding
 from ..auth import authenticate
+from .utils import setup_logging
 
 
 def main():
     usage = """usage: %prog [options] arg1=value, arg2=value
-    
+
     Enables sharding on the specified arctic library.
     """
+    setup_logging()
 
     parser = optparse.OptionParser(usage=usage)
     parser.add_option("--host", default='localhost', help="Hostname, or clustername. Default: localhost")