[Enhancement] Allow full logging customization through YAML configura…

…tion (bentoml#1009)

bentoml/configuration/default_bentoml.cfg

@@ -27,6 +27,8 @@ default_namespace = BENTOML
 prometheus_multiproc_dir = {BENTOML_HOME}/prometheus_multiproc_dir
 
 [logging]
+logging_config = {BENTOML_HOME}/logging.yaml
+
 level = INFO
 log_format = [%%(asctime)s] %%(levelname)s - %%(message)s
 dev_log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s

bentoml/utils/log.py

@@ -15,6 +15,7 @@
 import os
 import sys
 import logging.config
+import yaml
 from pathlib import Path
 
 from bentoml import config
@@ -33,6 +34,8 @@ def get_logging_config_dict(logging_level, base_log_directory):
     FEEDBACK_LOG_FILENAME = conf.get("feedback_log_filename")
     FEEDBACK_LOG_JSON_FORMAT = conf.get("feedback_log_json_format")
 
+    MEGABYTES = 1024 * 1024
+
     return {
         "version": 1,
         "disable_existing_loggers": False,
@@ -60,23 +63,23 @@ def get_logging_config_dict(logging_level, base_log_directory):
                 "formatter": "dev",
                 "class": "logging.handlers.RotatingFileHandler",
                 "filename": os.path.join(base_log_directory, "active.log"),
-                "maxBytes": 100 * 1000 * 1000,
+                "maxBytes": 100 * MEGABYTES,
                 "backupCount": 2,
             },
             "prediction": {
                 "class": "logging.handlers.RotatingFileHandler",
                 "formatter": "prediction",
                 "level": "INFO",
                 "filename": os.path.join(base_log_directory, PREDICTION_LOG_FILENAME),
-                "maxBytes": 100 * 1000 * 1000,
+                "maxBytes": 100 * MEGABYTES,
                 "backupCount": 10,
             },
             "feedback": {
                 "class": "logging.handlers.RotatingFileHandler",
                 "formatter": "feedback",
                 "level": "INFO",
                 "filename": os.path.join(base_log_directory, FEEDBACK_LOG_FILENAME),
-                "maxBytes": 100 * 1000 * 1000,
+                "maxBytes": 100 * MEGABYTES,
                 "backupCount": 10,
             },
         },
@@ -101,16 +104,21 @@ def get_logging_config_dict(logging_level, base_log_directory):
 
 
 def configure_logging(logging_level=None):
-    if logging_level is None:
-        logging_level = config("logging").get("LEVEL").upper()
-        if "LOGGING_LEVEL" in config("logging"):
-            # Support legacy config name e.g. BENTOML__LOGGING__LOGGING_LEVEL=debug
-            logging_level = config("logging").get("LOGGING_LEVEL").upper()
-
-    if get_debug_mode():
-        logging_level = logging.getLevelName(logging.DEBUG)
-
-    base_log_dir = os.path.expanduser(config("logging").get("BASE_LOG_DIR"))
-    Path(base_log_dir).mkdir(parents=True, exist_ok=True)
-    logging_config = get_logging_config_dict(logging_level, base_log_dir)
+    if os.path.exists(config("logging").get("logging_config")):
+        logging_config_path = config("logging").get("logging_config")
+        with open(logging_config_path, "rb") as f:
+            logging_config = yaml.safe_load(f.read())
+    else:
+        if logging_level is None:
+            logging_level = config("logging").get("LEVEL").upper()
+            if "LOGGING_LEVEL" in config("logging"):
+                # Support legacy config name e.g. BENTOML__LOGGING__LOGGING_LEVEL=debug
+                logging_level = config("logging").get("LOGGING_LEVEL").upper()
+
+        if get_debug_mode():
+            logging_level = logging.getLevelName(logging.DEBUG)
+
+        base_log_dir = os.path.expanduser(config("logging").get("BASE_LOG_DIR"))
+        Path(base_log_dir).mkdir(parents=True, exist_ok=True)
+        logging_config = get_logging_config_dict(logging_level, base_log_dir)
     logging.config.dictConfig(logging_config)

docs/source/guides/logging.rst

@@ -1,5 +1,87 @@
-Request Logging
-===============
+Logging
+=======
+
+BentoML provides basic logging customization through `bentoml.cfg` under the BentoML home directory. See below for configurable fields and default values.
+
+.. code-block:: init
+
+    [logging]
+    logging_config = {BENTOML_HOME}/logging.yaml
+
+    level = INFO
+    log_format = [%%(asctime)s] %%(levelname)s - %%(message)s
+    dev_log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s
+
+    # the base file directory where bentoml store all its log files
+    base_log_dir = {BENTOML_HOME}/logs/
+
+    log_request_image_files = True
+
+    prediction_log_filename = prediction.log
+    prediction_log_json_format = "%%(service_name)s %%(service_version)s %%(api)s %%(request_id)s %%(task)s %%(result)s %%(asctime)s"
+
+    feedback_log_filename = feedback.log
+    feedback_log_json_format = "%%(service_name)s %%(service_version)s %%(request_id)s %%(asctime)s"
+
+    yatai_web_server_log_filename = yatai_web_server.log    
+
+For advanced logging customization, user can provide full logging configuration in `logging.yaml`, placed under BentoML home directory. Please see below for the default logging configuration in yaml format for reference.
+
+.. code-block:: yaml
+
+    version: 1
+    disable_existing_loggers: False
+    formatters:
+        console:
+            format: '[%(asctime)s] %(levelname)s - %(message)s'
+        dev: 
+            format: '[%(asctime)s] {{%(filename)s:%(lineno)d}} %(levelname)s - %(message)s'
+        prediction:
+            (): pythonjsonlogger.jsonlogger.JsonFormatter
+            fmt: '%(service_name)s %(service_version)s %(api)s %(request_id)s %(task)s %(result)s %(asctime)s'
+        feedback:
+            (): pythonjsonlogger.jsonlogger.JsonFormatter
+            fmt: '%(service_name)s %(service_version)s %(request_id)s %(asctime)s'
+    handlers:
+        console:
+            level: INFO
+            formatter: console
+            class: logging.StreamHandler
+            stream: ext://sys.stdout
+        local:
+            level: INFO
+            formatter: dev
+            class: logging.handlers.RotatingFileHandler
+            filename: '{BENTOML_HOME}/logs/active.log'
+            maxBytes: 104857600
+            backupCount: 2
+        prediction:
+            class: logging.handlers.RotatingFileHandler
+            formatter: prediction
+            level: INFO
+            filename: '{BENTOML_HOME}/logs/prediction.log'
+            maxBytes: 104857600
+            backupCount: 10
+        feedback:
+            class: logging.handlers.RotatingFileHandler
+            formatter: feedback
+            level: INFO
+            filename: '{BENTOML_HOME}/logs/feedback.log'
+            maxBytes: 104857600
+            backupCount: 10
+    loggers:
+        bentoml:
+            handlers: [console, local]
+            level: INFO
+            propagate: False
+        bentoml.prediction:
+            handlers: [prediction, console]
+            level: INFO
+            propagate: False
+        bentoml.feedback:
+            handlers: [feedback, console]
+            level: INFO
+            propagate: False
 
 This part of the BentoML documentation is a work in progress. If you have any questions
 related to this, please join