diff --git a/bentoml/configuration/default_bentoml.cfg b/bentoml/configuration/default_bentoml.cfg index 4ce8878a6b0..e0b1afefdfa 100644 --- a/bentoml/configuration/default_bentoml.cfg +++ b/bentoml/configuration/default_bentoml.cfg @@ -27,6 +27,10 @@ default_namespace = BENTOML prometheus_multiproc_dir = {BENTOML_HOME}/prometheus_multiproc_dir [logging] +logging_config = {BENTOML_HOME}/logging.yml +console_logging_enabled = true +file_logging_enabled = true + level = INFO log_format = [%%(asctime)s] %%(levelname)s - %%(message)s dev_log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s @@ -34,8 +38,6 @@ dev_log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - % # the base file directory where bentoml store all its log files base_log_dir = {BENTOML_HOME}/logs/ -log_request_image_files = True - prediction_log_filename = prediction.log prediction_log_json_format = "%%(service_name)s %%(service_version)s %%(api)s %%(request_id)s %%(task)s %%(result)s %%(asctime)s" @@ -44,7 +46,6 @@ feedback_log_json_format = "%%(service_name)s %%(service_version)s %%(request_id yatai_web_server_log_filename = yatai_web_server.log - [tracing] # example: http://127.0.0.1:9411/api/v1/spans zipkin_api_url = diff --git a/bentoml/utils/log.py b/bentoml/utils/log.py index 21d11a90dd0..9d2eb8d5abb 100644 --- a/bentoml/utils/log.py +++ b/bentoml/utils/log.py @@ -19,6 +19,7 @@ from bentoml import config from bentoml.configuration import get_debug_mode +from bentoml.utils.ruamel_yaml import YAML def get_logging_config_dict(logging_level, base_log_directory): @@ -33,6 +34,61 @@ def get_logging_config_dict(logging_level, base_log_directory): FEEDBACK_LOG_FILENAME = conf.get("feedback_log_filename") FEEDBACK_LOG_JSON_FORMAT = conf.get("feedback_log_json_format") + MEGABYTES = 1024 * 1024 + + handlers = {} + bentoml_logger_handlers = [] + prediction_logger_handlers = [] + feedback_logger_handlers = [] + if conf.getboolean("console_logging_enabled"): + handlers.update( + { + "console": { + "level": logging_level, + "formatter": "console", + "class": "logging.StreamHandler", + "stream": sys.stdout, + } + } + ) + bentoml_logger_handlers.append("console") + prediction_logger_handlers.append("console") + feedback_logger_handlers.append("console") + if conf.getboolean("file_logging_enabled"): + handlers.update( + { + "local": { + "level": logging_level, + "formatter": "dev", + "class": "logging.handlers.RotatingFileHandler", + "filename": os.path.join(base_log_directory, "active.log"), + "maxBytes": 100 * MEGABYTES, + "backupCount": 2, + }, + "prediction": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "prediction", + "level": "INFO", + "filename": os.path.join( + base_log_directory, PREDICTION_LOG_FILENAME + ), + "maxBytes": 100 * MEGABYTES, + "backupCount": 10, + }, + "feedback": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "feedback", + "level": "INFO", + "filename": os.path.join(base_log_directory, FEEDBACK_LOG_FILENAME), + "maxBytes": 100 * MEGABYTES, + "backupCount": 10, + }, + } + ) + bentoml_logger_handlers.append("local") + prediction_logger_handlers.append("prediction") + feedback_logger_handlers.append("feedback") + return { "version": 1, "disable_existing_loggers": False, @@ -48,51 +104,20 @@ def get_logging_config_dict(logging_level, base_log_directory): "fmt": FEEDBACK_LOG_JSON_FORMAT, }, }, - "handlers": { - "console": { - "level": logging_level, - "formatter": "console", - "class": "logging.StreamHandler", - "stream": sys.stdout, - }, - "local": { - "level": logging_level, - "formatter": "dev", - "class": "logging.handlers.RotatingFileHandler", - "filename": os.path.join(base_log_directory, "active.log"), - "maxBytes": 100 * 1000 * 1000, - "backupCount": 2, - }, - "prediction": { - "class": "logging.handlers.RotatingFileHandler", - "formatter": "prediction", - "level": "INFO", - "filename": os.path.join(base_log_directory, PREDICTION_LOG_FILENAME), - "maxBytes": 100 * 1000 * 1000, - "backupCount": 10, - }, - "feedback": { - "class": "logging.handlers.RotatingFileHandler", - "formatter": "feedback", - "level": "INFO", - "filename": os.path.join(base_log_directory, FEEDBACK_LOG_FILENAME), - "maxBytes": 100 * 1000 * 1000, - "backupCount": 10, - }, - }, + "handlers": handlers, "loggers": { "bentoml": { - "handlers": ["console", "local"], + "handlers": bentoml_logger_handlers, "level": logging_level, "propagate": False, }, "bentoml.prediction": { - "handlers": ["prediction", "console"], + "handlers": prediction_logger_handlers, "level": "INFO", "propagate": False, }, "bentoml.feedback": { - "handlers": ["feedback", "console"], + "handlers": feedback_logger_handlers, "level": "INFO", "propagate": False, }, @@ -101,16 +126,29 @@ def get_logging_config_dict(logging_level, base_log_directory): def configure_logging(logging_level=None): - if logging_level is None: - logging_level = config("logging").get("LEVEL").upper() - if "LOGGING_LEVEL" in config("logging"): - # Support legacy config name e.g. BENTOML__LOGGING__LOGGING_LEVEL=debug - logging_level = config("logging").get("LOGGING_LEVEL").upper() - - if get_debug_mode(): - logging_level = logging.getLevelName(logging.DEBUG) - base_log_dir = os.path.expanduser(config("logging").get("BASE_LOG_DIR")) Path(base_log_dir).mkdir(parents=True, exist_ok=True) - logging_config = get_logging_config_dict(logging_level, base_log_dir) - logging.config.dictConfig(logging_config) + if os.path.exists(config("logging").get("logging_config")): + logging_config_path = config("logging").get("logging_config") + with open(logging_config_path, "rb") as f: + logging_config = YAML().load(f.read()) + logging.config.dictConfig(logging_config) + logging.getLogger(__name__).debug( + "Loaded logging configuration from %s." % logging_config_path + ) + else: + if logging_level is None: + logging_level = config("logging").get("LEVEL").upper() + if "LOGGING_LEVEL" in config("logging"): + # Support legacy config name e.g. BENTOML__LOGGING__LOGGING_LEVEL=debug + logging_level = config("logging").get("LOGGING_LEVEL").upper() + + if get_debug_mode(): + logging_level = logging.getLevelName(logging.DEBUG) + + logging_config = get_logging_config_dict(logging_level, base_log_dir) + logging.config.dictConfig(logging_config) + logging.getLogger(__name__).debug( + "Loaded logging configuration from default configuration " + + "and environment variables." + ) diff --git a/docs/source/guides/configuration.rst b/docs/source/guides/configuration.rst new file mode 100644 index 00000000000..e9c610b2e2b --- /dev/null +++ b/docs/source/guides/configuration.rst @@ -0,0 +1,63 @@ +.. _configuration-page: + +Configuration +============= + +BentoML can be configured through configuration properties defined in the `default_bentoml.cfg `_. +The values of configuration properties are applied in the following precedence order. + +- Environment Variables +- User Defined Configuration File +- BentoML Defaults + +Environment Variables +^^^^^^^^^^^^^^^^^^^^^ + +To override a configuration property, environment variables should be named in the following convention, +`BENTOML__
__`, in upper case letters. + +For example, to override the `level` property to `ERROR` in the `logging` section of the configuration, user +should define an environment variable named `BENTOML__LOGGING__LEVEL` with value `ERROR`. + + +.. code-block:: cfg + :caption: default_bentoml.cfg + + [logging] + level = INFO + +See Docker example below for setting the environment variable of logging level. + +.. code-block:: shell + + $ docker run -e BENTOML__LOGGING__LEVEL=ERROR + +User Defined Configuration File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A user defined configuration file, in the same format as the +`default_bentoml.cfg `_ +can be placed under the BentoML home directory with the file name `bentoml.cfg`, to override existing configuration +properties. + +The example below, overrides both `level` and `file_logging_enabled` properties in the `logging` section, to change +logging level to `WARN` and disable file based logging. + +.. code-block:: cfg + :caption: {BENTOML_HOME}/bentoml.cfg + + [logging] + level = WARN + file_logging_enabled = false + +See Docker example below for injecting the BentoML configuration file into the container. + +.. code-block:: shell + + $ docker run -v /local/path/to/bentoml.cfg:{BENTOML_HOME}/bentoml.cfg + +BentoML Defaults +^^^^^^^^^^^^^^^^ + +Any non-overridden properties will fallback to the default values defined in +`default_bentoml.cfg `_. \ No newline at end of file diff --git a/docs/source/guides/index.rst b/docs/source/guides/index.rst index 2ac0c02df6b..9e190f4e0a6 100644 --- a/docs/source/guides/index.rst +++ b/docs/source/guides/index.rst @@ -10,9 +10,10 @@ and ask in the bentoml-users channel. .. toctree:: :glob: + configuration + logging batch_serving monitoring - logging micro_batching custom_artifact custom_input_adapter diff --git a/docs/source/guides/logging.rst b/docs/source/guides/logging.rst index e77598d0858..d3ea7b6abe0 100644 --- a/docs/source/guides/logging.rst +++ b/docs/source/guides/logging.rst @@ -1,7 +1,154 @@ -Request Logging -=============== +Logging +======= -This part of the BentoML documentation is a work in progress. If you have any questions -related to this, please join -`the BentoML Slack community `_ -and ask in the bentoml-users channel. \ No newline at end of file +BentoML uses standard `Python logging `_ libraries and provides basic +logging customization through `logging` section in `bentoml.cfg` under the BentoML home directory. Refer to the +:ref:`configuration guide ` on how override configuration properties. See +`default_bentoml.cfg `_ +for a list of override-able properties. + +For advanced logging customization, user can provide full logging configurations in `logging.yml`, placed under +the BentoML home directory. For example, `logging.yml` configuration file can be injected into the Docker container +through the following command. + +.. code-block:: shell + + $ docker run -v /local/path/to/logging.yml:{BENTOML_HOME}/logging.yml + +Please see below configuration examples of different logging scenarios in YAML format. + +.. code-block:: yaml + :caption: Enable INFO+ console logging but only WARN+ file logging. + + version: 1 + disable_existing_loggers: False + formatters: + console: + format: '[%(asctime)s] %(levelname)s - %(message)s' + dev: + format: '[%(asctime)s] {{%(filename)s:%(lineno)d}} %(levelname)s - %(message)s' + handlers: + console: + level: INFO + formatter: console + class: logging.StreamHandler + stream: ext://sys.stdout + local: + level: WARN + formatter: dev + class: logging.handlers.RotatingFileHandler + filename: '/var/log/bentoml/active.log' + maxBytes: 104857600 + backupCount: 2 + loggers: + bentoml: + handlers: [console, local] + level: INFO + propagate: False + bentoml.prediction: + handlers: [console] + level: INFO + propagate: False + bentoml.feedback: + handlers: [console] + level: INFO + propagate: False + +.. code-block:: yaml + :caption: Disable all logging except prediction and feedback file logging. + + version: 1 + disable_existing_loggers: False + formatters: + prediction: + (): pythonjsonlogger.jsonlogger.JsonFormatter + fmt: '%(service_name)s %(service_version)s %(api)s %(request_id)s %(task)s %(result)s %(asctime)s' + feedback: + (): pythonjsonlogger.jsonlogger.JsonFormatter + fmt: '%(service_name)s %(service_version)s %(request_id)s %(asctime)s' + handlers: + prediction: + class: logging.handlers.RotatingFileHandler + formatter: prediction + level: INFO + filename: '/var/log/bentoml/prediction.log' + maxBytes: 104857600 + backupCount: 10 + feedback: + class: logging.handlers.RotatingFileHandler + formatter: feedback + level: INFO + filename: '/var/log/bentoml/feedback.log' + maxBytes: 104857600 + backupCount: 10 + loggers: + bentoml: + handlers: [] + level: INFO + propagate: False + bentoml.prediction: + handlers: [prediction] + level: INFO + propagate: False + bentoml.feedback: + handlers: [feedback] + level: INFO + propagate: False + + +.. code-block:: yaml + :caption: Default logging configuration. + + version: 1 + disable_existing_loggers: False + formatters: + console: + format: '[%(asctime)s] %(levelname)s - %(message)s' + dev: + format: '[%(asctime)s] {{%(filename)s:%(lineno)d}} %(levelname)s - %(message)s' + prediction: + (): pythonjsonlogger.jsonlogger.JsonFormatter + fmt: '%(service_name)s %(service_version)s %(api)s %(request_id)s %(task)s %(result)s %(asctime)s' + feedback: + (): pythonjsonlogger.jsonlogger.JsonFormatter + fmt: '%(service_name)s %(service_version)s %(request_id)s %(asctime)s' + handlers: + console: + level: INFO + formatter: console + class: logging.StreamHandler + stream: ext://sys.stdout + local: + level: INFO + formatter: dev + class: logging.handlers.RotatingFileHandler + filename: '/var/log/bentoml/active.log' + maxBytes: 104857600 + backupCount: 2 + prediction: + class: logging.handlers.RotatingFileHandler + formatter: prediction + level: INFO + filename: '/var/log/bentoml/prediction.log' + maxBytes: 104857600 + backupCount: 10 + feedback: + class: logging.handlers.RotatingFileHandler + formatter: feedback + level: INFO + filename: '/var/log/bentoml/feedback.log' + maxBytes: 104857600 + backupCount: 10 + loggers: + bentoml: + handlers: [console, local] + level: INFO + propagate: False + bentoml.prediction: + handlers: [console, prediction] + level: INFO + propagate: False + bentoml.feedback: + handlers: [console, feedback] + level: INFO + propagate: False diff --git a/tests/utils/test_log.py b/tests/utils/test_log.py new file mode 100644 index 00000000000..f59a33c2cfd --- /dev/null +++ b/tests/utils/test_log.py @@ -0,0 +1,144 @@ +import logging +import os +import tempfile + +from bentoml.utils.log import configure_logging + + +def test_configure_logging_default(): + configure_logging() + + bentoml_logger = logging.getLogger("bentoml") + assert bentoml_logger.level == logging.INFO + assert bentoml_logger.propagate is False + assert len(bentoml_logger.handlers) == 2 + assert bentoml_logger.handlers[0].name == "console" + assert bentoml_logger.handlers[1].name == "local" + + prediction_logger = logging.getLogger("bentoml.prediction") + assert prediction_logger.level == logging.INFO + assert prediction_logger.propagate is False + assert len(prediction_logger.handlers) == 2 + assert prediction_logger.handlers[0].name == "console" + assert prediction_logger.handlers[1].name == "prediction" + + feedback_logger = logging.getLogger("bentoml.feedback") + assert feedback_logger.level == logging.INFO + assert feedback_logger.propagate is False + assert len(feedback_logger.handlers) == 2 + assert feedback_logger.handlers[0].name == "console" + assert feedback_logger.handlers[1].name == "feedback" + + +def test_configure_logging_custom_level(): + configure_logging(logging.ERROR) + + bentoml_logger = logging.getLogger("bentoml") + assert bentoml_logger.level == logging.ERROR + assert bentoml_logger.propagate is False + assert len(bentoml_logger.handlers) == 2 + assert bentoml_logger.handlers[0].name == "console" + assert bentoml_logger.handlers[1].name == "local" + + prediction_logger = logging.getLogger("bentoml.prediction") + assert prediction_logger.level == logging.INFO + assert prediction_logger.propagate is False + assert len(prediction_logger.handlers) == 2 + assert prediction_logger.handlers[0].name == "console" + assert prediction_logger.handlers[1].name == "prediction" + + feedback_logger = logging.getLogger("bentoml.feedback") + assert feedback_logger.level == logging.INFO + assert feedback_logger.propagate is False + assert len(feedback_logger.handlers) == 2 + assert feedback_logger.handlers[0].name == "console" + assert feedback_logger.handlers[1].name == "feedback" + + +def test_configure_logging_file_disabled(): + os.environ["BENTOML__LOGGING__CONSOLE_LOGGING_ENABLED"] = "false" + + configure_logging() + + bentoml_logger = logging.getLogger("bentoml") + assert bentoml_logger.level == logging.INFO + assert bentoml_logger.propagate is False + assert len(bentoml_logger.handlers) == 1 + assert bentoml_logger.handlers[0].name == "local" + + prediction_logger = logging.getLogger("bentoml.prediction") + assert prediction_logger.level == logging.INFO + assert prediction_logger.propagate is False + assert len(prediction_logger.handlers) == 1 + assert prediction_logger.handlers[0].name == "prediction" + + feedback_logger = logging.getLogger("bentoml.feedback") + assert feedback_logger.level == logging.INFO + assert feedback_logger.propagate is False + assert len(feedback_logger.handlers) == 1 + assert feedback_logger.handlers[0].name == "feedback" + + del os.environ["BENTOML__LOGGING__CONSOLE_LOGGING_ENABLED"] + + +def test_configure_logging_console_disabled(): + os.environ["BENTOML__LOGGING__FILE_LOGGING_ENABLED"] = "false" + + configure_logging() + + bentoml_logger = logging.getLogger("bentoml") + assert bentoml_logger.level == logging.INFO + assert bentoml_logger.propagate is False + assert len(bentoml_logger.handlers) == 1 + assert bentoml_logger.handlers[0].name == "console" + + prediction_logger = logging.getLogger("bentoml.prediction") + assert prediction_logger.level == logging.INFO + assert prediction_logger.propagate is False + assert len(prediction_logger.handlers) == 1 + assert prediction_logger.handlers[0].name == "console" + + feedback_logger = logging.getLogger("bentoml.feedback") + assert feedback_logger.level == logging.INFO + assert feedback_logger.propagate is False + assert len(feedback_logger.handlers) == 1 + assert feedback_logger.handlers[0].name == "console" + + del os.environ["BENTOML__LOGGING__FILE_LOGGING_ENABLED"] + + +def test_configure_logging_yaml(): + logging_config = tempfile.NamedTemporaryFile(delete=False) + logging_config.write( + b""" +version: 1 +disable_existing_loggers: False +formatters: + test_formatter: + format: '[%(asctime)s] %(levelname)s - %(message)s' +handlers: + test_handler: + level: WARN + formatter: test_formatter + class: logging.StreamHandler + stream: ext://sys.stdout +loggers: + test_logger: + handlers: [test_handler] + level: WARN + propagate: False""" + ) + logging_config.close() + + os.environ["BENTOML__LOGGING__LOGGING_CONFIG"] = logging_config.name + + configure_logging() + + bentoml_logger = logging.getLogger("test_logger") + assert bentoml_logger.level == logging.WARN + assert bentoml_logger.propagate is False + assert len(bentoml_logger.handlers) == 1 + assert bentoml_logger.handlers[0].name == "test_handler" + + os.remove(logging_config.name) + del os.environ["BENTOML__LOGGING__LOGGING_CONFIG"]