img-processor/_log.py

# coding: utf-8

import logging
import types
import datetime
import os
from collections import namedtuple
from enum import Enum
import simplejson
from pythonjsonlogger import jsonlogger


###############################################################################

class ServiceType(Enum):
    AGENT = 1
    TASK = 2
    EXPORT = 3


class EventType(Enum):
    LOG = 1
    TASK_STARTED = 2
    TASK_FINISHED = 3
    TASK_STOPPED = 4
    TASK_CRASHED = 5
    STEP_COMPLETE = 6
    PROGRESS = 7
    METRICS = 8
    AGENT_READY = 9
    TASK_VERIFIED = 10
    TASK_REJECTED = 11
    TASK_SUBMITTED = 12
    TASK_SCHEDULED = 13
    AGENT_EXITED = 14
    LOGA = 15
    STEP_STARTED = 16
    FILES_UPLOADED = 17

###############################################################################
# predefined levels


# level name: level, default exc_info, description
LogLevelSpec = namedtuple('LogLevelSpec', [
    'int',
    'add_exc_info',
    'descr',
])

LOGGING_LEVELS = {
    'FATAL': LogLevelSpec(50, True, 'Critical error'),
    'ERROR': LogLevelSpec(40, True, 'Error'),  # may be shown to end user
    'WARN': LogLevelSpec(30, False, 'Warning'),  # may be shown to end user
    'INFO': LogLevelSpec(20, False, 'Info'),  # may be shown to end user
    'DEBUG': LogLevelSpec(10, False, 'Debug'),
    'TRACE': LogLevelSpec(5, False, 'Trace'),
}


def _set_logging_levels(levels, the_logger):
    for lvl_name, (lvl, def_exc_info, _) in levels.items():
        logging.addLevelName(lvl, lvl_name.upper())  # two mappings

        def construct_logger_member(lvl_val, default_exc_info):
            return lambda self, msg, *args, exc_info=default_exc_info, **kwargs: \
                self.log(lvl_val,
                         msg,
                         *args,
                         exc_info=exc_info,
                         **kwargs)

        func = construct_logger_member(lvl, def_exc_info)
        bound_method = types.MethodType(func, the_logger)
        setattr(the_logger, lvl_name.lower(), bound_method)


###############################################################################


def _get_default_logging_fields():
    supported_keys = [
        'asctime',
        # 'created',
        # 'filename',
        # 'funcName',
        'levelname',
        # 'levelno',
        # 'lineno',
        # 'module',
        # 'msecs',
        'message',
        # 'name',
        # 'pathname',
        # 'process',
        # 'processName',
        # 'relativeCreated',
        # 'thread',
        # 'threadName'
    ]
    return ' '.join(['%({0:s})'.format(k) for k in supported_keys])


def dumps_ignore_nan(obj, *args, **kwargs):
    return simplejson.dumps(obj, ignore_nan=True, ensure_ascii=False, *args, **kwargs)


class CustomJsonFormatter(jsonlogger.JsonFormatter):
    additional_fields = {}

    def __init__(self, format_string):
        super().__init__(format_string, json_serializer=dumps_ignore_nan)

    def process_log_record(self, log_record):
        log_record['timestamp'] = log_record.pop('asctime', None)

        levelname = log_record.pop('levelname', None)
        if levelname is not None:
            log_record['level'] = levelname.lower()

        e_info = log_record.pop('exc_info', None)
        if e_info is not None:
            if e_info == 'NoneType: None':  # python logger is not ok here
                pass
            else:
                log_record['stack'] = e_info.split('\n')

        return jsonlogger.JsonFormatter.process_log_record(self, log_record)

    def add_fields(self, log_record, record, message_dict):
        super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)

        for field, val in CustomJsonFormatter.additional_fields.items():
            if (val is not None) and (field not in log_record):
                log_record[field] = val

    def formatTime(self, record, datefmt=None):
        ct = datetime.datetime.fromtimestamp(record.created)
        t = ct.strftime('%Y-%m-%dT%H:%M:%S')
        s = '%s.%03dZ' % (t, record.msecs)
        return s


def _construct_logger(the_logger, loglevel_text):
    for handler in the_logger.handlers:
        the_logger.removeHandler(handler)

    _set_logging_levels(LOGGING_LEVELS, the_logger)

    the_logger.setLevel(loglevel_text.upper())

    log_handler = logging.StreamHandler()
    add_logger_handler(the_logger, log_handler)

    the_logger.propagate = False


###############################################################################


def add_logger_handler(the_logger, log_handler):  # default format
    logger_fmt_string = _get_default_logging_fields()
    formatter = CustomJsonFormatter(logger_fmt_string)
    log_handler.setFormatter(formatter)
    the_logger.addHandler(log_handler)


def add_default_logging_into_file(the_logger, log_dir):
    fname = 'log_{}.txt'.format(
        datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
    ofpath = os.path.join(log_dir, fname)

    log_handler_file = logging.FileHandler(filename=ofpath)
    add_logger_handler(the_logger, log_handler_file)


# runs on all formatters
def change_formatters_default_values(the_logger, field_name, value):
    for handler in the_logger.handlers:
        hfaf = handler.formatter.additional_fields
        if value is not None:
            hfaf[field_name] = value
        else:
            hfaf.pop(field_name, None)


def set_global_logger():
    loglevel = os.getenv('LOG_LEVEL', 'TRACE')  # use the env to set loglevel
    the_logger = logging.getLogger('logger')  # optional logger name
    _construct_logger(the_logger, loglevel)
    return the_logger


def get_task_logger(task_id):
    loglevel = os.getenv('LOG_LEVEL', 'TRACE')  # use the env to set loglevel
    logger_name = 'task_{}'.format(task_id)
    the_logger = logging.getLogger(logger_name)  # optional logger name
    _construct_logger(the_logger, loglevel)
    return the_logger


logger = set_global_logger()