diff --git a/thingsboard_gateway/config/logs.json b/thingsboard_gateway/config/logs.json index 0f28055c..037b4523 100644 --- a/thingsboard_gateway/config/logs.json +++ b/thingsboard_gateway/config/logs.json @@ -16,14 +16,14 @@ "stream": "ext://sys.stdout" }, "databaseHandler": { - "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/database.log", "backupCount": 1, "encoding": "utf-8" }, "serviceHandler": { - "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/service.log", "backupCount": 7, @@ -32,7 +32,7 @@ "encoding": "utf-8" }, "connectorHandler": { - "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/connector.log", "backupCount": 7, @@ -41,7 +41,7 @@ "encoding": "utf-8" }, "converterHandler": { - "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/converter.log", "backupCount": 7, @@ -50,7 +50,7 @@ "encoding": "utf-8" }, "tb_connectionHandler": { - "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/tb_connection.log", "backupCount": 7, @@ -59,7 +59,7 @@ "encoding": "utf-8" }, "storageHandler": { - "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/storage.log", "backupCount": 7, @@ -68,7 +68,7 @@ "encoding": "utf-8" }, "extensionHandler": { - "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler", + "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler", "formatter": "LogFormatter", "filename": "./logs/extension.log", "backupCount": 7, diff --git a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py index e79f367d..c7f6a01b 100644 --- a/thingsboard_gateway/connectors/bacnet/bacnet_connector.py +++ b/thingsboard_gateway/connectors/bacnet/bacnet_connector.py @@ -58,10 +58,10 @@ def __init__(self, gateway, config, connector_type): self.__log = init_logger(self.__gateway, self.name, log_level, enable_remote_logging=remote_logging, - is_connector_logger=True, connector_name=self.name) - self.__converter_log = init_logger(self.__gateway, self.name, log_level, + is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, self.name + '_converter', log_level, enable_remote_logging=remote_logging, - is_converter_logger=True, connector_name=self.name) + is_converter_logger=True, attr_name=self.name) self.__log.info('Starting BACnet connector...') if BackwardCompatibilityAdapter.is_old_config(config): diff --git a/thingsboard_gateway/connectors/ble/ble_connector.py b/thingsboard_gateway/connectors/ble/ble_connector.py index 751aa9b0..1ad5fb77 100644 --- a/thingsboard_gateway/connectors/ble/ble_connector.py +++ b/thingsboard_gateway/connectors/ble/ble_connector.py @@ -48,8 +48,14 @@ def __init__(self, gateway, config, connector_type): self.__config = config self.__id = self.__config.get('id') self.name = self.__config.get("name", 'BLE Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5))) - self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + self.__log = init_logger(self.__gateway, self.name, + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=self.name) self.daemon = True @@ -138,7 +144,7 @@ def __process_data(self): StatisticsService.count_connector_bytes(self.name, data, stat_parameter_name='connectorBytesReceived') try: - converter = converter(device_config, self.__log) + converter = converter(device_config, self.__converter_log) converted_data: ConvertedData = converter.convert(config, data) self.statistics['MessagesReceived'] = self.statistics['MessagesReceived'] + 1 self.__log.debug(converted_data) diff --git a/thingsboard_gateway/connectors/can/can_connector.py b/thingsboard_gateway/connectors/can/can_connector.py index fc3b5447..b59f6f8a 100644 --- a/thingsboard_gateway/connectors/can/can_connector.py +++ b/thingsboard_gateway/connectors/can/can_connector.py @@ -76,7 +76,12 @@ def __init__(self, gateway, config, connector_type): self.__config = config self.__id = self.__config.get('id') self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self._converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=self.name) self.__bus_conf = {} self.__bus = None self.__reconnect_count = 0 @@ -591,11 +596,11 @@ def __get_converter(self, config, need_uplink): else: if need_uplink: uplink = config.get("uplink") - return BytesCanUplinkConverter(self._log) if uplink is None \ + return BytesCanUplinkConverter(self._converter_log) if uplink is None \ else TBModuleLoader.import_module(self._connector_type, uplink) else: downlink = config.get("downlink") - return BytesCanDownlinkConverter(self._log) if downlink is None \ + return BytesCanDownlinkConverter(self._converter_log) if downlink is None \ else TBModuleLoader.import_module(self._connector_type, downlink) def get_config(self): diff --git a/thingsboard_gateway/connectors/ftp/ftp_connector.py b/thingsboard_gateway/connectors/ftp/ftp_connector.py index df36ad69..cb8720c2 100644 --- a/thingsboard_gateway/connectors/ftp/ftp_connector.py +++ b/thingsboard_gateway/connectors/ftp/ftp_connector.py @@ -49,7 +49,12 @@ def __init__(self, gateway, config, connector_type): self.__tls_support = self.__config.get("TLSSupport", False) self.name = self.__config.get("name", "".join(choice(ascii_lowercase) for _ in range(5))) self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=self.name) self.daemon = True self.__stopped = False self.__requests_in_progress = [] @@ -135,7 +140,7 @@ def __process_paths(self, ftp): time_point = timer() if time_point - path.last_polled_time >= path.poll_period or path.last_polled_time == 0: configuration = path.config - converter = FTPUplinkConverter(configuration, self.__log) + converter = FTPUplinkConverter(configuration, self.__converter_log) path.last_polled_time = time_point if '*' in path.path: diff --git a/thingsboard_gateway/connectors/modbus/modbus_connector.py b/thingsboard_gateway/connectors/modbus/modbus_connector.py index ed0898d7..a3dbe7a6 100644 --- a/thingsboard_gateway/connectors/modbus/modbus_connector.py +++ b/thingsboard_gateway/connectors/modbus/modbus_connector.py @@ -84,7 +84,12 @@ def __init__(self, gateway, config, connector_type): self.name = self.__config.get("name", 'Modbus Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5))) self.__log = init_logger(self.__gateway, config.get('name', self.name), config.get('logLevel', 'INFO'), - enable_remote_logging=config.get('enableRemoteLogging', False)) + enable_remote_logging=config.get('enableRemoteLogging', False), + is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, self.name + '_converter', + config.get('logLevel', 'INFO'), + enable_remote_logging=config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=self.name) self.__log.info('Starting Modbus Connector...') self.__id = self.__config.get('id') self.__connected = False diff --git a/thingsboard_gateway/connectors/modbus/slave.py b/thingsboard_gateway/connectors/modbus/slave.py index 0f41d7d1..6f87ce52 100644 --- a/thingsboard_gateway/connectors/modbus/slave.py +++ b/thingsboard_gateway/connectors/modbus/slave.py @@ -130,9 +130,9 @@ def __load_converter(self, config, converter_type, converter_config: Union[Dict, converter = config[converter_type + CONVERTER_PARAMETER] else: if converter_type == DOWNLINK_PREFIX: - converter = BytesModbusDownlinkConverter(converter_config, self._log) + converter = BytesModbusDownlinkConverter(converter_config, self.connector.__converter_log) else: - converter = BytesModbusUplinkConverter(converter_config, self._log) + converter = BytesModbusUplinkConverter(converter_config, self.connector.__converter_log) return converter except Exception as e: diff --git a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py index 1084104a..1b930dc5 100644 --- a/thingsboard_gateway/connectors/mqtt/mqtt_connector.py +++ b/thingsboard_gateway/connectors/mqtt/mqtt_connector.py @@ -118,6 +118,10 @@ def __init__(self, gateway, config, connector_type): config.get('logLevel', 'INFO'), enable_remote_logging=config.get('enableRemoteLogging', False), is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, config['name'] + '_converter', + config.get('logLevel', 'INFO'), + enable_remote_logging=config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=config['name']) # check if the configuration is in the old format using_old_config_format_detected = BackwardCompatibilityAdapter.is_old_config_format(config) @@ -454,7 +458,7 @@ def _on_connect(self, client, userdata, flags, result_code, *extra_params): if module: self.__log.debug('Converter %s for topic %s - found!', converter_class_name, mapping["topicFilter"]) - converter = module(mapping, self.__log) + converter = module(mapping, self.__converter_log) if sharing_id: self.__shared_custom_converters[sharing_id] = converter else: diff --git a/thingsboard_gateway/connectors/ocpp/ocpp_connector.py b/thingsboard_gateway/connectors/ocpp/ocpp_connector.py index 28484744..9a0f2ce4 100644 --- a/thingsboard_gateway/connectors/ocpp/ocpp_connector.py +++ b/thingsboard_gateway/connectors/ocpp/ocpp_connector.py @@ -69,7 +69,12 @@ def __init__(self, gateway, config, connector_type): self._gateway = gateway self.name = self._config.get("name", 'OCPP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5))) self._log = init_logger(self._gateway, self.name, self._config.get('logLevel', 'INFO'), - enable_remote_logging=self._config.get('enableRemoteLogging', False)) + enable_remote_logging=self._config.get('enableRemoteLogging', False), + is_connector_logger=True) + self._converter_log = init_logger(self._gateway, self.name + '_converter', + self._config.get('logLevel', 'INFO'), + enable_remote_logging=self._config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=self.name) self._default_converters = {'uplink': 'OcppUplinkConverter'} self._server = None @@ -184,7 +189,7 @@ async def on_connect(self, websocket, path): if is_valid: uplink_converter_name = cp_config.get('extension', self._default_converters['uplink']) cp = ChargePoint(charge_point_id, websocket, {**cp_config, 'uplink_converter_name': uplink_converter_name}, - OcppConnector._callback, self._log) + OcppConnector._callback, self._converter_log) cp.authorized = True self._log.info('Connected Charge Point with id: %s', charge_point_id) diff --git a/thingsboard_gateway/connectors/odbc/odbc_connector.py b/thingsboard_gateway/connectors/odbc/odbc_connector.py index b443ae98..85926af4 100644 --- a/thingsboard_gateway/connectors/odbc/odbc_connector.py +++ b/thingsboard_gateway/connectors/odbc/odbc_connector.py @@ -63,7 +63,12 @@ def __init__(self, gateway, config, connector_type): self.__config = config self.__id = self.__config.get('id') self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self._converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=self.name) self._connector_type = connector_type self.__stopped = False @@ -81,7 +86,7 @@ def __init__(self, gateway, config, connector_type): self.__attribute_columns = [] self.__timeseries_columns = [] - self.__converter = OdbcUplinkConverter(self._log) if not self.__config.get("converter", "") else \ + self.__converter = OdbcUplinkConverter(self._converter_log) if not self.__config.get("converter", "") else \ TBModuleLoader.import_module(self._connector_type, self.__config["converter"]) self.__configure_pyodbc() diff --git a/thingsboard_gateway/connectors/opcua/opcua_connector.py b/thingsboard_gateway/connectors/opcua/opcua_connector.py index b6f63d98..7135b393 100644 --- a/thingsboard_gateway/connectors/opcua/opcua_connector.py +++ b/thingsboard_gateway/connectors/opcua/opcua_connector.py @@ -83,7 +83,11 @@ def __init__(self, gateway: 'TBGatewayService', config, connector_type): self.__config.get('server', {}).get('mapping', [])))) self.__enable_remote_logging = self.__config.get('enableRemoteLogging', False) self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__enable_remote_logging) + enable_remote_logging=self.__enable_remote_logging, is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__enable_remote_logging, + is_converter_logger=True, attr_name=self.name) self.__replace_loggers() report_strategy = self.__config.get('reportStrategy') self.__connector_report_strategy_config = gateway.get_report_strategy_service().get_main_report_strategy() @@ -150,7 +154,7 @@ def __replace_loggers(self): 'ERROR', enable_remote_logging=self.__enable_remote_logging, is_connector_logger=True, - connector_name=self.name) + attr_name=self.name) def open(self): self.__stopped = False @@ -638,9 +642,9 @@ async def _create_new_devices(self): device_config = {**device_config, 'device_name': device_name, 'device_type': device_type} self.__device_nodes.append( Device(path=node, name=device_name, config=device_config, - converter=converter(device_config, self.__log), + converter=converter(device_config, self.__converter_log), converter_for_sub=converter(device_config, - self.__log) if self.__enable_subscriptions else None, + self.__converter_log) if self.__enable_subscriptions else None, logger=self.__log)) self.__log.info('Added device node: %s', device_name) self.__log.debug('Device nodes: %s', self.__device_nodes) diff --git a/thingsboard_gateway/connectors/request/request_connector.py b/thingsboard_gateway/connectors/request/request_connector.py index 85374969..0df80fc6 100644 --- a/thingsboard_gateway/connectors/request/request_connector.py +++ b/thingsboard_gateway/connectors/request/request_connector.py @@ -56,7 +56,12 @@ def __init__(self, gateway, config, connector_type): self.__gateway = gateway self.name = self.__config.get("name", "".join(choice(ascii_lowercase) for _ in range(5))) self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self._converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_converter_logger=True, attr_name=self.name) self.__security = HTTPBasicAuth(self.__config["security"]["username"], self.__config["security"]["password"]) if \ self.__config["security"]["type"] == "basic" else None self.__host = None @@ -173,7 +178,7 @@ def __fill_requests(self): module = TBModuleLoader.import_module(self._connector_type, endpoint["converter"]["extension"]) if module is not None: self._log.debug('Custom converter for url %s - found!', endpoint["url"]) - converter = module(endpoint, self._log) + converter = module(endpoint, self._converter_log) else: self._log.error( "\n\nCannot find extension module for %s url.\nPlease check your configuration.\n", @@ -191,18 +196,18 @@ def __fill_attribute_updates(self): for attribute_request in self.__config.get("attributeUpdates", []): if attribute_request.get("converter") is not None: converter = TBModuleLoader.import_module("request", attribute_request["converter"])(attribute_request, - self._log) + self._converter_log) else: - converter = JsonRequestDownlinkConverter(attribute_request, self._log) + converter = JsonRequestDownlinkConverter(attribute_request, self._converter_log) attribute_request_dict = {**attribute_request, "converter": converter} self.__attribute_updates.append(attribute_request_dict) def __fill_rpc_requests(self): for rpc_request in self.__config.get("serverSideRpc", []): if rpc_request.get("converter") is not None: - converter = TBModuleLoader.import_module("request", rpc_request["converter"])(rpc_request, self._log) + converter = TBModuleLoader.import_module("request", rpc_request["converter"])(rpc_request, self._converter_log) else: - converter = JsonRequestDownlinkConverter(rpc_request, self._log) + converter = JsonRequestDownlinkConverter(rpc_request, self._converter_log) rpc_request_dict = {**rpc_request, "converter": converter} self.__rpc_requests.append(rpc_request_dict) diff --git a/thingsboard_gateway/connectors/rest/rest_connector.py b/thingsboard_gateway/connectors/rest/rest_connector.py index 8ad445f6..bd245d69 100644 --- a/thingsboard_gateway/connectors/rest/rest_connector.py +++ b/thingsboard_gateway/connectors/rest/rest_connector.py @@ -71,7 +71,12 @@ def __init__(self, gateway, config, connector_type): self.name = config.get("name", 'REST Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5))) self.__gateway = gateway self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True, attr_name=self.name) self._default_downlink_converter = TBModuleLoader.import_module(self._connector_type, self._default_converters['downlink']) self._default_uplink_converter = TBModuleLoader.import_module(self._connector_type, @@ -130,7 +135,7 @@ def load_handlers(self): for http_method in mapping['HTTPMethods']: handler = data_handlers[security_type](self.collect_statistic_and_send, self.get_name(), self.get_id(), self.endpoints[mapping["endpoint"]], - self.__log, provider=self.__event_provider) + self.__converter_log, provider=self.__event_provider) handlers.append(web.route(http_method, mapping['endpoint'], handler)) except Exception as e: self.__log.error("Error on creating handlers - %s", str(e)) diff --git a/thingsboard_gateway/connectors/snmp/snmp_connector.py b/thingsboard_gateway/connectors/snmp/snmp_connector.py index 7e43c679..571a3a21 100644 --- a/thingsboard_gateway/connectors/snmp/snmp_connector.py +++ b/thingsboard_gateway/connectors/snmp/snmp_connector.py @@ -58,7 +58,12 @@ def __init__(self, gateway, config, connector_type): self.__id = self.__config.get('id') self.name = config.get("name", 'SNMP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5))) self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self._converter_log = init_logger(self.__gateway, self.name + "_converter", + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True, attr_name=self.name) self.__devices = self.__config["devices"] self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} @@ -235,7 +240,7 @@ def __fill_converters(self): device["uplink_converter"] = TBModuleLoader.import_module("snmp", device.get('converter', self._default_converters[ "uplink"]))(device, - self._log) + self._converter_log) device["downlink_converter"] = TBModuleLoader.import_module("snmp", device.get('converter', self._default_converters[ "downlink"]))(device) diff --git a/thingsboard_gateway/connectors/socket/socket_connector.py b/thingsboard_gateway/connectors/socket/socket_connector.py index 532391ed..0b1db57c 100644 --- a/thingsboard_gateway/connectors/socket/socket_connector.py +++ b/thingsboard_gateway/connectors/socket/socket_connector.py @@ -56,10 +56,10 @@ def __init__(self, gateway, config, connector_type): log_level = self.__config.get('logLevel', 'INFO') self.__log = init_logger(self.__gateway, self.name, log_level, enable_remote_logging=remote_logging, - is_connector_logger=True, connector_name=self.name) + is_connector_logger=True) self.__converter_log = init_logger(self.__gateway, self.name, log_level, enable_remote_logging=remote_logging, - is_converter_logger=True, connector_name=self.name) + is_converter_logger=True, attr_name=self.name) if is_using_old_config: self.__log.warning("Old Socket connector configuration format detected. Automatic conversion is applied.") diff --git a/thingsboard_gateway/connectors/xmpp/xmpp_connector.py b/thingsboard_gateway/connectors/xmpp/xmpp_connector.py index 99002c95..4996a1ee 100644 --- a/thingsboard_gateway/connectors/xmpp/xmpp_connector.py +++ b/thingsboard_gateway/connectors/xmpp/xmpp_connector.py @@ -60,7 +60,12 @@ def __init__(self, gateway, config, connector_type): self._devices_config = config.get('devices', []) self.name = config.get("name", 'XMPP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5))) self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'), - enable_remote_logging=self.__config.get('enableRemoteLogging', False)) + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True) + self.__converter_log = init_logger(self.__gateway, self.name + '_converter', + self.__config.get('logLevel', 'INFO'), + enable_remote_logging=self.__config.get('enableRemoteLogging', False), + is_connector_logger=True, attr_name=self.name) self._devices = {} self._reformat_devices_config() @@ -94,7 +99,7 @@ def _reformat_devices_config(self): attribute_updates=config.get('attributeUpdates', []), server_side_rpc=config.get('serverSideRpc', []) ) - self._devices[device_jid].set_converter(converter(config, self.__log)) + self._devices[device_jid].set_converter(converter(config, self.__converter_log)) except KeyError as e: self.__log.error('Invalid configuration %s with key error %s', config, e) continue diff --git a/thingsboard_gateway/tb_utility/tb_handler.py b/thingsboard_gateway/tb_utility/tb_handler.py index c1344329..7ef2ab89 100644 --- a/thingsboard_gateway/tb_utility/tb_handler.py +++ b/thingsboard_gateway/tb_utility/tb_handler.py @@ -14,10 +14,7 @@ import logging import logging.handlers -import os import threading -from os import environ -from pathlib import Path from queue import Queue, Empty from sys import stdout from time import time, sleep @@ -124,8 +121,8 @@ def handle(self, record): if record.levelno < remote_logging_level: return # Remote logging level set higher than record level - if logger and hasattr(logger, 'connector_name'): - name = logger.connector_name + if logger and hasattr(logger, 'attr_name'): + name = logger.attr_name if name: record = self.formatter.format(record) @@ -165,18 +162,3 @@ def get_logger_level_id(log_level): if isinstance(log_level, str): return 100 if log_level == 'NONE' else logging.getLevelName(log_level) return log_level - - -class TimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler): - def __init__(self, filename, when='h', interval=1, backupCount=0, - encoding=None, delay=False, utc=False): - config_path = environ.get('TB_GW_LOGS_PATH') - if config_path: - filename = config_path + os.sep + filename.split(os.sep)[-1] - - if not Path(filename).exists(): - with open(filename, 'w'): - pass - - super().__init__(filename, when=when, interval=interval, backupCount=backupCount, - encoding=encoding, delay=delay, utc=utc) diff --git a/thingsboard_gateway/tb_utility/tb_logger.py b/thingsboard_gateway/tb_utility/tb_logger.py index 24a30f3b..53ed45a7 100644 --- a/thingsboard_gateway/tb_utility/tb_logger.py +++ b/thingsboard_gateway/tb_utility/tb_logger.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from thingsboard_gateway.gateway.tb_gateway_service import TBGatewayService +from thingsboard_gateway.tb_utility.tb_rotating_file_handler import TimedRotatingFileHandler from thingsboard_gateway.gateway.statistics.statistics_service import StatisticsService TRACE_LOGGING_LEVEL = 5 @@ -27,7 +28,7 @@ def init_logger(gateway: 'TBGatewayService', name, level, enable_remote_logging=False, is_connector_logger=False, - is_converter_logger=False, connector_name=None): + is_converter_logger=False, attr_name=None): """ For creating a Logger with all config automatically Create a Logger manually only if you know what you are doing! @@ -37,12 +38,30 @@ def init_logger(gateway: 'TBGatewayService', name, level, enable_remote_logging= log.is_connector_logger = is_connector_logger log.is_converter_logger = is_converter_logger - if connector_name: - log.connector_name = connector_name + if attr_name: + log.attr_name = attr_name + '_ERRORS_COUNT' if hasattr(gateway, 'main_handler') and gateway.main_handler not in log.handlers: log.addHandler(gateway.main_handler) + # Add file handler to the connector or converter logger + # First check if it is a main module logger (for example OPC-UA connector logger) + # If it is, add a file handler to the main module logger + # If it is not (for example asyncua logger), add the main module file handler to the logger + if TbLogger.is_main_module_logger(name, attr_name, is_converter_logger): + if is_connector_logger: + file_handler = TimedRotatingFileHandler.get_connector_file_handler(log.name + '_connector') + + if is_converter_logger: + file_handler = TimedRotatingFileHandler.get_converter_file_handler(log.name) + + if file_handler: + log.addHandler(file_handler) + else: + main_file_handler = TimedRotatingFileHandler.get_time_rotating_file_handler_by_logger_name(attr_name) + if main_file_handler: + log.addHandler(main_file_handler) + from thingsboard_gateway.tb_utility.tb_handler import TBRemoteLoggerHandler if not hasattr(gateway, 'remote_handler'): gateway.remote_handler = TBRemoteLoggerHandler(gateway) @@ -78,23 +97,22 @@ class TbLogger(logging.Logger): PREVIOUS_ERRORS_RESET_TIME = 0 def __init__(self, name, level=logging.NOTSET, is_connector_logger=False, is_converter_logger=False, - connector_name=None): + attr_name=None): super(TbLogger, self).__init__(name=name, level=level) self.propagate = True self.parent = self.root - self.__previous_number_of_errors = -1 self.__is_connector_logger = is_connector_logger self.__is_converter_logger = is_converter_logger self.__previous_reset_errors_time = TbLogger.PREVIOUS_ERRORS_RESET_TIME logging.Logger.trace = TbLogger.trace - if connector_name: - self.__connector_name = connector_name + self.errors = 0 + + if attr_name: + self.attr_name = attr_name + '_ERRORS_COUNT' else: - self.__connector_name = name + self.attr_name = self.name + '_ERRORS_COUNT' - self.errors = 0 - self.attr_name = self.__connector_name + '_ERRORS_COUNT' self._is_on_init_state = True @property @@ -113,15 +131,6 @@ def is_converter_logger(self): def is_converter_logger(self, value): self.__is_converter_logger = value - @property - def connector_name(self): - return self.__connector_name - - @connector_name.setter - def connector_name(self, value): - self.__connector_name = value - self.attr_name = self.__connector_name + '_ERRORS_COUNT' - def reset(self): with TbLogger.ERRORS_MUTEX: TbLogger.ALL_ERRORS_COUNT = max(0, TbLogger.ALL_ERRORS_COUNT - self.errors) @@ -223,5 +232,9 @@ def send_errors_if_needed(cls, gateway): cls.__PREVIOUS_BATCH_TO_SEND.pop(key, None) cls.__PREVIOUS_BATCH_TO_SEND.update(batch_to_send) + @staticmethod + def is_main_module_logger(name, attr_name, is_converter_logger): + return name == attr_name or attr_name is None or (name != attr_name and is_converter_logger) + logging.setLoggerClass(TbLogger) diff --git a/thingsboard_gateway/tb_utility/tb_rotating_file_handler.py b/thingsboard_gateway/tb_utility/tb_rotating_file_handler.py new file mode 100644 index 00000000..c5de7348 --- /dev/null +++ b/thingsboard_gateway/tb_utility/tb_rotating_file_handler.py @@ -0,0 +1,63 @@ +import os +import logging +from os import environ +from pathlib import Path + + +class TimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler): + def __init__(self, filename, when='h', interval=1, backupCount=0, + encoding=None, delay=False, utc=False): + config_path = environ.get('TB_GW_LOGS_PATH') + if config_path: + filename = config_path + os.sep + filename.split(os.sep)[-1] + + if not Path(filename).exists(): + with open(filename, 'w'): + pass + + super().__init__(filename, when=when, interval=interval, backupCount=backupCount, + encoding=encoding, delay=delay, utc=utc) + + @staticmethod + def get_connector_file_handler(file_name): + return TimedRotatingFileHandler.__get_file_handler(file_name, 'connector') + + @staticmethod + def get_converter_file_handler(file_name): + return TimedRotatingFileHandler.__get_file_handler(file_name, 'converter') + + @staticmethod + def get_time_rotating_file_handler_by_logger_name(logger_name): + file_handler_filter = list(filter(lambda x: isinstance(x, TimedRotatingFileHandler), + logging.getLogger(logger_name).handlers)) + if len(file_handler_filter): + return file_handler_filter[0] + + @staticmethod + def __get_file_handler(file_name, logger_name): + file_handler = TimedRotatingFileHandler.get_time_rotating_file_handler_by_logger_name(logger_name) + + file_name = file_name + '.log' + + if file_handler: + return TimedRotatingFileHandler.__create_file_handler_copy(file_handler, file_name) + else: + return TimedRotatingFileHandler.__create_default_file_handler(file_name) + + @staticmethod + def __create_file_handler_copy(handler, file_name): + file_name = f'{os.sep}'.join(handler.baseFilename.split(os.sep)[:-1]) + os.sep + file_name + handler_copy = TimedRotatingFileHandler(file_name, + when=handler.when, + backupCount=handler.backupCount, + interval=handler.interval, + encoding=handler.encoding, + delay=handler.delay, + utc=handler.utc) + handler_copy.setFormatter(handler.formatter) + return handler_copy + + @staticmethod + def __create_default_file_handler(file_name): + file_name = '.' + os.sep + file_name + '.log' + return TimedRotatingFileHandler(file_name)