Merge branch 'thingsboard:master' into master

tests/integration/connectors/can/test_can_connector.py

@@ -32,7 +32,7 @@
 from thingsboard_gateway.tb_utility.tb_handler import TBRemoteLoggerHandler
 from thingsboard_gateway.tb_utility.tb_logger import TbLogger, init_logger
 
-try :
+try:
     from can import Notifier, BufferedReader, Bus, Message
 except (ImportError, ModuleNotFoundError):
     from thingsboard_gateway.tb_utility.tb_utility import TBUtility

thingsboard_gateway/config/logs.json

@@ -16,14 +16,14 @@
       "stream": "ext://sys.stdout"
     },
     "databaseHandler": {
-      "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
+      "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/database.log",
       "backupCount": 1,
       "encoding": "utf-8"
     },
     "serviceHandler": {
-      "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
+      "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/service.log",
       "backupCount": 7,
@@ -32,7 +32,7 @@
       "encoding": "utf-8"
     },
     "connectorHandler": {
-      "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
+      "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/connector.log",
       "backupCount": 7,
@@ -41,7 +41,7 @@
       "encoding": "utf-8"
     },
     "converterHandler": {
-      "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
+      "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/converter.log",
       "backupCount": 7,
@@ -50,7 +50,7 @@
       "encoding": "utf-8"
     },
     "tb_connectionHandler": {
-      "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
+      "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/tb_connection.log",
       "backupCount": 7,
@@ -59,7 +59,7 @@
       "encoding": "utf-8"
     },
     "storageHandler": {
-      "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
+      "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/storage.log",
       "backupCount": 7,
@@ -68,7 +68,7 @@
       "encoding": "utf-8"
     },
     "extensionHandler": {
-      "class": "thingsboard_gateway.tb_utility.tb_handler.TimedRotatingFileHandler",
+      "class": "thingsboard_gateway.tb_utility.tb_rotating_file_handler.TimedRotatingFileHandler",
       "formatter": "LogFormatter",
       "filename": "./logs/extension.log",
       "backupCount": 7,

thingsboard_gateway/connectors/bacnet/bacnet_connector.py

@@ -58,10 +58,10 @@ def __init__(self, gateway, config, connector_type):
 
         self.__log = init_logger(self.__gateway, self.name, log_level,
                                  enable_remote_logging=remote_logging,
-                                 is_connector_logger=True, connector_name=self.name)
-        self.__converter_log = init_logger(self.__gateway, self.name, log_level,
+                                 is_connector_logger=True)
+        self.__converter_log = init_logger(self.__gateway, self.name + '_converter', log_level,
                                            enable_remote_logging=remote_logging,
-                                           is_converter_logger=True, connector_name=self.name)
+                                           is_converter_logger=True, attr_name=self.name)
         self.__log.info('Starting BACnet connector...')
 
         if BackwardCompatibilityAdapter.is_old_config(config):
@@ -153,8 +153,11 @@ async def __discover_devices(self):
         for device_config in self.__config.get('devices', []):
             try:
                 DeviceObjectConfig.update_address_in_config_util(device_config)
-                await self.__application.do_who_is(device_address=device_config['address'])
+                result = await self.__application.do_who_is(device_address=device_config['address'])
                 self.__log.debug('WhoIs request sent to device %s', device_config['address'])
+
+                if result is None:
+                    self.__log.error('Device %s not found', device_config['address'])
             except Exception as e:
                 self.__log.error('Error discovering device %s: %s', device_config['address'], e)
 

thingsboard_gateway/connectors/bacnet/entities/device_object_config.py

@@ -23,7 +23,7 @@ def __init__(self, config):
         self.__network_number_quality = config.get('networkNumberQuality', 'configured')
         self.__max_apdu_length_accepted = int(config.get('maxApduLengthAccepted', 1024))
         self.__segmentation_supported = config.get('segmentationSupported', 'segmentedBoth')
-        self.__vendor_identifier = int(config.get('vendorIdentifier', 15))
+        self.__vendor_identifier = int(config.get('vendorIdentifier', 15) or 15)
 
     @property
     def device_object_config(self):

thingsboard_gateway/connectors/ble/ble_connector.py

@@ -48,8 +48,14 @@ def __init__(self, gateway, config, connector_type):
         self.__config = config
         self.__id = self.__config.get('id')
         self.name = self.__config.get("name", 'BLE Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))
-        self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'),
-                                 enable_remote_logging=self.__config.get('enableRemoteLogging', False))
+        self.__log = init_logger(self.__gateway, self.name,
+                                 self.__config.get('logLevel', 'INFO'),
+                                 enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                 is_connector_logger=True)
+        self.__converter_log = init_logger(self.__gateway, self.name + '_converter',
+                                           self.__config.get('logLevel', 'INFO'),
+                                           enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                           is_converter_logger=True, attr_name=self.name)
 
         self.daemon = True
 
@@ -138,7 +144,7 @@ def __process_data(self):
                 StatisticsService.count_connector_bytes(self.name, data, stat_parameter_name='connectorBytesReceived')
 
                 try:
-                    converter = converter(device_config, self.__log)
+                    converter = converter(device_config, self.__converter_log)
                     converted_data: ConvertedData = converter.convert(config, data)
                     self.statistics['MessagesReceived'] = self.statistics['MessagesReceived'] + 1
                     self.__log.debug(converted_data)

thingsboard_gateway/connectors/can/can_connector.py

@@ -76,7 +76,12 @@ def __init__(self, gateway, config, connector_type):
         self.__config = config
         self.__id = self.__config.get('id')
         self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'),
-                                enable_remote_logging=self.__config.get('enableRemoteLogging', False))
+                                enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                is_connector_logger=True)
+        self._converter_log = init_logger(self.__gateway, self.name + '_converter',
+                                          self.__config.get('logLevel', 'INFO'),
+                                          enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                          is_converter_logger=True, attr_name=self.name)
         self.__bus_conf = {}
         self.__bus = None
         self.__reconnect_count = 0
@@ -591,11 +596,11 @@ def __get_converter(self, config, need_uplink):
         else:
             if need_uplink:
                 uplink = config.get("uplink")
-                return BytesCanUplinkConverter(self._log) if uplink is None \
+                return BytesCanUplinkConverter(self._converter_log) if uplink is None \
                     else TBModuleLoader.import_module(self._connector_type, uplink)
             else:
                 downlink = config.get("downlink")
-                return BytesCanDownlinkConverter(self._log) if downlink is None \
+                return BytesCanDownlinkConverter(self._converter_log) if downlink is None \
                     else TBModuleLoader.import_module(self._connector_type, downlink)
 
     def get_config(self):

thingsboard_gateway/connectors/ftp/ftp_connector.py

@@ -49,7 +49,12 @@ def __init__(self, gateway, config, connector_type):
         self.__tls_support = self.__config.get("TLSSupport", False)
         self.name = self.__config.get("name", "".join(choice(ascii_lowercase) for _ in range(5)))
         self.__log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'),
-                                 enable_remote_logging=self.__config.get('enableRemoteLogging', False))
+                                 enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                 is_connector_logger=True)
+        self.__converter_log = init_logger(self.__gateway, self.name + '_converter',
+                                           self.__config.get('logLevel', 'INFO'),
+                                           enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                           is_converter_logger=True, attr_name=self.name)
         self.daemon = True
         self.__stopped = False
         self.__requests_in_progress = []
@@ -135,7 +140,7 @@ def __process_paths(self, ftp):
             time_point = timer()
             if time_point - path.last_polled_time >= path.poll_period or path.last_polled_time == 0:
                 configuration = path.config
-                converter = FTPUplinkConverter(configuration, self.__log)
+                converter = FTPUplinkConverter(configuration, self.__converter_log)
                 path.last_polled_time = time_point
 
                 if '*' in path.path:

thingsboard_gateway/connectors/modbus/modbus_connector.py

@@ -84,7 +84,12 @@ def __init__(self, gateway, config, connector_type):
         self.name = self.__config.get("name", 'Modbus Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))
         self.__log = init_logger(self.__gateway, config.get('name', self.name),
                                  config.get('logLevel', 'INFO'),
-                                 enable_remote_logging=config.get('enableRemoteLogging', False))
+                                 enable_remote_logging=config.get('enableRemoteLogging', False),
+                                 is_connector_logger=True)
+        self.converter_log = init_logger(self.__gateway, self.name + '_converter',
+                                         config.get('logLevel', 'INFO'),
+                                         enable_remote_logging=config.get('enableRemoteLogging', False),
+                                         is_converter_logger=True, attr_name=self.name)
         self.__log.info('Starting Modbus Connector...')
         self.__id = self.__config.get('id')
         self.__connected = False

thingsboard_gateway/connectors/modbus/slave.py

@@ -130,9 +130,9 @@ def __load_converter(self, config, converter_type, converter_config: Union[Dict,
                 converter = config[converter_type + CONVERTER_PARAMETER]
             else:
                 if converter_type == DOWNLINK_PREFIX:
-                    converter = BytesModbusDownlinkConverter(converter_config, self._log)
+                    converter = BytesModbusDownlinkConverter(converter_config, self.connector.converter_log)
                 else:
-                    converter = BytesModbusUplinkConverter(converter_config, self._log)
+                    converter = BytesModbusUplinkConverter(converter_config, self.connector.converter_log)
 
             return converter
         except Exception as e:

thingsboard_gateway/connectors/mqtt/mqtt_connector.py

@@ -118,6 +118,10 @@ def __init__(self, gateway, config, connector_type):
                                  config.get('logLevel', 'INFO'),
                                  enable_remote_logging=config.get('enableRemoteLogging', False),
                                  is_connector_logger=True)
+        self.__converter_log = init_logger(self.__gateway, config['name'] + '_converter',
+                                           config.get('logLevel', 'INFO'),
+                                           enable_remote_logging=config.get('enableRemoteLogging', False),
+                                           is_converter_logger=True, attr_name=config['name'])
 
         # check if the configuration is in the old format
         using_old_config_format_detected = BackwardCompatibilityAdapter.is_old_config_format(config)
@@ -454,7 +458,7 @@ def _on_connect(self, client, userdata, flags, result_code, *extra_params):
                         if module:
                             self.__log.debug('Converter %s for topic %s - found!', converter_class_name,
                                              mapping["topicFilter"])
-                            converter = module(mapping, self.__log)
+                            converter = module(mapping, self.__converter_log)
                             if sharing_id:
                                 self.__shared_custom_converters[sharing_id] = converter
                         else:

thingsboard_gateway/connectors/ocpp/ocpp_connector.py

@@ -69,7 +69,12 @@ def __init__(self, gateway, config, connector_type):
         self._gateway = gateway
         self.name = self._config.get("name", 'OCPP Connector ' + ''.join(choice(ascii_lowercase) for _ in range(5)))
         self._log = init_logger(self._gateway, self.name, self._config.get('logLevel', 'INFO'),
-                                enable_remote_logging=self._config.get('enableRemoteLogging', False))
+                                enable_remote_logging=self._config.get('enableRemoteLogging', False),
+                                is_connector_logger=True)
+        self._converter_log = init_logger(self._gateway, self.name + '_converter',
+                                          self._config.get('logLevel', 'INFO'),
+                                          enable_remote_logging=self._config.get('enableRemoteLogging', False),
+                                          is_converter_logger=True, attr_name=self.name)
 
         self._default_converters = {'uplink': 'OcppUplinkConverter'}
         self._server = None
@@ -184,7 +189,7 @@ async def on_connect(self, websocket, path):
         if is_valid:
             uplink_converter_name = cp_config.get('extension', self._default_converters['uplink'])
             cp = ChargePoint(charge_point_id, websocket, {**cp_config, 'uplink_converter_name': uplink_converter_name},
-                             OcppConnector._callback, self._log)
+                             OcppConnector._callback, self._converter_log)
             cp.authorized = True
 
             self._log.info('Connected Charge Point with id: %s', charge_point_id)

thingsboard_gateway/connectors/odbc/odbc_connector.py

@@ -63,7 +63,12 @@ def __init__(self, gateway, config, connector_type):
         self.__config = config
         self.__id = self.__config.get('id')
         self._log = init_logger(self.__gateway, self.name, self.__config.get('logLevel', 'INFO'),
-                                enable_remote_logging=self.__config.get('enableRemoteLogging', False))
+                                enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                is_connector_logger=True)
+        self._converter_log = init_logger(self.__gateway, self.name + '_converter',
+                                          self.__config.get('logLevel', 'INFO'),
+                                          enable_remote_logging=self.__config.get('enableRemoteLogging', False),
+                                          is_converter_logger=True, attr_name=self.name)
         self._connector_type = connector_type
         self.__stopped = False
 
@@ -81,7 +86,7 @@ def __init__(self, gateway, config, connector_type):
         self.__attribute_columns = []
         self.__timeseries_columns = []
 
-        self.__converter = OdbcUplinkConverter(self._log) if not self.__config.get("converter", "") else \
+        self.__converter = OdbcUplinkConverter(self._converter_log) if not self.__config.get("converter", "") else \
             TBModuleLoader.import_module(self._connector_type, self.__config["converter"])
 
         self.__configure_pyodbc()