[PLA-2013] Add config and LRU cache (#41)

.env.example

@@ -0,0 +1,4 @@
+NUMBER_OF_ISOLATES=
+DEFAULT_BIND_PORT=
+SPEC_PER_ISOLATE=
+SPEC_EXPIRE_DURATION=

.gitignore

@@ -1,4 +1,6 @@
 .idea
+.env
+bin/**.exe
 
 ### Dart template
 # See https://www.dartlang.org/guides/libraries/private-files

bin/server.dart

@@ -1,11 +1,42 @@
 import 'dart:io';
 import 'dart:isolate';
 
+import 'package:dotenv/dotenv.dart';
+import 'package:logging/logging.dart';
 import 'package:platform_decoder/handler/general.dart';
 import 'package:shelf_plus/shelf_plus.dart';
 
-void main() {
-  const numberOfIsolates = 8;
+int numberOfIsolates = 8;
+int defaultBindPort = 8090;
+
+final logger = Logger('Decoder');
+
+void loadConfig() {
+  var env = DotEnv(includePlatformEnvironment: true)..load();
+
+  numberOfIsolates =
+      int.tryParse(env.getOrElse('NUMBER_OF_ISOLATES', () => '8')) ?? 8;
+  defaultBindPort =
+      int.tryParse(env.getOrElse('DEFAULT_BIND_PORT', () => '8090')) ?? 8090;
+  int specPerIsolate =
+      int.tryParse(env.getOrElse('SPEC_PER_ISOLATE', () => '4')) ?? 4;
+  int specExpireDuration =
+      int.tryParse(env.getOrElse('SPEC_EXPIRE_DURATION', () => '300')) ?? 300;
+
+  logger.info('Starting Platform Decoder v2.1.0');
+  logger.info('Number of isolates: $numberOfIsolates');
+  logger.info('Default bind port: $defaultBindPort');
+  logger.info('Spec per isolate: $specPerIsolate');
+  logger.info('Spec expire duration: $specExpireDuration');
+}
+
+Future<void> main() async {
+  Logger.root.level = Level.ALL;
+  Logger.root.onRecord.listen((record) {
+    print('${record.time} [${record.level.name}] ${record.message}');
+  });
+
+  loadConfig();
 
   for (var i = 0; i < numberOfIsolates - 1; i++) {
     Isolate.spawn(spawnServer, null, debugName: i.toString()); // isolate 0..7
@@ -17,14 +48,19 @@ void spawnServer(_) => shelfRun(
       init,
       defaultBindAddress: InternetAddress.anyIPv4,
       defaultShared: true,
-      defaultBindPort: 8090,
+      defaultBindPort: defaultBindPort,
       defaultEnableHotReload: false,
     );
 
 Handler init() {
   var app = Router().plus;
   app.use(logRequests());
 
+  Logger.root.level = Level.ALL;
+  Logger.root.onRecord.listen((record) {
+    print('${record.time} [${record.level.name}] ${record.message}');
+  });
+
   app.get('/', () => 'Ok');
   app.get('/health', () => {'status': 'healthy'});
   app.post('/', (Request request) async {

lib/decoder/chain_info.dart

@@ -1,56 +1,60 @@
-// The first time a metadata is requested we instantiate it and keep it on memory to speed up the decoder
-// Otherwise we would have to decode the metadata every time we decode an extrinsic or event
-// TODO: Later we can make some kind of LRU cache so it doesn't stays there forever
-
+import 'package:dotenv/dotenv.dart';
+import 'package:logging/logging.dart';
+import 'package:lru_memory_cache/lru_memory_cache.dart';
 import 'package:platform_decoder/consts/enjin/enjin.dart' as enjin;
 import 'package:platform_decoder/consts/matrix/matrix.dart' as matrix;
 import 'package:substrate_metadata/core/metadata_decoder.dart';
 import 'package:substrate_metadata/models/models.dart';
 
-// {
-//   "enjin-matrixchain": {
-//     1000: ChainInfo,
-//     1001: ChainInfo,
-//   }
-// }
-final Map<String, Map<int, ChainInfo>> chainInfos = {};
+final logger = Logger('Decoder');
 
-ChainInfo getChainInfo(network, specVersion) {
-  final chainInfoMetadata = chainInfos[network]?[specVersion];
+var env = DotEnv(includePlatformEnvironment: true)..load();
+int specPerIsolate =
+    int.tryParse(env.getOrElse('SPEC_PER_ISOLATE', () => '4')) ?? 4;
+int specExpireDuration =
+    int.tryParse(env.getOrElse('SPEC_EXPIRE_DURATION', () => '300')) ?? 300;
+
+LRUMemoryCache<String, ChainInfo> cache = LRUMemoryCache(
+  generateKey: (k) =>
+      "${k.constants['System']!['Version']!.value['spec_name']}-${k.constants['System']!['Version']!.value['spec_version']}",
+  capacity: specPerIsolate,
+  expireMode: ExpireMode.onInteraction,
+);
 
-  if (chainInfoMetadata != null) {
-    return chainInfoMetadata;
+String getSpecName(String network, int specVersion) {
+  if (network == 'canary-matrixchain') {
+    return 'matrix-$specVersion';
+  }
+  if (network == 'enjin-matrixchain') {
+    return 'matrix-enjin-$specVersion';
+  }
+  if (network == 'canary-relaychain') {
+    return 'canary-$specVersion';
   }
 
-  // Instantiate a new metadata for relaychains
-  if (network == 'canary-relaychain' || network == 'enjin-relaychain') {
-    final metadata =
-        MetadataDecoder.instance.decode(enjin.metadata(network, specVersion));
-    final ChainInfo chain = ChainInfo.fromMetadata(metadata);
+  return 'enjin-$specVersion';
+}
 
-    if (chainInfos[network] == null) {
-      chainInfos[network] = {specVersion: chain};
-    } else {
-      chainInfos[network]![specVersion] = chain;
-    }
+DecodedMetadata getDecodedMetadata(String network, int specVersion) {
+  if (network == 'canary-relaychain' || network == 'enjin-relaychain') {
+    return MetadataDecoder.instance
+        .decode(enjin.metadata(network, specVersion));
+  }
 
-    print("Instantiated new metadata for $network $specVersion");
+  return MetadataDecoder.instance.decode(matrix.metadata(network, specVersion));
+}
 
-    return chain;
-  }
+ChainInfo getChainInfo(network, specVersion) {
+  ChainInfo? chainInfo = cache.get(getSpecName(network, specVersion));
 
-  // Instantiate a new metadata for matrixchains
-  final metadata =
-      MetadataDecoder.instance.decode(matrix.metadata(network, specVersion));
-  final ChainInfo chain = ChainInfo.fromMetadata(metadata);
+  if (chainInfo == null) {
+    logger.info("Metadata $network v$specVersion instantiated");
 
-  if (chainInfos[network] == null) {
-    chainInfos[network] = {specVersion: chain};
-  } else {
-    chainInfos[network]![specVersion] = chain;
+    final metadata = getDecodedMetadata(network, specVersion);
+    chainInfo = ChainInfo.fromMetadata(metadata);
   }
 
-  print("Instantiated new metadata for $network $specVersion");
+  cache.add(chainInfo, expiryDuration: Duration(seconds: specExpireDuration));
 
-  return chain;
+  return chainInfo;
 }

lib/handler/general.dart

@@ -1,10 +1,13 @@
 import 'dart:async';
 import 'dart:convert';
 
+import 'package:logging/logging.dart';
 import 'package:platform_decoder/decoder/decoder.dart';
 import 'package:shelf/shelf.dart';
 import 'package:substrate_metadata/utils/utils.dart';
 
+final logger = Logger('Decoder');
+
 int getLatestSpecVersion(network) {
   if (network == 'enjin-relaychain') {
     return 1032;
@@ -33,6 +36,8 @@ Future<dynamic> handleRequest(Request request) async {
 
       return extrinsic;
     } catch (e) {
+      logger.warning('Failed to decode extrinsic with reason: $e');
+      logger.warning('Extrinsic $network v$specVersion: ${body['extrinsic']}');
       return {"error": "Failed to decode extrinsic"};
     }
   }
@@ -50,6 +55,8 @@ Future<dynamic> handleRequest(Request request) async {
 
       return extrinsics.toList();
     } catch (e) {
+      logger.warning('Failed to decode extrinsics with reason: $e');
+      logger.warning('Extrinsic $network v$specVersion: ${body['extrinsics']}');
       return {"error": "Failed to decode extrinsics"};
     }
   }
@@ -64,6 +71,8 @@ Future<dynamic> handleRequest(Request request) async {
 
       return decoded.toList();
     } catch (e) {
+      logger.warning('Failed to decode events with reason: $e');
+      logger.warning('Extrinsic $network v$specVersion: ${body['events']}');
       return {"error": "Failed to decode events"};
     }
   }

pubspec.yaml

@@ -1,19 +1,25 @@
 name: platform_decoder
 description: Enjin Platform Decoder
-version: 1.10.1
+version: 2.1.0
 publish_to: none
 
 environment:
   sdk: ">=3.3.0 <4.0.0"
 
 dependencies:
-  shelf: ^1.4.2
+  shelf: ^1.4.1
   shelf_plus: ^1.9.2
   polkadart: ^0.4.7
   substrate_metadata: ^1.2.2
   polkadart_scale_codec: ^1.2.1
   compute: ^1.0.2
+  lru_memory_cache: ^1.0.3
+  dotenv: ^4.2.0
+  logging: ^1.2.0
 
 dev_dependencies:
   lints: ^4.0.0
   test: ^1.25.8
+
+assets:
+  - .env