OCT-1720: Reduce number of RPC calls

backend/app/extensions.py

@@ -5,8 +5,15 @@
 from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy
 from flask_caching import Cache
-from web3 import Web3
-from web3.middleware import geth_poa_middleware
+from web3 import Web3, middleware
+from web3._utils.caching import (
+    generate_cache_key,
+)
+
+from functools import partial
+import lru
+import threading
+import time
 
 from app.infrastructure.contracts import abi
 from app.infrastructure.contracts.epochs import Epochs
@@ -42,10 +49,82 @@
 gql_factory = GQLConnectionFactory()
 
 
+def is_get_block_by_number(app, method, params):
+    return method in ["eth_getBlockByNumber"]
+
+
+def is_epochs_or_deposits_contract(app, method, params):
+    return method in ["eth_call"] and params[0]["to"] in [
+        app.config["EPOCHS_CONTRACT_ADDRESS"],
+        app.config["DEPOSITS_CONTRACT_ADDRESS"],
+    ]
+
+
+def is_glm_or_projects_contract(app, method, params):
+    return method in ["eth_call"] and params[0]["to"] in [
+        app.config["PROJECTS_CONTRACT_ADDRESS"],
+        app.config["GLM_CONTRACT_ADDRESS"],
+    ]
+
+
+def _create_cache(app, cache_expiry_seconds, condition_fn, should_cache_fn):
+    cache_class = partial(lru.LRU, 256)
+
+    def cache_fn(make_request, web3):
+        cache_obj = cache_class()
+        lock = threading.Lock()
+
+        def middleware_fn(method, params):
+            lock_acquired = lock.acquire(blocking=False)
+
+            try:
+                if lock_acquired and condition_fn(app, method, params):
+                    cache_key = generate_cache_key((method, params))
+                    if cache_key in cache_obj:
+                        cached_at, cached_response = cache_obj[cache_key]
+                        cached_for = time.time() - cached_at
+
+                        if cached_for <= cache_expiry_seconds:
+                            return cached_response
+                        else:
+                            del cache_obj[cache_key]
+
+                    # # cache either missed or expired so make the request.
+                    response = make_request(method, params)
+
+                    if should_cache_fn(response):
+                        cache_obj[cache_key] = (time.time(), response)
+
+                    return response
+                else:
+                    return make_request(method, params)
+            finally:
+                if lock_acquired:
+                    lock.release()
+
+        return middleware_fn
+
+    return cache_fn
+
+
 def init_web3(app):
     w3.provider = app.config["WEB3_PROVIDER"]
-    if geth_poa_middleware not in w3.middleware_onion:
-        w3.middleware_onion.inject(geth_poa_middleware, layer=0)
+    if middleware.geth_poa_middleware not in w3.middleware_onion:
+        w3.middleware_onion.inject(middleware.geth_poa_middleware, layer=0)
+        # w3.middleware_onion.add(middleware.time_based_cache_middleware)
+        # w3.middleware_onion.add(middleware.latest_block_based_cache_middleware)
+        # w3.middleware_onion.add(middleware.simple_cache_middleware)
+        w3.middleware_onion.add(
+            _create_cache(
+                app, 600, is_glm_or_projects_contract, _generic_should_cache_web3
+            )
+        )
+        w3.middleware_onion.add(
+            _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True))
+        )
+        w3.middleware_onion.add(
+            _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3)
+        )
 
     glm.init_web3(w3, app.config["GLM_CONTRACT_ADDRESS"])
     epochs.init_web3(w3, app.config["EPOCHS_CONTRACT_ADDRESS"])
@@ -62,3 +141,13 @@ def init_scheduler(app):
     if app.config["SCHEDULER_ENABLED"] and app.config["ENV"] != "test":
         scheduler.init_app(app)
         scheduler.start()
+
+
+def _generic_should_cache_web3(response):
+    if "error" in response:
+        return False
+    elif "result" not in response:
+        return False
+    if response["result"] is None:
+        return False
+    return True

backend/app/infrastructure/database/pending_epoch_snapshot.py

@@ -4,7 +4,7 @@
 from typing_extensions import deprecated
 
 from app.infrastructure.database.models import PendingEpochSnapshot
-from app.extensions import db
+from app.extensions import db, cache
 from app import exceptions
 
 from decimal import Decimal
@@ -20,10 +20,12 @@ def get_by_epoch_num(epoch) -> PendingEpochSnapshot:
     return snapshot
 
 
+@cache.memoize(timeout=15)
 def get_by_epoch(epoch: int) -> Optional[PendingEpochSnapshot]:
     return PendingEpochSnapshot.query.filter_by(epoch=epoch).first()
 
 
+@cache.memoize(timeout=15)
 def get_last_snapshot() -> PendingEpochSnapshot:
     snapshot = (
         db.session.query(PendingEpochSnapshot)

localenv/anvil/Dockerfile

@@ -1,4 +1,4 @@
-FROM europe-docker.pkg.dev/wildland-dev/octant-test/foundry:latest
+FROM europe-docker.pkg.dev/wildland-dev/internal/octant/anvil:latest
 
 WORKDIR /home/foundry
 

localenv/apitest.yaml

@@ -27,6 +27,12 @@ services:
       CHAIN_ID: "1337"
       CHAIN_NAME: "localhost"
 
+      GC_PASSPORT_SCORER_API_KEY: "${GC_PASSPORT_SCORER_API_KEY}"
+      GC_PASSPORT_SCORER_ID: "${GC_PASSPORT_SCORER_ID}"
+
+      DELEGATION_SALT: "salt"
+      DELEGATION_SALT_PRIMARY: "salt_primary"
+
     depends_on:
       - anvil
       - graph-node

localenv/localenv.yaml

@@ -42,6 +42,12 @@ services:
       CHAIN_ID: "1337"
       CHAIN_NAME: "localhost"
 
+      GC_PASSPORT_SCORER_API_KEY: "${GC_PASSPORT_SCORER_API_KEY}"
+      GC_PASSPORT_SCORER_ID: "${GC_PASSPORT_SCORER_ID}"
+
+      DELEGATION_SALT: "salt"
+      DELEGATION_SALT_PRIMARY: "salt_primary"
+
     depends_on:
       - backend-postgres
       - anvil

package.json

@@ -22,7 +22,7 @@
     "apitest:up": "docker compose -p apitest -f ./localenv/docker-compose.yaml -f ./localenv/apitest.yaml up anvil ipfs graph-postgres graph-node multideployer",
     "apitest:down": "docker compose -p apitest -f ./localenv/docker-compose.yaml -f ./localenv/apitest.yaml down",
     "apitest:run": "docker compose -p apitest -f ./localenv/docker-compose.yaml -f ./localenv/apitest.yaml run backend-apitest",
-    "preapitest:up": "docker rm -v -f $(docker ps -qa --filter 'name=apitest')",
+    "preapitest:up": "docker rm -v -f $(docker ps -qa --filter 'name=apitest') || true",
     "preapitest:run": "yarn localenv:build-backend"
   },
   "repository": {