From aef12951e254228b070a0bc7efda847509071908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Thu, 4 Jul 2024 03:08:05 +0200 Subject: [PATCH 01/12] OCT-1720: Reduce number of RPC calls --- backend/app/extensions.py | 99 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 4 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 8a749b7c8a..f3d97f617f 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -5,8 +5,16 @@ from flask_socketio import SocketIO from flask_sqlalchemy import SQLAlchemy from flask_caching import Cache -from web3 import Web3 -from web3.middleware import geth_poa_middleware +from web3 import Web3, middleware +from web3._utils.rpc_abi import RPC +from web3._utils.caching import ( + generate_cache_key, +) + +from functools import wraps, partial +import lru +import threading +import time from app.infrastructure.contracts import abi from app.infrastructure.contracts.epochs import Epochs @@ -42,10 +50,83 @@ gql_factory = GQLConnectionFactory() +def is_get_block_by_number(app, method, params): + return method in ["eth_getBlockByNumber"] + + +def is_epochs_or_deposits_contract(app, method, params): + return method in ["eth_call"] and params[0]["to"] in [ + app.config["EPOCHS_CONTRACT_ADDRESS"], + app.config["DEPOSITS_CONTRACT_ADDRESS"], + ] + + +def is_glm_or_projects_contract(app, method, params): + return method in ["eth_call"] and params[0]["to"] in [ + app.config["PROJECTS_CONTRACT_ADDRESS"], + app.config["GLM_CONTRACT_ADDRESS"], + ] + + +def _create_cache(app, cache_expiry_seconds, condition_fn, should_cache_fn): + cache_class = partial(lru.LRU, 256) + + def cache_fn(make_request, web3): + cache_obj = cache_class() + lock = threading.Lock() + + def middleware_fn(method, params): + lock_acquired = lock.acquire(blocking=False) + + try: + if lock_acquired and condition_fn(app, method, params): + cache_key = generate_cache_key((method, params)) + if cache_key in cache_obj: + cached_at, cached_response = cache_obj[cache_key] + cached_for = time.time() - cached_at + + if cached_for <= cache_expiry_seconds: + return cached_response + else: + del cache_obj[cache_key] + + # # cache either missed or expired so make the request. + response = make_request(method, params) + + if should_cache_fn(response): + cache_obj[cache_key] = (time.time(), response) + + return response + else: + return make_request(method, params) + finally: + if lock_acquired: + lock.release() + + return middleware_fn + + return cache_fn + + def init_web3(app): w3.provider = app.config["WEB3_PROVIDER"] - if geth_poa_middleware not in w3.middleware_onion: - w3.middleware_onion.inject(geth_poa_middleware, layer=0) + if middleware.geth_poa_middleware not in w3.middleware_onion: + w3.middleware_onion.inject(middleware.geth_poa_middleware, layer=0) + + middleware.validation.METHODS_TO_VALIDATE.remove(RPC.eth_call) + + w3.middleware_onion.add(middleware.time_based_cache_middleware) + w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) + w3.middleware_onion.add(middleware.simple_cache_middleware) + w3.middleware_onion.add( + _create_cache(app, 600, is_glm_or_projects_contract, _generic_should_cache_web3) + ) + w3.middleware_onion.add( + _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) + ) + w3.middleware_onion.add( + _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) + ) glm.init_web3(w3, app.config["GLM_CONTRACT_ADDRESS"]) epochs.init_web3(w3, app.config["EPOCHS_CONTRACT_ADDRESS"]) @@ -62,3 +143,13 @@ def init_scheduler(app): if app.config["SCHEDULER_ENABLED"] and app.config["ENV"] != "test": scheduler.init_app(app) scheduler.start() + + +def _generic_should_cache_web3(response): + if "error" in response: + return False + elif "result" not in response: + return False + if response["result"] is None: + return False + return True From ca140d6fdd758ea9797f26d6c8f19dfe398c7495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Thu, 4 Jul 2024 22:24:28 +0200 Subject: [PATCH 02/12] OCT-1721: Cache often consecutive DB query. --- backend/app/extensions.py | 2 +- backend/app/infrastructure/database/pending_epoch_snapshot.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index f3d97f617f..0a5aa9c56b 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -11,7 +11,7 @@ generate_cache_key, ) -from functools import wraps, partial +from functools import partial import lru import threading import time diff --git a/backend/app/infrastructure/database/pending_epoch_snapshot.py b/backend/app/infrastructure/database/pending_epoch_snapshot.py index 60e8c18791..9f43afaab9 100644 --- a/backend/app/infrastructure/database/pending_epoch_snapshot.py +++ b/backend/app/infrastructure/database/pending_epoch_snapshot.py @@ -4,7 +4,7 @@ from typing_extensions import deprecated from app.infrastructure.database.models import PendingEpochSnapshot -from app.extensions import db +from app.extensions import db, cache from app import exceptions from decimal import Decimal @@ -20,10 +20,12 @@ def get_by_epoch_num(epoch) -> PendingEpochSnapshot: return snapshot +@cache.memoize(timeout=15) def get_by_epoch(epoch: int) -> Optional[PendingEpochSnapshot]: return PendingEpochSnapshot.query.filter_by(epoch=epoch).first() +@cache.memoize(timeout=15) def get_last_snapshot() -> PendingEpochSnapshot: snapshot = ( db.session.query(PendingEpochSnapshot) From 96bd00dc667077fa6056bf4275bcedda907bc540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Fri, 5 Jul 2024 11:14:05 +0200 Subject: [PATCH 03/12] Fix cache fn load --- backend/app/extensions.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 0a5aa9c56b..33c35ce5e7 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -6,7 +6,6 @@ from flask_sqlalchemy import SQLAlchemy from flask_caching import Cache from web3 import Web3, middleware -from web3._utils.rpc_abi import RPC from web3._utils.caching import ( generate_cache_key, ) @@ -112,21 +111,20 @@ def init_web3(app): w3.provider = app.config["WEB3_PROVIDER"] if middleware.geth_poa_middleware not in w3.middleware_onion: w3.middleware_onion.inject(middleware.geth_poa_middleware, layer=0) - - middleware.validation.METHODS_TO_VALIDATE.remove(RPC.eth_call) - - w3.middleware_onion.add(middleware.time_based_cache_middleware) - w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) - w3.middleware_onion.add(middleware.simple_cache_middleware) - w3.middleware_onion.add( - _create_cache(app, 600, is_glm_or_projects_contract, _generic_should_cache_web3) - ) - w3.middleware_onion.add( - _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) - ) - w3.middleware_onion.add( - _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) - ) + w3.middleware_onion.add(middleware.time_based_cache_middleware) + w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) + w3.middleware_onion.add(middleware.simple_cache_middleware) + w3.middleware_onion.add( + _create_cache( + app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 + ) + ) + w3.middleware_onion.add( + _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) + ) + w3.middleware_onion.add( + _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) + ) glm.init_web3(w3, app.config["GLM_CONTRACT_ADDRESS"]) epochs.init_web3(w3, app.config["EPOCHS_CONTRACT_ADDRESS"]) From b2601f80c29fae7bf985a7a33f3e8267889ff9f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Wed, 17 Jul 2024 01:36:21 +0200 Subject: [PATCH 04/12] test1 --- backend/app/extensions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 33c35ce5e7..67258868fd 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -114,11 +114,11 @@ def init_web3(app): w3.middleware_onion.add(middleware.time_based_cache_middleware) w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) w3.middleware_onion.add(middleware.simple_cache_middleware) - w3.middleware_onion.add( - _create_cache( - app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 - ) - ) + # w3.middleware_onion.add( + # _create_cache( + # app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 + # ) + # ) w3.middleware_onion.add( _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) ) From 43a8c95009a52587bd73ebe00a8fd7b00594a043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Wed, 17 Jul 2024 01:36:34 +0200 Subject: [PATCH 05/12] test2 --- backend/app/extensions.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 67258868fd..4a347a8e3b 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -114,17 +114,17 @@ def init_web3(app): w3.middleware_onion.add(middleware.time_based_cache_middleware) w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) w3.middleware_onion.add(middleware.simple_cache_middleware) - # w3.middleware_onion.add( - # _create_cache( - # app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 - # ) - # ) w3.middleware_onion.add( - _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) + _create_cache( + app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 + ) ) w3.middleware_onion.add( - _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) + _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) ) + # w3.middleware_onion.add( + # _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) + # ) glm.init_web3(w3, app.config["GLM_CONTRACT_ADDRESS"]) epochs.init_web3(w3, app.config["EPOCHS_CONTRACT_ADDRESS"]) From abf88b01bc5a1bcaa9ed0d12d524e2558047ceea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Wed, 17 Jul 2024 01:36:42 +0200 Subject: [PATCH 06/12] test3 --- backend/app/extensions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 4a347a8e3b..47b11dd59c 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -114,11 +114,11 @@ def init_web3(app): w3.middleware_onion.add(middleware.time_based_cache_middleware) w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) w3.middleware_onion.add(middleware.simple_cache_middleware) - w3.middleware_onion.add( - _create_cache( - app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 - ) - ) + # w3.middleware_onion.add( + # _create_cache( + # app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 + # ) + # ) w3.middleware_onion.add( _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) ) From 8bc52f1e3f904a9dcf3e0a1a09e9689968a27eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Wed, 17 Jul 2024 01:48:25 +0200 Subject: [PATCH 07/12] test1 --- backend/app/extensions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 47b11dd59c..67258868fd 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -122,9 +122,9 @@ def init_web3(app): w3.middleware_onion.add( _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) ) - # w3.middleware_onion.add( - # _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) - # ) + w3.middleware_onion.add( + _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) + ) glm.init_web3(w3, app.config["GLM_CONTRACT_ADDRESS"]) epochs.init_web3(w3, app.config["EPOCHS_CONTRACT_ADDRESS"]) From 56dbce144dca5c273accd54f435765c5c01e69f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Wed, 17 Jul 2024 01:51:59 +0200 Subject: [PATCH 08/12] testx --- backend/app/extensions.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 67258868fd..5eeec04d8f 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -111,20 +111,20 @@ def init_web3(app): w3.provider = app.config["WEB3_PROVIDER"] if middleware.geth_poa_middleware not in w3.middleware_onion: w3.middleware_onion.inject(middleware.geth_poa_middleware, layer=0) - w3.middleware_onion.add(middleware.time_based_cache_middleware) - w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) - w3.middleware_onion.add(middleware.simple_cache_middleware) + # w3.middleware_onion.add(middleware.time_based_cache_middleware) + # w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) + # w3.middleware_onion.add(middleware.simple_cache_middleware) # w3.middleware_onion.add( # _create_cache( # app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 # ) # ) - w3.middleware_onion.add( - _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) - ) - w3.middleware_onion.add( - _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) - ) + # w3.middleware_onion.add( + # _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) + # ) + # w3.middleware_onion.add( + # _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) + # ) glm.init_web3(w3, app.config["GLM_CONTRACT_ADDRESS"]) epochs.init_web3(w3, app.config["EPOCHS_CONTRACT_ADDRESS"]) From 7de67fa3bd877e810a4da5ac8fa8cca8f0ac8b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Kluczek?= Date: Wed, 17 Jul 2024 02:01:42 +0200 Subject: [PATCH 09/12] test --- backend/app/extensions.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 5eeec04d8f..21fe94daf2 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -114,17 +114,17 @@ def init_web3(app): # w3.middleware_onion.add(middleware.time_based_cache_middleware) # w3.middleware_onion.add(middleware.latest_block_based_cache_middleware) # w3.middleware_onion.add(middleware.simple_cache_middleware) - # w3.middleware_onion.add( - # _create_cache( - # app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 - # ) - # ) - # w3.middleware_onion.add( - # _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) - # ) - # w3.middleware_onion.add( - # _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) - # ) + w3.middleware_onion.add( + _create_cache( + app, 600, is_glm_or_projects_contract, _generic_should_cache_web3 + ) + ) + w3.middleware_onion.add( + _create_cache(app, 60, is_epochs_or_deposits_contract, (lambda x: True)) + ) + w3.middleware_onion.add( + _create_cache(app, 20, is_get_block_by_number, _generic_should_cache_web3) + ) glm.init_web3(w3, app.config["GLM_CONTRACT_ADDRESS"]) epochs.init_web3(w3, app.config["EPOCHS_CONTRACT_ADDRESS"]) From 7f947ede6b49a77c552f42625d355c27322eb92b Mon Sep 17 00:00:00 2001 From: Pawel Peregud Date: Tue, 23 Jul 2024 16:19:03 +0200 Subject: [PATCH 10/12] localenv: add missing env vars --- localenv/apitest.yaml | 6 ++++++ localenv/localenv.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/localenv/apitest.yaml b/localenv/apitest.yaml index be083c373e..a1444a0ca6 100644 --- a/localenv/apitest.yaml +++ b/localenv/apitest.yaml @@ -27,6 +27,12 @@ services: CHAIN_ID: "1337" CHAIN_NAME: "localhost" + GC_PASSPORT_SCORER_API_KEY: "${GC_PASSPORT_SCORER_API_KEY}" + GC_PASSPORT_SCORER_ID: "${GC_PASSPORT_SCORER_ID}" + + DELEGATION_SALT: "salt" + DELEGATION_SALT_PRIMARY: "salt_primary" + depends_on: - anvil - graph-node diff --git a/localenv/localenv.yaml b/localenv/localenv.yaml index 0e07271100..080f48b5f2 100644 --- a/localenv/localenv.yaml +++ b/localenv/localenv.yaml @@ -42,6 +42,12 @@ services: CHAIN_ID: "1337" CHAIN_NAME: "localhost" + GC_PASSPORT_SCORER_API_KEY: "${GC_PASSPORT_SCORER_API_KEY}" + GC_PASSPORT_SCORER_ID: "${GC_PASSPORT_SCORER_ID}" + + DELEGATION_SALT: "salt" + DELEGATION_SALT_PRIMARY: "salt_primary" + depends_on: - backend-postgres - anvil From 269fffc881784e51254ce530259fcc18aae2f514 Mon Sep 17 00:00:00 2001 From: Pawel Peregud Date: Tue, 23 Jul 2024 10:40:27 +0200 Subject: [PATCH 11/12] localenv: don't crash if there are no containers to cleanup --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index eef860c2dd..08bf136830 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "apitest:up": "docker compose -p apitest -f ./localenv/docker-compose.yaml -f ./localenv/apitest.yaml up anvil ipfs graph-postgres graph-node multideployer", "apitest:down": "docker compose -p apitest -f ./localenv/docker-compose.yaml -f ./localenv/apitest.yaml down", "apitest:run": "docker compose -p apitest -f ./localenv/docker-compose.yaml -f ./localenv/apitest.yaml run backend-apitest", - "preapitest:up": "docker rm -v -f $(docker ps -qa --filter 'name=apitest')", + "preapitest:up": "docker rm -v -f $(docker ps -qa --filter 'name=apitest') || true", "preapitest:run": "yarn localenv:build-backend" }, "repository": { From 1b306bb48518d231c01a6d3296e3f5f37631a04a Mon Sep 17 00:00:00 2001 From: Pawel Peregud Date: Tue, 23 Jul 2024 10:39:53 +0200 Subject: [PATCH 12/12] localenv: fix link to base image for anvil --- localenv/anvil/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localenv/anvil/Dockerfile b/localenv/anvil/Dockerfile index a6528f1038..721ada3450 100644 --- a/localenv/anvil/Dockerfile +++ b/localenv/anvil/Dockerfile @@ -1,4 +1,4 @@ -FROM europe-docker.pkg.dev/wildland-dev/octant-test/foundry:latest +FROM europe-docker.pkg.dev/wildland-dev/internal/octant/anvil:latest WORKDIR /home/foundry