Feat/dockerize spock

.env

@@ -1,10 +1,60 @@
+### PROD USAGE
 VL_DB_DATABASE=database
 VL_DB_USER=user
 VL_DB_PASSWORD=password
 VL_DB_HOST=localhost
 VL_DB_PORT=5433
-VL_CHAIN_HOST=https://eth-goerli.alchemyapi.io/v2/p7bY4ggxW60weHKPiAP7HXN2RNAAQZ8E
-VL_CHAIN_HOST_L2=https://arb-goerli.g.alchemy.com/v2/rN1vSB6tSdjfWGno6SSZdjOB8m8LvM_0
-VL_CONFIG_NAME=multi_goerli
+VL_CHAIN_HOST=https://eth-mainnet.g.alchemy.com/v2/<PROD KEY>
+VL_CHAIN_HOST_L2=https://arb-mainnet.g.alchemy.com/v2/<PROD KEY>
+VL_CONFIG_NAME=multi
+
+### STAGING USAGE
+# VL_DB_DATABASE=database
+# VL_DB_USER=user
+# VL_DB_PASSWORD=password
+# VL_DB_HOST=localhost
+# VL_DB_PORT=5433
+# VL_CHAIN_HOST=https://eth-mainnet.g.alchemy.com/v2/<STAGING KEY>
+# VL_CHAIN_HOST_L2=https://arb-mainnet.g.alchemy.com/v2/<STAGING KEY>
+# VL_CONFIG_NAME=multi
+
+### GOERLI USAGE
+# VL_DB_DATABASE=database
+# VL_DB_USER=user
+# VL_DB_PASSWORD=password
+# VL_DB_HOST=localhost
+# VL_DB_PORT=5433
+# VL_CHAIN_HOST=https://eth-goerli.alchemyapi.io/v2/<GOERLI KEY>
+# VL_CHAIN_HOST_L2=https://arb-goerli.g.alchemy.com/v2/<GOERLI KEY>
+# VL_CONFIG_NAME=multi_goerli
+# VL_LOGGING_LEVEL=3
+# STARTING_BLOCK_GOERLI=5273000
+
+### DOCKER USAGE
+# VL_DB_DATABASE=database
+# VL_DB_USER=user
+# VL_DB_PASSWORD=password
+# VL_DB_HOST=host.docker.internal
+# VL_DB_PORT=5432
+# VL_CHAIN_HOST=http://host.docker.internal:8545
+# VL_CHAIN_HOST_L2=http://host.docker.internal:8546
+# VL_CONFIG_NAME=docker_config
+# VL_LOGGING_LEVEL=3
+# STARTING_BLOCK_GOERLI=7810463
+# STARTING_BLOCK_ARB_TESTNET=793614
+
+
+### LOCAL USAGE
+# VL_DB_DATABASE=database
+# VL_DB_USER=user
+# VL_DB_PASSWORD=password
+# VL_DB_HOST=localhost
+# VL_DB_PORT=5432
+# VL_CHAIN_HOST=http://localhost:8545
+# VL_CHAIN_HOST_L2=http://localhost:8546
+# VL_CONFIG_NAME=docker_config
+# VL_LOGGING_LEVEL=3
+# STARTING_BLOCK_GOERLI=7810463
+# STARTING_BLOCK_ARB_TESTNET=793614
+
 
-VL_LOGGING_LEVEL=3

README.md

@@ -26,9 +26,14 @@ yarn start-api     # starts GraphQL API
 
 ```
 
-## Environment 
+### To Build the services in a docker container (optional)
+
+`docker build -t makerdaodux/govpolldb-app:latest .`
+
+## Environment
+
+Create the connection details for the database by setting the following environment variables. A default .env file is provided, which uses the following values:
 
-Create the connection details for the database by setting the following environment variables.  A default .env file is provided, which uses the following values:
 ```
 VL_DB_DATABASE=database
 VL_DB_USER=user
@@ -37,7 +42,7 @@ VL_DB_HOST=localhost
 VL_DB_PORT=5433
 ```
 
-You'll also need to set a `VL_CHAIN_HOST` env variable that points to an ethereum node.  Alchemy is recommended, Infura is not.  See the [spock readme](https://github.com/oasisdex/spock) for more information, and also for information about the `VL_LOGGING_LEVEL` env variable.  
+You'll also need to set a `VL_CHAIN_HOST` env variable that points to an ethereum node. Alchemy is recommended, Infura is not. See the [spock readme](https://github.com/oasisdex/spock) for more information, and also for information about the `VL_LOGGING_LEVEL` env variable.
 
 ## Using Postico (optional)
 

config.js

@@ -12,6 +12,9 @@ const voteProxyFactoryTransformer = require('./transformers/VoteProxyFactoryTran
 const esmTransformer = require('./transformers/EsmTransformer');
 const esmV2Transformer = require('./transformers/EsmV2Transformer');
 const voteDelegateFactoryTransformer = require('./transformers/VoteDelegateFactoryTransformer');
+const {
+  BlockGenerator,
+} = require('@makerdao-dux/spock-etl/dist/blockGenerator/blockGenerator');
 
 //mainnet
 const MKR_ADDRESS = '0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2';
@@ -52,6 +55,11 @@ const ARB_TESTNET_POLLING_ADDRESS =
 const CHAIN_HOST_L1 = process.env.VL_CHAIN_HOST;
 const CHAIN_HOST_L2 = process.env.VL_CHAIN_HOST_L2;
 
+const STARTING_BLOCK_GOERLI = parseInt(process.env.STARTING_BLOCK_GOERLI);
+const STARTING_BLOCK_ARB_TESTNET = parseInt(
+  process.env.STARTING_BLOCK_ARB_TESTNET
+);
+
 const goerli = {
   name: 'goerli',
   processorSchema: 'vulcan2x',
@@ -215,13 +223,56 @@ const arbitrumTestnet = {
     console.log(`Starting with these services: ${Object.keys(services)}`),
 };
 
+const dockerConfig = [
+  {
+    ...goerli,
+    onStart: async (services) => {
+      console.log(
+        `Starting Goerli config with these services: ${Object.keys(services)}`
+      );
+
+      // Blocknumbers can be provided to add certain events to the test database on an adhoc basis
+      if (process.env.SEED_BLOCKS) {
+        const blocks = process.env.SEED_BLOCKS.split(',');
+
+        console.log(`Initial seed blocks to scrape: ${blocks}`);
+
+        // Set the batch size to 1 so we only extract the single block we want
+        const modServices = {
+          ...services,
+          config: {
+            ...services.config,
+            blockGenerator: {
+              batch: 1,
+            },
+          },
+        };
+
+        const b = new BlockGenerator(modServices);
+        await b.init();
+        await Promise.all(
+          blocks.map((blockNumber) =>
+            b.run(parseInt(blockNumber), parseInt(blockNumber))
+          )
+        );
+
+        await b.deinit();
+      }
+    },
+  },
+  { ...arbitrumTestnet, startingBlock: STARTING_BLOCK_ARB_TESTNET },
+];
+
 let config;
 if (process.env.VL_CONFIG_NAME === 'multi') {
   console.log('Using Mainnet multi-chain config');
   config = [mainnet, arbitrum];
 } else if (process.env.VL_CONFIG_NAME === 'multi_goerli') {
   console.log('Using Goerli multi-chain config');
   config = [goerli, arbitrumTestnet];
+} else if (process.env.VL_CONFIG_NAME === 'docker_config') {
+  console.log('Using Docker multi-chain config');
+  config = dockerConfig;
 }
 
 module.exports.default = config;

package.json

@@ -9,6 +9,7 @@
     "start-sync": "yarn migrate",
     "migrate": "node -r ./loadenv.js ./node_modules/@makerdao-dux/spock-etl/dist/bin/migrate ./config.js",
     "start-api": "node -r ./loadenv.js ./node_modules/@makerdao-dux/spock-graphql-api/dist/index.js ./config.js",
+    "start-all": "yarn migrate && yarn start-api & yarn start-etl",
     "test": "jest",
     "test:ci": "./tests/run-tests.sh",
     "repl": "node --experimental-repl-await -r ./loadenv.js ./repl"