test: support .env.test to reuse env variables across the tests

.env.test

@@ -0,0 +1,9 @@
+# We use these images during sim and e2e tests
+GETH_DOCKER_IMAGE=ethereum/client-go:v1.13.11
+# Use either image or local binary for the testing
+GETH_BINARY_DIR=
+LIGHTHOUSE_DOCKER_IMAGE=sigp/lighthouse:v4.6.0-amd64-modern-dev
+NETHERMIND_DOCKER_IMAGE=nethermind/nethermind:1.25.3
+# We mostly use mainnet for unit testing
+# Changing this value may impact the tests which are written with mainnet in mind
+LODESTAR_PRESET=mainnet

.github/actions/dotenv/action.yml

@@ -0,0 +1,5 @@
+name: "Setup env variables using .env file"
+description: "Load .env file from root of repo and setup for CI runner"
+runs:
+  using: "node20"
+  main: index.js

.github/actions/dotenv/index.js

@@ -0,0 +1,29 @@
+const fs = require("fs");
+const core = require("@actions/core");
+const dotEnv = require("dotenv");
+const envFile = ".env.test";
+
+if (!fs.existsSync(envFile)) {
+  core.setFailed("File .env not found");
+}
+
+const result = dotEnv.config({path: envFile});
+if (result.error) {
+  core.setFailed(result.error.message);
+} else {
+  core.setOutput("env", result.parsed);
+  core.info("Env file loaded");
+  core.info("Populating env variables...");
+
+  for (const key in result.parsed) {
+    const value = result.parsed[key];
+    core.info(`${key}=${value}`);
+
+    // Export variable
+    core.exportVariable(key, value);
+
+    // Set to output so it can be used in as the input for the next job/step
+    core.setOutput(key, value);
+
+  }
+}

.github/workflows/test-sim.yml

@@ -22,11 +22,6 @@ on:
         type: number
         default: 40
 
-env:
-  GETH_DOCKER_IMAGE: ethereum/client-go:v1.13.11
-  LIGHTHOUSE_DOCKER_IMAGE: sigp/lighthouse:latest-amd64-modern-dev
-  NETHERMIND_DOCKER_IMAGE: nethermind/nethermind:1.25.3
-
 jobs:
   tests-sim:
     name: Sim tests
@@ -58,6 +53,9 @@ jobs:
         if: steps.cache-deps.outputs.cache-hit == 'true'
       # </common-build>
 
+      - name: Load env variables 
+        uses: ./.github/actions/dotenv
+
       - name: Download required docker images before running tests
         run: |
           docker pull ${{env.GETH_DOCKER_IMAGE}}

.github/workflows/test.yml

@@ -12,10 +12,6 @@ on:
   pull_request:
   workflow_dispatch:
 
-env:
-  GETH_DOCKER_IMAGE: ethereum/client-go:v1.11.6
-  NETHERMIND_DOCKER_IMAGE: nethermind/nethermind:1.18.0
-
 jobs:
   build:
     name: Build
@@ -225,6 +221,9 @@ jobs:
           key: ${{ runner.os }}-node-${{ matrix.node }}-${{ github.sha }}
           fail-on-cache-miss: true
 
+      - name: Load env variables 
+        uses: ./.github/actions/dotenv
+
       - name: Run the e2e test environment
         run: scripts/run_e2e_env.sh start
 

CONTRIBUTING.md

@@ -35,7 +35,7 @@ To run tests:
 Note that to run `test:e2e`, first ensure that the environment is correctly setup by running the `run_e2e_env.sh` script.
 
 ```sh
-GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6  NETHERMIND_DOCKER_IMAGE=nethermind/nethermind:1.18.0 ./scripts/run_e2e_env.sh start
+./scripts/run_e2e_env.sh start
 ```
 
 Similarly, run `yarn download-spec-tests` before running `yarn test:spec`.
@@ -63,9 +63,9 @@ If you observe following error running any of the test files that means you are
 - Spec tests often compare full expected vs actual states in JSON format.
 - A single logical error can cause many spec tests to fail. To focus on a single test at a time you can use vitest's option `--bail 1` to stop at the first failed test
 - To then run only that failed test you can run against a specific file as use vitest's filters option `-t <pattern>` to run only one case
+- Before running the tests, make sure to switch to the package directory (e.g. `packages/beacon-node`) to speed up test execution
 
 ```sh
-cd packages/beacon-node
 LODESTAR_PRESET=minimal yarn vitest --run --bail 1 --config vitest.spec.config.ts test/spec/presets/sanity.test.ts -t attester_slashing
 ```
 

docs/pages/contribution/testing/simulation-tests.md

@@ -12,7 +12,7 @@ There are a number of sim tests that are available and each has a slightly diffe
 
 ### Environment Variables
 
-To see what typical values for these are check out the `test-sim.yaml` workflow file in the `.github/workflows` directory.
+To see what typical values for these are check out the `.env.test` file in the root directory.
 
 - `GETH_DOCKER_IMAGE`: The geth docker image that will be used
 - `NETHERMIND_IMAGE`: The nethermind docker image that will be used
@@ -23,19 +23,15 @@ To see what typical values for these are check out the `test-sim.yaml` workflow
 The multi-fork sim test checks most of the functionality Lodestar provides. Is verifies that Lodestar is capable of peering, moving through all of the forks and using various sync methods in a testnet environment. Lodestar is tested with both Geth and Nethermind as the execution client. It also checks a Lighthouse/Geth node for cross client compatibility.
 
 ```sh
-GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 \
-    LIGHTHOUSE_DOCKER_IMAGE=sigp/lighthouse:latest-amd64-modern-dev \
-    NETHERMIND_DOCKER_IMAGE=nethermind/nethermind:1.18.0 \
-    yarn workspace @chainsafe/lodestar test:sim:multifork
+yarn workspace @chainsafe/lodestar test:sim:multifork
 ```
 
 ### `test:sim:endpoints`
 
 This tests that various endpoints of the beacon node and validator client are working as expected.
 
 ```sh
-GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 \
-    yarn workspace @chainsafe/lodestar test:sim:endpoints
+yarn workspace @chainsafe/lodestar test:sim:endpoints
 ```
 
 ### `test:sim:deneb`
@@ -47,9 +43,7 @@ This test is still included in our CI but is no longer as important as it once w
 Checks that Lodestar is compatible with other consensus validators and vice-versa. All tests use Geth as the EL.
 
 ```sh
-GETH_DOCKER_IMAGE=ethereum/client-go:v1.11.6 \
-    LIGHTHOUSE_DOCKER_IMAGE=sigp/lighthouse:latest-amd64-modern-dev \
-    yarn workspace @chainsafe/lodestar test:sim:mixedclient
+yarn workspace @chainsafe/lodestar test:sim:mixedclient
 ```
 
 ## Sim Test Infrastructure

package.json

@@ -39,13 +39,14 @@
     "test-coverage:e2e-sim": "c8 --config .c8rc.json --report-dir coverage/e2e-sim/ --all npm run test:e2e:sim",
     "test-coverage:spec": "c8 --config .c8rc.json --report-dir coverage/spec/ --all npm run test:spec",
     "benchmark": "yarn benchmark:files 'packages/*/test/perf/**/*.test.ts'",
-    "benchmark:files": "LODESTAR_PRESET=mainnet NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",
+    "benchmark:files": "NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",
     "release:create-rc": "node scripts/release/create_rc.mjs",
     "release:tag-rc": "node scripts/release/tag_rc.mjs",
     "release:tag-stable": "node scripts/release/tag_stable.mjs",
     "release:publish": "lerna publish from-package --yes --no-verify-access"
   },
   "devDependencies": {
+    "@actions/core": "^1.10.1",
     "@chainsafe/eslint-plugin-node": "^11.2.3",
     "@dapplion/benchmark": "^0.2.4",
     "@types/mocha": "^10.0.6",
@@ -56,6 +57,7 @@
     "@vitest/browser": "^1.2.1",
     "codecov": "^3.8.3",
     "crypto-browserify": "^3.12.0",
+    "dotenv": "^16.4.1",
     "electron": "^26.2.2",
     "eslint": "^8.50.0",
     "eslint-import-resolver-typescript": "^3.6.1",

packages/beacon-node/package.json

@@ -76,10 +76,10 @@
     "lint": "eslint --color --ext .ts src/ test/",
     "lint:fix": "yarn run lint --fix",
     "test": "yarn test:unit && yarn test:e2e",
-    "test:unit:minimal": "vitest --run --segfaultRetry 3 --dir test/unit/",
-    "test:unit:mainnet": "LODESTAR_PRESET=mainnet vitest --run --dir test/unit-mainnet",
+    "test:unit:minimal": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --dir test/unit/",
+    "test:unit:mainnet": "LODESTAR_PRESET=mainnet vitest --run --segfaultRetry 3 --dir test/unit-mainnet",
     "test:unit": "wrapper() { yarn test:unit:minimal $@ && yarn test:unit:mainnet $@; }; wrapper",
-    "test:e2e": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --config vitest.e2e.config.ts --dir test/e2e",
+    "test:e2e": "vitest --run --segfaultRetry 3 --config vitest.e2e.config.ts --dir test/e2e",
     "test:sim": "vitest --run test/sim/**/*.test.ts",
     "test:sim:merge-interop": "vitest --run test/sim/merge-interop.test.ts",
     "test:sim:mergemock": "vitest --run test/sim/mergemock.test.ts",

packages/cli/package.json

@@ -32,11 +32,11 @@
     "lint:fix": "yarn run lint --fix",
     "test:unit": "vitest --run --dir test/unit/",
     "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
-    "test:sim:multifork": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_fork.test.ts",
-    "test:sim:mixedclient": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/mixed_client.test.ts",
-    "test:sim:endpoints": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/endpoints.test.ts",
-    "test:sim:deneb": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/deneb.test.ts",
-    "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/backup_eth_provider.test.ts",
+    "test:sim:multifork": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/multi_fork.test.ts",
+    "test:sim:mixedclient": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/mixed_client.test.ts",
+    "test:sim:endpoints": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/endpoints.test.ts",
+    "test:sim:deneb": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/deneb.test.ts",
+    "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/backup_eth_provider.test.ts",
     "test": "yarn test:unit && yarn test:e2e",
     "coverage": "codecov -F lodestar",
     "check-readme": "typescript-docs-verifier"

packages/cli/test/utils/simulation/execution_clients/geth.ts

@@ -21,7 +21,7 @@ export const generateGethNode: ExecutionNodeGenerator<ExecutionClient.Geth> = (o
   const {id, mode, ttd, address, mining, clientOptions, nodeIndex} = opts;
   const ports = getNodePorts(nodeIndex);
 
-  const isDocker = process.env.GETH_DOCKER_IMAGE !== undefined;
+  const isDocker = !!process.env.GETH_DOCKER_IMAGE;
   const binaryPath = isDocker ? "" : `${process.env.GETH_BINARY_DIR}/geth`;
   const {rootDir, rootDirMounted, genesisFilePathMounted, logFilePath, jwtsecretFilePathMounted} = getNodeMountedPaths(
     opts.paths,

packages/logger/package.json

@@ -61,7 +61,7 @@
     "test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit",
     "test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit",
     "test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'",
-    "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e",
+    "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e",
     "check-readme": "typescript-docs-verifier"
   },
   "types": "lib/index.d.ts",

packages/params/package.json

@@ -58,7 +58,7 @@
     "test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit",
     "test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit",
     "test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'",
-    "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
+    "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
     "check-readme": "typescript-docs-verifier"
   },
   "repository": {

packages/params/test/e2e/overridePreset.test.ts

@@ -21,8 +21,7 @@ describe("Override preset", function () {
   vi.setConfig({testTimeout: 30_000});
 
   it("Should correctly override preset", async () => {
-    // These commands can not run with minimal preset
-    if (process.env.LODESTAR_PRESET === "minimal") delete process.env.LODESTAR_PRESET;
+    if (process.env.LODESTAR_PRESET) delete process.env.LODESTAR_PRESET;
 
     await exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.ok)}`);
   });

packages/params/test/e2e/setPreset.test.ts

@@ -21,8 +21,7 @@ describe("setPreset", function () {
   vi.setConfig({testTimeout: 30_000});
 
   it("Should correctly set preset", async () => {
-    // These commands can not run with minimal preset
-    if (process.env.LODESTAR_PRESET === "minimal") delete process.env.LODESTAR_PRESET;
+    if (process.env.LODESTAR_PRESET) delete process.env.LODESTAR_PRESET;
 
     await exec(`node --loader ts-node/esm ${path.join(__dirname, scriptNames.ok)}`);
   });

packages/prover/package.json

@@ -57,7 +57,7 @@
     "test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit",
     "test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit",
     "test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'",
-    "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e",
+    "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e",
     "check-readme": "typescript-docs-verifier",
     "generate-fixtures": "node --loader ts-node/esm scripts/generate_fixtures.ts"
   },

packages/types/package.json

@@ -64,7 +64,7 @@
     "test": "yarn test:unit",
     "test:constants:minimal": "LODESTAR_PRESET=minimal vitest --run --dir test/constants/",
     "test:constants:mainnet": "LODESTAR_PRESET=mainnet vitest --run --dir test/constants/",
-    "test:unit": "wrapper() { yarn test:constants:minimal $@ && yarn test:constants:mainnet $@ && vitest --run --dir test/unit/ $@; }; wrapper",
+    "test:unit": "wrapper() { yarn test:constants:minimal $@ && yarn test:constants:mainnet $@ && LODESTAR_PRESET=mainnet vitest --run --dir test/unit/ $@; }; wrapper",
     "test:browsers": "yarn test:browsers:chrome && yarn test:browsers:firefox && yarn test:browsers:electron",
     "test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit",
     "test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit",

packages/validator/package.json

@@ -30,7 +30,7 @@
     "test:unit": "vitest --run --dir test/unit/",
     "test": "yarn test:unit && yarn test:e2e",
     "test:spec": "vitest --run --config vitest.spec.config.ts --dir test/spec/",
-    "test:e2e": "LODESTAR_PRESET=mainnet vitest --run --config vitest.e2e.config.ts --dir test/e2e",
+    "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e",
     "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
     "coverage": "codecov -F lodestar-validator",
     "check-readme": "typescript-docs-verifier"

scripts/run_e2e_env.sh

@@ -2,6 +2,7 @@
 
 function start_app() {
   mkdir -p test-logs/e2e-test-env
+  source ../.env.test
   export LODESTAR_PRESET=minimal
   nohup node --loader ts-node/esm packages/cli/test/scripts/e2e_test_env.ts > test-logs/e2e-test-env/simulation.out 2>&1 &
   echo $! > test-logs/e2e-test-env/simulation.pid

scripts/vitest/setupFiles/dotenv.ts

@@ -0,0 +1,8 @@
+import path from "node:path";
+// It's a dev dependency
+// eslint-disable-next-line import/no-extraneous-dependencies
+import {config} from "dotenv";
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const __dirname = new URL(".", import.meta.url).pathname;
+
+config({path: path.join(__dirname, "../../../.env.test")});

vitest.base.browser.config.ts

@@ -24,7 +24,7 @@ export default defineConfig({
       "**/.{idea,git,cache,output,temp}/**",
       "**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build}.config.*",
     ],
-    setupFiles: [path.join(__dirname, "./scripts/vitest/customMatchers.ts")],
+    setupFiles: [path.join(__dirname, "./scripts/vitest/setupFiles/customMatchers.ts")],
     reporters: ["default", "hanging-process"],
     coverage: {
       enabled: false,

vitest.base.unit.config.ts

@@ -14,7 +14,10 @@ export default defineConfig({
       "**/.{idea,git,cache,output,temp}/**",
       "**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build}.config.*",
     ],
-    setupFiles: [path.join(__dirname, "./scripts/vitest/customMatchers.ts")],
+    setupFiles: [
+      path.join(__dirname, "./scripts/vitest/setupFiles/customMatchers.ts"),
+      path.join(__dirname, "./scripts/vitest/setupFiles/dotenv.ts"),
+    ],
     reporters: ["default", "hanging-process"],
     coverage: {
       enabled: process.env.CI === "true",

yarn.lock

@@ -22,6 +22,14 @@
     semver "^6.1.0"
     uuid "^3.3.3"
 
+"@actions/core@^1.10.1":
+  version "1.10.1"
+  resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.10.1.tgz#61108e7ac40acae95ee36da074fa5850ca4ced8a"
+  integrity sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==
+  dependencies:
+    "@actions/http-client" "^2.0.1"
+    uuid "^8.3.2"
+
 "@actions/core@^1.2.6":
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.10.0.tgz#44551c3c71163949a2f06e94d9ca2157a0cfac4f"
@@ -5617,6 +5625,11 @@ dotenv-expand@~10.0.0:
   resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-10.0.0.tgz#12605d00fb0af6d0a592e6558585784032e4ef37"
   integrity sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==
 
+dotenv@^16.4.1:
+  version "16.4.1"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.1.tgz#1d9931f1d3e5d2959350d1250efab299561f7f11"
+  integrity sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==
+
 dotenv@~16.3.1:
   version "16.3.1"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e"