validate ENV values at run-time (blockscout#1184)

* migrate from zod to yul schema

* describe full schema

* adjust docker integration

* make script for downloading app assets

* change links to downloaded assets in the config code

* docker integration

* validate external json configs

* better schemas and ts integration

* update docs and dev script

* gh workflow

* try to fail gh workflow

* install global deps and adjust script for gh workflow

* refinements

* make workflow to pass

* 🙈

* fix tests

.env.example

@@ -1,5 +1,5 @@
-NEXT_PUBLIC_SENTRY_DSN=xxx
-SENTRY_CSP_REPORT_URI=xxx
+NEXT_PUBLIC_SENTRY_DSN=https://sentry.io
+SENTRY_CSP_REPORT_URI=https://sentry.io
 NEXT_PUBLIC_WALLET_CONNECT_PROJECT_ID=xxx
 NEXT_PUBLIC_RE_CAPTCHA_APP_SITE_KEY=xxx
 NEXT_PUBLIC_GOOGLE_ANALYTICS_PROPERTY_ID=UA-XXXXXX-X

.github/workflows/checks.yml

@@ -50,9 +50,49 @@ jobs:
       - name: Compile TypeScript
         run: yarn lint:tsc
 
+  envs_validation:
+    name: ENV variables presets validation
+    runs-on: ubuntu-latest
+    needs: [ code_quality ]
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'yarn'
+
+      - name: Cache node_modules
+        uses: actions/cache@v3
+        id: cache-node-modules
+        with:
+          path: |
+            node_modules
+          key: node_modules-${{ runner.os }}-${{ hashFiles('yarn.lock') }}
+
+      - name: Install dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: yarn --frozen-lockfile
+
+      - name: Install script dependencies
+        run: cd ./deploy/tools/envs-validator && yarn --frozen-lockfile
+
+      - name: Copy secrets file
+        run: cp ./.env.example ./configs/envs/.env.secrets
+
+      - name: Run validation script
+        run: |
+          set +e
+          cd ./deploy/tools/envs-validator && yarn dev
+          exitcode="$?"
+          echo "exitcode=$exitcode" >> $GITHUB_OUTPUT
+          exit "$exitcode"
+
   jest_tests:
     name: Jest tests
-    needs: [ code_quality ]
+    needs: [ code_quality, envs_validation ]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -81,7 +121,7 @@ jobs:
 
   pw_tests:
     name: 'Playwright tests / Project: ${{ matrix.project }}'
-    needs: [ code_quality ]
+    needs: [ code_quality, envs_validation ]
     runs-on: ubuntu-latest
     container:
       image: mcr.microsoft.com/playwright:v1.35.1-focal

.gitignore

@@ -12,6 +12,7 @@
 # next.js
 /.next/
 /out/
+/public/assets/
 
 # production
 /build

Dockerfile

@@ -64,16 +64,13 @@ RUN yarn build
 # Copy dependencies and source code, then build
 COPY --from=deps /feature-reporter/node_modules ./deploy/tools/feature-reporter/node_modules
 RUN cd ./deploy/tools/feature-reporter && yarn compile_config
-RUN cd ./deploy/tools/feature-reporter &&  yarn build
+RUN cd ./deploy/tools/feature-reporter && yarn build
 
 
 ### ENV VARIABLES CHECKER
 # Copy dependencies and source code, then build 
-WORKDIR /envs-validator
-COPY --from=deps /envs-validator/node_modules ./node_modules
-COPY ./deploy/tools/envs-validator .
-COPY ./types/envs.ts .
-RUN yarn build
+COPY --from=deps /envs-validator/node_modules ./deploy/tools/envs-validator/node_modules
+RUN cd ./deploy/tools/envs-validator && yarn build
 
 
 # *****************************
@@ -95,14 +92,16 @@ RUN adduser --system --uid 1001 nextjs
 COPY --from=builder /app/next.config.js ./
 COPY --from=builder /app/public ./public
 COPY --from=builder /app/package.json ./package.json
-COPY --from=builder /envs-validator/index.js ./envs-validator.js
+COPY --from=builder /app/deploy/tools/envs-validator/index.js ./envs-validator.js
 COPY --from=builder /app/deploy/tools/feature-reporter/index.js ./feature-reporter.js
 
 # Copy scripts
 ## Entripoint
 COPY --chmod=+x ./deploy/scripts/entrypoint.sh .
 ## ENV replacer
 COPY --chmod=+x ./deploy/scripts/replace_envs.sh .
+## Assets downloader
+COPY --chmod=+x ./deploy/scripts/download_assets.sh .
 ## Favicon generator
 COPY --chmod=+x ./deploy/scripts/favicon_generator.sh .
 COPY ./deploy/tools/favicon-generator ./deploy/tools/favicon-generator

configs/app/features/adsBanner.ts

@@ -1,12 +1,12 @@
 import type { Feature } from './types';
 import type { AdButlerConfig } from 'types/client/adButlerConfig';
+import { SUPPORTED_AD_BANNER_PROVIDERS } from 'types/client/adProviders';
 import type { AdBannerProviders } from 'types/client/adProviders';
 
 import { getEnvValue, parseEnvJson } from '../utils';
 
 const provider: AdBannerProviders = (() => {
   const envValue = getEnvValue(process.env.NEXT_PUBLIC_AD_BANNER_PROVIDER) as AdBannerProviders;
-  const SUPPORTED_AD_BANNER_PROVIDERS: Array<AdBannerProviders> = [ 'slise', 'adbutler', 'coinzilla', 'none' ];
 
   return envValue && SUPPORTED_AD_BANNER_PROVIDERS.includes(envValue) ? envValue : 'slise';
 })();

configs/app/features/adsText.ts

@@ -1,13 +1,12 @@
 import type { Feature } from './types';
+import { SUPPORTED_AD_BANNER_PROVIDERS } from 'types/client/adProviders';
 import type { AdTextProviders } from 'types/client/adProviders';
 
 import { getEnvValue } from '../utils';
 
 const provider: AdTextProviders = (() => {
-  const envValue = getEnvValue(process.env.NEXT_PUBLIC_AD_TEXT_PROVIDER);
-  const SUPPORTED_AD_BANNER_PROVIDERS = [ 'coinzilla', 'none' ];
-
-  return envValue && SUPPORTED_AD_BANNER_PROVIDERS.includes(envValue) ? envValue as AdTextProviders : 'coinzilla';
+  const envValue = getEnvValue(process.env.NEXT_PUBLIC_AD_TEXT_PROVIDER) as AdTextProviders;
+  return envValue && SUPPORTED_AD_BANNER_PROVIDERS.includes(envValue) ? envValue : 'coinzilla';
 })();
 
 const title = 'Text ads';

configs/app/features/marketplace.ts

@@ -1,9 +1,10 @@
 import type { Feature } from './types';
 
 import chain from '../chain';
-import { getEnvValue } from '../utils';
+import { getEnvValue, getExternalAssetFilePath } from '../utils';
 
-const configUrl = getEnvValue(process.env.NEXT_PUBLIC_MARKETPLACE_CONFIG_URL);
+// config file will be downloaded at run-time and saved in the public folder
+const configUrl = getExternalAssetFilePath('NEXT_PUBLIC_MARKETPLACE_CONFIG_URL', process.env.NEXT_PUBLIC_MARKETPLACE_CONFIG_URL);
 const submitFormUrl = getEnvValue(process.env.NEXT_PUBLIC_MARKETPLACE_SUBMIT_FORM);
 
 const title = 'Marketplace';

configs/app/features/web3Wallet.ts

@@ -1,4 +1,5 @@
 import type { Feature } from './types';
+import { SUPPORTED_WALLETS } from 'types/client/wallets';
 import type { WalletType } from 'types/client/wallets';
 
 import { getEnvValue, parseEnvJson } from '../utils';
@@ -9,12 +10,6 @@ const wallets = ((): Array<WalletType> | undefined => {
     return;
   }
 
-  const SUPPORTED_WALLETS: Array<WalletType> = [
-    'metamask',
-    'coinbase',
-    'token_pocket',
-  ];
-
   const wallets = parseEnvJson<Array<WalletType>>(envValue)?.filter((type) => SUPPORTED_WALLETS.includes(type));
 
   if (!wallets || wallets.length === 0) {

configs/app/ui.ts

@@ -1,28 +1,28 @@
 import type { NavItemExternal } from 'types/client/navigation-items';
+import type { ChainIndicatorId } from 'types/homepage';
 import type { NetworkExplorer } from 'types/networks';
-import type { ChainIndicatorId } from 'ui/home/indicators/types';
 
 import * as views from './ui/views';
-import { getEnvValue, parseEnvJson } from './utils';
+import { getEnvValue, getExternalAssetFilePath, parseEnvJson } from './utils';
 
 // eslint-disable-next-line max-len
 const HOMEPAGE_PLATE_BACKGROUND_DEFAULT = 'radial-gradient(103.03% 103.03% at 0% 0%, rgba(183, 148, 244, 0.8) 0%, rgba(0, 163, 196, 0.8) 100%), var(--chakra-colors-blue-400)';
 
 const UI = Object.freeze({
   sidebar: {
     logo: {
-      'default': getEnvValue(process.env.NEXT_PUBLIC_NETWORK_LOGO),
-      dark: getEnvValue(process.env.NEXT_PUBLIC_NETWORK_LOGO_DARK),
+      'default': getExternalAssetFilePath('NEXT_PUBLIC_NETWORK_LOGO', process.env.NEXT_PUBLIC_NETWORK_LOGO),
+      dark: getExternalAssetFilePath('NEXT_PUBLIC_NETWORK_LOGO_DARK', process.env.NEXT_PUBLIC_NETWORK_LOGO_DARK),
     },
     icon: {
-      'default': getEnvValue(process.env.NEXT_PUBLIC_NETWORK_ICON),
-      dark: getEnvValue(process.env.NEXT_PUBLIC_NETWORK_ICON_DARK),
+      'default': getExternalAssetFilePath('NEXT_PUBLIC_NETWORK_ICON', process.env.NEXT_PUBLIC_NETWORK_ICON),
+      dark: getExternalAssetFilePath('NEXT_PUBLIC_NETWORK_ICON_DARK', process.env.NEXT_PUBLIC_NETWORK_ICON_DARK),
     },
     otherLinks: parseEnvJson<Array<NavItemExternal>>(getEnvValue(process.env.NEXT_PUBLIC_OTHER_LINKS)) || [],
-    featuredNetworks: getEnvValue(process.env.NEXT_PUBLIC_FEATURED_NETWORKS),
+    featuredNetworks: getExternalAssetFilePath('NEXT_PUBLIC_FEATURED_NETWORKS', process.env.NEXT_PUBLIC_FEATURED_NETWORKS),
   },
   footer: {
-    links: getEnvValue(process.env.NEXT_PUBLIC_FOOTER_LINKS),
+    links: getExternalAssetFilePath('NEXT_PUBLIC_FOOTER_LINKS', process.env.NEXT_PUBLIC_FOOTER_LINKS),
     frontendVersion: getEnvValue(process.env.NEXT_PUBLIC_GIT_TAG),
     frontendCommit: getEnvValue(process.env.NEXT_PUBLIC_GIT_COMMIT_SHA),
   },

configs/app/ui/views/block.ts

@@ -1,15 +1,8 @@
-import type { ArrayElement } from 'types/utils';
+import type { BlockFieldId } from 'types/views/block';
+import { BLOCK_FIELDS_IDS } from 'types/views/block';
 
 import { getEnvValue, parseEnvJson } from 'configs/app/utils';
 
-export const BLOCK_FIELDS_IDS = [
-  'burnt_fees',
-  'total_reward',
-  'nonce',
-] as const;
-
-export type BlockFieldId = ArrayElement<typeof BLOCK_FIELDS_IDS>;
-
 const blockHiddenFields = (() => {
   const parsedValue = parseEnvJson<Array<BlockFieldId>>(getEnvValue(process.env.NEXT_PUBLIC_VIEWS_BLOCK_HIDDEN_FIELDS)) || [];
 

configs/app/utils.ts

@@ -1,3 +1,5 @@
+import * as regexp from 'lib/regexp';
+
 export const getEnvValue = <T extends string>(env: T | undefined): T | undefined => env?.replaceAll('\'', '"') as T;
 
 export const parseEnvJson = <DataType>(env: string | undefined): DataType | null => {
@@ -7,3 +9,16 @@ export const parseEnvJson = <DataType>(env: string | undefined): DataType | null
     return null;
   }
 };
+
+export const getExternalAssetFilePath = (envName: string, envValue: string | undefined) => {
+  const parsedValue = getEnvValue(envValue);
+
+  if (!parsedValue) {
+    return;
+  }
+
+  const fileName = envName.replace(/^NEXT_PUBLIC_/, '').replace(/_URL$/, '').toLowerCase();
+  const fileExtension = parsedValue.match(regexp.FILE_EXTENSION)?.[1];
+
+  return `/assets/${ fileName }.${ fileExtension }`;
+};

configs/envs/.env.eth

@@ -35,7 +35,7 @@ NEXT_PUBLIC_APP_ENV=development
 NEXT_PUBLIC_GRAPHIQL_TRANSACTION=0xf7d4972356e6ae44ae948d0cf19ef2beaf0e574c180997e969a2837da15e349d
 NEXT_PUBLIC_HAS_BEACON_CHAIN=true
 NEXT_PUBLIC_IS_ACCOUNT_SUPPORTED=true
-NEXT_PUBLIC_AUTH_URL=http://localhost:3000
+# NEXT_PUBLIC_AUTH_URL=http://localhost:3000
 NEXT_PUBLIC_LOGOUT_URL=https://blockscoutcom.us.auth0.com/v2/logout
 NEXT_PUBLIC_STATS_API_HOST=https://stats-eth-main.k8s.blockscout.com
 NEXT_PUBLIC_VISUALIZE_API_HOST=https://visualizer.services.blockscout.com

configs/envs/.env.eth_goerli

@@ -37,7 +37,7 @@ NEXT_PUBLIC_APP_INSTANCE=local
 NEXT_PUBLIC_APP_ENV=development
 NEXT_PUBLIC_GRAPHIQL_TRANSACTION=0xf7d4972356e6ae44ae948d0cf19ef2beaf0e574c180997e969a2837da15e349d
 NEXT_PUBLIC_IS_ACCOUNT_SUPPORTED=true
-NEXT_PUBLIC_AUTH_URL=http://localhost:3000
+# NEXT_PUBLIC_AUTH_URL=http://localhost:3000
 NEXT_PUBLIC_LOGOUT_URL=https://blockscoutcom.us.auth0.com/v2/logout
 NEXT_PUBLIC_MARKETPLACE_CONFIG_URL=https://raw.githubusercontent.com/blockscout/frontend-configs/dev/configs/marketplace/eth-goerli.json
 NEXT_PUBLIC_MARKETPLACE_SUBMIT_FORM=https://airtable.com/shrqUAcjgGJ4jU88C

configs/envs/.env.localhost

@@ -36,5 +36,5 @@ NEXT_PUBLIC_NETWORK_EXPLORERS=[{'title':'Anyblock','baseUrl':'https://explorer.a
 NEXT_PUBLIC_APP_INSTANCE=local
 NEXT_PUBLIC_APP_ENV=development
 NEXT_PUBLIC_IS_ACCOUNT_SUPPORTED=true
-NEXT_PUBLIC_AUTH_URL=http://localhost:3000
+# NEXT_PUBLIC_AUTH_URL=http://localhost:3000
 NEXT_PUBLIC_LOGOUT_URL=https://blockscoutcom.us.auth0.com/v2/logout

configs/envs/.env.main

@@ -13,6 +13,7 @@ NEXT_PUBLIC_NETWORK_ID=5
 NEXT_PUBLIC_NETWORK_CURRENCY_NAME=Ether
 NEXT_PUBLIC_NETWORK_CURRENCY_SYMBOL=ETH
 NEXT_PUBLIC_NETWORK_CURRENCY_DECIMALS=18
+NEXT_PUBLIC_NETWORK_GOVERNANCE_TOKEN_SYMBOL=ETH
 NEXT_PUBLIC_NETWORK_RPC_URL=https://rpc.ankr.com/eth_goerli
 NEXT_PUBLIC_NETWORK_VERIFICATION_TYPE=validation
 NEXT_PUBLIC_IS_TESTNET=true
@@ -37,7 +38,7 @@ NEXT_PUBLIC_APP_INSTANCE=local
 NEXT_PUBLIC_APP_ENV=development
 NEXT_PUBLIC_GRAPHIQL_TRANSACTION=0xf7d4972356e6ae44ae948d0cf19ef2beaf0e574c180997e969a2837da15e349d
 NEXT_PUBLIC_IS_ACCOUNT_SUPPORTED=true
-NEXT_PUBLIC_AUTH_URL=http://localhost:3000
+# NEXT_PUBLIC_AUTH_URL=http://localhost:3000
 NEXT_PUBLIC_LOGOUT_URL=https://blockscoutcom.us.auth0.com/v2/logout
 NEXT_PUBLIC_MARKETPLACE_CONFIG_URL=https://raw.githubusercontent.com/blockscout/frontend-configs/dev/configs/marketplace/eth-goerli.json
 NEXT_PUBLIC_MARKETPLACE_SUBMIT_FORM=https://airtable.com/shrqUAcjgGJ4jU88C

configs/envs/.env.main.L2

@@ -39,7 +39,6 @@ NEXT_PUBLIC_GRAPHIQL_TRANSACTION=0x4a0ed8ddf751a7cb5297f827699117b0f6d21a0b29075
 NEXT_PUBLIC_WEB3_WALLETS=['coinbase']
 NEXT_PUBLIC_WEB3_DISABLE_ADD_TOKEN_TO_WALLET=true
 NEXT_PUBLIC_IS_ACCOUNT_SUPPORTED=true
-NEXT_PUBLIC_AUTH_URL=http://localhost:3000
 NEXT_PUBLIC_LOGOUT_URL=https://blockscoutcom.us.auth0.com/v2/logout
 NEXT_PUBLIC_MARKETPLACE_CONFIG_URL=https://raw.githubusercontent.com/blockscout/frontend-configs/dev/configs/marketplace/base-goerli.json
 NEXT_PUBLIC_MARKETPLACE_SUBMIT_FORM=https://airtable.com/shrqUAcjgGJ4jU88C

configs/envs/.env.poa_core

@@ -35,7 +35,7 @@ NEXT_PUBLIC_NETWORK_EXPLORERS=[{'title':'Anyblock','baseUrl':'https://explorer.a
 NEXT_PUBLIC_APP_INSTANCE=local
 NEXT_PUBLIC_APP_ENV=development
 NEXT_PUBLIC_IS_ACCOUNT_SUPPORTED=true
-NEXT_PUBLIC_AUTH_URL=http://localhost:3000
+# NEXT_PUBLIC_AUTH_URL=http://localhost:3000
 NEXT_PUBLIC_LOGOUT_URL=https://blockscoutcom.us.auth0.com/v2/logout
 NEXT_PUBLIC_NETWORK_VERIFICATION_TYPE=validation
 NEXT_PUBLIC_NETWORK_RPC_URL=https://core.poa.network

deploy/scripts/download_assets.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+echo
+echo "⬇️  Downloading external assets..."
+
+# Check if the number of arguments provided is correct
+if [ "$#" -ne 1 ]; then
+  echo "🛑 Error: incorrect amount of arguments. Usage: $0 <ASSETS_DIR>."
+  exit 1
+fi
+
+# Define the directory to save the downloaded assets
+ASSETS_DIR="$1"
+
+# Define a list of environment variables containing URLs of external assets
+ASSETS_ENVS=(
+    "NEXT_PUBLIC_MARKETPLACE_CONFIG_URL"
+    "NEXT_PUBLIC_FEATURED_NETWORKS"
+    "NEXT_PUBLIC_FOOTER_LINKS"
+    "NEXT_PUBLIC_NETWORK_LOGO"
+    "NEXT_PUBLIC_NETWORK_LOGO_DARK"
+    "NEXT_PUBLIC_NETWORK_ICON"
+    "NEXT_PUBLIC_NETWORK_ICON_DARK"
+)
+
+# Create the assets directory if it doesn't exist
+mkdir -p "$ASSETS_DIR"
+
+# Function to determine the target file name based on the environment variable
+get_target_filename() {
+    local env_var="$1"
+    local url="${!env_var}"
+
+    # Extract the middle part of the variable name (between "NEXT_PUBLIC_" and "_URL") in lowercase
+    local name_prefix="${env_var#NEXT_PUBLIC_}"
+    local name_suffix="${name_prefix%_URL}"
+    local name_lc="$(echo "$name_suffix" | tr '[:upper:]' '[:lower:]')"
+
+    # Extract the extension from the URL
+    local extension="${url##*.}"
+
+    # Construct the custom file name
+    echo "$name_lc.$extension"
+}
+
+# Function to download and save an asset
+download_and_save_asset() {
+    local env_var="$1"
+    local url="$2"
+    local filename="$3"
+    local destination="$ASSETS_DIR/$filename"
+
+    # Check if the environment variable is set
+    if [ -z "${!env_var}" ]; then
+        echo "   [.] Environment variable $env_var is not set. Skipping download."
+        return 1
+    fi
+
+    # Download the asset using curl
+    curl -s -o "$destination" "$url"
+
+    # Check if the download was successful
+    if [ $? -eq 0 ]; then
+        echo "   [+] Downloaded $env_var to $destination successfully."
+        return 0
+    else
+        echo "   [-] Failed to download $env_var from $url."
+        return 1
+    fi
+}
+
+# Iterate through the list and download assets
+for env_var in "${ASSETS_ENVS[@]}"; do
+    url="${!env_var}"
+    filename=$(get_target_filename "$env_var")
+    download_and_save_asset "$env_var" "$url" "$filename"
+done
+
+echo "✅ Done."
+echo