Skip to content

Commit

Permalink
feat: added configuration to use a local token registry
Browse files Browse the repository at this point in the history
  • Loading branch information
Kammerlo committed May 25, 2024
1 parent caa2316 commit a9f7cde
Show file tree
Hide file tree
Showing 7 changed files with 260 additions and 11 deletions.
4 changes: 4 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -113,3 +113,7 @@ COPY config/network/${NETWORK}/cardano-node /config/cardano-node/
WORKDIR /app/packages/server/dist
EXPOSE 3100
CMD ["node", "index.js"]

FROM cardanofoundation/cf-token-metadata-registry-api:latest as token-registry
ADD scripts/token-registry-init.sh /app/entrypoint.sh
ENTRYPOINT sh /app/entrypoint.sh
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,16 @@ docker compose -p preview down

</details>

### Use local Token Metadata Registry
The public Token metadata registry has a limit of daily requests, this can lead to long sync times, when resyncing from scratch.
To avoid this we introduced the possibility to use a local token metadata registry.
This will be started within docker-compose and will save all data in an additional schema within the local postgres database.
To start it use the following command:
```
docker compose -f docker-compose-local-registry.yml up -d
```
To build all containers from source use `--build`. All environment variables and extra commands can be used as described above.

### Upgrade Database to Postgres 14
If you are upgrading from Postgres 11 to 14: A resync will be needed.
To speed up the process you can use the following snapshots:
Expand Down
172 changes: 172 additions & 0 deletions docker-compose-local-registry.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
version: "3.8"

services:
postgres:
image: postgres:${POSTGRES_VERSION:-14.10-alpine}
environment:
- POSTGRES_LOGGING=true
- POSTGRES_DB_FILE=/run/secrets/postgres_db
- POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
- POSTGRES_USER_FILE=/run/secrets/postgres_user
ports:
- ${POSTGRES_PORT:-5432}:5432
secrets:
- postgres_db
- postgres_password
- postgres_user
shm_size: '2gb'
volumes:
- postgres14-data:/var/lib/postgresql/data
restart: on-failure
logging:
driver: "json-file"
options:
max-size: "200k"
max-file: "10"

cardano-node-ogmios:
platform: linux/x86_64
image: cardanosolutions/cardano-node-ogmios:${OGMIOS_VERSION:-v6.2.0}_${CARDANO_NODE_VERSION:-8.9.0}-${NETWORK:-mainnet}
logging:
driver: "json-file"
options:
max-size: "400k"
max-file: "20"
ports:
- ${OGMIOS_PORT:-1337}:1337
restart: on-failure
volumes:
- node-db:/db
- node-ipc:/ipc
# Uncomment if you want to use your own config files or the provided ones.
# Current configs are also available at: https://book.world.dev.cardano.org/
# - ./config/network/${NETWORK:-mainnet}:/config

cardano-db-sync:
platform: linux/x86_64
image: ghcr.io/intersectmbo/cardano-db-sync:${CARDANO_DB_SYNC_VERSION:-13.2.0.2}
command: [
"--config", "/config/cardano-db-sync/config.json",
"--socket-path", "/node-ipc/node.socket"
]
environment:
- POSTGRES_HOST=postgres
- POSTGRES_PORT=5432
- RESTORE_SNAPSHOT=${RESTORE_SNAPSHOT:-}
- RESTORE_RECREATE_DB=N
depends_on:
- cardano-node-ogmios
- postgres
secrets:
- postgres_password
- postgres_user
- postgres_db
volumes:
- ./config/network/${NETWORK:-mainnet}:/config
- db-sync-data:/var/lib/cexplorer
- node-ipc:/node-ipc
restart: on-failure
stop_signal: SIGINT
logging:
driver: "json-file"
options:
max-size: "200k"
max-file: "10"

hasura:
build:
context: ./packages/api-cardano-db-hasura/hasura
image: cardanofoundation/cardano-graphql-hasura:${CARDANO_GRAPHQL_VERSION:-8.0.1}
ports:
- ${HASURA_PORT:-8090}:8080
depends_on:
- "postgres"
restart: on-failure
environment:
- HASURA_GRAPHQL_ENABLE_CONSOLE=true
- HASURA_GRAPHQL_CORS_DOMAIN=http://localhost:9695
secrets:
- postgres_db
- postgres_password
- postgres_user
logging:
driver: "json-file"
options:
max-size: "200k"
max-file: "10"

background:
build:
cache_from: [ cardanofoundation/cardano-graphql-background:latest ]
context: .
target: background
image: cardanofoundation/cardano-graphql-background:${CARDANO_GRAPHQL_VERSION:-8.0.1}-${NETWORK:-mainnet}
depends_on:
- "hasura"
- "postgres"
environment:
- LOGGER_MIN_SEVERITY=${LOGGER_MIN_SEVERITY:-info}
- METADATA_SERVER_URI=${METADATA_SERVER_URI:-http://token-metadata-registry:8080}
restart: on-failure
secrets:
- postgres_db
- postgres_password
- postgres_user
logging:
driver: "json-file"
options:
max-size: "200k"
max-file: "10"

server:
platform: linux/x86_64
build:
args:
- NETWORK=${NETWORK:-mainnet}
cache_from: [ inputoutput/cardano-graphql-server:latest ]
context: .
target: server
image: cardanofoundation/cardano-graphql-server:${CARDANO_GRAPHQL_VERSION:-8.0.1}-${NETWORK:-mainnet}
environment:
- ALLOW_INTROSPECTION=true
- CACHE_ENABLED=true
- LOGGER_MIN_SEVERITY=${LOGGER_MIN_SEVERITY:-info}
expose:
- ${API_PORT:-3100}
ports:
- ${API_PORT:-3100}:3100
restart: on-failure
logging:
driver: "json-file"
options:
max-size: "200k"
max-file: "10"

token-metadata-registry:
build:
context: .
target: token-registry
ports:
- ${TOKEN_REGISTRY_PORT:-8080}:8080
environment:
- TOKEN_METADATA_SYNC_JOB=true
- POSTGRES_PORT=${POSTGRES_PORT:-5432}
- POSTGRES_HOST=postgres
- DB_SCHEMA=tokenregistry
secrets:
- postgres_db
- postgres_password
- postgres_user

secrets:
postgres_db:
file: ./placeholder-secrets/postgres_db
postgres_password:
file: ./placeholder-secrets/postgres_password
postgres_user:
file: ./placeholder-secrets/postgres_user
volumes:
db-sync-data:
node-db:
node-ipc:
postgres14-data:
11 changes: 11 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,17 @@ services:
max-size: "200k"
max-file: "10"

token-metadata-registry:
image: cardanofoundation/cf-token-metadata-registry-api:${TOKEN_REGISTRY_TAG:-latest}
ports:
- ${TOKEN_REGISTRY_PORT:-8080}:8080
environment:
- TOKEN_METADATA_SYNC_JOB=true
- DB_USERNAME=postgres
- DB_PASSWORD=doNoUseThisSecret!
- DB_NAME=cexplorer
- DB_URL=jdbc:postgresql://postgres:${POSTGRES_PORT:-5432}/${DB_NAME:-cexplorer}?currentSchema=token

secrets:
postgres_db:
file: ./placeholder-secrets/postgres_db
Expand Down
58 changes: 48 additions & 10 deletions packages/api-cardano-db-hasura/src/MetadataClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { errors, RunnableModuleState } from '@cardano-graphql/util'
import { dummyLogger, Logger } from 'ts-log'
import { AssetMetadata } from './AssetMetadata'
import { Asset } from './graphql_types'
import pRetry from 'p-retry'

const MODULE_NAME = 'MetadataFetchClient'

Expand All @@ -20,16 +21,51 @@ export class MetadataClient {
})
}

private async ensureMetadataServerIsAvailable (): Promise<void> {
try {
await this.axiosClient.get('/metadata/healthcheck')
} catch (error) {
if (error.code === 'ENOTFOUND') {
throw new errors.HostDoesNotExist('metadata server')
} else if (error.response?.status !== 404) {
throw error
private async ensureLocalMetadataServerIsAvailable (): Promise<void> {
await pRetry(
async () => {
try {
await this.axiosClient.get('/health')
} catch (error) {
if (error.code === 'ENOTFOUND') {
this.logger.info('Waiting for TokenRegistry to be available')
throw new errors.HostDoesNotExist('metadata server')
} else if (error.response?.status === 400) { // Needed until TokenRegistry is updated
this.logger.info('Token Registry is up')
} else if (error.response?.status !== 404) {
this.logger.info('Metadata Server unreachable.')
throw error
}
}
}, {
factor: 1.5,
retries: 10
}
}
)
}

private async waitForLocalMetadataServerSynced (): Promise<void> {
await pRetry(
async () => {
try {
const result = await this.axiosClient.get('/health')
if (!result.data.synced) {
this.logger.info('Metadata registry is still syncing. This can take up to 15min...')
throw new Error('')
}
} catch (error) {
if (error.response?.status === 400) {
this.logger.info('external Registry is up and running') // Needed until TokenRegistry is updated
} else {
throw new Error('')
}
}
}, {
factor: 1.5,
retries: 1000,
minTimeout: 60000 // first try after one minute
}
)
}

public async fetch (assetIds: Asset['assetId'][]): Promise<AssetMetadata[]> {
Expand Down Expand Up @@ -62,7 +98,9 @@ export class MetadataClient {
if (this.state !== null) return
this.state = 'initializing'
this.logger.info({ module: MODULE_NAME }, 'Initializing')
await this.ensureMetadataServerIsAvailable()
await this.ensureLocalMetadataServerIsAvailable()
this.logger.info({ module: MODULE_NAME }, 'Metadata Server is up and running. Checking Sync Status.')
await this.waitForLocalMetadataServerSynced()
this.state = 'initialized'
this.logger.info({ module: MODULE_NAME }, 'Initialized')
}
Expand Down
8 changes: 7 additions & 1 deletion scripts/export_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ case "$NETWORK" in
mainnet)
API_PORT=3100
HASURA_PORT=8090
METADATA_SERVER_URI="https://tokens.cardano.org"
METADATA_SERVER_URI="http://localhost:8080"
OGMIOS_PORT=1337
PG_ADMIN_PORT=8442
POSTGRES_PORT=5432
TOKEN_REGISTRY_PORT=8080
;;
testnet)
API_PORT=3101
Expand All @@ -23,6 +24,7 @@ case "$NETWORK" in
OGMIOS_PORT=1338
PG_ADMIN_PORT=8443
POSTGRES_PORT=5443
TOKEN_REGISTRY_PORT=8081
export CARDANO_NODE_VERSION=8.7.3
;;
preprod)
Expand All @@ -32,6 +34,7 @@ case "$NETWORK" in
OGMIOS_PORT=1339
PG_ADMIN_PORT=8444
POSTGRES_PORT=5444
TOKEN_REGISTRY_PORT=8082
;;
preview)
API_PORT=3103
Expand All @@ -40,6 +43,7 @@ case "$NETWORK" in
OGMIOS_PORT=1340
PG_ADMIN_PORT=8445
POSTGRES_PORT=5445
TOKEN_REGISTRY_PORT=8083
;;
vasil-dev)
API_PORT=3104
Expand All @@ -48,6 +52,7 @@ case "$NETWORK" in
OGMIOS_PORT=1341
PG_ADMIN_PORT=8446
POSTGRES_PORT=5446
TOKEN_REGISTRY_PORT=8084
;;
esac

Expand All @@ -70,3 +75,4 @@ export PG_ADMIN_PORT
export POSTGRES_PORT
export POSTGRES_USER_FILE=${SECRETS_DIR}/postgres_user
export POSTGRES_HOST=localhost
export TOKEN_REGISTRY_TAG=latest
8 changes: 8 additions & 0 deletions scripts/token-registry-init.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash

export DB_USERNAME=$(cat /run/secrets/postgres_user)
export DB_PASSWORD=$(cat /run/secrets/postgres_password)
export DB_NAME=$(cat /run/secrets/postgres_db)
export DB_URL=jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${DB_NAME}?currentSchema=${DB_SCHEMA}

java -jar /app/app.jar

0 comments on commit a9f7cde

Please sign in to comment.