Skip to content

Commit

Permalink
Feat add addrbook (#369)
Browse files Browse the repository at this point in the history
* feat: adds download-addrbook.sh script

* feat: adds addrbook download

* fix(CosmosFullNode): updates CRD with addrbook
  • Loading branch information
AntiTyping authored Oct 16, 2023
1 parent 0bdbc03 commit e00e01f
Show file tree
Hide file tree
Showing 8 changed files with 248 additions and 12 deletions.
21 changes: 21 additions & 0 deletions api/v1/cosmosfullnode_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,27 @@ type ChainSpec struct {
// +optional
LogFormat *string `json:"logFormat"`

// URL to address book file to download from the internet.
// The operator detects and properly handles the following file extensions:
// .json, .json.gz, .tar, .tar.gz, .tar.gzip, .zip
// Use AddrbookScript if the chain has an unconventional file format or address book location.
// +optional
AddrbookURL *string `json:"addrbookURL"`

// Specify shell (sh) script commands to properly download and save the address book file.
// Prefer AddrbookURL if the file is in a conventional format.
// The available shell commands are from docker image ghcr.io/strangelove-ventures/infra-toolkit, including wget and curl.
// Save the file to env var $ADDRBOOK_FILE.
// E.g. curl https://url-to-addrbook.com > $ADDRBOOK_FILE
// Takes precedence over AddrbookURL.
// Hint: Use "set -eux" in your script.
// Available env vars:
// $HOME: The home directory.
// $ADDRBOOK_FILE: The location of the final address book file.
// $CONFIG_DIR: The location of the config dir that houses the address book file. Used for extracting from archives. The archive must have a single file called "addrbook.json".
// +optional
AddrbookScript *string `json:"addrbookScript"`

// URL to genesis file to download from the internet.
// Although this field is optional, you will almost always want to set it.
// If not set, uses the genesis file created from the init subcommand. (This behavior may be desirable for new chains or testing.)
Expand Down
10 changes: 10 additions & 0 deletions api/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 21 additions & 0 deletions config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,27 @@ spec:
chain:
description: Blockchain-specific configuration.
properties:
addrbookScript:
description: 'Specify shell (sh) script commands to properly download
and save the address book file. Prefer AddrbookURL if the file
is in a conventional format. The available shell commands are
from docker image ghcr.io/strangelove-ventures/infra-toolkit,
including wget and curl. Save the file to env var $ADDRBOOK_FILE.
E.g. curl https://url-to-addrbook.com > $ADDRBOOK_FILE Takes
precedence over AddrbookURL. Hint: Use "set -eux" in your script.
Available env vars: $HOME: The home directory. $ADDRBOOK_FILE:
The location of the final address book file. $CONFIG_DIR: The
location of the config dir that houses the address book file.
Used for extracting from archives. The archive must have a single
file called "addrbook.json".'
type: string
addrbookURL:
description: 'URL to address book file to download from the internet.
The operator detects and properly handles the following file
extensions: .json, .json.gz, .tar, .tar.gz, .tar.gzip, .zip
Use AddrbookScript if the chain has an unconventional file format
or address book location.'
type: string
app:
description: App configuration applied to app.toml.
properties:
Expand Down
40 changes: 40 additions & 0 deletions internal/fullnode/addrbook.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package fullnode

import (
_ "embed"
"fmt"

cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
)

var (
//go:embed script/download-addrbook.sh
scriptDownloadAddrbook string
)

const addrbookScriptWrapper = `ls $CONFIG_DIR/addrbook.json 1> /dev/null 2>&1
ADDRBOOK_EXISTS=$?
if [ $ADDRBOOK_EXISTS -eq 0 ]; then
echo "Address book already exists"
exit 0
fi
ls -l $CONFIG_DIR/addrbook.json
%s
ls -l $CONFIG_DIR/addrbook.json
echo "Address book $ADDRBOOK_FILE downloaded"
`

// DownloadGenesisCommand returns a proper address book command for use in an init container.
func DownloadAddrbookCommand(cfg cosmosv1.ChainSpec) (string, []string) {
args := []string{"-c"}
switch {
case cfg.AddrbookScript != nil:
args = append(args, fmt.Sprintf(addrbookScriptWrapper, *cfg.AddrbookScript))
case cfg.AddrbookURL != nil:
args = append(args, fmt.Sprintf(addrbookScriptWrapper, scriptDownloadAddrbook), "-s", *cfg.AddrbookURL)
default:
args = append(args, "echo Using default address book")
}
return "sh", args
}
72 changes: 72 additions & 0 deletions internal/fullnode/addrbook_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package fullnode

import (
"testing"

cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
"github.com/stretchr/testify/require"
)

func TestDownloadAddrbookCommand(t *testing.T) {
t.Parallel()

requireValidScript := func(t *testing.T, script string) {
t.Helper()
require.NotEmpty(t, script)
require.Contains(t, script, `if [ $ADDRBOOK_EXISTS -eq 0 ]`)
}

t.Run("default", func(t *testing.T) {
var cfg cosmosv1.ChainSpec

cmd, args := DownloadAddrbookCommand(cfg)
require.Equal(t, "sh", cmd)

require.Len(t, args, 2)

require.Equal(t, "-c", args[0])

got := args[1]
require.NotContains(t, got, "ADDRBOOK_EXISTS")
require.Contains(t, got, "Using default address book")
})

t.Run("download", func(t *testing.T) {
cfg := cosmosv1.ChainSpec{
AddrbookURL: ptr("https://example.com/addrbook.json"),
}
cmd, args := DownloadAddrbookCommand(cfg)
require.Equal(t, "sh", cmd)

require.Len(t, args, 4)

require.Equal(t, "-c", args[0])
got := args[1]
requireValidScript(t, got)
require.Contains(t, got, `ADDRBOOK_URL`)
require.Contains(t, got, "download_json")

require.Equal(t, "-s", args[2])
require.Equal(t, "https://example.com/addrbook.json", args[3])
})

t.Run("custom", func(t *testing.T) {
cfg := cosmosv1.ChainSpec{
// Keeping this to assert that custom script takes precedence.
AddrbookURL: ptr("https://example.com/addrbook.json"),
AddrbookScript: ptr("echo hi"),
}
cmd, args := DownloadAddrbookCommand(cfg)
require.Equal(t, "sh", cmd)

require.Len(t, args, 2)

require.Equal(t, "-c", args[0])

got := args[1]
requireValidScript(t, got)

require.NotContains(t, got, "ADDRBOOK_URL")
require.Contains(t, got, "echo hi")
})
}
12 changes: 11 additions & 1 deletion internal/fullnode/pod_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ func envVars(crd *cosmosv1.CosmosFullNode) []corev1.EnvVar {
{Name: "HOME", Value: workDir},
{Name: "CHAIN_HOME", Value: home},
{Name: "GENESIS_FILE", Value: path.Join(home, "config", "genesis.json")},
{Name: "ADDRBOOK_FILE", Value: path.Join(home, "config", "addrbook.json")},
{Name: "CONFIG_DIR", Value: path.Join(home, "config")},
{Name: "DATA_DIR", Value: path.Join(home, "data")},
}
Expand All @@ -287,6 +288,7 @@ func initContainers(crd *cosmosv1.CosmosFullNode, moniker string) []corev1.Conta
tpl := crd.Spec.PodTemplate
binary := crd.Spec.ChainSpec.Binary
genesisCmd, genesisArgs := DownloadGenesisCommand(crd.Spec.ChainSpec)
addrbookCmd, addrbookArgs := DownloadAddrbookCommand(crd.Spec.ChainSpec)
env := envVars(crd)

initCmd := fmt.Sprintf("%s init %s --chain-id %s", binary, moniker, crd.Spec.ChainSpec.ChainID)
Expand Down Expand Up @@ -332,7 +334,15 @@ echo "Initializing into tmp dir for downstream processing..."
ImagePullPolicy: tpl.ImagePullPolicy,
WorkingDir: workDir,
},

{
Name: "addrbook-init",
Image: infraToolImage,
Command: []string{addrbookCmd},
Args: addrbookArgs,
Env: env,
ImagePullPolicy: tpl.ImagePullPolicy,
WorkingDir: workDir,
},
{
Name: "config-merge",
Image: infraToolImage,
Expand Down
31 changes: 20 additions & 11 deletions internal/fullnode/pod_builder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,12 @@ func TestPodBuilder(t *testing.T) {
require.Equal(t, startContainer.Env[1].Value, "/home/operator/cosmos")
require.Equal(t, startContainer.Env[2].Name, "GENESIS_FILE")
require.Equal(t, startContainer.Env[2].Value, "/home/operator/cosmos/config/genesis.json")
require.Equal(t, startContainer.Env[3].Name, "CONFIG_DIR")
require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config")
require.Equal(t, startContainer.Env[4].Name, "DATA_DIR")
require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/data")
require.Equal(t, startContainer.Env[3].Name, "ADDRBOOK_FILE")
require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config/addrbook.json")
require.Equal(t, startContainer.Env[4].Name, "CONFIG_DIR")
require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/config")
require.Equal(t, startContainer.Env[5].Name, "DATA_DIR")
require.Equal(t, startContainer.Env[5].Value, "/home/operator/cosmos/data")
require.Equal(t, envVars(&crd), startContainer.Env)

healthContainer := pod.Spec.Containers[1]
Expand All @@ -242,14 +244,15 @@ func TestPodBuilder(t *testing.T) {
}
require.Equal(t, healthPort, healthContainer.Ports[0])

require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 5)
require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 6)

wantInitImages := []string{
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"main-image:v1.2.3",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
}
require.Equal(t, wantInitImages, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string {
return c.Image
Expand All @@ -267,7 +270,11 @@ func TestPodBuilder(t *testing.T) {
require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$CHAIN_HOME"`)
require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$HOME/.tmp"`)

mergeConfig := pod.Spec.InitContainers[3]
mergeConfig1 := pod.Spec.InitContainers[3]
// The order of config-merge arguments is important. Rightmost takes precedence.
require.Contains(t, mergeConfig1.Args[1], `echo Using default address book`)

mergeConfig := pod.Spec.InitContainers[4]
// The order of config-merge arguments is important. Rightmost takes precedence.
require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/config.toml" "$OVERLAY_DIR/config-overlay.toml" > "$CONFIG_DIR/config.toml"`)
require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/app.toml" "$OVERLAY_DIR/app-overlay.toml" > "$CONFIG_DIR/app.toml`)
Expand All @@ -293,10 +300,12 @@ func TestPodBuilder(t *testing.T) {
require.Equal(t, container.Env[1].Value, "/home/operator/.osmosisd")
require.Equal(t, container.Env[2].Name, "GENESIS_FILE")
require.Equal(t, container.Env[2].Value, "/home/operator/.osmosisd/config/genesis.json")
require.Equal(t, container.Env[3].Name, "CONFIG_DIR")
require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config")
require.Equal(t, container.Env[4].Name, "DATA_DIR")
require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/data")
require.Equal(t, container.Env[3].Name, "ADDRBOOK_FILE")
require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config/addrbook.json")
require.Equal(t, container.Env[4].Name, "CONFIG_DIR")
require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/config")
require.Equal(t, container.Env[5].Name, "DATA_DIR")
require.Equal(t, container.Env[5].Value, "/home/operator/.osmosisd/data")

require.NotEmpty(t, pod.Spec.InitContainers)

Expand Down Expand Up @@ -554,7 +563,7 @@ gaiad start --home /home/operator/cosmos`
require.Equal(t, "/foo", extraVol[0].MountPath)

initConts := lo.SliceToMap(pod.Spec.InitContainers, func(c corev1.Container) (string, corev1.Container) { return c.Name, c })
require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "config-merge"}, lo.Keys(initConts))
require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "addrbook-init", "config-merge"}, lo.Keys(initConts))
require.Equal(t, "foo:latest", initConts["chain-init"].Image)
})

Expand Down
53 changes: 53 additions & 0 deletions internal/fullnode/script/download-addrbook.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
set -eu

# $ADDRBOOK_FILE and $CONFIG_DIR already set via pod env vars.

ADDRBOOK_URL="$1"

echo "Downloading address book file $ADDRBOOK_URL to $ADDRBOOK_FILE..."

download_json() {
echo "Downloading plain json..."
wget -c -O "$ADDRBOOK_FILE" "$ADDRBOOK_URL"
}

download_jsongz() {
echo "Downloading json.gz..."
wget -c -O - "$ADDRBOOK_URL" | gunzip -c >"$ADDRBOOK_FILE"
}

download_tar() {
echo "Downloading and extracting tar..."
wget -c -O - "$ADDRBOOK_URL" | tar -x -C "$CONFIG_DIR"
}

download_targz() {
echo "Downloading and extracting compressed tar..."
wget -c -O - "$ADDRBOOK_URL" | tar -xz -C "$CONFIG_DIR"
}

download_zip() {
echo "Downloading and extracting zip..."
wget -c -O tmp_genesis.zip "$ADDRBOOK_URL"
unzip tmp_genesis.zip
rm tmp_genesis.zip
mv genesis.json "$ADDRBOOK_FILE"
}

rm -f "$ADDRBOOK_FILE"

case "$ADDRBOOK_URL" in
*.json.gz) download_jsongz ;;
*.json) download_json ;;
*.tar.gz) download_targz ;;
*.tar.gzip) download_targz ;;
*.tar) download_tar ;;
*.zip) download_zip ;;
*)
echo "Unable to handle file extension for $ADDRBOOK_URL"
exit 1
;;
esac

echo "Saved address book file to $ADDRBOOK_FILE."
echo "Download address book file complete."

0 comments on commit e00e01f

Please sign in to comment.