Skip to content

Commit

Permalink
feat: adds addrbook download
Browse files Browse the repository at this point in the history
  • Loading branch information
AntiTyping committed Oct 16, 2023
1 parent 103f49d commit a4120ab
Show file tree
Hide file tree
Showing 7 changed files with 195 additions and 12 deletions.
21 changes: 21 additions & 0 deletions api/v1/cosmosfullnode_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,27 @@ type ChainSpec struct {
// +optional
LogFormat *string `json:"logFormat"`

// URL to address book file to download from the internet.
// The operator detects and properly handles the following file extensions:
// .json, .json.gz, .tar, .tar.gz, .tar.gzip, .zip
// Use AddrbookScript if the chain has an unconventional file format or address book location.
// +optional
AddrbookURL *string `json:"addrbookURL"`

// Specify shell (sh) script commands to properly download and save the address book file.
// Prefer AddrbookURL if the file is in a conventional format.
// The available shell commands are from docker image ghcr.io/strangelove-ventures/infra-toolkit, including wget and curl.
// Save the file to env var $ADDRBOOK_FILE.
// E.g. curl https://url-to-addrbook.com > $ADDRBOOK_FILE
// Takes precedence over AddrbookURL.
// Hint: Use "set -eux" in your script.
// Available env vars:
// $HOME: The home directory.
// $ADDRBOOK_FILE: The location of the final address book file.
// $CONFIG_DIR: The location of the config dir that houses the address book file. Used for extracting from archives. The archive must have a single file called "addrbook.json".
// +optional
AddrbookScript *string `json:"addrbookScript"`

// URL to genesis file to download from the internet.
// Although this field is optional, you will almost always want to set it.
// If not set, uses the genesis file created from the init subcommand. (This behavior may be desirable for new chains or testing.)
Expand Down
10 changes: 10 additions & 0 deletions api/v1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 21 additions & 0 deletions config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,27 @@ spec:
chain:
description: Blockchain-specific configuration.
properties:
addrbookScript:
description: 'Specify shell (sh) script commands to properly download
and save the address book file. Prefer AddrbookURL if the file is
in a conventional format. The available shell commands are from
docker image ghcr.io/strangelove-ventures/infra-toolkit, including
wget and curl. Save the file to env var $ADDRBOOK_FILE. E.g.
curl https://url-to-addrbook.com > $GENESIS_FILE
Takes precedence over AddrbookURL. Hint: Use "set -eux" in your
script. Available env vars: $HOME: The home directory. $ADDRBOOK_FILE:
The location of the final address book file. $CONFIG_DIR: The location
of the config dir that houses the address book file. Used for extracting
from archives. The archive must have a single file called "addrbook.json".'
type: string
addrbookURL:
description: 'URL to address book file to download from the internet.
If not set, uses the address book file created from the
init subcommand. The operator detects and properly handles the following
file extensions: .json, .json.gz, .tar, .tar.gz, .tar.gzip,
.zip Use AddrbookScript if the chain has an unconventional file
format or address book location.'
type: string
app:
description: App configuration applied to app.toml.
properties:
Expand Down
40 changes: 40 additions & 0 deletions internal/fullnode/addrbook.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package fullnode

import (
_ "embed"
"fmt"

cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
)

var (
//go:embed script/download-addrbook.sh
scriptDownloadAddrbook string
)

const addrbookScriptWrapper = `ls $CONFIG_DIR/addrbook.json 1> /dev/null 2>&1
ADDRBOOK_EXISTS=$?
if [ $ADDRBOOK_EXISTS -eq 0 ]; then
echo "Address book already exists"
exit 0
fi
ls -l $CONFIG_DIR/addrbook.json
%s
ls -l $CONFIG_DIR/addrbook.json
echo "Address book $ADDRBOOK_FILE downloaded"
`

// DownloadGenesisCommand returns a proper address book command for use in an init container.
func DownloadAddrbookCommand(cfg cosmosv1.ChainSpec) (string, []string) {
args := []string{"-c"}
switch {
case cfg.AddrbookScript != nil:
args = append(args, fmt.Sprintf(addrbookScriptWrapper, *cfg.AddrbookScript))
case cfg.AddrbookURL != nil:
args = append(args, fmt.Sprintf(addrbookScriptWrapper, scriptDownloadAddrbook), "-s", *cfg.AddrbookURL)
default:
args = append(args, "echo Using default address book")
}
return "sh", args
}
72 changes: 72 additions & 0 deletions internal/fullnode/addrbook_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package fullnode

import (
"testing"

cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
"github.com/stretchr/testify/require"
)

func TestDownloadAddrbookCommand(t *testing.T) {
t.Parallel()

requireValidScript := func(t *testing.T, script string) {
t.Helper()
require.NotEmpty(t, script)
require.Contains(t, script, `if [ $ADDRBOOK_EXISTS -eq 0 ]`)
}

t.Run("default", func(t *testing.T) {
var cfg cosmosv1.ChainSpec

cmd, args := DownloadAddrbookCommand(cfg)
require.Equal(t, "sh", cmd)

require.Len(t, args, 2)

require.Equal(t, "-c", args[0])

got := args[1]
require.NotContains(t, got, "ADDRBOOK_EXISTS")
require.Contains(t, got, "Using default address book")
})

t.Run("download", func(t *testing.T) {
cfg := cosmosv1.ChainSpec{
AddrbookURL: ptr("https://example.com/addrbook.json"),
}
cmd, args := DownloadAddrbookCommand(cfg)
require.Equal(t, "sh", cmd)

require.Len(t, args, 4)

require.Equal(t, "-c", args[0])
got := args[1]
requireValidScript(t, got)
require.Contains(t, got, `ADDRBOOK_URL`)
require.Contains(t, got, "download_json")

require.Equal(t, "-s", args[2])
require.Equal(t, "https://example.com/addrbook.json", args[3])
})

t.Run("custom", func(t *testing.T) {
cfg := cosmosv1.ChainSpec{
// Keeping this to assert that custom script takes precedence.
AddrbookURL: ptr("https://example.com/addrbook.json"),
AddrbookScript: ptr("echo hi"),
}
cmd, args := DownloadAddrbookCommand(cfg)
require.Equal(t, "sh", cmd)

require.Len(t, args, 2)

require.Equal(t, "-c", args[0])

got := args[1]
requireValidScript(t, got)

require.NotContains(t, got, "ADDRBOOK_URL")
require.Contains(t, got, "echo hi")
})
}
12 changes: 11 additions & 1 deletion internal/fullnode/pod_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ func envVars(crd *cosmosv1.CosmosFullNode) []corev1.EnvVar {
{Name: "HOME", Value: workDir},
{Name: "CHAIN_HOME", Value: home},
{Name: "GENESIS_FILE", Value: path.Join(home, "config", "genesis.json")},
{Name: "ADDRBOOK_FILE", Value: path.Join(home, "config", "addrbook.json")},
{Name: "CONFIG_DIR", Value: path.Join(home, "config")},
{Name: "DATA_DIR", Value: path.Join(home, "data")},
}
Expand All @@ -287,6 +288,7 @@ func initContainers(crd *cosmosv1.CosmosFullNode, moniker string) []corev1.Conta
tpl := crd.Spec.PodTemplate
binary := crd.Spec.ChainSpec.Binary
genesisCmd, genesisArgs := DownloadGenesisCommand(crd.Spec.ChainSpec)
addrbookCmd, addrbookArgs := DownloadAddrbookCommand(crd.Spec.ChainSpec)
env := envVars(crd)

initCmd := fmt.Sprintf("%s init %s --chain-id %s", binary, moniker, crd.Spec.ChainSpec.ChainID)
Expand Down Expand Up @@ -332,7 +334,15 @@ echo "Initializing into tmp dir for downstream processing..."
ImagePullPolicy: tpl.ImagePullPolicy,
WorkingDir: workDir,
},

{
Name: "addrbook-init",
Image: infraToolImage,
Command: []string{addrbookCmd},
Args: addrbookArgs,
Env: env,
ImagePullPolicy: tpl.ImagePullPolicy,
WorkingDir: workDir,
},
{
Name: "config-merge",
Image: infraToolImage,
Expand Down
31 changes: 20 additions & 11 deletions internal/fullnode/pod_builder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,12 @@ func TestPodBuilder(t *testing.T) {
require.Equal(t, startContainer.Env[1].Value, "/home/operator/cosmos")
require.Equal(t, startContainer.Env[2].Name, "GENESIS_FILE")
require.Equal(t, startContainer.Env[2].Value, "/home/operator/cosmos/config/genesis.json")
require.Equal(t, startContainer.Env[3].Name, "CONFIG_DIR")
require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config")
require.Equal(t, startContainer.Env[4].Name, "DATA_DIR")
require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/data")
require.Equal(t, startContainer.Env[3].Name, "ADDRBOOK_FILE")
require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config/addrbook.json")
require.Equal(t, startContainer.Env[4].Name, "CONFIG_DIR")
require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/config")
require.Equal(t, startContainer.Env[5].Name, "DATA_DIR")
require.Equal(t, startContainer.Env[5].Value, "/home/operator/cosmos/data")
require.Equal(t, envVars(&crd), startContainer.Env)

healthContainer := pod.Spec.Containers[1]
Expand All @@ -242,14 +244,15 @@ func TestPodBuilder(t *testing.T) {
}
require.Equal(t, healthPort, healthContainer.Ports[0])

require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 5)
require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 6)

wantInitImages := []string{
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"main-image:v1.2.3",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
"ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1",
}
require.Equal(t, wantInitImages, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string {
return c.Image
Expand All @@ -267,7 +270,11 @@ func TestPodBuilder(t *testing.T) {
require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$CHAIN_HOME"`)
require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$HOME/.tmp"`)

mergeConfig := pod.Spec.InitContainers[3]
mergeConfig1 := pod.Spec.InitContainers[3]
// The order of config-merge arguments is important. Rightmost takes precedence.
require.Contains(t, mergeConfig1.Args[1], `echo Using default address book`)

mergeConfig := pod.Spec.InitContainers[4]
// The order of config-merge arguments is important. Rightmost takes precedence.
require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/config.toml" "$OVERLAY_DIR/config-overlay.toml" > "$CONFIG_DIR/config.toml"`)
require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/app.toml" "$OVERLAY_DIR/app-overlay.toml" > "$CONFIG_DIR/app.toml`)
Expand All @@ -293,10 +300,12 @@ func TestPodBuilder(t *testing.T) {
require.Equal(t, container.Env[1].Value, "/home/operator/.osmosisd")
require.Equal(t, container.Env[2].Name, "GENESIS_FILE")
require.Equal(t, container.Env[2].Value, "/home/operator/.osmosisd/config/genesis.json")
require.Equal(t, container.Env[3].Name, "CONFIG_DIR")
require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config")
require.Equal(t, container.Env[4].Name, "DATA_DIR")
require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/data")
require.Equal(t, container.Env[3].Name, "ADDRBOOK_FILE")
require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config/addrbook.json")
require.Equal(t, container.Env[4].Name, "CONFIG_DIR")
require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/config")
require.Equal(t, container.Env[5].Name, "DATA_DIR")
require.Equal(t, container.Env[5].Value, "/home/operator/.osmosisd/data")

require.NotEmpty(t, pod.Spec.InitContainers)

Expand Down Expand Up @@ -554,7 +563,7 @@ gaiad start --home /home/operator/cosmos`
require.Equal(t, "/foo", extraVol[0].MountPath)

initConts := lo.SliceToMap(pod.Spec.InitContainers, func(c corev1.Container) (string, corev1.Container) { return c.Name, c })
require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "config-merge"}, lo.Keys(initConts))
require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "addrbook-init", "config-merge"}, lo.Keys(initConts))
require.Equal(t, "foo:latest", initConts["chain-init"].Image)
})

Expand Down

0 comments on commit a4120ab

Please sign in to comment.