Skip to content

Commit

Permalink
Use ipfs/go-test for general test logic (#2633)
Browse files Browse the repository at this point in the history
This replaces generic test code found in various locations in ipni.
  • Loading branch information
gammazero authored Jun 28, 2024
1 parent 31a4a62 commit d1ae68e
Show file tree
Hide file tree
Showing 16 changed files with 188 additions and 542 deletions.
180 changes: 30 additions & 150 deletions assigner/server/server_test.go
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
package server_test

import (
"bufio"
"bytes"
"context"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"

"github.com/ipfs/go-cid"
testcmd "github.com/ipfs/go-test/cmd"
"github.com/ipfs/go-test/random"
client "github.com/ipni/go-libipni/ingest/client"
"github.com/ipni/go-libipni/test"
adminclient "github.com/ipni/storetheindex/admin/client"
"github.com/ipni/storetheindex/assigner/config"
"github.com/ipni/storetheindex/assigner/core"
Expand All @@ -36,7 +33,10 @@ var pubIdent2 = sticfg.Identity{
PrivKey: "CAESQLtIPpQ0cFqLyK9Wnkd0J8lkslrd/g3KJSZOog7MLLt31PlBaXUpIIU5WaTuEJPioGK3+jEbDzFxDNrkQX5xKTg=",
}

const pubsubTopic = "/indexer/ingest/mainnet"
const (
indexerReadyMatch = "Indexer is ready"
pubsubTopic = "/indexer/ingest/mainnet"
)

func setupServer(t *testing.T, assigner *core.Assigner) *server.Server {
s, err := server.New("127.0.0.1:0", assigner)
Expand All @@ -57,80 +57,38 @@ func TestAssignOnAnnounce(t *testing.T) {
}
t.Parallel()

ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
e := &e2eTestRunner{
t: t,
dir: t.TempDir(),
ctx: ctx,

indexerReady: make(chan struct{}, 1),
}

// Use a clean environment, with the host's PATH, and a temporary HOME.
// We also tell "go install" to place binaries there.
hostEnv := os.Environ()
var filteredEnv []string
for _, env := range hostEnv {
if strings.Contains(env, "CC") || strings.Contains(env, "LDFLAGS") || strings.Contains(env, "CFLAGS") {
// Bring in the C compiler flags from the host. For example on a Nix
// machine, this compilation within the test will fail since the compiler
// will not find correct libraries.
filteredEnv = append(filteredEnv, env)
} else if strings.HasPrefix(env, "PATH") {
// Bring in the host's PATH.
filteredEnv = append(filteredEnv, env)
}
}
e.env = append(filteredEnv, []string{
"HOME=" + e.dir,
"GOBIN=" + e.dir,
}...)
if runtime.GOOS == "windows" {
const gopath = "C:\\Projects\\Go"
err := os.MkdirAll(gopath, 0666)
require.NoError(t, err)
e.env = append(e.env, fmt.Sprintf("GOPATH=%s", gopath))
}
t.Logf("Env: %s", strings.Join(e.env, " "))

// Reuse the host's build and module download cache.
// This should allow "go install" to reuse work.
for _, name := range []string{"GOCACHE", "GOMODCACHE"} {
out, err := exec.Command("go", "env", name).CombinedOutput()
require.NoError(t, err)
out = bytes.TrimSpace(out)
e.env = append(e.env, fmt.Sprintf("%s=%s", name, out))
}
rnr := testcmd.NewRunner(t, t.TempDir())

cwd, err := os.Getwd()
require.NoError(t, err)

err = os.Chdir(filepath.Dir(filepath.Dir(cwd)))
require.NoError(t, err)
e.run("go", "install", ".")

ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()

rnr.Run(ctx, "go", "install", ".")

err = os.Chdir(cwd)
require.NoError(t, err)

indexer := filepath.Join(e.dir, "storetheindex")
e.run(indexer, "init", "--pubsub-topic", pubsubTopic, "--no-bootstrap", "--use-assigner",
indexer := filepath.Join(rnr.Dir, "storetheindex")
rnr.Run(ctx, indexer, "init", "--pubsub-topic", pubsubTopic, "--no-bootstrap", "--use-assigner",
"--listen-admin=/ip4/127.0.0.1/tcp/3602",
"--listen-finder=/ip4/127.0.0.1/tcp/3600",
"--listen-ingest=/ip4/127.0.0.1/tcp/3601",
"--listen-p2p=/ip4/127.0.0.1/tcp/3603",
)
stiCfg, err := sticfg.Load(filepath.Join(e.dir, ".storetheindex", "config"))
stiCfg, err := sticfg.Load(filepath.Join(rnr.Dir, ".storetheindex", "config"))
require.NoError(t, err)
indexerID := stiCfg.Identity.PeerID
t.Log("Initialized indexer", indexerID)

cmdIndexer := e.start(indexer, "daemon")
select {
case <-e.indexerReady:
case <-ctx.Done():
t.Fatal("timed out waiting for indexer to start")
}
indexerReady := testcmd.NewStdoutWatcher(indexerReadyMatch)
cmdIndexer := rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady)
err = indexerReady.Wait(ctx)
require.NoError(t, err, "timed out waiting for indexer to start")

// Assign a peer, to test that assigner reads this at startup.
err = assign(ctx, "localhost:3602", pubIdent2.PeerID)
Expand All @@ -156,7 +114,7 @@ func TestAssignOnAnnounce(t *testing.T) {
require.NoError(t, err)
ai.ID = peerID

mhs := test.RandomMultihashes(1)
mhs := random.Multihashes(1)

assignChan, cancel := assigner.OnAssignment(peerID)
defer cancel()
Expand All @@ -173,29 +131,27 @@ func TestAssignOnAnnounce(t *testing.T) {
}

// Check assignment
outProvider := e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
outProvider := rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
expect := peerID.String()
require.Contains(t, string(outProvider), expect)

e.stop(cmdIndexer, 5*time.Second)
rnr.Stop(cmdIndexer, 5*time.Second)

// Start index again to check that assignment was persisted.
cmdIndexer = e.start(indexer, "daemon")
select {
case <-e.indexerReady:
case <-ctx.Done():
t.Fatal("timed out waiting for indexer to start")
}
outProvider = e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
cmdIndexer = rnr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady)
err = indexerReady.Wait(ctx)
require.NoError(t, err, "timed out waiting for indexer to start")

outProvider = rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
require.Contains(t, string(outProvider), expect)

outProvider = e.run(indexer, "admin", "unassign", "--indexer", cfg.IndexerPool[0].AdminURL, "-p", peerID.String())
outProvider = rnr.Run(ctx, indexer, "admin", "unassign", "--indexer", cfg.IndexerPool[0].AdminURL, "-p", peerID.String())
require.Contains(t, string(outProvider), expect)

outProvider = e.run(indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
outProvider = rnr.Run(ctx, indexer, "admin", "list-assigned", "--indexer", cfg.IndexerPool[0].AdminURL)
require.NotContains(t, string(outProvider), expect)

e.stop(cmdIndexer, 5*time.Second)
rnr.Stop(cmdIndexer, 5*time.Second)
}

// initAssigner initializes a new registry
Expand All @@ -220,82 +176,6 @@ func initAssigner(t *testing.T, trustedID string) (*core.Assigner, config.Assign
return assigner, cfg
}

type e2eTestRunner struct {
t *testing.T
dir string
ctx context.Context
env []string

indexerReady chan struct{}
}

func (e *e2eTestRunner) run(name string, args ...string) []byte {
e.t.Helper()

e.t.Logf("run: %s %s", name, strings.Join(args, " "))

cmd := exec.CommandContext(e.ctx, name, args...)
cmd.Env = e.env
out, err := cmd.CombinedOutput()
require.NoError(e.t, err, "err: %v, output: %s", err, out)
return out
}

func (e *e2eTestRunner) start(prog string, args ...string) *exec.Cmd {
e.t.Helper()

name := filepath.Base(prog)
e.t.Logf("run: %s %s", name, strings.Join(args, " "))

cmd := exec.CommandContext(e.ctx, prog, args...)
cmd.Env = e.env

stdout, err := cmd.StdoutPipe()
require.NoError(e.t, err)
cmd.Stderr = cmd.Stdout

scanner := bufio.NewScanner(stdout)

go func() {
for scanner.Scan() {
line := scanner.Text()

// Logging every single line via the test output is verbose,
// but helps see what's happening, especially when the test fails.
e.t.Logf("%s: %s", name, line)
if strings.Contains(line, "Indexer is ready") {
e.indexerReady <- struct{}{}
}
}
}()

err = cmd.Start()
require.NoError(e.t, err)
return cmd
}

func (e *e2eTestRunner) stop(cmd *exec.Cmd, timeout time.Duration) {
sig := os.Interrupt
if runtime.GOOS == "windows" {
// Windows can't send SIGINT.
sig = os.Kill
}
err := cmd.Process.Signal(sig)
require.NoError(e.t, err)

waitErr := make(chan error, 1)
go func() { waitErr <- cmd.Wait() }()

select {
case <-time.After(timeout):
e.t.Logf("killing command after %s: %s", timeout, cmd)
err := cmd.Process.Kill()
require.NoError(e.t, err)
case err := <-waitErr:
require.NoError(e.t, err)
}
}

func assign(ctx context.Context, indexer, peerIDStr string) error {
cl, err := adminclient.New(indexer)
if err != nil {
Expand Down
13 changes: 4 additions & 9 deletions carstore/carwriter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,17 @@ import (
"testing"

"github.com/ipfs/go-datastore"
"github.com/ipfs/go-test/random"
car "github.com/ipld/go-car/v2"
carblockstore "github.com/ipld/go-car/v2/blockstore"
carindex "github.com/ipld/go-car/v2/index"
"github.com/ipld/go-ipld-prime"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/ipni/go-libipni/ingest/schema"
"github.com/ipni/go-libipni/test"
"github.com/ipni/storetheindex/carstore"
"github.com/ipni/storetheindex/filestore"
crypto "github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
p2ptest "github.com/libp2p/go-libp2p/core/test"
"github.com/multiformats/go-multicodec"
"github.com/multiformats/go-multihash"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -233,7 +232,7 @@ func newRandomLinkedList(t *testing.T, lsys ipld.LinkSystem, size int) (ipld.Lin
var out []multihash.Multihash
var nextLnk ipld.Link
for i := 0; i < size; i++ {
mhs := test.RandomMultihashes(testEntriesChunkSize)
mhs := random.Multihashes(testEntriesChunkSize)
chunk := &schema.EntryChunk{
Entries: mhs,
Next: nextLnk,
Expand Down Expand Up @@ -271,11 +270,7 @@ func mkProvLinkSystem(ds datastore.Datastore) ipld.LinkSystem {
func storeRandomIndexAndAd(t *testing.T, eChunkCount int, metadata []byte, prevLink ipld.Link, dstore datastore.Datastore) (ipld.Link, *schema.Advertisement, []multihash.Multihash, peer.ID, crypto.PrivKey) {
lsys := mkProvLinkSystem(dstore)

priv, pubKey, err := p2ptest.RandTestKeyPair(crypto.Ed25519, 256)
require.NoError(t, err)

p, err := peer.IDFromPublicKey(pubKey)
require.NoError(t, err)
p, priv, _ := random.Identity()

ctxID := []byte("test-context-id")
if metadata == nil {
Expand All @@ -297,7 +292,7 @@ func storeRandomIndexAndAd(t *testing.T, eChunkCount int, metadata []byte, prevL
adv.Entries, mhs = newRandomLinkedList(t, lsys, eChunkCount)
}

err = adv.Sign(priv)
err := adv.Sign(priv)
require.NoError(t, err)

node, err := adv.ToNode()
Expand Down
10 changes: 5 additions & 5 deletions command/synthetic.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

agg "github.com/filecoin-project/go-dagaggregator-unixfs"
"github.com/ipfs/go-cid"
"github.com/ipni/go-libipni/test"
"github.com/ipfs/go-test/random"
"github.com/urfave/cli/v2"
)

Expand Down Expand Up @@ -131,7 +131,7 @@ func writeCidFile(fileName string, num int) error {
for curr < num {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}
if _, err = w.WriteString(cids[i].String()); err != nil {
Expand Down Expand Up @@ -171,7 +171,7 @@ func writeCidFileOfSize(fileName string, size int) error {
for curr < size {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}
c := cids[i]
Expand Down Expand Up @@ -212,7 +212,7 @@ func writeManifest(fileName string, num int) error {
for curr < num {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}

Expand Down Expand Up @@ -257,7 +257,7 @@ func writeManifestOfSize(fileName string, size int) error {
for curr < size {
if i == len(cids) {
// Refil cids
cids = test.RandomCids(100)
cids = random.Cids(100)
i = 0
}
c := cids[i]
Expand Down
1 change: 0 additions & 1 deletion config/identity.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ type Identity struct {
}

func (i Identity) Decode() (peer.ID, ic.PrivKey, error) {

privKey, err := i.DecodePrivateKey("")
if err != nil {
return "", nil, fmt.Errorf("could not decode private key: %w", err)
Expand Down
Loading

0 comments on commit d1ae68e

Please sign in to comment.