feat: seed-based automatic peering (#111)

CHANGELOG.md

@@ -15,6 +15,8 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- ✨ Now supports automatic peering with peers that have the same seed via `--seed-peering` (`RAINBOW_SEED_PEERING`). To enable this, you must configure `--seed` (`RAINBOW_SEED`) and `--seed-index` (`RAINBOW_SEED_INDEX`).
+
 ### Changed
 
 ### Removed

docs/environment-variables.md

@@ -10,6 +10,10 @@
   - [`RAINBOW_GC_THRESHOLD`](#rainbow_gc_threshold)
   - [`RAINBOW_IPNS_MAX_CACHE_TTL`](#rainbow_ipns_max_cache_ttl)
   - [`RAINBOW_PEERING`](#rainbow_peering)
+  - [`RAINBOW_SEED`](#rainbow_seed)
+  - [`RAINBOW_SEED_INDEX`](#rainbow_seed_index)
+  - [`RAINBOW_SEED_PEERING`](#rainbow_seed_peering)
+  - [`RAINBOW_SEED_PEERING_MAX_INDEX`](#rainbow_seed_peering_max_index)
   - [`RAINBOW_PEERING_SHARED_CACHE`](#rainbow_peering_shared_cache)
 - [Logging](#logging)
   - [`GOLOG_LOG_LEVEL`](#golog_log_level)
@@ -96,26 +100,65 @@ Default: No upper bound, [TTL from IPNS Record](https://specs.ipfs.tech/ipns/ipn
 
 A comma-separated list of [multiaddresses](https://docs.libp2p.io/concepts/fundamentals/addressing/) of peers to stay connected to.
 
-
-If `RAINBOW_SEED` is set and `/p2p/rainbow-seed/N` value is found here, Rainbow
-will replace it with a valid `/p2p/` for a peer ID generated from same seed
-and index `N`.
+> [!TIP]
+> If `RAINBOW_SEED` is set and `/p2p/rainbow-seed/N` value is found here, Rainbow
+> will replace it with a valid `/p2p/` for a peer ID generated from same seed
+> and index `N`. This is useful when `RAINBOW_SEED_PEERING` can't be used,
+> or when peer routing should be skipped and specific address should be used.
 
 Default: not set (no peering)
 
+### `RAINBOW_SEED`
+
+Base58 seed to derive PeerID from. Can be generated with `rainbow gen-seed`.
+If set, requires `RAINBOW_SEED_INDEX` to be set as well.
+
+Default: not set
+
+### `RAINBOW_SEED_INDEX`
+
+Index to derivate the PeerID identity from `RAINBOW_SEED`.
+
+Default: not set
+
+### `RAINBOW_SEED_PEERING`
+
+> [!WARNING]
+> Experimental feature.
+
+Automated version of `RAINBOW_PEERING` which does not require providing multiaddrs.
+
+Instead, it will set up peering with peers that share the same seed (requires `RAINBOW_SEED_INDEX` to be set up).
+
+> [!NOTE]
+> Runs a separate light DHT for peer routing with the main host if DHT routing is disabled.
+
+Default: `false` (disabled)
+
+### `RAINBOW_SEED_PEERING_MAX_INDEX`
+
+Informs the largest index to derive for `RAINBOW_SEED_PEERING`.
+If you have more instances than the default, increase it here.
+
+Default: 100
 
 ### `RAINBOW_PEERING_SHARED_CACHE`
 
-Enable sharing of local cache to peers safe-listed with `RAINBOW_PEERING`.
+> [!WARNING]
+> Experimental feature.
+
+Enable sharing of local cache to peers safe-listed with `RAINBOW_PEERING`
+or `RAINBOW_SEED_PEERING`.
 
 Once enabled, Rainbow will respond to [Bitswap](https://docs.ipfs.tech/concepts/bitswap/)
 queries from these safelisted peers, serving locally cached blocks if requested.
 
-The main use case for this feature is scaling and load balancing across a
-fleet of rainbow, or other bitswap-capable IPFS services. Cache sharing allows
-clustered services to check if any of the other instances has a requested CID.
-This saves resources as data cached on other instance can be fetched internally
-(e.g. LAN) rather than externally (WAN, p2p).
+> [!TIP]
+> The main use case for this feature is scaling and load balancing across a
+> fleet of rainbow, or other bitswap-capable IPFS services. Cache sharing allows
+> clustered services to check if any of the other instances has a requested CID.
+> This saves resources as data cached on other instance can be fetched internally
+> (e.g. LAN) rather than externally (WAN, p2p).
 
 Default: `false` (no cache sharing)
 

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/libp2p/go-libp2p-kad-dht v0.25.2
 	github.com/libp2p/go-libp2p-record v0.2.0
 	github.com/libp2p/go-libp2p-routing-helpers v0.7.3
+	github.com/libp2p/go-libp2p-testing v0.12.0
 	github.com/mitchellh/go-server-timing v1.0.1
 	github.com/mr-tron/base58 v1.2.0
 	github.com/multiformats/go-multiaddr v0.12.3

keys.go

@@ -9,6 +9,7 @@ import (
 	"io"
 
 	libp2p "github.com/libp2p/go-libp2p/core/crypto"
+	peer "github.com/libp2p/go-libp2p/core/peer"
 	"github.com/mr-tron/base58"
 	"golang.org/x/crypto/hkdf"
 )
@@ -25,7 +26,7 @@ func newSeed() (string, error) {
 	return base58.Encode(bs), nil
 }
 
-// derive derives libp2p keys from a b58-encoded seed.
+// deriveKey derives libp2p keys from a b58-encoded seed.
 func deriveKey(b58secret string, info []byte) (libp2p.PrivKey, error) {
 	secret, err := base58.Decode(b58secret)
 	if err != nil {
@@ -45,6 +46,32 @@ func deriveKey(b58secret string, info []byte) (libp2p.PrivKey, error) {
 	return libp2p.UnmarshalEd25519PrivateKey(key)
 }
 
+// derivePeerIDs derives the peer IDs of all the peers with the same seed up to
+// maxIndex. Our peer ID (with index 'ourIndex') is not generated.
+func derivePeerIDs(seed string, ourIndex int, maxIndex int) ([]peer.ID, error) {
+	peerIDs := []peer.ID{}
+
+	for i := 0; i <= maxIndex; i++ {
+		if i == ourIndex {
+			continue
+		}
+
+		peerPriv, err := deriveKey(seed, deriveKeyInfo(i))
+		if err != nil {
+			return nil, err
+		}
+
+		pid, err := peer.IDFromPrivateKey(peerPriv)
+		if err != nil {
+			return nil, err
+		}
+
+		peerIDs = append(peerIDs, pid)
+	}
+
+	return peerIDs, nil
+}
+
 func deriveKeyInfo(index int) []byte {
 	return []byte(fmt.Sprintf("rainbow-%d", index))
 }

main.go

@@ -91,6 +91,29 @@ Generate an identity seed and launch a gateway:
 			EnvVars: []string{"RAINBOW_SEED_INDEX"},
 			Usage:   "Index to derivate the peerID (needs --seed)",
 		},
+		&cli.BoolFlag{
+			Name:    "seed-peering",
+			Value:   false,
+			EnvVars: []string{"RAINBOW_SEED_PEERING"},
+			Usage:   "Automatic peering with peers with the same seed (requires --seed and --seed-index). Runs a separate light DHT for peer routing with the main host if --dht-routing or --dht-shared-host are disabled",
+			Action: func(ctx *cli.Context, b bool) error {
+				if !b {
+					return nil
+				}
+
+				if !ctx.IsSet("seed") || !ctx.IsSet("seed-index") {
+					return errors.New("--seed and --seed-index must be explicitly defined when --seed-peering is enabled")
+				}
+
+				return nil
+			},
+		},
+		&cli.IntFlag{
+			Name:    "seed-peering-max-index",
+			Value:   100,
+			EnvVars: []string{"RAINBOW_SEED_PEERING_MAX_INDEX"},
+			Usage:   "Largest index to derive automatic peering peer IDs for",
+		},
 		&cli.StringSliceFlag{
 			Name:    "gateway-domains",
 			Value:   cli.NewStringSlice(),
@@ -338,6 +361,10 @@ share the same seed as long as the indexes are different.
 			DenylistSubs:            cctx.StringSlice("denylists"),
 			Peering:                 peeringAddrs,
 			PeeringCache:            cctx.Bool("peering-shared-cache"),
+			Seed:                    seed,
+			SeedIndex:               index,
+			SeedPeering:             cctx.Bool("seed-peering"),
+			SeedPeeringMaxIndex:     cctx.Int("seed-peering-max-index"),
 			GCInterval:              cctx.Duration("gc-interval"),
 			GCThreshold:             cctx.Float64("gc-threshold"),
 		}