Add Peapod to NeoFS Lens (#2514)

CHANGELOG.md

@@ -16,6 +16,7 @@ minor release, the component will be purged, so be prepared (see `Updating` sect
 - New storage component for small objects named Peapod (#2453)
 - New `blobovnicza-to-peapod` tool providing blobovnicza-to-peapod data migration (#2453)
 - SN's version and capacity is announced via the attributes automatically but can be overwritten explicitly (#2455, #602)
+- `peapod` command for `neofs-lens` (#2507)
 
 ### Fixed
 - `neo-go` RPC connection loss handling (#1337)

cmd/neofs-lens/internal/peapod/inspect.go

@@ -0,0 +1,36 @@
+package peapod
+
+import (
+	common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
+	blobstorcommon "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	"github.com/spf13/cobra"
+)
+
+var inspectCMD = &cobra.Command{
+	Use:   "inspect",
+	Short: "Object inspection",
+	Long:  `Inspect specific object in a Peapod.`,
+	Run:   inspectFunc,
+}
+
+func init() {
+	common.AddAddressFlag(inspectCMD, &vAddress)
+	common.AddComponentPathFlag(inspectCMD, &vPath)
+	common.AddOutputFileFlag(inspectCMD, &vOut)
+}
+
+func inspectFunc(cmd *cobra.Command, _ []string) {
+	var getPrm blobstorcommon.GetPrm
+
+	err := getPrm.Address.DecodeString(vAddress)
+	common.ExitOnErr(cmd, common.Errf("failed to decode object address: %w", err))
+
+	ppd := openPeapod(cmd)
+	defer ppd.Close()
+
+	res, err := ppd.Get(getPrm)
+	common.ExitOnErr(cmd, common.Errf("failed to read object from Peapod: %w", err))
+
+	common.PrintObjectHeader(cmd, *res.Object)
+	common.WriteObjectToFile(cmd, vOut, res.RawData)
+}

cmd/neofs-lens/internal/peapod/list.go

@@ -0,0 +1,37 @@
+package peapod
+
+import (
+	"fmt"
+	"io"
+
+	common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
+	"github.com/spf13/cobra"
+)
+
+var listCMD = &cobra.Command{
+	Use:   "list",
+	Short: "Object listing",
+	Long:  `List all objects stored in a Peapod.`,
+	Run:   listFunc,
+}
+
+func init() {
+	common.AddComponentPathFlag(listCMD, &vPath)
+}
+
+func listFunc(cmd *cobra.Command, _ []string) {
+	// other targets can be supported
+	w := cmd.OutOrStderr()
+
+	wAddr := func(addr oid.Address) error {
+		_, err := io.WriteString(w, fmt.Sprintf("%s\n", addr))
+		return err
+	}
+
+	ppd := openPeapod(cmd)
+	defer ppd.Close()
+
+	err := ppd.IterateAddresses(wAddr)
+	common.ExitOnErr(cmd, common.Errf("Peapod iterator failure: %w", err))
+}

cmd/neofs-lens/internal/peapod/root.go

@@ -0,0 +1,41 @@
+package peapod
+
+import (
+	common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod"
+	"github.com/spf13/cobra"
+)
+
+var (
+	vAddress string
+	vPath    string
+	vOut     string
+)
+
+// Root defines root command for operations with Peapod.
+var Root = &cobra.Command{
+	Use:   "peapod",
+	Short: "Operations with a Peapod",
+}
+
+func init() {
+	Root.AddCommand(listCMD, inspectCMD)
+}
+
+// open and returns read-only peapod.Peapod located in vPath.
+func openPeapod(cmd *cobra.Command) *peapod.Peapod {
+	// interval prm doesn't matter for read-only usage, but must be positive
+	ppd := peapod.New(vPath, 0400, 1)
+	var compressCfg compression.Config
+
+	err := compressCfg.Init()
+	common.ExitOnErr(cmd, common.Errf("failed to init compression config: %w", err))
+
+	ppd.SetCompressor(&compressCfg)
+
+	err = ppd.Open(true)
+	common.ExitOnErr(cmd, common.Errf("failed to open Peapod: %w", err))
+
+	return ppd
+}

cmd/neofs-lens/root.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/blobovnicza"
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/meta"
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/peapod"
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/writecache"
 	"github.com/nspcc-dev/neofs-node/misc"
 	"github.com/nspcc-dev/neofs-node/pkg/util/gendoc"
@@ -35,6 +36,7 @@ func init() {
 	command.SetOut(os.Stdout)
 	command.Flags().Bool("version", false, "Application version")
 	command.AddCommand(
+		peapod.Root,
 		blobovnicza.Root,
 		meta.Root,
 		writecache.Root,

pkg/local_object_storage/blobstor/peapod/peapod.go

@@ -67,6 +67,13 @@ var errMissingRootBucket = errors.New("missing root bucket")
 // specified permissions.
 //
 // Specified flush interval MUST be positive (see Init).
+//
+// Note that resulting Peapod is NOT ready-to-go:
+//   - configure compression first (SetCompressor method)
+//   - then open the instance (Open method). Opened Peapod must be finally closed
+//   - initialize internal database structure (Init method). May be skipped for read-only usage
+//
+// Any other usage is unsafe and may lead to panic.
 func New(path string, perm fs.FileMode, flushInterval time.Duration) *Peapod {
 	if flushInterval <= 0 {
 		panic(fmt.Sprintf("non-positive flush interval %v", flushInterval))
@@ -437,6 +444,8 @@ func (x *Peapod) batch(ctx context.Context, fBktRoot func(bktRoot *bbolt.Bucket)
 
 // Iterate iterates over all objects stored in the underlying database and
 // passes them into LazyHandler or Handler. Break on f's false return.
+//
+// Use IterateAddresses to iterate over keys only.
 func (x *Peapod) Iterate(prm common.IteratePrm) (common.IterateRes, error) {
 	var addr oid.Address
 
@@ -492,3 +501,31 @@ func (x *Peapod) Iterate(prm common.IteratePrm) (common.IterateRes, error) {
 
 	return common.IterateRes{}, nil
 }
+
+// IterateAddresses iterates over all objects stored in the underlying database
+// and passes their addresses into f. If f returns an error, IterateAddresses
+// returns it and breaks.
+func (x *Peapod) IterateAddresses(f func(addr oid.Address) error) error {
+	var addr oid.Address
+
+	err := x.bolt.View(func(tx *bbolt.Tx) error {
+		bktRoot := tx.Bucket(rootBucket)
+		if bktRoot == nil {
+			return errMissingRootBucket
+		}
+
+		return bktRoot.ForEach(func(k, v []byte) error {
+			err := decodeKeyForObject(&addr, k)
+			if err != nil {
+				return fmt.Errorf("decode object address from bucket key: %w", err)
+			}
+
+			return f(addr)
+		})
+	})
+	if err != nil {
+		return fmt.Errorf("exec read-only BoltDB transaction: %w", err)
+	}
+
+	return nil
+}

pkg/local_object_storage/blobstor/peapod/peapod_test.go

@@ -298,3 +298,36 @@ func BenchmarkPeapod_Put(b *testing.B) {
 		})
 	}
 }
+
+func TestPeapod_IterateAddresses(t *testing.T) {
+	ppd := newTestPeapod(t)
+
+	mSrc := map[oid.Address]struct{}{
+		oidtest.Address(): {},
+		oidtest.Address(): {},
+		oidtest.Address(): {},
+	}
+
+	mDst := make(map[oid.Address]struct{})
+
+	f := func(addr oid.Address) error {
+		mDst[addr] = struct{}{}
+		return nil
+	}
+
+	err := ppd.IterateAddresses(f)
+	require.NoError(t, err)
+	require.Empty(t, mDst)
+
+	for addr := range mSrc {
+		_, err = ppd.Put(common.PutPrm{
+			Address: addr,
+			RawData: nil, // doesn't affect current test
+		})
+		require.NoError(t, err)
+	}
+
+	err = ppd.IterateAddresses(f)
+	require.NoError(t, err)
+	require.Equal(t, mSrc, mDst)
+}