Skip to content

Commit

Permalink
cli: add debug mode to upload-bin command
Browse files Browse the repository at this point in the history
Signed-off-by: Ekaterina Pavlova <ekt@morphbits.io>
  • Loading branch information
AliceInHunterland committed Nov 15, 2024
1 parent b97d0b2 commit 0fd9717
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 13 deletions.
6 changes: 5 additions & 1 deletion cli/util/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ func NewCommands() []*cli.Command {
return nil
},
},
&cli.BoolFlag{
Name: "debug",
Usage: "Enable debug mode with logging uploaded object ids",
},
}, options.RPC...)
uploadBinFlags = append(uploadBinFlags, options.Wallet...)
return []*cli.Command{
Expand Down Expand Up @@ -183,7 +187,7 @@ func NewCommands() []*cli.Command {
{
Name: "upload-bin",
Usage: "Fetch blocks from RPC node and upload them to the NeoFS container",
UsageText: "neo-go util upload-bin --fs-rpc-endpoint <address1>[,<address2>[...]] --container <cid> --block-attribute block --index-attribute index --rpc-endpoint <node> [--timeout <time>] --wallet <wallet> [--wallet-config <config>] [--address <address>] [--workers <num>] [--searchers <num>] [--index-file-size <size>] [--skip-blocks-uploading] [--retries <num>]",
UsageText: "neo-go util upload-bin --fs-rpc-endpoint <address1>[,<address2>[...]] --container <cid> --block-attribute block --index-attribute index --rpc-endpoint <node> [--timeout <time>] --wallet <wallet> [--wallet-config <config>] [--address <address>] [--workers <num>] [--searchers <num>] [--index-file-size <size>] [--skip-blocks-uploading] [--retries <num>] [--debug <bool>]",
Action: uploadBin,
Flags: uploadBinFlags,
},
Expand Down
33 changes: 22 additions & 11 deletions cli/util/upload_bin.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ func uploadBlocks(ctx *cli.Context, p *pool.Pool, rpc *rpcclient.Client, signer
fmt.Fprintf(ctx.App.Writer, "No new blocks to upload. Need to upload starting from %d, current height %d\n", oldestMissingBlockIndex, currentBlockHeight)
return nil
}
debug := ctx.Bool("debug")

Check warning on line 249 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L249

Added line #L249 was not covered by tests
for batchStart := oldestMissingBlockIndex; batchStart <= int(currentBlockHeight); batchStart += searchBatchSize {
var (
batchEnd = min(batchStart+searchBatchSize, int(currentBlockHeight)+1)
Expand Down Expand Up @@ -295,7 +296,14 @@ func uploadBlocks(ctx *cli.Context, p *pool.Pool, rpc *rpcclient.Client, signer

objBytes := bw.Bytes()
errRetr := retry(func() error {
return uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, objBytes, attrs, homomorphicHashingDisabled)
resOid, errUpload := uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, objBytes, attrs, homomorphicHashingDisabled)
if errUpload != nil {
return errUpload
}
if debug {
fmt.Fprintf(ctx.App.Writer, "Uploaded block %d with object ID: %s\n", blockIndex, resOid.String())
}
return errUpload

Check warning on line 306 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L299-L306

Added lines #L299 - L306 were not covered by tests
}, maxRetries)
if errRetr != nil {
select {
Expand Down Expand Up @@ -460,7 +468,8 @@ func uploadIndexFiles(ctx *cli.Context, p *pool.Pool, containerID cid.ID, accoun
*object.NewAttribute("IndexSize", strconv.Itoa(int(indexFileSize))),
}
err := retry(func() error {
return uploadObj(ctx.Context, p, signer, account.PrivateKey().GetScriptHash(), containerID, buffer, attrs, homomorphicHashingDisabled)
_, errUpload := uploadObj(ctx.Context, p, signer, account.PrivateKey().GetScriptHash(), containerID, buffer, attrs, homomorphicHashingDisabled)
return errUpload

Check warning on line 472 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L471-L472

Added lines #L471 - L472 were not covered by tests
}, maxRetries)
if err != nil {
select {
Expand Down Expand Up @@ -541,14 +550,15 @@ func searchObjects(ctx context.Context, p *pool.Pool, containerID cid.ID, accoun
}

// uploadObj uploads object to the container using provided settings.
func uploadObj(ctx context.Context, p *pool.Pool, signer user.Signer, owner util.Uint160, containerID cid.ID, objData []byte, attrs []object.Attribute, homomorphicHashingDisabled bool) error {
func uploadObj(ctx context.Context, p *pool.Pool, signer user.Signer, owner util.Uint160, containerID cid.ID, objData []byte, attrs []object.Attribute, homomorphicHashingDisabled bool) (oid.ID, error) {

Check warning on line 553 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L553

Added line #L553 was not covered by tests
var (
ownerID user.ID
hdr object.Object
chSHA256 checksum.Checksum
chHomomorphic checksum.Checksum
v = new(version.Version)
prmObjectPutInit client.PrmObjectPutInit
resOID = oid.ID{}

Check warning on line 561 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L561

Added line #L561 was not covered by tests
)

ownerID.SetScriptHash(owner)
Expand All @@ -569,31 +579,32 @@ func uploadObj(ctx context.Context, p *pool.Pool, signer user.Signer, owner util

err := hdr.SetIDWithSignature(signer)
if err != nil {
return err
return resOID, err

Check warning on line 582 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L582

Added line #L582 was not covered by tests
}
err = hdr.CheckHeaderVerificationFields()
if err != nil {
return err
return resOID, err

Check warning on line 586 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L586

Added line #L586 was not covered by tests
}

writer, err := p.ObjectPutInit(ctx, hdr, signer, prmObjectPutInit)
if err != nil {
return fmt.Errorf("failed to initiate object upload: %w", err)
return resOID, fmt.Errorf("failed to initiate object upload: %w", err)

Check warning on line 591 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L591

Added line #L591 was not covered by tests
}
_, err = writer.Write(objData)
if err != nil {
_ = writer.Close()
return fmt.Errorf("failed to write object data: %w", err)
return resOID, fmt.Errorf("failed to write object data: %w", err)

Check warning on line 596 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L596

Added line #L596 was not covered by tests
}
err = writer.Close()
if err != nil {
return fmt.Errorf("failed to close object writer: %w", err)
return resOID, fmt.Errorf("failed to close object writer: %w", err)

Check warning on line 600 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L600

Added line #L600 was not covered by tests
}
res := writer.GetResult()
if res.StoredObjectID().Equals(oid.ID{}) {
return fmt.Errorf("object ID is empty")
resOID = res.StoredObjectID()
if resOID.Equals(oid.ID{}) {
return resOID, fmt.Errorf("object ID is empty")

Check warning on line 605 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L603-L605

Added lines #L603 - L605 were not covered by tests
}
return nil
return resOID, nil

Check warning on line 607 in cli/util/upload_bin.go

View check run for this annotation

Codecov / codecov/patch

cli/util/upload_bin.go#L607

Added line #L607 was not covered by tests
}

func getBlockIndex(header object.Object, attribute string) (int, error) {
Expand Down
3 changes: 2 additions & 1 deletion docs/neofs-blockstorage.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ NAME:
neo-go util upload-bin - Fetch blocks from RPC node and upload them to the NeoFS container
USAGE:
neo-go util upload-bin --fs-rpc-endpoint <address1>[,<address2>[...]] --container <cid> --block-attribute block --index-attribute index --rpc-endpoint <node> [--timeout <time>] --wallet <wallet> [--wallet-config <config>] [--address <address>] [--workers <num>] [--searchers <num>] [--index-file-size <size>] [--skip-blocks-uploading] [--retries <num>]
neo-go util upload-bin --fs-rpc-endpoint <address1>[,<address2>[...]] --container <cid> --block-attribute block --index-attribute index --rpc-endpoint <node> [--timeout <time>] --wallet <wallet> [--wallet-config <config>] [--address <address>] [--workers <num>] [--searchers <num>] [--index-file-size <size>] [--skip-blocks-uploading] [--retries <num>] [--debug <bool>]
OPTIONS:
--fs-rpc-endpoint value, --fsr value [ --fs-rpc-endpoint value, --fsr value ] List of NeoFS storage node RPC addresses (comma-separated or multiple --fs-rpc-endpoint flags)
Expand All @@ -100,6 +100,7 @@ OPTIONS:
--searchers value Number of concurrent searches for blocks (default: 20)
--skip-blocks-uploading Skip blocks uploading and upload only index files (default: false)
--retries value Maximum number of Neo/NeoFS node request retries (default: 5)
--debug Enable debug mode with logging uploaded object ids (default: false)
--rpc-endpoint value, -r value RPC node address
--timeout value, -s value Timeout for the operation (default: 10s)
--wallet value, -w value Wallet to use to get the key for transaction signing; conflicts with --wallet-config flag
Expand Down

0 comments on commit 0fd9717

Please sign in to comment.