From 81981860d559018aeffb7173b05d39713f120615 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 10 Oct 2023 09:54:58 -0700 Subject: [PATCH 1/2] Allow advertisement or entries chain truncation When traversing an advertisement chain or entries chain, handle a NotFound error as an indication that the chain was intentionally truncated, and ingest the portion of the chain that was found. --- dagsync/ipnisync/sync.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dagsync/ipnisync/sync.go b/dagsync/ipnisync/sync.go index 89a3def..cf13990 100644 --- a/dagsync/ipnisync/sync.go +++ b/dagsync/ipnisync/sync.go @@ -276,6 +276,13 @@ func (s *Syncer) walkFetch(ctx context.Context, rootCid cid.Cid, sel selector.Se err = progress.WalkMatching(rootNode, sel, func(_ traversal.Progress, _ datamodel.Node) error { return nil }) if err != nil { + // Advertisement or entries block, not at start of chain, was not + // found. Consider this to mean the chain was truncated at this point, + // and return what was found so far. + if errors.Is(err, ipld.ErrNotExists{}) { + log.Warnw("stopping ipld traversal due to content not found") + return traversalOrder, nil + } return nil, err } return traversalOrder, nil From 949745791eb4a63771beed2c5025704989bbb3bf Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 12 Oct 2023 13:13:04 -0700 Subject: [PATCH 2/2] chain truncation recognition for dtsync --- dagsync/dtsync/syncer.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dagsync/dtsync/syncer.go b/dagsync/dtsync/syncer.go index e8a8141..9a8fbdd 100644 --- a/dagsync/dtsync/syncer.go +++ b/dagsync/dtsync/syncer.go @@ -2,6 +2,7 @@ package dtsync import ( "context" + "errors" "fmt" "io" @@ -110,9 +111,17 @@ func (s *Syncer) has(ctx context.Context, nextCid cid.Cid, sel ipld.Node) ([]cid if err != nil { return nil, false } - if err := progress.WalkMatching(rootNode, csel, func(p traversal.Progress, n datamodel.Node) error { + err = progress.WalkMatching(rootNode, csel, func(p traversal.Progress, n datamodel.Node) error { return nil - }); err != nil { + }) + if err != nil { + // Advertisement or entries block, not at start of chain, was not + // found. Consider this to mean the chain was truncated at this point, + // and return what was found so far. + if errors.Is(err, ipld.ErrNotExists{}) { + log.Warnw("stopping ipld traversal due to content not found") + return traversed, true + } return nil, false } return traversed, true