Retry on delete failure

ipni · Dec 8, 2023 · 0dafa38 · 0dafa38
1 parent 06ead39
commit 0dafa38
Showing 1 changed file with 17 additions and 3 deletions.
diff --git a/ipni-gc/reaper/reaper.go b/ipni-gc/reaper/reaper.go
@@ -44,7 +44,7 @@ var log = logging.Logger("ipni-gc")
 
 var ErrProviderNotFound = errors.New("provider not found")
 
-var errIndexerWrite = errors.New("delete from index valuestore failed")
+var errIndexerWrite = errors.New("delete from valuestore failed")
 
 type GCStats struct {
 	AdsProcessed    int
@@ -652,7 +652,14 @@ func (s *scythe) removeEntriesWithCar(ctx context.Context, adCid cid.Cid) (int,
 		}
 		if commit {
 			if err = indexer.Remove(value, chunk.Entries...); err != nil {
-				return mhCount, fmt.Errorf("%w: %w", errIndexerWrite, err)
+				if errors.Is(err, context.DeadlineExceeded) {
+					log.Errorw("Timed out removing indexes from valuestore, retrying", "indexes", len(chunk.Entries))
+					time.Sleep(100 * time.Millisecond)
+					err = indexer.Remove(value, chunk.Entries...)
+				}
+				if err != nil {
+					return mhCount, fmt.Errorf("%w: %w", errIndexerWrite, err)
+				}
 			}
 		}
 		mhCount += len(chunk.Entries)
@@ -696,7 +703,14 @@ func (s *scythe) removeEntriesWithPublisher(ctx context.Context, adCid cid.Cid)
 		}
 		if commit {
 			if err = indexer.Remove(value, chunk.Entries...); err != nil {
-				return mhCount, fmt.Errorf("%w: %w", errIndexerWrite, err)
+				if errors.Is(err, context.DeadlineExceeded) {
+					log.Errorw("Timed out removing indexes from valuestore, retrying", "indexes", len(chunk.Entries))
+					time.Sleep(100 * time.Millisecond)
+					err = indexer.Remove(value, chunk.Entries...)
+				}
+				if err != nil {
+					return mhCount, fmt.Errorf("%w: %w", errIndexerWrite, err)
+				}
 			}
 		}
 		mhCount += len(chunk.Entries)