Skip to content

Commit

Permalink
source-mongodb: adjust logging
Browse files Browse the repository at this point in the history
Reduces some of the logging noise by adjusting logs to debug level, and makes the frequency of
change stream progress updates 5 minutes instead of 1 minute.
  • Loading branch information
williamhbaker committed May 7, 2024
1 parent 328052f commit 3f806f1
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 6 deletions.
2 changes: 1 addition & 1 deletion source-mongodb/backfill.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func (c *capture) doBackfill(
"collection": binding.resource.Collection,
"estimatedTotalDocs": estimatedTotalDocs,
})
logEntry.Info("starting backfill for collection")
logEntry.Debug("starting backfill for collection")

// By not specifying a sort parameter, MongoDB uses natural sort to order documents. Natural
// sort is approximately insertion order (but not guaranteed). We hint to MongoDB to use the _id
Expand Down
8 changes: 4 additions & 4 deletions source-mongodb/change_stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (c *capture) initializeStreams(
return nil, fmt.Errorf("initializing change stream on database %q: %w", db, err)
}

logEntry.Info("intialized change stream")
logEntry.Debug("intialized change stream")

out = append(out, changeStream{
ms: ms,
Expand Down Expand Up @@ -236,9 +236,9 @@ func (c *capture) streamCatchup(

ts := time.Now()
logEntry := log.WithField("database", s.db)
logEntry.Info("started catching up stream for database")
logEntry.Debug("started catching up stream for database")
defer func() {
logEntry.WithField("took", time.Since(ts).String()).Info("finished catching up stream for database")
logEntry.WithField("took", time.Since(ts).String()).Debug("finished catching up stream for database")
}()

if err := c.tryStream(groupCtx, s, &opTime); err != nil {
Expand Down Expand Up @@ -273,7 +273,7 @@ func (c *capture) tryStream(
log.WithFields(log.Fields{
"database": s.db,
"eventClusterTime": ev.ClusterTime,
}).Info("catch up stream reached cluster OpTime high watermark")
}).Debug("catch up stream reached cluster OpTime high watermark")
// Early return without a checkpoint for the "post batch resume token", since by
// definition we have received a document that corresponds to a specific resume
// token.
Expand Down
2 changes: 1 addition & 1 deletion source-mongodb/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const (
// progress report is currently just the count of stream documents that have been processed,
// which provides a nice indication in the logs of what the capture is doing when it is reading
// change streams.
streamLoggerInterval = 1 * time.Minute
streamLoggerInterval = 5 * time.Minute
)

// Pull is a very long lived RPC through which the Flow runtime and a
Expand Down

0 comments on commit 3f806f1

Please sign in to comment.