⭐️ Randomize the start time of scanning in serve mode

This makes sure that scans get randomly distributed throughout the timer
time in an effort to spread out load against the api server.

apps/cnspec/cmd/backgroundjob/background.go

@@ -20,6 +20,9 @@ func New() (*BackgroundScanner, error) {
 type BackgroundScanner struct{}
 
 func (bs *BackgroundScanner) Run(runScanFn JobRunner) error {
- Serve(time.Duration(viper.GetInt64("timer"))*time.Minute, runScanFn)
+ Serve(
+ time.Duration(viper.GetInt64("timer"))*time.Minute,
+ time.Duration(viper.GetInt64("splay"))*time.Minute,
+ runScanFn)
  return nil
 }

apps/cnspec/cmd/backgroundjob/serve_unix.go

@@ -4,6 +4,7 @@
 package backgroundjob
 
 import (
+ "math/rand"
  "os"
  "os/signal"
  "sync"
@@ -13,19 +14,20 @@ import (
  "github.com/rs/zerolog/log"
 )
 
-func Serve(timer time.Duration, handler JobRunner) {
+func Serve(timer time.Duration, splay time.Duration, handler JobRunner) {
  log.Info().Msg("start cnspec background service")
- log.Info().Msgf("scan interval is %d minute(s)", int(timer.Minutes()))
+ log.Info().Msgf("scan interval is %d minute(s) with a splay of %d minutes(s)", int(timer.Minutes()), int(splay.Minutes()))
 
  quitChannel := make(chan os.Signal)
  signal.Notify(quitChannel, syscall.SIGINT, syscall.SIGTERM)
 
  shutdownChannel := make(chan struct{})
  waitGroup := &sync.WaitGroup{}
 
- initTick := time.Tick(1 * time.Second)
- defaultTick := time.Tick(timer)
- tick := initTick
+ t := time.NewTimer(time.Duration(rand.Int63n(int64(time.Minute))))
+ defer t.Stop()
+
+ //tick := initTick
  waitGroup.Add(1)
 
  go func(shutdownChannel chan struct{}, wg *sync.WaitGroup) {
@@ -40,14 +42,14 @@ func Serve(timer time.Duration, handler JobRunner) {
  }
 
  select {
- case <-tick:
- if tick == initTick {
- tick = defaultTick
- }
+ case <-t.C:
  err := handler()
  if err != nil {
  log.Error().Err(err).Send()
  }
+ nextRun := timer + time.Duration(rand.Int63n(int64(splay)))
+ log.Info().Msgf("next scan in %v", nextRun)
+ t.Reset(nextRun)
  case <-shutdownChannel:
  log.Info().Msg("stop worker")
  return

apps/cnspec/cmd/backgroundjob/serve_windows.go

@@ -42,9 +42,10 @@ type windowsService struct {
 func (m *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
  const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
  changes <- svc.Status{State: svc.StartPending}
- initTick := time.Tick(1 * time.Second)
- defaulttick := time.Tick(m.Timer)
- tick := initTick
+
+ t := time.NewTimer(time.Duration(rand.Int63n(int64(time.Minute))))
+ defer t.Stop()
+
  log.Info().Msg("schedule background scan")
  changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
 
@@ -68,15 +69,15 @@ func (m *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, chan
 loop:
  for {
  select {
- case <-tick:
- if tick == initTick {
- tick = defaulttick
- }
+ case <-t.C:
  select {
  case runChan <- struct{}{}:
  default:
  log.Error().Msg("scan not started. may be stuck")
  }
+ nextRun := timer + time.Duration(rand.Int63n(int64(splay)))
+ log.Info().Msgf("next scan in %v", nextRun)
+ t.Reset(nextRun)
  case c := <-r:
  switch c.Cmd {
  case svc.Interrogate:

apps/cnspec/cmd/serve.go

@@ -36,7 +36,9 @@ func init() {
  rootCmd.AddCommand(serveCmd)
  // background scan flags
  serveCmd.Flags().Int("timer", 60, "scan interval in minutes")
+ serveCmd.Flags().Int("splay", 60, "randomize the timer by up to this many minutes")
  serveCmd.Flags().MarkHidden("timer")
+ serveCmd.Flags().MarkHidden("splay")
  // set inventory
  serveCmd.Flags().String("inventory-file", "", "Set the path to the inventory file")
 }
@@ -47,6 +49,7 @@ var serveCmd = &cobra.Command{
 
  PreRun: func(cmd *cobra.Command, args []string) {
  viper.BindPFlag("timer", cmd.Flags().Lookup("timer"))
+ viper.BindPFlag("splay", cmd.Flags().Lookup("splay"))
  viper.BindPFlag("inventory-file", cmd.Flags().Lookup("inventory-file"))
  },
  Run: func(cmd *cobra.Command, args []string) {