implement prune command

Signed-off-by: Avi Deitcher <avi@deitcher.net>

cmd/common_test.go

@@ -17,12 +17,17 @@ func newMockExecs() *mockExecs {
 	return m
 }
 
-func (m *mockExecs) timerDump(opts core.DumpOptions, timerOpts core.TimerOptions) error {
-	args := m.Called(opts, timerOpts)
+func (m *mockExecs) dump(opts core.DumpOptions) error {
+	args := m.Called(opts)
 	return args.Error(0)
 }
 
 func (m *mockExecs) restore(target storage.Storage, targetFile string, dbconn database.Connection, databasesMap map[string]string, compressor compression.Compressor) error {
 	args := m.Called(target, targetFile, dbconn, databasesMap, compressor)
 	return args.Error(0)
 }
+
+func (m *mockExecs) prune(target storage.Storage, targetFile, retention string) error {
+	args := m.Called(target, targetFile, retention)
+	return args.Error(0)
+}

cmd/dump.go

@@ -141,6 +141,12 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
 				MaxAllowedPacket:    maxAllowedPacket,
 			}
 
+			// retention, if enabled
+			retention := v.GetString("retention")
+			if retention == "" {
+				retention = configuration.Prune.Retention
+			}
+
 			// timer options
 			once := v.GetBool("once")
 			if !v.IsSet("once") && configuration != nil {
@@ -164,12 +170,25 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
 				Begin:     begin,
 				Frequency: frequency,
 			}
-			dump := core.TimerDump
+			dump := core.Dump
+			prune := core.Prune
 			if execs != nil {
-				dump = execs.timerDump
+				dump = execs.dump
+				prune = execs.prune
 			}
-			if err := dump(dumpOpts, timerOpts); err != nil {
-				return err
+			if err := core.TimerCommand(timerOpts, func() error {
+				err := dump(dumpOpts)
+				if err != nil {
+					return fmt.Errorf("error running dump: %w", err)
+				}
+				if retention != "" {
+					if err := prune(targets[0], "", retention); err != nil {
+						return fmt.Errorf("error running prune: %w", err)
+					}
+				}
+				return nil
+			}); err != nil {
+				return fmt.Errorf("error running command: %w", err)
 			}
 			log.Info("Backup complete")
 			return nil

cmd/prune.go

@@ -0,0 +1,141 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/databacker/mysql-backup/pkg/core"
+	"github.com/databacker/mysql-backup/pkg/storage"
+	"github.com/databacker/mysql-backup/pkg/util"
+)
+
+func pruneCmd(execs execs) (*cobra.Command, error) {
+	var v *viper.Viper
+	var cmd = &cobra.Command{
+		Use:   "prune",
+		Short: "prune older backups",
+		Long: `Prune older backups based on a retention period. Can be number of backups or time-based.
+		For time-based, the format is: 1d, 1w, 1m, 1y for days, weeks, months, years, respectively.
+		For number-based, the format is: 1c, 2c, 3c, etc. for the count of backups to keep.
+		
+		For time-based, prune always converts the time to hours, and then rounds up. This means that 2d is treated as 48h, and
+		any backups must be at least 48 full hours ago to be pruned.
+		`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			bindFlags(cmd, v)
+		},
+		Args: cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			log.Debug("starting prune")
+			targetFile := args[0]
+			target := v.GetString("target")
+			retention := v.GetString("retention")
+
+			// target URL can reference one from the config file, or an absolute one
+			// if it's not in the config file, it's an absolute one
+			// if it is in the config file, it's a reference to one of the targets in the config file
+			u, err := util.SmartParse(target)
+			if err != nil {
+				return fmt.Errorf("invalid target url: %v", err)
+			}
+			var store storage.Storage
+			if u.Scheme == "config" {
+				// get the target name
+				targetName := u.Host
+				// get the target from the config file
+				if configuration == nil {
+					return fmt.Errorf("no configuration file found")
+				}
+				if target, ok := configuration.Targets[targetName]; !ok {
+					return fmt.Errorf("target %s not found in configuration", targetName)
+				} else {
+					if store, err = target.Storage.Storage(); err != nil {
+						return fmt.Errorf("error creating storage for target %s: %v", targetName, err)
+					}
+				}
+				if retention == "" {
+					retention = configuration.Prune.Retention
+				}
+				// need to add the path to the specific target file
+			} else {
+				// parse the target URL
+				store, err = storage.ParseURL(target, creds)
+				if err != nil {
+					return fmt.Errorf("invalid target url: %v", err)
+				}
+			}
+
+			if retention == "" {
+				return fmt.Errorf("retention period is required")
+			}
+
+			// timer options
+			once := v.GetBool("once")
+			if !v.IsSet("once") && configuration != nil {
+				once = configuration.Dump.Schedule.Once
+			}
+			cron := v.GetString("cron")
+			if cron == "" && configuration != nil {
+				cron = configuration.Dump.Schedule.Cron
+			}
+			begin := v.GetString("begin")
+			if begin == "" && configuration != nil {
+				begin = configuration.Dump.Schedule.Begin
+			}
+			frequency := v.GetInt("frequency")
+			if frequency == 0 && configuration != nil {
+				frequency = configuration.Dump.Schedule.Frequency
+			}
+			timerOpts := core.TimerOptions{
+				Once:      once,
+				Cron:      cron,
+				Begin:     begin,
+				Frequency: frequency,
+			}
+
+			prune := core.Prune
+			if execs != nil {
+				prune = execs.prune
+			}
+			if err := core.TimerCommand(timerOpts, func() error {
+				return prune(store, targetFile, retention)
+			}); err != nil {
+				return fmt.Errorf("error running prune: %w", err)
+			}
+			log.Info("Pruning complete")
+			return nil
+		},
+	}
+	// target - where the backup is
+	v = viper.New()
+	v.SetEnvPrefix("db_restore")
+	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	v.AutomaticEnv()
+
+	flags := cmd.Flags()
+	flags.String("target", "", "full URL target to the backup that you wish to restore")
+	if err := cmd.MarkFlagRequired("target"); err != nil {
+		return nil, err
+	}
+
+	// retention
+	flags.String("retention", "", "Retention period for backups. Can be number of backups or time-based. For time-based, the format is: 1d, 1w, 1m, 1y for days, weeks, months, years, respectively. For number-based, the format is: 1c, 2c, 3c, etc. for the count of backups to keep.")
+
+	// frequency
+	flags.Int("frequency", defaultFrequency, "how often to run prunes, in minutes")
+
+	// begin
+	flags.String("begin", defaultBegin, "What time to do the first prune. Must be in one of two formats: Absolute: HHMM, e.g. `2330` or `0415`; or Relative: +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half")
+
+	// cron
+	flags.String("cron", "", "Set the prune schedule using standard [crontab syntax](https://en.wikipedia.org/wiki/Cron), a single line.")
+
+	// once
+	flags.Bool("once", false, "Override all other settings and run the prune once immediately and exit. Useful if you use an external scheduler (e.g. as part of an orchestration solution like Cattle or Docker Swarm or [kubernetes cron jobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/)) and don't want the container to do the scheduling internally.")
+
+	return cmd, nil
+}

cmd/root.go

@@ -19,13 +19,14 @@ import (
 )
 
 type execs interface {
-	timerDump(opts core.DumpOptions, timerOpts core.TimerOptions) error
+	dump(opts core.DumpOptions) error
 	restore(target storage.Storage, targetFile string, dbconn database.Connection, databasesMap map[string]string, compressor compression.Compressor) error
+	prune(target storage.Storage, targetFile, retention string) error
 }
 
 type subCommand func(execs) (*cobra.Command, error)
 
-var subCommands = []subCommand{dumpCmd, restoreCmd}
+var subCommands = []subCommand{dumpCmd, restoreCmd, pruneCmd}
 
 const (
 	defaultPort = 3306

pkg/config/type.go

@@ -34,6 +34,7 @@ type Config struct {
 	Restore  Restore  `yaml:"restore"`
 	Database Database `yaml:"database"`
 	Targets  Targets  `yaml:"targets"`
+	Prune    Prune    `yaml:"prune"`
 }
 
 type Dump struct {
@@ -51,6 +52,10 @@ type Dump struct {
 	Targets          []string      `yaml:"targets"`
 }
 
+type Prune struct {
+	Retention string `yaml:"retention"`
+}
+
 type Schedule struct {
 	Once      bool   `yaml:"once"`
 	Cron      string `yaml:"cron"`

pkg/core/dump.go

@@ -19,25 +19,6 @@ const (
 	targetRenameCmd = "/scripts.d/target.sh"
 )
 
-// TimerDump runs a dump on a timer
-func TimerDump(opts DumpOptions, timerOpts TimerOptions) error {
-	c, err := Timer(timerOpts)
-	if err != nil {
-		log.Errorf("error creating timer: %v", err)
-		os.Exit(1)
-	}
-	// block and wait for it
-	for update := range c {
-		if err := Dump(opts); err != nil {
-			return fmt.Errorf("error backing up: %w", err)
-		}
-		if update.Last {
-			break
-		}
-	}
-	return nil
-}
-
 // Dump run a single dump, based on the provided opts
 func Dump(opts DumpOptions) error {
 	targets := opts.Targets