tool: add db upgrade command

Add a `db upgrade` command that ratches up the DB version to the
newest one.

tool/db.go

@@ -34,6 +34,7 @@ import (
 type dbT struct {
 	Root       *cobra.Command
 	Check      *cobra.Command
+	Upgrade    *cobra.Command
 	Checkpoint *cobra.Command
 	Get        *cobra.Command
 	Logs       *cobra.Command
@@ -107,6 +108,17 @@ database not be in use by another process.
 		Args: cobra.ExactArgs(1),
 		Run:  d.runCheck,
 	}
+	d.Upgrade = &cobra.Command{
+		Use:   "upgrade <dir>",
+		Short: "upgrade the DB internal format version",
+		Long: `
+Upgrades the DB internal format version to the latest version.
+It is recommended to make a backup copy of the DB directory before upgrading.
+Requires that the specified database not be in use by another process.
+`,
+		Args: cobra.ExactArgs(1),
+		Run:  d.runUpgrade,
+	}
 	d.Checkpoint = &cobra.Command{
 		Use:   "checkpoint <src-dir> <dest-dir>",
 		Short: "create a checkpoint",
@@ -201,10 +213,10 @@ specified database.
 		Run:  d.runIOBench,
 	}
 
-	d.Root.AddCommand(d.Check, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise, d.IOBench)
+	d.Root.AddCommand(d.Check, d.Upgrade, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise, d.IOBench)
 	d.Root.PersistentFlags().BoolVarP(&d.verbose, "verbose", "v", false, "verbose output")
 
-	for _, cmd := range []*cobra.Command{d.Check, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise} {
+	for _, cmd := range []*cobra.Command{d.Check, d.Upgrade, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise} {
 		cmd.Flags().StringVar(
 			&d.comparerName, "comparer", "", "comparer name (use default if empty)")
 		cmd.Flags().StringVar(
@@ -215,6 +227,10 @@ specified database.
 		cmd.Flags().Var(
 			&d.fmtValue, "value", "value formatter")
 	}
+	for _, cmd := range []*cobra.Command{d.Upgrade, d.Excise} {
+		cmd.Flags().BoolVarP(
+			&d.bypassPrompt, "yes", "y", false, "bypass prompt")
+	}
 
 	d.LSM.Flags().BoolVar(
 		&d.lsmURL, "url", false, "generate LSM viewer URL")
@@ -237,8 +253,6 @@ specified database.
 		&d.start, "start", "start key for the excised range")
 	d.Excise.Flags().Var(
 		&d.end, "end", "exclusive end key for the excised range")
-	d.Excise.Flags().BoolVar(
-		&d.bypassPrompt, "yes", false, "bypass prompt")
 
 	d.IOBench.Flags().BoolVar(
 		&d.allLevels, "all-levels", false, "if set, benchmark all levels (default is only L5/L6)")
@@ -391,13 +405,43 @@ func (d *dbT) runCheck(cmd *cobra.Command, args []string) {
 		stats.NumPoints, makePlural("point", stats.NumPoints), stats.NumTombstones, makePlural("tombstone", int64(stats.NumTombstones)))
 }
 
-type nonReadOnly struct{}
+func (d *dbT) runUpgrade(cmd *cobra.Command, args []string) {
+	stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
+	db, err := d.openDB(args[0], nonReadOnly{})
+	if err != nil {
+		fmt.Fprintf(stderr, "%s\n", err)
+		return
+	}
+	defer d.closeDB(stderr, db)
 
-func (n nonReadOnly) Apply(dirname string, opts *pebble.Options) {
-	opts.ReadOnly = false
-	// Increase the L0 compaction threshold to reduce the likelihood of an
-	// unintended compaction changing test output.
-	opts.L0CompactionThreshold = 10
+	targetVersion := pebble.FormatNewest
+	current := db.FormatMajorVersion()
+	if current >= targetVersion {
+		fmt.Fprintf(stdout, "DB is already at internal version %d.\n", current)
+		return
+	}
+	fmt.Fprintf(stdout, "Upgrading DB from internal version %d to %d.\n", current, targetVersion)
+
+	prompt := `WARNING!!!
+This DB will not be usable with older versions of Pebble!
+
+It is strongly recommended to back up the data before upgrading.
+`
+
+	if len(d.opts.BlockPropertyCollectors) == 0 {
+		prompt += `
+If this DB uses custom block property collectors, the upgrade should be invoked
+through a custom binary that configures them. Otherwise, any new tables created
+during upgrade will not have the relevant block properties.
+`
+	}
+	if !d.promptForConfirmation(prompt, cmd.InOrStdin(), stdout, stderr) {
+		return
+	}
+	if err := db.RatchetFormatMajorVersion(targetVersion); err != nil {
+		fmt.Fprintf(stderr, "error: %s\n", err)
+	}
+	fmt.Fprintf(stdout, "Upgrade complete.\n")
 }
 
 func (d *dbT) runCheckpoint(cmd *cobra.Command, args []string) {
@@ -589,23 +633,10 @@ func (d *dbT) runExcise(cmd *cobra.Command, args []string) {
 	fmt.Fprintf(stdout, "  start: %s\n", d.fmtKey.fn(span.Start))
 	fmt.Fprintf(stdout, "  end:   %s\n", d.fmtKey.fn(span.End))
 
-	if !d.bypassPrompt {
-		fmt.Fprintf(stdout, "WARNING!!!\n")
-		fmt.Fprintf(stdout, "This command will remove all keys in this range!\n")
-		reader := bufio.NewReader(cmd.InOrStdin())
-		for {
-			fmt.Fprintf(stdout, "Continue? [Y/N] ")
-			answer, _ := reader.ReadString('\n')
-			answer = strings.ToLower(strings.TrimSpace(answer))
-			if answer == "y" || answer == "yes" {
-				break
-			}
-
-			if answer == "n" || answer == "no" {
-				fmt.Fprintf(stderr, "Aborting\n")
-				return
-			}
-		}
+	prompt := `WARNING!!!
+This command will remove all keys in this range!`
+	if !d.promptForConfirmation(prompt, cmd.InOrStdin(), stdout, stderr) {
+		return
 	}
 
 	// Write a temporary sst that only has excise tombstones. We write it inside
@@ -814,6 +845,46 @@ func (d *dbT) runSet(cmd *cobra.Command, args []string) {
 	}
 }
 
+func (d *dbT) promptForConfirmation(prompt string, stdin io.Reader, stdout, stderr io.Writer) bool {
+	if d.bypassPrompt {
+		return true
+	}
+	if _, err := fmt.Fprintf(stdout, "%s\n", prompt); err != nil {
+		fmt.Fprintf(stderr, "Error: %v\n", err)
+		return false
+	}
+	reader := bufio.NewReader(stdin)
+	for {
+		if _, err := fmt.Fprintf(stdout, "Continue? [Y/N] "); err != nil {
+			fmt.Fprintf(stderr, "Error: %v\n", err)
+			return false
+		}
+		answer, err := reader.ReadString('\n')
+		if err != nil {
+			fmt.Fprintf(stderr, "Error: %v\n", err)
+			return false
+		}
+		answer = strings.ToLower(strings.TrimSpace(answer))
+		if answer == "y" || answer == "yes" {
+			return true
+		}
+
+		if answer == "n" || answer == "no" {
+			_, _ = fmt.Fprintf(stderr, "Aborting\n")
+			return false
+		}
+	}
+}
+
+type nonReadOnly struct{}
+
+func (n nonReadOnly) Apply(dirname string, opts *pebble.Options) {
+	opts.ReadOnly = false
+	// Increase the L0 compaction threshold to reduce the likelihood of an
+	// unintended compaction changing test output.
+	opts.L0CompactionThreshold = 10
+}
+
 func propArgs(props []props, getProp func(*props) interface{}) []interface{} {
 	args := make([]interface{}, 0, len(props))
 	for _, p := range props {

tool/testdata/db_upgrade

@@ -0,0 +1,59 @@
+create foo
+----
+
+db set foo blue blue-val
+----
+
+db set foo orange orange-val
+----
+
+db set foo green green-val
+----
+
+db set foo red red-val
+----
+
+db set foo yellow yellow-val
+----
+
+db get foo blue
+----
+[626c75652d76616c]
+
+db get foo yellow
+----
+[79656c6c6f772d76616c]
+
+db upgrade foo
+----
+----
+Upgrading DB from internal version 16 to 19.
+WARNING!!!
+This DB will not be usable with older versions of Pebble!
+
+It is strongly recommended to back up the data before upgrading.
+
+If this DB uses custom block property collectors, the upgrade should be invoked
+through a custom binary that configures them. Otherwise, any new tables created
+during upgrade will not have the relevant block properties.
+
+Continue? [Y/N] Error: EOF
+----
+----
+
+db upgrade foo --yes
+----
+Upgrading DB from internal version 16 to 19.
+Upgrade complete.
+
+db get foo blue
+----
+[626c75652d76616c]
+
+db get foo yellow
+----
+[79656c6c6f772d76616c]
+
+db upgrade foo
+----
+DB is already at internal version 19.