From e768f25ce578f7f1d000fcfe824de59bb4541d1d Mon Sep 17 00:00:00 2001 From: Radu Berinde Date: Tue, 31 Dec 2024 15:51:33 -0800 Subject: [PATCH] tool: add db upgrade command Add a `db upgrade` command that ratches up the DB version to the newest one. --- tool/db.go | 125 ++++++++++++++++++++++++++++++--------- tool/testdata/db_upgrade | 59 ++++++++++++++++++ 2 files changed, 157 insertions(+), 27 deletions(-) create mode 100644 tool/testdata/db_upgrade diff --git a/tool/db.go b/tool/db.go index 0642cfc342..ee2348177e 100644 --- a/tool/db.go +++ b/tool/db.go @@ -34,6 +34,7 @@ import ( type dbT struct { Root *cobra.Command Check *cobra.Command + Upgrade *cobra.Command Checkpoint *cobra.Command Get *cobra.Command Logs *cobra.Command @@ -107,6 +108,17 @@ database not be in use by another process. Args: cobra.ExactArgs(1), Run: d.runCheck, } + d.Upgrade = &cobra.Command{ + Use: "upgrade ", + Short: "upgrade the DB internal format version", + Long: ` +Upgrades the DB internal format version to the latest version. +It is recommended to make a backup copy of the DB directory before upgrading. +Requires that the specified database not be in use by another process. +`, + Args: cobra.ExactArgs(1), + Run: d.runUpgrade, + } d.Checkpoint = &cobra.Command{ Use: "checkpoint ", Short: "create a checkpoint", @@ -201,10 +213,10 @@ specified database. Run: d.runIOBench, } - d.Root.AddCommand(d.Check, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise, d.IOBench) + d.Root.AddCommand(d.Check, d.Upgrade, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise, d.IOBench) d.Root.PersistentFlags().BoolVarP(&d.verbose, "verbose", "v", false, "verbose output") - for _, cmd := range []*cobra.Command{d.Check, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise} { + for _, cmd := range []*cobra.Command{d.Check, d.Upgrade, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise} { cmd.Flags().StringVar( &d.comparerName, "comparer", "", "comparer name (use default if empty)") cmd.Flags().StringVar( @@ -215,6 +227,10 @@ specified database. cmd.Flags().Var( &d.fmtValue, "value", "value formatter") } + for _, cmd := range []*cobra.Command{d.Upgrade, d.Excise} { + cmd.Flags().BoolVarP( + &d.bypassPrompt, "yes", "y", false, "bypass prompt") + } d.LSM.Flags().BoolVar( &d.lsmURL, "url", false, "generate LSM viewer URL") @@ -237,8 +253,6 @@ specified database. &d.start, "start", "start key for the excised range") d.Excise.Flags().Var( &d.end, "end", "exclusive end key for the excised range") - d.Excise.Flags().BoolVar( - &d.bypassPrompt, "yes", false, "bypass prompt") d.IOBench.Flags().BoolVar( &d.allLevels, "all-levels", false, "if set, benchmark all levels (default is only L5/L6)") @@ -391,13 +405,43 @@ func (d *dbT) runCheck(cmd *cobra.Command, args []string) { stats.NumPoints, makePlural("point", stats.NumPoints), stats.NumTombstones, makePlural("tombstone", int64(stats.NumTombstones))) } -type nonReadOnly struct{} +func (d *dbT) runUpgrade(cmd *cobra.Command, args []string) { + stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr() + db, err := d.openDB(args[0], nonReadOnly{}) + if err != nil { + fmt.Fprintf(stderr, "%s\n", err) + return + } + defer d.closeDB(stderr, db) -func (n nonReadOnly) Apply(dirname string, opts *pebble.Options) { - opts.ReadOnly = false - // Increase the L0 compaction threshold to reduce the likelihood of an - // unintended compaction changing test output. - opts.L0CompactionThreshold = 10 + targetVersion := pebble.FormatNewest + current := db.FormatMajorVersion() + if current >= targetVersion { + fmt.Fprintf(stdout, "DB is already at internal version %d.\n", current) + return + } + fmt.Fprintf(stdout, "Upgrading DB from internal version %d to %d.\n", current, targetVersion) + + prompt := `WARNING!!! +This DB will not be usable with older versions of Pebble! + +It is strongly recommended to back up the data before upgrading. +` + + if len(d.opts.BlockPropertyCollectors) == 0 { + prompt += ` +If this DB uses custom block property collectors, the upgrade should be invoked +through a custom binary that configures them. Otherwise, any new tables created +during upgrade will not have the relevant block properties. +` + } + if !d.promptForConfirmation(prompt, cmd.InOrStdin(), stdout, stderr) { + return + } + if err := db.RatchetFormatMajorVersion(targetVersion); err != nil { + fmt.Fprintf(stderr, "error: %s\n", err) + } + fmt.Fprintf(stdout, "Upgrade complete.\n") } func (d *dbT) runCheckpoint(cmd *cobra.Command, args []string) { @@ -589,23 +633,10 @@ func (d *dbT) runExcise(cmd *cobra.Command, args []string) { fmt.Fprintf(stdout, " start: %s\n", d.fmtKey.fn(span.Start)) fmt.Fprintf(stdout, " end: %s\n", d.fmtKey.fn(span.End)) - if !d.bypassPrompt { - fmt.Fprintf(stdout, "WARNING!!!\n") - fmt.Fprintf(stdout, "This command will remove all keys in this range!\n") - reader := bufio.NewReader(cmd.InOrStdin()) - for { - fmt.Fprintf(stdout, "Continue? [Y/N] ") - answer, _ := reader.ReadString('\n') - answer = strings.ToLower(strings.TrimSpace(answer)) - if answer == "y" || answer == "yes" { - break - } - - if answer == "n" || answer == "no" { - fmt.Fprintf(stderr, "Aborting\n") - return - } - } + prompt := `WARNING!!! +This command will remove all keys in this range!` + if !d.promptForConfirmation(prompt, cmd.InOrStdin(), stdout, stderr) { + return } // Write a temporary sst that only has excise tombstones. We write it inside @@ -814,6 +845,46 @@ func (d *dbT) runSet(cmd *cobra.Command, args []string) { } } +func (d *dbT) promptForConfirmation(prompt string, stdin io.Reader, stdout, stderr io.Writer) bool { + if d.bypassPrompt { + return true + } + if _, err := fmt.Fprintf(stdout, "%s\n", prompt); err != nil { + fmt.Fprintf(stderr, "Error: %v\n", err) + return false + } + reader := bufio.NewReader(stdin) + for { + if _, err := fmt.Fprintf(stdout, "Continue? [Y/N] "); err != nil { + fmt.Fprintf(stderr, "Error: %v\n", err) + return false + } + answer, err := reader.ReadString('\n') + if err != nil { + fmt.Fprintf(stderr, "Error: %v\n", err) + return false + } + answer = strings.ToLower(strings.TrimSpace(answer)) + if answer == "y" || answer == "yes" { + return true + } + + if answer == "n" || answer == "no" { + _, _ = fmt.Fprintf(stderr, "Aborting\n") + return false + } + } +} + +type nonReadOnly struct{} + +func (n nonReadOnly) Apply(dirname string, opts *pebble.Options) { + opts.ReadOnly = false + // Increase the L0 compaction threshold to reduce the likelihood of an + // unintended compaction changing test output. + opts.L0CompactionThreshold = 10 +} + func propArgs(props []props, getProp func(*props) interface{}) []interface{} { args := make([]interface{}, 0, len(props)) for _, p := range props { diff --git a/tool/testdata/db_upgrade b/tool/testdata/db_upgrade new file mode 100644 index 0000000000..7c27d73c8f --- /dev/null +++ b/tool/testdata/db_upgrade @@ -0,0 +1,59 @@ +create foo +---- + +db set foo blue blue-val +---- + +db set foo orange orange-val +---- + +db set foo green green-val +---- + +db set foo red red-val +---- + +db set foo yellow yellow-val +---- + +db get foo blue +---- +[626c75652d76616c] + +db get foo yellow +---- +[79656c6c6f772d76616c] + +db upgrade foo +---- +---- +Upgrading DB from internal version 16 to 19. +WARNING!!! +This DB will not be usable with older versions of Pebble! + +It is strongly recommended to back up the data before upgrading. + +If this DB uses custom block property collectors, the upgrade should be invoked +through a custom binary that configures them. Otherwise, any new tables created +during upgrade will not have the relevant block properties. + +Continue? [Y/N] Error: EOF +---- +---- + +db upgrade foo --yes +---- +Upgrading DB from internal version 16 to 19. +Upgrade complete. + +db get foo blue +---- +[626c75652d76616c] + +db get foo yellow +---- +[79656c6c6f772d76616c] + +db upgrade foo +---- +DB is already at internal version 19.