Skip to content

Commit

Permalink
feat(restore): batch, order sstables by size
Browse files Browse the repository at this point in the history
This results in creating batches of sstables of
more similar size.

Fixes #3979
  • Loading branch information
Michal-Leszczynski committed Oct 3, 2024
1 parent c2fcec5 commit ac72b54
Showing 1 changed file with 22 additions and 0 deletions.
22 changes: 22 additions & 0 deletions pkg/service/restore/batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type batchDispatcher struct {
}

func newBatchDispatcher(workload []LocationWorkload, batchSize int, hostShardCnt map[string]uint, locationHosts map[Location][]string) *batchDispatcher {
sortWorkloadBySizeDesc(workload)
var size int64
for _, t := range workload {
size += t.Size
Expand Down Expand Up @@ -221,3 +222,24 @@ func (b *batchDispatcher) createBatch(l *LocationWorkload, t *TableWorkload, dir
SSTables: sstables,
}, true
}

func sortWorkloadBySizeDesc(workload []LocationWorkload) {
slices.SortFunc(workload, func(a, b LocationWorkload) int {
return int(b.Size - a.Size)
})
for _, loc := range workload {
slices.SortFunc(loc.Tables, func(a, b TableWorkload) int {
return int(b.Size - a.Size)
})
for _, tab := range loc.Tables {
slices.SortFunc(tab.RemoteDirs, func(a, b RemoteDirWorkload) int {
return int(b.Size - a.Size)
})
for _, dir := range tab.RemoteDirs {
slices.SortFunc(dir.SSTables, func(a, b RemoteSSTable) int {
return int(b.Size - a.Size)
})
}
}
}
}

0 comments on commit ac72b54

Please sign in to comment.