Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: sync glacier objects overwrite #734

Merged
merged 23 commits into from
Nov 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
f5a9d48
fix: sync glacier
4o4x Jul 4, 2024
9179a2c
fix: shouldSkipObject separated to src and dest
4o4x Jul 9, 2024
6b6b201
Revert "fix: shouldSkipObject separated to src and dest"
4o4x Jul 9, 2024
d33956b
Revert "Revert "fix: shouldSkipObject separated to src and dest""
4o4x Jul 9, 2024
ed780d8
fix: sync_test
4o4x Jul 9, 2024
6196e3c
Merge branch 'master' into fix-sync-glacier
igungor Jul 9, 2024
4e5370f
Merge branch 'master' into fix-sync-glacier
ilkinulas Jul 10, 2024
67c747e
Merge branch 'master' into fix-sync-glacier
igungor Jul 26, 2024
9ff4d40
test: added sync tests
4o4x Jul 29, 2024
137a5d9
Merge branch 'fix-sync-glacier' of https://github.com/4o4x/s5cmd into…
4o4x Jul 29, 2024
eb56286
test: added storageclass assertion to some of testes
4o4x Jul 29, 2024
8d7568a
fix: remove unnecessary print statements in sync_test.go
4o4x Jul 29, 2024
711db81
chore: remove 0.17 s3fake
4o4x Jul 30, 2024
efb2ace
chore: remove 0.16 s3fake from go.sum
4o4x Jul 30, 2024
4bb7d16
chore: revert unnecessary line delete in sync_strategy.go
4o4x Jul 30, 2024
0110fbe
chore: revert unnecessary line delete in sync_strategy_test.go
4o4x Jul 30, 2024
3932f27
chore: revert some unnecessary
4o4x Jul 30, 2024
957a0e5
chore: remove print statement in rm_test.go
4o4x Jul 30, 2024
67dab13
test: fix storageclass test using timestamps
4o4x Jul 31, 2024
75eb909
test: fix storageclass test using timestamps
4o4x Jul 31, 2024
a4a7311
chore: revert unnecessary changes in rm_test.go
4o4x Jul 31, 2024
8981b61
Merge branch 'master' into fix-sync-glacier
igungor Oct 18, 2024
5bedf2d
Merge branch 'master' into fix-sync-glacier
igungor Oct 24, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions command/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ func (s Sync) getSourceAndDestinationObjects(ctx context.Context, cancel context
log.Error(msg)
cancel()
}
if s.shouldSkipObject(st, true) {
if s.shouldSkipSrcObject(st, true) {
continue
}
filteredSrcObjectChannel <- *st
Expand Down Expand Up @@ -404,7 +404,7 @@ func (s Sync) getSourceAndDestinationObjects(ctx context.Context, cancel context
log.Error(msg)
cancel()
}
if s.shouldSkipObject(dt, false) {
if s.shouldSkipDstObject(dt, false) {
continue
}
filteredDstObjectChannel <- *dt
Expand Down Expand Up @@ -550,7 +550,7 @@ func generateDestinationURL(srcurl, dsturl *url.URL, isBatch bool) *url.URL {
}

// shouldSkipObject checks is object should be skipped.
func (s Sync) shouldSkipObject(object *storage.Object, verbose bool) bool {
func (s Sync) shouldSkipSrcObject(object *storage.Object, verbose bool) bool {
if object.Type.IsDir() || errorpkg.IsCancelation(object.Err) {
return true
}
Expand All @@ -572,6 +572,21 @@ func (s Sync) shouldSkipObject(object *storage.Object, verbose bool) bool {
return false
}

func (s Sync) shouldSkipDstObject(object *storage.Object, verbose bool) bool {
if object.Type.IsDir() || errorpkg.IsCancelation(object.Err) {
return true
}

if err := object.Err; err != nil {
if verbose {
printError(s.fullCommand, s.op, err)
}
return true
}

return false
}

// shouldStopSync determines whether a sync process should be stopped or not.
func (s Sync) shouldStopSync(err error) bool {
if err == storage.ErrNoObjectFound {
Expand Down
Loading
Loading