Skip to content

Commit

Permalink
Use lock to fix issue with parallel invocations (#638)
Browse files Browse the repository at this point in the history
Fixes issues with errors like under Linux:

```
Opening zip "/(...)/downloads/sha256/c97f(...)/bin/bazel (deleted)": open(): No such file or directory
FATAL: Failed to open '/(...)/downloads/sha256/c97f(...)/bin/bazel (deleted)' as a zip file: (error: 2): No such file or directory
```

and below ones under Windows:

```
could not download Bazel: failed to download bazel: failed to move (...) (...)\bin\bazel.exe: Access is denied.
```

See: #436

Tested using `parallel_bazelisk_bug.go` script under Linux and Windows
that this change fixes issue.
  • Loading branch information
hauserx authored Dec 2, 2024
1 parent 26309b3 commit 86ef884
Show file tree
Hide file tree
Showing 5 changed files with 45 additions and 1 deletion.
1 change: 1 addition & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ go_deps.from_file(go_mod = "//:go.mod")
use_repo(
go_deps,
"com_github_bgentry_go_netrc",
"com_github_gofrs_flock",
"com_github_hashicorp_go_version",
"com_github_mitchellh_go_homedir",
"org_golang_x_term",
Expand Down
1 change: 1 addition & 0 deletions core/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ go_library(
"//platforms",
"//versions",
"//ws",
"@com_github_gofrs_flock//:flock",
"@com_github_mitchellh_go_homedir//:go-homedir",
],
)
Expand Down
33 changes: 32 additions & 1 deletion core/core.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package core

import (
"bufio"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/json"
Expand All @@ -21,12 +22,14 @@ import (
"sort"
"strings"
"syscall"
"time"

"github.com/bazelbuild/bazelisk/config"
"github.com/bazelbuild/bazelisk/httputil"
"github.com/bazelbuild/bazelisk/platforms"
"github.com/bazelbuild/bazelisk/versions"
"github.com/bazelbuild/bazelisk/ws"
"github.com/gofrs/flock"
"github.com/mitchellh/go-homedir"
)

Expand Down Expand Up @@ -451,6 +454,34 @@ func atomicWriteFile(path string, contents []byte, perm os.FileMode) error {
return nil
}

// lockedRenameIfDstAbsent executes os.Rename under file lock to avoid issues
// of multiple bazelisk processes renaming file to the same destination file.
// See https://github.com/bazelbuild/bazelisk/issues/436.
func lockedRenameIfDstAbsent(src, dst string) error {
lockFile := dst + ".lock"
fileLock := flock.New(lockFile)

// Do not wait for lock forever to avoid hanging in any scenarios. This
// makes the lock best-effort.
lockCtx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()

ok, err := fileLock.TryLockContext(lockCtx, 50*time.Millisecond)
if !ok || err != nil {
log.Printf("WARNING: Unable to create lock during rename to %s, this may cause issues for parallel bazel executions: %s\n", lockFile, err)
} else {
defer func() {
_ = fileLock.Unlock()
}()
}

if _, err := os.Stat(dst); err == nil {
return nil
}

return os.Rename(src, dst)
}

func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, string, error) {
downloadsDir := filepath.Join(bazeliskHome, "downloads")
temporaryDownloadDir := filepath.Join(downloadsDir, "_tmp")
Expand Down Expand Up @@ -509,7 +540,7 @@ func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories
if err := os.Rename(tmpDestPath, tmpPathInCorrectDirectory); err != nil {
return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpDestPath, tmpPathInCorrectDirectory, err)
}
if err := os.Rename(tmpPathInCorrectDirectory, pathToBazelInCAS); err != nil {
if err := lockedRenameIfDstAbsent(tmpPathInCorrectDirectory, pathToBazelInCAS); err != nil {
return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpPathInCorrectDirectory, pathToBazelInCAS, err)
}

Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.22

require (
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
github.com/gofrs/flock v0.12.1
github.com/hashicorp/go-version v1.7.0
github.com/mitchellh/go-homedir v1.1.0
golang.org/x/term v0.26.0
Expand Down
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

0 comments on commit 86ef884

Please sign in to comment.