Merge branch 'main' of https://github.com/Azure/azure-storage-fuse in…

…to ashruti/disk

CHANGELOG.md

@@ -8,12 +8,15 @@
 - Do not allow to mount with non-empty directory provided for disk persistence in block-cache.
 - Rename file was calling an additional getProperties call.
 - Delete empty directories from local cache on rmdir operation.
+- [#1547](https://github.com/Azure/azure-storage-fuse/issues/1547) Truncate logic of file cache is modified to prevent downloading and uploading the entire file.
 
 **Features**
 - Added 'gen-config' command to auto generate Blobfuse2 config file.
 
 **Other Changes**
 - Stream config will be converted to block-cache config implicitly and 'stream' component is no longer used from this release onwards.
+- MSI login with object-id will not rely on azcli anymore, rather it will be supported by 'azidentity' SDK.
+- Version check is now moved to a static website hosted on a public container.
 - MSI login with object-id will not rely on 'azcli' anymore, rather it will be supported by 'azidentity' SDK.
 - 'df' command output will present memory availability in case of block-cache if disk is not configured.
 

blobfuse2-release.yaml

@@ -1744,8 +1744,22 @@ stages:
     condition: succeeded('ReleaseArtifacts')
     jobs:
       - job: UpdateVersion
+        timeoutInMinutes: 120
+        strategy:
+          matrix:
+            UBN20:
+              vmImage: 'ubuntu-20.04'
+              fuselib: 'libfuse3-dev'
+              tags: 'fuse3'
+              depends: 'fuse3'
+              container: 'test-cnt-ubn-20'
+              AgentName: "blobfuse-ubuntu20"
+
         pool:
-          vmImage: 'ubuntu-20.04'
+          name: "blobfuse-ubuntu-pool"
+          demands:
+            - ImageOverride -equals $(agentName)
+
         variables:
           - group: NightlyBlobFuse
           - name: root_dir
@@ -1769,20 +1783,51 @@ stages:
 
               # install blobfuse2
               - script: |
+                  sudo apt install fuse3 -y
                   cd $(Build.ArtifactStagingDirectory)/blobfuse2
                   ls | grep -i ubuntu-20.04
                   sudo apt-get install ./`ls | grep -i ubuntu-20.04` -y
                   blobfuse2 version
                 displayName: 'Installing blobfuse2'
 
               - script: |
-                  wget https://raw.githubusercontent.com/Azure/azure-storage-fuse/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/releaseVersionUpdate.py
-                  ls -l
-                displayName: 'Getting Python script'
-                workingDirectory: $(root_dir)
-
+                  mkdir -p ~/blob_mnt
+                  chmod 777 ~/blob_mnt
+                  mkdir -p ~/blobfusetemp
+                  chmod 777 ~/blobfusetemp
+                  blobfuse2 mount ~/blob_mnt --tmp-path=~/blobfusetemp --container-name="\$web"
+                  sleep 2
+                env:
+                  AZURE_STORAGE_ACCOUNT: "blobfuse2"
+                  AZURE_STORAGE_AUTH_TYPE: "msi"
+                  AZURE_STORAGE_IDENTITY_CLIENT_ID: $(BLOBFUSE2_MSI_CIENT_ID)
+                displayName: 'Mount release container for version update'
+              
               - script: |
-                  python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`blobfuse2 version`"
-                displayName: 'Updating version number'
-                workingDirectory: $(root_dir)
-
+                  cd ~/blob_mnt/release/latest/
+                  echo "Before cleanup"
+                  ls -l 
+                  rm -rf *
+                  echo "After cleanup"
+                  ls -l 
+                displayName: 'Delete older version file'
+                
+              - script: |
+                  cd ~/blob_mnt/release/latest/
+                  touch `blobfuse2 version | cut -d " " -f3`
+                  ls -l 
+                displayName: 'Create new version file'
+              
+              - script: |
+                  cd ~/blob_mnt/release/latest/
+                  version=`blobfuse2 version | cut -d " " -f3`
+                  echo "Release version: " $version
+                  echo -e "<xml>\n<latest>$version</latest>\n</xml>" > index.xml
+                  echo "Printing index.xml"
+                  cat index.xml
+                displayName: 'Update the XML file'    
+                
+              - script: |
+                  blobfuse2 unmount all
+                displayName: 'Unmount Blobfuse2'                              
+                

cmd/health-monitor_test.go

@@ -72,10 +72,10 @@ type hmonTestSuite struct {
 }
 
 func generateRandomPID() string {
-	rand.Seed(time.Now().UnixNano())
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	var randpid int
 	for i := 0; i <= 5; i++ {
-		randpid = rand.Intn(90000) + 10000
+		randpid = r.Intn(90000) + 10000
 		_, err := os.FindProcess(randpid)
 		if err != nil {
 			break

cmd/mount.go

@@ -455,13 +455,8 @@ var mountCmd = &cobra.Command{
 
 		pipeline, err = internal.NewPipeline(options.Components, !daemon.WasReborn())
 		if err != nil {
-			if err.Error() == "Azure CLI not found on path" {
-				log.Err("mount : failed to initialize new pipeline :: To authenticate using MSI with object-ID, ensure Azure CLI is installed. Alternatively, use app/client ID or resource ID for authentication. [%v]", err)
-				return Destroy(fmt.Sprintf("failed to initialize new pipeline :: To authenticate using MSI with object-ID, ensure Azure CLI is installed. Alternatively, use app/client ID or resource ID for authentication. [%s]", err.Error()))
-			} else {
-				log.Err("mount : failed to initialize new pipeline [%v]", err)
-				return Destroy(fmt.Sprintf("failed to initialize new pipeline [%s]", err.Error()))
-			}
+			log.Err("mount : failed to initialize new pipeline [%v]", err)
+			return Destroy(fmt.Sprintf("failed to initialize new pipeline [%s]", err.Error()))
 		}
 
 		common.ForegroundMount = options.Foreground

cmd/mount_test.go

@@ -294,7 +294,7 @@ func (suite *mountTestSuite) TestCliParamsV1() {
 	defer os.RemoveAll(tempLogDir)
 
 	op, err := executeCommandC(rootCmd, "mount", mntDir, fmt.Sprintf("--config-file=%s", confFileMntTest),
-		fmt.Sprintf("--log-file-path=%s", tempLogDir+"/blobfuse2.log"), "--invalidate-on-sync", "--pre-mount-validate", "--basic-remount-check")
+		fmt.Sprintf("--log-file-path=%s", tempLogDir+"/blobfuse2.log"), "--invalidate-on-sync", "--pre-mount-validate", "--basic-remount-check", "-o", "direct_io")
 	suite.assert.NotNil(err)
 	suite.assert.Contains(op, "failed to initialize new pipeline")
 }
@@ -310,7 +310,7 @@ func (suite *mountTestSuite) TestStreamAttrCacheOptionsV1() {
 	defer os.RemoveAll(tempLogDir)
 
 	op, err := executeCommandC(rootCmd, "mount", mntDir, fmt.Sprintf("--log-file-path=%s", tempLogDir+"/blobfuse2.log"),
-		"--streaming", "--use-attr-cache", "--invalidate-on-sync", "--pre-mount-validate", "--basic-remount-check")
+		"--streaming", "--use-attr-cache", "--invalidate-on-sync", "--pre-mount-validate", "--basic-remount-check", "-o", "direct_io")
 	suite.assert.NotNil(err)
 	suite.assert.Contains(op, "failed to initialize new pipeline")
 }
@@ -326,7 +326,7 @@ func (suite *mountTestSuite) TestInvalidLibfuseOption() {
 	// incorrect option
 	op, err := executeCommandC(rootCmd, "mount", mntDir, fmt.Sprintf("--config-file=%s", confFileMntTest),
 		"-o allow_other", "-o attr_timeout=120", "-o entry_timeout=120", "-o negative_timeout=120",
-		"-o ro", "-o default_permissions", "-o umask=755", "-o a=b=c")
+		"-o ro", "-o default_permissions", "-o umask=755", "-o uid=1000", "-o gid=1000", "-o direct_io", "-o a=b=c")
 	suite.assert.NotNil(err)
 	suite.assert.Contains(op, "invalid FUSE options")
 }
@@ -342,7 +342,7 @@ func (suite *mountTestSuite) TestUndefinedLibfuseOption() {
 	// undefined option
 	op, err := executeCommandC(rootCmd, "mount", mntDir, fmt.Sprintf("--config-file=%s", confFileMntTest),
 		"-o allow_other", "-o attr_timeout=120", "-o entry_timeout=120", "-o negative_timeout=120",
-		"-o ro", "-o allow_root", "-o umask=755", "-o random_option")
+		"-o ro", "-o allow_root", "-o umask=755", "-o uid=1000", "-o gid=1000", "-o direct_io", "-o random_option")
 	suite.assert.NotNil(err)
 	suite.assert.Contains(op, "invalid FUSE options")
 }
@@ -358,11 +358,41 @@ func (suite *mountTestSuite) TestInvalidUmaskValue() {
 	// incorrect umask value
 	op, err := executeCommandC(rootCmd, "mount", mntDir, fmt.Sprintf("--config-file=%s", confFileMntTest),
 		"-o allow_other", "-o attr_timeout=120", "-o entry_timeout=120", "-o negative_timeout=120",
-		"-o ro", "-o allow_root", "-o default_permissions", "-o umask=abcd")
+		"-o ro", "-o allow_root", "-o default_permissions", "-o uid=1000", "-o gid=1000", "-o direct_io", "-o umask=abcd")
 	suite.assert.NotNil(err)
 	suite.assert.Contains(op, "failed to parse umask")
 }
 
+func (suite *mountTestSuite) TestInvalidUIDValue() {
+	defer suite.cleanupTest()
+
+	mntDir, err := os.MkdirTemp("", "mntdir")
+	suite.assert.Nil(err)
+	defer os.RemoveAll(mntDir)
+
+	// incorrect umask value
+	op, err := executeCommandC(rootCmd, "mount", mntDir, fmt.Sprintf("--config-file=%s", confFileMntTest),
+		"-o allow_other", "-o attr_timeout=120", "-o entry_timeout=120", "-o negative_timeout=120",
+		"-o ro", "-o allow_root", "-o default_permissions", "-o umask=755", "-o gid=1000", "-o direct_io", "-o uid=abcd")
+	suite.assert.NotNil(err)
+	suite.assert.Contains(op, "failed to parse uid")
+}
+
+func (suite *mountTestSuite) TestInvalidGIDValue() {
+	defer suite.cleanupTest()
+
+	mntDir, err := os.MkdirTemp("", "mntdir")
+	suite.assert.Nil(err)
+	defer os.RemoveAll(mntDir)
+
+	// incorrect umask value
+	op, err := executeCommandC(rootCmd, "mount", mntDir, fmt.Sprintf("--config-file=%s", confFileMntTest),
+		"-o allow_other", "-o attr_timeout=120", "-o entry_timeout=120", "-o negative_timeout=120",
+		"-o ro", "-o allow_root", "-o default_permissions", "-o umask=755", "-o uid=1000", "-o direct_io", "-o gid=abcd")
+	suite.assert.NotNil(err)
+	suite.assert.Contains(op, "failed to parse gid")
+}
+
 // fuse option parsing validation
 func (suite *mountTestSuite) TestFuseOptions() {
 	defer suite.cleanupTest()

cmd/mountv1_test.go

@@ -101,9 +101,9 @@ func TestGenerateConfig(t *testing.T) {
 }
 
 func randomString(length int) string {
-	rand.Seed(time.Now().UnixNano())
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	b := make([]byte, length)
-	rand.Read(b)
+	r.Read(b)
 	return fmt.Sprintf("%x", b)[:length]
 }
 

cmd/root.go

@@ -50,8 +50,7 @@ import (
 )
 
 type VersionFilesList struct {
-	XMLName xml.Name `xml:"EnumerationResults"`
-	Blobs   []Blob   `xml:"Blobs>Blob"`
+	Version string `xml:"latest"`
 }
 
 type Blob struct {
@@ -104,18 +103,23 @@ func getRemoteVersion(req string) (string, error) {
 		return "", err
 	}
 
+	if len(body) > 50 {
+		log.Err("getRemoteVersion: something suspicious in the contents from remote version")
+		return "", fmt.Errorf("unable to get latest version")
+	}
+
 	var versionList VersionFilesList
 	err = xml.Unmarshal(body, &versionList)
 	if err != nil {
 		log.Err("getRemoteVersion: error unmarshalling xml response [%s]", err.Error())
 		return "", err
 	}
 
-	if len(versionList.Blobs) != 1 {
+	if len(versionList.Version) < 5 || len(versionList.Version) > 20 {
 		return "", fmt.Errorf("unable to get latest version")
 	}
 
-	versionName := strings.Split(versionList.Blobs[0].Name, "/")[1]
+	versionName := versionList.Version
 	return versionName, nil
 }
 
@@ -126,7 +130,7 @@ func beginDetectNewVersion() chan interface{} {
 	go func() {
 		defer close(completed)
 
-		latestVersionUrl := common.Blobfuse2ListContainerURL + "?restype=container&comp=list&prefix=latest/"
+		latestVersionUrl := common.Blobfuse2ListContainerURL + "/latest/index.xml"
 		remoteVersion, err := getRemoteVersion(latestVersionUrl)
 		if err != nil {
 			log.Err("beginDetectNewVersion: error getting latest version [%s]", err.Error())

cmd/root_test.go

@@ -147,7 +147,7 @@ func getDummyVersion() string {
 
 func (suite *rootCmdSuite) TestGetRemoteVersionValidContainer() {
 	defer suite.cleanupTest()
-	latestVersionUrl := common.Blobfuse2ListContainerURL + "?restype=container&comp=list&prefix=latest/"
+	latestVersionUrl := common.Blobfuse2ListContainerURL + "/latest/index.xml"
 	out, err := getRemoteVersion(latestVersionUrl)
 	suite.assert.NotEmpty(out)
 	suite.assert.Nil(err)

common/util_test.go

@@ -35,13 +35,12 @@ package common
 
 import (
 	"bytes"
+	"crypto/rand"
 	"fmt"
-	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
@@ -50,7 +49,6 @@ import (
 var home_dir, _ = os.UserHomeDir()
 
 func randomString(length int) string {
-	rand.Seed(time.Now().UnixNano())
 	b := make([]byte, length)
 	rand.Read(b)
 	return fmt.Sprintf("%x", b)[:length]

common/version.go

@@ -39,7 +39,7 @@ import (
 	"strings"
 )
 
-const Blobfuse2ListContainerURL = "https://blobfuse2.blob.core.windows.net/release"
+const Blobfuse2ListContainerURL = "https://blobfuse2.z13.web.core.windows.net/release"
 const BlobFuse2WarningsURL = "https://aka.ms/blobfuse2warnings"
 const BlobFuse2BlockingURL = "https://aka.ms/blobfuse2blockers"
 

component/azstorage/block_blob_test.go

@@ -40,14 +40,14 @@ import (
 	"bytes"
 	"container/list"
 	"context"
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"math"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"strings"
@@ -285,9 +285,8 @@ func (s *blockBlobTestSuite) TestDefault() {
 }
 
 func randomString(length int) string {
-	random := rand.New(rand.NewSource(time.Now().UnixNano()))
 	b := make([]byte, length)
-	random.Read(b)
+	rand.Read(b)
 	return fmt.Sprintf("%x", b)[:length]
 }
 

component/azstorage/datalake_test.go

@@ -39,11 +39,11 @@ package azstorage
 import (
 	"bytes"
 	"container/list"
+	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
-	"math/rand"
 	"os"
 	"strings"
 	"syscall"