Merge pull request #509 from latchbio/rteqs/ldata-download-cache

cache ldata download

pyproject.toml

@@ -43,15 +43,14 @@ dependencies = [
   "requests-toolbelt==0.10.1",
   "python-dateutil>=2.8",
   "GitPython==3.1.40",
-
   # for old latch develop, to be removed
   "aioconsole==0.6.1",
   "asyncssh==2.13.2",
   "websockets==11.0.3",
   "watchfiles==0.19.0",
-
   # marshmallow_jsonschema depends on setuptools but doesn't specify it so we have to do it for them yay :D
   "setuptools>=75.3.0",
+  "pyxattr>=0.8.1",
 ]
 classifiers = [
   "Development Status :: 4 - Beta",
@@ -85,7 +84,6 @@ Repository = "https://github.com/latchbio/latch"
 Issues = "https://github.com/latchbio/latch/issues"
 Changelog = "https://github.com/latchbio/latch/blob/main/CHANGELOG.md"
 
-
 [dependency-groups]
 dev = ["ruff>=0.7.0", "pytest>=8.3.3"]
 docs = [

src/latch/ldata/path.py

@@ -4,6 +4,7 @@
 import shutil
 from dataclasses import dataclass, field
 from pathlib import Path
+import sys
 from typing import Iterator, Optional, Type
 
 import gql
@@ -18,6 +19,7 @@
 )
 from flytekit.extend import TypeEngine, TypeTransformer
 from typing_extensions import Self
+import xattr
 
 from latch.ldata.type import LatchPathError, LDataNodeType
 from latch_cli.utils import urljoins
@@ -52,6 +54,7 @@ class _Cache:
     size: Optional[int] = None
     dir_size: Optional[int] = None
     content_type: Optional[str] = None
+    version_id: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -101,6 +104,7 @@ def fetch_metadata(self) -> None:
                             ldataObjectMeta {
                                 contentSize
                                 contentType
+                                versionId
                             }
                         }
                     }
@@ -131,6 +135,8 @@ def fetch_metadata(self) -> None:
                 None if meta["contentSize"] is None else int(meta["contentSize"])
             )
             self._cache.content_type = meta["contentType"]
+            self._cache.version_id = meta["versionId"]
+
 
     def _clear_cache(self):
         self._cache.path = None
@@ -140,6 +146,7 @@ def _clear_cache(self):
         self._cache.size = None
         self._cache.dir_size = None
         self._cache.content_type = None
+        self._cache.version_id = None
 
     def node_id(self, *, load_if_missing: bool = True) -> Optional[str]:
         match = node_id_regex.match(self.path)
@@ -188,6 +195,11 @@ def content_type(self, *, load_if_missing: bool = True) -> Optional[str]:
             self.fetch_metadata()
         return self._cache.content_type
 
+    def version_id(self, *, load_if_missing: bool = True) -> Optional[str]:
+        if self._cache.version_id is None and load_if_missing:
+            self.fetch_metadata()
+        return self._cache.version_id
+
     def is_dir(self, *, load_if_missing: bool = True) -> bool:
         return self.type(load_if_missing=load_if_missing) in _dir_types
 
@@ -291,7 +303,7 @@ def upload_from(self, src: Path, *, show_progress_bar: bool = False) -> None:
         self._clear_cache()
 
     def download(
-        self, dst: Optional[Path] = None, *, show_progress_bar: bool = False
+        self, dst: Optional[Path] = None, *, show_progress_bar: bool = False, cache: bool = False
     ) -> Path:
         """Download the file at this instance's path to the given destination.
 
@@ -306,7 +318,18 @@ def download(
             _download_idx += 1
             tmp_dir.mkdir(parents=True, exist_ok=True)
             atexit.register(lambda p: shutil.rmtree(p), tmp_dir)
-            dst = tmp_dir / self.name()
+            name = self.name()
+            if name is None:
+                raise Exception("unable get name of ldata node")
+            dst = tmp_dir / name
+
+        not_windows = sys.platform != "win32"
+        dst_str = str(dst)
+
+        self._clear_cache()
+        version_id = self.version_id()
+        if not_windows and cache and dst.exists() and version_id == xattr.getxattr(dst_str, 'user.version_id').decode():
+            return dst
 
         _download(
             self.path,
@@ -315,6 +338,10 @@ def download(
             verbose=False,
             confirm_overwrite=False,
         )
+
+        if not_windows:
+            xattr.setxattr(dst_str, 'user.version_id', version_id)
+
         return dst
 
     def __truediv__(self, other: object) -> "LPath":