From 1b0ff1413b568dccc8fce7ef4f97a0e19fe3124c Mon Sep 17 00:00:00 2001 From: Winston Chang Date: Mon, 28 Oct 2024 16:14:17 -0500 Subject: [PATCH] If local packages add new dependencies, add them to lock file --- scripts/pyodide_packages.py | 57 +++++++++++++++++++++++++++++++++++-- shinylive_lock.json | 21 +++++++++++--- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/scripts/pyodide_packages.py b/scripts/pyodide_packages.py index 712e6011..15da22ed 100755 --- a/scripts/pyodide_packages.py +++ b/scripts/pyodide_packages.py @@ -102,7 +102,7 @@ # ============================================= -# Data structures used in our requirements.json +# Data structures used in our shinylive_requirements.json # ============================================= class RequirementsPackage(TypedDict): name: str @@ -111,7 +111,7 @@ class RequirementsPackage(TypedDict): # ==================================================== -# Data structures used in our extra_packages_lock.json +# Data structures used in our shinylive_lock.json # ==================================================== class LockfileDependency(TypedDict): name: str @@ -214,6 +214,15 @@ class PyodidePackagesFile(TypedDict): packages: dict[str, PyodidePackageInfo] +# ================================================= +# Internal data structures +# ================================================= +# This BasicPackageInfo type is used internally for package resolution. +class BasicPackageInfo(TypedDict): + name: str + depends: list[str] + + # ============================================================================= # Functions for generating the lockfile from the requirements file. # ============================================================================= @@ -241,6 +250,16 @@ def update_lockfile_local() -> None: lockfile_info.update(required_package_info) + # If any dependencies in the lockfile_info are not in the union of {lockfile_info, + # orig_pyodide_lock()["packages"]}, then we need to add them to the lockfile_info. + # This can happen when a local package adds a new dependency. + print("Searching for new dependencies") + # basic_package_info is the union of lockfile_info and original pyodide packages. + basic_package_info = _to_basic_package_info(lockfile_info) + basic_package_info.update(_to_basic_package_info(orig_pyodide_lock()["packages"])) + _recurse_dependencies_lockfile(lockfile_info, basic_package_info) + + print(f"Writing {package_lock_file}") with open(package_lock_file, "w") as f: json.dump( _mark_no_indent(lockfile_info, _is_lockfile_dependency), @@ -277,12 +296,15 @@ def generate_lockfile() -> None: def _recurse_dependencies_lockfile( pkgs: dict[str, LockfilePackageInfo], + pyodide_packages_info: dict[str, BasicPackageInfo] | None = None, ) -> None: """ Recursively find all dependencies of the given packages. This will mutate the object passed in. """ - pyodide_packages_info = orig_pyodide_lock()["packages"] + if pyodide_packages_info is None: + pyodide_packages_info = _to_basic_package_info(orig_pyodide_lock()["packages"]) + i = 0 while i < len(pkgs): pkg_info = pkgs[list(pkgs.keys())[i]] @@ -506,6 +528,35 @@ def _dep_filter(dep_str: str) -> bool: return list(res) +def _to_basic_package_info( + packages: dict[str, PyodidePackageInfo] | dict[str, LockfilePackageInfo] +) -> dict[str, BasicPackageInfo]: + """ + Convert Pyodide package information to a simplified format. + + Args: + packages (dict[str, PyodidePackageInfo]): Original package information from Pyodide lock file + + Returns: + dict[str, BasicPackageInfo]: Simplified package information with name and depends + """ + basic_info: dict[str, BasicPackageInfo] = {} + for pkg_name, pkg_data in packages.items(): + depends = pkg_data["depends"] + depends_str_list: list[str] = [] + for dep in depends: + if isinstance(dep, str): + depends_str_list.append(dep) + else: + depends_str_list.append(dep["name"]) + + basic_info[pkg_name.lower()] = { + "name": pkg_name, + "depends": depends_str_list, + } + return basic_info + + # ============================================================================= # Functions for copying and downloading the wheel files. # ============================================================================= diff --git a/shinylive_lock.json b/shinylive_lock.json index 2aed2375..af84abf1 100644 --- a/shinylive_lock.json +++ b/shinylive_lock.json @@ -15,21 +15,23 @@ }, "shiny": { "name": "shiny", - "version": "1.1.0", - "filename": "shiny-1.1.0-py3-none-any.whl", + "version": "1.1.1.dev49", + "filename": "shiny-1.1.1.dev49-py3-none-any.whl", "sha256": null, "url": null, "depends": [ {"name": "typing-extensions", "specs": [[">=", "4.10.0"]]}, {"name": "starlette", "specs": []}, {"name": "python-multipart", "specs": []}, - {"name": "htmltools", "specs": [[">=", "0.5.2"]]}, + {"name": "htmltools", "specs": []}, {"name": "markdown-it-py", "specs": [[">=", "1.1.0"]]}, {"name": "mdit-py-plugins", "specs": [[">=", "0.3.0"]]}, {"name": "linkify-it-py", "specs": [[">=", "1.0"]]}, {"name": "appdirs", "specs": [[">=", "1.4.4"]]}, {"name": "asgiref", "specs": [[">=", "3.5.2"]]}, - {"name": "packaging", "specs": [[">=", "20.9"]]} + {"name": "packaging", "specs": [[">=", "20.9"]]}, + {"name": "narwhals", "specs": [[">=", "1.10.0"]]}, + {"name": "orjson", "specs": [[">=", "3.10.7"]]} ], "imports": [ "shiny" @@ -499,5 +501,16 @@ "imports": [ "sniffio" ] + }, + "narwhals": { + "name": "narwhals", + "version": "1.11.1", + "filename": "narwhals-1.11.1-py3-none-any.whl", + "sha256": "2bcf7c1649668f3b3ad1410c52f456da0a81b725a4c33f716062d0021354f049", + "url": "https://files.pythonhosted.org/packages/83/db/7e7863772deb3d283fd0579deecdc19fcbde311a8fa169fc59bfd7700c84/narwhals-1.11.1-py3-none-any.whl", + "depends": [], + "imports": [ + "narwhals" + ] } } \ No newline at end of file