Merge pull request #1910 from freakboy3742/sign-order

Correct signing order for macOS bundles

automation/src/automation/bootstraps/pyside6.py

@@ -49,4 +49,15 @@ def main():
     app = QtWidgets.QApplication(sys.argv)
     main_window = {{{{ cookiecutter.class_name }}}}()
     sys.exit(app.exec())
+"""
+
+    def pyproject_table_macOS(self):
+        # Include PySide6-Addons on macOS, even though it isn't used, as a way to
+        # exercise signing of apps that include nested frameworks and apps. See #1891
+        # for details.
+        return """\
+universal_build = true
+requires = [
+    "PySide6-Addons~=6.5",
+]
 """

changes/1891.bugfix.rst

@@ -0,0 +1 @@
+The order in which nested frameworks and apps are signed on macOS was corrected.

src/briefcase/integrations/file.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
+import itertools
 import os
 import shutil
 import sys
 import tempfile
 from contextlib import suppress
 from email.message import Message
 from pathlib import Path
+from typing import TYPE_CHECKING
 from urllib.parse import urlparse
 
 import requests.exceptions as requests_exceptions
@@ -19,6 +21,9 @@
 )
 from briefcase.integrations.base import Tool, ToolCache
 
+if TYPE_CHECKING:
+    from typing import Iterable, Sequence
+
 
 class File(Tool):
     name = "file"
@@ -34,6 +39,65 @@ def verify_install(cls, tools: ToolCache, **kwargs) -> File:
         tools.file = File(tools=tools)
         return tools.file
 
+    @classmethod
+    def sorted_depth_first(cls, paths: Sequence[Path]) -> Iterable[Path]:
+        """Sort a list of paths, so that they appear in lexical order, with
+        subdirectories of a folder sorting before files in the same directory.
+
+        :param paths: The list of paths to sort.
+        :returns: The sorted list of paths
+        """
+        # The sort key for a path is a triple - the parent of the path; whether the path
+        # is a directory or not; and finally, the actual path. We then sort this in
+        # reverse order.
+        #
+        # Sorting by the parent of the path first (in reverse order) guarantees that
+        # long paths are sorted first; so a file in a folder will come *after* the
+        # folder it is in.
+        #
+        # The second term (parent.is_dir()) guarantees that subfolders in a folder are
+        # found before files in the same folder (because in reverse order, booleans sort
+        # True before False).
+        #
+        # Lastly, we sort on the actual path itself. This provide a guaranteed lexical
+        # ordering inside any given folder. This isn't strictly required, but it's
+        # helpful for testing and reproducibility that the sort order is reliable and
+        # repeatable.
+        return sorted(paths, key=lambda p: (p.parent, p.is_dir(), p), reverse=True)
+
+    @classmethod
+    def sorted_depth_first_groups(
+        cls,
+        paths: Sequence[Path],
+    ) -> Iterable[Iterable[Path]]:
+        """Convert a list of paths into a collection of groups that are all at the same
+        "level", grouped depth-first.
+
+        Subfolders in a folder will be in a separate group, returned *before* files in
+        the same folder.
+
+        :param paths: The list of paths return grouped.
+        :returns: A generator returning a iterable groups of paths that are all at the
+            same sorting level.
+        """
+        # The sort function guarantees depth first ordering, with folders before files
+        # at the same level.
+        #
+        # The grouping key is based on the parent of the path (so that all objects in
+        # the same folder group together), with an additional discriminator to ensure
+        # that subfolders are in a different group to files.
+        #
+        # itertools.groupby guarantees a new group whenever the key changes; and the
+        # input sort order guarantees that directories come before files, so we get
+        # our desired sort grouping and ordering.
+        return (
+            group_paths
+            for _, group_paths in itertools.groupby(
+                cls.sorted_depth_first(paths),
+                lambda path: (path.parent, path.is_dir()),
+            )
+        )
+
     def is_archive(self, filename: str | os.PathLike) -> bool:
         """Can a file be unpacked via `shutil.unpack_archive()`?
 

src/briefcase/platforms/macOS/__init__.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import concurrent.futures
-import itertools
 import os
 import plistlib
 import re
@@ -601,32 +600,33 @@ def sign_app(self, app: AppConfig, identity: SigningIdentity):
         # Sign the bundle path itself
         sign_targets.append(bundle_path)
 
-        # Run signing through a ThreadPoolExecutor so that they run in parallel.
-        # However, we need to ensure that objects are signed from the inside out
-        # (i.e., a folder must be signed *after* all it's contents has been
-        # signed). To do this, we sort the list of signing targets in reverse
-        # lexicographic order, and then group all the signing targets by parent.
-        # This sorts all the signable files into folders; and sign all files in
-        # a folder before sorting the next group. This ensures that longer paths
-        # are signed first, and all files in a folder are signed before the
-        # folder is signed.
+        # Run signing through a ThreadPoolExecutor so that they run in parallel. We need
+        # to ensure that objects are signed from the inside out (i.e., a folder must be
+        # signed *after* all it's contents has been signed; files in a folder must be
+        # signed *after* all subfolders in that same folder); and an app that uses a
+        # library must be signed *after* all the libraries it uses. See
+        # https://developer.apple.com/documentation/xcode/creating-distribution-signed-code-for-the-mac#Determine-the-signing-order
+        # for details.
         #
-        # NOTE: We are relying on the fact that the final iteration order
-        # produced by groupby() reflects the order in which groups are found in
-        # the input data. The documentation for groupby() says that a new break
-        # is created every time a new group is found in the input data; sorting
-        # the input in reverse order ensures that only one group is found per folder,
-        # and that the deepest folder is found first.
+        # To do this, we utilize grouping on a sorted list, and rely on the fact that a
+        # new group is created whenever the grouping key changes. The sorting process
+        # guarantees depth-first ordering; the grouping is applied over the sorted
+        # content, ensuring that that folders are signed before files.
+        #
+        # This approach isn't perfect. It will fail if there's an embedded app that uses
+        # a library in a different framework, and the app is lexically sorted before the
+        # library (e.g. if app_packages/foobar/Alpha.framework/Helpers/My App.app
+        # depends on app_packages/footbar/Beta.framework/libbeta). However, if you're
+        # embedding apps in frameworks in Python libraries that are installed by pip,
+        # you're already making poor life choices; this approach is enough to satisfy
+        # the one use of app-in-framework embedding that we're aware of (PySide).
         progress_bar = self.input.progress_bar()
         task_id = progress_bar.add_task("Signing App", total=len(sign_targets))
         with progress_bar:
-            for _, names in itertools.groupby(
-                sorted(sign_targets, reverse=True),
-                lambda name: name.parent,
-            ):
+            for group in self.tools.file.sorted_depth_first_groups(sign_targets):
                 with concurrent.futures.ThreadPoolExecutor() as executor:
                     futures = []
-                    for path in names:
+                    for path in group:
                         future = executor.submit(
                             self.sign_file,
                             path,

tests/integrations/file/test_File__sorted_depth_first.py

@@ -0,0 +1,75 @@
+import pytest
+
+from briefcase.integrations.file import File
+
+from ...utils import create_file
+
+
+@pytest.mark.parametrize(
+    "files, sorted",
+    [
+        # Files in a directory are sorted lexically
+        (
+            ["foo/bar/a.txt", "foo/bar/c.txt", "foo/bar/b.txt"],
+            ["foo/bar/c.txt", "foo/bar/b.txt", "foo/bar/a.txt"],
+        ),
+        # Subfolders are sorted before files in that directory; but sorted lexically in themselves
+        (
+            [
+                "foo/bar/b",
+                "foo/bar/b/aaa.txt",
+                "foo/bar/b/zzz.txt",
+                "foo/bar/b/deeper",
+                "foo/bar/b/deeper/deeper_db2.txt",
+                "foo/bar/b/deeper/deeper_db1.txt",
+                "foo/bar/a.txt",
+                "foo/bar/c.txt",
+                "foo/bar/e.txt",
+                "foo/bar/d",
+                "foo/bar/d/deep_d2.txt",
+                "foo/bar/d/deep_d1.txt",
+            ],
+            [
+                "foo/bar/d/deep_d2.txt",
+                "foo/bar/d/deep_d1.txt",
+                "foo/bar/b/deeper/deeper_db2.txt",
+                "foo/bar/b/deeper/deeper_db1.txt",
+                "foo/bar/b/deeper",
+                "foo/bar/b/zzz.txt",
+                "foo/bar/b/aaa.txt",
+                "foo/bar/d",
+                "foo/bar/b",
+                "foo/bar/e.txt",
+                "foo/bar/c.txt",
+                "foo/bar/a.txt",
+            ],
+        ),
+        # If a folder contains both folders and files, the folders are returned first.
+        (
+            [
+                "foo/bar/a",
+                "foo/bar/b.txt",
+                "foo/bar/c",
+            ],
+            [
+                "foo/bar/c",
+                "foo/bar/a",
+                "foo/bar/b.txt",
+            ],
+        ),
+    ],
+)
+def test_sorted_depth_first(files, sorted, tmp_path):
+    # Convert the strings into paths in the temp folder
+    paths = [tmp_path / file_path for file_path in files]
+
+    # Create all the paths that have a suffix
+    for file_path in paths:
+        if file_path.suffix:
+            create_file(tmp_path / file_path, content=str(file_path))
+        else:
+            file_path.mkdir(parents=True, exist_ok=True)
+
+    assert File.sorted_depth_first(paths) == [
+        tmp_path / file_path for file_path in sorted
+    ]

tests/integrations/file/test_File__sorted_depth_first_groups.py

@@ -0,0 +1,78 @@
+import pytest
+
+from briefcase.integrations.file import File
+
+from ...utils import create_file
+
+
+@pytest.mark.parametrize(
+    "files, groups",
+    [
+        # Files in a single directory are sorted lexically into the same group
+        (
+            ["foo/bar/a.txt", "foo/bar/c.txt", "foo/bar/b.txt"],
+            [
+                ["foo/bar/c.txt", "foo/bar/b.txt", "foo/bar/a.txt"],
+            ],
+        ),
+        # Subfolders are sorted before files in that directory, and in a different
+        # group; but sorted lexically in themselves
+        (
+            [
+                "foo/bar/b",
+                "foo/bar/b/aaa.txt",
+                "foo/bar/b/zzz.txt",
+                "foo/bar/b/deeper",
+                "foo/bar/b/deeper/deeper_db2.txt",
+                "foo/bar/b/deeper/deeper_db1.txt",
+                "foo/bar/a.txt",
+                "foo/bar/c.txt",
+                "foo/bar/e.txt",
+                "foo/bar/d",
+                "foo/bar/d/deep_d2.txt",
+                "foo/bar/d/deep_d1.txt",
+            ],
+            [
+                ["foo/bar/d/deep_d2.txt", "foo/bar/d/deep_d1.txt"],
+                ["foo/bar/b/deeper/deeper_db2.txt", "foo/bar/b/deeper/deeper_db1.txt"],
+                ["foo/bar/b/deeper"],
+                ["foo/bar/b/zzz.txt", "foo/bar/b/aaa.txt"],
+                ["foo/bar/d", "foo/bar/b"],
+                ["foo/bar/e.txt", "foo/bar/c.txt", "foo/bar/a.txt"],
+            ],
+        ),
+        # If a folder contains both folders and files, the folders are in a different
+        # group the files, and are returned first.
+        (
+            [
+                "foo/bar/a",
+                "foo/bar/b.txt",
+                "foo/bar/c",
+            ],
+            [
+                [
+                    "foo/bar/c",
+                    "foo/bar/a",
+                ],
+                [
+                    "foo/bar/b.txt",
+                ],
+            ],
+        ),
+    ],
+)
+def test_sorted_depth_first_groups(files, groups, tmp_path):
+    # Convert the strings into paths in the temp folder
+    paths = [tmp_path / file_path for file_path in files]
+
+    # Create files for all paths that have a suffix; otherwise, ensure the folder
+    # exists.
+    for file_path in paths:
+        if file_path.suffix:
+            create_file(tmp_path / file_path, content=str(file_path))
+        else:
+            file_path.mkdir(parents=True, exist_ok=True)
+
+    assert list(list(group) for group in File.sorted_depth_first_groups(paths)) == [
+        [tmp_path / file_path for file_path in group] for group in groups
+    ]

tests/platforms/macOS/app/test_signing.py

@@ -713,6 +713,6 @@ def _codesign(args, **kwargs):
         # coverage we need to. However, win32 doesn't handle executable permissions
         # the same as linux/unix, `unknown.binary` is identified as a signing target.
         # We ignore this discrepancy for testing purposes.
-        assert len(output.strip("\n").split("\n")) == (7 if verbose else 1)
+        assert len(output.strip("\n").split("\n")) == (9 if verbose else 1)
     else:
-        assert len(output.strip("\n").split("\n")) == (6 if verbose else 1)
+        assert len(output.strip("\n").split("\n")) == (8 if verbose else 1)