Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Correct signing order for macOS bundles #1910

Merged
merged 9 commits into from
Jul 25, 2024
11 changes: 11 additions & 0 deletions automation/src/automation/bootstraps/pyside6.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,15 @@ def main():
app = QtWidgets.QApplication(sys.argv)
main_window = {{{{ cookiecutter.class_name }}}}()
sys.exit(app.exec())
"""

def pyproject_table_briefcase_app_extra_content(self):
# Include PySide6-Addons, even though it isn't used, as a way to exercise
# signing of apps that include nested frameworks and apps. See #1891 for
# details.
return """
requires = [
"PySide6-Essentials~=6.5",
"PySide6-Addons~=6.5",
]
"""
1 change: 1 addition & 0 deletions changes/1891.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The order in which nested frameworks and apps are signed on macOS was corrected.
38 changes: 38 additions & 0 deletions src/briefcase/integrations/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,44 @@ def verify_install(cls, tools: ToolCache, **kwargs) -> File:
tools.file = File(tools=tools)
return tools.file

@classmethod
def sorted_depth_first(cls, paths):
"""Sort a list of paths, so that they appear in lexical order, with
subdirectories of a folder sorting before files in the same directory.

:param paths: The list of paths to sort.
:param reverse: Should the list be returned in reverse sorting order
mhsmith marked this conversation as resolved.
Show resolved Hide resolved
:returns: The sorted list of paths
"""

# The sort key for a path is is a list of triples. The path is split into
# constituent parts; each part is converted into a triple of:
#
# (is not a dir, is a leaf, path to this part)
#
# For example, the path "/foo/bar/whiz.txt" would have the key:
#
# [ (0, 0, "/foo"), (0, 0, "/foo/bar"), (1, 1, "/foo/bar/whiz.txt")
mhsmith marked this conversation as resolved.
Show resolved Hide resolved
#
# To see how this works, consider a comparison when sorting the contents of the
# folder /foo. All subfolders of /foo/bar will return (0, 0, "/foo/bar") as the
# second entry in the key, so they'll sort as equal, with ties being resolved by
# later elements in the key. The folder /foo/bar itself will have a key of (0,
# 1, "/foo/bar"), so it will sort *after* any subfolder content. The file
# /foo/something.txt will have the key (1, 1, "foo/something.txt"); this means
mhsmith marked this conversation as resolved.
Show resolved Hide resolved
# that files in foo will come *after* any subfolders.
def sort_key(p):
return [
(
not Path(*p.parts[:t]).is_dir(),
t == len(p.parts),
Path(*p.parts[:t]),
)
for t in range(len(p.parts) + 1)
]

return sorted(paths, key=sort_key)

def is_archive(self, filename: str | os.PathLike) -> bool:
"""Can a file be unpacked via `shutil.unpack_archive()`?

Expand Down
26 changes: 11 additions & 15 deletions src/briefcase/platforms/macOS/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -602,26 +602,22 @@ def sign_app(self, app: AppConfig, identity: SigningIdentity):
sign_targets.append(bundle_path)

# Run signing through a ThreadPoolExecutor so that they run in parallel.
# However, we need to ensure that objects are signed from the inside out
# (i.e., a folder must be signed *after* all it's contents has been
# signed). To do this, we sort the list of signing targets in reverse
# lexicographic order, and then group all the signing targets by parent.
# This sorts all the signable files into folders; and sign all files in
# a folder before sorting the next group. This ensures that longer paths
# are signed first, and all files in a folder are signed before the
# folder is signed.
# However, we need to ensure that objects are signed from the inside out (i.e.,
# a folder must be signed *after* all it's contents has been signed, and files
# in a folder must be signed *after* all subfolders in that same folder). To do
# this, we sort the list of signing targets in depth-first directory order.
mhsmith marked this conversation as resolved.
Show resolved Hide resolved
#
# NOTE: We are relying on the fact that the final iteration order
# produced by groupby() reflects the order in which groups are found in
# the input data. The documentation for groupby() says that a new break
# is created every time a new group is found in the input data; sorting
# the input in reverse order ensures that only one group is found per folder,
# and that the deepest folder is found first.
# NOTE: We are relying on the fact that the final iteration order produced by
# groupby() reflects the order in which groups are found in the input data. The
# documentation for groupby() says that a new break is created every time a new
# group is found in the input data; sorting the input in reverse order ensures
# that only one group is found per folder, and that the deepest folder is found
# first.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But it doesn't guarantee that subfolders and files in the same folder are put into separate groups. For example, if there was no Helpers directory, the group separation would be:

.../Qt/lib/QtWebEngineCore.framework/Versions/A/QtWebEngineProcess.app/Contents/MacOS/QtWebEngineProcess
---
.../Qt/lib/QtWebEngineCore.framework/Versions/A/QtWebEngineProcess.app
.../Qt/lib/QtWebEngineCore.framework/Versions/A/QtWebEngineCore

Because the last two paths have the same parent, they would be signed in parallel.

Possible solution: make the key for both sorting and grouping be (path.parent, path.is_dir()), and sort with reverse=True. Those two elements map directly onto the two requirements stated in the comment.

If the grouping was factored out into a utility function, it could be covered by a test.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch - I hadn't considered that although the sort order will put the directory first, if they're grouped together parallelism won't guarantee their signed in that order.

I've modified the grouping as you've described (well - close - the sorting already puts directories before files, so we just need a different grouping key, not a full reverse sort order).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WIth reverse=True, I think the sorting and grouping key can both be (path.parent, path.is_dir()), and all the complexity of sorted_depth_first can be removed.

Copy link
Member

@mhsmith mhsmith Jul 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ordering within the group shouldn't matter, but to make the build a bit more reproducible (and pass the tests), it would also be a good idea to sort within each group by the full paths.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The key can't be exactly the same, because the sorting process needs to sort the actual files, whereas the grouping needs to sort on the containers of the files. So - we need to have a different key; the three-element key gives the sort order that matches a normal directory listings, so in any debug log it's going to be marginally less confusing. The 3-element key also matches the "complexity" of the actual search - sorting by "depth, then by directory vs file, then by filename" is literally what the algorithm is sorting on, rather than relying on a subtle edge effect of reverse interacting with the boolean value of is_dir().

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, so we can sort the whole list bykey=(path.parent, path.is_dir()), reverse=True, then group by the same key, then sort each group by path. That's still much easier to understand than the current version of sorted_depth_first, which creates keys which are lists of tuples.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unless the file sorting contains the actual path in the key, the ordering of files/folders inside any given group will be at least partially unpredictable. This won't matter to the signing order, but it makes testing a little harder because we can't completely guarantee the order of the output.

It also seems a little silly to me have a sort method that doesn't fully sort. Plus, we're already sorting, so requiring a second pass of sorting (in the test harness, and potentially in the usage of the final groups) seems a little silly.

However, I can't argue that the simple tuple is much easier to understand than the list of tuples, even if the resulting order isn't quite a "lexically pleasing" as the more complex option; and we can fix the filename sorting by adding the actual path to the initial sort key (using the simpler 2 part key for the grouping key).

progress_bar = self.input.progress_bar()
task_id = progress_bar.add_task("Signing App", total=len(sign_targets))
with progress_bar:
for _, names in itertools.groupby(
sorted(sign_targets, reverse=True),
self.tools.file.sorted_depth_first(sign_targets),
lambda name: name.parent,
):
with concurrent.futures.ThreadPoolExecutor() as executor:
Expand Down
60 changes: 60 additions & 0 deletions tests/integrations/file/test_File__sorted_depth_first.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import pytest

from briefcase.integrations.file import File

from ...utils import create_file


@pytest.mark.parametrize(
"files, sorted",
[
# Files in a directory are sorted lexically
(
["foo/bar/a.txt", "foo/bar/c.txt", "foo/bar/b.txt"],
["foo/bar/a.txt", "foo/bar/b.txt", "foo/bar/c.txt"],
),
# Subfolders are sorted before files in that directory; but sorted lexically in themselves
(
[
"foo/bar/b",
"foo/bar/b/aaa.txt",
"foo/bar/b/zzz.txt",
"foo/bar/b/deeper",
"foo/bar/b/deeper/deeper_db2.txt",
"foo/bar/b/deeper/deeper_db1.txt",
"foo/bar/a.txt",
"foo/bar/c.txt",
"foo/bar/e.txt",
"foo/bar/d",
"foo/bar/d/deep_d2.txt",
"foo/bar/d/deep_d1.txt",
],
[
"foo/bar/b/deeper/deeper_db1.txt",
"foo/bar/b/deeper/deeper_db2.txt",
"foo/bar/b/deeper",
"foo/bar/b/aaa.txt",
"foo/bar/b/zzz.txt",
"foo/bar/d/deep_d1.txt",
"foo/bar/d/deep_d2.txt",
"foo/bar/b",
"foo/bar/d",
"foo/bar/a.txt",
"foo/bar/c.txt",
"foo/bar/e.txt",
],
),
],
)
def test_sorted_depth_first(files, sorted, tmp_path):
# Convert the strings into paths in the temp folder
paths = [tmp_path / file_path for file_path in files]

# Create all the paths that have a suffix
for file_path in paths:
if file_path.suffix:
create_file(tmp_path / file_path, content=str(file_path))

assert File.sorted_depth_first(paths) == [
tmp_path / file_path for file_path in sorted
]
4 changes: 2 additions & 2 deletions tests/platforms/macOS/app/test_signing.py
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,6 @@ def _codesign(args, **kwargs):
# coverage we need to. However, win32 doesn't handle executable permissions
# the same as linux/unix, `unknown.binary` is identified as a signing target.
# We ignore this discrepancy for testing purposes.
assert len(output.strip("\n").split("\n")) == (7 if verbose else 1)
assert len(output.strip("\n").split("\n")) == (11 if verbose else 1)
else:
assert len(output.strip("\n").split("\n")) == (6 if verbose else 1)
assert len(output.strip("\n").split("\n")) == (10 if verbose else 1)