diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 910c817c8..22c3fa868 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,9 @@ repos: hooks: - id: ruff stages: [ push, commit, manual ] + # Run the formatter. + - id: ruff-format + stages: [ push, commit, manual ] - repo: https://github.com/pycqa/flake8 rev: '7.0.0' # pick a git hash / tag to point to hooks: diff --git a/pycheribuild/__main__.py b/pycheribuild/__main__.py index 5ec2e6f34..cf77654cc 100644 --- a/pycheribuild/__main__.py +++ b/pycheribuild/__main__.py @@ -85,19 +85,24 @@ def _update_check(config: DefaultCheriConfig, d: Path) -> None: run_command(["git", "fetch"], cwd=project_dir, timeout=5, config=config) branch_info = GitRepository.get_branch_info(d, config=config) if branch_info is not None and branch_info.upstream_branch == "master": - if query_yes_no(config, f"The local {branch_info.local_branch} branch is tracking the obsolete remote 'master'" - f" branch, would you like to switch to 'main'?", force_result=False): + if query_yes_no( + config, + f"The local {branch_info.local_branch} branch is tracking the obsolete remote 'master'" + " branch, would you like to switch to 'main'?", + force_result=False, + ): # Update the remote ref to point to "main". run_command("git", "branch", f"--set-upstream-to={branch_info.remote_name}/main", cwd=d, config=config) if branch_info.local_branch == "master": # And rename master to main if possible. run_command("git", "branch", "-m", "main", cwd=d, allow_unexpected_returncode=True, config=config) - output = run_command(["git", "status", "-uno"], cwd=project_dir, config=config, capture_output=True, - print_verbose_only=True).stdout + output = run_command( + ["git", "status", "-uno"], cwd=project_dir, config=config, capture_output=True, print_verbose_only=True + ).stdout behind_index = output.find(b"Your branch is behind ") if behind_index > 0: - msg_end = output.find(b"\n (use \"git pull\" to update your local branch)") + msg_end = output.find(b'\n (use "git pull" to update your local branch)') if msg_end > 0: output = output[behind_index:msg_end] status_update("Current", d.name, "checkout can be updated:", output.decode("utf-8")) @@ -122,16 +127,21 @@ def ensure_fd_is_blocking(fd) -> None: def check_not_root() -> None: if os.geteuid() == 0: - fatal_error("You are running cheribuild as root. This is dangerous, bad practice and can cause builds to fail." - " Please re-run as a non-root user.", pretend=False) + fatal_error( + "You are running cheribuild as root. This is dangerous, bad practice and can cause builds to fail." + " Please re-run as a non-root user.", + pretend=False, + ) # noinspection PyProtectedMember def get_config_option_value(handle: ConfigOptionHandle, config: DefaultCheriConfig) -> str: option = handle._get_option() if option.is_fallback_only: - raise LookupError(f"Option '{option.full_option_name}' cannot be queried since it is a generic fallback value" - f"for a target-specific option. Please use the target-suffixed on instead.") + raise LookupError( + f"Option '{option.full_option_name}' cannot be queried since it is a generic fallback value" + "for a target-specific option. Please use the target-suffixed on instead." + ) if option._owning_class is not None: project_cls: "type[SimpleProject]" = option._owning_class Target.instantiating_targets_should_warn = False @@ -199,8 +209,9 @@ def real_main() -> None: fatal_error(*e.args, pretend=False) sys.exit() - assert any(x in cheri_config.action for x in (CheribuildAction.TEST, CheribuildAction.BUILD, - CheribuildAction.BENCHMARK)) + assert any( + x in cheri_config.action for x in (CheribuildAction.TEST, CheribuildAction.BUILD, CheribuildAction.BENCHMARK) + ) if cheri_config.docker: cheribuild_dir = str(Path(__file__).absolute().parent.parent) @@ -223,12 +234,16 @@ def real_main() -> None: try: docker_dir_mappings = [ # map cheribuild and the sources read-only into the container - "-v", cheribuild_dir + ":/cheribuild:ro", - "-v", str(cheri_config.source_root.absolute()) + ":/source", + "-v", + cheribuild_dir + ":/cheribuild:ro", + "-v", + str(cheri_config.source_root.absolute()) + ":/source", # build and output are read-write: - "-v", str(cheri_config.build_root.absolute()) + ":/build", - "-v", str(cheri_config.output_root.absolute()) + ":/output", - ] + "-v", + str(cheri_config.build_root.absolute()) + ":/build", + "-v", + str(cheri_config.output_root.absolute()) + ":/output", + ] cheribuild_args = ["/cheribuild/cheribuild.py", "--skip-update", *filtered_cheribuild_args] if cheri_config.docker_reuse_container: # Use docker restart + docker exec instead of docker run @@ -241,8 +256,16 @@ def real_main() -> None: subprocess.check_call(start_cmd) docker_run_cmd = ["docker", "exec", cheri_config.docker_container, *cheribuild_args] else: - docker_run_cmd = ["docker", "run", "--user", str(os.getuid()) + ":" + str(os.getgid()), "--rm", - "--interactive", "--tty", *docker_dir_mappings] + docker_run_cmd = [ + "docker", + "run", + "--user", + str(os.getuid()) + ":" + str(os.getgid()), + "--rm", + "--interactive", + "--tty", + *docker_dir_mappings, + ] docker_run_cmd += [cheri_config.docker_container, *cheribuild_args] run_command(docker_run_cmd, config=cheri_config, give_tty_control=True) except subprocess.CalledProcessError as e: @@ -251,8 +274,10 @@ def real_main() -> None: status_update("It seems like the docker image", cheri_config.docker_container, "was not found.") status_update("In order to build the default docker image for cheribuild (cheribuild-docker) run:") print( - coloured(AnsiColour.blue, "cd", - cheribuild_dir + "/docker && docker build --tag cheribuild-docker .")) + coloured( + AnsiColour.blue, "cd", cheribuild_dir + "/docker && docker build --tag cheribuild-docker ." + ) + ) sys.exit(coloured(AnsiColour.red, "Failed to start docker!")) raise sys.exit() @@ -287,17 +312,27 @@ def real_main() -> None: print("Failed to check for updates:", e) # no-combine chosen_targets = target_manager.get_all_chosen_targets(cheri_config) if cheri_config.print_targets_only: - print("Will execute the following", len(chosen_targets), "targets:\n ", - "\n ".join(t.name for t in chosen_targets)) + print( + "Will execute the following", + len(chosen_targets), + "targets:\n ", + "\n ".join(t.name for t in chosen_targets), + ) # If --verbose is passed, also print the dependencies for each target if cheri_config.verbose: needed_by = {k.name: [] for k in chosen_targets} direct_deps = dict() for target in chosen_targets: # noinspection PyProtectedMember - direct_deps[target.name] = [t.name for t in target.project_class._direct_dependencies( - cheri_config, include_sdk_dependencies=True, include_toolchain_dependencies=True, - explicit_dependencies_only=False)] + direct_deps[target.name] = [ + t.name + for t in target.project_class._direct_dependencies( + cheri_config, + include_sdk_dependencies=True, + include_toolchain_dependencies=True, + explicit_dependencies_only=False, + ) + ] for dep in direct_deps[target.name]: needed_by[dep].append(target.name) for target in chosen_targets: @@ -321,7 +356,7 @@ def main() -> None: except Exception as e: # If we are currently debugging, raise the exception to allow e.g. PyCharm's # "break on exception that terminates execution" feature works. - debugger_attached = getattr(sys, 'gettrace', lambda: None)() is not None + debugger_attached = getattr(sys, "gettrace", lambda: None)() is not None if debugger_attached: raise e else: diff --git a/pycheribuild/boot_cheribsd/__init__.py b/pycheribuild/boot_cheribsd/__init__.py index 32955ca9d..0e8e6c420 100755 --- a/pycheribuild/boot_cheribsd/__init__.py +++ b/pycheribuild/boot_cheribsd/__init__.py @@ -64,14 +64,18 @@ assert str(_pexpect_dir.resolve()) in sys.path, str(_pexpect_dir) + " not found in " + str(sys.path) import pexpect # noqa: E402 -SUPPORTED_ARCHITECTURES = {x.generic_target_suffix: x for x in (CompilationTargets.CHERIBSD_RISCV_NO_CHERI, - CompilationTargets.CHERIBSD_RISCV_HYBRID, - CompilationTargets.CHERIBSD_RISCV_PURECAP, - CompilationTargets.CHERIBSD_X86_64, - CompilationTargets.CHERIBSD_AARCH64, - CompilationTargets.CHERIBSD_MORELLO_HYBRID, - CompilationTargets.CHERIBSD_MORELLO_PURECAP, - )} +SUPPORTED_ARCHITECTURES = { + x.generic_target_suffix: x + for x in ( + CompilationTargets.CHERIBSD_RISCV_NO_CHERI, + CompilationTargets.CHERIBSD_RISCV_HYBRID, + CompilationTargets.CHERIBSD_RISCV_PURECAP, + CompilationTargets.CHERIBSD_X86_64, + CompilationTargets.CHERIBSD_AARCH64, + CompilationTargets.CHERIBSD_MORELLO_HYBRID, + CompilationTargets.CHERIBSD_MORELLO_PURECAP, + ) +} # boot loader without lua: "Hit [Enter] to boot " # menu.lua before Sep 2019: ", hit [Enter] to boot " @@ -164,12 +168,20 @@ def interact(self, escape_character=chr(29), input_filter=None, output_filter=No info("Interacting with (fake) ", coloured(AnsiColour.yellow, commandline_to_str(self.cmd))) def sendcontrol(self, char): - info("Sending ", coloured(AnsiColour.yellow, "CTRL+", char), coloured(AnsiColour.blue, " to (fake) "), - coloured(AnsiColour.yellow, commandline_to_str(self.cmd))) - - def sendline(self, s=''): - info("Sending ", coloured(AnsiColour.yellow, s), coloured(AnsiColour.blue, " to (fake) "), - coloured(AnsiColour.yellow, commandline_to_str(self.cmd))) + info( + "Sending ", + coloured(AnsiColour.yellow, "CTRL+", char), + coloured(AnsiColour.blue, " to (fake) "), + coloured(AnsiColour.yellow, commandline_to_str(self.cmd)), + ) + + def sendline(self, s=""): + info( + "Sending ", + coloured(AnsiColour.yellow, s), + coloured(AnsiColour.blue, " to (fake) "), + coloured(AnsiColour.yellow, commandline_to_str(self.cmd)), + ) super().sendline(s) @@ -230,32 +242,55 @@ class CheriBSDSpawnMixin(MixinBase): def expect_exact_ignore_panic(self, patterns, *, timeout: int): return super().expect_exact(patterns, timeout=timeout) - def expect(self, patterns: PatternListType, timeout=-1, pretend_result=None, ignore_timeout=False, - log_patterns=True, timeout_msg="timeout", **kwargs): + def expect( + self, + patterns: PatternListType, + timeout=-1, + pretend_result=None, + ignore_timeout=False, + log_patterns=True, + timeout_msg="timeout", + **kwargs, + ): assert isinstance(patterns, list), "expected list and not " + str(patterns) if log_patterns: info("Expecting regex ", coloured(AnsiColour.cyan, str(patterns))) - return self._expect_and_handle_panic_impl(patterns, timeout_msg, ignore_timeout=ignore_timeout, - timeout=timeout, expect_fn=super().expect, **kwargs) - - def expect_exact(self, pattern_list: PatternListType, - timeout=-1, pretend_result=None, ignore_timeout=False, log_patterns=True, timeout_msg="timeout", - **kwargs): + return self._expect_and_handle_panic_impl( + patterns, timeout_msg, ignore_timeout=ignore_timeout, timeout=timeout, expect_fn=super().expect, **kwargs + ) + + def expect_exact( + self, + pattern_list: PatternListType, + timeout=-1, + pretend_result=None, + ignore_timeout=False, + log_patterns=True, + timeout_msg="timeout", + **kwargs, + ): assert isinstance(pattern_list, list), "expected list and not " + str(pattern_list) if log_patterns: info("Expecting literal ", coloured(AnsiColour.blue, str(pattern_list))) - return self._expect_and_handle_panic_impl(pattern_list, timeout_msg, timeout=timeout, - ignore_timeout=ignore_timeout, expect_fn=super().expect_exact, - **kwargs) + return self._expect_and_handle_panic_impl( + pattern_list, + timeout_msg, + timeout=timeout, + ignore_timeout=ignore_timeout, + expect_fn=super().expect_exact, + **kwargs, + ) def expect_prompt(self, timeout=-1, timeout_msg="timeout waiting for prompt", ignore_timeout=False, **kwargs): - result = self.expect_exact([PEXPECT_PROMPT], timeout=timeout, timeout_msg=timeout_msg, - ignore_timeout=ignore_timeout, **kwargs) + result = self.expect_exact( + [PEXPECT_PROMPT], timeout=timeout, timeout_msg=timeout_msg, ignore_timeout=ignore_timeout, **kwargs + ) time.sleep(0.05) # give QEMU a bit of time after printing the prompt (otherwise we might lose some input) return result - def _expect_and_handle_panic_impl(self, options: PatternListType, timeout_msg, *, ignore_timeout=True, - expect_fn, timeout, **kwargs): + def _expect_and_handle_panic_impl( + self, options: PatternListType, timeout_msg, *, ignore_timeout=True, expect_fn, timeout, **kwargs + ): panic_regexes = [PANIC, STOPPED, PANIC_KDB, PANIC_PAGE_FAULT, PANIC_MORELLO_CAP_ABORT, PANIC_IN_BACKTRACE] for i in panic_regexes: assert i not in options @@ -275,15 +310,32 @@ def _expect_and_handle_panic_impl(self, options: PatternListType, timeout_msg, * else: raise e - def run(self, cmd: str, *, expected_output=None, error_output=None, cheri_trap_fatal=True, ignore_cheri_trap=False, - timeout=600): - run_cheribsd_command(self, cmd, expected_output=expected_output, error_output=error_output, - cheri_trap_fatal=cheri_trap_fatal, ignore_cheri_trap=ignore_cheri_trap, timeout=timeout) - - def checked_run(self, cmd: str, *, timeout=600, ignore_cheri_trap=False, error_output: "Optional[str]" = None, - **kwargs): - checked_run_cheribsd_command(self, cmd, timeout=timeout, ignore_cheri_trap=ignore_cheri_trap, - error_output=error_output, **kwargs) + def run( + self, + cmd: str, + *, + expected_output=None, + error_output=None, + cheri_trap_fatal=True, + ignore_cheri_trap=False, + timeout=600, + ): + run_cheribsd_command( + self, + cmd, + expected_output=expected_output, + error_output=error_output, + cheri_trap_fatal=cheri_trap_fatal, + ignore_cheri_trap=ignore_cheri_trap, + timeout=timeout, + ) + + def checked_run( + self, cmd: str, *, timeout=600, ignore_cheri_trap=False, error_output: "Optional[str]" = None, **kwargs + ): + checked_run_cheribsd_command( + self, cmd, timeout=timeout, ignore_cheri_trap=ignore_cheri_trap, error_output=error_output, **kwargs + ) class CheriBSDInstance(CheriBSDSpawnMixin, pexpect.spawn): @@ -298,8 +350,7 @@ class QemuCheriBSDInstance(CheriBSDInstance): smb_dirs: "list[SmbMount]" = None flush_interval = None - def __init__(self, qemu_config: QemuOptions, *args, ssh_port: Optional[int], - ssh_pubkey: Optional[Path], **kwargs): + def __init__(self, qemu_config: QemuOptions, *args, ssh_port: Optional[int], ssh_pubkey: Optional[Path], **kwargs): super().__init__(qemu_config.xtarget, *args, **kwargs) self.qemu_config = qemu_config self.should_quit = False @@ -315,35 +366,61 @@ def __init__(self, qemu_config: QemuOptions, *args, ssh_port: Optional[int], @property def ssh_private_key(self): if self._ssh_private_key is None: - failure("Attempted to use SSH without specifying a key, please pass --test-ssh-key=/path/to/id_foo.pub to " - "cheribuild.", exit=True) + failure( + "Attempted to use SSH without specifying a key, please pass --test-ssh-key=/path/to/id_foo.pub to " + "cheribuild.", + exit=True, + ) assert self._ssh_private_key != self.ssh_public_key, (self._ssh_private_key, "!=", self.ssh_public_key) return self._ssh_private_key @staticmethod def _ssh_options(use_controlmaster: bool): - result = ["-o", "UserKnownHostsFile=/dev/null", - "-o", "StrictHostKeyChecking=no", - "-o", "NoHostAuthenticationForLocalhost=yes", - # "-o", "ConnectTimeout=20", - # "-o", "ConnectionAttempts=2", - ] + result = [ + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "StrictHostKeyChecking=no", + "-o", + "NoHostAuthenticationForLocalhost=yes", + # "-o", "ConnectTimeout=20", + # "-o", "ConnectionAttempts=2", + ] if use_controlmaster: # XXX: always use controlmaster for faster connections? controlmaster_dir = Path.home() / ".ssh/controlmasters" controlmaster_dir.mkdir(exist_ok=True) - result += ["-o", f"ControlPath={controlmaster_dir}/%r@%h:%p", - "-o", "ControlMaster=auto", - # Keep socket open for 10 min (600) or indefinitely (yes) - "-o", "ControlPersist=600"] + result += [ + "-o", + f"ControlPath={controlmaster_dir}/%r@%h:%p", + "-o", + "ControlMaster=auto", + # Keep socket open for 10 min (600) or indefinitely (yes) + "-o", + "ControlPersist=600", + ] return result - def run_command_via_ssh(self, command: "list[str]", *, stdout=None, stderr=None, check=True, verbose=False, - use_controlmaster=False, **kwargs) -> "subprocess.CompletedProcess[bytes]": + def run_command_via_ssh( + self, + command: "list[str]", + *, + stdout=None, + stderr=None, + check=True, + verbose=False, + use_controlmaster=False, + **kwargs, + ) -> "subprocess.CompletedProcess[bytes]": assert self.ssh_port is not None - ssh_command = ["ssh", "{user}@{host}".format(user=self.ssh_user, host="localhost"), - "-p", str(self.ssh_port), - "-i", str(self.ssh_private_key)] + ssh_command = [ + "ssh", + "{user}@{host}".format(user=self.ssh_user, host="localhost"), + "-p", + str(self.ssh_port), + "-i", + str(self.ssh_private_key), + ] if verbose: ssh_command.append("-v") ssh_command.extend(self._ssh_options(use_controlmaster=use_controlmaster)) @@ -354,8 +431,9 @@ def run_command_via_ssh(self, command: "list[str]", *, stdout=None, stderr=None, def check_ssh_connection(self, prefix="SSH connection:"): connection_test_start = datetime.datetime.utcnow() - result = self.run_command_via_ssh(["echo", "connection successful"], check=True, stdout=subprocess.PIPE, - verbose=True) + result = self.run_command_via_ssh( + ["echo", "connection successful"], check=True, stdout=subprocess.PIPE, verbose=True + ) connection_time = (datetime.datetime.utcnow() - connection_test_start).total_seconds() info(prefix, result.stdout) if result.stdout != b"connection successful\n": @@ -444,10 +522,11 @@ def is_newer(path1: Path, path2: Path): def prepend_ld_library_path(qemu: CheriBSDInstance, path: str): - qemu.run("export LD_LIBRARY_PATH=" + path + ":$LD_LIBRARY_PATH; echo \"$LD_LIBRARY_PATH\"", timeout=3) - qemu.run("export LD_64C_LIBRARY_PATH=" + path + ":$LD_64C_LIBRARY_PATH; echo \"$LD_64C_LIBRARY_PATH\"", timeout=3) - qemu.run("export LD_CHERI_LIBRARY_PATH=" + path + ":$LD_CHERI_LIBRARY_PATH; echo \"$LD_CHERI_LIBRARY_PATH\"", - timeout=3) + qemu.run("export LD_LIBRARY_PATH=" + path + ':$LD_LIBRARY_PATH; echo "$LD_LIBRARY_PATH"', timeout=3) + qemu.run("export LD_64C_LIBRARY_PATH=" + path + ':$LD_64C_LIBRARY_PATH; echo "$LD_64C_LIBRARY_PATH"', timeout=3) + qemu.run( + "export LD_CHERI_LIBRARY_PATH=" + path + ':$LD_CHERI_LIBRARY_PATH; echo "$LD_CHERI_LIBRARY_PATH"', timeout=3 + ) def set_ld_library_path_with_sysroot(qemu: CheriBSDInstance): @@ -457,8 +536,11 @@ def set_ld_library_path_with_sysroot(qemu: CheriBSDInstance): local_dir = "usr/local" if qemu.xtarget.target_info_cls.is_cheribsd(): local_dir += "/" + qemu.xtarget.generic_arch_suffix - qemu.run("export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" - "/sysroot/{prefix}/{l}:${var}".format(prefix=local_dir, l="lib", var="LD_LIBRARY_PATH"), timeout=3) + qemu.run( + "export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" + "/sysroot/{prefix}/{l}:${var}".format(prefix=local_dir, l="lib", var="LD_LIBRARY_PATH"), + timeout=3, + ) return purecap_install_prefix = "usr/local/" + qemu.xtarget.get_cheri_purecap_target().generic_arch_suffix @@ -467,21 +549,34 @@ def set_ld_library_path_with_sysroot(qemu: CheriBSDInstance): noncheri_ld_lib_path_var = "LD_LIBRARY_PATH" if not qemu.xtarget.is_cheri_purecap() else "LD_64_LIBRARY_PATH" cheri_ld_lib_path_var = "LD_LIBRARY_PATH" if qemu.xtarget.is_cheri_purecap() else "LD_64C_LIBRARY_PATH" - qemu.run("export {var}=/{lib}:/usr/{lib}:/usr/local/{lib}:/sysroot/{lib}:/sysroot/usr/{lib}:/sysroot/{hybrid}/lib:" - "/sysroot/usr/local/{lib}:/sysroot/{noncheri}/lib:${var}".format( - lib=non_cheri_libdir, hybrid=hybrid_install_prefix, noncheri=nocheri_install_prefix, - var=noncheri_ld_lib_path_var), timeout=3) - qemu.run("export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" - "/sysroot/{prefix}/lib:${var}".format(prefix=purecap_install_prefix, l=cheri_libdir, - var=cheri_ld_lib_path_var), timeout=3) + qemu.run( + "export {var}=/{lib}:/usr/{lib}:/usr/local/{lib}:/sysroot/{lib}:/sysroot/usr/{lib}:/sysroot/{hybrid}/lib:" + "/sysroot/usr/local/{lib}:/sysroot/{noncheri}/lib:${var}".format( + lib=non_cheri_libdir, + hybrid=hybrid_install_prefix, + noncheri=nocheri_install_prefix, + var=noncheri_ld_lib_path_var, + ), + timeout=3, + ) + qemu.run( + "export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" + "/sysroot/{prefix}/lib:${var}".format(prefix=purecap_install_prefix, l=cheri_libdir, var=cheri_ld_lib_path_var), + timeout=3, + ) if cheri_ld_lib_path_var == "LD_64C_LIBRARY_PATH": - qemu.run("export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" - "/sysroot/{prefix}/lib:${var}".format(prefix=purecap_install_prefix, l=cheri_libdir, - var="LD_CHERI_LIBRARY_PATH"), timeout=3) - - -def maybe_decompress(path: Path, force_decompression: bool, keep_archive=True, - args: "Optional[argparse.Namespace]" = None, *, what: str) -> Path: + qemu.run( + "export {var}=/{l}:/usr/{l}:/usr/local/{l}:/sysroot/{l}:/sysroot/usr/{l}:/sysroot/usr/local/{l}:" + "/sysroot/{prefix}/lib:${var}".format( + prefix=purecap_install_prefix, l=cheri_libdir, var="LD_CHERI_LIBRARY_PATH" + ), + timeout=3, + ) + + +def maybe_decompress( + path: Path, force_decompression: bool, keep_archive=True, args: "Optional[argparse.Namespace]" = None, *, what: str +) -> Path: # drop the suffix and then try decompressing def bunzip(archive): return decompress(archive, force_decompression, cmd=["bunzip2", "-v", "-f"], keep_archive=keep_archive) @@ -548,18 +643,30 @@ def debug_kernel_panic(qemu: CheriBSDSpawnMixin): SH_PROGRAM_NOT_FOUND = re.compile("/bin/sh: [/\\w\\d_-]+: not found") -RTLD_DSO_NOT_FOUND = re.compile("ld-elf[\\w\\d_-]*.so.1: Shared object \".+\" not found, required by \".+\"") - - -def run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, expected_output=None, error_output=None, - cheri_trap_fatal=True, ignore_cheri_trap=False, timeout=60): +RTLD_DSO_NOT_FOUND = re.compile('ld-elf[\\w\\d_-]*.so.1: Shared object ".+" not found, required by ".+"') + + +def run_cheribsd_command( + qemu: CheriBSDSpawnMixin, + cmd: str, + expected_output=None, + error_output=None, + cheri_trap_fatal=True, + ignore_cheri_trap=False, + timeout=60, +): qemu.sendline(cmd) # FIXME: allow ignoring CHERI traps if expected_output: qemu.expect([expected_output], timeout=timeout) - results = [SH_PROGRAM_NOT_FOUND, RTLD_DSO_NOT_FOUND, pexpect.TIMEOUT, - PEXPECT_PROMPT_RE, PEXPECT_CONTINUATION_PROMPT_RE] + results = [ + SH_PROGRAM_NOT_FOUND, + RTLD_DSO_NOT_FOUND, + pexpect.TIMEOUT, + PEXPECT_PROMPT_RE, + PEXPECT_CONTINUATION_PROMPT_RE, + ] error_output_index = -1 cheri_trap_indices = tuple() if error_output: @@ -597,11 +704,18 @@ def run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, expected_output=Non failure("Got CHERI TRAP!", exit=False) -def checked_run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, timeout=600, ignore_cheri_trap=False, - error_output: "Optional[str]" = None, **kwargs): +def checked_run_cheribsd_command( + qemu: CheriBSDSpawnMixin, + cmd: str, + timeout=600, + ignore_cheri_trap=False, + error_output: "Optional[str]" = None, + **kwargs, +): starttime = datetime.datetime.now() qemu.sendline( - cmd + " ;if test $? -eq 0; then echo '__COMMAND' 'SUCCESSFUL__'; else echo '__COMMAND' 'FAILED__'; fi") + cmd + " ;if test $? -eq 0; then echo '__COMMAND' 'SUCCESSFUL__'; else echo '__COMMAND' 'FAILED__'; fi" + ) cheri_trap_indices = tuple() error_output_index = None results = ["__COMMAND SUCCESSFUL__", "__COMMAND FAILED__", PEXPECT_CONTINUATION_PROMPT_RE, pexpect.TIMEOUT] @@ -622,25 +736,34 @@ def checked_run_cheribsd_command(qemu: CheriBSDSpawnMixin, cmd: str, timeout=600 elif i == 2: raise CheriBSDCommandFailed("Detected line continuation, cannot handle this yet! ", cmd, execution_time=runtime) elif i == 3: - raise CheriBSDCommandTimeout("timeout after ", runtime, " running '", cmd, "': ", str(qemu), - execution_time=runtime) + raise CheriBSDCommandTimeout( + "timeout after ", runtime, " running '", cmd, "': ", str(qemu), execution_time=runtime + ) elif i in cheri_trap_indices: # wait up to 20 seconds for a prompt to ensure the dump output has been printed qemu.expect_prompt(timeout=20, ignore_timeout=True) qemu.flush() - raise CheriBSDCommandFailed("Got CHERI trap running '", cmd, "' (after '", runtime.total_seconds(), "s)", - execution_time=runtime) + raise CheriBSDCommandFailed( + "Got CHERI trap running '", cmd, "' (after '", runtime.total_seconds(), "s)", execution_time=runtime + ) elif i == error_output_index: # wait up to 20 seconds for the shell prompt qemu.expect_prompt(timeout=20, ignore_timeout=True) qemu.flush() assert isinstance(error_output, str) - raise CheriBSDMatchedErrorOutput("Matched error output '" + error_output + "' running '", cmd, "' (after '", - runtime.total_seconds(), ")", execution_time=runtime) + raise CheriBSDMatchedErrorOutput( + "Matched error output '" + error_output + "' running '", + cmd, + "' (after '", + runtime.total_seconds(), + ")", + execution_time=runtime, + ) else: assert i < len(results), str(i) + " >= len(" + str(results) + ")" - raise CheriBSDCommandFailed("error running '", cmd, "' (after '", runtime.total_seconds(), "s)", - execution_time=runtime) + raise CheriBSDCommandFailed( + "error running '", cmd, "' (after '", runtime.total_seconds(), "s)", execution_time=runtime + ) def setup_ssh_for_root_login(qemu: QemuCheriBSDInstance): @@ -655,7 +778,7 @@ def setup_ssh_for_root_login(qemu: QemuCheriBSDInstance): ssh_pubkey_contents = pubkey.read_text(encoding="utf-8").strip() # Handle ssh-pubkeys that might be too long to send as a single line (write 150-char chunks instead): chunk_size = 150 - for part in (ssh_pubkey_contents[i:i + chunk_size] for i in range(0, len(ssh_pubkey_contents), chunk_size)): + for part in (ssh_pubkey_contents[i : i + chunk_size] for i in range(0, len(ssh_pubkey_contents), chunk_size)): qemu.run("printf %s " + shlex.quote(part) + " >> /root/.ssh/authorized_keys") # Add a final newline qemu.run("printf '\\n' >> /root/.ssh/authorized_keys") @@ -740,8 +863,10 @@ def interact(self, escape_character=chr(29), input_filter=None, output_filter=No def start_dhclient(qemu: CheriBSDSpawnMixin, network_iface: str): success("===> Setting up QEMU networking") qemu.sendline(f"ifconfig {network_iface} up && dhclient {network_iface}") - i = qemu.expect([pexpect.TIMEOUT, "DHCPACK from 10.0.2.2", "dhclient already running", - "interface ([\\w\\d]+) does not exist"], timeout=120) + i = qemu.expect( + [pexpect.TIMEOUT, "DHCPACK from 10.0.2.2", "dhclient already running", "interface ([\\w\\d]+) does not exist"], + timeout=120, + ) if i == 0: # Timeout failure("timeout awaiting dhclient ", str(qemu), exit=True) if i == 1: @@ -758,12 +883,24 @@ def start_dhclient(qemu: CheriBSDSpawnMixin, network_iface: str): qemu.expect_prompt(timeout=30) -def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kernel_image: Path, - disk_image: Optional[Path], ssh_port: Optional[int], - ssh_pubkey: Optional[Path], *, write_disk_image_changes: bool, expected_kernel_abi: str, - smp_args: "list[str]", smb_dirs: "Optional[list[SmbMount]]" = None, kernel_init_only=False, - trap_on_unrepresentable=False, skip_ssh_setup=False, bios_path: "Optional[Path]" = None, - boot_alternate_kernel_dir: "Optional[Path]" = None) -> QemuCheriBSDInstance: +def boot_cheribsd( + qemu_options: QemuOptions, + qemu_command: Optional[Path], + kernel_image: Path, + disk_image: Optional[Path], + ssh_port: Optional[int], + ssh_pubkey: Optional[Path], + *, + write_disk_image_changes: bool, + expected_kernel_abi: str, + smp_args: "list[str]", + smb_dirs: "Optional[list[SmbMount]]" = None, + kernel_init_only=False, + trap_on_unrepresentable=False, + skip_ssh_setup=False, + bios_path: "Optional[Path]" = None, + boot_alternate_kernel_dir: "Optional[Path]" = None, +) -> QemuCheriBSDInstance: user_network_args = "" if smb_dirs is None: smb_dirs = [] @@ -784,13 +921,17 @@ def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kerne bios_args = riscv_bios_arguments(qemu_options.xtarget, None) else: bios_args = [] - qemu_args = qemu_options.get_commandline(qemu_command=qemu_command, kernel_file=kernel_image, disk_image=disk_image, - bios_args=bios_args, user_network_args=user_network_args, - write_disk_image_changes=write_disk_image_changes, - add_network_device=True, - trap_on_unrepresentable=trap_on_unrepresentable, # For debugging - add_virtio_rng=True, # faster entropy gathering - ) + qemu_args = qemu_options.get_commandline( + qemu_command=qemu_command, + kernel_file=kernel_image, + disk_image=disk_image, + bios_args=bios_args, + user_network_args=user_network_args, + write_disk_image_changes=write_disk_image_changes, + add_network_device=True, + trap_on_unrepresentable=trap_on_unrepresentable, # For debugging + add_virtio_rng=True, # faster entropy gathering + ) qemu_args.extend(smp_args) kernel_commandline = [] if qemu_options.can_boot_kernel_directly and kernel_image and boot_alternate_kernel_dir: @@ -817,8 +958,16 @@ def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kerne qemu_cls = QemuCheriBSDInstance if get_global_config().pretend: qemu_cls = FakeQemuSpawn - child = qemu_cls(qemu_options, qemu_args[0], qemu_args[1:], ssh_port=ssh_port, ssh_pubkey=ssh_pubkey, - encoding="utf-8", echo=False, timeout=60) + child = qemu_cls( + qemu_options, + qemu_args[0], + qemu_args[1:], + ssh_port=ssh_port, + ssh_pubkey=ssh_pubkey, + encoding="utf-8", + echo=False, + timeout=60, + ) # child.logfile=sys.stdout.buffer child.smb_dirs = smb_dirs if QEMU_LOGFILE: @@ -842,9 +991,15 @@ def boot_cheribsd(qemu_options: QemuOptions, qemu_command: Optional[Path], kerne return child -def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=False, - network_iface: Optional[str], expected_kernel_abi_msg: Optional[str] = None, - loader_kernel_dir: "Optional[Path]" = None) -> None: +def boot_and_login( + child: CheriBSDSpawnMixin, + *, + starttime, + kernel_init_only=False, + network_iface: Optional[str], + expected_kernel_abi_msg: Optional[str] = None, + loader_kernel_dir: "Optional[Path]" = None, +) -> None: have_dhclient = False # ignore SIGINT for the python code, the child should still receive it # signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -882,8 +1037,9 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal if loader_kernel_dir: # Stop autoboot and enter console child.send("\x1b") - i = child.expect(loader_boot_prompt_messages, timeout=60, - timeout_msg="timeout before loader prompt") + i = child.expect( + loader_boot_prompt_messages, timeout=60, timeout_msg="timeout before loader prompt" + ) if i != loader_boot_prompt_messages.index(BOOT_LOADER_PROMPT): failure("failed to enter boot loader prompt after stopping autoboot", exit=True) # Fall through to BOOT_LOADER_PROMPT @@ -905,8 +1061,11 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal if i == boot_messages.index(expected_kernel_abi_msg): success(f"Booting correct kernel ABI: {expected_kernel_abi_msg}") else: - failure(f"Did not find expected kernel ABI message '{expected_kernel_abi_msg}'," - f" got '{child.match.group(0)}' instead.", exit=True) + failure( + f"Did not find expected kernel ABI message '{expected_kernel_abi_msg}'," + f" got '{child.match.group(0)}' instead.", + exit=True, + ) i = child.expect(boot_messages, timeout=10 * 60, timeout_msg="timeout mounting rootfs") if i == boot_messages.index(TRYING_TO_MOUNT_ROOT): @@ -919,28 +1078,40 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal success("===> init running (kernel startup time: ", userspace_starttime - starttime, ")") userspace_starttime = datetime.datetime.now() - boot_expect_strings: PatternListType = [LOGIN, LOGIN_AS_ROOT_MINIMAL, SHELL_OPEN, BOOT_FAILURE, - BOOT_FAILURE2, BOOT_FAILURE3] - i = child.expect([*boot_expect_strings, "DHCPACK from ", *FATAL_ERROR_MESSAGES], timeout=90 * 60, - timeout_msg="timeout awaiting login prompt") + boot_expect_strings: PatternListType = [ + LOGIN, + LOGIN_AS_ROOT_MINIMAL, + SHELL_OPEN, + BOOT_FAILURE, + BOOT_FAILURE2, + BOOT_FAILURE3, + ] + i = child.expect( + [*boot_expect_strings, "DHCPACK from ", *FATAL_ERROR_MESSAGES], + timeout=90 * 60, + timeout_msg="timeout awaiting login prompt", + ) if i == len(boot_expect_strings): # DHCPACK from have_dhclient = True success("===> got DHCPACK") # we have a network, keep waiting for the login prompt - i = child.expect(boot_expect_strings + FATAL_ERROR_MESSAGES, timeout=15 * 60, - timeout_msg="timeout awaiting login prompt") + i = child.expect( + boot_expect_strings + FATAL_ERROR_MESSAGES, timeout=15 * 60, timeout_msg="timeout awaiting login prompt" + ) if i == boot_expect_strings.index(LOGIN): success("===> got login prompt") child.sendline("root") - i = child.expect([INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=10 * 60, - timeout_msg="timeout awaiting command prompt ") # give CheriABI csh 3 minutes to start + i = child.expect( + [INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=10 * 60, timeout_msg="timeout awaiting command prompt " + ) # give CheriABI csh 3 minutes to start if i == 0: # /bin/csh prompt success("===> got csh command prompt, starting POSIX sh") # csh is weird, use the normal POSIX sh instead child.sendline("sh") - i = child.expect([INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=3 * 60, - timeout_msg="timeout starting /bin/sh") # give CheriABI sh 3 minutes to start + i = child.expect( + [INITIAL_PROMPT_CSH, INITIAL_PROMPT_SH], timeout=3 * 60, timeout_msg="timeout starting /bin/sh" + ) # give CheriABI sh 3 minutes to start if i == 0: # POSIX sh with PS1 set success("===> started POSIX sh (PS1 already set)") elif i == 1: # POSIX sh without PS1 @@ -956,8 +1127,7 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal child.expect([INITIAL_PROMPT_SH], timeout=3 * 60, timeout_msg="timeout logging in") # Note: the default shell in the minimal images is csh (but without the default prompt). child.sendline("sh") - child.expect([INITIAL_PROMPT_SH], timeout=3 * 60, - timeout_msg="timeout starting /bin/sh") + child.expect([INITIAL_PROMPT_SH], timeout=3 * 60, timeout_msg="timeout starting /bin/sh") success("===> /etc/rc completed, got command prompt") _set_pexpect_sh_prompt(child) else: # BOOT_FAILURE or FATAL_ERROR_MESSAGES @@ -978,9 +1148,13 @@ def boot_and_login(child: CheriBSDSpawnMixin, *, starttime, kernel_init_only=Fal return -def _do_test_setup(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_archives: "list[Path]", - test_ld_preload_files: "list[Path]", - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None): +def _do_test_setup( + qemu: QemuCheriBSDInstance, + args: argparse.Namespace, + test_archives: "list[Path]", + test_ld_preload_files: "list[Path]", + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, +): smb_dirs = qemu.smb_dirs setup_tests_starttime = datetime.datetime.now() # Print a backtrace and drop into the debugger on panic @@ -1005,7 +1179,8 @@ def _do_test_setup(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_ar # We can differentiate the two by checking if /boot/kernel/kernel exists since it will be missing in the minimal # image qemu.run( - "if [ ! -e /boot/kernel/kernel ]; then mkdir -p /usr/local && mount -t tmpfs -o size=300m tmpfs /usr/local; fi") + "if [ ! -e /boot/kernel/kernel ]; then mkdir -p /usr/local && mount -t tmpfs -o size=300m tmpfs /usr/local; fi" + ) # Or this: if [ "$(ls -A $DIR)" ]; then echo "Not Empty"; else echo "Empty"; fi qemu.run("if [ ! -e /opt ]; then mkdir -p /opt && mount -t tmpfs -o size=500m tmpfs /opt; fi") qemu.run("df -ih") @@ -1014,9 +1189,21 @@ def _do_test_setup(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_ar def do_scp(src, dst="/"): # CVE-2018-20685 -> Can no longer use '.' See # https://superuser.com/questions/1403473/scp-error-unexpected-filename - scp_cmd = ["scp", "-B", "-r", "-P", str(qemu.ssh_port), "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-i", str(qemu.ssh_private_key), str(src), "root@localhost:" + dst] + scp_cmd = [ + "scp", + "-B", + "-r", + "-P", + str(qemu.ssh_port), + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + str(qemu.ssh_private_key), + str(src), + "root@localhost:" + dst, + ] # use script for a fake tty to get progress output from scp if sys.platform.startswith("linux"): scp_cmd = ["script", "--quiet", "--return", "--command", " ".join(scp_cmd), "/dev/null"] @@ -1048,16 +1235,24 @@ def do_scp(src, dst="/"): mount_command = f"mount_smbfs -I 10.0.2.4 -N //10.0.2.4/qemu{index + 1} '{d.in_target}'" for trial in range(MAX_SMBFS_RETRY if not get_global_config().pretend else 1): # maximum of 3 trials try: - checked_run_cheribsd_command(qemu, mount_command, - error_output="unable to open connection: syserr = ", - pretend_result=0) + checked_run_cheribsd_command( + qemu, mount_command, error_output="unable to open connection: syserr = ", pretend_result=0 + ) qemu.smb_failed = False break except CheriBSDMatchedErrorOutput as e: # If the smbfs connection timed out try once more. This can happen when multiple libc++ test jobs are # running on the same jenkins slaves so one of them might time out - failure("QEMU SMBD failed to mount ", d.in_target, " after ", e.execution_time.total_seconds(), - " seconds. Trying ", (MAX_SMBFS_RETRY - trial - 1), " more time(s)", exit=False) + failure( + "QEMU SMBD failed to mount ", + d.in_target, + " after ", + e.execution_time.total_seconds(), + " seconds. Trying ", + (MAX_SMBFS_RETRY - trial - 1), + " more time(s)", + exit=False, + ) qemu.smb_failed = True info("Waiting for 2-10 seconds before retrying mount_smbfs...") if not get_global_config().pretend: @@ -1074,11 +1269,13 @@ def do_scp(src, dst="/"): # Ensure that the libraries exist checked_run_cheribsd_command(qemu, f"test -x '{lib}'") if ld_preload_target_paths: - checked_run_cheribsd_command(qemu, "export '{}={}'".format(args.test_ld_preload_variable, - ":".join(ld_preload_target_paths))) + checked_run_cheribsd_command( + qemu, "export '{}={}'".format(args.test_ld_preload_variable, ":".join(ld_preload_target_paths)) + ) if args.test_ld_preload_variable == "LD_64C_PRELOAD": - checked_run_cheribsd_command(qemu, "export '{}={}'".format("LD_CHERI_PRELOAD", - ":".join(ld_preload_target_paths))) + checked_run_cheribsd_command( + qemu, "export '{}={}'".format("LD_CHERI_PRELOAD", ":".join(ld_preload_target_paths)) + ) if args.extra_library_paths: prepend_ld_library_path(qemu, ":".join(args.extra_library_paths)) @@ -1089,10 +1286,14 @@ def do_scp(src, dst="/"): success("Additional test enviroment setup took ", datetime.datetime.now() - setup_tests_starttime) -def runtests(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_archives: "list[Path]", - test_ld_preload_files: "list[Path]", - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, - test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None) -> bool: +def runtests( + qemu: QemuCheriBSDInstance, + args: argparse.Namespace, + test_archives: "list[Path]", + test_ld_preload_files: "list[Path]", + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, + test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, +) -> bool: try: _do_test_setup(qemu, args, test_archives, test_ld_preload_files, test_setup_function) except KeyboardInterrupt: @@ -1127,13 +1328,13 @@ def runtests(qemu: QemuCheriBSDInstance, args: argparse.Namespace, test_archives test_command = args.test_command timeout = args.test_timeout - qemu.sendline(test_command + - " ;if test $? -eq 0; then echo 'TESTS' 'COMPLETED'; else echo 'TESTS' 'FAILED'; fi") + qemu.sendline(test_command + " ;if test $? -eq 0; then echo 'TESTS' 'COMPLETED'; else echo 'TESTS' 'FAILED'; fi") i = qemu.expect([pexpect.TIMEOUT, "TESTS COMPLETED", "TESTS UNSTABLE", "TESTS FAILED"], timeout=timeout) testtime = datetime.datetime.now() - run_tests_starttime if i == 0: # Timeout - return failure("timeout after ", testtime, "waiting for tests (command='", test_command, "'): ", str(qemu), - exit=False) + return failure( + "timeout after ", testtime, "waiting for tests (command='", test_command, "'): ", str(qemu), exit=False + ) elif i == 1 or i == 2: if i == 2: success("===> Tests completed (but with FAILURES)!") @@ -1156,64 +1357,113 @@ def default_ssh_key(): def get_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(allow_abbrev=False) - parser.add_argument("--architecture", help="CPU architecture to be used for this test", required=True, - choices=[x for x in SUPPORTED_ARCHITECTURES.keys()]) + parser.add_argument( + "--architecture", + help="CPU architecture to be used for this test", + required=True, + choices=[x for x in SUPPORTED_ARCHITECTURES.keys()], + ) parser.add_argument("--qemu-cmd", "--qemu", help="Path to QEMU (default: find matching on in $PATH)", default=None) parser.add_argument("--qemu-smp", "--smp", type=int, help="Run QEMU with SMP", default=None) parser.add_argument("--kernel", default=None) parser.add_argument("--bios", default=None) parser.add_argument("--disk-image", default=None) - parser.add_argument("--minimal-image", action="store_true", - help="Set this if tests are being run on the minimal disk image rather than the full one") + parser.add_argument( + "--minimal-image", + action="store_true", + help="Set this if tests are being run on the minimal disk image rather than the full one", + ) parser.add_argument("--extract-images-to", help="Path where the compressed images should be extracted to") parser.add_argument("--reuse-image", action="store_true") parser.add_argument("--keep-compressed-images", action="store_true", default=True, dest="keep_compressed_images") parser.add_argument("--no-keep-compressed-images", action="store_false", dest="keep_compressed_images") - parser.add_argument("--write-disk-image-changes", default=False, action="store_true", - help="Commit changes made to the disk image (by default the image is immutable)") + parser.add_argument( + "--write-disk-image-changes", + default=False, + action="store_true", + help="Commit changes made to the disk image (by default the image is immutable)", + ) parser.add_argument("--no-write-disk-image-changes", action="store_false", dest="write_disk_image_changes") - parser.add_argument("--trap-on-unrepresentable", action="store_true", - help="CHERI trap on unrepresentable caps instead of detagging") + parser.add_argument( + "--trap-on-unrepresentable", action="store_true", help="CHERI trap on unrepresentable caps instead of detagging" + ) parser.add_argument("--ssh-key", "--test-ssh-key", default=default_ssh_key()) parser.add_argument("--ssh-port", type=int, default=None) parser.add_argument("--use-smb-instead-of-ssh", action="store_true") - parser.add_argument("--smb-mount-directory", metavar="HOST_PATH:IN_TARGET", - help="Share a host directory with the QEMU guest via smb. This option can be passed multiple " - "times " - "to share more than one directory. The argument should be colon-separated as follows: " - "':'. Appending '@ro' to HOST_PATH will cause the " - "directory " - "to be mapped as a read-only smb share", action="append", - dest="smb_mount_directories", type=parse_smb_mount, default=[]) + parser.add_argument( + "--smb-mount-directory", + metavar="HOST_PATH:IN_TARGET", + help="Share a host directory with the QEMU guest via smb. This option can be passed multiple " + "times " + "to share more than one directory. The argument should be colon-separated as follows: " + "':'. Appending '@ro' to HOST_PATH will cause the " + "directory " + "to be mapped as a read-only smb share", + action="append", + dest="smb_mount_directories", + type=parse_smb_mount, + default=[], + ) parser.add_argument("--test-archive", "-t", action="append", nargs=1) parser.add_argument("--test-command", "-c") - parser.add_argument('--test-ld-preload', action="append", nargs=1, metavar='LIB', - help="Copy LIB to the guest and LD_PRELOAD it before running tests") - parser.add_argument('--extra-library-path', action="append", dest="extra_library_paths", metavar="DIR", - help="Add DIR as an additional LD_LIBRARY_PATH before running tests") - parser.add_argument('--test-ld-preload-variable', type=str, default=None, - help="The environment variable to set to LD_PRELOAD a library. should be set to either " - "LD_PRELOAD or LD_64C_PRELOAD") - parser.add_argument("--test-timeout", "-tt", type=int, default=60 * 60, - help="Timeout in seconds for running tests") + parser.add_argument( + "--test-ld-preload", + action="append", + nargs=1, + metavar="LIB", + help="Copy LIB to the guest and LD_PRELOAD it before running tests", + ) + parser.add_argument( + "--extra-library-path", + action="append", + dest="extra_library_paths", + metavar="DIR", + help="Add DIR as an additional LD_LIBRARY_PATH before running tests", + ) + parser.add_argument( + "--test-ld-preload-variable", + type=str, + default=None, + help="The environment variable to set to LD_PRELOAD a library. should be set to either " + "LD_PRELOAD or LD_64C_PRELOAD", + ) + parser.add_argument("--test-timeout", "-tt", type=int, default=60 * 60, help="Timeout in seconds for running tests") parser.add_argument("--qemu-logfile", help="File to write all interactions with QEMU to", type=Path) - parser.add_argument("--test-environment-only", action="store_true", - help="Setup mount paths + SSH for tests but don't actually run the tests (implies --interact)") - parser.add_argument("--skip-ssh-setup", action="store_true", - help="Don't start sshd on boot. Saves a few seconds of boot time if not needed.") - parser.add_argument("--pretend", "-p", action="store_true", - help="Don't actually boot CheriBSD just print what would happen") + parser.add_argument( + "--test-environment-only", + action="store_true", + help="Setup mount paths + SSH for tests but don't actually run the tests (implies --interact)", + ) + parser.add_argument( + "--skip-ssh-setup", + action="store_true", + help="Don't start sshd on boot. Saves a few seconds of boot time if not needed.", + ) + parser.add_argument( + "--pretend", "-p", action="store_true", help="Don't actually boot CheriBSD just print what would happen" + ) parser.add_argument("--interact", "-i", action="store_true") - parser.add_argument("--interact-on-kernel-panic", action="store_true", - help="Instead of exiting on kernel panic start interacting with QEMU") + parser.add_argument( + "--interact-on-kernel-panic", + action="store_true", + help="Instead of exiting on kernel panic start interacting with QEMU", + ) parser.add_argument("--test-kernel-init-only", action="store_true") parser.add_argument("--enable-coredumps", action="store_true", dest="enable_coredumps", default=False) parser.add_argument("--disable-coredumps", action="store_false", dest="enable_coredumps") - parser.add_argument("--alternate-kernel-rootfs-path", type=Path, default=None, - help="Path relative to the disk image pointing to the directory " + - "containing the alternate kernel to run and related kernel modules") - parser.add_argument("--expected-kernel-abi", choices=["any", "hybrid", "purecap"], default="any", - help="The kernel kind that is expected ('any' to skip checks)") + parser.add_argument( + "--alternate-kernel-rootfs-path", + type=Path, + default=None, + help="Path relative to the disk image pointing to the directory " + + "containing the alternate kernel to run and related kernel modules", + ) + parser.add_argument( + "--expected-kernel-abi", + choices=["any", "hybrid", "purecap"], + default="any", + help="The kernel kind that is expected ('any' to skip checks)", + ) # Ensure that we don't get a race when running multiple shards: # If we extract the disk image at the same time we might spawn QEMU just between when the # value extracted by one job is unlinked and when it is replaced with a new file @@ -1222,16 +1472,19 @@ def get_argument_parser() -> argparse.ArgumentParser: return parser -def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, - argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, - argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None): +def _main( + test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, + argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, + argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None, +): parser = get_argument_parser() if argparse_setup_callback: argparse_setup_callback(parser) try: # noinspection PyUnresolvedReferences import argcomplete + argcomplete.autocomplete(parser) except ImportError: pass @@ -1287,8 +1540,10 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac test_ld_preload_files: "list[Path]" = [] if not args.use_smb_instead_of_ssh and not args.skip_ssh_setup: if args.ssh_key is None: - failure("No SSH key specified, but test script needs SSH. Please pass --test-ssh-key=/path/to/id_foo.pub", - exit=True) + failure( + "No SSH key specified, but test script needs SSH. Please pass --test-ssh-key=/path/to/id_foo.pub", + exit=True, + ) elif not Path(args.ssh_key).exists(): failure("Specified SSH key do not exist: ", args.ssh_key, exit=True) if Path(args.ssh_key).suffix != ".pub": @@ -1340,22 +1595,37 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac keep_compressed_images = False kernel = None if args.kernel is not None: - kernel = maybe_decompress(Path(args.kernel), force_decompression, keep_archive=keep_compressed_images, - args=args, what="kernel") + kernel = maybe_decompress( + Path(args.kernel), force_decompression, keep_archive=keep_compressed_images, args=args, what="kernel" + ) diskimg = None if args.disk_image: - diskimg = maybe_decompress(Path(args.disk_image), force_decompression, keep_archive=keep_compressed_images, - args=args, what="disk image") + diskimg = maybe_decompress( + Path(args.disk_image), + force_decompression, + keep_archive=keep_compressed_images, + args=args, + what="disk image", + ) boot_starttime = datetime.datetime.now() - qemu = boot_cheribsd(qemu_options, qemu_command=args.qemu_cmd, kernel_image=kernel, disk_image=diskimg, - ssh_port=args.ssh_port, ssh_pubkey=Path(args.ssh_key) if args.ssh_key is not None else None, - smb_dirs=args.smb_mount_directories, kernel_init_only=args.test_kernel_init_only, - smp_args=["-smp", str(args.qemu_smp)] if args.qemu_smp else [], - trap_on_unrepresentable=args.trap_on_unrepresentable, skip_ssh_setup=args.skip_ssh_setup, - bios_path=args.bios, write_disk_image_changes=args.write_disk_image_changes, - boot_alternate_kernel_dir=args.alternate_kernel_rootfs_path, - expected_kernel_abi=args.expected_kernel_abi) + qemu = boot_cheribsd( + qemu_options, + qemu_command=args.qemu_cmd, + kernel_image=kernel, + disk_image=diskimg, + ssh_port=args.ssh_port, + ssh_pubkey=Path(args.ssh_key) if args.ssh_key is not None else None, + smb_dirs=args.smb_mount_directories, + kernel_init_only=args.test_kernel_init_only, + smp_args=["-smp", str(args.qemu_smp)] if args.qemu_smp else [], + trap_on_unrepresentable=args.trap_on_unrepresentable, + skip_ssh_setup=args.skip_ssh_setup, + bios_path=args.bios, + write_disk_image_changes=args.write_disk_image_changes, + boot_alternate_kernel_dir=args.alternate_kernel_rootfs_path, + expected_kernel_abi=args.expected_kernel_abi, + ) success("Booting CheriBSD took: ", datetime.datetime.now() - boot_starttime) tests_okay = True @@ -1366,8 +1636,14 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac setup_ssh_starttime = datetime.datetime.now() setup_ssh_for_root_login(qemu) info("Setting up SSH took: ", datetime.datetime.now() - setup_ssh_starttime) - tests_okay = runtests(qemu, args, test_archives=test_archives, test_function=test_function, - test_setup_function=test_setup_function, test_ld_preload_files=test_ld_preload_files) + tests_okay = runtests( + qemu, + args, + test_archives=test_archives, + test_function=test_function, + test_setup_function=test_setup_function, + test_ld_preload_files=test_ld_preload_files, + ) except CheriBSDCommandFailed as e: failure("Command failed while runnings tests: ", str(e), "\n", str(qemu), exit=False) traceback.print_exc(file=sys.stderr) @@ -1398,16 +1674,22 @@ def _main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespac sys.exit(2) # different exit code for test failures -def main(test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, - test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, - argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, - argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None): +def main( + test_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], bool]]" = None, + test_setup_function: "Optional[Callable[[CheriBSDInstance, argparse.Namespace], None]]" = None, + argparse_setup_callback: "Optional[Callable[[argparse.ArgumentParser], None]]" = None, + argparse_adjust_args_callback: "Optional[Callable[[argparse.Namespace], None]]" = None, +): # Some programs (such as QEMU) can mess up the TTY state if they don't exit cleanly with keep_terminal_sane(): run_and_kill_children_on_exit( - lambda: _main(test_function=test_function, test_setup_function=test_setup_function, - argparse_setup_callback=argparse_setup_callback, - argparse_adjust_args_callback=argparse_adjust_args_callback)) + lambda: _main( + test_function=test_function, + test_setup_function=test_setup_function, + argparse_setup_callback=argparse_setup_callback, + argparse_adjust_args_callback=argparse_adjust_args_callback, + ) + ) if __name__ == "__main__": diff --git a/pycheribuild/config/chericonfig.py b/pycheribuild/config/chericonfig.py index b99a9b725..f46bef9c7 100644 --- a/pycheribuild/config/chericonfig.py +++ b/pycheribuild/config/chericonfig.py @@ -152,94 +152,158 @@ class CheribuildActionEnum(Enum): class CheriConfig(ConfigBase): def __init__(self, loader, action_class: "type[CheribuildActionEnum]") -> None: - super().__init__(pretend=DoNotUseInIfStmt(), verbose=DoNotUseInIfStmt(), quiet=DoNotUseInIfStmt(), - force=DoNotUseInIfStmt()) + super().__init__( + pretend=DoNotUseInIfStmt(), verbose=DoNotUseInIfStmt(), quiet=DoNotUseInIfStmt(), force=DoNotUseInIfStmt() + ) self._cached_deps = collections.defaultdict(dict) assert isinstance(loader, ConfigLoaderBase) loader._cheri_config = self self.loader = loader - self.pretend = loader.add_commandline_only_bool_option("pretend", "p", - help="Only print the commands instead of running them") + self.pretend = loader.add_commandline_only_bool_option( + "pretend", "p", help="Only print the commands instead of running them" + ) # add the actions: self._action_class = action_class - self.action = loader.add_option("action", default=[], action="append", type=action_class, help_hidden=True, - help="The action to perform by cheribuild", group=loader.action_group) + self.action = loader.add_option( + "action", + default=[], + action="append", + type=action_class, + help_hidden=True, + help="The action to perform by cheribuild", + group=loader.action_group, + ) self.default_action: Optional[CheribuildActionEnum] = None # Add aliases (e.g. --test = --action=test): for action in action_class: if action.altname: - loader.action_group.add_argument(action.option_name, action.altname, help=action.help_message, - dest="action", action="append_const", const=action.actions) + loader.action_group.add_argument( + action.option_name, + action.altname, + help=action.help_message, + dest="action", + action="append_const", + const=action.actions, + ) else: - loader.action_group.add_argument(action.option_name, help=action.help_message, dest="action", - action="append_const", const=action.actions) + loader.action_group.add_argument( + action.option_name, + help=action.help_message, + dest="action", + action="append_const", + const=action.actions, + ) self.print_targets_only = loader.add_commandline_only_bool_option( - "print-targets-only", help_hidden=False, group=loader.action_group, - help="Don't run the build but instead only print the targets that would be executed") - - self.clang_path = loader.add_path_option("clang-path", shortname="-cc-path", - default=lambda c, _: latest_system_clang_tool(c, "clang", "cc"), - group=loader.path_group, - help="The C compiler to use for host binaries (must be compatible " - "with Clang >= 3.7)") - self.clang_plusplus_path = loader.add_path_option("clang++-path", shortname="-c++-path", - default=lambda c, _: latest_system_clang_tool(c, "clang++", - "c++"), - group=loader.path_group, - help="The C++ compiler to use for host binaries (must be " - "compatible with Clang >= 3.7)") - self.clang_cpp_path = loader.add_path_option("clang-cpp-path", shortname="-cpp-path", - default=lambda c, _: latest_system_clang_tool(c, "clang-cpp", - "cpp"), - group=loader.path_group, - help="The C preprocessor to use for host binaries (must be " - "compatible with Clang >= 3.7)") + "print-targets-only", + help_hidden=False, + group=loader.action_group, + help="Don't run the build but instead only print the targets that would be executed", + ) + + self.clang_path = loader.add_path_option( + "clang-path", + shortname="-cc-path", + default=lambda c, _: latest_system_clang_tool(c, "clang", "cc"), + group=loader.path_group, + help="The C compiler to use for host binaries (must be compatible " "with Clang >= 3.7)", + ) + self.clang_plusplus_path = loader.add_path_option( + "clang++-path", + shortname="-c++-path", + default=lambda c, _: latest_system_clang_tool(c, "clang++", "c++"), + group=loader.path_group, + help="The C++ compiler to use for host binaries (must be " "compatible with Clang >= 3.7)", + ) + self.clang_cpp_path = loader.add_path_option( + "clang-cpp-path", + shortname="-cpp-path", + default=lambda c, _: latest_system_clang_tool(c, "clang-cpp", "cpp"), + group=loader.path_group, + help="The C preprocessor to use for host binaries (must be " "compatible with Clang >= 3.7)", + ) self.pass_dash_k_to_make = loader.add_commandline_only_bool_option( - "pass-k-to-make", "k", help="Pass the -k flag to make to continue after the first error") - self.with_libstatcounters = loader.add_bool_option("with-libstatcounters", - group=loader.cross_compile_options_group, - help="Link cross compiled CHERI project with " - "libstatcounters.") + "pass-k-to-make", "k", help="Pass the -k flag to make to continue after the first error" + ) + self.with_libstatcounters = loader.add_bool_option( + "with-libstatcounters", + group=loader.cross_compile_options_group, + help="Link cross compiled CHERI project with " "libstatcounters.", + ) self.skip_world = loader.add_bool_option( - "skip-world", "-skip-buildworld", group=loader.freebsd_group, - help="Skip the buildworld-related steps when building FreeBSD or CheriBSD") + "skip-world", + "-skip-buildworld", + group=loader.freebsd_group, + help="Skip the buildworld-related steps when building FreeBSD or CheriBSD", + ) self.skip_kernel = loader.add_bool_option( - "skip-kernel", "-skip-buildkernel", group=loader.freebsd_group, - help="Skip the buildkernel step when building FreeBSD or CheriBSD") + "skip-kernel", + "-skip-buildkernel", + group=loader.freebsd_group, + help="Skip the buildkernel step when building FreeBSD or CheriBSD", + ) self.freebsd_kernconf = loader.add_commandline_only_option( - "kernel-config", "-kernconf", group=loader.freebsd_group, help_hidden=True, - help="Override the default FreeBSD/CheriBSD kernel config.") + "kernel-config", + "-kernconf", + group=loader.freebsd_group, + help_hidden=True, + help="Override the default FreeBSD/CheriBSD kernel config.", + ) self.freebsd_subdir = loader.add_commandline_only_option( - "freebsd-subdir", "-subdir", group=loader.freebsd_group, type=list, metavar="SUBDIRS", + "freebsd-subdir", + "-subdir", + group=loader.freebsd_group, + type=list, + metavar="SUBDIRS", help="Only build subdirs SUBDIRS of FreeBSD/CheriBSD instead of the full tree. Useful for quickly " - "rebuilding individual programs/libraries. If more than one dir is passed they will be processed in " - "order. Note: This will break if not all dependencies have been built.") + "rebuilding individual programs/libraries. If more than one dir is passed they will be processed in " + "order. Note: This will break if not all dependencies have been built.", + ) self.freebsd_host_tools_only = loader.add_commandline_only_bool_option( - "freebsd-host-tools-only", help_hidden=True, group=loader.freebsd_group, - help="Stop the FreeBSD/CheriBSD build after the host tools have been built") + "freebsd-host-tools-only", + help_hidden=True, + group=loader.freebsd_group, + help="Stop the FreeBSD/CheriBSD build after the host tools have been built", + ) self.buildenv = loader.add_commandline_only_bool_option( - "buildenv", group=loader.freebsd_group, + "buildenv", + group=loader.freebsd_group, help="Open a shell with the right environment for building the project. Currently only works for " - "FreeBSD/CheriBSD") + "FreeBSD/CheriBSD", + ) self.libcompat_buildenv = loader.add_commandline_only_bool_option( - "libcompat-buildenv", "-libcheri-buildenv", group=loader.freebsd_group, - help="Open a shell with the right environment for building compat libraries.") - - self.cheri_cap_table_abi = loader.add_option("cap-table-abi", help_hidden=True, - choices=("pcrel", "plt", "fn-desc"), - help="The ABI to use for cap-table mode") - self.cross_target_suffix = loader.add_option("cross-target-suffix", help_hidden=True, default="", - help="Add a suffix to the cross build and install directories.") - self.allow_running_as_root = loader.add_bool_option("allow-running-as-root", help_hidden=True, default=False, - help="Allow running cheribuild as root (not recommended!)") + "libcompat-buildenv", + "-libcheri-buildenv", + group=loader.freebsd_group, + help="Open a shell with the right environment for building compat libraries.", + ) + + self.cheri_cap_table_abi = loader.add_option( + "cap-table-abi", + help_hidden=True, + choices=("pcrel", "plt", "fn-desc"), + help="The ABI to use for cap-table mode", + ) + self.cross_target_suffix = loader.add_option( + "cross-target-suffix", + help_hidden=True, + default="", + help="Add a suffix to the cross build and install directories.", + ) + self.allow_running_as_root = loader.add_bool_option( + "allow-running-as-root", + help_hidden=True, + default=False, + help="Allow running cheribuild as root (not recommended!)", + ) # Attributes for code completion: self.verbose: Optional[bool] = None - self.debug_output = loader.add_commandline_only_bool_option("debug-output", "vv", - help="Extremely verbose output") + self.debug_output = loader.add_commandline_only_bool_option( + "debug-output", "vv", help="Extremely verbose output" + ) self.quiet: "Optional[bool] " = None self.clean: "Optional[bool] " = None self.force: "Optional[bool] " = None @@ -250,79 +314,121 @@ def __init__(self, loader, action_class: "type[CheribuildActionEnum]") -> None: self.skip_configure: "Optional[bool] " = None self.force_configure: "Optional[bool] " = None self.force_update: "Optional[bool] " = None - self.mips_float_abi = loader.add_option("mips-float-abi", default=MipsFloatAbi.SOFT, type=MipsFloatAbi, - group=loader.cross_compile_options_group, - help="The floating point ABI to use for building MIPS+CHERI programs") + self.mips_float_abi = loader.add_option( + "mips-float-abi", + default=MipsFloatAbi.SOFT, + type=MipsFloatAbi, + group=loader.cross_compile_options_group, + help="The floating point ABI to use for building MIPS+CHERI programs", + ) self.aarch64_fp_and_simd_options = loader.add_option( - "aarch64-fp-and-simd-options", default=AArch64FloatSimdOptions.DEFAULT, type=AArch64FloatSimdOptions, + "aarch64-fp-and-simd-options", + default=AArch64FloatSimdOptions.DEFAULT, + type=AArch64FloatSimdOptions, group=loader.cross_compile_options_group, - help="The floating point/SIMD mode to use for building AArch64 programs") - self.crosscompile_linkage = loader.add_option("cross-compile-linkage", default=Linkage.DEFAULT, type=Linkage, - group=loader.cross_compile_options_group, - enum_choices=(Linkage.DEFAULT, Linkage.DYNAMIC, Linkage.STATIC), - help="Whether to link cross-compile projects static or dynamic " - "by default") - self.csetbounds_stats = loader.add_bool_option("collect-csetbounds-stats", - group=loader.cross_compile_options_group, help_hidden=True, - help="Whether to log CSetBounds statistics in csv format") - self.subobject_bounds = loader.add_option("subobject-bounds", type=str, - group=loader.cross_compile_options_group, - choices=( - "conservative", "subobject-safe", "aggressive", "very-aggressive", - "everywhere-unsafe"), - help="Whether to add additional CSetBounds to subobject " - "references/&-operator") + help="The floating point/SIMD mode to use for building AArch64 programs", + ) + self.crosscompile_linkage = loader.add_option( + "cross-compile-linkage", + default=Linkage.DEFAULT, + type=Linkage, + group=loader.cross_compile_options_group, + enum_choices=(Linkage.DEFAULT, Linkage.DYNAMIC, Linkage.STATIC), + help="Whether to link cross-compile projects static or dynamic " "by default", + ) + self.csetbounds_stats = loader.add_bool_option( + "collect-csetbounds-stats", + group=loader.cross_compile_options_group, + help_hidden=True, + help="Whether to log CSetBounds statistics in csv format", + ) + self.subobject_bounds = loader.add_option( + "subobject-bounds", + type=str, + group=loader.cross_compile_options_group, + choices=("conservative", "subobject-safe", "aggressive", "very-aggressive", "everywhere-unsafe"), + help="Whether to add additional CSetBounds to subobject " "references/&-operator", + ) self.use_cheri_ubsan = loader.add_bool_option( - "use-cheri-ubsan", group=loader.cross_compile_options_group, - help="Add compiler flags to detect certain undefined CHERI behaviour at runtime") + "use-cheri-ubsan", + group=loader.cross_compile_options_group, + help="Add compiler flags to detect certain undefined CHERI behaviour at runtime", + ) self.use_cheri_ubsan_runtime = loader.add_bool_option( - "use-cheri-ubsan-runtime", group=loader.cross_compile_options_group, default=False, + "use-cheri-ubsan-runtime", + group=loader.cross_compile_options_group, + default=False, help="Use the UBSan runtime to provide more detailed information on undefined CHERI behaviour." - "If false (the default) the compiler will generate a trap instruction instead.") - self.subobject_debug = loader.add_bool_option("subobject-debug", group=loader.cross_compile_options_group, - default=True, help_hidden=False, - help="Clear software permission bit 2 when subobject bounds " - "reduced size" - " (Note: this should be turned off for benchmarks!)") - - self.clang_colour_diags = loader.add_bool_option("clang-colour-diags", "-clang-color-diags", default=True, - help="Force CHERI clang to emit coloured diagnostics") - self.use_sdk_clang_for_native_xbuild = loader.add_bool_option("use-sdk-clang-for-native-xbuild", - group=loader.cross_compile_options_group, - help="Compile cross-compile project with CHERI " - "clang from the SDK instead of host " - "compiler") - - self.configure_only = loader.add_bool_option("configure-only", - help="Only run the configure step (skip build and install)") + "If false (the default) the compiler will generate a trap instruction instead.", + ) + self.subobject_debug = loader.add_bool_option( + "subobject-debug", + group=loader.cross_compile_options_group, + default=True, + help_hidden=False, + help="Clear software permission bit 2 when subobject bounds " + "reduced size" + " (Note: this should be turned off for benchmarks!)", + ) + + self.clang_colour_diags = loader.add_bool_option( + "clang-colour-diags", + "-clang-color-diags", + default=True, + help="Force CHERI clang to emit coloured diagnostics", + ) + self.use_sdk_clang_for_native_xbuild = loader.add_bool_option( + "use-sdk-clang-for-native-xbuild", + group=loader.cross_compile_options_group, + help="Compile cross-compile project with CHERI " "clang from the SDK instead of host " "compiler", + ) + + self.configure_only = loader.add_bool_option( + "configure-only", help="Only run the configure step (skip build and install)" + ) self.skip_install = loader.add_bool_option("skip-install", help="Skip the install step (only do the build)") self.skip_build = loader.add_bool_option("skip-build", help="Skip the build step (only do the install)") self.skip_sdk = loader.add_bool_option( - "skip-sdk", group=loader.dependencies_group, + "skip-sdk", + group=loader.dependencies_group, help="When building with --include-dependencies ignore the SDK dependencies. Saves a lot of time " - "when building libc++, etc. with dependencies but the sdk is already up-to-date. " - "This is like --no-include-toolchain-depedencies but also skips the target that builds the sysroot.") + "when building libc++, etc. with dependencies but the sdk is already up-to-date. " + "This is like --no-include-toolchain-depedencies but also skips the target that builds the sysroot.", + ) self.skip_dependency_filters: "list[re.Pattern]" = loader.add_option( - "skip-dependency-filter", group=loader.dependencies_group, action="append", default=[], - type=_skip_dependency_filter_arg, metavar="REGEX", + "skip-dependency-filter", + group=loader.dependencies_group, + action="append", + default=[], + type=_skip_dependency_filter_arg, + metavar="REGEX", help="A regular expression to match against to target names that should be skipped when using" - "--include-dependency. Can be passed multiple times to add more patterns.") + "--include-dependency. Can be passed multiple times to add more patterns.", + ) self.trap_on_unrepresentable = loader.add_bool_option( - "trap-on-unrepresentable", default=False, group=loader.run_group, + "trap-on-unrepresentable", + default=False, + group=loader.run_group, help="Raise a CHERI exception when capabilities become unreprestable instead of detagging. Useful for " - "debugging, but deviates from the spec, and therefore off by default.") + "debugging, but deviates from the spec, and therefore off by default.", + ) self.debugger_on_cheri_trap = loader.add_bool_option( - "qemu-gdb-break-on-cheri-trap", default=False, group=loader.run_group, - help="Drop into GDB attached to QEMU when a CHERI exception is triggered (QEMU only).") + "qemu-gdb-break-on-cheri-trap", + default=False, + group=loader.run_group, + help="Drop into GDB attached to QEMU when a CHERI exception is triggered (QEMU only).", + ) self.qemu_debug_program = loader.add_option( - "qemu-gdb-debug-userspace-program", group=loader.run_group, - help="Print the command to debug the following userspace program in GDB attaced to QEMU") + "qemu-gdb-debug-userspace-program", + group=loader.run_group, + help="Print the command to debug the following userspace program in GDB attaced to QEMU", + ) self.include_dependencies: Optional[bool] = None self.include_toolchain_dependencies = True self.enable_hybrid_targets = False - self.only_dependencies = loader.add_bool_option("only-dependencies", - help="Only build dependencies of targets, " - "not the targets themselves") + self.only_dependencies = loader.add_bool_option( + "only-dependencies", help="Only build dependencies of targets, " "not the targets themselves" + ) self.start_with: Optional[str] = None self.start_after: Optional[str] = None self.make_without_nice: Optional[bool] = None @@ -339,135 +445,201 @@ def __init__(self, loader, action_class: "type[CheribuildActionEnum]") -> None: self.morello_sdk_dir: Optional[Path] = None self.other_tools_dir: Optional[Path] = None self.sysroot_output_root: Optional[Path] = None - self.docker = loader.add_bool_option("docker", help="Run the build inside a docker container", - group=loader.docker_group) - self.docker_container = loader.add_option("docker-container", help="Name of the docker container to use", - default="ctsrd/cheribuild-docker", group=loader.docker_group) - self.docker_reuse_container = loader.add_bool_option("docker-reuse-container", group=loader.docker_group, - help="Attach to the same container again (note: " - "docker-container option must be an id rather than " - "a container name") + self.docker = loader.add_bool_option( + "docker", help="Run the build inside a docker container", group=loader.docker_group + ) + self.docker_container = loader.add_option( + "docker-container", + help="Name of the docker container to use", + default="ctsrd/cheribuild-docker", + group=loader.docker_group, + ) + self.docker_reuse_container = loader.add_bool_option( + "docker-reuse-container", + group=loader.docker_group, + help="Attach to the same container again (note: " + "docker-container option must be an id rather than " + "a container name", + ) # compilation db options: self.create_compilation_db = loader.add_bool_option( - "compilation-db", "-cdb", help="Create a compile_commands.json file in the build dir " - "(requires Bear for non-CMake projects)") + "compilation-db", + "-cdb", + help="Create a compile_commands.json file in the build dir " "(requires Bear for non-CMake projects)", + ) self.copy_compilation_db_to_source_dir = None # False for jenkins, an option for cheribuild self.generate_cmakelists = False # False for jenkins, an option for cheribuild # Run QEMU options - self.wait_for_debugger = loader.add_bool_option("wait-for-debugger", group=loader.run_group, - help="Start QEMU in the 'wait for a debugger' state when" - "launching CheriBSD,FreeBSD, etc.") - - self.debugger_in_tmux_pane = loader.add_bool_option("debugger-in-tmux-pane", group=loader.run_group, - help="Start Qemu and gdb in another tmux split") - - self.gdb_random_port = loader.add_bool_option("gdb-random-port", default=True, group=loader.run_group, - help="Wait for gdb using a random port") - - self.run_under_gdb = loader.add_bool_option("run-under-gdb", group=loader.run_group, - help="Run tests/benchmarks under GDB. Note: currently most " - "targets ignore this flag.") + self.wait_for_debugger = loader.add_bool_option( + "wait-for-debugger", + group=loader.run_group, + help="Start QEMU in the 'wait for a debugger' state when" "launching CheriBSD,FreeBSD, etc.", + ) + + self.debugger_in_tmux_pane = loader.add_bool_option( + "debugger-in-tmux-pane", group=loader.run_group, help="Start Qemu and gdb in another tmux split" + ) + + self.gdb_random_port = loader.add_bool_option( + "gdb-random-port", default=True, group=loader.run_group, help="Wait for gdb using a random port" + ) + + self.run_under_gdb = loader.add_bool_option( + "run-under-gdb", + group=loader.run_group, + help="Run tests/benchmarks under GDB. Note: currently most " "targets ignore this flag.", + ) # Test options: self._test_ssh_key = loader.add_optional_path_option( - "test-ssh-key", group=loader.tests_group, + "test-ssh-key", + group=loader.tests_group, help="The SSH key to used to connect to the QEMU instance when running tests on CheriBSD. If not specified" - " a key will be generated in the build-root directory on-demand.") - self.use_minimal_benchmark_kernel = loader.add_bool_option("use-minimal-benchmark-kernel", - help="Use a CHERI BENCHMARK version of the " - "cheribsd-mfs-root-kernel (without " - "INVARIATES) for the " - "run-minimal target and for tests. This can " - "speed up longer running tests. This is the " - "default for " - "PostgreSQL and libc++ tests (passing " - "use-minimal-benchmark-kernel can force these " - "tests to use " - "an INVARIANTS kernel).", - group=loader.tests_group, default=False) - - self.test_extra_args = loader.add_commandline_only_option("test-extra-args", group=loader.tests_group, - type=list, - metavar="ARGS", - help="Additional flags to pass to the test script " - "in --test") - self.tests_interact = loader.add_commandline_only_bool_option("interact-after-tests", group=loader.tests_group, - help="Interact with the CheriBSD instance after " - "running the tests on QEMU (only for " - "--test)") - self.tests_env_only = loader.add_commandline_only_bool_option("test-environment-only", group=loader.tests_group, - help="Don't actually run the tests. Instead " - "setup a QEMU instance with the right " - "paths set up.") - self.test_ld_preload = loader.add_optional_path_option("test-ld-preload", group=loader.tests_group, - help="Preload the given library before running tests") + " a key will be generated in the build-root directory on-demand.", + ) + self.use_minimal_benchmark_kernel = loader.add_bool_option( + "use-minimal-benchmark-kernel", + help="Use a CHERI BENCHMARK version of the " + "cheribsd-mfs-root-kernel (without " + "INVARIATES) for the " + "run-minimal target and for tests. This can " + "speed up longer running tests. This is the " + "default for " + "PostgreSQL and libc++ tests (passing " + "use-minimal-benchmark-kernel can force these " + "tests to use " + "an INVARIANTS kernel).", + group=loader.tests_group, + default=False, + ) + + self.test_extra_args = loader.add_commandline_only_option( + "test-extra-args", + group=loader.tests_group, + type=list, + metavar="ARGS", + help="Additional flags to pass to the test script " "in --test", + ) + self.tests_interact = loader.add_commandline_only_bool_option( + "interact-after-tests", + group=loader.tests_group, + help="Interact with the CheriBSD instance after " "running the tests on QEMU (only for " "--test)", + ) + self.tests_env_only = loader.add_commandline_only_bool_option( + "test-environment-only", + group=loader.tests_group, + help="Don't actually run the tests. Instead " "setup a QEMU instance with the right " "paths set up.", + ) + self.test_ld_preload = loader.add_optional_path_option( + "test-ld-preload", group=loader.tests_group, help="Preload the given library before running tests" + ) self.benchmark_fpga_extra_args = loader.add_commandline_only_option( - "benchmark-fpga-extra-args", group=loader.benchmark_group, type=list, metavar="ARGS", - help="Extra options for the FPGA management script") - self.benchmark_clean_boot = loader.add_bool_option("benchmark-clean-boot", group=loader.benchmark_group, - help="Reboot the FPGA with a new bitfile and kernel before " - "running benchmarks. " - "If not set, assume the FPGA is running.") + "benchmark-fpga-extra-args", + group=loader.benchmark_group, + type=list, + metavar="ARGS", + help="Extra options for the FPGA management script", + ) + self.benchmark_clean_boot = loader.add_bool_option( + "benchmark-clean-boot", + group=loader.benchmark_group, + help="Reboot the FPGA with a new bitfile and kernel before " + "running benchmarks. " + "If not set, assume the FPGA is running.", + ) self.benchmark_extra_args = loader.add_commandline_only_option( - "benchmark-extra-args", group=loader.benchmark_group, type=list, - metavar="ARGS", help="Additional flags to pass to the program executed in --benchmark") + "benchmark-extra-args", + group=loader.benchmark_group, + type=list, + metavar="ARGS", + help="Additional flags to pass to the program executed in --benchmark", + ) self.benchmark_ssh_host = loader.add_option( - "benchmark-ssh-host", group=loader.benchmark_group, type=str, - default="cheri-fpga", help="The SSH hostname/IP for the benchmark FPGA") + "benchmark-ssh-host", + group=loader.benchmark_group, + type=str, + default="cheri-fpga", + help="The SSH hostname/IP for the benchmark FPGA", + ) self.benchmark_statcounters_suffix = loader.add_option( - "benchmark-csv-suffix", group=loader.benchmark_group, - help="Add a custom suffix for the statcounters CSV.") + "benchmark-csv-suffix", group=loader.benchmark_group, help="Add a custom suffix for the statcounters CSV." + ) self.benchmark_ld_preload = loader.add_optional_path_option( - "benchmark-ld-preload", group=loader.benchmark_group, - help="Preload the given library before running benchmarks") + "benchmark-ld-preload", + group=loader.benchmark_group, + help="Preload the given library before running benchmarks", + ) self.benchmark_with_debug_kernel = loader.add_bool_option( - "benchmark-with-debug-kernel", group=loader.benchmark_group, - help="Run the benchmark with a kernel that has assertions enabled.") + "benchmark-with-debug-kernel", + group=loader.benchmark_group, + help="Run the benchmark with a kernel that has assertions enabled.", + ) self.benchmark_lazy_binding = loader.add_bool_option( - "benchmark-lazy-binding", group=loader.benchmark_group, - help="Run the benchmark without setting LD_BIND_NOW.") + "benchmark-lazy-binding", + group=loader.benchmark_group, + help="Run the benchmark without setting LD_BIND_NOW.", + ) self.benchmark_iterations = loader.add_option( - "benchmark-iterations", type=int, group=loader.benchmark_group, - help="Override the number of iterations for the benchmark. " - "Note: not all benchmarks support this option") + "benchmark-iterations", + type=int, + group=loader.benchmark_group, + help="Override the number of iterations for the benchmark. " "Note: not all benchmarks support this option", + ) self.benchmark_with_qemu = loader.add_bool_option( - "benchmark-with-qemu", group=loader.benchmark_group, + "benchmark-with-qemu", + group=loader.benchmark_group, help="Run the benchmarks on QEMU instead of the FPGA (only useful to collect instruction counts or test " - "the benchmarks)") + "the benchmarks)", + ) self.shallow_clone = loader.add_bool_option( - "shallow-clone", default=True, + "shallow-clone", + default=True, help="Perform a shallow `git clone` when cloning new projects. This can save a lot of time for large" - "repositories such as FreeBSD or LLVM. Use `git fetch --unshallow` to convert to a non-shallow clone") + "repositories such as FreeBSD or LLVM. Use `git fetch --unshallow` to convert to a non-shallow clone", + ) self.fpga_custom_env_setup_script = loader.add_optional_path_option( - "beri-fpga-env-setup-script", group=loader.path_group, - help="Custom script to source to setup PATH and quartus, default to using cheri-cpu/cheri/setup.sh") + "beri-fpga-env-setup-script", + group=loader.path_group, + help="Custom script to source to setup PATH and quartus, default to using cheri-cpu/cheri/setup.sh", + ) self.local_arm_none_eabi_toolchain_relpath = Path("arm-none-eabi-sdk") self.arm_none_eabi_toolchain_prefix = loader.add_option( - "arm-none-eabi-prefix", default=ComputedDefaultValue(_default_arm_none_eabi_prefix, ""), + "arm-none-eabi-prefix", + default=ComputedDefaultValue(_default_arm_none_eabi_prefix, ""), group=loader.path_group, help="Prefix for arm-none-eabi-gcc binaries (e.g. /usr/bin/arm-none-eabi-). Available at" - "https://developer.arm.com/tools-and-software/open-source-software/" - "developer-tools/gnu-toolchain/gnu-rm/downloads") + "https://developer.arm.com/tools-and-software/open-source-software/" + "developer-tools/gnu-toolchain/gnu-rm/downloads", + ) self.build_morello_firmware_from_source = loader.add_bool_option( - "build-morello-firmware-from-source", help_hidden=False, - help="Build the firmware from source instead of downloading the latest release.") + "build-morello-firmware-from-source", + help_hidden=False, + help="Build the firmware from source instead of downloading the latest release.", + ) - self.list_kernels = loader.add_bool_option("list-kernels", group=loader.action_group, - help="List available kernel configs to run and exit") + self.list_kernels = loader.add_bool_option( + "list-kernels", group=loader.action_group, help="List available kernel configs to run and exit" + ) self.remote_morello_board = loader.add_option( - "remote-morello-board", help="SSH hostname of a Morello board. When set, some projects will run their " - "test suites on the remote board instead of QEMU.") + "remote-morello-board", + help="SSH hostname of a Morello board. When set, some projects will run their " + "test suites on the remote board instead of QEMU.", + ) self.targets: "Optional[list[str]]" = None - self.__optional_properties = ["internet_connection_last_checked_at", "start_after", "start_with", - "default_action"] + self.__optional_properties = [ + "internet_connection_last_checked_at", + "start_after", + "start_with", + "default_action", + ] def load(self) -> None: self.loader.load() @@ -494,8 +666,11 @@ def load(self) -> None: # flatten the potentially nested list if not self.action: if self.default_action is None: - fatal_error("Missing action, please pass one of", - ", ".join([str(action.option_name) for action in self._action_class]), pretend=False) + fatal_error( + "Missing action, please pass one of", + ", ".join([str(action.option_name) for action in self._action_class]), + pretend=False, + ) self.action = [self.default_action] else: assert isinstance(self.action, list) @@ -569,8 +744,20 @@ def test_ssh_key(self) -> Path: default_test_ssh_key_path = self.build_root / "insecure_test_ssh_key.pub" if not default_test_ssh_key_path.exists(): status_update("Generating SSH key for testing:", default_test_ssh_key_path) - run_command(["ssh-keygen", "-t", "ed25519", "-N", "", "-f", default_test_ssh_key_path.with_suffix(""), - "-C", "Test SSH key for cheribuild"], config=self) + run_command( + [ + "ssh-keygen", + "-t", + "ed25519", + "-N", + "", + "-f", + default_test_ssh_key_path.with_suffix(""), + "-C", + "Test SSH key for cheribuild", + ], + config=self, + ) return default_test_ssh_key_path def _ensure_required_properties_set(self) -> bool: @@ -600,7 +787,7 @@ def should_skip_dependency(self, target_name: str, requested_by: str) -> bool: # FIXME: not sure why this is needed def __getattribute__(self, item) -> "typing.Any": v = object.__getattribute__(self, item) - if hasattr(v, '__get__'): + if hasattr(v, "__get__"): # noinspection PyCallingNonCallable return v.__get__(self, self.__class__) # pytype: disable=attribute-error return v diff --git a/pycheribuild/config/compilation_targets.py b/pycheribuild/config/compilation_targets.py index fb283e3c6..85ce88e3d 100644 --- a/pycheribuild/config/compilation_targets.py +++ b/pycheribuild/config/compilation_targets.py @@ -182,7 +182,8 @@ def triple_for_target( config: "CheriConfig", *, include_version: bool, - ) -> str: ... + ) -> str: + ... def get_target_triple(self, *, include_version: bool) -> str: return self.triple_for_target(self.target, self.config, include_version=include_version) diff --git a/pycheribuild/config/computed_default_value.py b/pycheribuild/config/computed_default_value.py index 89fe81f08..f36bcdeed 100644 --- a/pycheribuild/config/computed_default_value.py +++ b/pycheribuild/config/computed_default_value.py @@ -33,20 +33,26 @@ T = typing.TypeVar("T") if typing.TYPE_CHECKING: # no-combine from ..utils import ConfigBase # no-combine + ConfigTy = typing.TypeVar("ConfigTy", bound=ConfigBase) # no-combine class ComputedDefaultValue(typing.Generic[T]): - def __init__(self, function: "Callable[[ConfigTy, Any], T]", - as_string: "Union[str, Callable[[Any], str]]", - as_readme_string: "Union[str, Callable[[Any], str], None]" = None, - inherit: "typing.Optional[ComputedDefaultValue[T]]" = None): + def __init__( + self, + function: "Callable[[ConfigTy, Any], T]", + as_string: "Union[str, Callable[[Any], str]]", + as_readme_string: "Union[str, Callable[[Any], str], None]" = None, + inherit: "typing.Optional[ComputedDefaultValue[T]]" = None, + ): if inherit is not None: + def inheriting_function(config, project): val = function(config, project) if val is None: val = inherit.function(config, project) return val + self.function = inheriting_function else: assert function is not None, "Must provide function or inherit" @@ -56,8 +62,10 @@ def inheriting_function(config, project): assert callable(as_string), "Inheriting only makes sense with callable as_string" if not callable(inherit.as_string): + def inherit_as_string_wrapper(cls): return inherit.as_string + inherited_as_string = inherit_as_string_wrapper else: inherited_as_string = inherit.as_string @@ -79,11 +87,14 @@ def inheriting_as_string(cls): def as_readme_string_none_wrapper(cls): return None + as_readme_string = as_readme_string_none_wrapper if not callable(inherit.as_readme_string): + def inherit_as_readme_string_wrapper(cls): return inherit.as_readme_string + inherited_as_readme_string = inherit_as_readme_string_wrapper else: inherited_as_readme_string = inherit.as_readme_string diff --git a/pycheribuild/config/config_loader_base.py b/pycheribuild/config/config_loader_base.py index ecba3f6fa..e5bcd600b 100644 --- a/pycheribuild/config/config_loader_base.py +++ b/pycheribuild/config/config_loader_base.py @@ -39,7 +39,7 @@ from .computed_default_value import ComputedDefaultValue from ..utils import ConfigBase, fatal_error, warning_message -T = typing.TypeVar('T') +T = typing.TypeVar("T") if typing.TYPE_CHECKING: import argparse @@ -73,8 +73,9 @@ class ConfigLoaderBase(ABC): # argparse groups used in the command line loader - def __init__(self, *, option_cls: "type[ConfigOptionBase]", - command_line_only_options_cls: "type[ConfigOptionBase]"): + def __init__( + self, *, option_cls: "type[ConfigOptionBase]", command_line_only_options_cls: "type[ConfigOptionBase]" + ): self.__option_cls: "type[ConfigOptionBase]" = option_cls self.__command_line_only_options_cls: "type[ConfigOptionBase]" = command_line_only_options_cls self.unknown_config_option_is_error = False @@ -84,7 +85,8 @@ def __init__(self, *, option_cls: "type[ConfigOptionBase]", self.dependencies_group = self.add_argument_group("Selecting which dependencies are built") self.path_group = self.add_argument_group("Configuration of default paths") self.cross_compile_options_group = self.add_argument_group( - "Adjust flags used when compiling MIPS/CHERI projects") + "Adjust flags used when compiling MIPS/CHERI projects" + ) self.tests_group = self.add_argument_group("Configuration for running tests") self.benchmark_group = self.add_argument_group("Configuration for running benchmarks") self.run_group = self.add_argument_group("Configuration for launching QEMU (and other simulators)") @@ -101,15 +103,30 @@ def add_commandline_only_option(self, *args, type: "Callable[[str], T]" = str, * def add_commandline_only_bool_option(self, *args, default=False, **kwargs) -> bool: assert default is False or kwargs.get("negatable") is True - return self.add_option(*args, option_cls=self.__command_line_only_options_cls, default=default, - negatable=kwargs.pop("negatable", False), type=bool, **kwargs) + return self.add_option( + *args, + option_cls=self.__command_line_only_options_cls, + default=default, + negatable=kwargs.pop("negatable", False), + type=bool, + **kwargs, + ) # noinspection PyShadowingBuiltins - def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callable[[str], T]]" = str, - default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, - _owning_class: "Optional[type]" = None, _fallback_names: "Optional[list[str]]" = None, - option_cls: "Optional[type[ConfigOptionBase[T]]]" = None, replaceable=False, - fallback_replaceable: "Optional[bool]" = None, **kwargs) -> T: + def add_option( + self, + name: str, + shortname=None, + *, + type: "Union[type[T], Callable[[str], T]]" = str, + default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, + _owning_class: "Optional[type]" = None, + _fallback_names: "Optional[list[str]]" = None, + option_cls: "Optional[type[ConfigOptionBase[T]]]" = None, + replaceable=False, + fallback_replaceable: "Optional[bool]" = None, + **kwargs, + ) -> T: if option_cls is None: option_cls = self.__option_cls if fallback_replaceable is None: @@ -124,10 +141,16 @@ def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callabl fallback_handle = self.option_handles.get(fallback_name) if fallback_handle is None or fallback_handle.replaceable: # Do not assign an owning class or a default value to this implicitly added fallback option. - self.add_option(fallback_name, type=type, option_cls=option_cls, replaceable=fallback_replaceable, - is_fallback=True) - option = option_cls(name, shortname, default, type, _owning_class, _loader=self, - _fallback_names=_fallback_names, **kwargs) + self.add_option( + fallback_name, + type=type, + option_cls=option_cls, + replaceable=fallback_replaceable, + is_fallback=True, + ) + option = option_cls( + name, shortname, default, type, _owning_class, _loader=self, _fallback_names=_fallback_names, **kwargs + ) if name in self.option_handles: self.option_handles[name]._replace_option(option, replaceable) else: @@ -138,14 +161,20 @@ def add_bool_option(self, name: str, shortname=None, default=False, **kwargs) -> # noinspection PyTypeChecker return self.add_option(name, shortname, default=default, type=bool, **kwargs) - def add_path_option(self, name: str, *, - default: "Union[ComputedDefaultValue[Path], Path, Callable[[ConfigBase, typing.Any], Path]]", - shortname=None, **kwargs) -> Path: + def add_path_option( + self, + name: str, + *, + default: "Union[ComputedDefaultValue[Path], Path, Callable[[ConfigBase, typing.Any], Path]]", + shortname=None, + **kwargs, + ) -> Path: # we have to make sure we resolve this to an absolute path because otherwise steps where CWD is different fail! return typing.cast(Path, self.add_option(name, shortname, type=Path, default=default, **kwargs)) - def add_optional_path_option(self, name: str, *, default: "Optional[Path]" = None, shortname=None, - **kwargs) -> Path: + def add_optional_path_option( + self, name: str, *, default: "Optional[Path]" = None, shortname=None, **kwargs + ) -> Path: # we have to make sure we resolve this to an absolute path because otherwise steps where CWD is different fail! return self.add_option(name, shortname, type=Path, default=default, **kwargs) @@ -193,8 +222,9 @@ def targets(self) -> "list[str]": class AbstractConfigOption(typing.Generic[T], metaclass=ABCMeta): @abstractmethod - def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: type, - return_none_if_default=False) -> T: + def load_option( + self, config: "ConfigBase", instance: "Optional[object]", _: type, return_none_if_default=False + ) -> T: ... @abstractmethod @@ -229,10 +259,19 @@ def _convert_type(self, loaded_result: _LoadedConfigValue) -> "Optional[T]": class ConfigOptionBase(AbstractConfigOption[T]): - def __init__(self, name: str, shortname: Optional[str], default, - value_type: "Union[type[T], Callable[[typing.Any], T]]", _owning_class=None, *, - _loader: "Optional[ConfigLoaderBase]" = None, _fallback_names: "Optional[list[str]]" = None, - _legacy_alias_names: "Optional[list[str]]" = None, is_fallback: bool = False): + def __init__( + self, + name: str, + shortname: Optional[str], + default, + value_type: "Union[type[T], Callable[[typing.Any], T]]", + _owning_class=None, + *, + _loader: "Optional[ConfigLoaderBase]" = None, + _fallback_names: "Optional[list[str]]" = None, + _legacy_alias_names: "Optional[list[str]]" = None, + is_fallback: bool = False, + ): self.name = name self.shortname = shortname self.default = default @@ -250,8 +289,9 @@ def __init__(self, name: str, shortname: Optional[str], default, self._is_default_value = False self.is_fallback_only = is_fallback - def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: type, - return_none_if_default=False) -> T: + def load_option( + self, config: "ConfigBase", instance: "Optional[object]", _: type, return_none_if_default=False + ) -> T: result = self._load_option_impl(config, self.full_option_name) # fall back from --qtbase-mips/foo to --qtbase/foo # Try aliases first: @@ -283,8 +323,16 @@ def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: typ try: result = self._convert_type(result) except ValueError as e: - fatal_error("Invalid value for option '", self.full_option_name, - "': could not convert '", result, "': ", str(e), sep="", pretend=config.pretend) + fatal_error( + "Invalid value for option '", + self.full_option_name, + "': could not convert '", + result, + "': ", + str(e), + sep="", + pretend=config.pretend, + ) sys.exit() return result @@ -305,9 +353,10 @@ def is_default_value(self) -> bool: return self._is_default_value def __get__(self, instance, owner) -> T: - assert instance is not None or not callable(self.default), \ - f"Tried to access read config option {self.full_option_name} without an object instance. " \ + assert instance is not None or not callable(self.default), ( + f"Tried to access read config option {self.full_option_name} without an object instance. " f"Config options using computed defaults can only be used with an object instance. Owner = {owner}" + ) # TODO: would be nice if this was possible (but too much depends on accessing values without instances) # if instance is None: @@ -335,8 +384,16 @@ def _convert_type(self, loaded_result: _LoadedConfigValue) -> "Optional[T]": if isinstance(self.value_type, type) and issubclass(self.value_type, collections.abc.Sequence): string_value = result result = shlex.split(string_value) - warning_message("Config option ", self.full_option_name, " (", string_value, ") should be a list, ", - "got a string instead -> assuming the correct value is ", result, sep="") + warning_message( + "Config option ", + self.full_option_name, + " (", + string_value, + ") should be a list, ", + "got a string instead -> assuming the correct value is ", + result, + sep="", + ) if isinstance(self.value_type, type) and issubclass(self.value_type, Path): expanded = os.path.expanduser(os.path.expandvars(str(result))) while expanded.startswith("//"): @@ -391,8 +448,9 @@ def _replace_option(self, option: "ConfigOptionBase[T]", replaceable: bool): def replaceable(self) -> bool: return self.__replaceable - def load_option(self, config: "ConfigBase", instance: "Optional[object]", _: type, - return_none_if_default=False) -> T: + def load_option( + self, config: "ConfigBase", instance: "Optional[object]", _: type, return_none_if_default=False + ) -> T: return self._get_option().load_option(config, instance, _, return_none_if_default) def _load_option_impl(self, config: "ConfigBase", target_option_name) -> "Optional[_LoadedConfigValue]": diff --git a/pycheribuild/config/defaultconfig.py b/pycheribuild/config/defaultconfig.py index 299b10166..8b01aad4a 100644 --- a/pycheribuild/config/defaultconfig.py +++ b/pycheribuild/config/defaultconfig.py @@ -42,12 +42,19 @@ class CheribuildAction(CheribuildActionEnum): BUILD = ("--build", "Run (usually build+install) chosen targets (default)") TEST = ("--test", "Run tests for the passed targets instead of building them", "--run-tests") BENCHMARK = ("--benchmark", "Run tests for the passed targets instead of building them") - BUILD_AND_TEST = ("--build-and-test", "Run chosen targets and then run any tests afterwards", None, - # can get the other instances yet -> use strings - ["build", "test"]) + BUILD_AND_TEST = ( + "--build-and-test", + "Run chosen targets and then run any tests afterwards", + None, + # can get the other instances yet -> use strings + ["build", "test"], + ) LIST_TARGETS = ("--list-targets", "List all available targets and exit") - DUMP_CONFIGURATION = ("--dump-configuration", "Print the current configuration as JSON. This can be saved to " - "~/.config/cheribuild.json to make it persistent") + DUMP_CONFIGURATION = ( + "--dump-configuration", + "Print the current configuration as JSON. This can be saved to " + "~/.config/cheribuild.json to make it persistent", + ) def __init__(self, option_name, help_message, altname=None, actions=None) -> None: self.option_name = option_name @@ -61,14 +68,18 @@ def __init__(self, option_name, help_message, altname=None, actions=None) -> Non class DefaultCheribuildConfigLoader(JsonAndCommandLineConfigLoader): def finalize_options(self, available_targets: "list[str]", **kwargs) -> None: - target_option = self._parser.add_argument("targets", metavar="TARGET", nargs=argparse.ZERO_OR_MORE, - help="The targets to build") + target_option = self._parser.add_argument( + "targets", metavar="TARGET", nargs=argparse.ZERO_OR_MORE, help="The targets to build" + ) if argcomplete and self.is_completing_arguments: # if OSInfo.IS_FREEBSD: # FIXME: for some reason this won't work self.completion_excludes = ["-t", "--skip-dependencies"] if sys.platform.startswith("freebsd"): - self.completion_excludes += ["--freebsd-builder-copy-only", "--freebsd-builder-hostname", - "--freebsd-builder-output-path"] + self.completion_excludes += [ + "--freebsd-builder-copy-only", + "--freebsd-builder-hostname", + "--freebsd-builder-output-path", + ] visible_targets = available_targets.copy() visible_targets.remove("__run_everything__") @@ -76,8 +87,14 @@ def finalize_options(self, available_targets: "list[str]", **kwargs) -> None: target_option.completer = target_completer # make sure we get target completion for the unparsed args too by adding another zero_or more options # not sure why this works but it's a nice hack - unparsed = self._parser.add_argument("targets", metavar="TARGET", type=list, nargs=argparse.ZERO_OR_MORE, - help=argparse.SUPPRESS, choices=available_targets) + unparsed = self._parser.add_argument( + "targets", + metavar="TARGET", + type=list, + nargs=argparse.ZERO_OR_MORE, + help=argparse.SUPPRESS, + choices=available_targets, + ) unparsed.completer = target_completer @@ -86,106 +103,152 @@ def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> super().__init__(loader, action_class=CheribuildAction) self.default_action = CheribuildAction.BUILD # The run mode: - self.get_config_option = loader.add_option("get-config-option", type=str, metavar="KEY", - group=loader.action_group, - help="Print the value of config option KEY and exit") + self.get_config_option = loader.add_option( + "get-config-option", + type=str, + metavar="KEY", + group=loader.action_group, + help="Print the value of config option KEY and exit", + ) # boolean flags self.quiet = loader.add_bool_option("quiet", "q", help="Don't show stdout of the commands that are executed") self.verbose = loader.add_bool_option("verbose", "v", help="Print all commmands that are executed") self.clean = loader.add_bool_option("clean", "c", help="Remove the build directory before build") self.force = loader.add_bool_option("force", "f", help="Don't prompt for user input but use the default action") - self.write_logfile = loader.add_bool_option("logfile", help="Write a logfile for the build steps", - default=False) + self.write_logfile = loader.add_bool_option( + "logfile", help="Write a logfile for the build steps", default=False + ) self.skip_update = loader.add_bool_option("skip-update", help="Skip the git pull step") self.skip_clone = False self.confirm_clone = loader.add_bool_option( - "confirm-clone", help="Ask for confirmation before cloning repositories.") - self.force_update = loader.add_bool_option("force-update", help="Always update (with autostash) even if there " - "are uncommitted changes") + "confirm-clone", help="Ask for confirmation before cloning repositories." + ) + self.force_update = loader.add_bool_option( + "force-update", help="Always update (with autostash) even if there " "are uncommitted changes" + ) self.presume_connectivity = loader.add_bool_option( "presume-connectivity", - help="Do not probe for network connectivity and just assume that we are suitably connected") + help="Do not probe for network connectivity and just assume that we are suitably connected", + ) # TODO: should replace this group with a tristate value configure_group = loader.add_mutually_exclusive_group() - self.skip_configure = loader.add_bool_option("skip-configure", help="Skip the configure step", - group=configure_group) - self.force_configure = loader.add_bool_option("reconfigure", "-force-configure", group=configure_group, - help="Always run the configure step, even for CMake projects " - "with a valid cache.") + self.skip_configure = loader.add_bool_option( + "skip-configure", help="Skip the configure step", group=configure_group + ) + self.force_configure = loader.add_bool_option( + "reconfigure", + "-force-configure", + group=configure_group, + help="Always run the configure step, even for CMake projects " "with a valid cache.", + ) self.include_dependencies = loader.add_commandline_only_bool_option( - "include-dependencies", "d", group=loader.dependencies_group, + "include-dependencies", + "d", + group=loader.dependencies_group, help="Also build the dependencies of targets passed on the command line. Targets passed on the command " - "line will be reordered and processed in an order that ensures dependencies are built before the " - "real target. (run --list-targets for more information). By default this does not build toolchain " - "targets such as LLVM. Pass --include-toolchain-dependencies to also build those.") + "line will be reordered and processed in an order that ensures dependencies are built before the " + "real target. (run --list-targets for more information). By default this does not build toolchain " + "targets such as LLVM. Pass --include-toolchain-dependencies to also build those.", + ) self.include_toolchain_dependencies = loader.add_bool_option( - "include-toolchain-dependencies", default=True, group=loader.dependencies_group, - help="Include toolchain targets such as LLVM and QEMU when --include-dependencies is set.") + "include-toolchain-dependencies", + default=True, + group=loader.dependencies_group, + help="Include toolchain targets such as LLVM and QEMU when --include-dependencies is set.", + ) self.enable_hybrid_targets = loader.add_bool_option( - "enable-hybrid-targets", default=False, help_hidden=True, + "enable-hybrid-targets", + default=False, + help_hidden=True, help="Enable building hybrid targets. This is highly discouraged, " - "only enable if you know what you're doing.") + "only enable if you know what you're doing.", + ) start_after_group = loader.dependencies_group.add_mutually_exclusive_group() self.start_with = loader.add_commandline_only_option( - "start-with", metavar="TARGET", group=start_after_group, - help="Start building at TARGET (useful when resuming an interrupted --include-depedencies build)") + "start-with", + metavar="TARGET", + group=start_after_group, + help="Start building at TARGET (useful when resuming an interrupted --include-depedencies build)", + ) self.start_after = loader.add_commandline_only_option( - "start-after", metavar="TARGET", group=start_after_group, - help="Start building after TARGET (useful when resuming an interrupted --include-depedencies build)") + "start-after", + metavar="TARGET", + group=start_after_group, + help="Start building after TARGET (useful when resuming an interrupted --include-depedencies build)", + ) self.copy_compilation_db_to_source_dir = loader.add_commandline_only_bool_option( "compilation-db-in-source-dir", - help="Generate a compile_commands.json and also copy it to the source directory") + help="Generate a compile_commands.json and also copy it to the source directory", + ) self.generate_cmakelists = loader.add_bool_option( "generate-cmakelists", - help="Generate a CMakeLists.txt that just calls cheribuild. Useful for IDEs that only support CMake") + help="Generate a CMakeLists.txt that just calls cheribuild. Useful for IDEs that only support CMake", + ) self.make_without_nice = loader.add_bool_option("make-without-nice", help="Run make/ninja without nice(1)") default_make_jobs = default_make_jobs_count() - default_make_jobs_computed = ComputedDefaultValue(lambda p, cls: default_make_jobs, - as_string=str(default_make_jobs), - as_readme_string="") - self.make_jobs: int = loader.add_option("make-jobs", "j", type=int, default=default_make_jobs_computed, - help="Number of jobs to use for compiling") + default_make_jobs_computed = ComputedDefaultValue( + lambda p, cls: default_make_jobs, as_string=str(default_make_jobs), as_readme_string="" + ) + self.make_jobs: int = loader.add_option( + "make-jobs", "j", type=int, default=default_make_jobs_computed, help="Number of jobs to use for compiling" + ) # configurable paths - self.source_root = loader.add_path_option("source-root", - default=Path(os.path.expanduser("~/cheri")), group=loader.path_group, - help="The directory to store all sources") - self.output_root = loader.add_path_option("output-root", - default=lambda p, cls: (p.source_root / "output"), - group=loader.path_group, - help="The directory to store all output (default: " - "'/output')") - self.build_root = loader.add_path_option("build-root", - default=lambda p, cls: (p.source_root / "build"), - group=loader.path_group, - help="The directory for all the builds (default: " - "'/build')") - self.tools_root = loader.add_path_option("tools-root", - default=lambda p, cls: p.output_root, group=loader.path_group, - help="The directory to find sdk and bootstrap tools (default: " - "'')") + self.source_root = loader.add_path_option( + "source-root", + default=Path(os.path.expanduser("~/cheri")), + group=loader.path_group, + help="The directory to store all sources", + ) + self.output_root = loader.add_path_option( + "output-root", + default=lambda p, cls: (p.source_root / "output"), + group=loader.path_group, + help="The directory to store all output (default: " "'/output')", + ) + self.build_root = loader.add_path_option( + "build-root", + default=lambda p, cls: (p.source_root / "build"), + group=loader.path_group, + help="The directory for all the builds (default: " "'/build')", + ) + self.tools_root = loader.add_path_option( + "tools-root", + default=lambda p, cls: p.output_root, + group=loader.path_group, + help="The directory to find sdk and bootstrap tools (default: " "'')", + ) default_morello_sdk = ComputedDefaultValue( function=lambda p, cls: (p.tools_root / p.default_morello_sdk_directory_name), - as_string="'/morello-sdk'") - self.morello_sdk_dir = loader.add_path_option("morello-sdk-root", - default=default_morello_sdk, group=loader.path_group, - help="The directory to find/install the Morello SDK") - self.sysroot_output_root = loader.add_path_option("sysroot-install-root", shortname="-sysroot-install-dir", - default=lambda p, cls: p.tools_root, group=loader.path_group, - help="Sysroot prefix (default: '')") + as_string="'/morello-sdk'", + ) + self.morello_sdk_dir = loader.add_path_option( + "morello-sdk-root", + default=default_morello_sdk, + group=loader.path_group, + help="The directory to find/install the Morello SDK", + ) + self.sysroot_output_root = loader.add_path_option( + "sysroot-install-root", + shortname="-sysroot-install-dir", + default=lambda p, cls: p.tools_root, + group=loader.path_group, + help="Sysroot prefix (default: '')", + ) # Hidden option to enable foo-hybrid-for-purecap-rootfs targets for all projects. This option is not actually # uses since we have to look at sys.argv[] directly due to depedency cycles. However, we do still need it since # we would otherwise get an invalid command line argument error. self.enable_hybrid_for_purecap_rootfs_targets = loader.add_commandline_only_bool_option( - "enable-hybrid-for-purecap-rootfs-targets", default=False, help_hidden=True) + "enable-hybrid-for-purecap-rootfs-targets", default=False, help_hidden=True + ) loader.finalize_options(available_targets) def load(self) -> None: diff --git a/pycheribuild/config/jenkinsconfig.py b/pycheribuild/config/jenkinsconfig.py index 9bb3a5133..692d6f372 100644 --- a/pycheribuild/config/jenkinsconfig.py +++ b/pycheribuild/config/jenkinsconfig.py @@ -90,74 +90,120 @@ class JenkinsConfig(CheriConfig): def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> None: super().__init__(loader, action_class=JenkinsAction) self.cpu = loader.add_commandline_only_option( - "cpu", default=os.getenv("CPU", "default"), - help="Only used for backwards compatibility with old jenkins jobs") + "cpu", + default=os.getenv("CPU", "default"), + help="Only used for backwards compatibility with old jenkins jobs", + ) self.workspace = loader.add_commandline_only_option( - "workspace", default=os.getenv("WORKSPACE"), type=Path, - help="The root directory for building (defaults to $WORKSPACE)") + "workspace", + default=os.getenv("WORKSPACE"), + type=Path, + help="The root directory for building (defaults to $WORKSPACE)", + ) self.compiler_archive_name = loader.add_commandline_only_option( - "compiler-archive", type=str, default="cheri-clang-llvm.tar.xz", - help="The name of the archive containing the compiler") + "compiler-archive", + type=str, + default="cheri-clang-llvm.tar.xz", + help="The name of the archive containing the compiler", + ) self.compiler_archive_output_path = loader.add_commandline_only_option( - "compiler-archive-output-path", type=Path, default=_infer_compiler_output_path, - help="The path where to extract the compiler") + "compiler-archive-output-path", + type=Path, + default=_infer_compiler_output_path, + help="The path where to extract the compiler", + ) self.compiler_type = loader.add_commandline_only_option( - "compiler-type", type=CompilerType, default=CompilerType.CHERI_LLVM, + "compiler-type", + type=CompilerType, + default=CompilerType.CHERI_LLVM, enum_choices=[CompilerType.CHERI_LLVM, CompilerType.MORELLO_LLVM, CompilerType.UPSTREAM_LLVM], - help="The type of the compiler to extract (used to infer the output " - " path)") + help="The type of the compiler to extract (used to infer the output " " path)", + ) self.sysroot_archive_name = loader.add_commandline_only_option( - "sysroot-archive", type=str, default="cheribsd-sysroot.tar.xz", - help="The name of the archive containing the sysroot") + "sysroot-archive", + type=str, + default="cheribsd-sysroot.tar.xz", + help="The name of the archive containing the sysroot", + ) self.sysroot_archive_output_path = loader.add_commandline_only_option( - "sysroot-archive-output-path", type=Path, - default=ComputedDefaultValue(lambda c, _: c.compiler_archive_output_path / "sysroot", - as_string="/sysroot"), - help="The path where to extract the sysroot (default=") + "sysroot-archive-output-path", + type=Path, + default=ComputedDefaultValue( + lambda c, _: c.compiler_archive_output_path / "sysroot", as_string="/sysroot" + ), + help="The path where to extract the sysroot (default=", + ) self.keep_install_dir = loader.add_commandline_only_bool_option( - "keep-install-dir", help="Don't delete the install dir prior to build") + "keep-install-dir", help="Don't delete the install dir prior to build" + ) self.keep_sdk_dir = loader.add_commandline_only_bool_option( - "keep-sdk-dir", help="Don't delete existing SDK dir even if there is a newer archive") + "keep-sdk-dir", help="Don't delete existing SDK dir even if there is a newer archive" + ) self.force_update = loader.add_commandline_only_bool_option( - "force-update", help="Do the updating (not recommended in jenkins!)") + "force-update", help="Do the updating (not recommended in jenkins!)" + ) self.copy_compilation_db_to_source_dir = False self.make_without_nice = False - self.make_jobs = loader.add_commandline_only_option("make-jobs", "j", type=int, - default=default_jenkins_make_jobs_count, - help="Number of jobs to use for compiling") - self.use_all_cores = loader.add_commandline_only_bool_option("use-all-cores", - help="Use all available cores for building (" - "Note: Should only be used for LLVM or " - "short-running jobs!)") + self.make_jobs = loader.add_commandline_only_option( + "make-jobs", + "j", + type=int, + default=default_jenkins_make_jobs_count, + help="Number of jobs to use for compiling", + ) + self.use_all_cores = loader.add_commandline_only_bool_option( + "use-all-cores", + help="Use all available cores for building (" + "Note: Should only be used for LLVM or " + "short-running jobs!)", + ) self.installation_prefix = loader.add_commandline_only_option( - "install-prefix", type=absolute_path_only, default=default_install_prefix, - help="The install prefix for cross compiled projects (the path in the install image)") + "install-prefix", + type=absolute_path_only, + default=default_install_prefix, + help="The install prefix for cross compiled projects (the path in the install image)", + ) self.use_system_compiler_for_native = loader.add_commandline_only_bool_option( - "use-system-compiler-for-native", "-without-sdk", - help="Don't use the CHERI SDK -> only /usr (for native builds)") + "use-system-compiler-for-native", + "-without-sdk", + help="Don't use the CHERI SDK -> only /usr (for native builds)", + ) self.strip_elf_files = loader.add_commandline_only_bool_option( - "strip-elf-files", help="Strip ELF files before creating the tarball", default=True, negatable=True) + "strip-elf-files", help="Strip ELF files before creating the tarball", default=True, negatable=True + ) self._cheri_sdk_dir_override = loader.add_commandline_only_option( - "cheri-sdk-path", default=None, type=Path, - help="Override the path to the CHERI SDK (default is $WORKSPACE/cherisdk)") + "cheri-sdk-path", + default=None, + type=Path, + help="Override the path to the CHERI SDK (default is $WORKSPACE/cherisdk)", + ) self._morello_sdk_dir_override = loader.add_commandline_only_option( - "morello-sdk-path", default=None, type=Path, - help="Override the path to the Morello SDK (default is $WORKSPACE/morello-sdk)") + "morello-sdk-path", + default=None, + type=Path, + help="Override the path to the Morello SDK (default is $WORKSPACE/morello-sdk)", + ) self.extract_compiler_only = loader.add_commandline_only_bool_option( - "extract-compiler-only", help="Don't attempt to extract a sysroot") + "extract-compiler-only", help="Don't attempt to extract a sysroot" + ) self.tarball_name = loader.add_commandline_only_option( - "tarball-name", default=lambda conf, cls: conf.targets[0] + "-" + conf.cpu + ".tar.xz") + "tarball-name", default=lambda conf, cls: conf.targets[0] + "-" + conf.cpu + ".tar.xz" + ) self.default_output_path = "tarball" - default_output = ComputedDefaultValue(lambda c, _: c.workspace / c.default_output_path, - "$WORKSPACE/" + self.default_output_path) - self.output_root = loader.add_commandline_only_option("output-path", default=default_output, type=Path, - help="Path for the output (relative to $WORKSPACE)") + default_output = ComputedDefaultValue( + lambda c, _: c.workspace / c.default_output_path, "$WORKSPACE/" + self.default_output_path + ) + self.output_root = loader.add_commandline_only_option( + "output-path", default=default_output, type=Path, help="Path for the output (relative to $WORKSPACE)" + ) self.sysroot_output_root = loader.add_commandline_only_option( "sysroot-output-path", default=ComputedDefaultValue(function=lambda c, _: c.workspace, as_string="$WORKSPACE"), - type=Path, help="Path for the installed sysroot (defaults to the same value as --output-path)") + type=Path, + help="Path for the installed sysroot (defaults to the same value as --output-path)", + ) # self.strip_install_prefix_from_archive = loader.add_commandline_only_bool_option( # "strip-install-prefix-from-archive", # help="Only put the files inside the install prefix into the tarball (stripping the leading @@ -167,8 +213,9 @@ def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> self.confirm_clone = False self.verbose = True self.quiet = False - self.clean = loader.add_commandline_only_bool_option("clean", default=True, negatable=True, - help="Clean build directory before building") + self.clean = loader.add_commandline_only_bool_option( + "clean", default=True, negatable=True, help="Clean build directory before building" + ) self.force = True # no user input in jenkins self.write_logfile = False # jenkins stores the output anyway self.skip_configure = loader.add_bool_option("skip-configure", help="Skip the configure step") @@ -178,7 +225,8 @@ def __init__(self, loader: ConfigLoaderBase, available_targets: "list[str]") -> self.allow_more_than_one_target = loader.add_commandline_only_bool_option( "allow-more-than-one-target", # help_hidden=True, Note: setting this to True seems to break argparse help="Allow more than one target on the command line. This should only be used for testing since " - "dependencies are not resolved!") + "dependencies are not resolved!", + ) loader.finalize_options(available_targets) self.FS = FileSystemUtils(self) @@ -208,8 +256,12 @@ def load(self) -> None: super().load() if not self.workspace or not self.workspace.is_dir(): - fatal_error("WORKSPACE is not set to a valid directory:", self.workspace, pretend=self.pretend, - fatal_when_pretending=True) + fatal_error( + "WORKSPACE is not set to a valid directory:", + self.workspace, + pretend=self.pretend, + fatal_when_pretending=True, + ) self.source_root = self.workspace self.build_root = self.workspace if self.output_root != self.workspace / self.default_output_path: @@ -219,8 +271,14 @@ def load(self) -> None: if os.path.relpath(str(self.output_root), str(self.workspace)).startswith(".."): fatal_error("Output path", self.output_root, "must be inside workspace", self.workspace, pretend=False) if os.path.relpath(str(self.sysroot_output_root), str(self.workspace)).startswith(".."): - fatal_error("Sysroot output path", self.sysroot_output_root, "must be inside workspace", self.workspace, - pretend=False, fatal_when_pretending=True) + fatal_error( + "Sysroot output path", + self.sysroot_output_root, + "must be inside workspace", + self.workspace, + pretend=False, + fatal_when_pretending=True, + ) # expect the CheriBSD disk images in the workspace root self.cheribsd_image_root = self.workspace @@ -254,14 +312,26 @@ def load(self) -> None: self.clang_plusplus_path = Path(os.getenv("HOST_CXX", self.clang_plusplus_path)) self.clang_cpp_path = Path(os.getenv("HOST_CPP", self.clang_cpp_path)) if not self.clang_path.exists(): - fatal_error("C compiler", self.clang_path, - "does not exit. Pass --clang-path or set $HOST_CC", pretend=self.pretend) + fatal_error( + "C compiler", + self.clang_path, + "does not exit. Pass --clang-path or set $HOST_CC", + pretend=self.pretend, + ) if not self.clang_plusplus_path.exists(): - fatal_error("C++ compiler", self.clang_plusplus_path, - "does not exit. Pass --clang++-path or set $HOST_CXX", pretend=self.pretend) + fatal_error( + "C++ compiler", + self.clang_plusplus_path, + "does not exit. Pass --clang++-path or set $HOST_CXX", + pretend=self.pretend, + ) if not self.clang_cpp_path.exists(): - fatal_error("C pre-processor", self.clang_cpp_path, - "does not exit. Pass --clang-cpp-path or set $HOST_CPP", pretend=self.pretend) + fatal_error( + "C pre-processor", + self.clang_cpp_path, + "does not exit. Pass --clang-cpp-path or set $HOST_CPP", + pretend=self.pretend, + ) else: # always use the CHERI clang built by jenkins (if available) # Prefix $WORKSPACE/native-sdk, but fall back to CHERI/Morello LLVM if that does not exist @@ -285,6 +355,7 @@ def load(self) -> None: assert self._ensure_required_properties_set() if os.getenv("DEBUG") is not None: import pprint + for k, v in self.__dict__.items(): if hasattr(v, "__get__"): # noinspection PyCallingNonCallable diff --git a/pycheribuild/config/loader.py b/pycheribuild/config/loader.py index a4c970f80..d5a2b8cb9 100644 --- a/pycheribuild/config/loader.py +++ b/pycheribuild/config/loader.py @@ -55,14 +55,13 @@ from ..colour import AnsiColour, coloured from ..utils import ConfigBase, error_message, fatal_error, status_update, warning_message -T = typing.TypeVar('T') -EnumTy = typing.TypeVar('EnumTy', bound=Enum) +T = typing.TypeVar("T") +EnumTy = typing.TypeVar("EnumTy", bound=Enum) # From https://bugs.python.org/issue25061 class _EnumArgparseType(typing.Generic[EnumTy]): - """Factory for creating enum object types - """ + """Factory for creating enum object types""" def __init__(self, enumclass: "type[EnumTy]"): self.enums: "type[EnumTy]" = enumclass @@ -74,8 +73,15 @@ def __init__(self, enumclass: "type[EnumTy]"): continue if c.isalpha() and c.isupper(): continue - raise RuntimeError("Invalid character '" + c + "' found in enum " + str(enumclass) + - " member " + member.name + ": must all be upper case letters or _ or digits.") + raise RuntimeError( + "Invalid character '" + + c + + "' found in enum " + + str(enumclass) + + " member " + + member.name + + ": must all be upper case letters or _ or digits." + ) # self.action = action def __call__(self, astring: "Union[str, list[str], EnumTy]") -> "Union[EnumTy, list[EnumTy]]": @@ -94,16 +100,16 @@ def __call__(self, astring: "Union[str, list[str], EnumTy]") -> "Union[EnumTy, l return e v = self.enums[enum_value_name] except KeyError: - msg = ', '.join([t.name.lower() for t in self.enums]) - msg = f'{name}: use one of {{{msg}}}' + msg = ", ".join([t.name.lower() for t in self.enums]) + msg = f"{name}: use one of {{{msg}}}" raise argparse.ArgumentTypeError(msg) # else: # self.action.choices = None # hugly hack to prevent post validation from choices return v def __repr__(self) -> str: - astr = ', '.join([t.name.lower() for t in self.enums]) - return f'{self.enums.__name__}({astr})' + astr = ", ".join([t.name.lower() for t in self.enums]) + return f"{self.enums.__name__}({astr})" # custom encoder to handle pathlib.Path and _LoadedConfigValue objects @@ -151,66 +157,123 @@ def get_argcomplete_prefix() -> str: # Based on Python 3.9 BooleanOptionalAction, but places the "no" after the first / class BooleanNegatableAction(argparse.Action): # noinspection PyShadowingBuiltins - def __init__(self, option_strings: "list[str]", dest, default=None, type=None, choices=None, required=False, - help=None, metavar=None): + def __init__( + self, + option_strings: "list[str]", + dest, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None, + ): # Add the negated option, placing the "no" after the / instead of the start -> --cheribsd/no-build-tests def collect_option_strings(original_strings): for opt in original_strings: all_option_strings.append(opt) - if opt.startswith('--'): + if opt.startswith("--"): slash_index = opt.rfind("/") if slash_index == -1: negated_opt = "--no-" + opt[2:] else: - negated_opt = opt[:slash_index + 1] + "no-" + opt[slash_index + 1:] + negated_opt = opt[: slash_index + 1] + "no-" + opt[slash_index + 1 :] all_option_strings.append(negated_opt) self._negated_option_strings.append(negated_opt) + all_option_strings = [] self._negated_option_strings = [] collect_option_strings(option_strings) # Don't show the alias options in --help output self.displayed_option_count = len(all_option_strings) - super().__init__(option_strings=all_option_strings, dest=dest, nargs=0, - default=default, type=type, choices=choices, required=required, help=help, metavar=metavar) + super().__init__( + option_strings=all_option_strings, + dest=dest, + nargs=0, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar, + ) def __call__(self, parser, namespace, values, option_string=None) -> None: if option_string in self.option_strings: setattr(namespace, self.dest, option_string not in self._negated_option_strings) def format_usage(self) -> str: - return ' | '.join(self.option_strings[:self.displayed_option_count]) + return " | ".join(self.option_strings[: self.displayed_option_count]) # argparse._StoreAction but with a possible list of aliases class StoreActionWithPossibleAliases(argparse.Action): # noinspection PyShadowingBuiltins - def __init__(self, option_strings: "list[str]", dest, nargs=None, default=None, type=None, choices=None, - required=False, help=None, metavar=None): + def __init__( + self, + option_strings: "list[str]", + dest, + nargs=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None, + ): if nargs == 1: raise ValueError("nargs for store actions must be 1") self.displayed_option_count = len(option_strings) - super().__init__(option_strings=option_strings, dest=dest, nargs=nargs, default=default, type=type, - choices=choices, required=required, help=help, metavar=metavar) + super().__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar, + ) def __call__(self, parser, namespace, values, option_string=None) -> None: setattr(namespace, self.dest, values) def format_usage(self) -> str: - return ' | '.join(self.option_strings[:self.displayed_option_count]) + return " | ".join(self.option_strings[: self.displayed_option_count]) class CommandLineConfigOption(ConfigOptionBase[T]): _loader: "JsonAndCommandLineConfigLoader" # noinspection PyProtectedMember,PyUnresolvedReferences - def __init__(self, name: str, shortname: "Optional[str]", default, - value_type: "Union[type[T], Callable[[Any], T]]", _owning_class, *, - _loader: "JsonAndCommandLineConfigLoader", help_hidden: bool, - group: "Optional[argparse._ArgumentGroup]", _fallback_names: "Optional[list[str]]" = None, - _legacy_alias_names: "Optional[list[str]]" = None, is_fallback: bool = False, **kwargs): - super().__init__(name, shortname, default, value_type, _owning_class, _loader=_loader, - _fallback_names=_fallback_names, _legacy_alias_names=_legacy_alias_names, - is_fallback=is_fallback) + def __init__( + self, + name: str, + shortname: "Optional[str]", + default, + value_type: "Union[type[T], Callable[[Any], T]]", + _owning_class, + *, + _loader: "JsonAndCommandLineConfigLoader", + help_hidden: bool, + group: "Optional[argparse._ArgumentGroup]", + _fallback_names: "Optional[list[str]]" = None, + _legacy_alias_names: "Optional[list[str]]" = None, + is_fallback: bool = False, + **kwargs, + ): + super().__init__( + name, + shortname, + default, + value_type, + _owning_class, + _loader=_loader, + _fallback_names=_fallback_names, + _legacy_alias_names=_legacy_alias_names, + is_fallback=is_fallback, + ) # hide obscure options unless --help-hidden/--help/all is passed if help_hidden and not self._loader.show_all_help: kwargs["help"] = argparse.SUPPRESS @@ -230,7 +293,8 @@ def __init__(self, name: str, shortname: "Optional[str]", default, self.default_str = "[]" else: assert isinstance(value_type, _EnumArgparseType), "default is enum but value type isn't: " + str( - value_type) + value_type + ) assert isinstance(default, Enum), "Should use enum constant for default and not " + str(default) self.default_str = default.name.lower() else: @@ -265,7 +329,7 @@ def _add_argparse_action(self, name, shortname, group, **kwargs) -> "argparse.Ac action = parser_obj.add_argument("--" + name, **kwargs) if self.default is not None and action.help is not None and has_default_help_text: if action.help != argparse.SUPPRESS: - action.help = action.help + " (default: \'" + self.default_str + "\')" + action.help = action.help + " (default: '" + self.default_str + "')" action.default = None # we don't want argparse default values! assert not action.type # we handle the type of the value manually return action @@ -305,8 +369,15 @@ def _load_option_impl(self, config: ConfigBase, target_option_name: str): # self.debug_msg(full_option_name, "from JSON:", from_json) if from_json is not None: if not config.quiet: - status_update("Overriding default value for", target_option_name, "with value from JSON key", - from_json.used_key, "->", from_json.value, file=sys.stderr) + status_update( + "Overriding default value for", + target_option_name, + "with value from JSON key", + from_json.used_key, + "->", + from_json.value, + file=sys.stderr, + ) return from_json return None # not found -> fall back to default @@ -348,8 +419,9 @@ def _load_from_json(self, full_option_name: str) -> "Optional[_LoadedConfigValue class DefaultValueOnlyConfigLoader(ConfigLoaderBase): def __init__(self) -> None: - super().__init__(option_cls=DefaultValueOnlyConfigOption, - command_line_only_options_cls=DefaultValueOnlyConfigOption) + super().__init__( + option_cls=DefaultValueOnlyConfigOption, command_line_only_options_cls=DefaultValueOnlyConfigOption + ) # Ignore options stored in other classes self.option_handles = dict() @@ -383,42 +455,63 @@ def dict_raise_on_duplicates_and_store_src(ordered_pairs, src_file) -> "dict[Any class ArgparseSetGivenAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None) -> None: setattr(namespace, self.dest, values) - setattr(namespace, self.dest + '_given', True) + setattr(namespace, self.dest + "_given", True) class CommandLineConfigLoader(ConfigLoaderBase): _parsed_args: argparse.Namespace - show_all_help: bool = any( - s in sys.argv for s in ("--help-all", "--help-hidden")) or ConfigLoaderBase.is_completing_arguments + show_all_help: bool = ( + any(s in sys.argv for s in ("--help-all", "--help-hidden")) or ConfigLoaderBase.is_completing_arguments + ) _argcomplete_prefix: "Optional[str]" = ( - get_argcomplete_prefix() if ConfigLoaderBase.is_completing_arguments else None) + get_argcomplete_prefix() if ConfigLoaderBase.is_completing_arguments else None + ) _argcomplete_prefix_includes_slash: bool = "/" in _argcomplete_prefix if _argcomplete_prefix else False - def __init__(self, argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, *, - option_cls=CommandLineConfigOption, command_line_only_options_cls=CommandLineConfigOption): + def __init__( + self, + argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, + *, + option_cls=CommandLineConfigOption, + command_line_only_options_cls=CommandLineConfigOption, + ): if self.is_completing_arguments or self.is_running_unit_tests: self._parser = argparser_class(formatter_class=NoOpHelpFormatter) else: terminal_width = shutil.get_terminal_size(fallback=(120, 24))[0] self._parser = argparser_class( - formatter_class=lambda prog: argparse.HelpFormatter(prog, width=terminal_width)) + formatter_class=lambda prog: argparse.HelpFormatter(prog, width=terminal_width) + ) super().__init__(option_cls=option_cls, command_line_only_options_cls=command_line_only_options_cls) - self._parser.add_argument("--help-all", "--help-hidden", action="help", help="Show all help options, including" - " the target-specific ones.") + self._parser.add_argument( + "--help-all", + "--help-hidden", + action="help", + help="Show all help options, including" " the target-specific ones.", + ) # noinspection PyShadowingBuiltins - def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callable[[str], T]]" = str, - default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, - group=None, help_hidden=False, **kwargs) -> T: + def add_option( + self, + name: str, + shortname=None, + *, + type: "Union[type[T], Callable[[str], T]]" = str, + default: "Union[ComputedDefaultValue[T], Optional[T], Callable[[ConfigBase, typing.Any], T]]" = None, + group=None, + help_hidden=False, + **kwargs, + ) -> T: if not self.is_needed_for_completion(name, shortname, type): # We are autocompleting and there is a prefix that won't match this option, so we just return the # default value since it won't be displayed anyway. This should noticeably speed up tab-completion. return default # pytype: disable=bad-return-type if isinstance(type, builtins.type) and issubclass(type, Enum): # Handle enums as the argparse type - assert "action" not in kwargs or kwargs[ - "action"] == "append", "action should be none or append for Enum options" + assert ( + "action" not in kwargs or kwargs["action"] == "append" + ), "action should be none or append for Enum options" assert "choices" not in kwargs, "for enum options choices are the enum names (or set enum_choices)!" if "enum_choices" in kwargs: kwargs["choices"] = tuple(t.name.lower().replace("_", "-") for t in kwargs["enum_choices"]) @@ -432,8 +525,9 @@ def add_option(self, name: str, shortname=None, *, type: "Union[type[T], Callabl # noinspection PyTypeChecker kwargs["choices"] = tuple(t.name.lower() for t in type) type = _EnumArgparseType(type) - return super().add_option(name, shortname, default=default, type=type, group=group, help_hidden=help_hidden, - **kwargs) + return super().add_option( + name, shortname, default=default, type=type, group=group, help_hidden=help_hidden, **kwargs + ) def debug_msg(self, *args, sep=" ", **kwargs) -> None: if self._parsed_args and self._parsed_args.verbose is True: @@ -453,7 +547,8 @@ def _load_command_line_args(self) -> None: exclude=self.completion_excludes, # hide these options from the output print_suppressed=True, # also include target-specific options output_stream=output, - exit_method=sys.exit) # ensure that cprofile data is written + exit_method=sys.exit, + ) # ensure that cprofile data is written else: argcomplete.autocomplete( self._parser, @@ -470,8 +565,9 @@ def _load_command_line_args(self) -> None: for x in trailing: # filter out unknown options (like -b) # exit with error - if x.startswith('-'): + if x.startswith("-"): import difflib + # There is no officially supported API to get back all option strings, but fortunately we store # all the actions here anyway all_options = getattr(self._parser, "_option_string_actions", {}).keys() @@ -512,7 +608,7 @@ def is_needed_for_completion(self, name: str, shortname: str, option_type) -> bo return True # Okay, prefix matches shortname elif option_type is bool and (comp_prefix.startswith("--no-") or self._argcomplete_prefix_includes_slash): slash_index = name.rfind("/") - negated_name = name[:slash_index + 1] + "no-" + name[slash_index + 1:] + negated_name = name[: slash_index + 1] + "no-" + name[slash_index + 1 :] if negated_name.startswith(comp_prefix[2:]): # self.debug_msg("comp_prefix '", comp_prefix, "' matches negated option: ", negated_name, sep="") return True # Okay, prefix matches negated long name @@ -524,10 +620,16 @@ def load(self) -> None: class JsonAndCommandLineConfigLoader(CommandLineConfigLoader): - def __init__(self, argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, *, - option_cls=JsonAndCommandLineConfigOption, command_line_only_options_cls=CommandLineConfigOption): - super().__init__(argparser_class, option_cls=option_cls, - command_line_only_options_cls=command_line_only_options_cls) + def __init__( + self, + argparser_class: "type[argparse.ArgumentParser]" = argparse.ArgumentParser, + *, + option_cls=JsonAndCommandLineConfigOption, + command_line_only_options_cls=CommandLineConfigOption, + ): + super().__init__( + argparser_class, option_cls=option_cls, command_line_only_options_cls=command_line_only_options_cls + ) self._config_path: "Optional[Path]" = None # Choose the default config file based on argv[0] # This allows me to have symlinks for e.g. stable-cheribuild.py release-cheribuild.py debug-cheribuild.py @@ -535,10 +637,16 @@ def __init__(self, argparser_class: "type[argparse.ArgumentParser]" = argparse.A cheribuild_rootdir = Path(__file__).absolute().parent.parent.parent self._inferred_config_prefix = self.get_config_prefix() self.default_config_path = Path(cheribuild_rootdir, self._inferred_config_prefix + "cheribuild.json") - self.path_group.add_argument("--config-file", metavar="FILE", type=str, default=str(self.default_config_path), - action=ArgparseSetGivenAction, - help="The config file that is used to load the default settings (default: '" + - str(self.default_config_path) + "')") + self.path_group.add_argument( + "--config-file", + metavar="FILE", + type=str, + default=str(self.default_config_path), + action=ArgparseSetGivenAction, + help="The config file that is used to load the default settings (default: '" + + str(self.default_config_path) + + "')", + ) @staticmethod def get_config_prefix() -> str: @@ -546,7 +654,7 @@ def get_config_prefix() -> str: suffixes = ["cheribuild", "cheribuild.py"] for suffix in suffixes: if program.endswith(suffix): - return program[0:-len(suffix)] + return program[0 : -len(suffix)] return "" def __load_json_with_comments(self, config_path: Path) -> "dict[str, Any]": @@ -565,15 +673,24 @@ def __load_json_with_comments(self, config_path: Path) -> "dict[str, Any]": result = dict() status_update("JSON config file", config_path, "was empty.") else: - result = json.loads("".join(json_lines), - object_pairs_hook=lambda o: dict_raise_on_duplicates_and_store_src(o, config_path)) - self.debug_msg("Parsed", config_path, "as", - coloured(AnsiColour.cyan, json.dumps(result, cls=MyJsonEncoder))) + result = json.loads( + "".join(json_lines), + object_pairs_hook=lambda o: dict_raise_on_duplicates_and_store_src(o, config_path), + ) + self.debug_msg( + "Parsed", config_path, "as", coloured(AnsiColour.cyan, json.dumps(result, cls=MyJsonEncoder)) + ) return result # Based on https://stackoverflow.com/a/7205107/894271 - def merge_dict_recursive(self, a: "dict[str, _LoadedConfigValue]", b: "dict[str, _LoadedConfigValue]", - included_file: Path, base_file: Path, path=None) -> dict: + def merge_dict_recursive( + self, + a: "dict[str, _LoadedConfigValue]", + b: "dict[str, _LoadedConfigValue]", + included_file: Path, + base_file: Path, + path=None, + ) -> dict: """merges b into a""" if path is None: path = [] @@ -585,8 +702,16 @@ def merge_dict_recursive(self, a: "dict[str, _LoadedConfigValue]", b: "dict[str, self.merge_dict_recursive(a[key].value, b[key].value, included_file, base_file, [*path, str(key)]) elif a[key] != b[key]: if self._parsed_args: - self.debug_msg("Overriding '" + '.'.join([*path, str(key)]) + "' value", b[key], " from", - included_file, "with value ", a[key], "from", base_file) + self.debug_msg( + "Overriding '" + ".".join([*path, str(key)]) + "' value", + b[key], + " from", + included_file, + "with value ", + a[key], + "from", + base_file, + ) else: pass # same leaf value else: @@ -599,8 +724,9 @@ def __load_json_with_includes(self, config_path: Path): result = self.__load_json_with_comments(config_path) except Exception as e: error_message("Could not load config file ", config_path, ": ", e, sep="") - if not sys.__stdin__.isatty() or not input("Invalid config file " + str(config_path) + - ". Continue? y/[N]").lower().startswith("y"): + if not sys.__stdin__.isatty() or not input( + "Invalid config file " + str(config_path) + ". Continue? y/[N]" + ).lower().startswith("y"): raise include_value = result.get("#include") if include_value: @@ -632,31 +758,51 @@ def _load_json_config_file(self) -> None: # XXX: Ideally we would always load this file and merge the two if # both exist, with the bundled config file setting new defaults. configdir = os.getenv("XDG_CONFIG_HOME") or os.path.expanduser("~/.config") - print("Checking", Path(configdir, self._config_path.name), "since", self._config_path, "doesn't exist", - file=sys.stderr) + print( + "Checking", + Path(configdir, self._config_path.name), + "since", + self._config_path, + "doesn't exist", + file=sys.stderr, + ) self._config_path = Path(configdir, self._config_path.name) if self._inferred_config_prefix: - print(coloured(AnsiColour.green, "Note: Configuration file path inferred as"), - coloured(AnsiColour.blue, self._config_path), - coloured(AnsiColour.green, "based on command name"), - file=sys.stderr) + print( + coloured(AnsiColour.green, "Note: Configuration file path inferred as"), + coloured(AnsiColour.blue, self._config_path), + coloured(AnsiColour.green, "based on command name"), + file=sys.stderr, + ) if self._config_path.exists(): self._json = self.__load_json_with_includes(self._config_path) else: if self._inferred_config_prefix: # If the user invoked foo-cheribuild.py but foo-cheribuild.json does not exist that is almost # certainly an error. Report it as such and don't - print(coloured(AnsiColour.green, "Note: Configuration file path inferred as"), - coloured(AnsiColour.blue, self._config_path), - coloured(AnsiColour.green, "based on command name"), - file=sys.stderr) - fatal_error("Configuration file ", self._config_path, "matching prefixed command was not found.", - "If this is intended pass an explicit `--config-file=/dev/null` argument.", - pretend=False) + print( + coloured(AnsiColour.green, "Note: Configuration file path inferred as"), + coloured(AnsiColour.blue, self._config_path), + coloured(AnsiColour.green, "based on command name"), + file=sys.stderr, + ) + fatal_error( + "Configuration file ", + self._config_path, + "matching prefixed command was not found.", + "If this is intended pass an explicit `--config-file=/dev/null` argument.", + pretend=False, + ) raise FileNotFoundError(self._parsed_args.config_file) - print(coloured(AnsiColour.green, "Note: Configuration file", self._config_path, - "does not exist, using only command line arguments."), - file=sys.stderr) + print( + coloured( + AnsiColour.green, + "Note: Configuration file", + self._config_path, + "does not exist, using only command line arguments.", + ), + file=sys.stderr, + ) def load(self) -> None: super().load() diff --git a/pycheribuild/config/target_info.py b/pycheribuild/config/target_info.py index d93a8d7c5..cdc24dd08 100644 --- a/pycheribuild/config/target_info.py +++ b/pycheribuild/config/target_info.py @@ -51,7 +51,8 @@ "DefaultInstallDir", "MipsFloatAbi", "NativeTargetInfo", - "TargetInfo"] + "TargetInfo", +] class CPUArchitecture(Enum): @@ -101,6 +102,7 @@ class CompilerType(Enum): """ Used by the jenkins script to detect which compiler directory should be used """ + DEFAULT_COMPILER = "default-compiler" # Default system compiler (i.e. the argument passed to cheribuild) CHERI_LLVM = "cheri-llvm" # Compile with CHERI LLVM built by cheribuild MORELLO_LLVM = "morello-llvm" # Compile with Morello LLVM built by cheribuild @@ -120,8 +122,10 @@ def clang_flags(self) -> "list[str]": if self is None: return [] # Equivalent to -ftrivial-auto-var-init=uninitialized elif self is AutoVarInit.ZERO: - return ["-ftrivial-auto-var-init=zero", - "-enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang"] + return [ + "-ftrivial-auto-var-init=zero", + "-enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang", + ] elif self is AutoVarInit.PATTERN: return ["-ftrivial-auto-var-init=pattern"] else: @@ -152,6 +156,7 @@ class DefaultInstallDir(Enum): class AbstractProject(FileSystemUtils): """A base class for (Simple)Project that exposes only the fields/methods needed in target_info.""" + _xtarget: "ClassVar[Optional[CrossCompileTarget]]" = None default_architecture: "ClassVar[Optional[CrossCompileTarget]]" needs_sysroot: "ClassVar[bool]" @@ -184,8 +189,13 @@ def warning(*args, **kwargs) -> None: warning_message(*args, **kwargs) def fatal(self, *args, sep=" ", fixit_hint=None, fatal_when_pretending=False) -> None: - fatal_error(*args, sep=sep, fixit_hint=fixit_hint, fatal_when_pretending=fatal_when_pretending, - pretend=self.config.pretend) + fatal_error( + *args, + sep=sep, + fixit_hint=fixit_hint, + fatal_when_pretending=fatal_when_pretending, + pretend=self.config.pretend, + ) @classmethod def get_crosscompile_target(cls) -> "CrossCompileTarget": @@ -194,9 +204,12 @@ def get_crosscompile_target(cls) -> "CrossCompileTarget": return target @classmethod - def get_instance(cls: "type[_AnyProject]", caller: "Optional[AbstractProject]", - config: "Optional[CheriConfig]" = None, - cross_target: "Optional[CrossCompileTarget]" = None) -> "_AnyProject": + def get_instance( + cls: "type[_AnyProject]", + caller: "Optional[AbstractProject]", + config: "Optional[CheriConfig]" = None, + cross_target: "Optional[CrossCompileTarget]" = None, + ) -> "_AnyProject": raise NotImplementedError() @classmethod @@ -316,20 +329,34 @@ def strip_tool(self) -> Path: @classmethod @abstractmethod - def essential_compiler_and_linker_flags_impl(cls, instance: "TargetInfo", *, xtarget: "CrossCompileTarget", - perform_sanity_checks=True, default_flags_only=False, - softfloat: Optional[bool] = None) -> "list[str]": + def essential_compiler_and_linker_flags_impl( + cls, + instance: "TargetInfo", + *, + xtarget: "CrossCompileTarget", + perform_sanity_checks=True, + default_flags_only=False, + softfloat: Optional[bool] = None, + ) -> "list[str]": """ :return: flags such as -target + -mabi which are needed for both compiler and linker """ ... - def get_essential_compiler_and_linker_flags(self, xtarget: "Optional[CrossCompileTarget]" = None, - perform_sanity_checks=True, default_flags_only=False, - softfloat: Optional[bool] = None) -> "list[str]": - return self.essential_compiler_and_linker_flags_impl(self, perform_sanity_checks=perform_sanity_checks, - xtarget=xtarget if xtarget is not None else self.target, - default_flags_only=default_flags_only, softfloat=softfloat) + def get_essential_compiler_and_linker_flags( + self, + xtarget: "Optional[CrossCompileTarget]" = None, + perform_sanity_checks=True, + default_flags_only=False, + softfloat: Optional[bool] = None, + ) -> "list[str]": + return self.essential_compiler_and_linker_flags_impl( + self, + perform_sanity_checks=perform_sanity_checks, + xtarget=xtarget if xtarget is not None else self.target, + default_flags_only=default_flags_only, + softfloat=softfloat, + ) @property def additional_executable_link_flags(self) -> "list[str]": @@ -412,8 +439,9 @@ def must_link_statically(self) -> bool: return False @final - def get_rootfs_project(self, *, t: "type[_AnyProject]", caller: AbstractProject, - xtarget: "Optional[CrossCompileTarget]" = None) -> _AnyProject: + def get_rootfs_project( + self, *, t: "type[_AnyProject]", caller: AbstractProject, xtarget: "Optional[CrossCompileTarget]" = None + ) -> _AnyProject: if xtarget is None: xtarget = self.target xtarget = xtarget.get_rootfs_target() @@ -448,11 +476,20 @@ def is_freebsd(cls) -> bool: def is_cheribsd(cls) -> bool: return False - def run_cheribsd_test_script(self, script_name, *script_args, kernel_path=None, disk_image_path=None, - mount_builddir=True, mount_sourcedir=False, mount_sysroot=False, - use_full_disk_image=False, mount_installdir=False, - use_benchmark_kernel_by_default=False, - rootfs_alternate_kernel_dir=None) -> None: + def run_cheribsd_test_script( + self, + script_name, + *script_args, + kernel_path=None, + disk_image_path=None, + mount_builddir=True, + mount_sourcedir=False, + mount_sysroot=False, + use_full_disk_image=False, + mount_installdir=False, + use_benchmark_kernel_by_default=False, + rootfs_alternate_kernel_dir=None, + ) -> None: raise ValueError("run_cheribsd_test_script only supports CheriBSD targets") @classmethod @@ -512,8 +549,11 @@ def host_c_preprocessor(config: CheriConfig) -> Path: def pkgconfig_candidates(self, prefix: Path) -> "list[str]": """:return: a list of potential candidates for pkgconfig .pc files inside prefix""" - return [str(prefix / self.default_libdir / "pkgconfig"), str(prefix / "share/pkgconfig"), - str(prefix / "libdata/pkgconfig")] + return [ + str(prefix / self.default_libdir / "pkgconfig"), + str(prefix / "share/pkgconfig"), + str(prefix / "libdata/pkgconfig"), + ] class NativeTargetInfo(TargetInfo): @@ -646,8 +686,9 @@ def _compat_abi_suffix(self) -> str: # Directory suffix for compat ABI (currently only "64"/"" should be valid) if _is_native_purecap() and not self.target.is_cheri_purecap(): return "64" - assert _is_native_purecap() == self.target.is_cheri_purecap(), \ - "Building purecap natively is only supported on purecap installations" + assert ( + _is_native_purecap() == self.target.is_cheri_purecap() + ), "Building purecap natively is only supported on purecap installations" return "" @property @@ -689,9 +730,15 @@ def pkgconfig_candidates(self, prefix: Path) -> "list[str]": return result @classmethod - def essential_compiler_and_linker_flags_impl(cls, instance: "TargetInfo", *, xtarget: "CrossCompileTarget", - perform_sanity_checks=True, default_flags_only=False, - softfloat: Optional[bool] = None) -> "list[str]": + def essential_compiler_and_linker_flags_impl( + cls, + instance: "TargetInfo", + *, + xtarget: "CrossCompileTarget", + perform_sanity_checks=True, + default_flags_only=False, + softfloat: Optional[bool] = None, + ) -> "list[str]": result = [] if instance.project.auto_var_init != AutoVarInit.NONE: compiler = instance.project.get_compiler_info(instance.c_compiler) @@ -704,8 +751,9 @@ def essential_compiler_and_linker_flags_impl(cls, instance: "TargetInfo", *, xta if valid_clang_version: result += instance.project.auto_var_init.clang_flags() else: - instance.project.fatal("Requested automatic variable initialization, but don't know how to for", - compiler) + instance.project.fatal( + "Requested automatic variable initialization, but don't know how to for", compiler + ) if xtarget.is_cheri_hybrid(): if xtarget.is_aarch64(include_purecap=True): result.append("-mabi=aapcs") @@ -718,17 +766,25 @@ class CrossCompileTarget: # Currently the same for all targets DEFAULT_SUBOBJECT_BOUNDS: str = "conservative" - def __init__(self, arch_suffix: str, cpu_architecture: CPUArchitecture, target_info_cls: "type[TargetInfo]", - *, is_cheri_purecap=False, is_cheri_hybrid=False, extra_target_suffix: str = "", - check_conflict_with: "Optional[CrossCompileTarget]" = None, - rootfs_target: "Optional[CrossCompileTarget]" = None, - non_cheri_target: "Optional[CrossCompileTarget]" = None, - hybrid_target: "Optional[CrossCompileTarget]" = None, - purecap_target: "Optional[CrossCompileTarget]" = None, - non_cheri_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None, - non_cheri_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, - hybrid_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, - purecap_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None) -> None: + def __init__( + self, + arch_suffix: str, + cpu_architecture: CPUArchitecture, + target_info_cls: "type[TargetInfo]", + *, + is_cheri_purecap=False, + is_cheri_hybrid=False, + extra_target_suffix: str = "", + check_conflict_with: "Optional[CrossCompileTarget]" = None, + rootfs_target: "Optional[CrossCompileTarget]" = None, + non_cheri_target: "Optional[CrossCompileTarget]" = None, + hybrid_target: "Optional[CrossCompileTarget]" = None, + purecap_target: "Optional[CrossCompileTarget]" = None, + non_cheri_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None, + non_cheri_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, + hybrid_for_purecap_rootfs_target: "Optional[CrossCompileTarget]" = None, + purecap_for_hybrid_rootfs_target: "Optional[CrossCompileTarget]" = None, + ) -> None: assert not arch_suffix.startswith("-"), arch_suffix assert not extra_target_suffix or extra_target_suffix.startswith("-"), extra_target_suffix name_prefix = target_info_cls.shortname @@ -774,9 +830,15 @@ def __init__(self, arch_suffix: str, cpu_architecture: CPUArchitecture, target_i def _set_from(self, other_target: "CrossCompileTarget") -> None: if self is other_target: return - for attr in ("_hybrid_target", "_non_cheri_target", "_purecap_target", "_non_cheri_for_hybrid_rootfs_target", - "_non_cheri_for_purecap_rootfs_target", "_hybrid_for_purecap_rootfs_target", - "_purecap_for_hybrid_rootfs_target"): + for attr in ( + "_hybrid_target", + "_non_cheri_target", + "_purecap_target", + "_non_cheri_for_hybrid_rootfs_target", + "_non_cheri_for_purecap_rootfs_target", + "_hybrid_for_purecap_rootfs_target", + "_purecap_for_hybrid_rootfs_target", + ): if getattr(self, attr) is None and getattr(other_target, attr) is not None: setattr(self, attr, getattr(other_target, attr)) # noinspection PyProtectedMember @@ -787,10 +849,13 @@ def _set_for(self, other_target: "Optional[CrossCompileTarget]", also_set_other= if other_target is not None and self is not other_target: if self._is_cheri_hybrid: if self._rootfs_target is not None: - assert self._rootfs_target._is_cheri_purecap, \ - "Only support purecap separate rootfs for hybrid targets" - assert other_target._hybrid_for_purecap_rootfs_target is None or \ - other_target._hybrid_for_purecap_rootfs_target is self, "Already set?" + assert ( + self._rootfs_target._is_cheri_purecap + ), "Only support purecap separate rootfs for hybrid targets" + assert ( + other_target._hybrid_for_purecap_rootfs_target is None + or other_target._hybrid_for_purecap_rootfs_target is self + ), "Already set?" other_target._hybrid_for_purecap_rootfs_target = self self._hybrid_for_purecap_rootfs_target = self else: @@ -799,10 +864,13 @@ def _set_for(self, other_target: "Optional[CrossCompileTarget]", also_set_other= self._hybrid_target = self elif self._is_cheri_purecap: if self._rootfs_target is not None: - assert self._rootfs_target._is_cheri_hybrid, \ - "Only support hybrid separate rootfs for purecap targets" - assert other_target._purecap_for_hybrid_rootfs_target is None or \ - other_target._purecap_for_hybrid_rootfs_target is self, "Already set?" + assert ( + self._rootfs_target._is_cheri_hybrid + ), "Only support hybrid separate rootfs for purecap targets" + assert ( + other_target._purecap_for_hybrid_rootfs_target is None + or other_target._purecap_for_hybrid_rootfs_target is self + ), "Already set?" other_target._purecap_for_hybrid_rootfs_target = self self._purecap_for_hybrid_rootfs_target = self else: @@ -812,20 +880,25 @@ def _set_for(self, other_target: "Optional[CrossCompileTarget]", also_set_other= else: if self._rootfs_target is not None: if self._rootfs_target._is_cheri_hybrid: - assert other_target._non_cheri_for_hybrid_rootfs_target is None or \ - other_target._non_cheri_for_hybrid_rootfs_target is self, "Already set?" + assert ( + other_target._non_cheri_for_hybrid_rootfs_target is None + or other_target._non_cheri_for_hybrid_rootfs_target is self + ), "Already set?" other_target._non_cheri_for_hybrid_rootfs_target = self self._non_cheri_for_hybrid_rootfs_target = self else: assert self._rootfs_target._is_cheri_purecap, "Separate non-CHERI rootfs for non-CHERI target?" - assert other_target._non_cheri_for_purecap_rootfs_target is None or \ - other_target._non_cheri_for_purecap_rootfs_target is self, "Already set?" + assert ( + other_target._non_cheri_for_purecap_rootfs_target is None + or other_target._non_cheri_for_purecap_rootfs_target is self + ), "Already set?" other_target._non_cheri_for_purecap_rootfs_target = self self._non_cheri_for_purecap_rootfs_target = self else: assert self._rootfs_target is None, "Separate rootfs targets only supported for CHERI targets" - assert other_target._non_cheri_target is None or other_target._non_cheri_target is self, \ - "Already set?" + assert ( + other_target._non_cheri_target is None or other_target._non_cheri_target is self + ), "Already set?" other_target._non_cheri_target = self self._non_cheri_target = self if also_set_other: @@ -962,16 +1035,24 @@ def get_non_cheri_target(self) -> "CrossCompileTarget": raise ValueError("Don't know non-CHERI version of " + repr(self)) def get_non_cheri_for_hybrid_rootfs_target(self) -> "CrossCompileTarget": - if not self._is_cheri_purecap and not self._is_cheri_hybrid and self._rootfs_target is not None and \ - self._rootfs_target._is_cheri_hybrid: + if ( + not self._is_cheri_purecap + and not self._is_cheri_hybrid + and self._rootfs_target is not None + and self._rootfs_target._is_cheri_hybrid + ): return self elif self._non_cheri_for_hybrid_rootfs_target is not None: return self._non_cheri_for_hybrid_rootfs_target raise ValueError("Don't know non-CHERI for hybrid rootfs version of " + repr(self)) def get_non_cheri_for_purecap_rootfs_target(self) -> "CrossCompileTarget": - if not self._is_cheri_purecap and not self._is_cheri_hybrid and self._rootfs_target is not None and \ - self._rootfs_target._is_cheri_purecap: + if ( + not self._is_cheri_purecap + and not self._is_cheri_hybrid + and self._rootfs_target is not None + and self._rootfs_target._is_cheri_purecap + ): return self elif self._non_cheri_for_purecap_rootfs_target is not None: return self._non_cheri_for_purecap_rootfs_target @@ -1006,9 +1087,16 @@ def __repr__(self) -> str: def _dump_target_relations(self) -> None: self_repr = repr(self) - for n in ('non_cheri', 'hybrid', 'purecap', 'non_cheri_for_hybrid_rootfs', 'non_cheri_for_purecap_rootfs', - 'hybrid_for_purecap_rootfs', 'purecap_for_hybrid_rootfs'): - k = '_' + n + '_target' + for n in ( + "non_cheri", + "hybrid", + "purecap", + "non_cheri_for_hybrid_rootfs", + "non_cheri_for_purecap_rootfs", + "hybrid_for_purecap_rootfs", + "purecap_for_hybrid_rootfs", + ): + k = "_" + n + "_target" v = self.__dict__[k] print(self_repr + "." + n + ": " + repr(v)) @@ -1041,14 +1129,20 @@ class BasicCompilationTargets: # Some projects (LLVM, QEMU, GDB, etc.) don't build as purecap binaries, so we have to build them hybrid instead. if _is_native_purecap(): NATIVE = CrossCompileTarget("native", _native_cpu_arch(), NativeTargetInfo, is_cheri_purecap=True) - NATIVE_HYBRID = CrossCompileTarget("native-hybrid", _native_cpu_arch(), NativeTargetInfo, is_cheri_hybrid=True, - purecap_target=NATIVE, check_conflict_with=NATIVE) + NATIVE_HYBRID = CrossCompileTarget( + "native-hybrid", + _native_cpu_arch(), + NativeTargetInfo, + is_cheri_hybrid=True, + purecap_target=NATIVE, + check_conflict_with=NATIVE, + ) NATIVE_NON_PURECAP = NATIVE_HYBRID ALL_NATIVE = (NATIVE, NATIVE_HYBRID) else: NATIVE = CrossCompileTarget("native", _native_cpu_arch(), NativeTargetInfo) NATIVE_NON_PURECAP = NATIVE - ALL_NATIVE = (NATIVE, ) + ALL_NATIVE = (NATIVE,) NATIVE_IF_FREEBSD = ALL_NATIVE if OSInfo.IS_FREEBSD else tuple() NATIVE_IF_LINUX = ALL_NATIVE if OSInfo.IS_LINUX else tuple() NATIVE_IF_MACOS = ALL_NATIVE if OSInfo.IS_MAC else tuple() diff --git a/pycheribuild/filesystemutils.py b/pycheribuild/filesystemutils.py index 4d6fc5722..7abc7e427 100644 --- a/pycheribuild/filesystemutils.py +++ b/pycheribuild/filesystemutils.py @@ -55,7 +55,7 @@ def _delete_directories(self, *dirs) -> None: run_command("rm", "-rf", *dirs, config=self.config) def clean_directory(self, path: Path, keep_root=False, ensure_dir_exists=True) -> None: - """ After calling this function path will be an empty directory + """After calling this function path will be an empty directory :param path: the directory to delete :param keep_root: Whether to keep the root directory (e.g. for NFS exported mountpoints) :param ensure_dir_exists: Create the cleaned directory if it doesn't exist @@ -84,8 +84,9 @@ def run(self) -> None: except Exception as e: warning_message("Could not remove directory", self.path, e) - def async_clean_directory(self, path: Path, *, keep_root=False, - keep_dirs: "Optional[list[str]]" = None) -> ThreadJoiner: + def async_clean_directory( + self, path: Path, *, keep_root=False, keep_dirs: "Optional[list[str]]" = None + ) -> ThreadJoiner: """ Delete a directory in the background (e.g. deleting the cheribsd build directory delays the build a lot) :: @@ -194,8 +195,9 @@ def read_file(self, file: Path) -> str: with file.open("r", encoding="utf-8") as f: return f.read() - def write_file(self, file: Path, contents: str, *, overwrite: bool, never_print_cmd=False, mode=None, - print_verbose_only=True) -> None: + def write_file( + self, file: Path, contents: str, *, overwrite: bool, never_print_cmd=False, mode=None, print_verbose_only=True + ) -> None: """ :param file: The target path to write contents to :param contents: the contents of the new file @@ -205,8 +207,14 @@ def write_file(self, file: Path, contents: str, *, overwrite: bool, never_print_ :param print_verbose_only: only print contents in verbose mode """ if not never_print_cmd: - print_command("echo", contents, colour=AnsiColour.green, output_file=file, - print_verbose_only=print_verbose_only, config=self.config) + print_command( + "echo", + contents, + colour=AnsiColour.green, + output_file=file, + print_verbose_only=print_verbose_only, + config=self.config, + ) if self.config.pretend: return if not overwrite and file.exists(): @@ -221,8 +229,9 @@ def write_file(self, file: Path, contents: str, *, overwrite: bool, never_print_ # would require create_symlinks to inherit some of ln's heuristics about # whether to create a new file called src.basename() inside dest, whether # to use dest.parent or dest, etc. - def create_symlink(self, src: Path, dest: Path, *, relative=True, cwd: "Optional[Path]" = None, - print_verbose_only=True): + def create_symlink( + self, src: Path, dest: Path, *, relative=True, cwd: "Optional[Path]" = None, print_verbose_only=True + ): assert dest.is_absolute() or cwd is not None if not cwd: cwd = dest.parent @@ -235,8 +244,15 @@ def create_symlink(self, src: Path, dest: Path, *, relative=True, cwd: "Optional else: run_command("ln", "-fsn", src, dest, cwd=cwd, print_verbose_only=print_verbose_only, config=self.config) - def create_symlinks(self, srcs: typing.Iterable[Path], destdir: Path, *, relative=True, - cwd: "Optional[Path]" = None, print_verbose_only=True): + def create_symlinks( + self, + srcs: typing.Iterable[Path], + destdir: Path, + *, + relative=True, + cwd: "Optional[Path]" = None, + print_verbose_only=True, + ): assert destdir.is_absolute() or cwd is not None if not cwd: cwd = destdir @@ -247,8 +263,15 @@ def create_symlinks(self, srcs: typing.Iterable[Path], destdir: Path, *, relativ destdir = destdir.relative_to(cwd) srcs = list(srcs) if srcs: - run_command("ln", "-fs", *srcs, str(destdir) + "/", cwd=cwd, print_verbose_only=print_verbose_only, - config=self.config) + run_command( + "ln", + "-fs", + *srcs, + str(destdir) + "/", + cwd=cwd, + print_verbose_only=print_verbose_only, + config=self.config, + ) def move_file(self, src: Path, dest: Path, force=False, create_dirs=True) -> None: if not src.exists(): @@ -258,8 +281,9 @@ def move_file(self, src: Path, dest: Path, force=False, create_dirs=True) -> Non self.makedirs(dest.parent) run_command([*cmd, str(src), str(dest)], config=self.config) - def install_file(self, src: Path, dest: Path, *, force=False, create_dirs=True, print_verbose_only=True, - mode=None) -> None: + def install_file( + self, src: Path, dest: Path, *, force=False, create_dirs=True, print_verbose_only=True, mode=None + ) -> None: if force: print_command("cp", "-f", src, dest, print_verbose_only=print_verbose_only, config=self.config) else: @@ -293,7 +317,7 @@ def rewrite_file(self, file: Path, rewrite: "Callable[[typing.Iterable[str]], ty with file.open("r+", encoding="utf-8") as f: lines = list(rewrite(f.read().splitlines())) f.seek(0) - f.writelines(map(lambda line: line + '\n', lines)) + f.writelines(map(lambda line: line + "\n", lines)) f.truncate() def add_unique_line_to_file(self, file: Path, line: str) -> None: @@ -306,6 +330,7 @@ def do_replace(old_lines: "typing.Iterable[str]"): for old, new in replacements.items(): line = line.replace(old, new) yield line + status_update("Remapping ", replacements, " in ", file, sep="") self.rewrite_file(file, do_replace) @@ -313,8 +338,13 @@ def do_replace(old_lines: "typing.Iterable[str]"): def triple_prefixes_for_binaries(self) -> typing.Iterable[str]: raise ValueError("Must override triple_prefixes_for_binaries to use create_triple_prefixed_symlinks!") - def create_triple_prefixed_symlinks(self, tool_path: Path, tool_name: "Optional[str]" = None, - create_unprefixed_link: bool = False, cwd: "Optional[str]" = None) -> None: + def create_triple_prefixed_symlinks( + self, + tool_path: Path, + tool_name: "Optional[str]" = None, + create_unprefixed_link: bool = False, + cwd: "Optional[str]" = None, + ) -> None: """ Create mips4-unknown-freebsd, cheri-unknown-freebsd and mips64-unknown-freebsd prefixed symlinks for build tools like clang, ld, etc. @@ -328,8 +358,9 @@ def create_triple_prefixed_symlinks(self, tool_path: Path, tool_name: "Optional[ if not tool_name: tool_name = tool_path.name if not tool_path.is_file(): - fatal_error("Attempting to create symlink to non-existent build tool_path:", tool_path, - pretend=self.config.pretend) + fatal_error( + "Attempting to create symlink to non-existent build tool_path:", tool_path, pretend=self.config.pretend + ) # a prefixed tool_path was installed -> create link such as mips4-unknown-freebsd-ld -> ld if create_unprefixed_link: @@ -342,8 +373,9 @@ def create_triple_prefixed_symlinks(self, tool_path: Path, tool_name: "Optional[ # if self.config.verbose: # print(coloured(AnsiColour.yellow, "Not overwriting", link, "because it is the target")) continue - run_command("ln", "-fsn", tool_path.name, target + tool_name, cwd=cwd, print_verbose_only=True, - config=self.config) + run_command( + "ln", "-fsn", tool_path.name, target + tool_name, cwd=cwd, print_verbose_only=True, config=self.config + ) @staticmethod # Not cached since another target could write to this dir: @functools.lru_cache(maxsize=20) @@ -364,6 +396,7 @@ def realpath(p: Path) -> Path: def sha256sum(self, file: Path) -> str: # Based on https://stackoverflow.com/a/44873382/894271 import hashlib # rarely need, so imported on demand to reduce startup time + h = hashlib.sha256() b = bytearray(128 * 1024) mv = memoryview(b) @@ -371,7 +404,7 @@ def sha256sum(self, file: Path) -> str: fatal_error("Cannot hash", file, "since it does not exist", pretend=self.config.pretend) if self.config.pretend: return "0" - with file.open('rb', buffering=0) as f: + with file.open("rb", buffering=0) as f: # PyCharm thinks .readinto is not supported. # noinspection PyUnresolvedReferences for n in iter(lambda: f.readinto(mv), 0): # pytype: disable=wrong-arg-types diff --git a/pycheribuild/jenkins.py b/pycheribuild/jenkins.py index c980914bc..901e75d33 100644 --- a/pycheribuild/jenkins.py +++ b/pycheribuild/jenkins.py @@ -65,8 +65,12 @@ def load(self) -> None: def finalize_options(self, available_targets: "list[str]", **kwargs) -> None: target_option = self._parser.add_argument( - "targets", metavar="TARGET", nargs=argparse.ZERO_OR_MORE, help="The target to build", - choices=[*available_targets, EXTRACT_SDK_TARGET, RUN_EVERYTHING_TARGET]) + "targets", + metavar="TARGET", + nargs=argparse.ZERO_OR_MORE, + help="The target to build", + choices=[*available_targets, EXTRACT_SDK_TARGET, RUN_EVERYTHING_TARGET], + ) if self.is_completing_arguments: try: import argcomplete @@ -78,12 +82,19 @@ def finalize_options(self, available_targets: "list[str]", **kwargs) -> None: self._parser, always_complete_options=None, # don't print -/-- by default print_suppressed=True, # also include target-specific options - ) + ) class SdkArchive: - def __init__(self, cheri_config: JenkinsConfig, name, *, required_globs: "Optional[list[str]]" = None, - extra_args: "Optional[list[str]]" = None, output_dir: Path): + def __init__( + self, + cheri_config: JenkinsConfig, + name, + *, + required_globs: "Optional[list[str]]" = None, + extra_args: "Optional[list[str]]" = None, + output_dir: Path, + ): self.output_dir = output_dir self.cheri_config = cheri_config self.archive = cheri_config.workspace / name @@ -93,8 +104,11 @@ def __init__(self, cheri_config: JenkinsConfig, name, *, required_globs: "Option def extract(self) -> None: assert self.archive.exists(), str(self.archive) self.cheri_config.FS.makedirs(self.output_dir) - run_command(["tar", "xf", self.archive, "-C", self.output_dir, *self.extra_args], - cwd=self.cheri_config.workspace, config=self.cheri_config) + run_command( + ["tar", "xf", self.archive, "-C", self.output_dir, *self.extra_args], + cwd=self.cheri_config.workspace, + config=self.cheri_config, + ) self.check_required_files() def check_required_files(self, fatal=True) -> bool: @@ -116,9 +130,13 @@ def __repr__(self) -> str: def get_sdk_archives(cheri_config, needs_cheribsd_sysroot: bool) -> "list[SdkArchive]": - clang_archive = SdkArchive(cheri_config, cheri_config.compiler_archive_name, - output_dir=cheri_config.compiler_archive_output_path, - required_globs=["bin/clang"], extra_args=["--strip-components", "1"]) + clang_archive = SdkArchive( + cheri_config, + cheri_config.compiler_archive_name, + output_dir=cheri_config.compiler_archive_output_path, + required_globs=["bin/clang"], + extra_args=["--strip-components", "1"], + ) all_archives = [] if clang_archive.archive.exists(): all_archives.append(clang_archive) @@ -128,12 +146,19 @@ def get_sdk_archives(cheri_config, needs_cheribsd_sysroot: bool) -> "list[SdkArc return all_archives # only need the clang archive # if we only extracted the compiler, extract the sysroot now extra_args = ["--strip-components", "1"] - sysroot_archive = SdkArchive(cheri_config, cheri_config.sysroot_archive_name, - output_dir=cheri_config.sysroot_archive_output_path, - required_globs=["usr/include"], extra_args=extra_args) + sysroot_archive = SdkArchive( + cheri_config, + cheri_config.sysroot_archive_name, + output_dir=cheri_config.sysroot_archive_output_path, + required_globs=["usr/include"], + extra_args=extra_args, + ) if not sysroot_archive.archive.exists(): - warning_message("Project needs a sysroot archive but ", sysroot_archive.archive, - "is missing. Will attempt to build anyway but build will most likely fail.") + warning_message( + "Project needs a sysroot archive but ", + sysroot_archive.archive, + "is missing. Will attempt to build anyway but build will most likely fail.", + ) run_command("ls", "-la", cwd=cheri_config.workspace, config=cheri_config) return all_archives else: @@ -141,7 +166,7 @@ def get_sdk_archives(cheri_config, needs_cheribsd_sysroot: bool) -> "list[SdkArc # Old sysroot archives had a leading ./, newer ones don't anymore # TODO: remove when master has been updated contents = run_command("tar", "tf", sysroot_archive.archive, capture_output=True, config=cheri_config) - if contents.stdout.startswith(b'./'): + if contents.stdout.startswith(b"./"): warning_message("Old sysroot archive detected, stripping one more path component") sysroot_archive.extra_args = ["--strip-components", "2"] return all_archives @@ -157,24 +182,27 @@ def extract_sdk_archives(cheri_config: JenkinsConfig, archives: "list[SdkArchive archive.extract() if not expected_bindir.exists(): - fatal_error("SDK bin dir", expected_bindir, "does not exist after extracting sysroot archives!", - pretend=cheri_config.pretend) + fatal_error( + "SDK bin dir", + expected_bindir, + "does not exist after extracting sysroot archives!", + pretend=cheri_config.pretend, + ) # Use llvm-ar/llvm-ranlib or the host ar/ranlib if they ar/ranlib are missing from archive for tool in ("ar", "ranlib", "nm"): if not (expected_bindir / tool).exists(): # If llvm-ar/ranlib/nm exists use that if (expected_bindir / ("llvm-" + tool)).exists(): - cheri_config.FS.create_symlink(expected_bindir / ("llvm-" + tool), - expected_bindir / tool, relative=True) + cheri_config.FS.create_symlink( + expected_bindir / ("llvm-" + tool), expected_bindir / tool, relative=True + ) else: # otherwise fall back to the /usr/bin version - cheri_config.FS.create_symlink(Path(shutil.which(tool)), expected_bindir / tool, - relative=False) + cheri_config.FS.create_symlink(Path(shutil.which(tool)), expected_bindir / tool, relative=False) if not (expected_bindir / "ld").exists(): status_update("Adding missing $SDK/ld link to ld.lld") - cheri_config.FS.create_symlink(expected_bindir / "ld.lld", - expected_bindir / "ld", relative=True) + cheri_config.FS.create_symlink(expected_bindir / "ld.lld", expected_bindir / "ld", relative=True) def create_sdk_from_archives(cheri_config: JenkinsConfig, needs_cheribsd_sysroot, extract_all: bool) -> None: @@ -219,7 +247,8 @@ def _jenkins_main() -> None: # special target to extract the sdk if JenkinsAction.EXTRACT_SDK in cheri_config.action or ( - len(cheri_config.targets) > 0 and cheri_config.targets[0] == EXTRACT_SDK_TARGET): + len(cheri_config.targets) > 0 and cheri_config.targets[0] == EXTRACT_SDK_TARGET + ): create_sdk_from_archives(cheri_config, not cheri_config.extract_compiler_only, extract_all=True) sys.exit() @@ -284,7 +313,9 @@ def create_tarball(cheri_config) -> None: tar_flags = ["--invalid-flag"] if bsdtar_path: bsdtar_version = get_program_version( - Path(bsdtar_path), regex=b"bsdtar\\s+(\\d+)\\.(\\d+)\\.?(\\d+)?", config=cheri_config, + Path(bsdtar_path), + regex=b"bsdtar\\s+(\\d+)\\.(\\d+)\\.?(\\d+)?", + config=cheri_config, ) if bsdtar_version > (3, 0, 0): # Only newer versions support --uid/--gid diff --git a/pycheribuild/jenkins_utils.py b/pycheribuild/jenkins_utils.py index 26570880e..1480f59d9 100644 --- a/pycheribuild/jenkins_utils.py +++ b/pycheribuild/jenkins_utils.py @@ -41,7 +41,8 @@ def jenkins_override_install_dirs_hack(cheri_config: CheriConfig, install_prefix expected_install_path = Path(f"{cheri_config.output_root}{install_prefix}") # Ugly workaround to override all install dirs to go to the tarball all_targets = [ - x for x in target_manager.targets(cheri_config) + x + for x in target_manager.targets(cheri_config) if not isinstance(x, (SimpleTargetAlias, MultiArchTargetAlias)) and issubclass(x.project_class, Project) ] for target in all_targets: diff --git a/pycheribuild/mtree.py b/pycheribuild/mtree.py index c4bd28bfd..c73995a2a 100644 --- a/pycheribuild/mtree.py +++ b/pycheribuild/mtree.py @@ -95,6 +95,7 @@ def escape(s): # Note: we only handle spaces here since we haven't seen any other special characters being use. If they do # exist in practise we can just update this code to handle them too. return s.replace(" ", "\\s") + return escape(self.path) + " " + " ".join(k + "=" + shlex.quote(v) for k, v in self.attributes.items()) def __repr__(self) -> str: @@ -102,8 +103,13 @@ def __repr__(self) -> str: class MtreeFile: - def __init__(self, *, verbose: bool, file: "Union[io.StringIO, Path, typing.IO, None]" = None, - contents_root: "Optional[Path]" = None): + def __init__( + self, + *, + verbose: bool, + file: "Union[io.StringIO, Path, typing.IO, None]" = None, + contents_root: "Optional[Path]" = None, + ): self.verbose = verbose self._mtree: "dict[str, MtreeEntry]" = OrderedDict() if file: @@ -173,8 +179,17 @@ def infer_mode_string(path: Path, should_be_dir) -> str: return "0600" return result - def add_file(self, file: "Optional[Path]", path_in_image, mode=None, uname="root", gname="wheel", - print_status=True, parent_dir_mode=None, symlink_dest: "Optional[str]" = None): + def add_file( + self, + file: "Optional[Path]", + path_in_image, + mode=None, + uname="root", + gname="wheel", + print_status=True, + parent_dir_mode=None, + symlink_dest: "Optional[str]" = None, + ): if isinstance(path_in_image, Path): path_in_image = str(path_in_image) assert not path_in_image.startswith("/") @@ -203,8 +218,14 @@ def add_file(self, file: "Optional[Path]", path_in_image, mode=None, uname="root # now add the actual entry (with contents=/path/to/file) contents_path = str(file.absolute()) last_attrib = ("contents", contents_path) - self.add_dir(str(Path(path_in_image).parent), mode=parent_dir_mode, uname=uname, gname=gname, - reference_dir=reference_dir, print_status=print_status) + self.add_dir( + str(Path(path_in_image).parent), + mode=parent_dir_mode, + uname=uname, + gname=gname, + reference_dir=reference_dir, + print_status=print_status, + ) attribs = OrderedDict([("type", mtree_type), ("uname", uname), ("gname", gname), ("mode", mode), last_attrib]) if print_status: if "link" in attribs: @@ -269,7 +290,7 @@ def exclude_matching(self, globs, exceptions=None, print_status=False) -> None: # glob must be anchored at the root (./) or start with a pattern assert glob[:2] == "./" or glob[:1] == "?" or glob[:1] == "*" paths_to_remove = set() - for (path, entry) in self._mtree.items(): + for path, entry in self._mtree.items(): for glob in globs: if fnmatch.fnmatch(path, glob): delete = True @@ -286,6 +307,7 @@ def exclude_matching(self, globs, exceptions=None, print_status=False) -> None: def __repr__(self) -> str: import pprint + return "" def write(self, output: "Union[io.StringIO,Path,typing.IO]", *, pretend): diff --git a/pycheribuild/processutils.py b/pycheribuild/processutils.py index 2b1cbdcfd..e13df1c89 100644 --- a/pycheribuild/processutils.py +++ b/pycheribuild/processutils.py @@ -68,7 +68,8 @@ "print_command", "run_and_kill_children_on_exit", "run_command", - "set_env"] + "set_env", +] def __filter_env(env: "dict[str, str]") -> "dict[str, str]": @@ -178,8 +179,16 @@ def _restore_flags(self) -> None: new_flags = fcntl.fcntl(self.fd, fcntl.F_GETFL) if new_flags == self.flags: return - warning_message(self.context, "FD flags for", self.fd.name, "changed from", hex(self.flags), - "to", hex(new_flags), "- resetting them.") + warning_message( + self.context, + "FD flags for", + self.fd.name, + "changed from", + hex(self.flags), + "to", + hex(new_flags), + "- resetting them.", + ) fcntl.fcntl(self.fd, fcntl.F_SETFL, self.flags) new_flags = fcntl.fcntl(self.fd, fcntl.F_GETFL) if new_flags != self.flags: @@ -226,9 +235,18 @@ def keep_terminal_sane(gave_tty_control=False, command: Optional[list] = None): stderr_state.restore() -def print_command(arg1: "Union[str, typing.Sequence[typing.Any]]", *remaining_args, output_file=None, - colour=AnsiColour.yellow, cwd=None, env=None, sep=" ", print_verbose_only=False, - config: ConfigBase, **kwargs): +def print_command( + arg1: "Union[str, typing.Sequence[typing.Any]]", + *remaining_args, + output_file=None, + colour=AnsiColour.yellow, + cwd=None, + env=None, + sep=" ", + print_verbose_only=False, + config: ConfigBase, + **kwargs, +): if config.quiet or (print_verbose_only and not config.verbose): return # also allow passing a single string @@ -290,11 +308,13 @@ def check_call_handle_noexec(cmdline: "list[str]", **kwargs): if interpreter: with keep_terminal_sane(command=cmdline): return subprocess.check_call(interpreter + cmdline, **kwargs) - raise _make_called_process_error(e.errno, cmdline, cwd=kwargs.get("cwd", None), - stderr=str(e).encode("utf-8")) from e + raise _make_called_process_error( + e.errno, cmdline, cwd=kwargs.get("cwd", None), stderr=str(e).encode("utf-8") + ) from e except FileNotFoundError as e: - raise _make_called_process_error(e.errno, cmdline, cwd=kwargs.get("cwd", None), - stderr=str(e).encode("utf-8")) from e + raise _make_called_process_error( + e.errno, cmdline, cwd=kwargs.get("cwd", None), stderr=str(e).encode("utf-8") + ) from e def popen_handle_noexec(cmdline: "list[str]", **kwargs) -> subprocess.Popen: @@ -304,11 +324,13 @@ def popen_handle_noexec(cmdline: "list[str]", **kwargs) -> subprocess.Popen: interpreter = get_interpreter(cmdline) if interpreter: return subprocess.Popen(interpreter + cmdline, **kwargs) - raise _make_called_process_error(e.errno, cmdline, cwd=kwargs.get("cwd", None), - stderr=str(e).encode("utf-8")) from e + raise _make_called_process_error( + e.errno, cmdline, cwd=kwargs.get("cwd", None), stderr=str(e).encode("utf-8") + ) from e except FileNotFoundError as e: - raise _make_called_process_error(e.errno, cmdline, cwd=kwargs.get("cwd", None), - stderr=str(e).encode("utf-8")) from e + raise _make_called_process_error( + e.errno, cmdline, cwd=kwargs.get("cwd", None), stderr=str(e).encode("utf-8") + ) from e @contextlib.contextmanager @@ -335,7 +357,7 @@ def _new_tty_foreground_process_group() -> None: raise e with suppress_sigttou(): try: - with scoped_open('/dev/tty', os.O_RDWR, ignore_open_error=True) as tty: + with scoped_open("/dev/tty", os.O_RDWR, ignore_open_error=True) as tty: if tty is not None: os.tcsetpgrp(tty, os.getpgrp()) except Exception as e: @@ -362,10 +384,12 @@ def __exit__(self, *args) -> None: pass -def popen(cmdline, print_verbose_only=False, run_in_pretend_mode=False, *, config: ConfigBase, - **kwargs) -> subprocess.Popen: - print_command(cmdline, cwd=kwargs.get("cwd"), env=kwargs.get("env"), config=config, - print_verbose_only=print_verbose_only) +def popen( + cmdline, print_verbose_only=False, run_in_pretend_mode=False, *, config: ConfigBase, **kwargs +) -> subprocess.Popen: + print_command( + cmdline, cwd=kwargs.get("cwd"), env=kwargs.get("env"), config=config, print_verbose_only=print_verbose_only + ) if not run_in_pretend_mode and config.pretend: # noinspection PyTypeChecker return FakePopen() # pytype: disable=bad-return-type @@ -373,11 +397,24 @@ def popen(cmdline, print_verbose_only=False, run_in_pretend_mode=False, *, confi # noinspection PyShadowingBuiltins -def run_command(*args, capture_output=False, capture_error=False, input: "Optional[Union[str, bytes]]" = None, - timeout=None, print_verbose_only=False, run_in_pretend_mode=False, raise_in_pretend_mode=False, - no_print=False, replace_env=False, give_tty_control=False, expected_exit_code=0, - allow_unexpected_returncode=False, config: ConfigBase, - env: "Optional[dict[str, str]]" = None, **kwargs) -> "CompletedProcess[bytes]": +def run_command( + *args, + capture_output=False, + capture_error=False, + input: "Optional[Union[str, bytes]]" = None, + timeout=None, + print_verbose_only=False, + run_in_pretend_mode=False, + raise_in_pretend_mode=False, + no_print=False, + replace_env=False, + give_tty_control=False, + expected_exit_code=0, + allow_unexpected_returncode=False, + config: ConfigBase, + env: "Optional[dict[str, str]]" = None, + **kwargs, +) -> "CompletedProcess[bytes]": if len(args) == 1 and isinstance(args[0], (list, tuple)): cmdline = args[0] # list with parameters was passed else: @@ -386,8 +423,9 @@ def run_command(*args, capture_output=False, capture_error=False, input: "Option cmdline = list(map(str, cmdline)) # ensure it's all strings so that subprocess can handle it # When running scripts from a noexec filesystem try to read the interpreter and run that if not no_print: - print_command(cmdline, cwd=kwargs.get("cwd"), env=kwargs.get("env"), print_verbose_only=print_verbose_only, - config=config) + print_command( + cmdline, cwd=kwargs.get("cwd"), env=kwargs.get("env"), print_verbose_only=print_verbose_only, config=config + ) if "cwd" in kwargs: kwargs["cwd"] = str(kwargs["cwd"]) else: @@ -401,7 +439,7 @@ def run_command(*args, capture_output=False, capture_error=False, input: "Option # actually run the process now: if input is not None: assert "stdin" not in kwargs # we need to use stdin here - kwargs['stdin'] = subprocess.PIPE + kwargs["stdin"] = subprocess.PIPE if not isinstance(input, bytes): input = str(input).encode("utf-8") if capture_output: @@ -454,8 +492,14 @@ def run_command(*args, capture_output=False, capture_error=False, input: "Option if exc is not None: if config.pretend and not raise_in_pretend_mode: cwd = (". Working directory was ", kwargs["cwd"]) if "cwd" in kwargs else () - fatal_error("Command ", "`" + commandline_to_str(process.args) + - "` failed with unexpected exit code ", retcode, *cwd, sep="", pretend=config.pretend) + fatal_error( + "Command ", + "`" + commandline_to_str(process.args) + "` failed with unexpected exit code ", + retcode, + *cwd, + sep="", + pretend=config.pretend, + ) else: raise exc stdout = typing.cast(bytes, stdout) @@ -480,8 +524,16 @@ def commandline_to_str(args: "typing.Iterable[Union[str,Path]]") -> str: class CompilerInfo: - def __init__(self, path: Path, compiler: str, version: "tuple[int, ...]", version_str: str, default_target: str, - *, config: ConfigBase): + def __init__( + self, + path: Path, + compiler: str, + version: "tuple[int, ...]", + version_str: str, + default_target: str, + *, + config: ConfigBase, + ): self.path = path self.compiler = compiler self.version = version @@ -501,15 +553,33 @@ def get_resource_dir(self) -> Path: return Path("/unknown/resource/dir") # avoid failing in jenkins # Clang 5.0 added the -print-resource-dir flag if self.is_clang and self.version >= (5, 0): - resource_dir = run_command(self.path, "-print-resource-dir", config=self.config, - print_verbose_only=True, capture_output=True, - run_in_pretend_mode=True).stdout.decode("utf-8").strip() + resource_dir = ( + run_command( + self.path, + "-print-resource-dir", + config=self.config, + print_verbose_only=True, + capture_output=True, + run_in_pretend_mode=True, + ) + .stdout.decode("utf-8") + .strip() + ) assert resource_dir, "-print-resource-dir no longer works?" self._resource_dir = Path(resource_dir) else: # pretend to compile an existing source file and capture the -resource-dir output - cc1_cmd = run_command(self.path, "-###", "-xc", "-c", "/dev/null", config=self.config, - capture_error=True, print_verbose_only=True, run_in_pretend_mode=True) + cc1_cmd = run_command( + self.path, + "-###", + "-xc", + "-c", + "/dev/null", + config=self.config, + capture_error=True, + print_verbose_only=True, + run_in_pretend_mode=True, + ) match = re.compile(b'"-cc1".+"-resource-dir" "([^"]+)"').search(cc1_cmd.stderr) assert match is not None, f"Could not find -resource dir in {cc1_cmd.stderr}" self._resource_dir = Path(match.group(1).decode("utf-8")) @@ -521,9 +591,18 @@ def get_include_dirs(self, basic_flags: "list[str]") -> "list[Path]": if not self.path.exists(): return [Path("/unknown/include/dir")] # avoid failing in jenkins # pretend to compile an existing source file and capture the -resource-dir output - output = run_command(self.path, "-E", "-Wp,-v", "-xc", "/dev/null", config=self.config, - stdout=subprocess.DEVNULL, capture_error=True, print_verbose_only=True, - run_in_pretend_mode=True).stderr + output = run_command( + self.path, + "-E", + "-Wp,-v", + "-xc", + "/dev/null", + config=self.config, + stdout=subprocess.DEVNULL, + capture_error=True, + print_verbose_only=True, + run_in_pretend_mode=True, + ).stderr found_start = False include_dirs = [] for line in io.BytesIO(output).readlines(): @@ -546,8 +625,16 @@ def _supports_flag(self, flag: str, other_args: "list[str]") -> bool: try: if not self.path.exists(): return False # avoid failing in jenkins - result = run_command(self.path, *other_args, flag, print_verbose_only=True, run_in_pretend_mode=True, - capture_error=True, allow_unexpected_returncode=True, config=self.config) + result = run_command( + self.path, + *other_args, + flag, + print_verbose_only=True, + run_in_pretend_mode=True, + capture_error=True, + allow_unexpected_returncode=True, + config=self.config, + ) except (subprocess.CalledProcessError, OSError) as e: warning_message("Failed to check for", flag, "support:", e) return False @@ -557,8 +644,9 @@ def supports_sanitizer_flag(self, sanitzer_flag: str, arch_flags: "list[str]"): result = self._supported_sanitizer_flags.get((sanitzer_flag, tuple(*arch_flags))) if result is None: assert sanitzer_flag.startswith("-fsanitize") - result = self._supports_flag(sanitzer_flag, - [*arch_flags, "-c", "-xc", "/dev/null", "-Werror", "-o", "/dev/null"]) + result = self._supports_flag( + sanitzer_flag, [*arch_flags, "-c", "-xc", "/dev/null", "-Werror", "-o", "/dev/null"] + ) self._supported_sanitizer_flags[(sanitzer_flag, tuple(*arch_flags))] = result return result @@ -606,7 +694,7 @@ def get_matching_binutil(self, binutil) -> Optional[Path]: version_suffix = "" for basename in ("clang++", "clang-cpp", "clang"): if name.startswith(basename): - version_suffix = name[len(basename):] + version_suffix = name[len(basename) :] # Try to find a binutil with the same version suffix first real_compiler_path = self.path.resolve() if self.path.exists() else self.path result = real_compiler_path.parent / (binutil + version_suffix) @@ -670,9 +758,16 @@ def get_compiler_info(compiler: "Union[str, Path]", *, config: ConfigBase) -> Co try: # Use -v instead of --version to support both gcc and clang # Note: for clang-cpp/cpp we need to have stdin as devnull - version_cmd = run_command(compiler, "-v", capture_error=True, print_verbose_only=True, - run_in_pretend_mode=True, config=config, - stdin=subprocess.DEVNULL, capture_output=True) + version_cmd = run_command( + compiler, + "-v", + capture_error=True, + print_verbose_only=True, + run_in_pretend_mode=True, + config=config, + stdin=subprocess.DEVNULL, + capture_output=True, + ) except subprocess.CalledProcessError as e: stderr = e.stderr if e.stderr else b"FAILED: " + str(e).encode("utf-8") version_cmd = CompletedProcess(e.cmd, e.returncode, e.output, stderr) @@ -717,16 +812,28 @@ def get_version_output(program: Path, command_args: Optional[tuple] = None, *, c command_args = ["--version"] if program == Path(): raise ValueError("Empty path?") - prog = run_command([str(program), *list(command_args)], config=config, stdin=subprocess.DEVNULL, - stderr=subprocess.STDOUT, capture_output=True, - run_in_pretend_mode=True, raise_in_pretend_mode=True) + prog = run_command( + [str(program), *list(command_args)], + config=config, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + capture_output=True, + run_in_pretend_mode=True, + raise_in_pretend_mode=True, + ) return prog.stdout @functools.lru_cache(maxsize=20) -def get_program_version(program: Path, command_args: Optional[tuple] = None, component_kind: "type[Type_T]" = int, - regex=None, program_name: Optional[bytes] = None, *, - config: ConfigBase) -> "tuple[Type_T, ...]": +def get_program_version( + program: Path, + command_args: Optional[tuple] = None, + component_kind: "type[Type_T]" = int, + regex=None, + program_name: Optional[bytes] = None, + *, + config: ConfigBase, +) -> "tuple[Type_T, ...]": if program_name is None: program_name = program.name.encode("utf-8") try: @@ -738,8 +845,12 @@ def get_program_version(program: Path, command_args: Optional[tuple] = None, com # extract the version component from program output such as "git version 2.7.4" -def extract_version(output: bytes, component_kind: "type[Type_T]" = int, regex: "Optional[typing.Pattern]" = None, - program_name: bytes = b"") -> "tuple[Type_T, ...]": +def extract_version( + output: bytes, + component_kind: "type[Type_T]" = int, + regex: "Optional[typing.Pattern]" = None, + program_name: bytes = b"", +) -> "tuple[Type_T, ...]": if regex is None: prefix = re.escape(program_name) + b" " if program_name else b"" regex = re.compile(prefix + b"version\\s+(\\d+)\\.(\\d+)\\.?(\\d+)?") @@ -753,8 +864,7 @@ def extract_version(output: bytes, component_kind: "type[Type_T]" = int, regex: return tuple(component_kind(x) for x in match.groups() if x is not None) -def latest_system_clang_tool(config: ConfigBase, basename: str, - fallback_basename: "Optional[str]") -> Optional[Path]: +def latest_system_clang_tool(config: ConfigBase, basename: str, fallback_basename: "Optional[str]") -> Optional[Path]: if "_ARGCOMPLETE" in os.environ: # Avoid expensive lookup when tab-completing return None if fallback_basename is None else Path(fallback_basename) @@ -802,7 +912,7 @@ def run_and_kill_children_on_exit(fn: "Callable[[], typing.Any]"): os.setpgrp() # Preserve whether our process group is the terminal leader with suppress_sigttou(): - with scoped_open('/dev/tty', os.O_RDWR, ignore_open_error=True) as tty: + with scoped_open("/dev/tty", os.O_RDWR, ignore_open_error=True) as tty: if tty is not None and os.tcgetpgrp(tty) == opgrp: os.tcsetpgrp(tty, os.getpgrp()) fn() @@ -816,12 +926,19 @@ def run_and_kill_children_on_exit(fn: "Callable[[], typing.Any]"): extra_msg += ("\nStandard error was:\n", err.stderr.decode("utf-8")) # If we are currently debugging, raise the exception to allow e.g. PyCharm's # "break on exception that terminates execution" feature works. - debugger_attached = getattr(sys, 'gettrace', lambda: None)() is not None + debugger_attached = getattr(sys, "gettrace", lambda: None)() is not None if debugger_attached: raise err else: - fatal_error("Command ", "`" + commandline_to_str(err.cmd) + "` failed with non-zero exit code ", - err.returncode, *extra_msg, sep="", exit_code=err.returncode, pretend=False) + fatal_error( + "Command ", + "`" + commandline_to_str(err.cmd) + "` failed with non-zero exit code ", + err.returncode, + *extra_msg, + sep="", + exit_code=err.returncode, + pretend=False, + ) finally: if error: signal.signal(signal.SIGTERM, signal.SIG_IGN) diff --git a/pycheribuild/projects/bear.py b/pycheribuild/projects/bear.py index c17fa8e24..1d7fafb79 100644 --- a/pycheribuild/projects/bear.py +++ b/pycheribuild/projects/bear.py @@ -32,9 +32,12 @@ class BuildBear(CMakeProject): - repository = GitRepository("https://github.com/rizsotto/Bear.git", force_branch=True, - # default_branch="devel") - default_branch="master") + repository = GitRepository( + "https://github.com/rizsotto/Bear.git", + force_branch=True, + # default_branch="devel") + default_branch="master", + ) native_install_dir = DefaultInstallDir.BOOTSTRAP_TOOLS @property diff --git a/pycheribuild/projects/binutils.py b/pycheribuild/projects/binutils.py index f0d1485ce..5680e34b1 100644 --- a/pycheribuild/projects/binutils.py +++ b/pycheribuild/projects/binutils.py @@ -35,11 +35,13 @@ class BuildGnuBinutils(AutotoolsProject): target = "gnu-binutils" - repository = GitRepository("https://github.com/CTSRD-CHERI/binutils.git", default_branch="cheribsd", - force_branch=True) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/binutils.git", default_branch="cheribsd", force_branch=True + ) native_install_dir = DefaultInstallDir.CHERI_SDK - full_install = BoolConfigOption("install-all-tools", help="Whether to install all binutils tools instead" - "of only as, ld and objdump") + full_install = BoolConfigOption( + "install-all-tools", help="Whether to install all binutils tools instead" "of only as, ld and objdump" + ) def setup(self): super().setup() @@ -61,24 +63,26 @@ def setup(self): # elf64ltsmip_fbsd # elf32btsmipn32_fbsd # elf32ltsmipn32_fbsd - self.configure_args.extend([ - # on cheri gcc -dumpmachine returns mips64-undermydesk-freebsd, however this is not accepted by BFD - # if we just pass --target=mips64 this apparently defaults to mips64-unknown-elf on freebsd - # and also on Linux, but let's be explicit in case it assumes ELF binaries to target linux - # "--target=mips64-undermydesk-freebsd", # binutils for MIPS64/CHERI - "--target=mips64-unknown-freebsd", # binutils for MIPS64/FreeBSD - "--disable-werror", # -Werror won't work with recent compilers - "--enable-ld", # enable linker (is default, but just be safe) - "--enable-libssp", # not sure if this is needed - "--enable-64-bit-bfd", # Make sure we always have 64 bit support - "--enable-targets=all", - "--disable-gprof", - "--disable-gold", - "--disable-nls", - "--disable-info", - # "--program-prefix=cheri-unknown-freebsd-", - "MAKEINFO=missing", # don't build docs, this will fail on recent Linux systems - ]) + self.configure_args.extend( + [ + # on cheri gcc -dumpmachine returns mips64-undermydesk-freebsd, however this is not accepted by BFD + # if we just pass --target=mips64 this apparently defaults to mips64-unknown-elf on freebsd + # and also on Linux, but let's be explicit in case it assumes ELF binaries to target linux + # "--target=mips64-undermydesk-freebsd", # binutils for MIPS64/CHERI + "--target=mips64-unknown-freebsd", # binutils for MIPS64/FreeBSD + "--disable-werror", # -Werror won't work with recent compilers + "--enable-ld", # enable linker (is default, but just be safe) + "--enable-libssp", # not sure if this is needed + "--enable-64-bit-bfd", # Make sure we always have 64 bit support + "--enable-targets=all", + "--disable-gprof", + "--disable-gold", + "--disable-nls", + "--disable-info", + # "--program-prefix=cheri-unknown-freebsd-", + "MAKEINFO=missing", # don't build docs, this will fail on recent Linux systems + ] + ) self.configure_args.append("--disable-shared") # newer compilers will default to -std=c99 which will break binutils: cflags = "-std=gnu89 -O2" @@ -124,8 +128,10 @@ def triple_prefixes_for_binaries(self) -> typing.Iterable[str]: return ["cheri-unknown-freebsd-"] # compat only def process(self): - self.warning("GNU binutils should only be built if you know what you are doing since the linker " - "is incredibly buggy and the assembler doesn't support all features that clang does.") + self.warning( + "GNU binutils should only be built if you know what you are doing since the linker " + "is incredibly buggy and the assembler doesn't support all features that clang does." + ) if not self.query_yes_no("Are you sure you want to build this code?"): return super().process() diff --git a/pycheribuild/projects/bluespec_compiler.py b/pycheribuild/projects/bluespec_compiler.py index e98739ec1..01d6076cc 100644 --- a/pycheribuild/projects/bluespec_compiler.py +++ b/pycheribuild/projects/bluespec_compiler.py @@ -74,11 +74,13 @@ def install(self, **kwargs): ) if OSInfo.IS_MAC: self.info( - "Alternatively, try running:", self.source_dir / ".github/workflows/install_dependencies_macos.sh", + "Alternatively, try running:", + self.source_dir / ".github/workflows/install_dependencies_macos.sh", ) elif OSInfo.is_ubuntu(): self.info( - "Alternatively, try running:", self.source_dir / ".github/workflows/install_dependencies_ubuntu.sh", + "Alternatively, try running:", + self.source_dir / ".github/workflows/install_dependencies_ubuntu.sh", ) raise diff --git a/pycheribuild/projects/bmake.py b/pycheribuild/projects/bmake.py index f782bf234..01553cfcd 100644 --- a/pycheribuild/projects/bmake.py +++ b/pycheribuild/projects/bmake.py @@ -46,5 +46,6 @@ def configure(self, **kwargs): super().configure() def compile(self, **kwargs): - self.run_with_logfile(["sh", self.build_dir / "make-bootstrap.sh"], cwd=self.build_dir, - logfile_name="build.log") + self.run_with_logfile( + ["sh", self.build_dir / "make-bootstrap.sh"], cwd=self.build_dir, logfile_name="build.log" + ) diff --git a/pycheribuild/projects/build_qemu.py b/pycheribuild/projects/build_qemu.py index 3255dd669..35dafe8c4 100644 --- a/pycheribuild/projects/build_qemu.py +++ b/pycheribuild/projects/build_qemu.py @@ -75,14 +75,23 @@ class BuildQEMUBase(AutotoolsProject): "targets will fail without smbd support)", ) gui = BoolConfigOption( - "gui", show_help=False, default=False, help="Build a the graphical UI bits for QEMU (SDL,VNC)", + "gui", + show_help=False, + default=False, + help="Build a the graphical UI bits for QEMU (SDL,VNC)", ) build_profiler = BoolConfigOption( - "build-profiler", show_help=False, default=False, help="Enable QEMU internal profiling", + "build-profiler", + show_help=False, + default=False, + help="Enable QEMU internal profiling", ) enable_plugins = BoolConfigOption("enable-plugins", show_help=False, default=False, help="Enable QEMU TCG plugins") prefer_full_lto_over_thin_lto = BoolConfigOption( - "full-lto", show_help=False, default=True, help="Prefer full LTO over LLVM ThinLTO when using LTO", + "full-lto", + show_help=False, + default=True, + help="Prefer full LTO over LLVM ThinLTO when using LTO", ) @classmethod @@ -101,7 +110,10 @@ def setup_config_options(cls, **kwargs): cls.qemu_targets = typing.cast( str, cls.add_config_option( - "targets", show_help=True, help="Build QEMU for the following targets", default=cls.default_targets, + "targets", + show_help=True, + help="Build QEMU for the following targets", + default=cls.default_targets, ), ) @@ -137,20 +149,31 @@ def should_strip_elf_file(self, f: Path) -> bool: def check_system_dependencies(self) -> None: super().check_system_dependencies() self.check_required_system_tool( - "glibtoolize" if self.target_info.is_macos() else "libtoolize", default="libtool", + "glibtoolize" if self.target_info.is_macos() else "libtoolize", + default="libtool", ) self.check_required_system_tool("autoreconf", default="autoconf") self.check_required_system_tool("aclocal", default="automake") self.check_required_pkg_config( - "pixman-1", homebrew="pixman", zypper="libpixman-1-0-devel", apt="libpixman-1-dev", freebsd="pixman", + "pixman-1", + homebrew="pixman", + zypper="libpixman-1-0-devel", + apt="libpixman-1-dev", + freebsd="pixman", ) self.check_required_pkg_config( - "glib-2.0", homebrew="glib", zypper="glib2-devel", apt="libglib2.0-dev", freebsd="glib", + "glib-2.0", + homebrew="glib", + zypper="glib2-devel", + apt="libglib2.0-dev", + freebsd="glib", ) # Tests require GNU sed self.check_required_system_tool( - "sed" if self.target_info.is_linux() else "gsed", homebrew="gnu-sed", freebsd="gsed", + "sed" if self.target_info.is_linux() else "gsed", + homebrew="gnu-sed", + freebsd="gsed", ) def add_asan_flags(self): diff --git a/pycheribuild/projects/cherios.py b/pycheribuild/projects/cherios.py index cf0450e37..f5f84df63 100644 --- a/pycheribuild/projects/cherios.py +++ b/pycheribuild/projects/cherios.py @@ -90,12 +90,14 @@ def setup(self): self._project_specific_options = ["-no-reboot", "-global", "virtio-mmio.force-legacy=false"] if cherios.build_net: - self._after_disk_options.extend([ - "-netdev", - "tap,id=tap0,ifname=cherios_tap,script=no,downscript=no", - "-device", - "virtio-net-device,netdev=tap0", - ]) + self._after_disk_options.extend( + [ + "-netdev", + "tap,id=tap0,ifname=cherios_tap,script=no,downscript=no", + "-device", + "virtio-net-device,netdev=tap0", + ] + ) if cherios.smp_cores > 1: self._project_specific_options.append("-smp") diff --git a/pycheribuild/projects/cmake.py b/pycheribuild/projects/cmake.py index 02a66bcf5..e36d39da8 100644 --- a/pycheribuild/projects/cmake.py +++ b/pycheribuild/projects/cmake.py @@ -51,16 +51,20 @@ # pulled in the fixes. class BuildLibuv(CrossCompileCMakeProject): target = "libuv" - repository = GitRepository("https://github.com/libuv/libuv.git", - temporary_url_override="https://github.com/arichardson/libuv.git", - url_override_reason="https://github.com/libuv/libuv/pull/3756") + repository = GitRepository( + "https://github.com/libuv/libuv.git", + temporary_url_override="https://github.com/arichardson/libuv.git", + url_override_reason="https://github.com/libuv/libuv/pull/3756", + ) # Not really autotools but same sequence of commands (other than the script being call bootstrap instead of configure) class BuildCMake(AutotoolsProject): - repository = GitRepository("https://github.com/Kitware/CMake", # a lot faster than "https://cmake.org/cmake.git" - # track the stable release branch - default_branch="release") + repository = GitRepository( + "https://github.com/Kitware/CMake", # a lot faster than "https://cmake.org/cmake.git" + # track the stable release branch + default_branch="release", + ) default_architecture = CompilationTargets.NATIVE native_install_dir = DefaultInstallDir.BOOTSTRAP_TOOLS make_kind = MakeCommandKind.Ninja @@ -125,15 +129,23 @@ def setup(self): self.add_cmake_options(CMAKE_USE_SYSTEM_LIBRARY_LIBUV=True) # CMake can't find the static libuv due to a different libname self.add_cmake_options( - LibUV_LIBRARY=BuildLibuv.get_install_dir(self) / self.target_info.default_libdir / "libuv_a.a") + LibUV_LIBRARY=BuildLibuv.get_install_dir(self) / self.target_info.default_libdir / "libuv_a.a" + ) def run_tests(self): # TODO: generate JUnit output once https://gitlab.kitware.com/cmake/cmake/-/merge_requests/6020 is merged # Can't run the testsuite since many tests depend on having a C compiler installed. test_command = "cd /build && ./bin/ctest -N" - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), - mount_builddir=True, mount_sourcedir=True, mount_sysroot=True) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", + "--test-command", + test_command, + "--test-timeout", + str(120 * 60), + mount_builddir=True, + mount_sourcedir=True, + mount_sysroot=True, + ) # Add a cmake-native target for consistency. diff --git a/pycheribuild/projects/cmake_project.py b/pycheribuild/projects/cmake_project.py index 876971017..95374ee05 100644 --- a/pycheribuild/projects/cmake_project.py +++ b/pycheribuild/projects/cmake_project.py @@ -50,6 +50,7 @@ class CMakeProject(_CMakeAndMesonSharedLogic): Sets configure command to CMake, adds -DCMAKE_INSTALL_PREFIX=installdir and checks that CMake is installed """ + do_not_add_to_targets: bool = True compile_db_requires_bear: bool = False # cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON does it generate_cmakelists: bool = False # There is already a CMakeLists.txt @@ -79,8 +80,16 @@ def _bool_to_str(self, value: bool) -> str: return "TRUE" if value else "FALSE" def _configure_tool_install_instructions(self) -> InstallInstructions: - return OSInfo.install_instructions("cmake", False, default="cmake", homebrew="cmake", zypper="cmake", - apt="cmake", freebsd="cmake", cheribuild_target="cmake") + return OSInfo.install_instructions( + "cmake", + False, + default="cmake", + homebrew="cmake", + zypper="cmake", + apt="cmake", + freebsd="cmake", + cheribuild_target="cmake", + ) @property def _get_version_args(self) -> dict: @@ -94,8 +103,9 @@ def _build_type_basic_compiler_flags(self): @classmethod def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) - cls.cmake_options = cls.add_list_option("cmake-options", metavar="OPTIONS", - help="Additional command line options to pass to CMake") + cls.cmake_options = cls.add_list_option( + "cmake-options", metavar="OPTIONS", help="Additional command line options to pass to CMake" + ) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -163,18 +173,20 @@ def setup(self) -> None: # TODO: set CMAKE_STRIP, CMAKE_NM, CMAKE_OBJDUMP, CMAKE_READELF, CMAKE_DLLTOOL, CMAKE_DLLTOOL, # CMAKE_ADDR2LINE self.add_cmake_options( - _CMAKE_TOOLCHAIN_LOCATION=self.target_info.sdk_root_dir / "bin", - CMAKE_LINKER=self.target_info.linker) + _CMAKE_TOOLCHAIN_LOCATION=self.target_info.sdk_root_dir / "bin", CMAKE_LINKER=self.target_info.linker + ) if self.force_static_linkage: self.add_cmake_options( CMAKE_SHARED_LIBRARY_SUFFIX=".a", CMAKE_FIND_LIBRARY_SUFFIXES=".a", CMAKE_EXTRA_SHARED_LIBRARY_SUFFIXES=".a", - CMAKE_SKIP_RPATH=True) + CMAKE_SKIP_RPATH=True, + ) if self.compiling_for_host(): self.add_cmake_options( - CMAKE_PROJECT_INCLUDE=Path(__file__).absolute().parent.parent / "files/ForceStaticLibraries.cmake") + CMAKE_PROJECT_INCLUDE=Path(__file__).absolute().parent.parent / "files/ForceStaticLibraries.cmake" + ) else: # Use $ORIGIN in the build RPATH (this should make it easier to run tests without having the absolute # build directory mounted). @@ -187,7 +199,8 @@ def setup(self) -> None: # Ninja can't change the RPATH when installing: https://gitlab.kitware.com/cmake/cmake/issues/13934 # Fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/6240 (3.21.20210625) self.add_cmake_options( - CMAKE_BUILD_WITH_INSTALL_RPATH=self._get_configure_tool_version() < (3, 21, 20210625)) + CMAKE_BUILD_WITH_INSTALL_RPATH=self._get_configure_tool_version() < (3, 21, 20210625) + ) # NB: Don't add the user provided options here, we append add them in setup_late() so that they are put last. def setup_late(self): @@ -223,8 +236,12 @@ def setup_late(self): flags = " " + commandline_to_str(self.optimization_flags) if self.build_type.is_release: flags += " -DNDEBUG" - self.add_cmake_options(**{f"CMAKE_C_FLAGS{self.build_type_var_suffix}": flags, - f"CMAKE_CXX_FLAGS{self.build_type_var_suffix}": flags}) + self.add_cmake_options( + **{ + f"CMAKE_C_FLAGS{self.build_type_var_suffix}": flags, + f"CMAKE_CXX_FLAGS{self.build_type_var_suffix}": flags, + } + ) # Add the options from the config file now so that they are added after child class setup() calls. self.configure_args.extend(self.cmake_options) @@ -236,8 +253,12 @@ def check_system_dependencies(self) -> None: self.check_required_system_tool("make") def add_cmake_options(self, *, _include_empty_vars=False, _replace=True, **kwargs) -> None: - return self._add_configure_options(_config_file_options=self.cmake_options, _replace=_replace, - _include_empty_vars=_include_empty_vars, **kwargs) + return self._add_configure_options( + _config_file_options=self.cmake_options, + _replace=_replace, + _include_empty_vars=_include_empty_vars, + **kwargs, + ) def set_minimum_cmake_version(self, major: int, minor: int, patch: int = 0) -> None: new_version = (major, minor, patch) @@ -287,8 +308,9 @@ def configure(self, **kwargs) -> None: self.generate_cmake_toolchain_file(self._toolchain_file) super().configure(**kwargs) if self.config.copy_compilation_db_to_source_dir and (self.build_dir / "compile_commands.json").exists(): - self.install_file(self.build_dir / "compile_commands.json", self.source_dir / "compile_commands.json", - force=True) + self.install_file( + self.build_dir / "compile_commands.json", self.source_dir / "compile_commands.json", force=True + ) def install(self, *, _stdout_filter=_default_stdout_filter) -> None: if _stdout_filter is _default_stdout_filter: @@ -299,26 +321,41 @@ def run_tests(self) -> None: if (self.build_dir / "CTestTestfile.cmake").exists() or self.config.pretend: # We can run tests using CTest if self.compiling_for_host(): - self.run_cmd(shutil.which(os.getenv("CTEST_COMMAND", "ctest")) or "ctest", "-V", "--output-on-failure", - cwd=self.build_dir, env=self.ctest_environment) + self.run_cmd( + shutil.which(os.getenv("CTEST_COMMAND", "ctest")) or "ctest", + "-V", + "--output-on-failure", + cwd=self.build_dir, + env=self.ctest_environment, + ) else: try: cmake_xtarget = self.crosscompile_target # Use a string here instead of BuildCrossCompiledCMake to avoid a cyclic import. cmake_target = target_manager.get_target( - "cmake-crosscompiled", required_arch=cmake_xtarget, config=self.config, caller=self, + "cmake-crosscompiled", + required_arch=cmake_xtarget, + config=self.config, + caller=self, ) cmake_project = cmake_target.project_class.get_instance(self, cross_target=cmake_xtarget) expected_ctest_path = cmake_project.install_dir / "bin/ctest" if not expected_ctest_path.is_file(): - self.dependency_error(f"cannot find CTest binary ({expected_ctest_path}) to run tests.", - cheribuild_target=cmake_project.target, cheribuild_xtarget=cmake_xtarget) + self.dependency_error( + f"cannot find CTest binary ({expected_ctest_path}) to run tests.", + cheribuild_target=cmake_project.target, + cheribuild_xtarget=cmake_xtarget, + ) # --output-junit needs version 3.21 min_version = "3.21" if not list(cmake_project.install_dir.glob("share/*/Help/release/" + min_version + ".rst")): - self.dependency_error("cannot find release notes for CMake", min_version, - "- installed CMake version is too old", - cheribuild_target=cmake_project.target, cheribuild_xtarget=cmake_xtarget) + self.dependency_error( + "cannot find release notes for CMake", + min_version, + "- installed CMake version is too old", + cheribuild_target=cmake_project.target, + cheribuild_xtarget=cmake_xtarget, + ) except LookupError: self.warning("Do not know how to cross-compile CTest for", self.target_info, "-> cannot run tests") return @@ -326,13 +363,22 @@ def run_tests(self) -> None: for var, value in self.ctest_environment.items(): args.append("--test-setup-command=export " + shlex.quote(var + "=" + value)) args.extend(self.ctest_script_extra_args) - self.target_info.run_cheribsd_test_script("run_ctest_tests.py", *args, mount_builddir=True, - mount_sysroot=True, mount_sourcedir=True, - use_full_disk_image=self.tests_need_full_disk_image) + self.target_info.run_cheribsd_test_script( + "run_ctest_tests.py", + *args, + mount_builddir=True, + mount_sysroot=True, + mount_sourcedir=True, + use_full_disk_image=self.tests_need_full_disk_image, + ) else: if self.has_optional_tests: - self.fatal("Can't run tests for projects that were built with tests disabled. ", - "Please re-run build the target with --", self.get_config_option_name("build_tests"), sep="") + self.fatal( + "Can't run tests for projects that were built with tests disabled. ", + "Please re-run build the target with --", + self.get_config_option_name("build_tests"), + sep="", + ) self.warning("Do not know how to run tests for", self.target) def find_package(self, name: str) -> bool: @@ -346,6 +392,7 @@ def find_package(self, name: str) -> bool: class MakefileProject(Project): """A very simple project that just set some defualt variables such as CC/CXX, etc""" + do_not_add_to_targets: bool = True build_in_source_dir: bool = True # Most makefile projects don't support out-of-source builds # Default to GNU make since that's what most makefile projects require. diff --git a/pycheribuild/projects/cross/apache.py b/pycheribuild/projects/cross/apache.py index 768060de7..938473466 100644 --- a/pycheribuild/projects/cross/apache.py +++ b/pycheribuild/projects/cross/apache.py @@ -42,8 +42,7 @@ class BuildPcre(CrossCompileAutotoolsProject): target = "pcre" - repository = SubversionRepository("svn://vcs.pcre.org/pcre", - default_branch="code/trunk") + repository = SubversionRepository("svn://vcs.pcre.org/pcre", default_branch="code/trunk") native_install_dir = DefaultInstallDir.BOOTSTRAP_TOOLS @@ -54,8 +53,7 @@ def configure(self, **kwargs): class BuildApr(CrossCompileAutotoolsProject): target = "apr" - repository = GitRepository("https://github.com/CTSRD-CHERI/apr.git", - default_branch="cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/apr.git", default_branch="cheri") dependencies = ("libexpat",) @@ -63,12 +61,14 @@ class BuildApr(CrossCompileAutotoolsProject): def setup(self): super().setup() - self.configure_args.extend([ - "--enable-threads", - "--enable-posix-shm", - "--with-devrandom", - "--with-expat=" + str(BuildExpat.get_install_dir(self)), - ]) + self.configure_args.extend( + [ + "--enable-threads", + "--enable-posix-shm", + "--with-devrandom", + "--with-expat=" + str(BuildExpat.get_install_dir(self)), + ] + ) if self.build_type.is_debug: self.configure_args.append("--enable-debug") @@ -79,17 +79,18 @@ def setup(self): if not self.compiling_for_host(): # Can't determine these when cross-compiling - self.configure_environment.update(ac_cv_file__dev_zero="yes", - ac_cv_mmap__dev_zero="yes", - # XXX: This might be yes on Linux - ac_cv_func_setpgrp_void="no", - ac_cv_struct_rlimit="yes", - ac_cv_func_sem_open="yes", - apr_cv_process_shared_works="yes", - apr_cv_mutex_robust_shared="yes", - # XXX: This might be yes on Linux - apr_cv_tcp_nodelay_with_cork="no", - ) + self.configure_environment.update( + ac_cv_file__dev_zero="yes", + ac_cv_mmap__dev_zero="yes", + # XXX: This might be yes on Linux + ac_cv_func_setpgrp_void="no", + ac_cv_struct_rlimit="yes", + ac_cv_func_sem_open="yes", + apr_cv_process_shared_works="yes", + apr_cv_mutex_robust_shared="yes", + # XXX: This might be yes on Linux + apr_cv_tcp_nodelay_with_cork="no", + ) def configure(self, **kwargs): self.run_cmd("./buildconf", cwd=self.source_dir) @@ -98,21 +99,22 @@ def configure(self, **kwargs): class BuildApache(CrossCompileAutotoolsProject): target = "apache" - repository = GitRepository("https://github.com/CTSRD-CHERI/apache-httpd.git", - default_branch="2.4.x-cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/apache-httpd.git", default_branch="2.4.x-cheri") dependencies = ("apr", "pcre") def setup(self): super().setup() - self.configure_args.extend([ - "--enable-layout=FreeBSD", - "--enable-http", - "--enable-mod-ssl", - "--with-expat=" + str(BuildExpat.get_install_dir(self)), - "--with-pcre=" + str(BuildPcre.get_install_dir(self)), - "--with-apr=" + str(BuildApr.get_install_dir(self)), - ]) + self.configure_args.extend( + [ + "--enable-layout=FreeBSD", + "--enable-http", + "--enable-mod-ssl", + "--with-expat=" + str(BuildExpat.get_install_dir(self)), + "--with-pcre=" + str(BuildPcre.get_install_dir(self)), + "--with-apr=" + str(BuildApr.get_install_dir(self)), + ] + ) if self.build_type.is_debug: self.configure_args.append("--enable-debugger-mode") @@ -130,12 +132,16 @@ def configure(self, **kwargs): # gen_test_char rules in server/ assume a native build if not self.compiling_for_host(): - self.run_cmd(str(self.host_CC), "-DCROSS_COMPILE", "-c", - self.source_dir / "server" / "gen_test_char.c", - "-o", "gen_test_char.lo", - cwd=self.build_dir / "server") - self.run_cmd(str(self.host_CC), "gen_test_char.lo", "-o", - "gen_test_char", cwd=self.build_dir / "server") + self.run_cmd( + str(self.host_CC), + "-DCROSS_COMPILE", + "-c", + self.source_dir / "server" / "gen_test_char.c", + "-o", + "gen_test_char.lo", + cwd=self.build_dir / "server", + ) + self.run_cmd(str(self.host_CC), "gen_test_char.lo", "-o", "gen_test_char", cwd=self.build_dir / "server") class BuildSSLProc(CrossCompileCMakeProject): @@ -163,4 +169,4 @@ def setup(self): super().setup() self.configure_args.append( "--with-sslproc=" + str(BuildSSLProc.get_install_dir(self)), - ) + ) diff --git a/pycheribuild/projects/cross/bash.py b/pycheribuild/projects/cross/bash.py index 517714f58..1e6eea723 100644 --- a/pycheribuild/projects/cross/bash.py +++ b/pycheribuild/projects/cross/bash.py @@ -31,12 +31,12 @@ class BuildBash(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/bash", - default_branch="cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/bash", default_branch="cheri") cross_install_dir = DefaultInstallDir.ROOTFS_OPTBASE path_in_rootfs = "/usr/local" - set_as_root_shell = BoolConfigOption("set-as-root-shell", show_help=True, - help="Set root's shell to bash (in the target rootfs)") + set_as_root_shell = BoolConfigOption( + "set-as-root-shell", show_help=True, help="Set root's shell to bash (in the target rootfs)" + ) def setup(self): super().setup() @@ -63,12 +63,13 @@ def install(self, **kwargs): self.create_symlink(Path("/usr/local/bin/bash"), self.destdir / "bin/bash", relative=False) self.add_unique_line_to_file(self.destdir / "etc/shells", "/usr/local/bin/bash") if self.set_as_root_shell: + def rewrite(old): new = [] for line in old: - fields = line.split(':') + fields = line.split(":") if len(fields) == 10 and fields[0] == "root": - line = ':'.join(fields[0:9] + ["/usr/local/bin/bash"]) + line = ":".join(fields[0:9] + ["/usr/local/bin/bash"]) new.append(line) return new diff --git a/pycheribuild/projects/cross/benchmark_mixin.py b/pycheribuild/projects/cross/benchmark_mixin.py index adc94e0e2..91fd610d1 100644 --- a/pycheribuild/projects/cross/benchmark_mixin.py +++ b/pycheribuild/projects/cross/benchmark_mixin.py @@ -48,8 +48,9 @@ # We also build benchmarks for hybrid to see whether those compilation flags change the results class BenchmarkMixin(_BenchmarkMixinBase): - supported_architectures = (CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID_FOR_PURECAP_ROOTFS + - CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID_FOR_PURECAP_ROOTFS + CompilationTargets.ALL_NATIVE + ) default_build_type = BuildType.RELEASE prefer_full_lto_over_thin_lto = True @@ -59,10 +60,15 @@ def optimization_flags(self): return ["-O3"] return super().optimization_flags - def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str]" = None, - benchmark_script: "Optional[str]" = None, - benchmark_script_args: "Optional[list[str]]" = None, - extra_runbench_args: "Optional[list[str]]" = None): + def run_fpga_benchmark( + self, + benchmarks_dir: Path, + *, + output_file: "Optional[str]" = None, + benchmark_script: "Optional[str]" = None, + benchmark_script_args: "Optional[list[str]]" = None, + extra_runbench_args: "Optional[list[str]]" = None, + ): assert benchmarks_dir is not None assert output_file is not None, "output_file must be set to a valid value" xtarget = self.crosscompile_target @@ -80,12 +86,16 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str basic_args = [] if self.config.benchmark_with_qemu: from ...projects.build_qemu import BuildQEMU + qemu_path = BuildQEMU.qemu_binary(self) qemu_ssh_socket = find_free_port() if not qemu_path.exists(): self.fatal("QEMU binary", qemu_path, "doesn't exist") - basic_args += ["--use-qemu-instead-of-fpga", "--qemu-path=" + str(qemu_path), - "--qemu-ssh-port=" + str(qemu_ssh_socket.port)] + basic_args += [ + "--use-qemu-instead-of-fpga", + "--qemu-path=" + str(qemu_path), + "--qemu-ssh-port=" + str(qemu_ssh_socket.port), + ] elif not self.compiling_for_mips(include_purecap=True): self.fatal("run_fpga_benchmark has not been updated for RISC-V/AArch64") return @@ -101,11 +111,13 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str env_var = "LD_64_PRELOAD" else: env_var = "LD_PRELOAD" - pre_cmd = "export {}={};".format(env_var, - shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name)) + pre_cmd = "export {}={};".format( + env_var, shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name) + ) if env_var == "LD_64C_PRELOAD": - pre_cmd += "export {}={};".format("LD_CHERI_PRELOAD", - shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name)) + pre_cmd += "export {}={};".format( + "LD_CHERI_PRELOAD", shlex.quote("/tmp/benchdir/" + self.config.benchmark_ld_preload.name) + ) runbench_args.append("--pre-command=" + pre_cmd) if self.config.benchmark_fpga_extra_args: basic_args.extend(self.config.benchmark_fpga_extra_args) @@ -115,6 +127,7 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str runbench_args.append("--interact") from ...projects.cross.cheribsd import BuildCheriBsdMfsKernel, ConfigPlatform + if self.config.benchmark_with_qemu: # When benchmarking with QEMU we always spawn a new instance target = self.target_info @@ -128,10 +141,12 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str # use a bitfile from jenkins. TODO: add option for overriding assert xtarget.is_riscv(include_purecap=True) basic_args.append("--jenkins-bitfile") - mfs_kernel = BuildCheriBsdMfsKernel.get_instance_for_cross_target(xtarget.get_rootfs_target(), self.config, - caller=self) - kernel_config = mfs_kernel.default_kernel_config(ConfigPlatform.GFE, - benchmark=not self.config.benchmark_with_debug_kernel) + mfs_kernel = BuildCheriBsdMfsKernel.get_instance_for_cross_target( + xtarget.get_rootfs_target(), self.config, caller=self + ) + kernel_config = mfs_kernel.default_kernel_config( + ConfigPlatform.GFE, benchmark=not self.config.benchmark_with_debug_kernel + ) kernel_image = mfs_kernel.get_kernel_install_path(kernel_config) basic_args.append("--kernel-img=" + str(kernel_image)) else: @@ -150,4 +165,5 @@ def run_fpga_benchmark(self, benchmarks_dir: Path, *, output_file: "Optional[str qemu_ssh_socket.socket.close() self.run_cmd( [str(cheribuild_path / "vcu118-bsd-boot.py"), *basic_args, "-vvvvv", "runbench", *runbench_args], - give_tty_control=True) + give_tty_control=True, + ) diff --git a/pycheribuild/projects/cross/benchmarks.py b/pycheribuild/projects/cross/benchmarks.py index 058db9767..d558293c0 100644 --- a/pycheribuild/projects/cross/benchmarks.py +++ b/pycheribuild/projects/cross/benchmarks.py @@ -67,13 +67,20 @@ class BuildMibench(BenchmarkMixin, CrossCompileProject): @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.benchmark_size = cls.add_config_option("benchmark-size", choices=("small", "large"), default="large", - kind=str, help="Size of benchmark input data to use") + cls.benchmark_size = cls.add_config_option( + "benchmark-size", + choices=("small", "large"), + default="large", + kind=str, + help="Size of benchmark input data to use", + ) @property def bundle_dir(self): - return Path(self.build_dir, self.crosscompile_target.generic_target_suffix + - self.build_configuration_suffix() + "-bundle") + return Path( + self.build_dir, + self.crosscompile_target.generic_target_suffix + self.build_configuration_suffix() + "-bundle", + ) @property def benchmark_version(self): @@ -88,18 +95,21 @@ def benchmark_version(self): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI256_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI256_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): # We can't fall back to /usr/bin/ar here since that breaks on MacOS if not self.compiling_for_host(): self.make_args.set(AR=str(self.sdk_bindir / "llvm-ar") + " rc") self.make_args.set(AR2=str(self.sdk_bindir / "llvm-ranlib")) self.make_args.set(RANLIB=str(self.sdk_bindir / "llvm-ranlib")) - self.make_args.set(MIPS_SYSROOT=self.sdk_sysroot, CHERI128_SYSROOT=self.sdk_sysroot, - CHERI256_SYSROOT=self.sdk_sysroot) + self.make_args.set( + MIPS_SYSROOT=self.sdk_sysroot, CHERI128_SYSROOT=self.sdk_sysroot, CHERI256_SYSROOT=self.sdk_sysroot + ) self.make_args.set(ADDITIONAL_CFLAGS=self.commandline_to_str(self.default_compiler_flags)) self.make_args.set(ADDITIONAL_LDFLAGS=self.commandline_to_str(self.default_ldflags)) @@ -144,9 +154,11 @@ def run_tests(self): return # testing, not benchmarking -> run only once: (-s small / -s large?) test_command = "cd '/build/{dirname}' && ./run_jenkins-bluehive.sh -d0 -r1 -s {size} {version}".format( - dirname=self.bundle_dir.name, size=self.benchmark_size, version=self.benchmark_version) - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), mount_builddir=True) + dirname=self.bundle_dir.name, size=self.benchmark_size, version=self.benchmark_version + ) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) def run_benchmarks(self): if not self.compiling_for_mips(include_purecap=True): @@ -158,11 +170,19 @@ def run_benchmarks(self): if not (benchmark_dir / "run_jenkins-bluehive.sh").exists(): self.fatal("Created invalid benchmark bundle...") num_iterations = self.config.benchmark_iterations or 10 - self.run_fpga_benchmark(benchmark_dir, output_file=self.default_statcounters_csv_name, - benchmark_script_args=["-d1", "-r" + str(num_iterations), - "-s", self.benchmark_size, - "-o", self.default_statcounters_csv_name, - self.benchmark_version]) + self.run_fpga_benchmark( + benchmark_dir, + output_file=self.default_statcounters_csv_name, + benchmark_script_args=[ + "-d1", + "-r" + str(num_iterations), + "-s", + self.benchmark_size, + "-o", + self.default_statcounters_csv_name, + self.benchmark_version, + ], + ) class BuildMiBenchNew(BuildLLVMTestSuiteBase): @@ -171,19 +191,19 @@ class BuildMiBenchNew(BuildLLVMTestSuiteBase): def setup(self): super().setup() - self.add_cmake_options(TEST_SUITE_SUBDIRS="MultiSource/Benchmarks/MiBench", - TEST_SUITE_COPY_DATA=True) + self.add_cmake_options(TEST_SUITE_SUBDIRS="MultiSource/Benchmarks/MiBench", TEST_SUITE_COPY_DATA=True) def compile(self, **kwargs): super().compile(**kwargs) - self.install_file(self.source_dir / "MultiSource/lit.local.cfg", - self.build_dir / "MultiSource/lit.local.cfg", force=True) + self.install_file( + self.source_dir / "MultiSource/lit.local.cfg", self.build_dir / "MultiSource/lit.local.cfg", force=True + ) def install(self, **kwargs): root_dir = str(self.build_dir / "MultiSource/Benchmarks/MiBench") for curdir, dirnames, filenames in os.walk(root_dir): # We don't run some benchmarks (e.g. consumer-typeset or consumer-lame) yet - for ignored_dirname in ('CMakeFiles', 'consumer-typeset', 'consumer-lame', 'office-ispell', 'telecomm-gsm'): + for ignored_dirname in ("CMakeFiles", "consumer-typeset", "consumer-lame", "office-ispell", "telecomm-gsm"): if ignored_dirname in dirnames: dirnames.remove(ignored_dirname) relpath = os.path.relpath(curdir, root_dir) @@ -208,10 +228,12 @@ class BuildOlden(BenchmarkMixin, CrossCompileProject): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI256_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI256_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): if not self.compiling_for_host(): self.make_args.set(SYSROOT_DIRNAME=self.cross_sysroot_path.name) @@ -228,8 +250,9 @@ def compile(self, **kwargs): self.fatal("Unknown target: ", self.crosscompile_target) # copy asan libraries and the run script to the bin dir to ensure that we can run with --test from the # build directory. - self.install_file(self.source_dir / "run_jenkins-bluehive.sh", - self.build_dir / "bin/run_jenkins-bluehive.sh", force=True) + self.install_file( + self.source_dir / "run_jenkins-bluehive.sh", self.build_dir / "bin/run_jenkins-bluehive.sh", force=True + ) if self.compiling_for_mips(include_purecap=False) and self.use_asan: self.copy_asan_dependencies(self.build_dir / "bin/lib") @@ -267,9 +290,9 @@ def run_tests(self): return # testing, not benchmarking -> run only once: (-s small / -s large?) test_command = f"cd /build/bin && ./run_jenkins-bluehive.sh -d0 -r1 {self.test_arch_suffix}" - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), - mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) def run_benchmarks(self): if not self.compiling_for_mips(include_purecap=True): @@ -282,10 +305,17 @@ def run_benchmarks(self): if not (benchmark_dir / "run_jenkins-bluehive.sh").exists(): self.fatal("Created invalid benchmark bundle...") num_iterations = self.config.benchmark_iterations or 15 - self.run_fpga_benchmark(benchmark_dir, output_file=self.default_statcounters_csv_name, - benchmark_script_args=["-d1", "-r" + str(num_iterations), "-o", - self.default_statcounters_csv_name, - self.test_arch_suffix]) + self.run_fpga_benchmark( + benchmark_dir, + output_file=self.default_statcounters_csv_name, + benchmark_script_args=[ + "-d1", + "-r" + str(num_iterations), + "-o", + self.default_statcounters_csv_name, + self.test_arch_suffix, + ], + ) class BuildSpec2006New(BuildLLVMTestSuiteBase): @@ -297,8 +327,9 @@ class BuildSpec2006New(BuildLLVMTestSuiteBase): @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.spec_iso_path = cls.add_optional_path_option("iso-path", altname="spec-sources", - help="Path to the SPEC2006 ISO image or extracted sources") + cls.spec_iso_path = cls.add_optional_path_option( + "iso-path", altname="spec-sources", help="Path to the SPEC2006 ISO image or extracted sources" + ) cls.fast_benchmarks_only = cls.add_bool_option("fast-benchmarks-only", default=False) cls.workload = cls.add_config_option("workload", choices=("test", "train", "ref"), default="test") cls.benchmark_override = cls.add_list_option("benchmarks", help="override the list of benchmarks to run") @@ -328,7 +359,7 @@ def __init__(self, *args, **kwargs): "471.omnetpp", # 3 runs = 0:05:09 -> ~1:45min per run "473.astar", # 3 runs = 0:31:41 -> ~10:30 mins per run "483.xalancbmk", # 3 runs = 0:00:55 -> ~20 secs per run" - ] + ] self.complete_benchmark_list = [*self.working_benchmark_list, "400.perlbench", "403.gcc", "429.mcf"] self.fast_list = ["471.omnetpp", "483.xalancbmk", "456.hmmer", "462.libquantum"] if self.benchmark_override: @@ -345,16 +376,19 @@ def setup(self): super().setup() # Only build spec2006 # self.add_cmake_options(TEST_SUITE_SUBDIRS="External/SPEC/CINT2006;External/SPEC/CFP2006", - self.add_cmake_options(TEST_SUITE_SUBDIRS="External/SPEC/CINT2006", - TEST_SUITE_COPY_DATA=True, - TEST_SUITE_RUN_TYPE=self.workload, - TEST_SUITE_SPEC2006_ROOT=self.extracted_spec_sources) + self.add_cmake_options( + TEST_SUITE_SUBDIRS="External/SPEC/CINT2006", + TEST_SUITE_COPY_DATA=True, + TEST_SUITE_RUN_TYPE=self.workload, + TEST_SUITE_SPEC2006_ROOT=self.extracted_spec_sources, + ) def _check_broken_bsdtar(self, bsdtar: Path) -> "tuple[bool, tuple[int, ...]]": if self.config.pretend and not bsdtar.exists(): return False, (0, 0, 0) - bsdtar_version = get_program_version(bsdtar, regex=rb"bsdtar\s+(\d+)\.(\d+)\.?(\d+)? \- libarchive", - config=self.config) + bsdtar_version = get_program_version( + bsdtar, regex=rb"bsdtar\s+(\d+)\.(\d+)\.?(\d+)? \- libarchive", config=self.config + ) # At least version 3.3.2 of libarchive fails to extract the SPEC ISO image correctly (at least two files # are missing). This does not appear to be a problem with Ubuntu 18.04's version 3.2.2. return (3, 3) <= bsdtar_version < (3, 5), bsdtar_version @@ -370,9 +404,13 @@ def extract_spec_iso_image(self): bsdtar = libarchive_path / "bin/bsdtar" bsdtar_broken, bsdtar_version = self._check_broken_bsdtar(bsdtar) if bsdtar_broken: - self.fatal("The installed version of libarchive (", ".".join(map(str, bsdtar_version)), - ") has a bug that results in some files not being extracted from the SPEC ISO.", - fixit_hint="Please update bsdtar to at least 3.5.0", sep="") + self.fatal( + "The installed version of libarchive (", + ".".join(map(str, bsdtar_version)), + ") has a bug that results in some files not being extracted from the SPEC ISO.", + fixit_hint="Please update bsdtar to at least 3.5.0", + sep="", + ) self.run_cmd(bsdtar, "xf", self.spec_iso_path, "-C", self.extracted_spec_sources, cwd=self.build_dir) # Some of the files in that archive are not user-writable; go pave @@ -394,7 +432,7 @@ def install(self, **kwargs): def install_benchmark_dir(self, root_dir: str): for curdir, dirnames, filenames in os.walk(root_dir): # We don't run some benchmarks (e.g. consumer-typeset or consumer-lame) yet - for ignored_dirname in ('CMakeFiles', ): + for ignored_dirname in ("CMakeFiles",): if ignored_dirname in dirnames: dirnames.remove(ignored_dirname) relpath = os.path.relpath(curdir, root_dir) @@ -427,8 +465,10 @@ def setup_config_options(cls, **kwargs): @property def bundle_dir(self): - return Path(self.build_dir, "lmbench-" + self.crosscompile_target.generic_target_suffix + - self.build_configuration_suffix() + "-bundle") + return Path( + self.build_dir, + "lmbench-" + self.crosscompile_target.generic_target_suffix + self.build_configuration_suffix() + "-bundle", + ) @property def benchmark_version(self): @@ -443,9 +483,11 @@ def benchmark_version(self): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): self.make_args.set(CC="clang") if not self.compiling_for_host(): @@ -461,14 +503,11 @@ def compile(self, **kwargs): def _create_benchmark_dir(self, install_dir: Path): self.makedirs(install_dir) - self.clean_directory(install_dir / "bin", keep_root=False, - ensure_dir_exists=False) - self.clean_directory(install_dir / "scripts", keep_root=False, - ensure_dir_exists=False) + self.clean_directory(install_dir / "bin", keep_root=False, ensure_dir_exists=False) + self.clean_directory(install_dir / "scripts", keep_root=False, ensure_dir_exists=False) self.copy_directory(self.build_dir / "bin", install_dir / "bin") self.copy_directory(self.build_dir / "scripts", install_dir / "scripts") - self.install_file(self.source_dir / "src" / "Makefile", - install_dir / "src" / "Makefile") + self.install_file(self.source_dir / "src" / "Makefile", install_dir / "src" / "Makefile") self.install_file(self.source_dir / "Makefile", install_dir / "Makefile") def install(self, **kwargs): @@ -484,8 +523,9 @@ def run_tests(self): return # testing, not benchmarking -> run only once test_command = f"cd '/build/{self.bundle_dir.name}' && ./run_jenkins-bluehive.sh -d0 -r1 -s" - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) class BuildUnixBench(BenchmarkMixin, CrossCompileProject): @@ -505,13 +545,18 @@ class BuildUnixBench(BenchmarkMixin, CrossCompileProject): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.fixed_iterations = cls.add_bool_option( - "fixed-iterations", default=False, - help="Run benchmarks for given number of iterations instead of duration.") + "fixed-iterations", default=False, help="Run benchmarks for given number of iterations instead of duration." + ) @property def bundle_dir(self): - return Path(self.build_dir, "unixbench-" + self.crosscompile_target.generic_target_suffix + - self.build_configuration_suffix() + "-bundle") + return Path( + self.build_dir, + "unixbench-" + + self.crosscompile_target.generic_target_suffix + + self.build_configuration_suffix() + + "-bundle", + ) @property def benchmark_version(self): @@ -526,9 +571,11 @@ def benchmark_version(self): def compile(self, **kwargs): new_env = dict() if not self.compiling_for_host(): - new_env = dict(MIPS_SDK=self.target_info.sdk_root_dir, - CHERI128_SDK=self.target_info.sdk_root_dir, - CHERI_SDK=self.target_info.sdk_root_dir) + new_env = dict( + MIPS_SDK=self.target_info.sdk_root_dir, + CHERI128_SDK=self.target_info.sdk_root_dir, + CHERI_SDK=self.target_info.sdk_root_dir, + ) with self.set_env(**new_env): self.make_args.set(CC="clang") if self.compiling_for_mips(include_purecap=True): @@ -550,8 +597,7 @@ def compile(self, **kwargs): def _create_benchmark_dir(self, install_dir: Path): self.makedirs(install_dir) - self.clean_directory(install_dir / "pgms", keep_root=False, - ensure_dir_exists=False) + self.clean_directory(install_dir / "pgms", keep_root=False, ensure_dir_exists=False) self.copy_directory(self.build_dir / "UnixBench" / "pgms", install_dir / "pgms") self.install_file(self.source_dir / "run.sh", install_dir / "run.sh") @@ -569,16 +615,21 @@ class NetPerfBench(BenchmarkMixin, CrossCompileAutotoolsProject): # Keep the old bundles when cleaning _extra_git_clean_excludes = ["--exclude=*-bundle"] # The makefiles here can't support any other tagets: - supported_architectures = (CompilationTargets.CHERIBSD_RISCV_NO_CHERI, - CompilationTargets.CHERIBSD_RISCV_HYBRID, - CompilationTargets.CHERIBSD_RISCV_PURECAP) + supported_architectures = ( + CompilationTargets.CHERIBSD_RISCV_NO_CHERI, + CompilationTargets.CHERIBSD_RISCV_HYBRID, + CompilationTargets.CHERIBSD_RISCV_PURECAP, + ) @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.hw_counters = cls.add_config_option("enable-hw-counters", - choices=("pmc", "statcounters"), default="statcounters", - help="Use hardware performance counters") + cls.hw_counters = cls.add_config_option( + "enable-hw-counters", + choices=("pmc", "statcounters"), + default="statcounters", + help="Use hardware performance counters", + ) def configure(self, **kwargs): self.configure_args.append("--enable-unixdomain") @@ -588,8 +639,7 @@ def configure(self, **kwargs): super().configure(**kwargs) def process(self): - if (self.compiling_for_riscv(include_purecap=True) and - self.hw_counters == "pmc"): + if self.compiling_for_riscv(include_purecap=True) and self.hw_counters == "pmc": self.fatal("hwpmc not supported on riscv") return super().process() diff --git a/pycheribuild/projects/cross/bodiagsuite.py b/pycheribuild/projects/cross/bodiagsuite.py index 2a1adc2a0..ed962c406 100644 --- a/pycheribuild/projects/cross/bodiagsuite.py +++ b/pycheribuild/projects/cross/bodiagsuite.py @@ -43,8 +43,9 @@ class BuildBODiagSuite(CrossCompileCMakeProject): target = "bodiagsuite" - repository = GitRepository("https://github.com/CTSRD-CHERI/bodiagsuite", - old_urls=[b"https://github.com/nwf/bodiagsuite"]) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/bodiagsuite", old_urls=[b"https://github.com/nwf/bodiagsuite"] + ) default_install_dir = DefaultInstallDir.DO_NOT_INSTALL default_build_type = BuildType.DEBUG default_use_asan = True @@ -69,19 +70,27 @@ def build_dir_suffix(self): @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.use_valgrind = cls.add_bool_option("use-valgrind", help="Run tests using valgrind (native only)", - only_add_for_targets=(CompilationTargets.NATIVE,)) - cls.use_stack_protector = cls.add_bool_option("use-stack-protector", - help="Compile tests with stack-protector (non-CHERI only)") - cls.use_fortify_source = cls.add_bool_option("use-fortify-source", - help="Compile tests with _DFORTIFY_SOURCE=2 (no effect on " - "FreeBSD)") - cls.use_softboundcets = cls.add_bool_option("use-softboundcets", - help="Compile tests with SoftBoundCETS (native only)", - only_add_for_targets=(CompilationTargets.NATIVE,)) - cls.use_effectivesan = cls.add_bool_option("use-effectivesan", - help="Compile tests with EffectiveSan (native only)", - only_add_for_targets=(CompilationTargets.NATIVE,)) + cls.use_valgrind = cls.add_bool_option( + "use-valgrind", + help="Run tests using valgrind (native only)", + only_add_for_targets=(CompilationTargets.NATIVE,), + ) + cls.use_stack_protector = cls.add_bool_option( + "use-stack-protector", help="Compile tests with stack-protector (non-CHERI only)" + ) + cls.use_fortify_source = cls.add_bool_option( + "use-fortify-source", help="Compile tests with _DFORTIFY_SOURCE=2 (no effect on " "FreeBSD)" + ) + cls.use_softboundcets = cls.add_bool_option( + "use-softboundcets", + help="Compile tests with SoftBoundCETS (native only)", + only_add_for_targets=(CompilationTargets.NATIVE,), + ) + cls.use_effectivesan = cls.add_bool_option( + "use-effectivesan", + help="Compile tests with EffectiveSan (native only)", + only_add_for_targets=(CompilationTargets.NATIVE,), + ) @property def CC(self): # noqa: N802 @@ -136,8 +145,9 @@ def process(self): assert "-lsoftboundcets_rt" in self.default_ldflags # FIXME: add option to disable FORTIFY_SOURCE if self.build_type != BuildType.DEBUG: - self.warning("BODiagsuite contains undefined behaviour that might be optimized away unless you compile" - " at -O0.") + self.warning( + "BODiagsuite contains undefined behaviour that might be optimized away unless you compile" " at -O0." + ) self.ask_for_confirmation("Are you sure you want to continue?") super().process() @@ -187,10 +197,19 @@ def run_tests(self): extra_args.extend(["--junit-testsuite-name", testsuite_prefix]) if self.compiling_for_host(): - extra_args.extend(["--test-native", "--bmake-path", bmake, - "--jobs", str(self.config.make_jobs), - "--build-dir", self.build_dir]) + extra_args.extend( + [ + "--test-native", + "--bmake-path", + bmake, + "--jobs", + str(self.config.make_jobs), + "--build-dir", + self.build_dir, + ] + ) self.run_cmd(self.get_test_script_path("run_bodiagsuite.py"), *extra_args) else: - self.target_info.run_cheribsd_test_script("run_bodiagsuite.py", *extra_args, - mount_sourcedir=False, mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_bodiagsuite.py", *extra_args, mount_sourcedir=False, mount_builddir=True + ) diff --git a/pycheribuild/projects/cross/boost.py b/pycheribuild/projects/cross/boost.py index f50db36db..ea48e2acd 100644 --- a/pycheribuild/projects/cross/boost.py +++ b/pycheribuild/projects/cross/boost.py @@ -39,5 +39,6 @@ def setup(self): included_libraries = [ "range", # used by kactivitymanagerd ] - self.add_cmake_options(BOOST_EXCLUDE_LIBRARIES=";".join(skipped_libraries), - BOOST_INCLUDE_LIBRARIES=";".join(included_libraries)) + self.add_cmake_options( + BOOST_EXCLUDE_LIBRARIES=";".join(skipped_libraries), BOOST_INCLUDE_LIBRARIES=";".join(included_libraries) + ) diff --git a/pycheribuild/projects/cross/cheri_exercises.py b/pycheribuild/projects/cross/cheri_exercises.py index c318d8bf5..111d1e1d0 100644 --- a/pycheribuild/projects/cross/cheri_exercises.py +++ b/pycheribuild/projects/cross/cheri_exercises.py @@ -47,8 +47,7 @@ class BuildCheriExercises(CrossCompileProject): target = "cheri-exercises" repository = GitRepository("https://github.com/CTSRD-CHERI/cheri-exercises.git") - supported_architectures = (CompilationTargets.CHERIBSD_RISCV_PURECAP, - CompilationTargets.CHERIBSD_MORELLO_PURECAP) + supported_architectures = (CompilationTargets.CHERIBSD_RISCV_PURECAP, CompilationTargets.CHERIBSD_MORELLO_PURECAP) default_install_dir = DefaultInstallDir.ROOTFS_OPTBASE path_in_rootfs = "/opt/cheri-exercises" @@ -58,82 +57,102 @@ def __init__(self, *args, **kwargs): def _compile_file(self, output: Path, *args, target_override: "Optional[CrossCompileTarget]" = None): assert isinstance(self.target_info, CheriBSDTargetInfo) - target_flags = self.target_info.get_essential_compiler_and_linker_flags(xtarget=target_override, - default_flags_only=True) + target_flags = self.target_info.get_essential_compiler_and_linker_flags( + xtarget=target_override, default_flags_only=True + ) warning_flags = ["-Wall", "-Wcheri"] - self.run_cmd([str(self.CC), *target_flags, *warning_flags, "-g", "-fuse-ld=lld", "-o", output, *args], - print_verbose_only=False) + self.run_cmd( + [str(self.CC), *target_flags, *warning_flags, "-g", "-fuse-ld=lld", "-o", output, *args], + print_verbose_only=False, + ) self.compiled_files.append(output) def _compile_for_cheri_and_non_cheri(self, output_name_prefix: str, *src_and_args): non_cheri_target = self.crosscompile_target.get_non_cheri_target() - self._compile_file(self.build_dir / (output_name_prefix + "-" + non_cheri_target.generic_arch_suffix), - *src_and_args, target_override=non_cheri_target) + self._compile_file( + self.build_dir / (output_name_prefix + "-" + non_cheri_target.generic_arch_suffix), + *src_and_args, + target_override=non_cheri_target, + ) self._compile_file(self.build_dir / (output_name_prefix + "-cheri"), *src_and_args) def compile(self, **kwargs): # Compile and run RISC-V and CHERI-RISC-V programs self._compile_for_cheri_and_non_cheri( - "print-pointer", self.source_dir / "src/exercises/compile-and-run/print-pointer.c") - self._compile_file(self.build_dir / "print-capability", - self.source_dir / "src/exercises/compile-and-run/print-capability.c") + "print-pointer", self.source_dir / "src/exercises/compile-and-run/print-pointer.c" + ) + self._compile_file( + self.build_dir / "print-capability", self.source_dir / "src/exercises/compile-and-run/print-capability.c" + ) # Exercise sundry inter-object buffer overflows (needs -G0) self._compile_for_cheri_and_non_cheri( "buffer-overflow-global", - self.source_dir / "src/exercises/buffer-overflow-global/buffer-overflow-global.c", "-G0") + self.source_dir / "src/exercises/buffer-overflow-global/buffer-overflow-global.c", + "-G0", + ) self._compile_for_cheri_and_non_cheri( - "buffer-overflow-heap", - self.source_dir / "src/exercises/buffer-overflow-heap/buffer-overflow-heap.c", "-G0") + "buffer-overflow-heap", self.source_dir / "src/exercises/buffer-overflow-heap/buffer-overflow-heap.c", "-G0" + ) self._compile_for_cheri_and_non_cheri( "buffer-overflow-stack", - self.source_dir / "src/exercises/buffer-overflow-stack/buffer-overflow-stack.c", "-G0") + self.source_dir / "src/exercises/buffer-overflow-stack/buffer-overflow-stack.c", + "-G0", + ) # Exercise a subobject buffer overflow self._compile_for_cheri_and_non_cheri( - "subobject-bounds", - self.source_dir / "src/exercises/subobject-bounds/buffer-overflow-subobject.c") + "subobject-bounds", self.source_dir / "src/exercises/subobject-bounds/buffer-overflow-subobject.c" + ) self._compile_file( self.build_dir / "subobject-bounds-cheri-subobject-safe", self.source_dir / "src/exercises/subobject-bounds/buffer-overflow-subobject.c", - "-Xclang", "-cheri-bounds=subobject-safe") # compile another version with subobject bounds + "-Xclang", + "-cheri-bounds=subobject-safe", + ) # compile another version with subobject bounds # Corrupt a data pointer by improperly manipulating it. self._compile_for_cheri_and_non_cheri( - "corrupt-pointer", self.source_dir / "src/exercises/cheri-tags/corrupt-pointer.c") + "corrupt-pointer", self.source_dir / "src/exercises/cheri-tags/corrupt-pointer.c" + ) # Corrupt a control-flow pointer using a subobject buffer overflow self._compile_for_cheri_and_non_cheri( - "buffer-overflow-fnptr", self.source_dir / "src/exercises/control-flow-pointer/buffer-overflow-fnptr.c") + "buffer-overflow-fnptr", self.source_dir / "src/exercises/control-flow-pointer/buffer-overflow-fnptr.c" + ) # Exercise integer-pointer type confusion bug self._compile_for_cheri_and_non_cheri( - "union-int-ptr", self.source_dir / "src/exercises/type-confusion/union-int-ptr.c") + "union-int-ptr", self.source_dir / "src/exercises/type-confusion/union-int-ptr.c" + ) # Demonstrate pointer injection self._compile_for_cheri_and_non_cheri( - "long-over-pipe", self.source_dir / "src/exercises/pointer-injection/long-over-pipe.c") + "long-over-pipe", self.source_dir / "src/exercises/pointer-injection/long-over-pipe.c" + ) self._compile_for_cheri_and_non_cheri( - "ptr-over-pipe", self.source_dir / "src/exercises/pointer-injection/ptr-over-pipe.c") + "ptr-over-pipe", self.source_dir / "src/exercises/pointer-injection/ptr-over-pipe.c" + ) # Demonstrate pointer revocation self._compile_file( self.build_dir / "pointer-revocation-temporal-control", - self.source_dir / "src/exercises/pointer-revocation/temporal-control.c") + self.source_dir / "src/exercises/pointer-revocation/temporal-control.c", + ) self._compile_file( self.build_dir / "pointer-revocation-temporal-control-caprevoke", self.source_dir / "src/exercises/pointer-revocation/temporal-control.c", - "-DCAPREVOKE") + "-DCAPREVOKE", + ) # Demonstrate various CheriABI properties self._compile_for_cheri_and_non_cheri( - "kern-read-over", self.source_dir / "src/exercises/cheriabi/kern-read-over.c") + "kern-read-over", self.source_dir / "src/exercises/cheriabi/kern-read-over.c" + ) - self._compile_for_cheri_and_non_cheri( - "perm-vmem", self.source_dir / "src/exercises/cheriabi/perm-vmem.c") + self._compile_for_cheri_and_non_cheri("perm-vmem", self.source_dir / "src/exercises/cheriabi/perm-vmem.c") - self._compile_for_cheri_and_non_cheri( - "print-more", self.source_dir / "src/exercises/cheriabi/print-more.c") + self._compile_for_cheri_and_non_cheri("print-more", self.source_dir / "src/exercises/cheriabi/print-more.c") # TODO: Demonstrate pointer revocation (however that needs caprevoke) @@ -147,8 +166,10 @@ def install(self, **kwargs): self.install_file(i, self.install_dir / i.name, print_verbose_only=False) # Also install them to the hybrid rootfs: hybrid_target = self.crosscompile_target.get_cheri_hybrid_target() - hybrid_install_dir = self.target_info.get_rootfs_project( - t=Project, xtarget=hybrid_target, caller=self).install_dir / self.path_in_rootfs[1:] + hybrid_install_dir = ( + self.target_info.get_rootfs_project(t=Project, xtarget=hybrid_target, caller=self).install_dir + / self.path_in_rootfs[1:] + ) self.makedirs(hybrid_install_dir) if self.with_clean: self.clean_directory(hybrid_install_dir, ensure_dir_exists=True) diff --git a/pycheribuild/projects/cross/cheribsd.py b/pycheribuild/projects/cross/cheribsd.py index e903354ef..cbe3ad2b3 100644 --- a/pycheribuild/projects/cross/cheribsd.py +++ b/pycheribuild/projects/cross/cheribsd.py @@ -68,6 +68,7 @@ def inner(config: CheriConfig, project: Project): if not isinstance(project, BuildCHERIBSD) and xtarget.is_hybrid_or_purecap_cheri(): raise ValueError(f"{project.target} should not build for CHERI architectures") return config.output_root / (prefix + project.build_configuration_suffix(xtarget)) + return ComputedDefaultValue(function=inner, as_string="$INSTALL_ROOT/" + prefix + "-") @@ -111,8 +112,19 @@ class CheriBSDConfig: Cheribuild configuration descriptor for a CheriBSD kernel configuration file """ - def __init__(self, kernconf: str, platforms: "set[ConfigPlatform]", kernel_abi=KernelABI.NOCHERI, default=False, - nocaprevoke=False, mfsroot=False, debug=False, benchmark=False, fuzzing=False, fett=False): + def __init__( + self, + kernconf: str, + platforms: "set[ConfigPlatform]", + kernel_abi=KernelABI.NOCHERI, + default=False, + nocaprevoke=False, + mfsroot=False, + debug=False, + benchmark=False, + fuzzing=False, + fett=False, + ): self.kernconf = kernconf self.platforms = platforms self.kernel_abi = kernel_abi @@ -127,12 +139,14 @@ def __init__(self, kernconf: str, platforms: "set[ConfigPlatform]", kernel_abi=K def __repr__(self) -> str: flags = [key for key, val in self.__dict__.items() if isinstance(val, bool) and val] return "CheriBSDConfig({kernconf} {platform}:{kernel_abi} [{flags}])".format( - kernconf=self.kernconf, platform=self.platforms, kernel_abi=self.kernel_abi.value, flags=" ".join(flags)) + kernconf=self.kernconf, platform=self.platforms, kernel_abi=self.kernel_abi.value, flags=" ".join(flags) + ) class KernelConfigFactory: - kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict([(k, None) for k in ( - "kabi_name", "nocaprevoke", "platform_name", "flags")]) + kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict( + [(k, None) for k in ("kabi_name", "nocaprevoke", "platform_name", "flags")] + ) separator: str = "_" platform_name_map: "dict[ConfigPlatform, Optional[str]]" = {} @@ -151,8 +165,16 @@ def get_platform_name(self, platforms: "set[ConfigPlatform]") -> Optional[str]: return self.platform_name_map[platform] assert False, "Should not be reached..." - def get_flag_names(self, platforms: "set[ConfigPlatform]", kernel_abi: KernelABI, mfsroot=False, fuzzing=False, - benchmark=False, default=False, nocaprevoke=False): + def get_flag_names( + self, + platforms: "set[ConfigPlatform]", + kernel_abi: KernelABI, + mfsroot=False, + fuzzing=False, + benchmark=False, + default=False, + nocaprevoke=False, + ): flags = [] if mfsroot: flags.append(f"MFS{self.separator}ROOT") @@ -186,8 +208,9 @@ def make_config(self, platforms: "set[ConfigPlatform]", kernel_abi, base_context class RISCVKernelConfigFactory(KernelConfigFactory): - kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict([(k, None) for k in ( - "kabi_name", "nocaprevoke", "platform_name", "flags")]) + kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict( + [(k, None) for k in ("kabi_name", "nocaprevoke", "platform_name", "flags")] + ) separator: str = "-" platform_name_map: "dict[ConfigPlatform, Optional[str]]" = { ConfigPlatform.QEMU: "QEMU", @@ -195,16 +218,27 @@ class RISCVKernelConfigFactory(KernelConfigFactory): ConfigPlatform.AWS: None, } - def get_flag_names(self, platforms: "set[ConfigPlatform]", kernel_abi: KernelABI, default=False, nocaprevoke=False, - mfsroot=False, debug=False, benchmark=False, fuzzing=False, fett=False): + def get_flag_names( + self, + platforms: "set[ConfigPlatform]", + kernel_abi: KernelABI, + default=False, + nocaprevoke=False, + mfsroot=False, + debug=False, + benchmark=False, + fuzzing=False, + fett=False, + ): if ConfigPlatform.GFE in platforms: # Suppress mfsroot flag as it is implied for GFE configurations mfsroot = False flags = [] if fett: flags.append("FETT") - flags += super().get_flag_names(platforms, kernel_abi, mfsroot=mfsroot, fuzzing=fuzzing, benchmark=benchmark, - nocaprevoke=nocaprevoke) + flags += super().get_flag_names( + platforms, kernel_abi, mfsroot=mfsroot, fuzzing=fuzzing, benchmark=benchmark, nocaprevoke=nocaprevoke + ) return flags def make_all(self) -> "list[CheriBSDConfig]": @@ -215,12 +249,14 @@ def make_all(self) -> "list[CheriBSDConfig]": configs.append(self.make_config({ConfigPlatform.QEMU}, kernel_abi, benchmark=True, default=True)) configs.append(self.make_config({ConfigPlatform.QEMU}, kernel_abi, mfsroot=True, default=True)) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, mfsroot=True, benchmark=True, default=True)) + self.make_config({ConfigPlatform.QEMU}, kernel_abi, mfsroot=True, benchmark=True, default=True) + ) # Generate FPGA kernels for kernel_abi in KernelABI: configs.append(self.make_config({ConfigPlatform.GFE}, kernel_abi, mfsroot=True, default=True)) configs.append( - self.make_config({ConfigPlatform.GFE}, kernel_abi, mfsroot=True, benchmark=True, default=True)) + self.make_config({ConfigPlatform.GFE}, kernel_abi, mfsroot=True, benchmark=True, default=True) + ) configs.append(self.make_config({ConfigPlatform.AWS}, kernel_abi, fett=True)) configs.append(self.make_config({ConfigPlatform.AWS}, kernel_abi, fett=True, benchmark=True)) @@ -231,20 +267,25 @@ def make_all(self) -> "list[CheriBSDConfig]": for kernel_abi in KernelABI: configs.append(self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, default=True)) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, default=True)) + self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, default=True) + ) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, mfsroot=True, default=True)) + self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, mfsroot=True, default=True) + ) configs.append( - self.make_config({ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, mfsroot=True, - default=True)) + self.make_config( + {ConfigPlatform.QEMU}, kernel_abi, nocaprevoke=True, benchmark=True, mfsroot=True, default=True + ) + ) configs.append(self.make_config({ConfigPlatform.GFE}, kernel_abi, nocaprevoke=True, mfsroot=True)) return configs class AArch64KernelConfigFactory(KernelConfigFactory): - kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict([(k, None) for k in ( - "platform_name", "kabi_name", "nocaprevoke", "flags")]) + kernconf_components: "typing.OrderedDict[str, Optional[str]]" = OrderedDict( + [(k, None) for k in ("platform_name", "kabi_name", "nocaprevoke", "flags")] + ) separator: str = "-" platform_name_map: "dict[ConfigPlatform, Optional[str]]" = { ConfigPlatform.QEMU: "GENERIC", @@ -264,22 +305,37 @@ def make_all(self) -> "list[CheriBSDConfig]": # Generate QEMU/FVP kernels for kernel_abi in KernelABI: configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True)) - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - benchmark=True)) - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - mfsroot=True)) + configs.append( + self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, benchmark=True) + ) + configs.append( + self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, mfsroot=True) + ) # Caprevoke kernels for kernel_abi in KernelABI: - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - nocaprevoke=True)) - configs.append(self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, - nocaprevoke=True, benchmark=True)) + configs.append( + self.make_config({ConfigPlatform.QEMU, ConfigPlatform.FVP}, kernel_abi, default=True, nocaprevoke=True) + ) + configs.append( + self.make_config( + {ConfigPlatform.QEMU, ConfigPlatform.FVP}, + kernel_abi, + default=True, + nocaprevoke=True, + benchmark=True, + ) + ) return configs -def filter_kernel_configs(configs: "list[CheriBSDConfig]", *, platform: "Optional[ConfigPlatform]", - kernel_abi: Optional[KernelABI], **filter_kwargs) -> "typing.Sequence[CheriBSDConfig]": +def filter_kernel_configs( + configs: "list[CheriBSDConfig]", + *, + platform: "Optional[ConfigPlatform]", + kernel_abi: Optional[KernelABI], + **filter_kwargs, +) -> "typing.Sequence[CheriBSDConfig]": """ Helper function to filter kernel configuration lists. Keyword filter arguments are mapped to CheriBSDConfig properties. @@ -346,17 +402,22 @@ def get_default(cls, xtarget, platform: ConfigPlatform, kernel_abi: KernelABI, * It is a fatal failure if 0 or more than one configurations exist. """ configs = cls.get_configs(xtarget, platform=platform, kernel_abi=kernel_abi, default=True, **filter_kwargs) - assert len(configs) != 0, f"No matching default kernel configuration for xtarget={xtarget}, " \ - f"platform={platform.name}, kernel_abi={kernel_abi.name}, " \ - f"filter_kwargs={filter_kwargs}" - assert len(configs) == 1, f"Too many default kernel configurations {configs} for xtarget={xtarget}, " \ - f"platform={platform.name}, kernel_abi={kernel_abi.name}, " \ - f"filter_kwargs={filter_kwargs}" + assert len(configs) != 0, ( + f"No matching default kernel configuration for xtarget={xtarget}, " + f"platform={platform.name}, kernel_abi={kernel_abi.name}, " + f"filter_kwargs={filter_kwargs}" + ) + assert len(configs) == 1, ( + f"Too many default kernel configurations {configs} for xtarget={xtarget}, " + f"platform={platform.name}, kernel_abi={kernel_abi.name}, " + f"filter_kwargs={filter_kwargs}" + ) return configs[0] @classmethod - def get_configs(cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", - **filter_kwargs): + def get_configs( + cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", **filter_kwargs + ): """ Return all configurations for a combination of target, platform and kernel ABI. This filters out all the specialized configuration flags defaulting all of them @@ -371,14 +432,16 @@ def get_configs(cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_ab return cls.get_all_configs(xtarget, platform=platform, kernel_abi=kernel_abi, **filter_kwargs) @classmethod - def get_all_configs(cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", - **filter_kwargs): + def get_all_configs( + cls, xtarget, *, platform: "Optional[ConfigPlatform]", kernel_abi: "Optional[KernelABI]", **filter_kwargs + ): """ Return all available configurations for a combination of target, group and kernel ABI filtered using kwargs. """ - return filter_kernel_configs(cls.get_target_configs(xtarget), platform=platform, kernel_abi=kernel_abi, - **filter_kwargs) + return filter_kernel_configs( + cls.get_target_configs(xtarget), platform=platform, kernel_abi=kernel_abi, **filter_kwargs + ) class BuildFreeBSDBase(Project): @@ -425,20 +488,27 @@ def crossbuild(self) -> bool: def setup_config_options(cls, kernel_only_target=False, **kwargs) -> None: super().setup_config_options(**kwargs) cls.extra_make_args = cls.add_list_option( - "build-options", default=cls.default_extra_make_options, metavar="OPTIONS", + "build-options", + default=cls.default_extra_make_options, + metavar="OPTIONS", help="Additional make options to be passed to make when building FreeBSD/CheriBSD. See `man src.conf` " - "for more info.", show_help=True) - cls.debug_kernel = cls.add_bool_option("debug-kernel", help="Build the kernel with -O0 and verbose boot output", - show_help=False) + "for more info.", + show_help=True, + ) + cls.debug_kernel = cls.add_bool_option( + "debug-kernel", help="Build the kernel with -O0 and verbose boot output", show_help=False + ) if kernel_only_target: cls.minimal = False cls.build_tests = False return # The remaining options only affect the userspace build if "minimal" not in cls.__dict__: - cls.minimal = cls.add_bool_option("minimal", show_help=False, - help="Don't build all of FreeBSD, just what is needed for running most " - "CHERI tests/benchmarks") + cls.minimal = cls.add_bool_option( + "minimal", + show_help=False, + help="Don't build all of FreeBSD, just what is needed for running most " "CHERI tests/benchmarks", + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() @@ -455,8 +525,10 @@ def setup(self) -> None: if self.crossbuild: # Use the script that I added for building on Linux/MacOS: - self.make_args.set_command(self.source_dir / "tools/build/make.py", - early_args=["--bootstrap-toolchain"] if self.use_bootstrapped_toolchain else []) + self.make_args.set_command( + self.source_dir / "tools/build/make.py", + early_args=["--bootstrap-toolchain"] if self.use_bootstrapped_toolchain else [], + ) self.make_args.set( DB_FROM_SRC=True, # don't use the system passwd file @@ -468,8 +540,19 @@ def setup(self) -> None: self.make_args.set_with_options(CLEAN=False) if self.minimal: - self.make_args.set_with_options(MAN=False, KERBEROS=False, SVN=False, SVNLITE=False, MAIL=False, ZFS=False, - SENDMAIL=False, EXAMPLES=False, LOCALES=False, NLS=False, CDDL=False) + self.make_args.set_with_options( + MAN=False, + KERBEROS=False, + SVN=False, + SVNLITE=False, + MAIL=False, + ZFS=False, + SENDMAIL=False, + EXAMPLES=False, + LOCALES=False, + NLS=False, + CDDL=False, + ) # tests off by default because they take a long time and often seems to break # the creation of disk-image (METALOG is invalid) @@ -489,9 +572,15 @@ def setup(self) -> None: for option in self.extra_make_args: if not self.crosscompile_target.is_hybrid_or_purecap_cheri() and "CHERI_" in option: - self.warning("Should not be adding CHERI specific make option", option, "for", self.target, - " -- consider setting separate", self.get_config_option_name("extra_make_args"), - "in the config file.") + self.warning( + "Should not be adding CHERI specific make option", + option, + "for", + self.target, + " -- consider setting separate", + self.get_config_option_name("extra_make_args"), + "in the config file.", + ) if "=" in option: key, value = option.split("=", 1) args = {key: value} @@ -548,20 +637,32 @@ def get_rootfs_dir(cls, caller, cross_target: "Optional[CrossCompileTarget]" = N return cls.get_install_dir(caller, cross_target) @classmethod - def setup_config_options(cls, bootstrap_toolchain=False, use_upstream_llvm: Optional[bool] = None, - debug_info_by_default=True, kernel_only_target=False, **kwargs) -> None: + def setup_config_options( + cls, + bootstrap_toolchain=False, + use_upstream_llvm: Optional[bool] = None, + debug_info_by_default=True, + kernel_only_target=False, + **kwargs, + ) -> None: super().setup_config_options(kernel_only_target=kernel_only_target, **kwargs) if cls._xtarget: # KERNCONF always depends on the target, so we don't inherit this config option. The only exception is # the global --kernel-config option that is provided for convenience and backwards compat. cls.kernel_config = cls.add_config_option( - "kernel-config", metavar="CONFIG", show_help=True, extra_fallback_config_names=["kernel-config"], + "kernel-config", + metavar="CONFIG", + show_help=True, + extra_fallback_config_names=["kernel-config"], default=ComputedDefaultValue( - function=lambda _, p: - p.default_kernel_config() if p.has_default_buildkernel_kernel_config else None, - as_string="target-dependent, usually GENERIC"), + function=lambda _, p: p.default_kernel_config() + if p.has_default_buildkernel_kernel_config + else None, + as_string="target-dependent, usually GENERIC", + ), use_default_fallback_config_names=False, # - help="The kernel configuration to use for `make buildkernel`") + help="The kernel configuration to use for `make buildkernel`", + ) if cls._xtarget is not None and cls._xtarget.is_hybrid_or_purecap_cheri(): # When targeting CHERI we have to use CHERI LLVM @@ -579,25 +680,40 @@ def setup_config_options(cls, bootstrap_toolchain=False, use_upstream_llvm: Opti else: # Prefer using system clang for FreeBSD builds rather than a self-built snapshot of LLVM since that might # have new warnings that break the -Werror build. - cls.build_toolchain = typing.cast(CompilerType, cls.add_config_option( - "toolchain", kind=CompilerType, default=CompilerType.DEFAULT_COMPILER, - enum_choice_strings=[t.value for t in CompilerType], - help="The toolchain to use for building FreeBSD. When set to 'custom', the 'toolchain-path' option " - "must also be set")) + cls.build_toolchain = typing.cast( + CompilerType, + cls.add_config_option( + "toolchain", + kind=CompilerType, + default=CompilerType.DEFAULT_COMPILER, + enum_choice_strings=[t.value for t in CompilerType], + help="The toolchain to use for building FreeBSD. When set to 'custom', the 'toolchain-path' option " + "must also be set", + ), + ) cls._cross_toolchain_root = cls.add_optional_path_option( - "toolchain-path", help="Path to the cross toolchain tools") + "toolchain-path", help="Path to the cross toolchain tools" + ) # override in CheriBSD - cls.linker_for_world = cls.add_config_option("linker-for-world", default="lld", choices=["bfd", "lld"], - help="The linker to use for world") - cls.linker_for_kernel = cls.add_config_option("linker-for-kernel", default="lld", choices=["bfd", "lld"], - help="The linker to use for the kernel") - - cls.with_debug_info = cls.add_bool_option("debug-info", default=debug_info_by_default, show_help=True, - help="pass make flags for building with debug info") - cls.with_debug_files = cls.add_bool_option("debug-files", default=True, - help="Use split DWARF debug files if building with debug info") + cls.linker_for_world = cls.add_config_option( + "linker-for-world", default="lld", choices=["bfd", "lld"], help="The linker to use for world" + ) + cls.linker_for_kernel = cls.add_config_option( + "linker-for-kernel", default="lld", choices=["bfd", "lld"], help="The linker to use for the kernel" + ) + + cls.with_debug_info = cls.add_bool_option( + "debug-info", + default=debug_info_by_default, + show_help=True, + help="pass make flags for building with debug info", + ) + cls.with_debug_files = cls.add_bool_option( + "debug-files", default=True, help="Use split DWARF debug files if building with debug info" + ) cls.fast_rebuild = cls.add_bool_option( - "fast", help="Skip some (usually) unnecessary build steps to speed up rebuilds") + "fast", help="Skip some (usually) unnecessary build steps to speed up rebuilds" + ) if kernel_only_target: cls.build_lib32 = False @@ -605,26 +721,39 @@ def setup_config_options(cls, bootstrap_toolchain=False, use_upstream_llvm: Opti cls.with_manpages = False return # The remaining options only affect the userspace build - subdir_default = ComputedDefaultValue(function=lambda config, proj: config.freebsd_subdir, - as_string="the value of the global --freebsd-subdir options") + subdir_default = ComputedDefaultValue( + function=lambda config, proj: config.freebsd_subdir, + as_string="the value of the global --freebsd-subdir options", + ) cls.explicit_subdirs_only = cls.add_list_option( - "subdir", metavar="SUBDIRS", show_help=True, default=subdir_default, + "subdir", + metavar="SUBDIRS", + show_help=True, + default=subdir_default, help="Only build subdirs SUBDIRS instead of the full tree. Useful for quickly rebuilding individual" - " programs/libraries. If more than one dir is passed, they will be processed in order. Note: This" - " will break if not all dependencies have been built.") + " programs/libraries. If more than one dir is passed, they will be processed in order. Note: This" + " will break if not all dependencies have been built.", + ) cls.keep_old_rootfs = cls.add_bool_option( - "keep-old-rootfs", help="Don't remove the whole old rootfs directory. This can speed up installing but" - " may cause strange errors so is off by default.") - cls.with_manpages = cls.add_bool_option("with-manpages", help="Also install manpages. This is off by default" - " since they can just be read from the host.") - cls.build_drm_kmod = cls.add_bool_option("build-drm-kmod", help="Also build drm-kmod during buildkernel", - show_help=False) + "keep-old-rootfs", + help="Don't remove the whole old rootfs directory. This can speed up installing but" + " may cause strange errors so is off by default.", + ) + cls.with_manpages = cls.add_bool_option( + "with-manpages", + help="Also install manpages. This is off by default" " since they can just be read from the host.", + ) + cls.build_drm_kmod = cls.add_bool_option( + "build-drm-kmod", help="Also build drm-kmod during buildkernel", show_help=False + ) if cls._xtarget is None or not cls._xtarget.cpu_architecture.is_32bit(): cls.build_lib32 = cls.add_bool_option( - "build-lib32", default=False, - help="Build the 32-bit compatibility userspace libraries (if supported for the current architecture)") + "build-lib32", + default=False, + help="Build the 32-bit compatibility userspace libraries (if supported for the current architecture)", + ) else: # XXX: this is not correct if we were to support a CHERI-64 userspace assert not cls._xtarget.is_hybrid_or_purecap_cheri() @@ -743,20 +872,31 @@ def _try_find_compatible_system_clang(self) -> "tuple[Optional[Path], Optional[s if cc_info.is_clang and not cc_info.is_apple_clang and cc_info.version >= min_version: compiler_path = cc_info.path else: - return (None, "Could not find a system installation of clang.", - "Please install a recent upstream clang or use the 'custom' or 'upstream-llvm' toolchain " - "option.") + return ( + None, + "Could not find a system installation of clang.", + "Please install a recent upstream clang or use the 'custom' or 'upstream-llvm' toolchain " + "option.", + ) self.info("Checking if", compiler_path, "can be used to build FreeBSD...") cc_info = self.get_compiler_info(compiler_path) if cc_info.is_apple_clang: - return (None, "Cannot build FreeBSD with Apple clang.", - "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " - "or 'upstream-llvm' toolchain option.") + return ( + None, + "Cannot build FreeBSD with Apple clang.", + "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " + "or 'upstream-llvm' toolchain option.", + ) if cc_info.version < min_version: - return (None, "Cannot build FreeBSD with Clang older than " + ".".join(map(str, min_version)) + - ". Found clang = " + str(compiler_path), - "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " - "or 'upstream-llvm' toolchain option.") + return ( + None, + "Cannot build FreeBSD with Clang older than " + + ".".join(map(str, min_version)) + + ". Found clang = " + + str(compiler_path), + "Please install a recent upstream clang (e.g. with homebrew) or use the 'custom' " + "or 'upstream-llvm' toolchain option.", + ) # Note: FreeBSD installs shell script wrappers for clang, so we can't just use # Path(compiler_path).resolve().parent.parent since that will try to use /usr/local/bin/clang. Instead # we print the resource dir (/lib/clang/) and go up three levels from there. @@ -785,8 +925,7 @@ def setup(self) -> None: def build_toolchain_root_dir(self) -> "Optional[Path]": if self.build_toolchain == CompilerType.BOOTSTRAPPED: return self.objdir / "tmp/usr" - elif self.build_toolchain in (CompilerType.UPSTREAM_LLVM, CompilerType.CHERI_LLVM, - CompilerType.MORELLO_LLVM): + elif self.build_toolchain in (CompilerType.UPSTREAM_LLVM, CompilerType.CHERI_LLVM, CompilerType.MORELLO_LLVM): return BuildLLVMMonoRepoBase.get_install_dir_for_type(self, self.build_toolchain) elif self.build_toolchain == CompilerType.SYSTEM_LLVM: system_clang_root, errmsg, fixit = self._try_find_compatible_system_clang() @@ -795,8 +934,11 @@ def build_toolchain_root_dir(self) -> "Optional[Path]": return system_clang_root elif self.build_toolchain == CompilerType.CUSTOM: if self._cross_toolchain_root is None: - self.fatal("Requested custom toolchain but", self.get_config_option_name("_cross_toolchain_root"), - "is not set.") + self.fatal( + "Requested custom toolchain but", + self.get_config_option_name("_cross_toolchain_root"), + "is not set.", + ) return self._cross_toolchain_root else: assert self.build_toolchain == CompilerType.DEFAULT_COMPILER @@ -814,10 +956,14 @@ def _setup_cross_toolchain_config(self) -> None: self.cross_toolchain_config.set_with_options( # TODO: should we have an option to include a compiler in the target system? - GCC=False, CLANG=False, LLD=False, # Take a long time and not needed in the target system + GCC=False, + CLANG=False, + LLD=False, # Take a long time and not needed in the target system LLDB=False, # may be useful but means we need to build LLVM # Bootstrap compiler/ linker are not needed: - GCC_BOOTSTRAP=False, CLANG_BOOTSTRAP=False, LLD_BOOTSTRAP=False, + GCC_BOOTSTRAP=False, + CLANG_BOOTSTRAP=False, + LLD_BOOTSTRAP=False, ) if not self.build_lib32: # takes a long time and usually not needed. @@ -838,7 +984,9 @@ def _setup_cross_toolchain_config(self) -> None: if not xccinfo.is_clang: self.ask_for_confirmation("Cross compiler is not clang, are you sure you want to continue?") self.cross_toolchain_config.set_env( - XCC=self.CC, XCXX=self.CXX, XCPP=self.CPP, + XCC=self.CC, + XCXX=self.CXX, + XCPP=self.CPP, X_COMPILER_TYPE=xccinfo.compiler, # This is needed otherwise the build assumes it should build with $CC ) if not self.use_llvm_binutils: @@ -898,8 +1046,10 @@ def _setup_arch_specific_options(self) -> str: # TODO: should probably set that inside CheriBSD makefiles instead if self.target_info.is_cheribsd(): target_flags += " -mcpu=beri" - self.cross_toolchain_config.set_with_options(RESCUE=False, # Won't compile with CHERI clang yet - BOOT=False) # bootloaders won't link with LLD yet + self.cross_toolchain_config.set_with_options( + RESCUE=False, # Won't compile with CHERI clang yet + BOOT=False, + ) # bootloaders won't link with LLD yet elif self.compiling_for_riscv(include_purecap=True): target_flags = "" else: @@ -965,6 +1115,7 @@ def kernel_make_args_for_config(self, kernconfs: "list[str]", extra_make_args) - kernel_options.remove_var("X" + binutil_name) if self.build_drm_kmod: from .drm_kmod import BuildDrmKMod + drm_kmod = BuildDrmKMod.get_instance(self) kernel_options.set(LOCAL_MODULES=drm_kmod.source_dir.name, LOCAL_MODULES_DIR=drm_kmod.source_dir.parent) kernel_options.set(KERNCONF=" ".join(kernconfs)) @@ -985,7 +1136,8 @@ def clean(self) -> ThreadJoiner: cleaning_kerneldir = True if kernel_dir.exists() and self.build_dir.exists(): assert not os.path.relpath(str(kernel_dir.resolve()), str(self.build_dir.resolve())).startswith( - ".."), builddir + ".." + ), builddir else: self.warning("Do not know the full path to the kernel build directory, will clean the whole tree!") if self.crossbuild: @@ -1001,20 +1153,28 @@ def _list_kernel_configs(self) -> None: blacklist = ["NOTES", "LINT", "DEFAULTS"] self.info("Valid kernel configuration files for --" + self.target + "/kernel-config:") for conf in configs: - if (conf.name in blacklist or conf.name.startswith("std.") or conf.name.endswith(".hints") or - conf.name.endswith("~")): + if ( + conf.name in blacklist + or conf.name.startswith("std.") + or conf.name.endswith(".hints") + or conf.name.endswith("~") + ): continue self.info(conf.name) - def _buildkernel(self, kernconfs: "list[str]", mfs_root_image: "Optional[Path]" = None, extra_make_args=None, - ignore_skip_kernel=False) -> None: + def _buildkernel( + self, + kernconfs: "list[str]", + mfs_root_image: "Optional[Path]" = None, + extra_make_args=None, + ignore_skip_kernel=False, + ) -> None: # Check that --skip-kernel is respected. However, we ignore it for the cheribsd-mfs-root-kernel targets # since those targets only build a kernel. assert not self.config.skip_kernel or ignore_skip_kernel, "--skip-kernel set but building kernel" kernel_make_args = self.kernel_make_args_for_config(kernconfs, extra_make_args) if not self.use_bootstrapped_toolchain and not self.CC.exists(): - self.fatal("Requested build of kernel with external toolchain, but", self.CC, - "doesn't exist!") + self.fatal("Requested build of kernel with external toolchain, but", self.CC, "doesn't exist!") if self.debug_kernel: if any(x.endswith(("_BENCHMARK", "-NODEBUG")) for x in kernconfs): if not self.query_yes_no("Trying to build BENCHMARK kernel without optimization. Continue?"): @@ -1035,11 +1195,15 @@ def _buildkernel(self, kernconfs: "list[str]", mfs_root_image: "Optional[Path]" self.run_make("kernel-toolchain", options=kernel_toolchain_opts) self.kernel_toolchain_exists = True self.info("Building kernels for configs:", " ".join(kernconfs)) - self.run_make("buildkernel", options=kernel_make_args, - compilation_db_name="compile_commands_" + " ".join(kernconfs).replace(" ", "_") + ".json") + self.run_make( + "buildkernel", + options=kernel_make_args, + compilation_db_name="compile_commands_" + " ".join(kernconfs).replace(" ", "_") + ".json", + ) - def _installkernel(self, kernconfs: "list[str]", *, install_dir: Path, extra_make_args=None, - ignore_skip_kernel=False) -> None: + def _installkernel( + self, kernconfs: "list[str]", *, install_dir: Path, extra_make_args=None, ignore_skip_kernel=False + ) -> None: # Check that --skip-kernel is respected. However, we ignore it for the cheribsd-mfs-root-kernel targets # since those targets only build a kernel. assert not self.config.skip_kernel or ignore_skip_kernel, "--skip-kernel set but building kernel" @@ -1057,8 +1221,15 @@ def _installkernel(self, kernconfs: "list[str]", *, install_dir: Path, extra_mak self.delete_file(install_dir / "METALOG.kernel") # Ensure that METALOG does not contain stale values. self.run_make("installkernel", options=install_kernel_args, parallel=False) - def _copykernel(self, kernconfs: "list[str]", rootfs_dir: Path, dest_dir: Path, *, - dest_kernel_suffix: str = "", dest_all_extra: bool = False) -> None: + def _copykernel( + self, + kernconfs: "list[str]", + rootfs_dir: Path, + dest_dir: Path, + *, + dest_kernel_suffix: str = "", + dest_all_extra: bool = False, + ) -> None: """ Copy kernels from the rootfs to the given directory (for MFS-ROOT kernels and Jenkins) """ @@ -1124,10 +1295,24 @@ def _remove_old_rootfs(self) -> None: if os.getuid() == 0: # if we installed as root remove the schg flag from files before cleaning (otherwise rm will fail) self._remove_schg_flag( - "lib/libc.so.7", "lib/libcrypt.so.5", "lib/libthr.so.3", - "libexec/ld-elf.so.1", "sbin/init", "usr/bin/chpass", "usr/bin/chsh", "usr/bin/ypchpass", - "usr/bin/ypchfn", "usr/bin/ypchsh", "usr/bin/login", "usr/bin/opieinfo", "usr/bin/opiepasswd", - "usr/bin/passwd", "usr/bin/yppasswd", "usr/bin/su", "usr/bin/crontab", "usr/lib/librt.so.1", + "lib/libc.so.7", + "lib/libcrypt.so.5", + "lib/libthr.so.3", + "libexec/ld-elf.so.1", + "sbin/init", + "usr/bin/chpass", + "usr/bin/chsh", + "usr/bin/ypchpass", + "usr/bin/ypchfn", + "usr/bin/ypchsh", + "usr/bin/login", + "usr/bin/opieinfo", + "usr/bin/opiepasswd", + "usr/bin/passwd", + "usr/bin/yppasswd", + "usr/bin/su", + "usr/bin/crontab", + "usr/lib/librt.so.1", "var/empty", ) # We keep 3rd-party programs (anything installed in /usr/local + /opt), but delete everything else prior @@ -1163,19 +1348,30 @@ def _query_make_var(self, args, var) -> Optional[Path]: return None query_args = args.copy() query_args.set_command(bmake_binary) - bw_flags = [*query_args.all_commandline_args(self.config), - "BUILD_WITH_STRICT_TMPPATH=0", - "-f", self.source_dir / "Makefile.inc1", - "-m", self.source_dir / "share/mk", - "showconfig", - "-D_NO_INCLUDE_COMPILERMK", # avoid calling ${CC} --version - "-V", var] + bw_flags = [ + *query_args.all_commandline_args(self.config), + "BUILD_WITH_STRICT_TMPPATH=0", + "-f", + self.source_dir / "Makefile.inc1", + "-m", + self.source_dir / "share/mk", + "showconfig", + "-D_NO_INCLUDE_COMPILERMK", # avoid calling ${CC} --version + "-V", + var, + ] if not self.source_dir.exists(): assert self.config.pretend, "This should only happen when running in a test environment" return None # https://github.com/freebsd/freebsd/commit/1edb3ba87657e28b017dffbdc3d0b3a32999d933 - cmd = self.run_cmd([bmake_binary, *bw_flags], env=args.env_vars, cwd=self.source_dir, - run_in_pretend_mode=True, capture_output=True, print_verbose_only=True) + cmd = self.run_cmd( + [bmake_binary, *bw_flags], + env=args.env_vars, + cwd=self.source_dir, + run_in_pretend_mode=True, + capture_output=True, + print_verbose_only=True, + ) lines = cmd.stdout.strip().split(b"\n") last_line = lines[-1].decode("utf-8").strip() if last_line.startswith("/") and cmd.returncode == 0: @@ -1193,8 +1389,9 @@ def objdir(self) -> Path: if result is None: result = Path() if self.realpath(result) == self.realpath(self.source_dir): - self.warning("bmake claims the build dir for", self.target, "is the source dir, assuming", - self.build_dir, "instead.") + self.warning( + "bmake claims the build dir for", self.target, "is the source dir, assuming", self.build_dir, "instead." + ) return self.build_dir if not result or result == Path(): # just clean the whole directory instead @@ -1239,8 +1436,10 @@ def install(self, *, kernconfs: "Optional[list[str]]" = None, sysroot_only=False # support building old versions of cheribsd before _compiler-metadata was renamed to _build-metadata self.run_make("_compiler-metadata", options=install_world_args) except subprocess.CalledProcessError: - self.warning("Failed to run either target _compiler-metadata or " - "_build_metadata, build system has changed!") + self.warning( + "Failed to run either target _compiler-metadata or " + "_build_metadata, build system has changed!" + ) # By default also create a sysroot when installing world installsysroot_args = install_world_args.copy() if self.has_installsysroot_target: @@ -1271,18 +1470,19 @@ def install(self, *, kernconfs: "Optional[list[str]]" = None, sysroot_only=False def rewrite_passwd(old): new = [] for line in old: - fields = line.split(':') + fields = line.split(":") if len(fields) == 10 and fields[0] == "toor" and fields[1] == "*" and not fields[9]: fields[1] = "" fields[9] = "/bin/sh" - line = ':'.join(fields) + line = ":".join(fields) new.append(line) return new pwd_mkdb_cmd = self.objdir / "tmp/legacy/usr/bin/pwd_mkdb" self.rewrite_file(self.destdir / "etc/master.passwd", rewrite_passwd) - self.run_cmd([pwd_mkdb_cmd, "-p", "-d", self.install_dir / "etc", - self.install_dir / "etc/master.passwd"]) + self.run_cmd( + [pwd_mkdb_cmd, "-p", "-d", self.install_dir / "etc", self.install_dir / "etc/master.passwd"] + ) assert not sysroot_only, "Should not end up here" if self.config.skip_kernel: @@ -1298,8 +1498,12 @@ def rewrite_passwd(old): self._copykernel(kernconfs=kernconfs, rootfs_dir=self.install_dir, dest_dir=self.config.output_root) def add_cross_build_options(self) -> None: - self.make_args.set_env(CC=self.host_CC, CXX=self.host_CXX, CPP=self.host_CPP, - STRIPBIN=shutil.which("strip") or shutil.which("llvm-strip") or "strip") + self.make_args.set_env( + CC=self.host_CC, + CXX=self.host_CXX, + CPP=self.host_CPP, + STRIPBIN=shutil.which("strip") or shutil.which("llvm-strip") or "strip", + ) if self.use_bootstrapped_toolchain: assert "XCC" not in self.make_args.env_vars # We have to provide the default X* values so that Makefile.inc1 does not disable MK_CLANG_BOOTSTRAP and @@ -1342,22 +1546,39 @@ def process(self) -> None: args = self.buildworld_args args.remove_flag("-s") # buildenv should not be silent if "bash" in os.getenv("SHELL", ""): - args.set(BUILDENV_SHELL="env -u PROMPT_COMMAND 'PS1=" + self.target + "-buildenv:\\w> ' " + - shutil.which("bash") + " --norc --noprofile") + args.set( + BUILDENV_SHELL="env -u PROMPT_COMMAND 'PS1=" + + self.target + + "-buildenv:\\w> ' " + + shutil.which("bash") + + " --norc --noprofile" + ) else: args.set(BUILDENV_SHELL="/bin/sh") buildenv_target = "buildenv" if self.config.libcompat_buildenv and self.libcompat_name(): buildenv_target = self.libcompat_name() + "buildenv" - self.run_cmd([self.make_args.command, *args.all_commandline_args(self.config), buildenv_target], - env=args.env_vars, cwd=self.source_dir) + self.run_cmd( + [self.make_args.command, *args.all_commandline_args(self.config), buildenv_target], + env=args.env_vars, + cwd=self.source_dir, + ) else: super().process() - def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_clean=None, skip_install=None, - install_to_internal_sysroot=True, libcompat_only=False, noncheri_only=False) -> None: + def build_and_install_subdir( + self, + make_args, + subdir, + skip_build=False, + skip_clean=None, + skip_install=None, + install_to_internal_sysroot=True, + libcompat_only=False, + noncheri_only=False, + ) -> None: is_lib = subdir.startswith("lib/") or "/lib/" in subdir or subdir.endswith("/lib") - make_in_subdir = "make -C \"" + subdir + "\" " + make_in_subdir = 'make -C "' + subdir + '" ' if skip_clean is None: skip_clean = not self.with_clean if skip_install is None: @@ -1367,10 +1588,10 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle install_to_sysroot_cmd = "" # We have to override INSTALL so that the sysroot installations don't end up in METALOG # This happens after https://github.com/freebsd/freebsd/commit/5496ab2ac950813edbd55d73c967184e033bea2f - install_nometalog_cmd = "INSTALL=\"install -N " + str(self.source_dir / "etc") + " -U\" METALOG=/dev/null" + install_nometalog_cmd = 'INSTALL="install -N ' + str(self.source_dir / "etc") + ' -U" METALOG=/dev/null' if is_lib and install_to_internal_sysroot: # Due to all the bmake + shell escaping I need 4 dollars here to get it to expand SYSROOT - sysroot_var = "\"$$$${SYSROOT}\"" + sysroot_var = '"$$$${SYSROOT}"' install_to_sysroot_cmd = ( f"if [ -n {sysroot_var} ]; then" f" {make_in_subdir} install {install_nometalog_cmd} MK_TESTS=no DESTDIR={sysroot_var}; " @@ -1380,7 +1601,7 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle if install_to_sysroot_cmd: install_cmd = install_to_sysroot_cmd else: - install_cmd = "echo \" Skipping make install\"" + install_cmd = 'echo " Skipping make install"' else: # if we are building a library also install to the sysroot so that other targets afterwards use the # updated static lib @@ -1391,11 +1612,14 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle # Note that installconfig doesn't fail if there is no configuration in a subdirectory. install_cmd = install_to_sysroot_cmd + make_in_subdir + "install installconfig " + install_nometalog_cmd colour_diags = "export CLANG_FORCE_COLOR_DIAGNOSTICS=always; " if self.config.clang_colour_diags else "" - build_cmd = "{colour_diags} {clean} && {build} && {install} && echo \" Done.\"".format( - build=make_in_subdir + "all " + self.commandline_to_str( - self.jflag) if not skip_build else "echo \" Skipping make all\"", - clean=make_in_subdir + "clean" if not skip_clean else "echo \" Skipping make clean\"", - install=install_cmd, colour_diags=colour_diags) + build_cmd = '{colour_diags} {clean} && {build} && {install} && echo " Done."'.format( + build=make_in_subdir + "all " + self.commandline_to_str(self.jflag) + if not skip_build + else 'echo " Skipping make all"', + clean=make_in_subdir + "clean" if not skip_clean else 'echo " Skipping make clean"', + install=install_cmd, + colour_diags=colour_diags, + ) make_args.set(BUILDENV_SHELL="sh -ex -c '" + build_cmd + "' || exit 1") # If --libcompat-buildenv was passed skip the MIPS lib has_libcompat = self.crosscompile_target.is_hybrid_or_purecap_cheri() and is_lib # TODO: handle lib32 @@ -1403,8 +1627,11 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle self.info("Skipping default ABI build of", subdir, "since --libcompat-buildenv was passed.") else: self.info("Building", subdir, "using buildenv target") - self.run_cmd([self.make_args.command, *make_args.all_commandline_args(self.config), "buildenv"], - env=make_args.env_vars, cwd=self.source_dir) + self.run_cmd( + [self.make_args.command, *make_args.all_commandline_args(self.config), "buildenv"], + env=make_args.env_vars, + cwd=self.source_dir, + ) # If we are building a library, we want to build both the CHERI and the mips version (unless the # user explicitly specified --libcompat-buildenv) if has_libcompat and not noncheri_only and self.libcompat_name(): @@ -1413,10 +1640,13 @@ def build_and_install_subdir(self, make_args, subdir, skip_build=False, skip_cle extra_flags = ["MK_TESTS=no"] # don't build tests since they will overwrite the non-compat ones self.run_cmd( [self.make_args.command, *make_args.all_commandline_args(self.config), *extra_flags, compat_target], - env=make_args.env_vars, cwd=self.source_dir) + env=make_args.env_vars, + cwd=self.source_dir, + ) - def _get_kernel_rootfs_install_path(self, kernconf: "Optional[str]", rootfs: Path, - default_kernconf: "Optional[str]") -> Path: + def _get_kernel_rootfs_install_path( + self, kernconf: "Optional[str]", rootfs: Path, default_kernconf: "Optional[str]" + ) -> Path: if kernconf is None or kernconf == default_kernconf: kerndir = "kernel" else: @@ -1473,17 +1703,20 @@ class BuildFreeBSDWithDefaultOptions(BuildFreeBSD): def clean(self) -> ThreadJoiner: # Bootstrapping LLVM takes forever with FreeBSD makefiles if self.use_bootstrapped_toolchain and not self.query_yes_no( - "You are about to do a clean FreeBSD build (without external toolchain). This will rebuild all of " - "LLVM and take a long time. Are you sure?", default_result=True): + "You are about to do a clean FreeBSD build (without external toolchain). This will rebuild all of " + "LLVM and take a long time. Are you sure?", + default_result=True, + ): return ThreadJoiner(None) return super().clean() @classmethod def setup_config_options(cls, install_directory_help=None, **kwargs) -> None: super().setup_config_options(bootstrap_toolchain=True) - cls.include_llvm = cls.add_bool_option("build-target-llvm", - help="Build LLVM for the target architecture. Note: this adds " - "significant time to the build") + cls.include_llvm = cls.add_bool_option( + "build-target-llvm", + help="Build LLVM for the target architecture. Note: this adds " "significant time to the build", + ) def add_cross_build_options(self) -> None: # Just try to build as much as possible (but using make.py) @@ -1516,20 +1749,27 @@ def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) cls.tinderbox = cls.add_bool_option("tinderbox", help="Use `make tinderbox` instead of `make universe`") cls.worlds_only = cls.add_bool_option("worlds-only", help="Only build worlds (skip building kernels)") - cls.kernels_only = cls.add_bool_option("kernels-only", help="Only build kernels (skip building worlds)", - default=ComputedDefaultValue( - function=lambda conf, proj: conf.skip_world, - as_string="true if --skip-world is set")) + cls.kernels_only = cls.add_bool_option( + "kernels-only", + help="Only build kernels (skip building worlds)", + default=ComputedDefaultValue( + function=lambda conf, proj: conf.skip_world, as_string="true if --skip-world is set" + ), + ) - cls.jflag_in_subjobs = cls.add_config_option("jflag-in-subjobs", - help="Number of jobs in each world/kernel build", - kind=int, default=ComputedDefaultValue(jflag_in_subjobs, - "default -j flag / 2")) + cls.jflag_in_subjobs = cls.add_config_option( + "jflag-in-subjobs", + help="Number of jobs in each world/kernel build", + kind=int, + default=ComputedDefaultValue(jflag_in_subjobs, "default -j flag / 2"), + ) - cls.jflag_for_universe = cls.add_config_option("jflag-for-universe", - help="Number of parallel world/kernel builds", - kind=int, default=ComputedDefaultValue(jflag_for_universe, - "default -j flag / 4")) + cls.jflag_for_universe = cls.add_config_option( + "jflag-for-universe", + help="Number of parallel world/kernel builds", + kind=int, + default=ComputedDefaultValue(jflag_for_universe, "default -j flag / 4"), + ) def compile(self, **kwargs) -> None: # The build seems to behave differently when -j1 is passed (it still complains about parallel make failures) @@ -1574,8 +1814,9 @@ class BuildCHERIBSD(BuildFreeBSD): default_directory_basename: str = "cheribsd" target: str = "cheribsd" can_build_with_system_clang: bool = False # We need CHERI LLVM for most architectures - repository: GitRepository = GitRepository("https://github.com/CTSRD-CHERI/cheribsd.git", - old_branches={"master": "main"}) + repository: GitRepository = GitRepository( + "https://github.com/CTSRD-CHERI/cheribsd.git", old_branches={"master": "main"} + ) _default_install_dir_fn: ComputedDefaultValue[Path] = _arch_suffixed_custom_install_dir("rootfs") supported_architectures = CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID is_sdk_target: bool = True @@ -1595,46 +1836,70 @@ class BuildCHERIBSD(BuildFreeBSD): def setup_config_options(cls, kernel_only_target=False, install_directory_help=None, **kwargs) -> None: if install_directory_help is None: install_directory_help = "Install directory for CheriBSD root file system" - super().setup_config_options(install_directory_help=install_directory_help, use_upstream_llvm=False, - kernel_only_target=kernel_only_target) + super().setup_config_options( + install_directory_help=install_directory_help, + use_upstream_llvm=False, + kernel_only_target=kernel_only_target, + ) fpga_targets = CompilationTargets.ALL_CHERIBSD_RISCV_TARGETS - cls.build_fpga_kernels = cls.add_bool_option("build-fpga-kernels", show_help=True, _allow_unknown_targets=True, - only_add_for_targets=fpga_targets, - help="Also build kernels for the FPGA.") - cls.build_fett_kernels = cls.add_bool_option("build-fett-kernels", show_help=False, _allow_unknown_targets=True, - only_add_for_targets=fpga_targets, - help="Also build kernels for FETT.") + cls.build_fpga_kernels = cls.add_bool_option( + "build-fpga-kernels", + show_help=True, + _allow_unknown_targets=True, + only_add_for_targets=fpga_targets, + help="Also build kernels for the FPGA.", + ) + cls.build_fett_kernels = cls.add_bool_option( + "build-fett-kernels", + show_help=False, + _allow_unknown_targets=True, + only_add_for_targets=fpga_targets, + help="Also build kernels for FETT.", + ) cls.mfs_root_image = cls.add_optional_path_option( - "mfs-root-image", help="Path to an MFS root image to be embedded in the kernel for booting") + "mfs-root-image", help="Path to an MFS root image to be embedded in the kernel for booting" + ) cls.default_kernel_abi = cls.add_config_option( - "default-kernel-abi", show_help=True, _allow_unknown_targets=True, + "default-kernel-abi", + show_help=True, + _allow_unknown_targets=True, only_add_for_targets=cls.purecap_kernel_targets, - kind=KernelABI, default=KernelABI.HYBRID, + kind=KernelABI, + default=KernelABI.HYBRID, enum_choices=[KernelABI.HYBRID, KernelABI.PURECAP], - help="Select default kernel to build") + help="Select default kernel to build", + ) # We also want to add this config option to the fake "cheribsd" target (to keep the config file manageable) cls.build_alternate_abi_kernels = cls.add_bool_option( - "build-alternate-abi-kernels", show_help=True, + "build-alternate-abi-kernels", + show_help=True, _allow_unknown_targets=True, only_add_for_targets=cls.purecap_kernel_targets, default=True, - help="Also build kernels with non-default ABI (purecap or hybrid)") + help="Also build kernels with non-default ABI (purecap or hybrid)", + ) - cls.build_bench_kernels = cls.add_bool_option("build-bench-kernels", show_help=True, - _allow_unknown_targets=True, - help="Also build benchmark kernels") + cls.build_bench_kernels = cls.add_bool_option( + "build-bench-kernels", show_help=True, _allow_unknown_targets=True, help="Also build benchmark kernels" + ) cls.build_nocaprevoke_kernels = cls.add_bool_option( - "build-nocaprevoke-kernels", show_help=True, _allow_unknown_targets=True, + "build-nocaprevoke-kernels", + show_help=True, + _allow_unknown_targets=True, only_add_for_targets=CompilationTargets.ALL_CHERIBSD_CHERI_TARGETS_WITH_HYBRID, - help="Build kernels without caprevoke support") + help="Build kernels without caprevoke support", + ) if kernel_only_target: return # The remaining options only affect the userspace build - cls.sysroot_only = cls.add_bool_option("sysroot-only", show_help=False, - help="Only build a sysroot instead of the full system. This will only " - "build the libraries and skip all binaries") + cls.sysroot_only = cls.add_bool_option( + "sysroot-only", + show_help=False, + help="Only build a sysroot instead of the full system. This will only " + "build the libraries and skip all binaries", + ) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -1655,16 +1920,22 @@ def get_default_kernel_abi(self) -> KernelABI: kernel_abi = KernelABI.NOCHERI return kernel_abi - def _get_config_variants(self, platforms: "set[ConfigPlatform]", kernel_abis: "list[KernelABI]", - combine_flags: "list[str]", **filter_kwargs) -> "list[CheriBSDConfig]": + def _get_config_variants( + self, + platforms: "set[ConfigPlatform]", + kernel_abis: "list[KernelABI]", + combine_flags: "list[str]", + **filter_kwargs, + ) -> "list[CheriBSDConfig]": flag_values = itertools.product([False, True], repeat=len(combine_flags)) combine_tuples = list(itertools.product(platforms, kernel_abis, flag_values)) configs = [] for platform, kernel_abi, flag_tuple in combine_tuples: combined_filter = {flag: v for flag, v in zip(combine_flags, flag_tuple)} filter_kwargs.update(combined_filter) - configs += CheriBSDConfigTable.get_configs(self.crosscompile_target, platform=platform, - kernel_abi=kernel_abi, **filter_kwargs) + configs += CheriBSDConfigTable.get_configs( + self.crosscompile_target, platform=platform, kernel_abi=kernel_abi, **filter_kwargs + ) return configs def _get_kernel_abis_to_build(self) -> "list[KernelABI]": @@ -1692,8 +1963,9 @@ def _get_all_kernel_configs(self) -> "list[CheriBSDConfig]": combinations.append("fett") configs = self._get_config_variants({platform}, kernel_abis, combinations) if self.build_fpga_kernels: - configs += self._get_config_variants(ConfigPlatform.fpga_platforms(), kernel_abis, - [*combinations, "mfsroot"]) + configs += self._get_config_variants( + ConfigPlatform.fpga_platforms(), kernel_abis, [*combinations, "mfsroot"] + ) return configs def default_kernel_config(self, platform: "Optional[ConfigPlatform]" = None, **filter_kwargs) -> str: @@ -1750,8 +2022,9 @@ def compile(self, **kwargs) -> None: def install(self, **kwargs) -> None: # When building we build MFS and - super().install(kernconfs=self.kernconf_list() + self.extra_kernels_with_mfs, - sysroot_only=self.sysroot_only, **kwargs) + super().install( + kernconfs=self.kernconf_list() + self.extra_kernels_with_mfs, sysroot_only=self.sysroot_only, **kwargs + ) class BuildCheriBsdMfsKernel(BuildCHERIBSD): @@ -1763,9 +2036,9 @@ class BuildCheriBsdMfsKernel(BuildCHERIBSD): *CompilationTargets.ALL_CHERIBSD_MORELLO_TARGETS, *CompilationTargets.ALL_CHERIBSD_RISCV_TARGETS, ) - default_build_dir: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=cheribsd_reuse_build_dir, - as_string=lambda cls: BuildCHERIBSD.project_build_dir_help()) + default_build_dir: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=cheribsd_reuse_build_dir, as_string=lambda cls: BuildCHERIBSD.project_build_dir_help() + ) # This exists specifically for this target has_default_buildkernel_kernel_config: bool = False # We want the CheriBSD config options as well, so that defaults (e.g. build-alternate-abi-kernels) are inherited. @@ -1780,6 +2053,7 @@ def __init__(self, *args, **kwargs) -> None: @cached_property def image(self) -> Path: from ..disk_image import BuildMfsRootCheriBSDDiskImage + return BuildMfsRootCheriBSDDiskImage.get_instance(self).disk_image_path @classmethod @@ -1806,16 +2080,23 @@ def _build_and_install_kernel_binaries(self, kernconfs: "list[str]", image: Path # Install to a temporary directory and then copy the kernel to OUTPUT_ROOT # Don't bother with modules for the MFS kernels: extra_make_args = dict(NO_MODULES="yes") - self._buildkernel(kernconfs=kernconfs, mfs_root_image=image, extra_make_args=extra_make_args, - ignore_skip_kernel=True) + self._buildkernel( + kernconfs=kernconfs, mfs_root_image=image, extra_make_args=extra_make_args, ignore_skip_kernel=True + ) with tempfile.TemporaryDirectory(prefix="cheribuild-" + self.target + "-") as td: temp_rootfs = Path(td) build_suffix = self.crosscompile_target.build_suffix(self.config, include_os=False) - self._installkernel(kernconfs=kernconfs, install_dir=temp_rootfs, extra_make_args=extra_make_args, - ignore_skip_kernel=True) - self._copykernel(kernconfs=kernconfs, rootfs_dir=temp_rootfs, dest_dir=self.config.cheribsd_image_root, - dest_kernel_suffix=build_suffix, dest_all_extra=True) + self._installkernel( + kernconfs=kernconfs, install_dir=temp_rootfs, extra_make_args=extra_make_args, ignore_skip_kernel=True + ) + self._copykernel( + kernconfs=kernconfs, + rootfs_dir=temp_rootfs, + dest_dir=self.config.cheribsd_image_root, + dest_kernel_suffix=build_suffix, + dest_all_extra=True, + ) def _get_all_kernel_configs(self) -> "list[CheriBSDConfig]": kernel_abis = self._get_kernel_abis_to_build() @@ -1827,8 +2108,9 @@ def _get_all_kernel_configs(self) -> "list[CheriBSDConfig]": combinations.append("nocaprevoke") configs = self._get_config_variants({platform}, kernel_abis, combinations, mfsroot=True) if self.build_fpga_kernels: - configs += self._get_config_variants(ConfigPlatform.fpga_platforms(), kernel_abis, - combinations, mfsroot=True) + configs += self._get_config_variants( + ConfigPlatform.fpga_platforms(), kernel_abis, combinations, mfsroot=True + ) return configs def default_kernel_config(self, platform: "Optional[ConfigPlatform]" = None, **filter_kwargs) -> str: @@ -1846,10 +2128,10 @@ def get_kernel_configs(self, platform: "Optional[ConfigPlatform]") -> "list[str] return [c.kernconf for c in filter_kernel_configs(configs, platform=platform, kernel_abi=None)] def get_kernel_install_path(self, kernconf: "Optional[str]" = None) -> Path: - """ Get the installed kernel path for an MFS kernel config that has been built. """ + """Get the installed kernel path for an MFS kernel config that has been built.""" path = self.config.cheribsd_image_root / ( - "kernel" + self.crosscompile_target.build_suffix(self.config, include_os=False) + - "." + kernconf) + "kernel" + self.crosscompile_target.build_suffix(self.config, include_os=False) + "." + kernconf + ) return path @@ -1946,8 +2228,9 @@ def process(self) -> None: self.objdir / "tmp/legacy/bin", self.objdir / "tmp/obj-tools/usr.sbin/makefs", ] - release_args.set_env(PATH=":".join(map(str, extra_path_entries)) + ":" + - release_args.env_vars.get("PATH", os.getenv("PATH"))) + release_args.set_env( + PATH=":".join(map(str, extra_path_entries)) + ":" + release_args.env_vars.get("PATH", os.getenv("PATH")) + ) # DESTDIR for install target is where to install the media, as you'd # expect, unlike the release target where it leaks into installworld @@ -1965,9 +2248,9 @@ class BuildFreeBSDRelease(BuildFreeBSDReleaseMixin, BuildFreeBSD): dependencies: "tuple[str, ...]" = ("freebsd",) repository: ReuseOtherProjectRepository = ReuseOtherProjectRepository(source_project=BuildFreeBSD) _always_add_suffixed_targets: bool = True - default_build_dir: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=freebsd_reuse_build_dir, - as_string=lambda cls: BuildFreeBSD.project_build_dir_help()) + default_build_dir: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=freebsd_reuse_build_dir, as_string=lambda cls: BuildFreeBSD.project_build_dir_help() + ) _default_install_dir_fn: ComputedDefaultValue[Path] = _arch_suffixed_custom_install_dir("freebsd-release") # We want the FreeBSD config options as well so the release installworld, # distributeworld etc. calls match what was built. @@ -1979,9 +2262,9 @@ class BuildCheriBSDRelease(BuildFreeBSDReleaseMixin, BuildCHERIBSD): dependencies: "tuple[str, ...]" = ("cheribsd",) repository: ReuseOtherProjectRepository = ReuseOtherProjectRepository(source_project=BuildCHERIBSD) _always_add_suffixed_targets: bool = True - default_build_dir: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=cheribsd_reuse_build_dir, - as_string=lambda cls: BuildCHERIBSD.project_build_dir_help()) + default_build_dir: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=cheribsd_reuse_build_dir, as_string=lambda cls: BuildCHERIBSD.project_build_dir_help() + ) _default_install_dir_fn: ComputedDefaultValue[Path] = _arch_suffixed_custom_install_dir("cheribsd-release") # We want the CheriBSD config options as well so the release installworld, # distributeworld etc. calls match what was built. @@ -2014,25 +2297,35 @@ def __init__(self, *args, **kwargs) -> None: def check_system_dependencies(self) -> None: super().check_system_dependencies() self.check_required_system_tool("bsdtar", cheribuild_target="bsdtar", apt="libarchive-tools") - if not OSInfo.IS_FREEBSD and not self.remote_path and not self.rootfs_source_class.get_instance( - self).crossbuild: + if ( + not OSInfo.IS_FREEBSD + and not self.remote_path + and not self.rootfs_source_class.get_instance(self).crossbuild + ): config_option = "'--" + self.get_config_option_name("remote_path") + "'" - self.fatal("Path to the remote SDK is not set, option", config_option, - "must be set to a path that scp understands (e.g. vica:~foo/cheri/output/sdk)") + self.fatal( + "Path to the remote SDK is not set, option", + config_option, + "must be set to a path that scp understands (e.g. vica:~foo/cheri/output/sdk)", + ) if not self.config.pretend: sys.exit("Cannot continue...") @classmethod def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) - cls.copy_remote_sysroot = cls.add_bool_option("copy-remote-sysroot", - help="Copy sysroot from remote server instead of from local " - "machine") - cls.remote_path = cls.add_config_option("remote-sdk-path", show_help=True, metavar="PATH", - help="The path to the CHERI SDK on the remote FreeBSD machine (e.g. " - "vica:~foo/cheri/output/sdk)") + cls.copy_remote_sysroot = cls.add_bool_option( + "copy-remote-sysroot", help="Copy sysroot from remote server instead of from local " "machine" + ) + cls.remote_path = cls.add_config_option( + "remote-sdk-path", + show_help=True, + metavar="PATH", + help="The path to the CHERI SDK on the remote FreeBSD machine (e.g. " "vica:~foo/cheri/output/sdk)", + ) cls.install_dir_override = cls.add_optional_path_option( - "install-directory", help="Override for the sysroot install directory") + "install-directory", help="Override for the sysroot install directory" + ) @property def cross_sysroot_path(self) -> Path: @@ -2045,7 +2338,8 @@ def copy_sysroot_from_remote_machine(self) -> None: self.info("Copying sysroot from remote system.") if not self.remote_path: self.fatal( - "Missing remote SDK path: Please set --cheribsd-sysroot/remote-sdk-path (or --freebsd/crossbuild)") + "Missing remote SDK path: Please set --cheribsd-sysroot/remote-sdk-path (or --freebsd/crossbuild)" + ) if self.config.pretend: self.remote_path = "someuser@somehose:this/path/does/not/exist" # noinspection PyAttributeOutsideInit @@ -2073,18 +2367,32 @@ def create_sysroot(self) -> None: bsdtar_path = shutil.which(str(self.bsdtar_cmd)) if not bsdtar_path: bsdtar_path = str(self.bsdtar_cmd) - tar_cmd = [bsdtar_path, "cf", "-", "--include=./lib/", "--include=./usr/include/", - "--include=./usr/lib/", "--include=./usr/libdata/", - "--include=./usr/lib32", "--include=./usr/lib64", - "--include=./usr/lib64c", "--include=./usr/lib64cb", - "--include=./usr/lib128", "--include=./usr/lib128g", - # only pack those files that are mentioned in METALOG - "@METALOG.world"] + tar_cmd = [ + bsdtar_path, + "cf", + "-", + "--include=./lib/", + "--include=./usr/include/", + "--include=./usr/lib/", + "--include=./usr/libdata/", + "--include=./usr/lib32", + "--include=./usr/lib64", + "--include=./usr/lib64c", + "--include=./usr/lib64cb", + "--include=./usr/lib128", + "--include=./usr/lib128g", + # only pack those files that are mentioned in METALOG + "@METALOG.world", + ] rootfs_target = self.rootfs_source_class.get_instance(self) rootfs_dir = rootfs_target.real_install_root_dir if not (rootfs_dir / "lib/libc.so.7").is_file(): - self.fatal("Sysroot source directory", rootfs_dir, "does not contain libc.so.7", - fixit_hint="Run `cheribuild.py " + rootfs_target.target + "` first") + self.fatal( + "Sysroot source directory", + rootfs_dir, + "does not contain libc.so.7", + fixit_hint="Run `cheribuild.py " + rootfs_target.target + "` first", + ) print_command(tar_cmd, cwd=rootfs_dir, config=self.config) if not self.config.pretend: tar_cwd = str(rootfs_dir) @@ -2095,8 +2403,9 @@ def create_sysroot(self) -> None: # create an archive to make it easier to copy the sysroot to another machine self.delete_file(self.sysroot_archive, print_verbose_only=True) - self.run_cmd("tar", "-caf", self.sysroot_archive, self.cross_sysroot_path.name, - cwd=self.cross_sysroot_path.parent) + self.run_cmd( + "tar", "-caf", self.sysroot_archive, self.cross_sysroot_path.name, cwd=self.cross_sysroot_path.parent + ) self.info("Successfully populated sysroot") def process(self) -> None: diff --git a/pycheribuild/projects/cross/cheritest.py b/pycheribuild/projects/cross/cheritest.py index 13dde3dc7..d20b277d0 100644 --- a/pycheribuild/projects/cross/cheritest.py +++ b/pycheribuild/projects/cross/cheritest.py @@ -43,10 +43,12 @@ class _BuildCheriMipsTestBase(Project): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.single_test = cls.add_config_option("single-test", help="Run a single test instead of all of them") - cls.run_tests_with_build = cls.add_bool_option("run-tests-with-build", - help="Run tests as part of the --build step (option is useful " - "for jenkins)", - show_help=False, default=True) + cls.run_tests_with_build = cls.add_bool_option( + "run-tests-with-build", + help="Run tests as part of the --build step (option is useful " "for jenkins)", + show_help=False, + default=True, + ) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/compiler_rt.py b/pycheribuild/projects/cross/compiler_rt.py index 4efac8370..757235c23 100644 --- a/pycheribuild/projects/cross/compiler_rt.py +++ b/pycheribuild/projects/cross/compiler_rt.py @@ -63,8 +63,9 @@ def setup(self): if is_jenkins_build(): llvm_tools_bindir = self.llvm_project.get_native_install_path(self.config) / "bin" else: - llvm_tools_bindir = self.llvm_project.get_build_dir( - self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin" + llvm_tools_bindir = ( + self.llvm_project.get_build_dir(self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin" + ) self.add_cmake_options( LLVM_CONFIG_PATH=llvm_tools_bindir / "llvm-config", LLVM_EXTERNAL_LIT=llvm_tools_bindir / "llvm-lit", @@ -80,7 +81,7 @@ def setup(self): COMPILER_RT_DEFAULT_TARGET_ONLY=not self.compiling_for_host(), # Per-target runtime directories don't add the purecap suffix so can't be used right now. LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=False, - ) + ) if self.should_include_debug_info: self.add_cmake_options(COMPILER_RT_DEBUG=True) @@ -94,7 +95,8 @@ def install(self, **kwargs): # HACK: we don't really need the ubsan runtime but the toolchain pulls it in automatically # TODO: is there an easier way to create an empty archive? ubsan_runtime_path = self.install_dir / ( - "lib/freebsd/libclang_rt.ubsan_standalone-mips64c" + self.config.mips_cheri_bits_str + ".a") + "lib/freebsd/libclang_rt.ubsan_standalone-mips64c" + self.config.mips_cheri_bits_str + ".a" + ) if not ubsan_runtime_path.exists(): self.warning("Did not install ubsan runtime", ubsan_runtime_path) if self.target_info.is_rtems(): @@ -103,8 +105,9 @@ def install(self, **kwargs): self.warning("Did not install compiler runtime", rt_runtime_path.exists) else: print(self.target_info.sysroot_dir) - self.create_symlink(rt_runtime_path, - self.target_info.sysroot_dir / "lib/libclang_rt.builtins-riscv64.a") + self.create_symlink( + rt_runtime_path, self.target_info.sysroot_dir / "lib/libclang_rt.builtins-riscv64.a" + ) def run_tests(self): self.run_make("check-compiler-rt") @@ -127,10 +130,12 @@ class BuildCompilerRtBuiltins(CrossCompileCMakeProject): is_sdk_target = True root_cmakelists_subdirectory = Path("lib/builtins") needs_sysroot = False # We don't need a complete sysroot - supported_architectures = (CompilationTargets.ALL_SUPPORTED_BAREMETAL_TARGETS + - CompilationTargets.ALL_SUPPORTED_RTEMS_TARGETS + - CompilationTargets.ALL_FREESTANDING_TARGETS + - CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_SUPPORTED_BAREMETAL_TARGETS + + CompilationTargets.ALL_SUPPORTED_RTEMS_TARGETS + + CompilationTargets.ALL_FREESTANDING_TARGETS + + CompilationTargets.ALL_NATIVE + ) # Note: needs to be @classproperty since it is called before __init__ @classproperty @@ -182,8 +187,9 @@ def install(self, **kwargs): self.move_file(self.install_dir / "lib/rtems5" / libname, self.install_dir / "lib" / libname) elif self.target_info.is_baremetal(): self.move_file(self.install_dir / "lib/baremetal" / libname, self.real_install_root_dir / "lib" / libname) - self.create_symlink(self.install_dir / "lib" / libname, self.install_dir / "lib/libgcc.a", - print_verbose_only=False) + self.create_symlink( + self.install_dir / "lib" / libname, self.install_dir / "lib/libgcc.a", print_verbose_only=False + ) class BuildUpstreamCompilerRtBuiltins(BuildCompilerRtBuiltins): diff --git a/pycheribuild/projects/cross/crosscompileproject.py b/pycheribuild/projects/cross/crosscompileproject.py index f1c17df8e..3fad44fc1 100644 --- a/pycheribuild/projects/cross/crosscompileproject.py +++ b/pycheribuild/projects/cross/crosscompileproject.py @@ -70,7 +70,8 @@ "Linkage", "MakeCommandKind", "SubversionRepository", - "commandline_to_str"] + "commandline_to_str", +] if typing.TYPE_CHECKING: @@ -180,8 +181,13 @@ def configure(self, **kwargs): env = {k: v for k, v in self.configure_environment.items() if v} self.configure_environment.clear() self.configure_environment.update(env) - self.print(coloured(AnsiColour.yellow, "Cross configure environment:\n\t", - "\n\t".join(k + "=" + str(v) for k, v in self.configure_environment.items()))) + self.print( + coloured( + AnsiColour.yellow, + "Cross configure environment:\n\t", + "\n\t".join(k + "=" + str(v) for k, v in self.configure_environment.items()), + ) + ) super().configure(**kwargs) def process(self): diff --git a/pycheribuild/projects/cross/current_directory.py b/pycheribuild/projects/cross/current_directory.py index 6f64b2f92..075cf3ac5 100644 --- a/pycheribuild/projects/cross/current_directory.py +++ b/pycheribuild/projects/cross/current_directory.py @@ -48,8 +48,7 @@ def _cwd_directory_basename(_, _1): class CurrentDirectoryMixin: do_not_add_to_targets = True - default_directory_basename = ComputedDefaultValue(function=_cwd_directory_basename, - as_string="$SOURCE_DIR_NAME") + default_directory_basename = ComputedDefaultValue(function=_cwd_directory_basename, as_string="$SOURCE_DIR_NAME") inherit_default_directory_basename = True repository = ExternallyManagedSourceRepository() source_dir = ComputedDefaultValue(function=_cwd_source_dir, as_string="$CWD") @@ -83,11 +82,11 @@ class BuildCurrent_Directory(CurrentDirectoryMixin, CrossCompileSimpleProject): @classmethod def dependencies(cls, _) -> "tuple[str, ...]": classes = [ - BuildCurrent_Directory_Autotools, - BuildCurrent_Directory_CMake, - BuildCurrent_Directory_Makefile, - BuildCurrent_Directory_Meson, - ] + BuildCurrent_Directory_Autotools, + BuildCurrent_Directory_CMake, + BuildCurrent_Directory_Makefile, + BuildCurrent_Directory_Meson, + ] for c in classes: for f in c.autodetect_files: if (_cwd_path / f).is_file(): diff --git a/pycheribuild/projects/cross/dbus.py b/pycheribuild/projects/cross/dbus.py index f889cd830..a3628517a 100644 --- a/pycheribuild/projects/cross/dbus.py +++ b/pycheribuild/projects/cross/dbus.py @@ -28,8 +28,10 @@ class BuildDBus(CrossCompileCMakeProject): target = "dbus" supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE - repository = GitRepository("https://gitlab.freedesktop.org/dbus/dbus.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/dbus.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/dbus/dbus.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/dbus.git"], + ) dependencies = ("libexpat",) ctest_script_extra_args = ["--test-timeout", str(120 * 60)] # Tests can take a long time to run @@ -52,11 +54,17 @@ def setup(self): def install(self, **kwargs): super().install() if not self.compiling_for_host() and self.target_info.is_freebsd(): - self.write_file(self.rootfs_dir / "etc/rc.conf.d/dbus", contents="dbus_enable=\"YES\"\n", - overwrite=True, print_verbose_only=False) + self.write_file( + self.rootfs_dir / "etc/rc.conf.d/dbus", + contents='dbus_enable="YES"\n', + overwrite=True, + print_verbose_only=False, + ) # Slightly modified version of https://cgit.freebsd.org/ports/plain/devel/dbus/files/dbus.in # to add the necessary users on-demand and chmod/chown the rsync'd files - self.write_file(self.rootfs_dir / self.target_info.localbase / "etc/rc.d/dbus", contents=f"""#!/bin/sh + self.write_file( + self.rootfs_dir / self.target_info.localbase / "etc/rc.d/dbus", + contents=f"""#!/bin/sh # PROVIDE: dbus # REQUIRE: DAEMON ldconfig @@ -103,4 +111,7 @@ def install(self, **kwargs): load_rc_config ${{name}} run_rc_command "$1" -""", overwrite=True, mode=0o755) +""", + overwrite=True, + mode=0o755, + ) diff --git a/pycheribuild/projects/cross/device_model.py b/pycheribuild/projects/cross/device_model.py index c5fc73b8a..146d72c6a 100644 --- a/pycheribuild/projects/cross/device_model.py +++ b/pycheribuild/projects/cross/device_model.py @@ -47,13 +47,12 @@ def compile(self, **kwargs): self.run_make("purecap") def install(self, **kwargs): - self.install_file(self.build_dir / "obj/device-model-riscv.bin", - self.real_install_root_dir / "device-model-riscv.bin") + self.install_file( + self.build_dir / "obj/device-model-riscv.bin", self.real_install_root_dir / "device-model-riscv.bin" + ) def setup(self): super().setup() cc = self.config.cheri_sdk_bindir / "clang" objcopy = self.config.cheri_sdk_bindir / "objcopy" - self.make_args.env_vars = {"CC": str(cc), - "AS": str(cc), - "OBJCOPY": str(objcopy)} + self.make_args.env_vars = {"CC": str(cc), "AS": str(cc), "OBJCOPY": str(objcopy)} diff --git a/pycheribuild/projects/cross/dlmalloc.py b/pycheribuild/projects/cross/dlmalloc.py index 59a68dbad..44d896483 100644 --- a/pycheribuild/projects/cross/dlmalloc.py +++ b/pycheribuild/projects/cross/dlmalloc.py @@ -47,19 +47,19 @@ def setup_config_options(cls, **kwargs): cls.cheri_set_bounds = cls.add_bool_option("cheri-bounds", default=True, help="Set bounds on allocations") - cls.qmabs = cls.add_config_option("qmabs", kind=int, - help="Quarantine memory absolute threshold") + cls.qmabs = cls.add_config_option("qmabs", kind=int, help="Quarantine memory absolute threshold") - cls.qmratio = cls.add_config_option("qmratio", kind=float, - help="Quarantine memory ratio threshold") + cls.qmratio = cls.add_config_option("qmratio", kind=float, help="Quarantine memory ratio threshold") - cls.qmmin = cls.add_config_option("qmmin", kind=int, - help="Minimum amount quarantined to trigger a revocation based on ratio") + cls.qmmin = cls.add_config_option( + "qmmin", kind=int, help="Minimum amount quarantined to trigger a revocation based on ratio" + ) cls.revoke = cls.add_bool_option("revoke", help="Revoke quarantine before reusing") - cls.consolidate_on_free = cls.add_bool_option("consolidate", default=True, - help="Consolidate memory when quarantining") + cls.consolidate_on_free = cls.add_bool_option( + "consolidate", default=True, help="Consolidate memory when quarantining" + ) cls.zero_memory = cls.add_bool_option("zero-memory", help="Zero allocated memory") @@ -67,11 +67,12 @@ def setup_config_options(cls, **kwargs): cls.unmap_support = cls.add_bool_option("unmap-support", default=True, help="support for unmapping") - cls.unmap_threshold = cls.add_config_option("unmap-threshold", kind=int, - help="Threshold (in pages) at which interior pages of quanantined " - "chunks are unmapped") - cls.quar_unsafe = cls.add_bool_option("unsafe-quarantine", - help="Don't isolate quarantine structures") + cls.unmap_threshold = cls.add_config_option( + "unmap-threshold", + kind=int, + help="Threshold (in pages) at which interior pages of quanantined " "chunks are unmapped", + ) + cls.quar_unsafe = cls.add_bool_option("unsafe-quarantine", help="Don't isolate quarantine structures") def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/doom.py b/pycheribuild/projects/cross/doom.py index 8c36f418a..7cca81fc9 100644 --- a/pycheribuild/projects/cross/doom.py +++ b/pycheribuild/projects/cross/doom.py @@ -32,9 +32,12 @@ class BuildChocolateDoom(CrossCompileAutotoolsProject): target = "chocolate-doom" - repository = GitRepository("https://github.com/chocolate-doom/chocolate-doom.git", - old_urls=[b"https://github.com/jrtc27/chocolate-doom.git"], - default_branch="master", force_branch=True) + repository = GitRepository( + "https://github.com/chocolate-doom/chocolate-doom.git", + old_urls=[b"https://github.com/jrtc27/chocolate-doom.git"], + default_branch="master", + force_branch=True, + ) dependencies = ("sdl", "sdl-mixer", "sdl-net", "libpng") _can_use_autogen_sh = False # Can't use autogen.sh since it will run configure in the wrong dir @@ -55,14 +58,14 @@ class BuildFreedoom(CrossCompileProject): version = "0.12.1" url_prefix: str = f"https://github.com/freedoom/freedoom/releases/download/v{version}/" packages: "dict[str, list[str]]" = { - 'freedoom': ['freedoom1', 'freedoom2'], - 'freedm': ['freedm'], + "freedoom": ["freedoom1", "freedoom2"], + "freedm": ["freedm"], } def compile(self, **kwargs): for pkgname, wads in self.packages.items(): filename = f"{pkgname}-{self.version}.zip" - wadfiles = ['*/' + wad + ".wad" for wad in wads] + wadfiles = ["*/" + wad + ".wad" for wad in wads] if not (self.build_dir / filename).is_file(): self.download_file(self.build_dir / filename, self.url_prefix + filename) self.run_cmd("unzip", "-jo", filename, *wadfiles, cwd=self.build_dir) @@ -73,6 +76,11 @@ def install(self, **kwargs): wadfile = wad + ".wad" wadpath = Path("share/doom") / wadfile self.install_file(self.build_dir / wadfile, self.install_dir / wadpath) - self.write_file(self.install_dir / "bin" / wad, overwrite=True, mode=0o755, - contents="#!/bin/sh\nexec {0}/bin/chocolate-doom -iwad {0}/{1} \"$@\"\n".format( - self.install_prefix, wadpath)) + self.write_file( + self.install_dir / "bin" / wad, + overwrite=True, + mode=0o755, + contents='#!/bin/sh\nexec {0}/bin/chocolate-doom -iwad {0}/{1} "$@"\n'.format( + self.install_prefix, wadpath + ), + ) diff --git a/pycheribuild/projects/cross/drm_kmod.py b/pycheribuild/projects/cross/drm_kmod.py index e41246878..d589016c7 100644 --- a/pycheribuild/projects/cross/drm_kmod.py +++ b/pycheribuild/projects/cross/drm_kmod.py @@ -29,7 +29,8 @@ def setup(self) -> None: MODULES_OVERRIDE="linuxkpi", ) self.kernel_make_args = self.freebsd_project.kernel_make_args_for_config( - self.freebsd_project.kernel_config, extra_make_args, + self.freebsd_project.kernel_config, + extra_make_args, ) assert self.kernel_make_args.kind == MakeCommandKind.BsdMake @@ -38,7 +39,11 @@ def clean(self, **kwargs) -> None: if self.use_buildenv: self.info("Cleaning drm-kmod modules for configs:", self.freebsd_project.kernel_config) self.freebsd_project.build_and_install_subdir( - self.kernel_make_args, str(self.source_dir), skip_build=True, skip_clean=False, skip_install=True, + self.kernel_make_args, + str(self.source_dir), + skip_build=True, + skip_clean=False, + skip_install=True, ) else: self.info("Clean not supported yet") @@ -48,11 +53,18 @@ def compile(self, **kwargs) -> None: self.info("Building drm-kmod modules for configs:", self.freebsd_project.kernel_config) if self.use_buildenv: self.freebsd_project.build_and_install_subdir( - self.kernel_make_args, str(self.source_dir), skip_build=False, skip_clean=True, skip_install=True, + self.kernel_make_args, + str(self.source_dir), + skip_build=False, + skip_clean=True, + skip_install=True, ) else: self.run_make( - "buildkernel", options=self.kernel_make_args, cwd=self.freebsd_project.source_dir, parallel=True, + "buildkernel", + options=self.kernel_make_args, + cwd=self.freebsd_project.source_dir, + parallel=True, ) def install(self, **kwargs) -> None: @@ -65,9 +77,16 @@ def install(self, **kwargs) -> None: make_args.set_env(METALOG=self.real_install_root_dir / "METALOG.drm-kmod") if self.use_buildenv: self.freebsd_project.build_and_install_subdir( - make_args, str(self.source_dir), skip_build=True, skip_clean=True, skip_install=False, + make_args, + str(self.source_dir), + skip_build=True, + skip_clean=True, + skip_install=False, ) else: self.run_make_install( - target="installkernel", options=make_args, cwd=self.freebsd_project.source_dir, parallel=False, + target="installkernel", + options=make_args, + cwd=self.freebsd_project.source_dir, + parallel=False, ) diff --git a/pycheribuild/projects/cross/ffmpeg.py b/pycheribuild/projects/cross/ffmpeg.py index 0110520a5..c90efd94c 100644 --- a/pycheribuild/projects/cross/ffmpeg.py +++ b/pycheribuild/projects/cross/ffmpeg.py @@ -28,9 +28,11 @@ class BuildFfmpeg(CrossCompileAutotoolsProject): target = "ffmpeg" - repository = GitRepository("https://github.com/FFmpeg/FFmpeg.git", - temporary_url_override="https://github.com/arichardson/FFmpeg.git", - url_override_reason="Needs --disable-neon workarounds") + repository = GitRepository( + "https://github.com/FFmpeg/FFmpeg.git", + temporary_url_override="https://github.com/arichardson/FFmpeg.git", + url_override_reason="Needs --disable-neon workarounds", + ) ctest_script_extra_args = ["--test-timeout", str(180 * 60)] # Tests take a long time to run add_host_target_build_config_options = False # doesn't understand --host _configure_supports_variables_on_cmdline = False # not really an autotools project @@ -39,31 +41,35 @@ class BuildFfmpeg(CrossCompileAutotoolsProject): def setup(self): super().setup() cflags = self.default_compiler_flags - self.configure_args.extend([ - f"--ar={self.target_info.ar}", - f"--as={self.CC}", - f"--cc={self.CC}", - f"--cxx={self.CXX}", - f"--ld={self.CC}", - f"--nm={self.target_info.nm}", - f"--ranlib={self.target_info.ranlib}", - f"--strip={self.target_info.strip_tool}", - f"--extra-cflags={self.commandline_to_str(cflags + self.CFLAGS)}", - f"--extra-cxxflags={self.commandline_to_str(cflags + self.CXXFLAGS)}", - f"--extra-ldflags={self.commandline_to_str(self.default_ldflags + self.LDFLAGS)}", - "--enable-pic", - "--disable-doc", - "--disable-ffplay", - "--disable-ffprobe", - ]) + self.configure_args.extend( + [ + f"--ar={self.target_info.ar}", + f"--as={self.CC}", + f"--cc={self.CC}", + f"--cxx={self.CXX}", + f"--ld={self.CC}", + f"--nm={self.target_info.nm}", + f"--ranlib={self.target_info.ranlib}", + f"--strip={self.target_info.strip_tool}", + f"--extra-cflags={self.commandline_to_str(cflags + self.CFLAGS)}", + f"--extra-cxxflags={self.commandline_to_str(cflags + self.CXXFLAGS)}", + f"--extra-ldflags={self.commandline_to_str(self.default_ldflags + self.LDFLAGS)}", + "--enable-pic", + "--disable-doc", + "--disable-ffplay", + "--disable-ffprobe", + ] + ) if not self.compiling_for_host(): - self.configure_args.extend([ - "--enable-cross-compile", - f"--host-cc={self.host_CC}", - f"--host-ld={self.host_CC}", - f"--arch={self.crosscompile_target.cpu_architecture.value}", - f"--target-os={self.target_info.cmake_system_name.lower()}", - ]) + self.configure_args.extend( + [ + "--enable-cross-compile", + f"--host-cc={self.host_CC}", + f"--host-ld={self.host_CC}", + f"--arch={self.crosscompile_target.cpu_architecture.value}", + f"--target-os={self.target_info.cmake_system_name.lower()}", + ] + ) if self.compiling_for_cheri(): self.configure_args.append("--disable-neon") # NEON asm needs some adjustments diff --git a/pycheribuild/projects/cross/freertos.py b/pycheribuild/projects/cross/freertos.py index 951d7e88e..7c88d115d 100644 --- a/pycheribuild/projects/cross/freertos.py +++ b/pycheribuild/projects/cross/freertos.py @@ -40,22 +40,25 @@ class BuildFreeRTOS(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/FreeRTOS-mirror", - force_branch=True, default_branch="cheri") + repository = GitRepository( + "https://github.com/CTSRD-CHERI/FreeRTOS-mirror", force_branch=True, default_branch="cheri" + ) target = "freertos" dependencies = ("newlib", "compiler-rt-builtins") is_sdk_target = True needs_sysroot = False # We don't need a complete sysroot supported_architectures = ( CompilationTargets.BAREMETAL_NEWLIB_RISCV64_PURECAP, - CompilationTargets.BAREMETAL_NEWLIB_RISCV64) + CompilationTargets.BAREMETAL_NEWLIB_RISCV64, + ) default_install_dir = DefaultInstallDir.ROOTFS_LOCALBASE # FreeRTOS Demos to build supported_freertos_demos = [ # Generic/simple (CHERI-)RISC-V Demo that runs main_blinky on simulators # and simple SoCs - "RISC-V-Generic"] + "RISC-V-Generic" + ] # Map Demos and the FreeRTOS apps we support building/running for supported_demo_apps = {"RISC-V-Generic": ["main_blinky"]} @@ -68,9 +71,11 @@ class BuildFreeRTOS(CrossCompileAutotoolsProject): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.default_demo_app = "qemu_virt-" + self.target_info.get_riscv_arch_string(self.crosscompile_target, - softfloat=True) + \ - self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + self.default_demo_app = ( + "qemu_virt-" + + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + + self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) def setup(self): super().setup() @@ -100,26 +105,47 @@ def setup(self): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.demo: str = typing.cast(str, cls.add_config_option( - "demo", metavar="DEMO", show_help=True, default=cls.default_demo, - help="The FreeRTOS Demo build.")) - - cls.demo_app: str = typing.cast(str, cls.add_config_option( - "prog", metavar="PROG", show_help=True, default=cls.default_demo_app, - help="The FreeRTOS program to build.")) - - cls.demo_bsp: str = typing.cast(str, cls.add_config_option( - "bsp", metavar="BSP", show_help=True, - default=ComputedDefaultValue(function=lambda _, p: p.default_demo_bsp(), - as_string="target-dependent default"), - help="The FreeRTOS BSP to build. This is only valid for the " - "paramterized RISC-V-Generic. The BSP option chooses " - "platform, RISC-V arch and RISC-V abi in the " - "$platform-$arch-$abi format. See RISC-V-Generic/README for more details")) + cls.demo: str = typing.cast( + str, + cls.add_config_option( + "demo", metavar="DEMO", show_help=True, default=cls.default_demo, help="The FreeRTOS Demo build." + ), + ) + + cls.demo_app: str = typing.cast( + str, + cls.add_config_option( + "prog", + metavar="PROG", + show_help=True, + default=cls.default_demo_app, + help="The FreeRTOS program to build.", + ), + ) + + cls.demo_bsp: str = typing.cast( + str, + cls.add_config_option( + "bsp", + metavar="BSP", + show_help=True, + default=ComputedDefaultValue( + function=lambda _, p: p.default_demo_bsp(), as_string="target-dependent default" + ), + help="The FreeRTOS BSP to build. This is only valid for the " + "paramterized RISC-V-Generic. The BSP option chooses " + "platform, RISC-V arch and RISC-V abi in the " + "$platform-$arch-$abi format. See RISC-V-Generic/README for more details", + ), + ) def default_demo_bsp(self): - return "qemu_virt-" + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + "-" + \ - self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + return ( + "qemu_virt-" + + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + + "-" + + self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) def compile(self, **kwargs): self.make_args.set(BSP=self.demo_bsp) @@ -129,8 +155,10 @@ def compile(self, **kwargs): self.make_args.set(PROG=self.demo_app) self.run_make("clean", cwd=self.source_dir / str("FreeRTOS/Demo/" + self.demo)) self.run_make(cwd=self.source_dir / str("FreeRTOS/Demo/" + self.demo)) - self.move_file(self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo_app + ".elf"), - self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo + self.demo_app + ".elf")) + self.move_file( + self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo_app + ".elf"), + self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo + self.demo_app + ".elf"), + ) def configure(self): pass @@ -141,7 +169,8 @@ def needs_configure(self): def install(self, **kwargs): self.install_file( self.source_dir / str("FreeRTOS/Demo/" + self.demo + "/" + self.demo + self.demo_app + ".elf"), - self.real_install_root_dir / str("FreeRTOS/Demo/" + self.demo + "_" + self.demo_app + ".elf")) + self.real_install_root_dir / str("FreeRTOS/Demo/" + self.demo + "_" + self.demo_app + ".elf"), + ) def process(self): if self.demo not in self.supported_freertos_demos: @@ -150,17 +179,21 @@ def process(self): if self.demo_app not in self.supported_demo_apps[self.demo]: self.fatal(self.demo + " Demo doesn't support/have " + self.demo_app) - with self.set_env(PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), - # Add compiler-rt location to the search path - LDFLAGS="-L" + str(BuildCompilerRtBuiltins.get_install_dir(self) / "lib")): + with self.set_env( + PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), + # Add compiler-rt location to the search path + LDFLAGS="-L" + str(BuildCompilerRtBuiltins.get_install_dir(self) / "lib"), + ): super().process() class LaunchFreeRTOSQEMU(LaunchQEMUBase): target = "run-freertos" dependencies = ("freertos",) - supported_architectures = (CompilationTargets.BAREMETAL_NEWLIB_RISCV64_PURECAP, - CompilationTargets.BAREMETAL_NEWLIB_RISCV64) + supported_architectures = ( + CompilationTargets.BAREMETAL_NEWLIB_RISCV64_PURECAP, + CompilationTargets.BAREMETAL_NEWLIB_RISCV64, + ) forward_ssh_port = False qemu_user_networking = False _enable_smbfs_support = False @@ -179,26 +212,47 @@ def setup(self): def setup_config_options(cls, **kwargs): super().setup_config_options(defaultSshPort=None, **kwargs) - cls.demo: str = typing.cast(str, cls.add_config_option( - "demo", metavar="DEMO", show_help=True, default=cls.default_demo, - help="The FreeRTOS Demo to run.")) - - cls.demo_app: str = typing.cast(str, cls.add_config_option( - "prog", metavar="PROG", show_help=True, default=cls.default_demo_app, - help="The FreeRTOS program to run.")) - - cls.demo_bsp = typing.cast(str, cls.add_config_option( - "bsp", metavar="BSP", show_help=True, - default=ComputedDefaultValue(function=lambda _, p: p.default_demo_bsp(), - as_string="target-dependent default"), - help="The FreeRTOS BSP to run. This is only valid for the " - "paramterized RISC-V-Generic. The BSP option chooses " - "platform, RISC-V arch and RISC-V abi in the " - "$platform-$arch-$abi format. See RISC-V-Generic/README for more details")) + cls.demo: str = typing.cast( + str, + cls.add_config_option( + "demo", metavar="DEMO", show_help=True, default=cls.default_demo, help="The FreeRTOS Demo to run." + ), + ) + + cls.demo_app: str = typing.cast( + str, + cls.add_config_option( + "prog", + metavar="PROG", + show_help=True, + default=cls.default_demo_app, + help="The FreeRTOS program to run.", + ), + ) + + cls.demo_bsp = typing.cast( + str, + cls.add_config_option( + "bsp", + metavar="BSP", + show_help=True, + default=ComputedDefaultValue( + function=lambda _, p: p.default_demo_bsp(), as_string="target-dependent default" + ), + help="The FreeRTOS BSP to run. This is only valid for the " + "paramterized RISC-V-Generic. The BSP option chooses " + "platform, RISC-V arch and RISC-V abi in the " + "$platform-$arch-$abi format. See RISC-V-Generic/README for more details", + ), + ) def default_demo_bsp(self): - return "qemu_virt-" + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + "-" + \ - self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + return ( + "qemu_virt-" + + self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True) + + "-" + + self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) def get_riscv_bios_args(self) -> "list[str]": # Use -bios none to ensure the FreeRTOS demo application runs in machine mode. diff --git a/pycheribuild/projects/cross/freetype.py b/pycheribuild/projects/cross/freetype.py index 0341c0a94..1d2208e63 100644 --- a/pycheribuild/projects/cross/freetype.py +++ b/pycheribuild/projects/cross/freetype.py @@ -33,8 +33,9 @@ class BuildFreeType2(CrossCompileMesonProject): target = "freetype2" supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE - repository = GitRepository("https://gitlab.freedesktop.org/freetype/freetype", - old_urls=[b"https://github.com/freetype/freetype2.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/freetype/freetype", old_urls=[b"https://github.com/freetype/freetype2.git"] + ) dependencies = ("libpng",) def setup(self): @@ -53,7 +54,8 @@ class BuildFontConfig(CrossCompileMesonProject): repository = GitRepository( "https://gitlab.freedesktop.org/fontconfig/fontconfig", temporary_url_override="https://gitlab.freedesktop.org/arichardson/fontconfig", - url_override_reason="Needs pointer provenance fixes (no PR posted yet)") + url_override_reason="Needs pointer provenance fixes (no PR posted yet)", + ) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/gdb.py b/pycheribuild/projects/cross/gdb.py index a8491136b..ebb5a7070 100644 --- a/pycheribuild/projects/cross/gdb.py +++ b/pycheribuild/projects/cross/gdb.py @@ -106,22 +106,24 @@ def setup(self) -> None: install_root = self.install_dir if self.compiling_for_host() else self.install_prefix # See https://github.com/bsdjhb/kdbg/blob/master/gdb/build # ./configure flags - self.configure_args.extend([ - "--disable-nls", - "--enable-tui", - "--disable-ld", - "--disable-gold", - "--disable-sim", - "--enable-64-bit-bfd", - "--without-gnu-as", - "--mandir=" + str(install_root / "man"), - "--infodir=" + str(install_root / "info"), - "--disable-werror", - "MAKEINFO=" + str(shutil.which("false")), - "--with-gdb-datadir=" + str(install_root / "share/gdb"), - "--disable-libstdcxx", - "--with-guile=no", - ]) + self.configure_args.extend( + [ + "--disable-nls", + "--enable-tui", + "--disable-ld", + "--disable-gold", + "--disable-sim", + "--enable-64-bit-bfd", + "--without-gnu-as", + "--mandir=" + str(install_root / "man"), + "--infodir=" + str(install_root / "info"), + "--disable-werror", + "MAKEINFO=" + str(shutil.which("false")), + "--with-gdb-datadir=" + str(install_root / "share/gdb"), + "--disable-libstdcxx", + "--with-guile=no", + ] + ) if self.target_info.is_freebsd(): self.configure_args.append("--with-separate-debug-dir=/usr/lib/debug") @@ -162,14 +164,15 @@ def setup(self) -> None: self.configure_args.append("--with-expat") else: self.configure_args.extend(["--without-python", "--without-expat", "--without-libunwind-ia64"]) - self.configure_environment.update(gl_cv_func_gettimeofday_clobber="no", - lt_cv_sys_max_cmd_len="262144", - # The build system run CC without any flags to detect dependency style... - # (ZW_PROG_COMPILER_DEPENDENCIES([CC])) -> for gcc3 mode which seems - # correct - am_cv_CC_dependencies_compiler_type="gcc3", - MAKEINFO="/bin/false", - ) + self.configure_environment.update( + gl_cv_func_gettimeofday_clobber="no", + lt_cv_sys_max_cmd_len="262144", + # The build system run CC without any flags to detect dependency style... + # (ZW_PROG_COMPILER_DEPENDENCIES([CC])) -> for gcc3 mode which seems + # correct + am_cv_CC_dependencies_compiler_type="gcc3", + MAKEINFO="/bin/false", + ) self.configure_args.append("--with-gmp=" + str(BuildGmp.get_install_dir(self))) self.configure_args.append("--with-mpfr=" + str(BuildMpfr.get_install_dir(self))) # GDB > 12 only uses --with-gmp @@ -241,8 +244,14 @@ def install(self, **kwargs): self.install_file(self.build_dir / "binutils/strip-new", bindir / "gstrip") if self.native_target_prefix is not None: - gdbfiles = ("man/man5/gdbinit.5", "man/man1/gdbserver.1", "man/man1/gdb.1", "man/man1/gdb-add-index.1", - "bin/gdb", "bin/gdb-add-index") + gdbfiles = ( + "man/man5/gdbinit.5", + "man/man1/gdbserver.1", + "man/man1/gdb.1", + "man/man1/gdb-add-index.1", + "bin/gdb", + "bin/gdb-add-index", + ) for file in gdbfiles: dst = self.install_dir / file src = dst.parent / (self.native_target_prefix + dst.name) @@ -253,12 +262,13 @@ class BuildUpstreamGDB(BuildGDBBase): repository = GitRepository("git://sourceware.org/git/binutils-gdb.git") target = "upstream-gdb" _default_install_dir_fn = ComputedDefaultValue( - function=lambda config, proj: - config.output_root / "upstream-gdb" if proj._xtarget.is_native() else None, + function=lambda config, proj: config.output_root / "upstream-gdb" if proj._xtarget.is_native() else None, # NB: We set default_architecture so the unsuffixed target is native - as_string=lambda cls: - "$INSTALL_ROOT/upstream-gdb" if cls._xtarget is None or cls._xtarget.is_native() else None, - inherit=BuildGDBBase._default_install_dir_fn) + as_string=lambda cls: "$INSTALL_ROOT/upstream-gdb" + if cls._xtarget is None or cls._xtarget.is_native() + else None, + inherit=BuildGDBBase._default_install_dir_fn, + ) class BuildGDB(BuildGDBBase): @@ -269,12 +279,15 @@ class BuildGDB(BuildGDBBase): "https://github.com/CTSRD-CHERI/gdb.git", # Branch name is changed for every major GDB release: default_branch=default_branch, - old_branches={"mips_cheri_7.12": default_branch, - "mips_cheri-8.0.1": default_branch, - "mips_cheri-8.2": default_branch, - "mips_cheri-8.3": default_branch, - "morello-8.3": default_branch}, - old_urls=[b'https://github.com/bsdjhb/gdb.git']) + old_branches={ + "mips_cheri_7.12": default_branch, + "mips_cheri-8.0.1": default_branch, + "mips_cheri-8.2": default_branch, + "mips_cheri-8.3": default_branch, + "morello-8.3": default_branch, + }, + old_urls=[b"https://github.com/bsdjhb/gdb.git"], + ) class BuildKGDB(BuildGDB): @@ -284,4 +297,5 @@ class BuildKGDB(BuildGDB): # Branch name is changed for every major GDB release: default_branch=default_branch, old_branches={"mips_cheri-8.3-kgdb": default_branch}, - old_urls=[b'https://github.com/bsdjhb/gdb.git']) + old_urls=[b"https://github.com/bsdjhb/gdb.git"], + ) diff --git a/pycheribuild/projects/cross/git.py b/pycheribuild/projects/cross/git.py index d053da7cb..a03ae1c90 100644 --- a/pycheribuild/projects/cross/git.py +++ b/pycheribuild/projects/cross/git.py @@ -60,10 +60,8 @@ def setup(self): ac_cv_snprintf_returns_bogus="no", ) # Doesn't use pkg-config - self.configure_args.extend([ - "--with-curl=" + str(BuildCurl.get_install_dir(self)), - "--with-expat=" + str(BuildExpat.get_install_dir(self)), - ]) + self.configure_args.append(f"--with-curl={BuildCurl.get_install_dir(self)}") + self.configure_args.append(f"--with-expat={BuildExpat.get_install_dir(self)}") # Build-time detection of uname to determine more properties # Only S and R seem to be used currently, but provide sensible # values or, for V, a dummy kernconf (and format it like a release diff --git a/pycheribuild/projects/cross/gkermit.py b/pycheribuild/projects/cross/gkermit.py index 93d62d6ef..ee9f63dfd 100644 --- a/pycheribuild/projects/cross/gkermit.py +++ b/pycheribuild/projects/cross/gkermit.py @@ -40,8 +40,7 @@ def setup(self): super().setup() self.common_warning_flags.append("-Wno-unused-value") self.common_warning_flags.append("-Wno-non-literal-null-conversion") - self.make_args.set_env(KFLAGS=self.commandline_to_str( - [*self.default_compiler_flags, "-include", "unistd.h"])) + self.make_args.set_env(KFLAGS=self.commandline_to_str([*self.default_compiler_flags, "-include", "unistd.h"])) def update(self): filename = "gku201.tar.gz" diff --git a/pycheribuild/projects/cross/glib.py b/pycheribuild/projects/cross/glib.py index 4bd714c62..2df90c749 100644 --- a/pycheribuild/projects/cross/glib.py +++ b/pycheribuild/projects/cross/glib.py @@ -29,11 +29,14 @@ class BuildGlib(CrossCompileMesonProject): target = "glib" dependencies = ("pcre2", "libffi", "dbus") - repository = GitRepository("https://gitlab.gnome.org/GNOME/glib.git", - temporary_url_override="https://gitlab.gnome.org/arichardson/glib.git", - old_urls=[b"https://github.com/CTSRD-CHERI/glib.git"], - url_override_reason="Lots of CHERI incompatibilities", - default_branch="main-with-cheri-fixes", force_branch=True) + repository = GitRepository( + "https://gitlab.gnome.org/GNOME/glib.git", + temporary_url_override="https://gitlab.gnome.org/arichardson/glib.git", + old_urls=[b"https://github.com/CTSRD-CHERI/glib.git"], + url_override_reason="Lots of CHERI incompatibilities", + default_branch="main-with-cheri-fixes", + force_branch=True, + ) def setup(self) -> None: super().setup() diff --git a/pycheribuild/projects/cross/gmp.py b/pycheribuild/projects/cross/gmp.py index 39b87cf3e..8c4763395 100644 --- a/pycheribuild/projects/cross/gmp.py +++ b/pycheribuild/projects/cross/gmp.py @@ -30,9 +30,12 @@ class BuildGmp(CrossCompileAutotoolsProject): repository = MercurialRepository("https://gmplib.org/repo/gmp") - supported_architectures = (CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + - CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + - CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + + CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + + CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + + CompilationTargets.ALL_NATIVE + ) native_install_dir = DefaultInstallDir.CHERI_SDK def check_system_dependencies(self) -> None: diff --git a/pycheribuild/projects/cross/juliet_test_suite.py b/pycheribuild/projects/cross/juliet_test_suite.py index 55e5f3ce0..29cc004f6 100644 --- a/pycheribuild/projects/cross/juliet_test_suite.py +++ b/pycheribuild/projects/cross/juliet_test_suite.py @@ -52,8 +52,10 @@ def setup_config_options(cls, **kwargs): def process(self): if self.build_type != BuildType.DEBUG: - self.warning("The Juliet test suite contains undefined behaviour that might be optimized away unless " - "you compile at -O0.") + self.warning( + "The Juliet test suite contains undefined behaviour that might be optimized away unless " + "you compile at -O0." + ) self.ask_for_confirmation("Are you sure you want to continue?") super().process() @@ -112,78 +114,80 @@ def run_tests(self): # For stdin redirection args.append("--test-setup-command=touch /tmp/in.txt") - self.target_info.run_cheribsd_test_script("run_juliet_tests.py", *args, mount_sourcedir=True, - mount_sysroot=True, mount_builddir=True) + self.target_info.run_cheribsd_test_script( + "run_juliet_tests.py", *args, mount_sourcedir=True, mount_sysroot=True, mount_builddir=True + ) class BuildJulietCWE121(BuildJulietCWESubdir): target = "juliet-cwe-121" cwe_number = 121 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE121_Stack_Based_Buffer_Overflow") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE121_Stack_Based_Buffer_Overflow" + ) cwe_setup_commands = [ - "echo 500 > /tmp/in.txt", - ] + "echo 500 > /tmp/in.txt", + ] class BuildJulietCWE122(BuildJulietCWESubdir): target = "juliet-cwe-122" cwe_number = 122 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE122_Heap_Based_Buffer_Overflow") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE122_Heap_Based_Buffer_Overflow" + ) cwe_setup_commands = [ - "echo 500 > /tmp/in.txt", - ] + "echo 500 > /tmp/in.txt", + ] class BuildJulietCWE124(BuildJulietCWESubdir): target = "juliet-cwe-124" cwe_number = 124 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE124_Buffer_Underwrite") + repository = ReuseOtherProjectRepository(BuildJulietTestSuite, subdirectory="testcases/CWE124_Buffer_Underwrite") cwe_setup_commands = [ - "echo -500 > /tmp/in.txt", - ] + "echo -500 > /tmp/in.txt", + ] class BuildJulietCWE126(BuildJulietCWESubdir): target = "juliet-cwe-126" cwe_number = 126 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE126_Buffer_Overread") + repository = ReuseOtherProjectRepository(BuildJulietTestSuite, subdirectory="testcases/CWE126_Buffer_Overread") cwe_setup_commands = [ - "echo 500 > /tmp/in.txt", - ] + "echo 500 > /tmp/in.txt", + ] class BuildJulietCWE127(BuildJulietCWESubdir): target = "juliet-cwe-127" cwe_number = 127 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE127_Buffer_Underread") + repository = ReuseOtherProjectRepository(BuildJulietTestSuite, subdirectory="testcases/CWE127_Buffer_Underread") cwe_setup_commands = [ - "echo -500 > /tmp/in.txt", - ] + "echo -500 > /tmp/in.txt", + ] class BuildJulietCWE134(BuildJulietCWESubdir): target = "juliet-cwe-134" cwe_number = 134 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE134_Uncontrolled_Format_String") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE134_Uncontrolled_Format_String" + ) cwe_warning_flags = ["-Wno-error=format-security"] cwe_setup_commands = [ - "export ADD=%s%d%s", - "echo Format string: %s %d %s > /tmp/file.txt", - "echo Format string: %s %d %s > /tmp/in.txt", - ] + "export ADD=%s%d%s", + "echo Format string: %s %d %s > /tmp/file.txt", + "echo Format string: %s %d %s > /tmp/in.txt", + ] class BuildJulietCWE188(BuildJulietCWESubdir): target = "juliet-cwe-188" cwe_number = 188 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE188_Reliance_on_Data_Memory_Layout") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE188_Reliance_on_Data_Memory_Layout" + ) def setup(self): super().setup() @@ -206,31 +210,33 @@ class BuildJulietCWE416(BuildJulietCWESubdir): class BuildJulietCWE587(BuildJulietCWESubdir): target = "juliet-cwe-587" cwe_number = 587 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE587_Assignment_of_Fixed_Address_to_Pointer") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE587_Assignment_of_Fixed_Address_to_Pointer" + ) class BuildJulietCWE588(BuildJulietCWESubdir): target = "juliet-cwe-588" cwe_number = 588 repository = ReuseOtherProjectRepository( - BuildJulietTestSuite, - subdirectory="testcases/CWE588_Attempt_to_Access_Child_of_Non_Structure_Pointer") + BuildJulietTestSuite, subdirectory="testcases/CWE588_Attempt_to_Access_Child_of_Non_Structure_Pointer" + ) class BuildJulietCWE680(BuildJulietCWESubdir): target = "juliet-cwe-680" cwe_number = 680 - repository = ReuseOtherProjectRepository(BuildJulietTestSuite, - subdirectory="testcases/CWE680_Integer_Overflow_to_Buffer_Overflow") + repository = ReuseOtherProjectRepository( + BuildJulietTestSuite, subdirectory="testcases/CWE680_Integer_Overflow_to_Buffer_Overflow" + ) class BuildJulietCWE685(BuildJulietCWESubdir): target = "juliet-cwe-685" cwe_number = 685 repository = ReuseOtherProjectRepository( - BuildJulietTestSuite, - subdirectory="testcases/CWE685_Function_Call_With_Incorrect_Number_of_Arguments") + BuildJulietTestSuite, subdirectory="testcases/CWE685_Function_Call_With_Incorrect_Number_of_Arguments" + ) cwe_warning_flags = ["-Wno-error=format-insufficient-args"] diff --git a/pycheribuild/projects/cross/kde.py b/pycheribuild/projects/cross/kde.py index a0e022581..71e9bb835 100644 --- a/pycheribuild/projects/cross/kde.py +++ b/pycheribuild/projects/cross/kde.py @@ -90,16 +90,19 @@ def ctest_script_extra_args(self): kde_prefix = self.install_prefix if self.tests_need_full_disk_image: return ["--test-setup-command", ". /build/prefix.sh && env | sort"] - return ["--extra-library-path", "/build/bin", "--extra-library-path", "/build/lib", - # Add the libraries from other frameworks - "--extra-library-path", "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", - # Also need the X11 libraries for most tests - "--extra-library-path", "/sysroot" + str(BuildLibXCB.get_instance(self).install_prefix) + "/lib", - # And of course QtCore/QtTest - "--extra-library-path", "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib", - "--test-setup-command", - f"mkdir -p {kde_prefix.parent} && ln -sn /sysroot{kde_prefix} {kde_prefix}", - "--test-setup-command", ". /build/prefix.sh && env | sort"] + # fmt: off + return [ + "--extra-library-path", "/build/bin", "--extra-library-path", "/build/lib", + # Add the libraries from other frameworks + "--extra-library-path", "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", + # Also need the X11 libraries for most tests + "--extra-library-path", "/sysroot" + str(BuildLibXCB.get_instance(self).install_prefix) + "/lib", + # And of course QtCore/QtTest + "--extra-library-path", "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib", + "--test-setup-command", + f"mkdir -p {kde_prefix.parent} && ln -sn /sysroot{kde_prefix} {kde_prefix}", + "--test-setup-command", ". /build/prefix.sh && env | sort" + ] # fmt: on def setup(self): super().setup() @@ -117,10 +120,17 @@ def setup(self): # We need native tools (e.g. desktoptojson/kconfig_compiler) for some projects native_project = BuildKCoreAddons.get_instance(self, cross_target=CompilationTargets.NATIVE) self.add_cmake_options( - KF5_HOST_TOOLING=native_project.install_dir / native_project.target_info.default_libdir / "cmake") - dep_names = " ".join(x.name for x in self._direct_dependencies(self.config, include_sdk_dependencies=False, - include_toolchain_dependencies=False, - explicit_dependencies_only=True)) + KF5_HOST_TOOLING=native_project.install_dir / native_project.target_info.default_libdir / "cmake" + ) + dep_names = " ".join( + x.name + for x in self._direct_dependencies( + self.config, + include_sdk_dependencies=False, + include_toolchain_dependencies=False, + explicit_dependencies_only=True, + ) + ) if "qtx11extras" in dep_names: self.warning("Adding include path as workaround for broken QtX11Extras") self.COMMON_FLAGS.append("-I" + str(BuildLibXCB.get_install_dir(self) / "include")) @@ -138,8 +148,10 @@ def setup(self): self.add_cmake_options(CMAKE_DISABLE_FIND_PACKAGE_KF5DocTools=True) def run_tests(self): - self.info("To debug failing tests, you can increase verbosity by setting", - coloured(AnsiColour.yellow, 'export QT_LOGGING_RULES="*.debug=true"')) + self.info( + "To debug failing tests, you can increase verbosity by setting", + coloured(AnsiColour.yellow, 'export QT_LOGGING_RULES="*.debug=true"'), + ) super().run_tests() @@ -178,16 +190,18 @@ class BuildGettext(CrossCompileAutotoolsProject): def setup(self): super().setup() - self.configure_args.extend([ - "--enable-relocatable", - "--disable-csharp", - "--disable-java", - "--disable-libasprintf", - "--disable-openmp", - "--without-emacs", - "--with-included-gettext", - "ac_cv_lib_rt_sched_yield=no", - ]) + self.configure_args.extend( + [ + "--enable-relocatable", + "--disable-csharp", + "--disable-java", + "--disable-libasprintf", + "--disable-openmp", + "--without-emacs", + "--with-included-gettext", + "ac_cv_lib_rt_sched_yield=no", + ] + ) def clean(self): if not (self.source_dir / "Makefile").exists(): @@ -204,9 +218,13 @@ def process(self): new_env = dict() if OSInfo.IS_MAC: # /usr/bin/bison and /usr/bin/sed on macOS are not compatible with this build system - new_env["PATH"] = ":".join([str(self.get_homebrew_prefix("gnu-sed") / "libexec/gnubin"), - str(self.get_homebrew_prefix("bison") / "bin"), - os.getenv("PATH")]) + new_env["PATH"] = ":".join( + [ + str(self.get_homebrew_prefix("gnu-sed") / "libexec/gnubin"), + str(self.get_homebrew_prefix("bison") / "bin"), + os.getenv("PATH"), + ] + ) with self.set_env(**new_env): super().process() @@ -226,8 +244,13 @@ class BuildKWayland(KDECMakeProject): @classmethod def dependencies(cls, config) -> "tuple[str, ...]": - result = (*super().dependencies(config), "libglvnd", "wayland-protocols", "qtwayland", - "plasma-wayland-protocols") + result = ( + *super().dependencies(config), + "libglvnd", + "wayland-protocols", + "qtwayland", + "plasma-wayland-protocols", + ) if cls.get_crosscompile_target().target_info_cls.is_freebsd(): result += ("linux-input-h",) return result @@ -278,10 +301,14 @@ def install(self, **kwargs): if not self.compiling_for_host(): # TODO: should probably just install Qt and KDE files in the same directory install_prefix = self.install_prefix - all_prefixes = " ".join(shlex.quote("/" + str(s.relative_to(self.rootfs_dir))) for s in - self.dependency_install_prefixes) - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-common", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh + all_prefixes = " ".join( + shlex.quote("/" + str(s.relative_to(self.rootfs_dir))) for s in self.dependency_install_prefixes + ) + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-common", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh set -xe # QML disk caching is currently broken export QML_DISABLE_DISK_CACHE=1 @@ -342,21 +369,33 @@ def install(self, **kwargs): else exec sh fi; -""") - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh +""", + ) + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh export DISPLAY=:0 export QT_QPA_PLATFORM=xcb exec "/{self.target_info.localbase / "bin/kde-shell-common"}" -""") - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-wayland", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh +""", + ) + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-wayland", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh export QT_QPA_PLATFORM=wayland exec "/{self.target_info.localbase / "bin/kde-shell-common"}" -""") - - self.write_file(self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11-smbfs", - overwrite=True, mode=0o755, contents=f"""#!/bin/sh +""", + ) + + self.write_file( + self.rootfs_dir / self.target_info.localbase / "bin/kde-shell-x11-smbfs", + overwrite=True, + mode=0o755, + contents=f"""#!/bin/sh set -xe if df -t smbfs,nfs "{install_prefix}" >/dev/null 2>/dev/null; then echo "{install_prefix} is already mounted from the host, skipping" @@ -367,7 +406,8 @@ def install(self, **kwargs): fi set +xe exec "/{self.target_info.localbase / "bin/kde-shell-x11"}" -""") +""", + ) class BuildKConfig(KDECMakeProject): @@ -523,8 +563,15 @@ class BuildKNotifications(KDECMakeProject): @classmethod def dependencies(cls, config) -> "tuple[str, ...]": - result = ("qtdeclarative", "kwindowsystem", "kconfig", "kconfig-native", "kcoreaddons", "kcoreaddons-native", - "phonon") + result = ( + "qtdeclarative", + "kwindowsystem", + "kconfig", + "kconfig-native", + "kcoreaddons", + "kcoreaddons-native", + "phonon", + ) if cls.get_crosscompile_target().target_info_cls.is_macos(): return (*result, "qtmacextras") return (*result, "qtx11extras") @@ -533,7 +580,8 @@ def dependencies(cls, config) -> "tuple[str, ...]": class BuildKPackage(KDECMakeProject): dependencies = ("karchive", "ki18n", "kcoreaddons", "kcoreaddons-native") repository = KF5GitRepository( - "https://invent.kde.org/frameworks/kpackage.git", old_urls=[b"https://invent.kde.org/arichardson/kpackage.git"], + "https://invent.kde.org/frameworks/kpackage.git", + old_urls=[b"https://invent.kde.org/arichardson/kpackage.git"], ) @@ -563,8 +611,15 @@ class BuildKBookmarks(KDECMakeProject): class BuildKCMUtils(KDECMakeProject): - dependencies = ("kitemviews", "kconfigwidgets", "kservice", "kxmlgui", "kdeclarative", "kauth", - "kcmutils-tools-native") + dependencies = ( + "kitemviews", + "kconfigwidgets", + "kservice", + "kxmlgui", + "kdeclarative", + "kauth", + "kcmutils-tools-native", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kcmutils.git") @@ -581,8 +636,16 @@ def setup(self): class BuildKConfigWidgets(KDECMakeProject): - dependencies = ("kauth", "kcoreaddons", "kcodecs", "kconfig", "kguiaddons", "ki18n", "kwidgetsaddons", - "kconfig-native") + dependencies = ( + "kauth", + "kcoreaddons", + "kcodecs", + "kconfig", + "kguiaddons", + "ki18n", + "kwidgetsaddons", + "kconfig-native", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kconfigwidgets.git") _has_qt_designer_plugin = True @@ -595,17 +658,34 @@ class BuildKConfigWidgets(KDECMakeProject): # frameworks/kemoticons: frameworks/kservice # frameworks/kjs: frameworks/kdoctools class BuildKNewStuff(KDECMakeProject): - dependencies = ("attica", "kitemviews", "kiconthemes", "ktextwidgets", "kxmlgui", - "solid", "kio", "kbookmarks", "kpackage", "kpackage-native", "ksyndication", "kirigami") + dependencies = ( + "attica", + "kitemviews", + "kiconthemes", + "ktextwidgets", + "kxmlgui", + "solid", + "kio", + "kbookmarks", + "kpackage", + "kpackage-native", + "ksyndication", + "kirigami", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/knewstuff.git") _needs_newer_bison = True class BuildKService(KDECMakeProject): - dependencies = ("kconfig", "kcoreaddons", "kcrash", "kdbusaddons", "ki18n", - "kcoreaddons-native", # desktoptojson - "kconfig-native", # kconfig_compiler - ) + dependencies = ( + "kconfig", + "kcoreaddons", + "kcrash", + "kdbusaddons", + "ki18n", + "kcoreaddons-native", # desktoptojson + "kconfig-native", # kconfig_compiler + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kservice.git") _needs_newer_bison = True @@ -640,24 +720,50 @@ def dependencies(cls, config) -> "tuple[str, ...]": class BuildKXMLGUI(KDECMakeProject): - dependencies = ("kitemviews", "kconfig", "kconfig-native", "kglobalaccel", - "kconfigwidgets", "ki18n", "kiconthemes", - "ktextwidgets", "kwidgetsaddons", "kwindowsystem") + dependencies = ( + "kitemviews", + "kconfig", + "kconfig-native", + "kglobalaccel", + "kconfigwidgets", + "ki18n", + "kiconthemes", + "ktextwidgets", + "kwidgetsaddons", + "kwindowsystem", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kxmlgui.git") _has_qt_designer_plugin = True class BuildKDeclarative(KDECMakeProject): repository = KF5GitRepository("https://invent.kde.org/frameworks/kdeclarative.git") - dependencies = ("kpackage", "kpackage-native", "kio", "kiconthemes", "knotifications", "qtdeclarative", "kio", - "libepoxy") + dependencies = ( + "kpackage", + "kpackage-native", + "kio", + "kiconthemes", + "knotifications", + "qtdeclarative", + "kio", + "libepoxy", + ) _has_qt_designer_plugin = True class BuildKInit(KDECMakeProject): target = "kinit" - dependencies = ("kio", "kservice", "kcrash", "kjobwidgets", "solid", "kdbusaddons", "kwindowsystem", "libx11", - "libxcb") + dependencies = ( + "kio", + "kservice", + "kcrash", + "kjobwidgets", + "solid", + "kdbusaddons", + "kwindowsystem", + "libx11", + "libxcb", + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kinit.git") @@ -675,12 +781,33 @@ class BuildKDED(KDECMakeProject): class BuildKIO(KDECMakeProject): target = "kio" - dependencies = ("kauth", "kdbusaddons", "ki18n", "kguiaddons", "kconfigwidgets", "kitemviews", "kcoreaddons", - "kwidgetsaddons", "kservice", "karchive", "qtx11extras", "solid", - "kjobwidgets", "kiconthemes", "kwindowsystem", "kcrash", "kcompletion", "ktextwidgets", - "kxmlgui", "kbookmarks", "kconfig", "kconfig-native", "knotifications", "kded", - # optional: "kwallet" - ) + dependencies = ( + "kauth", + "kdbusaddons", + "ki18n", + "kguiaddons", + "kconfigwidgets", + "kitemviews", + "kcoreaddons", + "kwidgetsaddons", + "kservice", + "karchive", + "qtx11extras", + "solid", + "kjobwidgets", + "kiconthemes", + "kwindowsystem", + "kcrash", + "kcompletion", + "ktextwidgets", + "kxmlgui", + "kbookmarks", + "kconfig", + "kconfig-native", + "knotifications", + "kded", + # optional: "kwallet" + ) repository = KF5GitRepository("https://invent.kde.org/frameworks/kio.git") _has_qt_designer_plugin = True @@ -759,9 +886,11 @@ class BuildKioExtras(KDECMakeProject): # This includes e.g. the thumbnail provider for dolphin target = "kio-extras" dependencies = ("kio", "ksyntaxhighlighting") - repository = GitRepository("https://invent.kde.org/network/kio-extras.git", - temporary_url_override="https://invent.kde.org/arichardson/kio-extras.git", - url_override_reason="https://invent.kde.org/network/kio-extras/-/merge_requests/110") + repository = GitRepository( + "https://invent.kde.org/network/kio-extras.git", + temporary_url_override="https://invent.kde.org/arichardson/kio-extras.git", + url_override_reason="https://invent.kde.org/network/kio-extras/-/merge_requests/110", + ) def setup(self): super().setup() @@ -802,15 +931,29 @@ class BuildKirigami(KDECMakeProject): target = "kirigami" dependencies = ("qtquickcontrols2", "extra-cmake-modules", "qtgraphicaleffects") repository = KF5GitRepository( - "https://invent.kde.org/frameworks/kirigami.git", old_urls=[b"https://invent.kde.org/arichardson/kirigami.git"], + "https://invent.kde.org/frameworks/kirigami.git", + old_urls=[b"https://invent.kde.org/arichardson/kirigami.git"], ) class BuildPlasmaFramework(KDECMakeProject): target = "plasma-framework" - dependencies = ("kio", "kconfigwidgets", "kactivities", "kdbusaddons", "kglobalaccel", "kpackage", "kdeclarative", - "qtquickcontrols", "qtquickcontrols2", "kxmlgui", "threadweaver", "kirigami", "kwayland", - "libglvnd") + dependencies = ( + "kio", + "kconfigwidgets", + "kactivities", + "kdbusaddons", + "kglobalaccel", + "kpackage", + "kdeclarative", + "qtquickcontrols", + "qtquickcontrols2", + "kxmlgui", + "threadweaver", + "kirigami", + "kwayland", + "libglvnd", + ) repository = KF5GitRepository( "https://invent.kde.org/frameworks/plasma-framework.git", old_urls=[b"https://invent.kde.org/arichardson/plasma-framework.git"], @@ -838,8 +981,17 @@ class BuildKFrameworkIntegration(KDECMakeProject): class BuildBreezeStyle(KDECMakeProject): target = "breeze" repository = KDEPlasmaGitRepository("https://invent.kde.org/plasma/breeze.git") - dependencies = ("kdecoration", "kconfig", "kcoreaddons", "kguiaddons", "kiconthemes", "kconfigwidgets", - "kwindowsystem", "kcmutils", "kframeworkintegration") + dependencies = ( + "kdecoration", + "kconfig", + "kcoreaddons", + "kguiaddons", + "kiconthemes", + "kconfigwidgets", + "kwindowsystem", + "kcmutils", + "kframeworkintegration", + ) class BuildKIdleTime(KDECMakeProject): @@ -885,8 +1037,16 @@ class BuildKScreenLocker(KDECMakeProject): class BuildKDECliTools(KDECMakeProject): target = "kde-cli-tools" repository = KDEPlasmaGitRepository("https://invent.kde.org/plasma/kde-cli-tools.git") - dependencies = ("kconfig", "kiconthemes", "ki18n", "kcmutils", "kio", "kservice", "kwindowsystem", - "kactivities") # optional: "kdesu" + dependencies = ( + "kconfig", + "kiconthemes", + "ki18n", + "kcmutils", + "kio", + "kservice", + "kwindowsystem", + "kactivities", + ) # optional: "kdesu" class BuildKWin(KDECMakeProject): @@ -905,8 +1065,21 @@ class BuildKWin(KDECMakeProject): @classmethod def dependencies(cls, config) -> "tuple[str, ...]": - result = (*super().dependencies(config), "kdecoration", "qtx11extras", "breeze", "kcmutils", "kscreenlocker", - "plasma-framework", "libinput", "qttools", "libepoxy", "libxcvt", "lcms2", "krunner") + result = ( + *super().dependencies(config), + "kdecoration", + "qtx11extras", + "breeze", + "kcmutils", + "kscreenlocker", + "plasma-framework", + "libinput", + "qttools", + "libepoxy", + "libxcvt", + "lcms2", + "krunner", + ) if cls.use_mesa: result += ("mesa", "libdrm") if cls.get_crosscompile_target().target_info_cls.is_freebsd(): @@ -920,10 +1093,14 @@ def setup(self): self.add_cmake_options(KWIN_BUILD_DRM_BACKEND=self.use_mesa, KWIN_BUILD_WAYLAND_EGL=self.use_mesa) if not self.compiling_for_host(): # We need to find the host Qt libraries for qwaylandscanner_kde. - self.add_cmake_options(NATIVE_PREFIX=";".join([ - str(BuildQtBase.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), - str(BuildKCoreAddons.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), - ])) + self.add_cmake_options( + NATIVE_PREFIX=";".join( + [ + str(BuildQtBase.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), + str(BuildKCoreAddons.get_install_dir(self, cross_target=CompilationTargets.NATIVE)), + ] + ) + ) class BuildLibKScreen(KDECMakeProject): @@ -973,11 +1150,38 @@ class BuildPlasmaWorkspace(KDECMakeProject): _uses_wayland_scanner = True repository = KDEPlasmaGitRepository( "https://invent.kde.org/plasma/plasma-workspace.git", - old_urls=[b"https://invent.kde.org/arichardson/plasma-workspace.git"]) - dependencies = ("xprop", "xsetroot", "plasma-framework", "kwin", "breeze", "kidletime", "kitemmodels", "kcmutils", - "knotifyconfig", "kded", "kinit", "kscreenlocker", "libkscreen", "libxft", "libxtst", "kpeople", - "kparts", "prison", "krunner", "kactivities-stats", "libksysguard", "kunitconversion", "kwallet", - "ktexteditor", "kwayland", "layer-shell-qt", "kquickcharts", "kactivitymanagerd") + old_urls=[b"https://invent.kde.org/arichardson/plasma-workspace.git"], + ) + dependencies = ( + "xprop", + "xsetroot", + "plasma-framework", + "kwin", + "breeze", + "kidletime", + "kitemmodels", + "kcmutils", + "knotifyconfig", + "kded", + "kinit", + "kscreenlocker", + "libkscreen", + "libxft", + "libxtst", + "kpeople", + "kparts", + "prison", + "krunner", + "kactivities-stats", + "libksysguard", + "kunitconversion", + "kwallet", + "ktexteditor", + "kwayland", + "layer-shell-qt", + "kquickcharts", + "kactivitymanagerd", + ) def setup(self): super().setup() @@ -1005,9 +1209,17 @@ class BuildPlasmaDesktop(KDECMakeProject): repository = KDEPlasmaGitRepository( "https://invent.kde.org/plasma/plasma-desktop.git", temporary_url_override="https://invent.kde.org/arichardson/plasma-desktop.git", - url_override_reason="https://invent.kde.org/plasma/plasma-desktop/-/merge_requests/944") - dependencies = ("plasma-workspace", "kirigami", "krunner", "kwallet", "qqc2-desktop-style", - "libxkbfile", "xkeyboard-config") + url_override_reason="https://invent.kde.org/plasma/plasma-desktop/-/merge_requests/944", + ) + dependencies = ( + "plasma-workspace", + "kirigami", + "krunner", + "kwallet", + "qqc2-desktop-style", + "libxkbfile", + "xkeyboard-config", + ) def setup(self): super().setup() @@ -1031,8 +1243,12 @@ class BuildDoplhin(KDECMakeProject): class BuildLibPng(CrossCompileCMakeProject): supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE - repository = GitRepository("https://github.com/glennrp/libpng", old_urls=[b"https://github.com/CTSRD-CHERI/libpng"], - default_branch="libpng16", force_branch=True) + repository = GitRepository( + "https://github.com/glennrp/libpng", + old_urls=[b"https://github.com/CTSRD-CHERI/libpng"], + default_branch="libpng16", + force_branch=True, + ) target = "libpng" # The tests take a really long time to run (~2.5 hours on purecap RISC-V) ctest_script_extra_args = ("--test-timeout", 5 * 60 * 60) @@ -1085,8 +1301,17 @@ class BuildKImageAnnotator(KDECMakeProject): class BuildGwenview(KDECMakeProject): target = "gwenview" - dependencies = ("qtsvg", "kitemmodels", "kimageformats", "kio", "kparts", "lcms2", "libpng", "exiv2", - "kimageannotator") + dependencies = ( + "qtsvg", + "kitemmodels", + "kimageformats", + "kio", + "kparts", + "lcms2", + "libpng", + "exiv2", + "kimageannotator", + ) repository = GitRepository("https://invent.kde.org/graphics/gwenview.git") @@ -1105,8 +1330,10 @@ def setup(self): class BuildPoppler(CrossCompileCMakeProject): target = "poppler" dependencies = ("freetype2", "fontconfig", "qtbase", "libtiff", "openjpeg") - repository = GitRepository("https://gitlab.freedesktop.org/poppler/poppler.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/poppler.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/poppler/poppler.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/poppler.git"], + ) def setup(self): super().setup() @@ -1123,10 +1350,16 @@ def update(self): @property def ctest_script_extra_args(self): - return ["--extra-library-path", "/build/bin", - "--extra-library-path", "/build/lib", - "--extra-library-path", "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", - "--extra-library-path", "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib"] + return [ + "--extra-library-path", + "/build/bin", + "--extra-library-path", + "/build/lib", + "--extra-library-path", + "/sysroot" + str(self.install_prefix) + "/lib:/sysroot/usr/lib:/sysroot/lib", + "--extra-library-path", + "/sysroot" + str(BuildQtBase.get_instance(self).install_prefix) + "/lib", + ] class BuildThreadWeaver(KDECMakeProject): @@ -1137,7 +1370,8 @@ class BuildThreadWeaver(KDECMakeProject): class BuildKPty(KDECMakeProject): target = "kpty" repository = KF5GitRepository( - "https://invent.kde.org/frameworks/kpty.git", old_urls=[b"https://invent.kde.org/arichardson/kpty"], + "https://invent.kde.org/frameworks/kpty.git", + old_urls=[b"https://invent.kde.org/arichardson/kpty"], ) def setup(self): @@ -1149,10 +1383,32 @@ def setup(self): class BuildKonsole(KDECMakeProject): target = "konsole" repository = GitRepository("https://invent.kde.org/utilities/konsole.git") - dependencies = ("extra-cmake-modules", "kbookmarks", "kconfig", "kconfigwidgets", "kcoreaddons", "kcrash", - "kglobalaccel", "kguiaddons", "kdbusaddons", "ki18n", "kiconthemes", "kio", "knewstuff", - "knotifications", "knotifyconfig", "kparts", "kpty", "kservice", "ktextwidgets", "kwidgetsaddons", - "kwindowsystem", "kxmlgui", "qtbase", "qtmultimedia") + dependencies = ( + "extra-cmake-modules", + "kbookmarks", + "kconfig", + "kconfigwidgets", + "kcoreaddons", + "kcrash", + "kglobalaccel", + "kguiaddons", + "kdbusaddons", + "ki18n", + "kiconthemes", + "kio", + "knewstuff", + "knotifications", + "knotifyconfig", + "kparts", + "kpty", + "kservice", + "ktextwidgets", + "kwidgetsaddons", + "kwindowsystem", + "kxmlgui", + "qtbase", + "qtmultimedia", + ) # TODO: fails to build due to exiv2 usage of auto_ptr @@ -1169,8 +1425,9 @@ class BuildOkular(KDECMakeProject): target = "okular" dependencies = ("poppler", "threadweaver", "kparts", "kio", "kiconthemes", "kpty", "kwallet", "libtiff") # TODO: after the next exiv2 release add "libkexiv2" (currently fails to build due to auto_ptr). - repository = GitRepository("https://invent.kde.org/graphics/okular.git", - old_urls=[b"https://invent.kde.org/arichardson/okular.git"]) + repository = GitRepository( + "https://invent.kde.org/graphics/okular.git", old_urls=[b"https://invent.kde.org/arichardson/okular.git"] + ) def setup(self): super().setup() @@ -1181,7 +1438,8 @@ def setup(self): "KF5Purpose", # Could add this framework (only required for enabling the share menu). "Qt5TextToSpeech", # We don't need speech features. "LibSpectre", # We don't need postscript support. - "CHM", "LibZip", # We don't need CHM support. + "CHM", + "LibZip", # We don't need CHM support. "DjVuLibre", # We don't need DjVu support. "Discount", # We don't need markdown support. "EPub", # We don't need EPub support. @@ -1195,22 +1453,64 @@ def setup(self): class BuildKTextEditor(KDECMakeProject): target = "ktexteditor" repository = KF5GitRepository("https://invent.kde.org/frameworks/ktexteditor.git") - dependencies = ("editorconfig-core-c", "extra-cmake-modules", "karchive", "kauth", "kconfig", "kguiaddons", "ki18n", - "kio", "kparts", "ksyntaxhighlighting", "ktextwidgets", "sonnet", "qtbase", "qtdeclarative") + dependencies = ( + "editorconfig-core-c", + "extra-cmake-modules", + "karchive", + "kauth", + "kconfig", + "kguiaddons", + "ki18n", + "kio", + "kparts", + "ksyntaxhighlighting", + "ktextwidgets", + "sonnet", + "qtbase", + "qtdeclarative", + ) class BuildKate(KDECMakeProject): target = "kate" repository = GitRepository("https://invent.kde.org/utilities/kate.git") - dependencies = ("extra-cmake-modules", "kactivities", "kconfig", "kcoreaddons", "kcrash", "kdbusaddons", - "kguiaddons", "ki18n", "kiconthemes", "kitemmodels", "kitemviews", "knewstuff", "kwallet", - "ksyntaxhighlighting", "ktexteditor", "ktextwidgets", "kwindowsystem", "qtbase") + dependencies = ( + "extra-cmake-modules", + "kactivities", + "kconfig", + "kcoreaddons", + "kcrash", + "kdbusaddons", + "kguiaddons", + "ki18n", + "kiconthemes", + "kitemmodels", + "kitemviews", + "knewstuff", + "kwallet", + "ksyntaxhighlighting", + "ktexteditor", + "ktextwidgets", + "kwindowsystem", + "qtbase", + ) class BuildKDEX11Desktop(TargetAliasWithDependencies): target = "kde-x11-desktop" supported_architectures = CompilationTargets.ALL_SUPPORTED_CHERIBSD_AND_HOST_TARGETS # TODO: "systemsettings" - now needs a newer plasma-workspace - dependencies = ("plasma-desktop", "dolphin", "konsole", "okular", "gwenview", "kate", - "xvnc-server", "systemsettings", - "xeyes", "twm", "xev", "xauth") # Add some basic X11 things as a fallback + dependencies = ( + "plasma-desktop", + "dolphin", + "konsole", + "okular", + "gwenview", + "kate", + "xvnc-server", + "systemsettings", + "xeyes", + "twm", + "xev", + "xauth", + ) # Add some basic X11 things as a fallback diff --git a/pycheribuild/projects/cross/littlekernel.py b/pycheribuild/projects/cross/littlekernel.py index 47da0a42a..35ec64563 100644 --- a/pycheribuild/projects/cross/littlekernel.py +++ b/pycheribuild/projects/cross/littlekernel.py @@ -44,9 +44,11 @@ class BuildLittleKernel(CrossCompileMakefileProject): CompilationTargets.FREESTANDING_RISCV64, CompilationTargets.FREESTANDING_RISCV64_PURECAP, ) - repository = GitRepository("https://github.com/littlekernel/lk", - temporary_url_override="https://github.com/arichardson/lk.git", - url_override_reason="Fixes to allow building with Clang") + repository = GitRepository( + "https://github.com/littlekernel/lk", + temporary_url_override="https://github.com/arichardson/lk.git", + url_override_reason="Fixes to allow building with Clang", + ) default_install_dir = DefaultInstallDir.DO_NOT_INSTALL set_pkg_config_path = False needs_sysroot = False @@ -73,15 +75,19 @@ def build_dir_suffix(self): def compiler_rt_builtins_path(self) -> Path: path = BuildCompilerRtBuiltins.get_build_dir(self) / f"lib/baremetal/libclang_rt.builtins-{self.triple_arch}.a" if not path.exists(): - self.dependency_error("Compiler builtins library", path, "does not exist", - cheribuild_target="compiler-rt-builtins") + self.dependency_error( + "Compiler builtins library", path, "does not exist", cheribuild_target="compiler-rt-builtins" + ) return path @property def essential_compiler_and_linker_flags(self) -> "list[str]": if self.compiling_for_cheri(): - return [*self.target_info.get_essential_compiler_and_linker_flags(softfloat=False), - "-Werror=cheri-capability-misuse", "-Werror=shorten-cap-to-int"] + return [ + *self.target_info.get_essential_compiler_and_linker_flags(softfloat=False), + "-Werror=cheri-capability-misuse", + "-Werror=shorten-cap-to-int", + ] return ["--target=" + self.target_info.target_triple] def setup(self): @@ -103,8 +109,11 @@ def setup(self): self.set_make_cmd_with_args("LD", self.target_info.linker, ["--unresolved-symbols=report-all"]) if self.crosscompile_target.is_riscv(include_purecap=True) and self.use_mmu: self.make_args.set(RISCV_MMU="sv39", RISCV_MODE="supervisor") - self.make_args.set(TOOLCHAIN_PREFIX=toolchain_prefix, ARCH_arm64_TOOLCHAIN_PREFIX=toolchain_prefix, - ARCH_riscv64_TOOLCHAIN_PREFIX=toolchain_prefix) + self.make_args.set( + TOOLCHAIN_PREFIX=toolchain_prefix, + ARCH_arm64_TOOLCHAIN_PREFIX=toolchain_prefix, + ARCH_riscv64_TOOLCHAIN_PREFIX=toolchain_prefix, + ) def setup_late(self) -> None: super().setup_late() @@ -121,16 +130,43 @@ def kernel_path(self) -> Path: def run_tests(self): if self.compiling_for_aarch64(include_purecap=True): - cmd = [self.config.qemu_bindir / "qemu-system-aarch64", "-cpu", - "cortex-a53", "-m", "512", "-smp", "1", "-machine", "virt", - "-net", "none", "-nographic", - "-kernel", self.kernel_path] + cmd = [ + self.config.qemu_bindir / "qemu-system-aarch64", + "-cpu", + "cortex-a53", + "-m", + "512", + "-smp", + "1", + "-machine", + "virt", + "-net", + "none", + "-nographic", + "-kernel", + self.kernel_path, + ] elif self.compiling_for_riscv(include_purecap=True): bios_args = ["-bios", "none"] if self.use_mmu: bios_args = riscv_bios_arguments(self.crosscompile_target, self) - cmd = [self.config.qemu_bindir / "qemu-system-riscv64cheri", "-cpu", "rv64", "-m", "512", "-smp", "1", - "-machine", "virt", "-net", "none", "-nographic", "-kernel", self.kernel_path, *bios_args] + cmd = [ + self.config.qemu_bindir / "qemu-system-riscv64cheri", + "-cpu", + "rv64", + "-m", + "512", + "-smp", + "1", + "-machine", + "virt", + "-net", + "none", + "-nographic", + "-kernel", + self.kernel_path, + *bios_args, + ] else: return self.fatal("Unsupported arch") self.run_cmd(cmd, cwd=self.build_dir, give_tty_control=True) diff --git a/pycheribuild/projects/cross/llvm_test_suite.py b/pycheribuild/projects/cross/llvm_test_suite.py index a537082cf..2aa9262e0 100644 --- a/pycheribuild/projects/cross/llvm_test_suite.py +++ b/pycheribuild/projects/cross/llvm_test_suite.py @@ -93,7 +93,8 @@ def setup(self): TEST_SUITE_LIT=self.llvm_lit, TEST_SUITE_COLLECT_CODE_SIZE=self.collect_stats, TEST_SUITE_COLLECT_COMPILE_TIME=self.collect_stats, - TEST_SUITE_COLLECT_STATS=self.collect_stats) + TEST_SUITE_COLLECT_STATS=self.collect_stats, + ) if self.compiling_for_host() and self.target_info.is_linux() and shutil.which("perf") is not None: self.add_cmake_options(TEST_SUITE_USE_PERF=True) @@ -103,8 +104,9 @@ def setup(self): self.add_cmake_options(TEST_SUITE_CXX_LIBRARY="-lc++;-lgcc_eh") self.add_cmake_options(BENCHMARK_USE_LIBCXX=True) if self.can_run_binaries_on_remote_morello_board(): - self.add_cmake_options(TEST_SUITE_RUN_BENCHMARKS=True, - TEST_SUITE_REMOTE_HOST=self.config.remote_morello_board) + self.add_cmake_options( + TEST_SUITE_RUN_BENCHMARKS=True, TEST_SUITE_REMOTE_HOST=self.config.remote_morello_board + ) else: self.add_cmake_options(TEST_SUITE_RUN_BENCHMARKS=False) # Would need to set up custom executor if self.crosscompile_target.is_any_x86(): @@ -148,8 +150,10 @@ class BuildLLVMTestSuiteCheriBSDUpstreamLLVM(BuildLLVMTestSuite): @property def custom_c_preprocessor(self): - return self.llvm_project.get_install_dir( - self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin/clang-cpp" + return ( + self.llvm_project.get_install_dir(self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) + / "bin/clang-cpp" + ) @property def custom_c_compiler(self): @@ -157,5 +161,6 @@ def custom_c_compiler(self): @property def custom_cxx_compiler(self): - return self.llvm_project.get_install_dir( - self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin/clang++" + return ( + self.llvm_project.get_install_dir(self, cross_target=CompilationTargets.NATIVE_NON_PURECAP) / "bin/clang++" + ) diff --git a/pycheribuild/projects/cross/mesa.py b/pycheribuild/projects/cross/mesa.py index 11835cdf1..8abf68ac7 100644 --- a/pycheribuild/projects/cross/mesa.py +++ b/pycheribuild/projects/cross/mesa.py @@ -35,9 +35,11 @@ class BuildLibDrm(CrossCompileMesonProject): target = "libdrm" dependencies = ("libpciaccess", "xorg-pthread-stubs") - repository = GitRepository("https://gitlab.freedesktop.org/mesa/drm.git", - temporary_url_override="https://gitlab.freedesktop.org/arichardson/drm.git", - url_override_reason="Lots of uinptr_t != u64 fun") + repository = GitRepository( + "https://gitlab.freedesktop.org/mesa/drm.git", + temporary_url_override="https://gitlab.freedesktop.org/arichardson/drm.git", + url_override_reason="Lots of uinptr_t != u64 fun", + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): @@ -55,8 +57,10 @@ def setup(self): class BuildLibGlvnd(CrossCompileMesonProject): target = "libglvnd" dependencies = ("libx11", "libxext") - repository = GitRepository("https://gitlab.freedesktop.org/glvnd/libglvnd.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/libglvnd.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/glvnd/libglvnd.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/libglvnd.git"], + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): @@ -73,11 +77,14 @@ def setup(self): class BuildMesa(CrossCompileMesonProject): target = "mesa" - repository = GitRepository("https://gitlab.freedesktop.org/mesa/mesa.git", - temporary_url_override="https://github.com/CTSRD-CHERI/mesa", - url_override_reason="Various changes to allow purecap compilation", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/mesa.git"], - force_branch=True, default_branch="mesa-22.3.7-cheriabi") + repository = GitRepository( + "https://gitlab.freedesktop.org/mesa/mesa.git", + temporary_url_override="https://github.com/CTSRD-CHERI/mesa", + url_override_reason="Various changes to allow purecap compilation", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/mesa.git"], + force_branch=True, + default_branch="mesa-22.3.7-cheriabi", + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE include_x11 = True include_wayland = True @@ -96,8 +103,9 @@ def check_system_dependencies(self): try: self.run_cmd(sys.executable, "-c", "import mako") except subprocess.CalledProcessError: - self.dependency_error("Missing python module mako", - install_instructions=InstallInstructions("pip3 install --user mako")) + self.dependency_error( + "Missing python module mako", install_instructions=InstallInstructions("pip3 install --user mako") + ) def setup(self): super().setup() @@ -116,8 +124,16 @@ def setup(self): meson_args["gallium-drivers"].append("panfrost") # Does not compile yet: meson_args["vulkan-drivers"].append("panfrost") meson_args = {k: str(v) for k, v in meson_args.items()} - self.add_meson_options(gbm="enabled", egl="enabled", glvnd=True, llvm="disabled", osmesa=False, - platforms=str(platforms), _include_empty_vars=True, **meson_args) + self.add_meson_options( + gbm="enabled", + egl="enabled", + glvnd=True, + llvm="disabled", + osmesa=False, + platforms=str(platforms), + _include_empty_vars=True, + **meson_args, + ) # threads_posix.h:274:13: error: releasing mutex 'mtx' that was not held [-Werror,-Wthread-safety-analysis] self.cross_warning_flags.append("-Wno-thread-safety-analysis") # There are quite a lot of -Wcheri-capability-misuse warnings, but for now we just want the library to exist @@ -131,8 +147,9 @@ def setup(self): class BuildLibEpoxy(CrossCompileMesonProject): target = "libepoxy" dependencies = ("libglvnd",) - repository = GitRepository("https://github.com/anholt/libepoxy", - old_urls=[b"https://github.com/arichardson/libepoxy"]) + repository = GitRepository( + "https://github.com/anholt/libepoxy", old_urls=[b"https://github.com/arichardson/libepoxy"] + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): diff --git a/pycheribuild/projects/cross/mpfr.py b/pycheribuild/projects/cross/mpfr.py index 7b52740c9..dbf00be0b 100644 --- a/pycheribuild/projects/cross/mpfr.py +++ b/pycheribuild/projects/cross/mpfr.py @@ -34,9 +34,12 @@ class BuildMpfr(CrossCompileAutotoolsProject): repository = GitRepository("https://gitlab.inria.fr/mpfr/mpfr.git") - supported_architectures = (CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + - CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + - CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + CompilationTargets.ALL_NATIVE) + supported_architectures = ( + CompilationTargets.ALL_CHERIBSD_TARGETS_WITH_HYBRID + + CompilationTargets.ALL_CHERIBSD_HYBRID_FOR_PURECAP_ROOTFS_TARGETS + + CompilationTargets.ALL_SUPPORTED_FREEBSD_TARGETS + + CompilationTargets.ALL_NATIVE + ) native_install_dir = DefaultInstallDir.CHERI_SDK def check_system_dependencies(self) -> None: diff --git a/pycheribuild/projects/cross/mrs.py b/pycheribuild/projects/cross/mrs.py index 04926b8e5..9c2f42b11 100644 --- a/pycheribuild/projects/cross/mrs.py +++ b/pycheribuild/projects/cross/mrs.py @@ -46,32 +46,40 @@ def setup_config_options(cls, **kwargs): cls.build_target = cls.add_config_option("build-target", kind=str, help="specify a target to build, or all") cls.debug = cls.add_bool_option("debug", help="enable debug output") - cls.offload_quarantine = cls.add_bool_option("offload-quarantine", - help="process the quarantine in a separate worker thread") + cls.offload_quarantine = cls.add_bool_option( + "offload-quarantine", help="process the quarantine in a separate worker thread" + ) cls.bypass_quarantine = cls.add_bool_option("bypass-quarantine", help="MADV_FREE freed page-size allocations") cls.clear_on_alloc = cls.add_bool_option("clear-on-alloc", help="zero regions during allocation") cls.clear_on_free = cls.add_bool_option("clear-on-free", help="zero regions as they come out of quarantine") cls.print_stats = cls.add_bool_option("print-stats", help="print heap statistics on exit") cls.print_caprevoke = cls.add_bool_option("print-caprevoke", help="print per-revocation statistics") - cls.concurrent_revocation_passes = cls.add_config_option("concurrent-revocation-passes", kind=int, - help="enable N concurrent revocation passes before " - "the stop-the-world pass") - cls.revoke_on_free = cls.add_bool_option("revoke-on-free", - help="perform revocation on free rather than during allocation " - "routines") + cls.concurrent_revocation_passes = cls.add_config_option( + "concurrent-revocation-passes", + kind=int, + help="enable N concurrent revocation passes before " "the stop-the-world pass", + ) + cls.revoke_on_free = cls.add_bool_option( + "revoke-on-free", help="perform revocation on free rather than during allocation " "routines" + ) cls.just_interpose = cls.add_bool_option("just-interpose", help="just call the real functions") cls.just_bookkeeping = cls.add_bool_option("just-bookkeeping", help="just update data structures") cls.just_quarantine = cls.add_bool_option("just-quarantine", help="do bookkeeping and quarantining") - cls.just_paint_bitmap = cls.add_bool_option("just-paint-bitmap", - help="do bookkeeping, quarantining, and bitmap painting") + cls.just_paint_bitmap = cls.add_bool_option( + "just-paint-bitmap", help="do bookkeeping, quarantining, and bitmap painting" + ) - cls.quarantine_ratio = cls.add_config_option("quarantine-ratio", kind=int, - help="limit the quarantine size to 1/QUARANTINE_RATIO times the " - "size of the heap") - cls.quarantine_highwater = cls.add_config_option("quarantine-highwater", kind=int, - help="limit the quarantine size to QUARANTINE_HIGHWATER " - "bytes (supersedes QUARANTINE_RATIO)") + cls.quarantine_ratio = cls.add_config_option( + "quarantine-ratio", + kind=int, + help="limit the quarantine size to 1/QUARANTINE_RATIO times the " "size of the heap", + ) + cls.quarantine_highwater = cls.add_config_option( + "quarantine-highwater", + kind=int, + help="limit the quarantine size to QUARANTINE_HIGHWATER " "bytes (supersedes QUARANTINE_RATIO)", + ) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/neomutt.py b/pycheribuild/projects/cross/neomutt.py index 9cbc49771..362ad2c3f 100644 --- a/pycheribuild/projects/cross/neomutt.py +++ b/pycheribuild/projects/cross/neomutt.py @@ -29,15 +29,14 @@ class BuildNeomutt(CrossCompileAutotoolsProject): repository = GitRepository("https://github.com/neomutt/neomutt.git") - dependencies = ('libxml2',) + dependencies = ("libxml2",) def setup(self): super().setup() # neomutt's build system doesn't use autotools, it justs pretends to look the same # - but it doesn't implement the --target option, so we strip it - self.configure_args[:] = [arg for arg in self.configure_args if not arg.startswith('--target=')] + self.configure_args[:] = [arg for arg in self.configure_args if not arg.startswith("--target=")] # enable OpenSSL (in base system), disable internationalisation libs we don't have - self.configure_args.extend(['--disable-nls', '--disable-idn', - '--disable-doc', '--ssl']) + self.configure_args.extend(["--disable-nls", "--disable-idn", "--disable-doc", "--ssl"]) diff --git a/pycheribuild/projects/cross/newlib.py b/pycheribuild/projects/cross/newlib.py index 6a442d1b5..4fa5446a3 100644 --- a/pycheribuild/projects/cross/newlib.py +++ b/pycheribuild/projects/cross/newlib.py @@ -84,7 +84,7 @@ def add_configure_vars(self, **kwargs): self.add_configure_env_arg(k, v) # self.make_args.env_vars[k] = str(v) if k.endswith("_FOR_BUILD"): - k2 = k[0:-len("_FOR_BUILD")] + k2 = k[0 : -len("_FOR_BUILD")] self.add_configure_env_arg(k2, v) # self.make_args.env_vars[k2] = str(v) @@ -101,10 +101,13 @@ def setup(self): AS_FOR_TARGET=str(self.CC), # + target_cflags, CC_FOR_TARGET=str(self.CC), # + target_cflags, CXX_FOR_TARGET=str(self.CXX), # + target_cflags, - AR_FOR_TARGET=self.target_info.ar, STRIP_FOR_TARGET=self.target_info.strip_tool, - OBJCOPY_FOR_TARGET=bindir / "objcopy", RANLIB_FOR_TARGET=self.target_info.ranlib, + AR_FOR_TARGET=self.target_info.ar, + STRIP_FOR_TARGET=self.target_info.strip_tool, + OBJCOPY_FOR_TARGET=bindir / "objcopy", + RANLIB_FOR_TARGET=self.target_info.ranlib, OBJDUMP_FOR_TARGET=bindir / "llvm-objdump", - READELF_FOR_TARGET=bindir / "readelf", NM_FOR_TARGET=self.target_info.nm, + READELF_FOR_TARGET=bindir / "readelf", + NM_FOR_TARGET=self.target_info.nm, # Set all the flags: CFLAGS_FOR_TARGET=target_cflags, CCASFLAGS_FOR_TARGET=target_cflags, @@ -115,7 +118,7 @@ def setup(self): CPP_FOR_BUILD=self.host_CPP, LD_FOR_TARGET=str(self.target_info.linker), LDFLAGS_FOR_TARGET=commandline_to_str(self.default_ldflags), - ) + ) if self.compiling_for_mips(include_purecap=True): # long double is the same as double for MIPS @@ -127,36 +130,36 @@ def setup(self): self.configure_args.append("--disable-libgloss") if self.target_info.is_baremetal(): - self.configure_args.extend([ - "--enable-malloc-debugging", - "--enable-newlib-long-time_t", # we want time_t to be long and not int! - "--enable-newlib-io-c99-formats", - "--enable-newlib-io-long-long", - # --enable-newlib-io-pos-args (probably not needed) - "--disable-newlib-io-long-double", # we don't need this, MIPS long double == double - "--enable-newlib-io-float", - # "--disable-newlib-supplied-syscalls" - "--disable-libstdcxx", # not sure if this is needed - - # we don't have any multithreading support on baremetal - "--disable-newlib-multithread", - - "--enable-newlib-global-atexit", # TODO: is this needed? - # --enable-newlib-nano-malloc (should we do this?) - "--disable-multilib", - - # TODO: smaller lib? "--enable-target-optspace" - - # FIXME: these don't seem to work - "--enable-serial-build-configure", - "--enable-serial-target-configure", - "--enable-serial-host-configure", - ]) + self.configure_args.extend( + [ + "--enable-malloc-debugging", + "--enable-newlib-long-time_t", # we want time_t to be long and not int! + "--enable-newlib-io-c99-formats", + "--enable-newlib-io-long-long", + # --enable-newlib-io-pos-args (probably not needed) + "--disable-newlib-io-long-double", # we don't need this, MIPS long double == double + "--enable-newlib-io-float", + # "--disable-newlib-supplied-syscalls" + "--disable-libstdcxx", # not sure if this is needed + # we don't have any multithreading support on baremetal + "--disable-newlib-multithread", + "--enable-newlib-global-atexit", # TODO: is this needed? + # --enable-newlib-nano-malloc (should we do this?) + "--disable-multilib", + # TODO: smaller lib? "--enable-target-optspace" + # FIXME: these don't seem to work + "--enable-serial-build-configure", + "--enable-serial-target-configure", + "--enable-serial-host-configure", + ] + ) elif self.target_info.is_rtems(): - self.configure_args.extend([ - "--enable-newlib-io-c99-formats", - "--disable-libstdcxx", # not sure if this is needed - ]) + self.configure_args.extend( + [ + "--enable-newlib-io-c99-formats", + "--disable-libstdcxx", # not sure if this is needed + ] + ) if self.locale_support: # needed for locale support @@ -182,21 +185,37 @@ def install(self, **kwargs): def run_tests(self): with tempfile.TemporaryDirectory(prefix="cheribuild-" + self.target + "-") as td: - self.write_file(Path(td, "main.c"), contents=""" + self.write_file( + Path(td, "main.c"), + contents=""" #include int main(int argc, char** argv) { for (int i = 0; i < argc; i++) { printf("argv[%d] = '%s'\\n", i, argv[i]); } } -""", overwrite=True) +""", + overwrite=True, + ) test_exe = Path(td, "test.exe") # FIXME: CHERI helloworld - compiler_flags = self.essential_compiler_and_linker_flags + self.COMMON_FLAGS + [ - "-Wl,-T,qemu-malta.ld", "-Wl,-verbose", "--sysroot=" + str(self.sdk_sysroot)] + compiler_flags = ( + self.essential_compiler_and_linker_flags + + self.COMMON_FLAGS + + ["-Wl,-T,qemu-malta.ld", "-Wl,-verbose", "--sysroot=" + str(self.sdk_sysroot)] + ) self.run_cmd([self.sdk_bindir / "clang", "main.c", "-o", test_exe, *compiler_flags, "-###"], cwd=td) self.run_cmd([self.sdk_bindir / "clang", "main.c", "-o", test_exe, *compiler_flags], cwd=td) self.run_cmd(self.sdk_bindir / "llvm-readobj", "-h", test_exe) from ..build_qemu import BuildQEMU - self.run_cmd(self.sdk_sysroot / "bin/run_with_qemu.py", "--qemu", BuildQEMU.qemu_binary(self), - "--timeout", "20", test_exe, "HELLO", "WORLD") + + self.run_cmd( + self.sdk_sysroot / "bin/run_with_qemu.py", + "--qemu", + BuildQEMU.qemu_binary(self), + "--timeout", + "20", + test_exe, + "HELLO", + "WORLD", + ) diff --git a/pycheribuild/projects/cross/nginx.py b/pycheribuild/projects/cross/nginx.py index 8baed06ea..cf02f8a22 100644 --- a/pycheribuild/projects/cross/nginx.py +++ b/pycheribuild/projects/cross/nginx.py @@ -48,12 +48,15 @@ def setup(self): super().setup() self.configure_command = self.source_dir / "auto/configure" if not self.compiling_for_host(): - self.COMMON_FLAGS.extend(["-pedantic", - "-Wno-gnu-statement-expression", - "-Wno-flexible-array-extensions", # TODO: could this cause errors? - # "-Wno-extended-offsetof", - "-Wno-format-pedantic", - ]) + self.COMMON_FLAGS.extend( + [ + "-pedantic", + "-Wno-gnu-statement-expression", + "-Wno-flexible-array-extensions", # TODO: could this cause errors? + # "-Wno-extended-offsetof", + "-Wno-format-pedantic", + ] + ) self.configure_environment["AR"] = str(self.sdk_bindir / "cheri-unknown-freebsd-ar") # The makefile expects the current working directory to be the source dir. Therefore we add -f $build/Makefile # This is also in the makefile generated in the source dir but it doesn't work with multiple build dirs @@ -67,9 +70,10 @@ def install(self, **kwargs): # install the benchmark script benchmark = self.read_file(self.source_dir / "nginx-benchmark.sh") if not self.compiling_for_host(): - benchmark = re.sub(r'NGINX=.*', "NGINX=\"" + str(self.install_prefix / "sbin/nginx") + "\"", benchmark) - benchmark = re.sub(r'FETCHBENCH=.*', "FETCHBENCH=\"" + str(self.install_prefix / "sbin/fetchbench") + "\"", - benchmark) + benchmark = re.sub(r"NGINX=.*", 'NGINX="' + str(self.install_prefix / "sbin/nginx") + '"', benchmark) + benchmark = re.sub( + r"FETCHBENCH=.*", 'FETCHBENCH="' + str(self.install_prefix / "sbin/fetchbench") + '"', benchmark + ) self.write_file(self.real_install_root_dir / "nginx-benchmark.sh", benchmark, overwrite=True, mode=0o755) def needs_configure(self): @@ -78,21 +82,28 @@ def needs_configure(self): def configure(self): if self.should_include_debug_info: self.configure_args.append("--with-debug") - self.configure_args.extend(["--without-pcre", - "--without-http_rewrite_module", - "--with-http_v2_module", - "--with-http_ssl_module", - "--without-http_gzip_module", - "--without-http_rewrite_module", - "--without-pcre", - "--builddir=" + str(self.build_dir)]) + self.configure_args.extend( + [ + "--without-pcre", + "--without-http_rewrite_module", + "--with-http_v2_module", + "--with-http_ssl_module", + "--without-http_gzip_module", + "--without-http_rewrite_module", + "--without-pcre", + "--builddir=" + str(self.build_dir), + ] + ) if not self.compiling_for_host(): self.LDFLAGS.append("-v") - self.configure_args.extend(["--crossbuild=FreeBSD:12.0-CURRENT:mips", - "--with-cc-opt=" + self.commandline_to_str(self.default_compiler_flags), - "--with-ld-opt=" + self.commandline_to_str(self.default_ldflags), - "--sysroot=" + str(self.sdk_sysroot), - ]) + self.configure_args.extend( + [ + "--crossbuild=FreeBSD:12.0-CURRENT:mips", + "--with-cc-opt=" + self.commandline_to_str(self.default_compiler_flags), + "--with-ld-opt=" + self.commandline_to_str(self.default_ldflags), + "--sysroot=" + str(self.sdk_sysroot), + ] + ) self.configure_environment["CC_TEST_FLAGS"] = self.commandline_to_str(self.default_compiler_flags) self.configure_environment["NGX_TEST_LD_OPT"] = self.commandline_to_str(self.default_ldflags) self.configure_environment["NGX_SIZEOF_int"] = "4" diff --git a/pycheribuild/projects/cross/opensbi.py b/pycheribuild/projects/cross/opensbi.py index d3998add9..02b05739f 100644 --- a/pycheribuild/projects/cross/opensbi.py +++ b/pycheribuild/projects/cross/opensbi.py @@ -61,11 +61,12 @@ class BuildOpenSBI(Project): CompilationTargets.FREESTANDING_RISCV64_HYBRID, CompilationTargets.FREESTANDING_RISCV64, # Won't compile yet: CompilationTargets.FREESTANDING_RISCV64_PURECAP - ) + ) make_kind = MakeCommandKind.GnuMake _always_add_suffixed_targets = True - _default_install_dir_fn = ComputedDefaultValue(function=opensbi_install_dir, - as_string="$SDK_ROOT/opensbi/riscv{32,64}{-hybrid,-purecap,}") + _default_install_dir_fn = ComputedDefaultValue( + function=opensbi_install_dir, as_string="$SDK_ROOT/opensbi/riscv{32,64}{-hybrid,-purecap,}" + ) @classproperty def needs_sysroot(self): @@ -99,7 +100,7 @@ def setup(self): PLATFORM_RISCV_ABI=self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True), PLATFORM_RISCV_ISA=self.target_info.get_riscv_arch_string(self.crosscompile_target, softfloat=True), PLATFORM_RISCV_XLEN=64, - ) + ) if self.config.verbose: self.make_args.set(V=True) @@ -135,14 +136,21 @@ def install(self, **kwargs): # Install into the QEMU firware directory so that `-bios default` works qemu_fw_dir = BuildQEMU.get_install_dir(self, cross_target=CompilationTargets.NATIVE) / "share/qemu/" self.makedirs(qemu_fw_dir) - self.run_cmd(self.sdk_bindir / "llvm-objcopy", "-S", "-O", "binary", - self._fw_jump_path(), qemu_fw_dir / "opensbi-riscv64cheri-virt-fw_jump.bin", - print_verbose_only=False) + self.run_cmd( + self.sdk_bindir / "llvm-objcopy", + "-S", + "-O", + "binary", + self._fw_jump_path(), + qemu_fw_dir / "opensbi-riscv64cheri-virt-fw_jump.bin", + print_verbose_only=False, + ) def _fw_jump_path(self) -> Path: # share/opensbi/lp64/generic/firmware//fw_payload.bin return self.install_dir / "share/opensbi/{abi}/generic/firmware/fw_jump.elf".format( - abi=self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True)) + abi=self.target_info.get_riscv_abi(self.crosscompile_target, softfloat=True) + ) @classmethod def get_nocap_instance(cls, caller, cpu_arch=CPUArchitecture.RISCV64) -> "BuildOpenSBI": @@ -170,20 +178,27 @@ class BuildOpenSBIGFE(BuildOpenSBI): def setup(self): super().setup() - self.make_args.set(FW_TEXT_START=0xc0000000) + self.make_args.set(FW_TEXT_START=0xC0000000) class BuildUpstreamOpenSBI(BuildOpenSBI): target = "upstream-opensbi" _default_install_dir_fn = ComputedDefaultValue( function=lambda config, p: config.cheri_sdk_dir / "upstream-opensbi/riscv64", - as_string="$SDK_ROOT/upstream-opensbi/riscv64") + as_string="$SDK_ROOT/upstream-opensbi/riscv64", + ) repository = GitRepository("https://github.com/riscv-software-src/opensbi.git") supported_architectures = (CompilationTargets.FREESTANDING_RISCV64,) def run_tests(self): options = QemuOptions(self.crosscompile_target) - self.run_cmd(options.get_commandline( - qemu_command=BuildQEMU.qemu_binary(self), add_network_device=False, bios_args=["-bios", "none"], - kernel_file=self.install_dir / "share/opensbi/lp64/generic/firmware//fw_payload.elf"), - give_tty_control=True, cwd="/") + self.run_cmd( + options.get_commandline( + qemu_command=BuildQEMU.qemu_binary(self), + add_network_device=False, + bios_args=["-bios", "none"], + kernel_file=self.install_dir / "share/opensbi/lp64/generic/firmware//fw_payload.elf", + ), + give_tty_control=True, + cwd="/", + ) diff --git a/pycheribuild/projects/cross/openssl.py b/pycheribuild/projects/cross/openssl.py index b7df43bfa..b452fb112 100644 --- a/pycheribuild/projects/cross/openssl.py +++ b/pycheribuild/projects/cross/openssl.py @@ -38,8 +38,9 @@ class BuildOpenSSL(CrossCompileProject): # nginx looks in /usr/local path_in_rootfs = "/usr/local" - repository = GitRepository("https://github.com/CTSRD-CHERI/openssl.git", - default_branch="OpenSSL_1_1_1-stable-cheri") + repository = GitRepository( + "https://github.com/CTSRD-CHERI/openssl.git", default_branch="OpenSSL_1_1_1-stable-cheri" + ) native_install_dir = DefaultInstallDir.DO_NOT_INSTALL cross_install_dir = DefaultInstallDir.ROOTFS_OPTBASE diff --git a/pycheribuild/projects/cross/picolibc.py b/pycheribuild/projects/cross/picolibc.py index 635087300..93f2aa8dd 100644 --- a/pycheribuild/projects/cross/picolibc.py +++ b/pycheribuild/projects/cross/picolibc.py @@ -120,7 +120,9 @@ def compile(self, **kwargs): # Symlink libgcc.a to the build dir to allow linking against it without adding all of /lib. self.makedirs(self.build_dir / "local-libgcc") self.create_symlink( - self.sdk_sysroot / "lib/libgcc.a", self.build_dir / "local-libgcc/libgcc.a", print_verbose_only=False, + self.sdk_sysroot / "lib/libgcc.a", + self.build_dir / "local-libgcc/libgcc.a", + print_verbose_only=False, ) super().compile(**kwargs) diff --git a/pycheribuild/projects/cross/pkg.py b/pycheribuild/projects/cross/pkg.py index 9c06b49d0..0d21b11ba 100644 --- a/pycheribuild/projects/cross/pkg.py +++ b/pycheribuild/projects/cross/pkg.py @@ -57,8 +57,11 @@ def setup(self): if self.build_type.should_include_debug_info: self.COMMON_FLAGS.append("-g") - self.make_args.set_env(CPPFLAGS=self.commandline_to_str( - self.COMMON_FLAGS + self.compiler_warning_flags + self.optimization_flags + self.COMMON_FLAGS)) + self.make_args.set_env( + CPPFLAGS=self.commandline_to_str( + self.COMMON_FLAGS + self.compiler_warning_flags + self.optimization_flags + self.COMMON_FLAGS + ) + ) self.make_args.set_env(LDFLAGS=self.commandline_to_str(self.default_ldflags)) def compile(self, **kwargs): diff --git a/pycheribuild/projects/cross/postgres.py b/pycheribuild/projects/cross/postgres.py index 854868f46..42115cc15 100644 --- a/pycheribuild/projects/cross/postgres.py +++ b/pycheribuild/projects/cross/postgres.py @@ -35,8 +35,9 @@ class BuildPostgres(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/postgres.git", - default_branch="96-cheri", force_branch=True) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/postgres.git", default_branch="96-cheri", force_branch=True + ) # we have to build in the source directory, out-of-source is broken # build_in_source_dir = True make_kind = MakeCommandKind.GnuMake @@ -50,9 +51,14 @@ def setup(self): # self.COMMON_FLAGS.append("-DLOCK_DEBUG=1") self.configure_args.append("--enable-cassert") - self.common_warning_flags.extend(["-pedantic", "-Wno-gnu-statement-expression", - "-Wno-flexible-array-extensions", # TODO: could this cause errors? - "-Wno-format-pedantic"]) + self.common_warning_flags.extend( + [ + "-pedantic", + "-Wno-gnu-statement-expression", + "-Wno-flexible-array-extensions", # TODO: could this cause errors? + "-Wno-format-pedantic", + ] + ) self.LDFLAGS.append("-pthread") if OSInfo.IS_FREEBSD and self.compiling_for_host(): # Something werid is happending with the locale code (somehow not being built -FPIC?): @@ -99,10 +105,11 @@ def install(self, **kwargs): pg_root = str(self.install_prefix) else: pg_root = str(self.install_dir) - benchmark = re.sub(r'POSTGRES_ROOT=".*"', "POSTGRES_ROOT=\"" + pg_root + "\"", benchmark) + benchmark = re.sub(r'POSTGRES_ROOT=".*"', 'POSTGRES_ROOT="' + pg_root + '"', benchmark) self.write_file(self.real_install_root_dir / benchname, benchmark, overwrite=True, mode=0o755) - self.install_file(self.source_dir / "run-postgres-tests.sh", - self.real_install_root_dir / "run-postgres-tests.sh") + self.install_file( + self.source_dir / "run-postgres-tests.sh", self.real_install_root_dir / "run-postgres-tests.sh" + ) @property def default_ldflags(self): @@ -124,9 +131,14 @@ def run_tests(self): # self.run_make("check", cwd=self.build_dir / "src/interfaces/ecpg/test", stdout_filter=None) else: locale_dir = self.rootfs_dir / "usr/share/locale" - self.target_info.run_cheribsd_test_script("run_postgres_tests.py", "--smb-mount-directory", - str(self.install_dir) + ":" + str(self.install_prefix), - "--locale-files-dir", locale_dir, mount_builddir=False, - # long running test -> speed up by using a kernel without - # invariants - use_benchmark_kernel_by_default=True) + self.target_info.run_cheribsd_test_script( + "run_postgres_tests.py", + "--smb-mount-directory", + str(self.install_dir) + ":" + str(self.install_prefix), + "--locale-files-dir", + locale_dir, + mount_builddir=False, + # long running test -> speed up by using a kernel without + # invariants + use_benchmark_kernel_by_default=True, + ) diff --git a/pycheribuild/projects/cross/python.py b/pycheribuild/projects/cross/python.py index 3848bb74e..ee644b6d3 100644 --- a/pycheribuild/projects/cross/python.py +++ b/pycheribuild/projects/cross/python.py @@ -72,12 +72,16 @@ def configure(self, **kwargs): if not self.compiling_for_host(): self.configure_args.append("--without-doc-strings") # should reduce size - native_python = self.get_instance_for_cross_target(CompilationTargets.NATIVE_NON_PURECAP, - self.config).install_dir / "bin/python3" + native_python = ( + self.get_instance_for_cross_target(CompilationTargets.NATIVE_NON_PURECAP, self.config).install_dir + / "bin/python3" + ) if not native_python.exists(): - self.dependency_error("Native python3 doesn't exist, you must build the `python-native` target first.", - cheribuild_target="python", - cheribuild_xtarget=CompilationTargets.NATIVE_NON_PURECAP) + self.dependency_error( + "Native python3 doesn't exist, you must build the `python-native` target first.", + cheribuild_target="python", + cheribuild_xtarget=CompilationTargets.NATIVE_NON_PURECAP, + ) self.add_configure_vars( ac_cv_buggy_getaddrinfo="no", # Doesn't work since that remove all flags, need to set PATH instead @@ -98,10 +102,21 @@ def run_tests(self): # python build system adds .exe for case-insensitive dirs suffix = "" if is_case_sensitive_dir(self.build_dir) else ".exe" if self.compiling_for_host(): - self.run_cmd(self.build_dir / ("python" + suffix), "-m", "test", "-w", "--junit-xml=python-tests.xml", - self.config.make_j_flag, cwd=self.build_dir) + self.run_cmd( + self.build_dir / ("python" + suffix), + "-m", + "test", + "-w", + "--junit-xml=python-tests.xml", + self.config.make_j_flag, + cwd=self.build_dir, + ) else: # Python executes tons of system calls, hopefully using the benchmark kernel helps - self.target_info.run_cheribsd_test_script("run_python_tests.py", "--buildexe-suffix=" + suffix, - mount_installdir=True, - mount_sourcedir=True, use_benchmark_kernel_by_default=True) + self.target_info.run_cheribsd_test_script( + "run_python_tests.py", + "--buildexe-suffix=" + suffix, + mount_installdir=True, + mount_sourcedir=True, + use_benchmark_kernel_by_default=True, + ) diff --git a/pycheribuild/projects/cross/qt5.py b/pycheribuild/projects/cross/qt5.py index b3c4f114b..793725fcf 100644 --- a/pycheribuild/projects/cross/qt5.py +++ b/pycheribuild/projects/cross/qt5.py @@ -129,16 +129,21 @@ def check_system_dependencies(self) -> None: super().check_system_dependencies() self.check_required_system_tool("xmlto", homebrew="xmlto", apt="xmlto") self.check_required_system_tool( - "xmllint", homebrew="libxml2", apt="libxml2-utils", cheribuild_target="libxml2-native", + "xmllint", + homebrew="libxml2", + apt="libxml2-utils", + cheribuild_target="libxml2-native", ) self.check_required_system_tool("msgfmt", freebsd="gettext-tools") # no way to disable translations def setup(self): super().setup() - self.add_meson_options(**{ - "update-mimedb": True, - "build-tools": self._can_build_tools, - }) + self.add_meson_options( + **{ + "update-mimedb": True, + "build-tools": self._can_build_tools, + } + ) if not self._can_build_tools: # Ensure that we have update-mime-database available as it will be used in a post-install action. self.get_update_mime_database_path(self) @@ -184,21 +189,23 @@ def dependencies(cls, config: CheriConfig) -> "tuple[str, ...]": return tuple(deps) if cls.use_x11: # The system X11 libraries might be too old, so add the cheribuild-provided ones as a dependency - deps.extend([ - "libx11", - "libxcb", - "libxkbcommon", - "libxcb-cursor", - "libxcb-util", - "libxcb-image", - "libice", - "libsm", - "libxext", - "libxtst", - "libxcb-render-util", - "libxcb-wm", - "libxcb-keysyms", - ]) + deps.extend( + [ + "libx11", + "libxcb", + "libxkbcommon", + "libxcb-cursor", + "libxcb-util", + "libxcb-image", + "libice", + "libsm", + "libxext", + "libxtst", + "libxcb-render-util", + "libxcb-wm", + "libxcb-keysyms", + ] + ) # Always use our patched image/sql libraries instead of the host ones: deps.extend(["libpng", "libjpeg-turbo"]) if not cls.get_crosscompile_target().is_native(): @@ -243,10 +250,16 @@ def setup_config_options(cls, **kwargs): # Link against X11 libs by default if we aren't compiling for macOS native_is_macos = cls._xtarget is not None and cls._xtarget.target_info_cls.is_macos() cls.use_x11 = cls.add_bool_option( - "use-x11", default=not native_is_macos, show_help=False, help="Build Qt with the XCB backend.", + "use-x11", + default=not native_is_macos, + show_help=False, + help="Build Qt with the XCB backend.", ) cls.use_opengl = cls.add_bool_option( - "use-opengl", default=True, show_help=False, help="Build Qt with OpenGL support", + "use-opengl", + default=True, + show_help=False, + help="Build Qt with OpenGL support", ) def configure(self, **kwargs): @@ -285,23 +298,25 @@ def configure(self, **kwargs): linker_flags = filter(lambda s: not s.startswith("--sysroot"), linker_flags) compiler_flags = filter(lambda s: not s.startswith("--sysroot"), compiler_flags) cross_tools_prefix = self.target_info.get_target_triple(include_version=False) - self.configure_args.extend([ - "-device", - "freebsd-generic-clang", - "-device-option", - f"CROSS_COMPILE={self.sdk_bindir}/{cross_tools_prefix}-", - "-device-option", - "COMPILER_FLAGS=" + self.commandline_to_str(compiler_flags), - "-device-option", - "LINKER_FLAGS=" + self.commandline_to_str(linker_flags), - "-sysroot", - self.cross_sysroot_path, - "-prefix", - self.install_prefix, - # The prefix for host tools such as qmake - "-hostprefix", - str(self.qt_host_tools_path), - ]) + self.configure_args.extend( + [ + "-device", + "freebsd-generic-clang", + "-device-option", + f"CROSS_COMPILE={self.sdk_bindir}/{cross_tools_prefix}-", + "-device-option", + "COMPILER_FLAGS=" + self.commandline_to_str(compiler_flags), + "-device-option", + "LINKER_FLAGS=" + self.commandline_to_str(linker_flags), + "-sysroot", + self.cross_sysroot_path, + "-prefix", + self.install_prefix, + # The prefix for host tools such as qmake + "-hostprefix", + str(self.qt_host_tools_path), + ] + ) xcb = BuildLibXCB.get_instance(self) if xcb.install_prefix != self.install_prefix: self.configure_args.append( @@ -316,25 +331,25 @@ def configure(self, **kwargs): if self.use_asan: self.configure_args.extend(["-sanitize", "address", "-sanitize", "undefined"]) - self.configure_args.extend([ - ( - # To ensure the host and cross-compiled version is the same also disable opengl - "-opengl" - if self.use_opengl - else "-no-opengl" - ), - # Since the cross-compiled version doesn't have glib, also disable it for the native on - "-no-glib", - # Needed for webkit: - # "-icu", - # "-no-Werror", - "-no-use-gold-linker", - "-no-iconv", - "-no-headersclean", - # Don't embed the mimetype DB in libQt5Core.so. It's huge and results in lots XML parsing. Instead, we just - # ensure that the binary cache exists in the disk image. - "-no-mimetype-database", - ]) + self.configure_args.extend( + [ + ( + # To ensure the host and cross-compiled version is the same also disable opengl + "-opengl" if self.use_opengl else "-no-opengl" + ), + # Since the cross-compiled version doesn't have glib, also disable it for the native on + "-no-glib", + # Needed for webkit: + # "-icu", + # "-no-Werror", + "-no-use-gold-linker", + "-no-iconv", + "-no-headersclean", + # Don't embed the mimetype DB in libQt5Core.so. It's huge and results in lots XML parsing. + # Instead, we just ensure that the binary cache exists in the disk image. + "-no-mimetype-database", + ] + ) if self.build_tests: self.configure_args.append("-developer-build") if self.target_info.is_macos(): @@ -387,16 +402,18 @@ def configure(self, **kwargs): # self.configure_args.append("-reduce-relocations") if self.minimal: - self.configure_args.extend([ - "-no-widgets", - "-no-glib", - "-no-gtk", - "-no-opengl", - "-no-cups", - "-no-syslog", - "-no-gui", - "-no-iconv", - ]) + self.configure_args.extend( + [ + "-no-widgets", + "-no-glib", + "-no-gtk", + "-no-opengl", + "-no-cups", + "-no-syslog", + "-no-gui", + "-no-iconv", + ] + ) else: self.configure_args.append("-dbus") # we want to build QtDBus if not self.target_info.is_macos(): @@ -436,7 +453,10 @@ class BuildQtBaseDev(CrossCompileCMakeProject): gui: bool = BoolConfigOption("gui", show_help=True, default=True, help="Include QtGui") use_opengl: bool = BoolConfigOption("opengl", show_help=True, default=False, help="Include QtOpenGl") minimal: bool = BoolConfigOption( - "minimal", show_help=True, default=True, help="Don't build QtWidgets or QtSql, etc", + "minimal", + show_help=True, + default=True, + help="Don't build QtWidgets or QtSql, etc", ) @classmethod @@ -446,22 +466,24 @@ def dependencies(cls, config: CheriConfig) -> "tuple[str, ...]": deps.append(BuildSharedMimeInfo.get_class_for_target(rootfs_target).target) if cls.gui: # The system X11 libraries might be too old, so add the cheribuild-provided ones as a dependency - deps.extend([ - "libx11", - "libxkbcommon", - "libinput", - "libxcb", - "libxcb-cursor", - "libxcb-util", - "libxcb-image", - "libice", - "libsm", - "libxext", - "libxtst", - "libxcb-render-util", - "libxcb-wm", - "libxcb-keysyms", - ]) + deps.extend( + [ + "libx11", + "libxkbcommon", + "libinput", + "libxcb", + "libxcb-cursor", + "libxcb-util", + "libxcb-image", + "libice", + "libsm", + "libxext", + "libxtst", + "libxcb-render-util", + "libxcb-wm", + "libxcb-keysyms", + ] + ) # Always use our patched image/sql libraries instead of the host ones: deps.extend(["libpng", "libjpeg-turbo"]) if not cls.get_crosscompile_target().is_native(): @@ -573,7 +595,10 @@ def run_tests(self): else: # TODO: run `ctest --show-only=json-v1` to get list of tests self.target_info.run_cheribsd_test_script( - "run_qtbase_tests.py", use_benchmark_kernel_by_default=False, mount_sysroot=True, mount_sourcedir=True, + "run_qtbase_tests.py", + use_benchmark_kernel_by_default=False, + mount_sysroot=True, + mount_sourcedir=True, ) @@ -585,7 +610,9 @@ class BuildQt5(BuildQtWithConfigureScript): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.all_modules = cls.add_bool_option( - "all-modules", show_help=True, help="Build all modules (even those that don't make sense for CHERI)", + "all-modules", + show_help=True, + help="Build all modules (even those that don't make sense for CHERI)", ) def configure(self, **kwargs): @@ -670,7 +697,8 @@ def install(self, **kwargs): self.clean_directory(qt_fonts_dir, ensure_dir_exists=False) self.makedirs(qt_fonts_dir.parent) dejavu_fonts = InstallDejaVuFonts.get_instance( - self, cross_target=self.crosscompile_target.get_rootfs_target(), + self, + cross_target=self.crosscompile_target.get_rootfs_target(), ) self.create_symlink(dejavu_fonts.fonts_dir, qt_fonts_dir, print_verbose_only=False) @@ -700,7 +728,9 @@ def run_tests(self): print_verbose_only=False, ) self.create_symlink( - mimedb_tests_dir / "shared-mime-info-2.1", mimedb_tests_dir / "s-m-i", print_verbose_only=False, + mimedb_tests_dir / "shared-mime-info-2.1", + mimedb_tests_dir / "s-m-i", + print_verbose_only=False, ) self._compile_relevant_tests() if self.compiling_for_host(): @@ -800,15 +830,19 @@ class BuildQtMacExtras(BuildQtModuleWithQMake): class BuildQtDeclarative(BuildQtModuleWithQMake): target = "qtdeclarative" repository = GitRepository( - "https://github.com/CTSRD-CHERI/qtdeclarative.git", default_branch="5.15", force_branch=True, + "https://github.com/CTSRD-CHERI/qtdeclarative.git", + default_branch="5.15", + force_branch=True, ) def setup(self): super().setup() - self.configure_args.extend([ - "-no-qml-debug", # debugger not compatibale with CHERI purecap - "-quick-designer", # needed for quickcontrols2 - ]) + self.configure_args.extend( + [ + "-no-qml-debug", # debugger not compatibale with CHERI purecap + "-quick-designer", # needed for quickcontrols2 + ] + ) class BuildQtTools(BuildQtModuleWithQMake): @@ -825,14 +859,16 @@ def setup(self): super().setup() # No need to build all the developer GUI tools, we only want programs that are # useful inside the disk image. - self.configure_args.extend([ - "-no-feature-assistant", - # Some KDE programs/libraries install designer plugins, so for now we install it by default. - # This avoids having to patch those projects to make the feature optional. - # "-no-feature-designer", - # kColorPicker (which is used by Gwenview needs linguist, so we build this too). - # "-no-feature-linguist", - ]) + self.configure_args.extend( + [ + "-no-feature-assistant", + # Some KDE programs/libraries install designer plugins, so for now we install it by default. + # This avoids having to patch those projects to make the feature optional. + # "-no-feature-designer", + # kColorPicker (which is used by Gwenview needs linguist, so we build this too). + # "-no-feature-linguist", + ] + ) class BuildQtWayland(BuildQtModuleWithQMake): @@ -848,10 +884,12 @@ class BuildQtWayland(BuildQtModuleWithQMake): def setup(self): super().setup() - self.configure_args.extend([ - "-feature-wayland-client", - "-feature-wayland-server", - ]) + self.configure_args.extend( + [ + "-feature-wayland-client", + "-feature-wayland-server", + ] + ) # def compile(self, **kwargs): # self.run_make() @@ -1002,7 +1040,9 @@ def setup(self): class BuildQtWebkit(CrossCompileCMakeProject): repository = GitRepository( - "https://github.com/CTSRD-CHERI/qtwebkit", default_branch="qtwebkit-5.212-cheri", force_branch=True, + "https://github.com/CTSRD-CHERI/qtwebkit", + default_branch="qtwebkit-5.212-cheri", + force_branch=True, ) is_large_source_repository = True dependencies = ("qtbase", "icu4c", "libxml2", "sqlite") @@ -1087,7 +1127,9 @@ def setup(self): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.build_jsc_only = cls.add_bool_option( - "build-jsc-only", show_help=True, help="only build the JavaScript interpreter executable", + "build-jsc-only", + show_help=True, + help="only build the JavaScript interpreter executable", ) def compile(self, **kwargs): @@ -1096,7 +1138,10 @@ def compile(self, **kwargs): update_mime = BuildSharedMimeInfo.get_update_mime_database_path(self) mime_info_src = BuildQtBase.get_source_dir(self) / "src/corelib/mimetypes/mime/packages/freedesktop.org.xml" self.install_file( - mime_info_src, Path(td, "mime/packages/freedesktop.org.xml"), force=True, print_verbose_only=False, + mime_info_src, + Path(td, "mime/packages/freedesktop.org.xml"), + force=True, + print_verbose_only=False, ) self.run_cmd(update_mime, "-V", Path(td, "mime"), cwd="/") @@ -1104,10 +1149,16 @@ def compile(self, **kwargs): self.fatal("Could not generated shared-mime-info cache!") # install mime.cache and freedesktop.org.xml into the build dir for tests self.install_file( - mime_info_src, self.build_dir / "freedesktop.org.xml", force=True, print_verbose_only=False, + mime_info_src, + self.build_dir / "freedesktop.org.xml", + force=True, + print_verbose_only=False, ) self.install_file( - Path(td, "mime/mime.cache"), self.build_dir / "mime.cache", force=True, print_verbose_only=False, + Path(td, "mime/mime.cache"), + self.build_dir / "mime.cache", + force=True, + print_verbose_only=False, ) # TODO: get https://github.com/annulen/webkit-test-fonts to run the full testsuite if self.build_jsc_only: @@ -1120,7 +1171,9 @@ def install(self, **kwargs): if not self.build_jsc_only: dump_render_tree = self.build_dir / "bin/DumpRenderTree" self.maybe_strip_elf_file( - dump_render_tree, output_path=dump_render_tree.with_suffix(".stripped"), print_verbose_only=False, + dump_render_tree, + output_path=dump_render_tree.with_suffix(".stripped"), + print_verbose_only=False, ) jsc = self.build_dir / "bin/jsc" self.maybe_strip_elf_file(jsc, output_path=jsc.with_suffix(".stripped"), print_verbose_only=False) diff --git a/pycheribuild/projects/cross/rlbox.py b/pycheribuild/projects/cross/rlbox.py index 3442dd349..ad787f545 100644 --- a/pycheribuild/projects/cross/rlbox.py +++ b/pycheribuild/projects/cross/rlbox.py @@ -46,8 +46,9 @@ def run_tests(self): self.run_make("test") else: args = ["--verbose"] if self.config.verbose else [] - self.target_info.run_cheribsd_test_script("run_rlbox_tests.py", *args, mount_builddir=True, - mount_sourcedir=True, mount_sysroot=True) + self.target_info.run_cheribsd_test_script( + "run_rlbox_tests.py", *args, mount_builddir=True, mount_sourcedir=True, mount_sysroot=True + ) class BuildCatch2(CrossCompileCMakeProject): diff --git a/pycheribuild/projects/cross/ros2.py b/pycheribuild/projects/cross/ros2.py index 816aec3d6..02b6d7496 100644 --- a/pycheribuild/projects/cross/ros2.py +++ b/pycheribuild/projects/cross/ros2.py @@ -32,8 +32,9 @@ class BuildRos2(CrossCompileCMakeProject): target = "ros2" - repository = GitRepository("https://github.com/dodsonmg/ros2_dashing_minimal.git", default_branch="master", - force_branch=True) + repository = GitRepository( + "https://github.com/dodsonmg/ros2_dashing_minimal.git", default_branch="master", force_branch=True + ) # it may eventually be useful to install to rootfs or sysroot depending on whether we want to use ROS2 # as a library for building other applications using cheribuild # therefore, the _install_dir doesn't do anything, but cheribuild requires them @@ -49,21 +50,28 @@ def _ignore_packages(self): def _run_vcs(self): # this is the meta version control system used by ros for downloading and unpacking repos if not shutil.which("vcs"): - self.dependency_error("Missing vcs command", - install_instructions=InstallInstructions("pip3 install --user vcstool")) + self.dependency_error( + "Missing vcs command", install_instructions=InstallInstructions("pip3 install --user vcstool") + ) cmdline = ["vcs", "import", "--input", "ros2_minimal.repos", "src"] self.run_cmd(cmdline, cwd=self.source_dir) def _run_colcon(self, **kwargs): # colcon is the meta build system (on top of cmake) used by ros if not shutil.which("colcon"): - self.dependency_error("Missing colcon command", - install_instructions=InstallInstructions( - "pip3 install --user colcon-common-extensions")) - colcon_cmd = ["colcon", "build", - "--install-base", self.install_dir, - "--build-base", self.build_dir, - "--merge-install"] + self.dependency_error( + "Missing colcon command", + install_instructions=InstallInstructions("pip3 install --user colcon-common-extensions"), + ) + colcon_cmd = [ + "colcon", + "build", + "--install-base", + self.install_dir, + "--build-base", + self.build_dir, + "--merge-install", + ] colcon_args = ["--no-warn-unused-cli", "--packages-skip-build-finished"] cmake_args = ["--cmake-args", "-DBUILD_TESTING=NO"] if not self.compiling_for_host(): @@ -134,7 +142,7 @@ def _set_env(self): endif setenv LD_LIBRARY_PATH {ld_library_path}:${{LD_LIBRARY_PATH}} """ - self.write_file(self.install_dir / 'cheri_setup.csh', csh_script, overwrite=True) + self.write_file(self.install_dir / "cheri_setup.csh", csh_script, overwrite=True) # write LD_64C_LIBRARY_PATH to a text file to source from sh in CheriBSD posix_sh_script = f"""#!/bin/sh rootdir=`pwd` @@ -142,7 +150,7 @@ def _set_env(self): export LD_CHERI_LIBRARY_PATH={ld_library_path}:${{LD_LIBRARY_PATH}} export LD_LIBRARY_PATH={ld_library_path}:${{LD_LIBRARY_PATH}} """ - self.write_file(self.install_dir / 'cheri_setup.sh', posix_sh_script, overwrite=True) + self.write_file(self.install_dir / "cheri_setup.sh", posix_sh_script, overwrite=True) def update(self): super().update() @@ -171,7 +179,6 @@ def install(self, **kwargs): def run_tests(self): # only test when not compiling for host if not self.compiling_for_host(): - self.target_info.run_cheribsd_test_script("run_ros2_tests.py", - mount_sourcedir=True, - mount_installdir=True, - mount_sysroot=True) + self.target_info.run_cheribsd_test_script( + "run_ros2_tests.py", mount_sourcedir=True, mount_installdir=True, mount_sysroot=True + ) diff --git a/pycheribuild/projects/cross/rtems.py b/pycheribuild/projects/cross/rtems.py index cfeb34036..d76ac745a 100644 --- a/pycheribuild/projects/cross/rtems.py +++ b/pycheribuild/projects/cross/rtems.py @@ -36,8 +36,7 @@ class BuildRtems(CrossCompileProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/rtems", - force_branch=True, default_branch="cheri_waf1") + repository = GitRepository("https://github.com/CTSRD-CHERI/rtems", force_branch=True, default_branch="cheri_waf1") target = "rtems" include_os_in_target_suffix = False dependencies = ("newlib", "compiler-rt-builtins") @@ -64,11 +63,12 @@ def _run_waf(self, *args, **kwargs): return self.run_cmd(cmdline, cwd=self.source_dir, **kwargs) def configure(self, **kwargs): - waf_run = self._run_waf("bsp_defaults", "--rtems-bsps=" + ",".join(self.rtems_bsps), "--rtems-compiler=clang", - capture_output=True) + waf_run = self._run_waf( + "bsp_defaults", "--rtems-bsps=" + ",".join(self.rtems_bsps), "--rtems-compiler=clang", capture_output=True + ) # waf configure reads config.ini by default to read RTEMS flags from - self.write_file(self.source_dir / "config.ini", str(waf_run.stdout, 'utf-8'), overwrite=True) + self.write_file(self.source_dir / "config.ini", str(waf_run.stdout, "utf-8"), overwrite=True) self._run_waf("configure", "--prefix", self.destdir) def compile(self, **kwargs): @@ -78,9 +78,11 @@ def install(self, **kwargs): self._run_waf("install") def process(self): - with self.set_env(PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), - CFLAGS="--sysroot=" + str(self.sdk_sysroot), - LDFLAGS="--sysroot=" + str(self.sdk_sysroot)): + with self.set_env( + PATH=str(self.sdk_bindir) + ":" + os.getenv("PATH", ""), + CFLAGS="--sysroot=" + str(self.sdk_sysroot), + LDFLAGS="--sysroot=" + str(self.sdk_sysroot), + ): super().process() diff --git a/pycheribuild/projects/cross/simple_benchmark.py b/pycheribuild/projects/cross/simple_benchmark.py index cdc65297f..f62d58bee 100644 --- a/pycheribuild/projects/cross/simple_benchmark.py +++ b/pycheribuild/projects/cross/simple_benchmark.py @@ -42,9 +42,17 @@ class BuildSimpleCheriBenchmarks(BenchmarkMixin, CrossCompileCMakeProject): def create_test_dir(self, outdir: Path): self.clean_directory(outdir) - for f in ("run_jenkins-bluehive.sh", "libqsort_default.so", "test_qsort_default", "test_qsort_static", - "benchmark_qsort", "malloc_bench_shared", "malloc_bench_static", "malloc_benchmark.sh", - "run_cheribsd.sh"): + for f in ( + "run_jenkins-bluehive.sh", + "libqsort_default.so", + "test_qsort_default", + "test_qsort_static", + "benchmark_qsort", + "malloc_bench_shared", + "malloc_bench_static", + "malloc_benchmark.sh", + "run_cheribsd.sh", + ): self.install_file(self.build_dir / f, outdir / f, force=True, print_verbose_only=False) return outdir @@ -59,9 +67,11 @@ def run_tests(self): self.create_test_dir(self.build_dir / "test-dir") # testing, not benchmarking -> run only once: (-s small / -s large?) test_command = "cd /build/test-dir && ./run_jenkins-bluehive.sh -d0 -r1 -o {output} -a {tgt}".format( - tgt=self.archname_column, output=self.default_statcounters_csv_name) - self.target_info.run_cheribsd_test_script("run_simple_tests.py", "--test-command", test_command, - "--test-timeout", str(120 * 60), mount_builddir=True) + tgt=self.archname_column, output=self.default_statcounters_csv_name + ) + self.target_info.run_cheribsd_test_script( + "run_simple_tests.py", "--test-command", test_command, "--test-timeout", str(120 * 60), mount_builddir=True + ) def run_benchmarks(self): if not self.compiling_for_mips(include_purecap=True): @@ -69,6 +79,15 @@ def run_benchmarks(self): return with tempfile.TemporaryDirectory() as td: benchmarks_dir = self.create_test_dir(Path(td)) - self.run_fpga_benchmark(benchmarks_dir, output_file=self.default_statcounters_csv_name, - benchmark_script_args=["-d1", "-r10", "-o", self.default_statcounters_csv_name, - "-a", self.archname_column]) + self.run_fpga_benchmark( + benchmarks_dir, + output_file=self.default_statcounters_csv_name, + benchmark_script_args=[ + "-d1", + "-r10", + "-o", + self.default_statcounters_csv_name, + "-a", + self.archname_column, + ], + ) diff --git a/pycheribuild/projects/cross/snmalloc.py b/pycheribuild/projects/cross/snmalloc.py index 54c077c05..c718190fd 100644 --- a/pycheribuild/projects/cross/snmalloc.py +++ b/pycheribuild/projects/cross/snmalloc.py @@ -48,26 +48,27 @@ def setup_config_options(cls, **kwargs): cls.check_client = cls.add_bool_option("check-client", help="Don't accept malformed input to free") - cls.pagemap_pointers = cls.add_bool_option("pagemap-pointers", - help="Change pagemap data structure to store pointers") - cls.pagemap_rederive = cls.add_bool_option("pagemap-rederive", - help="Rederive internal pointers using the pagemap") + cls.pagemap_pointers = cls.add_bool_option( + "pagemap-pointers", help="Change pagemap data structure to store pointers" + ) + cls.pagemap_rederive = cls.add_bool_option( + "pagemap-rederive", help="Rederive internal pointers using the pagemap" + ) cls.cheri_align = cls.add_bool_option("cheri-align", help="Align sizes for CHERI bounds setting") cheri_bounds_default = cls._xtarget is not None and cls._xtarget.is_cheri_purecap() - cls.cheri_bounds = cls.add_bool_option("cheri-bounds", default=cheri_bounds_default, - help="Set bounds on returned allocations") + cls.cheri_bounds = cls.add_bool_option( + "cheri-bounds", default=cheri_bounds_default, help="Set bounds on returned allocations" + ) cls.quarantine = cls.add_bool_option("quarantine", help="Quarantine deallocations") - cls.qpathresh = cls.add_config_option("qpathresh", kind=int, - help="Quarantine physical memory per allocator threshold") - cls.qpacthresh = cls.add_config_option("qpacthresh", kind=int, - help="Quarantine chunk per allocator threshold") - cls.qcsc = cls.add_config_option("qcsc", kind=int, - help="Quarantine chunk size class") + cls.qpathresh = cls.add_config_option( + "qpathresh", kind=int, help="Quarantine physical memory per allocator threshold" + ) + cls.qpacthresh = cls.add_config_option("qpacthresh", kind=int, help="Quarantine chunk per allocator threshold") + cls.qcsc = cls.add_config_option("qcsc", kind=int, help="Quarantine chunk size class") - cls.decommit = cls.add_config_option("decommit", kind=str, - help="Specify memory decommit policy") + cls.decommit = cls.add_config_option("decommit", kind=str, help="Specify memory decommit policy") cls.zero = cls.add_bool_option("zero", help="Specify memory decommit policy") @@ -106,8 +107,9 @@ def __init__(self, *args, **kwargs): self.COMMON_FLAGS.append("-DSNMALLOC_REVOKE_PARANOIA=%d" % self.revoke_paranoia) self.COMMON_FLAGS.append("-DSNMALLOC_REVOKE_THROUGHPUT=%d" % self.revoke_tput) self.COMMON_FLAGS.append("-DSNMALLOC_QUARANTINE_CHATTY=%d" % self.revoke_verbose) - self.COMMON_FLAGS.append("-DSNMALLOC_DEFAULT_ZERO=%s" % - ("ZeroMem::YesZero" if self.zero else "ZeroMem::NoZero")) + self.COMMON_FLAGS.append( + "-DSNMALLOC_DEFAULT_ZERO=%s" % ("ZeroMem::YesZero" if self.zero else "ZeroMem::NoZero") + ) if self.decommit is not None: self.COMMON_FLAGS.append("-DUSE_DECOMMIT_STRATEGY=%s" % self.decommit) diff --git a/pycheribuild/projects/cross/sqlite.py b/pycheribuild/projects/cross/sqlite.py index e33454ed5..60c855def 100644 --- a/pycheribuild/projects/cross/sqlite.py +++ b/pycheribuild/projects/cross/sqlite.py @@ -31,8 +31,9 @@ class BuildSQLite(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/sqlite.git", - default_branch="3.22.0-cheri", force_branch=True) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/sqlite.git", default_branch="3.22.0-cheri", force_branch=True + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() diff --git a/pycheribuild/projects/cross/u_boot.py b/pycheribuild/projects/cross/u_boot.py index a2c2616de..11fecb13c 100644 --- a/pycheribuild/projects/cross/u_boot.py +++ b/pycheribuild/projects/cross/u_boot.py @@ -51,8 +51,7 @@ def uboot_install_dir(config: CheriConfig, project: "BuildUBoot") -> Path: class BuildUBoot(Project): target = "u-boot" - repository = GitRepository("https://github.com/CTSRD-CHERI/u-boot", - default_branch="cheri") + repository = GitRepository("https://github.com/CTSRD-CHERI/u-boot", default_branch="cheri") dependencies = ("compiler-rt-builtins",) needs_sysroot = False # We don't need a complete sysroot default_install_dir = DefaultInstallDir.CUSTOM_INSTALL_DIR @@ -64,9 +63,9 @@ class BuildUBoot(Project): ) make_kind = MakeCommandKind.GnuMake _always_add_suffixed_targets = True - _default_install_dir_fn: ComputedDefaultValue[Path] = \ - ComputedDefaultValue(function=uboot_install_dir, - as_string="$SDK_ROOT/u-boot/riscv{32,64}{,-hybrid,-purecap}") + _default_install_dir_fn: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=uboot_install_dir, as_string="$SDK_ROOT/u-boot/riscv{32,64}{,-hybrid,-purecap}" + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() @@ -95,7 +94,7 @@ def setup(self) -> None: # any relocations (and BFD lets it do this). OBJCOPY=str(self.sdk_bindir / "llvm-objcopy") + " --allow-broken-links", OBJDUMP=self.sdk_bindir / "llvm-objdump", - ) + ) self.kconfig_overrides = { "CONFIG_SREC": False, @@ -124,8 +123,9 @@ def firmware_path(self) -> Path: return self.install_dir / "u-boot" @classmethod - def get_firmware_path(cls, caller, config: "Optional[CheriConfig]" = None, - cross_target: "Optional[CrossCompileTarget]" = None): + def get_firmware_path( + cls, caller, config: "Optional[CheriConfig]" = None, cross_target: "Optional[CrossCompileTarget]" = None + ): return cls.get_instance(caller, config=config, cross_target=cross_target).firmware_path def configure(self, **kwargs): @@ -156,7 +156,7 @@ def install(self, **kwargs): self.install_file(self.build_dir / "u-boot", qemu_fw_path) def run_make(self, *args, **kwargs): - if 'cwd' in kwargs: - assert kwargs['cwd'] == self.build_dir - del kwargs['cwd'] + if "cwd" in kwargs: + assert kwargs["cwd"] == self.build_dir + del kwargs["cwd"] super().run_make(*args, **kwargs, cwd=self.source_dir) diff --git a/pycheribuild/projects/cross/wayland.py b/pycheribuild/projects/cross/wayland.py index 401492b68..477baf3b8 100644 --- a/pycheribuild/projects/cross/wayland.py +++ b/pycheribuild/projects/cross/wayland.py @@ -40,9 +40,11 @@ class BuildEPollShim(CrossCompileCMakeProject): target = "epoll-shim" - repository = GitRepository("https://github.com/jiixyj/epoll-shim", - temporary_url_override="https://github.com/arichardson/epoll-shim", - url_override_reason="https://github.com/jiixyj/epoll-shim/pull/36") + repository = GitRepository( + "https://github.com/jiixyj/epoll-shim", + temporary_url_override="https://github.com/arichardson/epoll-shim", + url_override_reason="https://github.com/jiixyj/epoll-shim/pull/36", + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def configure(self, **kwargs): @@ -70,8 +72,9 @@ def run_tests(self): class BuildLibUdevDevd(CrossCompileMesonProject): target = "libudev-devd" - repository = GitRepository("https://github.com/wulf7/libudev-devd", - old_urls=[b"https://github.com/FreeBSDDesktop/libudev-devd"]) + repository = GitRepository( + "https://github.com/wulf7/libudev-devd", old_urls=[b"https://github.com/FreeBSDDesktop/libudev-devd"] + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.NATIVE_IF_FREEBSD dependencies = ("linux-input-h",) @@ -96,8 +99,12 @@ def process(self): dev_evdev_h = src_headers / "dev/evdev" / header if not dev_evdev_h.is_file(): self.fatal("Missing evdev header:", dev_evdev_h) - self.write_file(self.include_install_dir / "linux" / header, contents=f"#include \n", - overwrite=True, print_verbose_only=False) + self.write_file( + self.include_install_dir / "linux" / header, + contents=f"#include \n", + overwrite=True, + print_verbose_only=False, + ) @property def include_install_dir(self) -> Path: @@ -168,17 +175,22 @@ def setup(self): class BuildDejaGNU(AutotoolsProject): - repository = GitRepository("https://git.savannah.gnu.org/git/dejagnu.git", - temporary_url_override="https://github.com/arichardson/dejagnu.git", - url_override_reason="Remote test execution is broken(-ish) upstream") + repository = GitRepository( + "https://git.savannah.gnu.org/git/dejagnu.git", + temporary_url_override="https://github.com/arichardson/dejagnu.git", + url_override_reason="Remote test execution is broken(-ish) upstream", + ) native_install_dir = DefaultInstallDir.BOOTSTRAP_TOOLS class BuildLibFFI(CrossCompileAutotoolsProject): - repository = GitRepository("https://github.com/libffi/libffi.git", - temporary_url_override="https://github.com/CTSRD-CHERI/libffi.git", - default_branch="v3.4.4-cheriabi", force_branch=True, - url_override_reason="Needs lots of CHERI fixes") + repository = GitRepository( + "https://github.com/libffi/libffi.git", + temporary_url_override="https://github.com/CTSRD-CHERI/libffi.git", + default_branch="v3.4.4-cheriabi", + force_branch=True, + url_override_reason="Needs lots of CHERI fixes", + ) target = "libffi" supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE @@ -193,29 +205,46 @@ def setup(self): def run_tests(self): runtest_cmd = shutil.which("runtest") if not runtest_cmd: - self.dependency_error("DejaGNU is not installed.", - install_instructions=OSInfo.install_instructions("runtest", False, default="dejagnu", - apt="dejagnu", homebrew="deja-gnu"), - cheribuild_target="dejagnu", cheribuild_xtarget=CompilationTargets.NATIVE) + self.dependency_error( + "DejaGNU is not installed.", + install_instructions=OSInfo.install_instructions( + "runtest", False, default="dejagnu", apt="dejagnu", homebrew="deja-gnu" + ), + cheribuild_target="dejagnu", + cheribuild_xtarget=CompilationTargets.NATIVE, + ) runtest_flags = "-a" if self.config.debug_output: runtest_flags += " -v -v -v" elif self.config.verbose: runtest_flags += " -v" if self.compiling_for_host(): - self.run_cmd("make", "check", f"RUNTESTFLAGS={runtest_flags}", cwd=self.build_dir, - env=dict(DEJAGNU=self.source_dir / ".ci/site.exp", BOARDSDIR=self.source_dir / ".ci")) + self.run_cmd( + "make", + "check", + f"RUNTESTFLAGS={runtest_flags}", + cwd=self.build_dir, + env=dict(DEJAGNU=self.source_dir / ".ci/site.exp", BOARDSDIR=self.source_dir / ".ci"), + ) elif self.target_info.is_cheribsd(): # We need two minor fixes for SSH execution: - runtest_ver = get_program_version(Path(runtest_cmd or "runtest"), program_name=b"DejaGnu", - config=self.config) + runtest_ver = get_program_version( + Path(runtest_cmd or "runtest"), program_name=b"DejaGnu", config=self.config + ) if runtest_ver < (1, 6, 4): - self.dependency_error("DejaGnu version", runtest_ver, "cannot be used to run tests remotely,", - "please install a newer version with cheribuild", - cheribuild_target="dejagnu", cheribuild_xtarget=CompilationTargets.NATIVE) + self.dependency_error( + "DejaGnu version", + runtest_ver, + "cannot be used to run tests remotely,", + "please install a newer version with cheribuild", + cheribuild_target="dejagnu", + cheribuild_xtarget=CompilationTargets.NATIVE, + ) if self.can_run_binaries_on_remote_morello_board(): - self.write_file(self.build_dir / "site.exp", contents=f""" + self.write_file( + self.build_dir / "site.exp", + contents=f""" if ![info exists boards_dir] {{ set boards_dir {{}} }} @@ -223,11 +252,15 @@ def run_tests(self): verbose "Global Config File: target_triplet is $target_triplet" 2 global target_list set target_list "remote-cheribsd" -""", overwrite=True) +""", + overwrite=True, + ) ssh_options = "-o NoHostAuthenticationForLocalhost=yes" ssh_port = ssh_config_parameters(self.config.remote_morello_board, self.config).get("port", "22") ssh_user = ssh_config_parameters(self.config.remote_morello_board, self.config).get("user", "root") - self.write_file(self.build_dir / "remote-cheribsd.exp", contents=f""" + self.write_file( + self.build_dir / "remote-cheribsd.exp", + contents=f""" load_generic_config "unix" set_board_info connect ssh set_board_info hostname {self.config.remote_morello_board} @@ -239,15 +272,24 @@ def run_tests(self): # set_board_info exec_shell "gdb-run-noninteractive.sh" # Build tests statically linked so they pick up the local libffi library set TOOL_OPTIONS -static -""", overwrite=True) - self.run_cmd(["make", "check", - f"RUNTESTFLAGS={runtest_flags} --target-board remote-cheribsd --xml"], - env=dict(BOARDSDIR=self.build_dir, DEJAGNU=self.build_dir / "site.exp"), - cwd=str(self.build_dir)) +""", + overwrite=True, + ) + self.run_cmd( + ["make", "check", f"RUNTESTFLAGS={runtest_flags} --target-board remote-cheribsd --xml"], + env=dict(BOARDSDIR=self.build_dir, DEJAGNU=self.build_dir / "site.exp"), + cwd=str(self.build_dir), + ) else: - self.target_info.run_cheribsd_test_script("run_libffi_tests.py", "--test-timeout", str(120 * 60), - mount_builddir=True, mount_sourcedir=True, - mount_sysroot=False, use_full_disk_image=True) + self.target_info.run_cheribsd_test_script( + "run_libffi_tests.py", + "--test-timeout", + str(120 * 60), + mount_builddir=True, + mount_sourcedir=True, + mount_sysroot=False, + use_full_disk_image=True, + ) class BuildWayland(CrossCompileMesonProject): @@ -264,8 +306,13 @@ def dependencies(cls, config: CheriConfig) -> "tuple[str, ...]": if target.target_info_cls.is_freebsd(): deps += ("epoll-shim",) return deps - repository = GitRepository("https://gitlab.freedesktop.org/wayland/wayland.git", default_branch="main", - force_branch=True, old_urls=[b"https://github.com/CTSRD-CHERI/wayland"]) + + repository = GitRepository( + "https://gitlab.freedesktop.org/wayland/wayland.git", + default_branch="main", + force_branch=True, + old_urls=[b"https://github.com/CTSRD-CHERI/wayland"], + ) supported_architectures = CompilationTargets.ALL_FREEBSD_AND_CHERIBSD_TARGETS + CompilationTargets.ALL_NATIVE def setup(self): diff --git a/pycheribuild/projects/cross/webkit.py b/pycheribuild/projects/cross/webkit.py index f0a472b79..48ea78eba 100644 --- a/pycheribuild/projects/cross/webkit.py +++ b/pycheribuild/projects/cross/webkit.py @@ -41,30 +41,39 @@ class JsBackend(Enum): class BuildMorelloWebkit(CrossCompileCMakeProject): - repository = GitRepository("https://github.com/CTSRD-CHERI/webkit", - default_branch="master") + repository = GitRepository("https://github.com/CTSRD-CHERI/webkit", default_branch="master") default_directory_basename = "webkit" target = "morello-webkit" dependencies = ("icu4c",) native_install_dir = DefaultInstallDir.DO_NOT_INSTALL cross_install_dir = DefaultInstallDir.ROOTFS_OPTBASE tier2ptrliterals = BoolConfigOption( - "tier2ptrliterals", default=True, show_help=True, + "tier2ptrliterals", + default=True, + show_help=True, help="When true pointers are represented as atomic literals and loaded as data and when false pointers " - "are represented as numeric values which can be splitted and are encoded into instructions. " - "This option only affects the non-purecap tier2 backend.") + "are represented as numeric values which can be splitted and are encoded into instructions. " + "This option only affects the non-purecap tier2 backend.", + ) jsheapoffsets = BoolConfigOption( - "jsheapoffsets", default=False, show_help=True, + "jsheapoffsets", + default=False, + show_help=True, help="Use offsets into the JS heap for object references instead of capabilities. " - "This option only affects the purecap backends.") + "This option only affects the purecap backends.", + ) @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.backend = cls.add_config_option( - "backend", kind=JsBackend, - default=JsBackend.CLOOP, enum_choice_strings=[t.value for t in JsBackend], - show_help=True, help="The JavaScript backend to use for building WebKit") + "backend", + kind=JsBackend, + default=JsBackend.CLOOP, + enum_choice_strings=[t.value for t in JsBackend], + show_help=True, + help="The JavaScript backend to use for building WebKit", + ) @property def build_dir_suffix(self): @@ -109,7 +118,7 @@ def setup(self): ENABLE_YARR_JIT=False, ENABLE_SAMPLING_PROFILER=False, CHERI_PURE_CAPABILITY=self.crosscompile_target.is_cheri_purecap(), - ) + ) if self.crosscompile_target.is_cheri_purecap(): # TODO: we can get this from the pre-processor instead @@ -122,19 +131,29 @@ def setup(self): # Add options for each backend if self.backend == JsBackend.CLOOP: - self.add_cmake_options(ENABLE_C_LOOP=True, ENABLE_ASSEMBLER=False, ENABLE_JIT=False, - ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=False) + self.add_cmake_options( + ENABLE_C_LOOP=True, + ENABLE_ASSEMBLER=False, + ENABLE_JIT=False, + ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=False, + ) elif self.backend == JsBackend.TIER1ASM: - self.add_cmake_options(ENABLE_C_LOOP=False, ENABLE_ASSEMBLER=True, ENABLE_JIT=False, - ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True) + self.add_cmake_options( + ENABLE_C_LOOP=False, + ENABLE_ASSEMBLER=True, + ENABLE_JIT=False, + ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True, + ) elif self.backend == JsBackend.TIER2ASM: - self.add_cmake_options(ENABLE_C_LOOP=False, ENABLE_ASSEMBLER=True, ENABLE_DISASSEMBLER=True, - ENABLE_JIT=True) + self.add_cmake_options( + ENABLE_C_LOOP=False, ENABLE_ASSEMBLER=True, ENABLE_DISASSEMBLER=True, ENABLE_JIT=True + ) if self.crosscompile_target.is_cheri_purecap(): if self.jsheapoffsets: self.warning("integer heap offsets are not yet supported for the tier 2 backend!") - self.add_cmake_options(ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True, - ENABLE_JSHEAP_CHERI_OFFSET_REFS=False) + self.add_cmake_options( + ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=True, ENABLE_JSHEAP_CHERI_OFFSET_REFS=False + ) else: self.add_cmake_options(ENABLE_JIT_ARM64_EMBED_POINTERS_AS_ALIGNED_LITERALS=self.tier2ptrliterals) @@ -143,5 +162,6 @@ def run_tests(self): self.fatal("Running host tests not implemented") else: # full disk image to get icu library - self.target_info.run_cheribsd_test_script("run_morello_webkit_tests.py", mount_sourcedir=True, - use_full_disk_image=True) + self.target_info.run_cheribsd_test_script( + "run_morello_webkit_tests.py", mount_sourcedir=True, use_full_disk_image=True + ) diff --git a/pycheribuild/projects/cross/x11.py b/pycheribuild/projects/cross/x11.py index d17295cf6..a6b9ab9f4 100644 --- a/pycheribuild/projects/cross/x11.py +++ b/pycheribuild/projects/cross/x11.py @@ -110,8 +110,9 @@ def compile(self, **kwargs): if self.config.pretend and not shutil.which("automake"): automake_version = (0, 0, 0) else: - automake_version = get_program_version(Path("automake"), config=self.config, - regex=rb"automake\s+\(GNU automake\)\s+(\d+)\.(\d+)\.?(\d+)?") + automake_version = get_program_version( + Path("automake"), config=self.config, regex=rb"automake\s+\(GNU automake\)\s+(\d+)\.(\d+)\.?(\d+)?" + ) if automake_version >= (1, 16, 4): self.info("Working around https://www.mail-archive.com/bug-automake@gnu.org/msg04957.html") self.replace_in_file(self.build_dir / "xcb-proto.pc", {"${PYTHON_PREFIX}": str(self.install_prefix)}) @@ -311,8 +312,9 @@ def install(self, **kwargs): super().install(**kwargs) if not self.compiling_for_host() and not self.crosscompile_target.is_libcompat_target(): # Ensure that xauth is in the default $PATH, so that ssh -X works - self.create_symlink(self.install_dir / "bin/xauth", self.rootfs_dir / "usr/local/bin/xauth", - print_verbose_only=False) + self.create_symlink( + self.install_dir / "bin/xauth", self.rootfs_dir / "usr/local/bin/xauth", print_verbose_only=False + ) class BuildXEyes(X11AutotoolsProject): @@ -329,12 +331,14 @@ class BuildLibXKBCommon(X11MesonProject): def setup(self): # avoid wayland dep for now super().setup() - self.add_meson_options(**{ - "enable-x11": True, - "enable-xkbregistry": False, # Avoid dependency on libxml2 - "enable-docs": False, # Avoid lots of dependencies to build docs - "enable-tools": False, # No need for wayland deps just for the xkbcli tool - }) + self.add_meson_options( + **{ + "enable-x11": True, + "enable-xkbregistry": False, # Avoid dependency on libxml2 + "enable-docs": False, # Avoid lots of dependencies to build docs + "enable-tools": False, # No need for wayland deps just for the xkbcli tool + } + ) def process(self): newpath = os.getenv("PATH") @@ -353,8 +357,10 @@ class BuildXorgFontUtil(X11AutotoolsProject): class BuildPixman(X11MesonProject): target = "pixman" dependencies = ("libpng",) - repository = GitRepository("https://gitlab.freedesktop.org/pixman/pixman.git", - old_urls=[b"https://gitlab.freedesktop.org/arichardson/pixman.git"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/pixman/pixman.git", + old_urls=[b"https://gitlab.freedesktop.org/arichardson/pixman.git"], + ) def setup(self): super().setup() @@ -417,8 +423,10 @@ class BuildLibXScrnSaver(X11AutotoolsProject): class BuildLibJpegTurbo(X11CMakeProject): target = "libjpeg-turbo" - repository = GitRepository("https://github.com/libjpeg-turbo/libjpeg-turbo.git", - old_urls=[b"https://github.com/arichardson/libjpeg-turbo.git"]) + repository = GitRepository( + "https://github.com/libjpeg-turbo/libjpeg-turbo.git", + old_urls=[b"https://github.com/arichardson/libjpeg-turbo.git"], + ) def setup(self): super().setup() @@ -487,14 +495,28 @@ class BuildXVncServer(X11AutotoolsProject): target = "xvnc-server" # The actual XVnc source code is part of TigerVNC and not included in the xserver repository. # It also depends on build artifacts from an existing tigervnc build - dependencies = ("libx11", "xorg-font-util", "libxrender", "libxfont", "libxkbfile", "tigervnc", "xkeyboard-config", - "xkbcomp", "dbus") + dependencies = ( + "libx11", + "xorg-font-util", + "libxrender", + "libxfont", + "libxkbfile", + "tigervnc", + "xkeyboard-config", + "xkbcomp", + "dbus", + ) # The tigervnc code requires the 1.20 release - repository = GitRepository("https://gitlab.freedesktop.org/xorg/xserver.git", - default_branch="server-1.20-branch", force_branch=True, - temporary_url_override="https://gitlab.freedesktop.org/arichardson/xserver.git", - url_override_reason=["https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/721", - "https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/720"]) + repository = GitRepository( + "https://gitlab.freedesktop.org/xorg/xserver.git", + default_branch="server-1.20-branch", + force_branch=True, + temporary_url_override="https://gitlab.freedesktop.org/arichardson/xserver.git", + url_override_reason=[ + "https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/721", + "https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/720", + ], + ) def install(self, **kwargs): """ @@ -507,14 +529,22 @@ def install(self, **kwargs): super().install() # Install a script to start the Xvnc so I don't have to remember the arguments # TODO: should we install a service that we can start with `service xvnc start`? - self.write_file(self.install_dir / "bin/startxvnc", overwrite=True, mode=0o755, - contents="#!/bin/sh\nXvnc -geometry 1024x768 -SecurityTypes=None \"$@\"\n") + self.write_file( + self.install_dir / "bin/startxvnc", + overwrite=True, + mode=0o755, + contents='#!/bin/sh\nXvnc -geometry 1024x768 -SecurityTypes=None "$@"\n', + ) if not self.compiling_for_host() and not self.crosscompile_target.is_libcompat_target(): # Ensure that Xvnc is in the default $PATH - self.create_symlink(self.install_dir / "bin/Xvnc", self.rootfs_dir / "usr/local/bin/Xvnc", - print_verbose_only=False) - self.create_symlink(self.install_dir / "bin/startxvnc", self.rootfs_dir / "usr/local/bin/startxvnc", - print_verbose_only=False) + self.create_symlink( + self.install_dir / "bin/Xvnc", self.rootfs_dir / "usr/local/bin/Xvnc", print_verbose_only=False + ) + self.create_symlink( + self.install_dir / "bin/startxvnc", + self.rootfs_dir / "usr/local/bin/startxvnc", + print_verbose_only=False, + ) def update(self): super().update() @@ -523,12 +553,27 @@ def update(self): self.create_symlink(tigervnc_source / "unix/xserver/hw/vnc", self.source_dir / "hw/vnc") try: # Check if the patch was already applied with --dry-run - self.run_cmd("patch", "-p1", "--forward", "--dry-run", "-i", - tigervnc_source / "unix/xserver120.patch", cwd=self.source_dir, capture_error=True, - capture_output=True) - self.run_cmd("patch", "-p1", "--forward", "-i", - tigervnc_source / "unix/xserver120.patch", cwd=self.source_dir, capture_error=True, - capture_output=True) + self.run_cmd( + "patch", + "-p1", + "--forward", + "--dry-run", + "-i", + tigervnc_source / "unix/xserver120.patch", + cwd=self.source_dir, + capture_error=True, + capture_output=True, + ) + self.run_cmd( + "patch", + "-p1", + "--forward", + "-i", + tigervnc_source / "unix/xserver120.patch", + cwd=self.source_dir, + capture_error=True, + capture_output=True, + ) except subprocess.CalledProcessError as e: if b"Skipping patch" in e.stdout: return @@ -537,23 +582,37 @@ def update(self): def setup(self): super().setup() fonts_dir = Path("/", self.target_info.sysroot_install_prefix_relative, "share/fonts") - self.configure_args.extend([ - "--without-dtrace", "--enable-static", "--disable-dri", "--disable-unit-tests", - "--disable-xinerama", "--disable-xvfb", "--disable-xnest", "--disable-xorg", - "--disable-dmx", "--disable-xwin", "--disable-xephyr", "--disable-kdrive", - "--disable-libdrm", - "--disable-config-dbus", "--disable-config-hal", - "--disable-dri", "--disable-dri2", "--disable-dri3", - "--enable-install-libxf86config", - "--disable-glx", # "--enable-glx", - "-with-default-font-path=catalogue:" + str(fonts_dir) + ",built-ins", - "--with-serverconfig-path=" + str(self.install_prefix / "lib/X11"), - "--disable-selective-werror", - "--disable-xwayland", - "--with-fontrootdir=" + str(fonts_dir), - "--with-xkb-path=" + str(BuildXKeyboardConfig.get_instance(self).install_prefix / "share/X11/xkb"), - "--with-xkb-bin-directory=" + str(BuildXKkbcomp.get_instance(self).install_prefix / "bin"), - ]) + self.configure_args.extend( + [ + "--without-dtrace", + "--enable-static", + "--disable-dri", + "--disable-unit-tests", + "--disable-xinerama", + "--disable-xvfb", + "--disable-xnest", + "--disable-xorg", + "--disable-dmx", + "--disable-xwin", + "--disable-xephyr", + "--disable-kdrive", + "--disable-libdrm", + "--disable-config-dbus", + "--disable-config-hal", + "--disable-dri", + "--disable-dri2", + "--disable-dri3", + "--enable-install-libxf86config", + "--disable-glx", # "--enable-glx", + "-with-default-font-path=catalogue:" + str(fonts_dir) + ",built-ins", + "--with-serverconfig-path=" + str(self.install_prefix / "lib/X11"), + "--disable-selective-werror", + "--disable-xwayland", + "--with-fontrootdir=" + str(fonts_dir), + "--with-xkb-path=" + str(BuildXKeyboardConfig.get_instance(self).install_prefix / "share/X11/xkb"), + "--with-xkb-bin-directory=" + str(BuildXKkbcomp.get_instance(self).install_prefix / "bin"), + ] + ) tigervnc = BuildTigerVNC.get_instance(self) self.make_args.set(TIGERVNC_SRCDIR=tigervnc.source_dir, TIGERVNC_BUILDDIR=tigervnc.build_dir) self.COMMON_LDFLAGS.append("-Wl,-rpath," + str(BuildFreeType2.get_instance(self).install_prefix / "lib")) @@ -611,10 +670,17 @@ def setup(self): # Slightly more functional window manager than TWM class BuildIceWM(X11CMakeProject): target = "icewm" - dependencies = ("fontconfig", "libxcomposite", "libxdamage", "libpng", "libjpeg-turbo", - "libxpm", "libxft", "libxrandr") - repository = GitRepository("https://github.com/bbidulock/icewm", - old_urls=[b"https://github.com/arichardson/icewm"]) + dependencies = ( + "fontconfig", + "libxcomposite", + "libxdamage", + "libpng", + "libjpeg-turbo", + "libxpm", + "libxft", + "libxrandr", + ) + repository = GitRepository("https://github.com/bbidulock/icewm", old_urls=[b"https://github.com/arichardson/icewm"]) def setup(self): super().setup() diff --git a/pycheribuild/projects/cross/zlib.py b/pycheribuild/projects/cross/zlib.py index e6e0973ff..978be83a0 100644 --- a/pycheribuild/projects/cross/zlib.py +++ b/pycheribuild/projects/cross/zlib.py @@ -47,5 +47,8 @@ def setup(self): if not self.compiling_for_host(): # If we don't set this, the build will use the macOS host libtool instead of llvm-ar and then complain # because the .o files are not macOS object files. - self.add_configure_vars(uname=self.target_info.cmake_system_name, - AR=self.sdk_bindir / "llvm-ar", RANLIB=self.sdk_bindir / "llvm-ranlib") + self.add_configure_vars( + uname=self.target_info.cmake_system_name, + AR=self.sdk_bindir / "llvm-ar", + RANLIB=self.sdk_bindir / "llvm-ranlib", + ) diff --git a/pycheribuild/projects/disk_image.py b/pycheribuild/projects/disk_image.py index c37cfed6b..6406808a4 100644 --- a/pycheribuild/projects/disk_image.py +++ b/pycheribuild/projects/disk_image.py @@ -121,8 +121,8 @@ def _default_disk_image_name(_: CheriConfig, directory: Path, project: "BuildDis def _default_disk_image_hostname(prefix: str) -> "ComputedDefaultValue[str]": # noinspection PyProtectedMember return ComputedDefaultValue( - function=lambda conf, proj: prefix + proj.build_configuration_suffix(), - as_string=prefix + "-") + function=lambda conf, proj: prefix + proj.build_configuration_suffix(), as_string=prefix + "-" + ) class FileSystemType(Enum): @@ -139,7 +139,8 @@ class BuildDiskImageBase(SimpleProject): disk_image_prefix: str = None default_disk_image_path = ComputedDefaultValue( function=lambda conf, proj: _default_disk_image_name(conf, conf.output_root, proj), - as_string=lambda cls: "$OUTPUT_ROOT/" + cls.disk_image_prefix + "--disk.img depending on architecture") + as_string=lambda cls: "$OUTPUT_ROOT/" + cls.disk_image_prefix + "--disk.img depending on architecture", + ) @classproperty def default_architecture(self) -> CrossCompileTarget: @@ -156,39 +157,66 @@ def dependencies(cls, config: CheriConfig) -> "tuple[str, ...]": @classmethod def setup_config_options(cls, *, default_hostname, extra_files_suffix="", **kwargs): super().setup_config_options() - cls.extra_files_dir = cls.add_path_option("extra-files", show_help=True, - default=lambda config, project: ( - config.source_root / ("extra-files" + extra_files_suffix)), - help="A directory with additional files that will be added to the " - "image (default: " - "'$SOURCE_ROOT/extra-files" + extra_files_suffix + "')", - metavar="DIR") - cls.hostname = cls.add_config_option("hostname", show_help=False, default=default_hostname, metavar="HOSTNAME", - help="The hostname to use for the disk image") + cls.extra_files_dir = cls.add_path_option( + "extra-files", + show_help=True, + default=lambda config, project: (config.source_root / ("extra-files" + extra_files_suffix)), + help="A directory with additional files that will be added to the " + "image (default: " + "'$SOURCE_ROOT/extra-files" + extra_files_suffix + "')", + metavar="DIR", + ) + cls.hostname = cls.add_config_option( + "hostname", + show_help=False, + default=default_hostname, + metavar="HOSTNAME", + help="The hostname to use for the disk image", + ) if "use_qcow2" not in cls.__dict__: - cls.use_qcow2 = cls.add_bool_option("use-qcow2", - help="Convert the disk image to QCOW2 format instead of raw") - cls.rootfs_type = cls.add_config_option("rootfs-type", show_help=True, - kind=FileSystemType, default=FileSystemType.UFS, - enum_choices=[FileSystemType.UFS, FileSystemType.ZFS], - help="Select the type of the root file system image.") - cls.remote_path = cls.add_config_option("remote-path", show_help=False, metavar="PATH", - help="When set rsync will be used to update the image from " - "the remote server instead of building it locally.") - cls.wget_via_tmp = cls.add_bool_option("wget-via-tmp", - help="Use a directory in /tmp for recursive wget operations;" - "of interest in rare cases, like extra-files on smbfs.") - cls.include_gdb = cls.add_bool_option("include-gdb", default=True, - help="Include GDB in the disk image (if it exists)") - cls.include_kgdb = cls.add_bool_option("include-kgdb", default=False, - help="Include KGDB in the disk image (if it exists)") + cls.use_qcow2 = cls.add_bool_option( + "use-qcow2", help="Convert the disk image to QCOW2 format instead of raw" + ) + cls.rootfs_type = cls.add_config_option( + "rootfs-type", + show_help=True, + kind=FileSystemType, + default=FileSystemType.UFS, + enum_choices=[FileSystemType.UFS, FileSystemType.ZFS], + help="Select the type of the root file system image.", + ) + cls.remote_path = cls.add_config_option( + "remote-path", + show_help=False, + metavar="PATH", + help="When set rsync will be used to update the image from " + "the remote server instead of building it locally.", + ) + cls.wget_via_tmp = cls.add_bool_option( + "wget-via-tmp", + help="Use a directory in /tmp for recursive wget operations;" + "of interest in rare cases, like extra-files on smbfs.", + ) + cls.include_gdb = cls.add_bool_option( + "include-gdb", default=True, help="Include GDB in the disk image (if it exists)" + ) + cls.include_kgdb = cls.add_bool_option( + "include-kgdb", default=False, help="Include KGDB in the disk image (if it exists)" + ) assert cls.default_disk_image_path is not None - cls.disk_image_path = cls.add_path_option("path", default=cls.default_disk_image_path, metavar="IMGPATH", - help="The output path for the disk image", show_help=True) - cls.force_overwrite = cls.add_bool_option("force-overwrite", default=True, - help="Overwrite an existing disk image without prompting") - cls.no_autoboot = cls.add_bool_option("no-autoboot", default=False, - help="Disable autoboot and boot menu for targets that use loader(8)") + cls.disk_image_path = cls.add_path_option( + "path", + default=cls.default_disk_image_path, + metavar="IMGPATH", + help="The output path for the disk image", + show_help=True, + ) + cls.force_overwrite = cls.add_bool_option( + "force-overwrite", default=True, help="Overwrite an existing disk image without prompting" + ) + cls.no_autoboot = cls.add_bool_option( + "no-autoboot", default=False, help="Disable autoboot and boot menu for targets that use loader(8)" + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() @@ -235,8 +263,17 @@ def setup(self) -> None: def _get_source_class_target(self): return self.crosscompile_target - def add_file_to_image(self, file: Path, *, base_directory: "Optional[Path]" = None, user="root", group="wheel", - mode=None, path_in_target=None, strip_binaries: "Optional[bool]" = None): + def add_file_to_image( + self, + file: Path, + *, + base_directory: "Optional[Path]" = None, + user="root", + group="wheel", + mode=None, + path_in_target=None, + strip_binaries: "Optional[bool]" = None, + ): if path_in_target is None: assert base_directory is not None, "Either base_directory or path_in_target must be set!" path_in_target = os.path.relpath(str(file), str(base_directory)) @@ -259,8 +296,9 @@ def add_file_to_image(self, file: Path, *, base_directory: "Optional[Path]" = No if file in self.extra_files: self.extra_files.remove(file) # remove it from extra_files so we don't install it twice - def create_file_for_image(self, path_in_image: str, *, contents: str = "\n", show_contents_non_verbose=False, - mode=None): + def create_file_for_image( + self, path_in_image: str, *, contents: str = "\n", show_contents_non_verbose=False, mode=None + ): if path_in_image.startswith("/"): path_in_image = path_in_image[1:] assert not path_in_image.startswith("/") @@ -275,8 +313,14 @@ def create_file_for_image(self, path_in_image: str, *, contents: str = "\n", sho target_file = self.tmpdir / path_in_image base_dir = self.tmpdir if self.config.verbose or (show_contents_non_verbose and not self.config.quiet): - print("Generating /", path_in_image, " with the following contents:\n", - coloured(AnsiColour.green, contents), sep="", end="") + print( + "Generating /", + path_in_image, + " with the following contents:\n", + coloured(AnsiColour.green, contents), + sep="", + end="", + ) self.write_file(target_file, contents, never_print_cmd=True, overwrite=False, mode=mode) self.add_file_to_image(target_file, base_directory=base_dir) @@ -306,27 +350,29 @@ def prepare_rootfs(self): if self.include_swap_partition: fstab_contents += "/dev/gpt/swap none swap sw 0 0\n" fstab_contents += self.file_templates.get_fstab_template() - self.create_file_for_image("/etc/fstab", contents=fstab_contents, - mode=0o644, show_contents_non_verbose=True) + self.create_file_for_image("/etc/fstab", contents=fstab_contents, mode=0o644, show_contents_non_verbose=True) # enable ssh and set hostname # TODO: use separate file in /etc/rc.conf.d/ ? rc_conf_contents = self.file_templates.get_rc_conf_template().format(hostname=self.hostname) - self.create_file_for_image("/etc/rc.conf", contents=rc_conf_contents, - mode=0o644, show_contents_non_verbose=False) + self.create_file_for_image( + "/etc/rc.conf", contents=rc_conf_contents, mode=0o644, show_contents_non_verbose=False + ) - cshrc_contents = self.file_templates.get_cshrc_template().format(SRCPATH=self.config.source_root, - ROOTFS_DIR=self.rootfs_dir) + cshrc_contents = self.file_templates.get_cshrc_template().format( + SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir + ) self.create_file_for_image("/etc/csh.cshrc", contents=cshrc_contents, mode=0o644) # Basic .bashrc/.bash_profile template - dot_bashrc_contents = self.file_templates.get_dot_bashrc_template().format(SRCPATH=self.config.source_root, - ROOTFS_DIR=self.rootfs_dir) + dot_bashrc_contents = self.file_templates.get_dot_bashrc_template().format( + SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir + ) self.create_file_for_image("/root/.bashrc", contents=dot_bashrc_contents, mode=0o644) self.create_file_for_image("/usr/share/skel/dot.bashrc", contents=dot_bashrc_contents, mode=0o644) dot_bash_profile_contents = self.file_templates.get_dot_bash_profile_template().format( - SRCPATH=self.config.source_root, - ROOTFS_DIR=self.rootfs_dir) + SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir + ) self.create_file_for_image("/root/.bash_profile", contents=dot_bash_profile_contents, mode=0o644) self.create_file_for_image("/usr/share/skel/dot.bash_profile", contents=dot_bash_profile_contents, mode=0o644) @@ -343,8 +389,12 @@ def path_relative_to_outputroot(xtarget) -> Path: try: return install_dir.relative_to(self.config.output_root) except ValueError: - self.info(install_dir, "is not relative to", self.config.output_root, - "-- qemu-mount-rootfs.sh may not mount it") + self.info( + install_dir, + "is not relative to", + self.config.output_root, + "-- qemu-mount-rootfs.sh may not mount it", + ) return Path("/invalid/path") if self.crosscompile_target.is_hybrid_or_purecap_cheri(): @@ -352,33 +402,55 @@ def path_relative_to_outputroot(xtarget) -> Path: hybrid_cheri_dirname = path_relative_to_outputroot(self.crosscompile_target.get_cheri_hybrid_target()) purecap_cheri_dirname = path_relative_to_outputroot(self.crosscompile_target.get_cheri_purecap_target()) mount_rootfs_script = include_local_file("files/cheribsd/qemu-mount-rootfs.sh.in").format( - SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir, - NOCHERI_ROOTFS_DIRNAME=non_cheri_dirname, HYBRID_ROOTFS_DIRNAME=hybrid_cheri_dirname, - PURECAP_ROOTFS_DIRNAME=purecap_cheri_dirname) - self.create_file_for_image("/sbin/qemu-mount-rootfs.sh", contents=mount_rootfs_script, - mode=0o755, show_contents_non_verbose=False) + SRCPATH=self.config.source_root, + ROOTFS_DIR=self.rootfs_dir, + NOCHERI_ROOTFS_DIRNAME=non_cheri_dirname, + HYBRID_ROOTFS_DIRNAME=hybrid_cheri_dirname, + PURECAP_ROOTFS_DIRNAME=purecap_cheri_dirname, + ) + self.create_file_for_image( + "/sbin/qemu-mount-rootfs.sh", contents=mount_rootfs_script, mode=0o755, show_contents_non_verbose=False + ) mount_sources_script = include_local_file("files/cheribsd/qemu-mount-sources.sh.in").format( - SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir) - self.create_file_for_image("/sbin/qemu-mount-sources.sh", contents=mount_sources_script, - mode=0o755, show_contents_non_verbose=False) + SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir + ) + self.create_file_for_image( + "/sbin/qemu-mount-sources.sh", contents=mount_sources_script, mode=0o755, show_contents_non_verbose=False + ) do_reroot_script = include_local_file("files/cheribsd/qemu-do-reroot.sh.in").format( - SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir) - self.create_file_for_image("/sbin/qemu-do-reroot.sh", contents=do_reroot_script, - mode=0o755, show_contents_non_verbose=False) - self.create_file_for_image("/sbin/startup-benchmark.sh", mode=0o755, show_contents_non_verbose=False, - contents=include_local_file("files/cheribsd/startup-benchmark.sh")) + SRCPATH=self.config.source_root, ROOTFS_DIR=self.rootfs_dir + ) + self.create_file_for_image( + "/sbin/qemu-do-reroot.sh", contents=do_reroot_script, mode=0o755, show_contents_non_verbose=False + ) + self.create_file_for_image( + "/sbin/startup-benchmark.sh", + mode=0o755, + show_contents_non_verbose=False, + contents=include_local_file("files/cheribsd/startup-benchmark.sh"), + ) # Add a script to launch gdb, run a program and get a backtrace: - self.create_file_for_image("/usr/bin/gdb-run.sh", contents=include_local_file("files/cheribsd/gdb-run.sh"), - mode=0o755, show_contents_non_verbose=False) + self.create_file_for_image( + "/usr/bin/gdb-run.sh", + contents=include_local_file("files/cheribsd/gdb-run.sh"), + mode=0o755, + show_contents_non_verbose=False, + ) # And another one for non-interactive use: - self.create_file_for_image("/usr/bin/gdb-run-noninteractive.sh", - contents=include_local_file("files/cheribsd/gdb-run-noninteractive.sh"), - mode=0o755, show_contents_non_verbose=False) + self.create_file_for_image( + "/usr/bin/gdb-run-noninteractive.sh", + contents=include_local_file("files/cheribsd/gdb-run-noninteractive.sh"), + mode=0o755, + show_contents_non_verbose=False, + ) # Add a script to turn of network and stop running services: - self.create_file_for_image("/usr/bin/prepare-benchmark-environment.sh", - contents=include_local_file("files/cheribsd/prepare-benchmark-environment.sh"), - mode=0o755, show_contents_non_verbose=False) + self.create_file_for_image( + "/usr/bin/prepare-benchmark-environment.sh", + contents=include_local_file("files/cheribsd/prepare-benchmark-environment.sh"), + mode=0o755, + show_contents_non_verbose=False, + ) # Update test suite config to skip tests disabled in FreeBSD CI and skip slow tests by default. # For example, the mkimg tests take almost 6 hours out of 22 total on RISCV purecap. @@ -391,8 +463,9 @@ def path_relative_to_outputroot(xtarget) -> Path: # make sure we can login as root with pubkey auth: new_kyua_config_contents = self.read_file(kyua_config) new_kyua_config_contents += include_local_file("files/cheribsd/kyua.conf.append") - self.create_file_for_image("/" + kyua_config_path, contents=new_kyua_config_contents, - mode=0o644, show_contents_non_verbose=False) + self.create_file_for_image( + "/" + kyua_config_path, contents=new_kyua_config_contents, mode=0o644, show_contents_non_verbose=False + ) # make sure that the disk image always has the same SSH host keys # If they don't exist the system will generate one on first boot and we have to accept them every time @@ -407,23 +480,26 @@ def path_relative_to_outputroot(xtarget) -> Path: new_sshd_config_contents = self.read_file(sshd_config) new_sshd_config_contents += "\n# Allow root login with pubkey auth:\nPermitRootLogin without-password\n" new_sshd_config_contents += "\n# Major speedup to SSH performance:\n UseDNS no\n" - self.create_file_for_image("/etc/ssh/sshd_config", contents=new_sshd_config_contents, - mode=0o644, show_contents_non_verbose=False) + self.create_file_for_image( + "/etc/ssh/sshd_config", contents=new_sshd_config_contents, mode=0o644, show_contents_non_verbose=False + ) # now try adding the right ~/.ssh/authorized_keys authorized_keys = self.extra_files_dir / "root/.ssh/authorized_keys" if not authorized_keys.is_file(): ssh_keys = list(Path(os.path.expanduser("~/.ssh/")).glob("*.pub")) if len(ssh_keys) > 0: print("Found the following ssh keys:", list(map(str, ssh_keys))) - if self.query_yes_no("Would you like to add them to /root/.ssh/authorized_keys in the image?", - default_result=True): + if self.query_yes_no( + "Would you like to add them to /root/.ssh/authorized_keys in the image?", default_result=True + ): contents = "" for pubkey in ssh_keys: contents += self.read_file(pubkey) self.create_file_for_image("/root/.ssh/authorized_keys", contents=contents, mode=0o600) - if self.query_yes_no("Should this authorized_keys file be used by default? " - "(You can always change them by editing/deleting '" + - str(authorized_keys) + "')?"): + if self.query_yes_no( + "Should this authorized_keys file be used by default? " + "(You can always change them by editing/deleting '" + str(authorized_keys) + "')?" + ): self.install_file(self.tmpdir / "root/.ssh/authorized_keys", authorized_keys) # SSHD complains and rejects all connections if /root or /root/.ssh is not 0700 self.run_cmd("chmod", "0700", authorized_keys.parent.parent, authorized_keys.parent) @@ -431,14 +507,14 @@ def path_relative_to_outputroot(xtarget) -> Path: loader_conf_contents = "" if self.is_x86: - loader_conf_contents += "console=\"comconsole\"\nautoboot_delay=0\n" + loader_conf_contents += 'console="comconsole"\nautoboot_delay=0\n' if self.no_autoboot: if self.crosscompile_target.is_aarch64(include_purecap=True) or self.is_x86: - loader_conf_contents += "autoboot_delay=\"NO\"\nbeastie_disable=\"YES\"\n" + loader_conf_contents += 'autoboot_delay="NO"\nbeastie_disable="YES"\n' else: self.warning("--no-autoboot is not supported for this target, ignoring.") if self.rootfs_type == FileSystemType.ZFS: - loader_conf_contents += "zfs_load=\"YES\"\n" + loader_conf_contents += 'zfs_load="YES"\n' self.create_file_for_image("/boot/loader.conf", contents=loader_conf_contents, mode=0o644) # Avoid long boot time on first start due to missing entropy: @@ -499,7 +575,7 @@ def add_gdb(self): def add_all_files_in_dir(self, root_dir: Path): for root, dirnames, filenames in os.walk(str(root_dir)): - for ignored_dirname in ('.svn', '.git', '.idea'): + for ignored_dirname in (".svn", ".git", ".idea"): if ignored_dirname in dirnames: dirnames.remove(ignored_dirname) # Symlinks that point to directories are included in dirnames as a @@ -517,8 +593,10 @@ def is_x86(self): def run_mkimg(self, cmd: "list[str]", **kwargs): if not self.mkimg_cmd or not self.mkimg_cmd.exists(): - self.fatal(f"Missing mkimg command ('{self.mkimg_cmd}')! Should be found in FreeBSD build dir.", - fixit_hint="Pass an explicit path to mkimg by setting the MKIMG_CMD environment variable") + self.fatal( + f"Missing mkimg command ('{self.mkimg_cmd}')! Should be found in FreeBSD build dir.", + fixit_hint="Pass an explicit path to mkimg by setting the MKIMG_CMD environment variable", + ) self.run_cmd([self.mkimg_cmd, *cmd], **kwargs) @property @@ -560,13 +638,20 @@ def make_x86_disk_image(self, out_img: Path): raise ValueError("Invalid FileSystemType") # See mk_nogeli_gpt_ufs_legacy in tools/boot/rootgen.sh in FreeBSD - self.run_mkimg(["-s", "gpt", # use GUID Partition Table (GPT) - # "-f", "raw", # raw disk image instead of qcow2 - "-b", self.rootfs_dir / "boot/pmbr", # bootload (MBR) - *mkimg_bootfs_args, - *mkimg_rootfs_args, - "-o", out_img, # output file - ], cwd=self.rootfs_dir) + self.run_mkimg( + [ + "-s", + "gpt", # use GUID Partition Table (GPT) + # "-f", "raw", # raw disk image instead of qcow2 + "-b", + self.rootfs_dir / "boot/pmbr", # bootload (MBR) + *mkimg_bootfs_args, + *mkimg_rootfs_args, + "-o", + out_img, # output file + ], + cwd=self.rootfs_dir, + ) finally: self.delete_file(root_partition) # no need to keep the partition now that we have built the full image @@ -594,13 +679,19 @@ def make_gpt_disk_image(self, out_img: Path): mkimg_rootfs_args = ["-p", f"freebsd-{self.rootfs_type.value}:={root_partition}"] self.make_rootfs_image(root_partition) - self.run_mkimg(["-s", "gpt", # use GUID Partition Table (GPT) - # "-f", "raw", # raw disk image instead of qcow2 - *mkimg_efi_args, - *mkimg_rootfs_args, - *mkimg_swap_args, - "-o", out_img, # output file - ], cwd=self.rootfs_dir) + self.run_mkimg( + [ + "-s", + "gpt", # use GUID Partition Table (GPT) + # "-f", "raw", # raw disk image instead of qcow2 + *mkimg_efi_args, + *mkimg_rootfs_args, + *mkimg_swap_args, + "-o", + out_img, # output file + ], + cwd=self.rootfs_dir, + ) finally: self.delete_file(root_partition) # no need to keep the partition now that we have built the full image if efi_partition is not None: @@ -628,19 +719,32 @@ def make_efi_partition(self, efi_partition: Path): if use_makefs: # Makefs doesn't handle contents= right now efi_mtree = MtreeFile(verbose=self.config.verbose) - efi_mtree.add_file(self.rootfs_dir / "boot" / loader_file, path_in_image="efi/boot/" + efi_file.lower(), - mode=0o644) + efi_mtree.add_file( + self.rootfs_dir / "boot" / loader_file, path_in_image="efi/boot/" + efi_file.lower(), mode=0o644 + ) efi_mtree.write(tmp_mtree, pretend=self.config.pretend) tmp_mtree.flush() # ensure the file is actually written self.run_cmd("cat", tmp_mtree.name) # Note: it appears msdosfs makefs only works if you pass a fixed size, so use 2m which is large # enough to fit either loader_simp.efi or loader.efi. - self.run_cmd([self.makefs_cmd, "-t", "msdos", "-s", "2m", - # "-d", "0x2fffffff", # super verbose output - # "-d", "0x20000000", # MSDOSFS debug output - "-B", "le", # byte order little endian - "-N", self.user_group_db_dir, - str(efi_partition), str(tmp_mtree.name)], cwd=self.rootfs_dir) + self.run_cmd( + [ + self.makefs_cmd, + "-t", + "msdos", + "-s", + "2m", + # "-d", "0x2fffffff", # super verbose output + # "-d", "0x20000000", # MSDOSFS debug output + "-B", + "le", # byte order little endian + "-N", + self.user_group_db_dir, + str(efi_partition), + str(tmp_mtree.name), + ], + cwd=self.rootfs_dir, + ) else: # Use this (and mtools) instead: https://wiki.osdev.org/UEFI_Bare_Bones#Creating_the_FAT_image if not (mtools_bin / "mformat").exists(): @@ -649,8 +753,13 @@ def make_efi_partition(self, efi_partition: Path): self.run_cmd(mtools_bin / "mformat", "-i", efi_partition, "-f", "2048", "::") self.run_cmd(mtools_bin / "mmd", "-i", efi_partition, "::/EFI") self.run_cmd(mtools_bin / "mmd", "-i", efi_partition, "::/EFI/BOOT") - self.run_cmd(mtools_bin / "mcopy", "-i", efi_partition, - self.rootfs_dir / "boot" / loader_file, "::/EFI/BOOT/" + efi_file.upper()) + self.run_cmd( + mtools_bin / "mcopy", + "-i", + efi_partition, + self.rootfs_dir / "boot" / loader_file, + "::/EFI/BOOT/" + efi_file.upper(), + ) if (mtools_bin / "minfo").exists(): # Get some information about the created image information: self.run_cmd(mtools_bin / "minfo", "-i", efi_partition) @@ -666,9 +775,12 @@ def make_rootfs_image(self, rootfs_img: Path): makefs_flags = [] if self.rootfs_type == FileSystemType.ZFS: makefs_flags = [ - "-t", "zfs", - "-o", "poolname=zroot,rootpath=/,bootfs=zroot", - "-s", "5g", + "-t", + "zfs", + "-o", + "poolname=zroot,rootpath=/,bootfs=zroot", + "-s", + "5g", ] elif self.rootfs_type == FileSystemType.UFS: debug_options = [] @@ -685,31 +797,52 @@ def make_rootfs_image(self, rootfs_img: Path): if self.is_x86: # x86: -t ffs -f 200000 -s 8g -o version=2,bsize=32768,fsize=4096 extra_flags = ["-o", "bsize=32768,fsize=4096,label=root"] - makefs_flags = debug_options + extra_flags + [ - "-t", "ffs", # BSD fast file system - "-o", "version=2,label=root", # UFS2 - "-o", "softupdates=1", # Enable soft updates journaling - "-Z", # sparse file output - "-b", free_blocks, - "-f", "1k" if self.is_minimal else "200k", - # minimum 1024 free inodes for minimal, otherwise at least 1M - "-R", "4m", # round up size to the next 4m multiple - "-M", self.minimum_image_size, - "-B", "be" if self.big_endian else "le", # byte order - ] + makefs_flags = ( + debug_options + + extra_flags + + [ + "-t", + "ffs", # BSD fast file system + "-o", + "version=2,label=root", # UFS2 + "-o", + "softupdates=1", # Enable soft updates journaling + "-Z", # sparse file output + "-b", + free_blocks, + "-f", + "1k" if self.is_minimal else "200k", + # minimum 1024 free inodes for minimal, otherwise at least 1M + "-R", + "4m", # round up size to the next 4m multiple + "-M", + self.minimum_image_size, + "-B", + "be" if self.big_endian else "le", # byte order + ] + ) try: - self.run_cmd([self.makefs_cmd, *makefs_flags, - "-N", self.user_group_db_dir, - # use master.passwd from the cheribsd source not the current systems passwd file - # which makes sure that the numeric UID values are correct - rootfs_img, # output file - self.manifest_file, # use METALOG as the manifest for the disk image - ], cwd=self.rootfs_dir) + self.run_cmd( + [ + self.makefs_cmd, + *makefs_flags, + "-N", + self.user_group_db_dir, + # use master.passwd from the cheribsd source not the current systems passwd file + # which makes sure that the numeric UID values are correct + rootfs_img, # output file + self.manifest_file, # use METALOG as the manifest for the disk image + ], + cwd=self.rootfs_dir, + ) except Exception: - self.warning("makefs failed, if it reports an issue with METALOG report a bug (could be either cheribuild" - " or cheribsd) and attach the METALOG file.") - self.query_yes_no("About to delete the temporary directory. Copy any files you need before pressing enter.", - yes_no_str="") + self.warning( + "makefs failed, if it reports an issue with METALOG report a bug (could be either cheribuild" + " or cheribsd) and attach the METALOG file." + ) + self.query_yes_no( + "About to delete the temporary directory. Copy any files you need before pressing enter.", yes_no_str="" + ) raise def make_disk_image(self): @@ -746,11 +879,16 @@ def make_disk_image(self): if not qemu_img_command.exists(): self.fatal("Cannot create QCOW2 image without qemu-img command!") # create a qcow2 version from the raw image: - self.run_cmd(qemu_img_command, "convert", - "-f", "raw", # input file is in raw format (not required as QEMU can detect it - "-O", "qcow2", # convert to qcow2 format - raw_img, # input file - self.disk_image_path) # output file + self.run_cmd( + qemu_img_command, + "convert", + "-f", + "raw", # input file is in raw format (not required as QEMU can detect it + "-O", + "qcow2", # convert to qcow2 format + raw_img, # input file + self.disk_image_path, + ) # output file self.delete_file(raw_img, print_verbose_only=True) if self.config.verbose: self.run_cmd(qemu_img_command, "info", self.disk_image_path) @@ -784,10 +922,13 @@ def __process(self): # with --clean always delete the image opt = self.get_config_option_name("force_overwrite") self.info("An image already exists (" + str(self.disk_image_path) + "). ", end="") - self.info("Note: Pass", coloured(AnsiColour.yellow, "--" + opt), - coloured(AnsiColour.cyan, "to skip this prompt or add"), - coloured(AnsiColour.yellow, "\"" + opt + "\": true"), - coloured(AnsiColour.cyan, "to", self.config.loader.config_file_path)) + self.info( + "Note: Pass", + coloured(AnsiColour.yellow, "--" + opt), + coloured(AnsiColour.cyan, "to skip this prompt or add"), + coloured(AnsiColour.yellow, '"' + opt + '": true'), + coloured(AnsiColour.cyan, "to", self.config.loader.config_file_path), + ) if not self.query_yes_no("Overwrite?", default_result=True): return # we are done here self.delete_file(self.disk_image_path) @@ -812,9 +953,12 @@ def __process(self): self.mkimg_cmd = mkimg_xtool if not self.makefs_cmd or not self.makefs_cmd.exists(): - self.fatal("Missing makefs command ('{}')! Should be found in FreeBSD build dir ({})".format( - self.makefs_cmd, freebsd_builddir), - fixit_hint="Pass an explicit path to makefs by setting the MAKEFS_CMD environment variable") + self.fatal( + "Missing makefs command ('{}')! Should be found in FreeBSD build dir ({})".format( + self.makefs_cmd, freebsd_builddir + ), + fixit_hint="Pass an explicit path to makefs by setting the MAKEFS_CMD environment variable", + ) self.info("Disk image will be saved to", self.disk_image_path) self.info("Disk image root fs is", self.rootfs_dir) self.info("Extra files for the disk image will be copied from", self.extra_files_dir) @@ -870,8 +1014,9 @@ def add_unlisted_files_to_metalog(self): print("Found the following files in the rootfs that are not listed in METALOG:") for i in unlisted_files: print("\t", i[1]) - if self.query_yes_no("Should these files also be added to the image?", default_result=True, - force_result=True): + if self.query_yes_no( + "Should these files also be added to the image?", default_result=True, force_result=True + ): for i in unlisted_files: self.mtree.add_file(i[0], i[1], print_status=self.config.verbose) @@ -888,9 +1033,15 @@ def generate_ssh_host_keys(self): private_key = ssh_dir / private_key_name public_key = ssh_dir / (private_key_name + ".pub") if not private_key.is_file(): - self.run_cmd("ssh-keygen", "-t", key_type, - "-N", "", # no passphrase - "-f", str(private_key)) + self.run_cmd( + "ssh-keygen", + "-t", + key_type, + "-N", + "", # no passphrase + "-f", + str(private_key), + ) self.add_file_to_image(private_key, base_directory=self.extra_files_dir, mode="0600") self.add_file_to_image(public_key, base_directory=self.extra_files_dir, mode="0644") @@ -908,17 +1059,23 @@ def get_rc_conf_template(self): @classmethod def setup_config_options(cls, **kwargs): - super().setup_config_options(default_hostname=_default_disk_image_hostname("cheribsd-minimal"), - extra_files_suffix="-minimal", **kwargs) + super().setup_config_options( + default_hostname=_default_disk_image_hostname("cheribsd-minimal"), extra_files_suffix="-minimal", **kwargs + ) cls.strip_binaries = cls.add_bool_option( - "strip", default=True, help="strip ELF files to reduce size of generated image") + "strip", default=True, help="strip ELF files to reduce size of generated image" + ) cls.include_cheribsdtest = cls.add_bool_option( - "include-cheribsdtest", default=True, help="Also add static cheribsdtest base variants to the disk image") - cls.include_pmc = cls.add_bool_option("include-pmc", default=True, - help="Include PMC tools and kernel modules in the disk image") - cls.kernels = cls.add_list_option("kernel-names", default=[""], - help="Kernel(s) to include in the image; empty string or '/' for " - "/boot/kernel/, X for /boot/kernel.X/") + "include-cheribsdtest", default=True, help="Also add static cheribsdtest base variants to the disk image" + ) + cls.include_pmc = cls.add_bool_option( + "include-pmc", default=True, help="Include PMC tools and kernel modules in the disk image" + ) + cls.kernels = cls.add_list_option( + "kernel-names", + default=[""], + help="Kernel(s) to include in the image; empty string or '/' for " "/boot/kernel/, X for /boot/kernel.X/", + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -959,8 +1116,10 @@ def process_files_list(self, files_list): def add_unlisted_files_to_metalog(self): # Now add all the files from *.files to the image: self.verbose_print("Adding files from rootfs to minimal image:") - files_to_add = [include_local_file("files/minimal-image/base.files"), - include_local_file("files/minimal-image/etc.files")] + files_to_add = [ + include_local_file("files/minimal-image/base.files"), + include_local_file("files/minimal-image/etc.files"), + ] if self._have_cplusplus_support(["lib", "usr/lib"]): files_to_add.append(include_local_file("files/minimal-image/need-cplusplus.files")) @@ -1000,8 +1159,9 @@ def add_unlisted_files_to_metalog(self): fullpath = self.rootfs_dir / "usr" / libcompat_dir if fullpath.is_symlink(): # add the libcompat symlinks to ensure that we can always use lib64/lib64c in test scripts - self.mtree.add_symlink(src_symlink=self.rootfs_dir / "usr" / libcompat_dir, - path_in_image="usr/" + libcompat_dir) + self.mtree.add_symlink( + src_symlink=self.rootfs_dir / "usr" / libcompat_dir, path_in_image="usr/" + libcompat_dir + ) if (self.rootfs_dir / libcompat_dir).is_symlink(): self.mtree.add_symlink(src_symlink=self.rootfs_dir / libcompat_dir, path_in_image=libcompat_dir) elif (fullpath / "libc.so").exists(): @@ -1063,13 +1223,16 @@ def add_required_libraries(self, libdirs: "list[str]", ignore_required: bool = F # required, but versions were bumped with changes to ncurses optional_libs += [ # needed by /bin/sh & /bin/csh (if we included the purecap sh/csh) - "libedit.so.7", "libedit.so.8", + "libedit.so.7", + "libedit.so.8", ] # required, but versions were bumped for OpenSSL 3 optional_libs += [ # cheribsdbox depends on SSL - "libcrypto.so.111", "libcrypto.so.30", - "libssl.so.111", "libssl.so.30", + "libcrypto.so.111", + "libcrypto.so.30", + "libssl.so.111", + "libssl.so.30", ] # additional cheribsdbox dependencies (PAM+SSL+BSM) # We don't know what ABI cheribsdbox is built for so let's just add the libraries for all ABIs @@ -1113,25 +1276,47 @@ def add_required_libraries(self, libdirs: "list[str]", ignore_required: bool = F else: prefix = "{" + ",".join(libdirs) + "}/" if required and not ignore_required: - self.fatal("Could not find required library '", prefix + library_basename, "' in rootfs ", - self.rootfs_dir, sep="") + self.fatal( + "Could not find required library '", + prefix + library_basename, + "' in rootfs ", + self.rootfs_dir, + sep="", + ) else: - self.info("Could not find optional library '", prefix + library_basename, "' in rootfs ", - self.rootfs_dir, sep="") + self.info( + "Could not find optional library '", + prefix + library_basename, + "' in rootfs ", + self.rootfs_dir, + sep="", + ) continue self.add_file_to_image(full_lib_path, base_directory=self.rootfs_dir) def prepare_rootfs(self): super().prepare_rootfs() # Add the additional sysctl configs - self.create_file_for_image("/etc/pam.d/system", mode=0o644, show_contents_non_verbose=False, - contents=include_local_file("files/minimal-image/pam.d/system")) + self.create_file_for_image( + "/etc/pam.d/system", + mode=0o644, + show_contents_non_verbose=False, + contents=include_local_file("files/minimal-image/pam.d/system"), + ) # disable coredumps (since there is almost no space on the image) - self.create_file_for_image("/etc/sysctl.conf", mode=0o644, show_contents_non_verbose=False, - contents=include_local_file("files/minimal-image/etc/sysctl.conf")) + self.create_file_for_image( + "/etc/sysctl.conf", + mode=0o644, + show_contents_non_verbose=False, + contents=include_local_file("files/minimal-image/etc/sysctl.conf"), + ) # The actual minimal startup file: - self.create_file_for_image("/etc/rc", mode=0o644, show_contents_non_verbose=False, - contents=include_local_file("files/minimal-image/etc/rc")) + self.create_file_for_image( + "/etc/rc", + mode=0o644, + show_contents_non_verbose=False, + contents=include_local_file("files/minimal-image/etc/rc"), + ) def make_rootfs_image(self, rootfs_img: Path): # update cheribsdbox link in case we stripped it: @@ -1147,8 +1332,10 @@ def make_rootfs_image(self, rootfs_img: Path): self.delete_file(dummy_hardlink) os.link(str(cheribsdbox_path), str(dummy_hardlink)) if Path(cheribsdbox_path).stat().st_nlink < 2: - self.fatal("Need at least one hardlink to cheribsdbox so that makefs can detect deduplicate. " - "This should have been created by cheribuild but something must have gone wrong") + self.fatal( + "Need at least one hardlink to cheribsdbox so that makefs can detect deduplicate. " + "This should have been created by cheribuild but something must have gone wrong" + ) print("Relocating mtree path ./bin/cheribsdbox to use", cheribsdbox_path) # noinspection PyProtectedMember for i in self.mtree._mtree.values(): @@ -1217,7 +1404,8 @@ class BuildCheriBSDTarball(BuildCheriBSDDiskImage): default_disk_image_path = ComputedDefaultValue( function=lambda conf, proj: _default_tar_name(conf, conf.output_root, proj), - as_string=lambda cls: "$OUTPUT_ROOT/" + cls.disk_image_prefix + "-.tar.xz depending on architecture") + as_string=lambda cls: "$OUTPUT_ROOT/" + cls.disk_image_prefix + "-.tar.xz depending on architecture", + ) def check_system_dependencies(self) -> None: super().check_system_dependencies() @@ -1234,10 +1422,13 @@ def make_disk_image(self): try: self.run_cmd([bsdtar_path, "acf", self.disk_image_path, "@" + str(self.manifest_file)], cwd=self.rootfs_dir) except Exception: - self.warning("bsdtar failed, if it reports an issue with METALOG report a bug (could be either cheribuild" - " or cheribsd) and attach the METALOG file.") - self.query_yes_no("About to delete the temporary directory. Copy any files you need before pressing enter.", - yes_no_str="") + self.warning( + "bsdtar failed, if it reports an issue with METALOG report a bug (could be either cheribuild" + " or cheribsd) and attach the METALOG file." + ) + self.query_yes_no( + "About to delete the temporary directory. Copy any files you need before pressing enter.", yes_no_str="" + ) raise diff --git a/pycheribuild/projects/docker_adduser.py b/pycheribuild/projects/docker_adduser.py index e95d33676..f8423d7b7 100644 --- a/pycheribuild/projects/docker_adduser.py +++ b/pycheribuild/projects/docker_adduser.py @@ -65,11 +65,15 @@ def process(self): gid = uid # Create a Dockerfile that will contain this user's name, gid, uid - self.write_file(self.build_dir / "Dockerfile", overwrite=True, contents=f""" + self.write_file( + self.build_dir / "Dockerfile", + overwrite=True, + contents=f""" FROM {self.config.docker_container} RUN addgroup --gid {gid} {user} && \ adduser --uid {uid} --ingroup {user} {user} -""") +""", + ) # Build a new image from our installed image with this user try: @@ -78,8 +82,6 @@ def process(self): except subprocess.CalledProcessError: # if the image is missing print a helpful error message: - error = "Failed to add your user to the docker image " + \ - self.config.docker_container - hint = "Ensure you have " + self.config.docker_container + \ - "available (check using docker image ls)" + error = "Failed to add your user to the docker image " + self.config.docker_container + hint = "Ensure you have " + self.config.docker_container + "available (check using docker image ls)" self.fatal(error, fixit_hint=hint) diff --git a/pycheribuild/projects/effectivesan.py b/pycheribuild/projects/effectivesan.py index 69da01025..71343cfae 100644 --- a/pycheribuild/projects/effectivesan.py +++ b/pycheribuild/projects/effectivesan.py @@ -46,8 +46,8 @@ def get_native_install_path(cls, config: CheriConfig): default_directory_basename = "EffectiveSan" repository = GitRepository("https://github.com/GJDuck/EffectiveSan") _default_install_dir_fn = ComputedDefaultValue( - function=lambda config, project: project.get_native_install_path(config), - as_string="$INSTALL_ROOT/effectivesan") + function=lambda config, project: project.get_native_install_path(config), as_string="$INSTALL_ROOT/effectivesan" + ) skip_cheri_symlinks = True root_cmakelists_subdirectory = Path("llvm-4.0.1.src") diff --git a/pycheribuild/projects/fvp_firmware.py b/pycheribuild/projects/fvp_firmware.py index 6abb83f5c..bcb50918e 100644 --- a/pycheribuild/projects/fvp_firmware.py +++ b/pycheribuild/projects/fvp_firmware.py @@ -75,15 +75,27 @@ def process(self): self.fatal("Unsupported CPU architecture") filename = "gcc-arm-none-eabi-9-2020-q2-update-mac.tar.bz2" if filename is None: - self.fatal("Cannot infer download URL for current OS:", platform.platform(), - fixit_hint="Please visit https://developer.arm.com/tools-and-software/open-source-software/" - "developer-tools/gnu-toolchain/gnu-rm/downloads and select the appropriate download.") + self.fatal( + "Cannot infer download URL for current OS:", + platform.platform(), + fixit_hint="Please visit https://developer.arm.com/tools-and-software/open-source-software/" + "developer-tools/gnu-toolchain/gnu-rm/downloads and select the appropriate download.", + ) return if not (self.config.build_root / filename).is_file() or self.with_clean: self.download_file(self.config.build_root / filename, url_prefix + filename, "-O") with self.async_clean_directory(self.config.output_root / self.config.local_arm_none_eabi_toolchain_relpath): - self.run_cmd(["tar", "xf", self.config.build_root / filename, "--strip-components", "1", "-C", - self.config.output_root / self.config.local_arm_none_eabi_toolchain_relpath]) + self.run_cmd( + [ + "tar", + "xf", + self.config.build_root / filename, + "--strip-components", + "1", + "-C", + self.config.output_root / self.config.local_arm_none_eabi_toolchain_relpath, + ] + ) class MorelloFirmwareBase(CrossCompileMakefileProject): @@ -92,8 +104,9 @@ class MorelloFirmwareBase(CrossCompileMakefileProject): cross_install_dir = DefaultInstallDir.CUSTOM_INSTALL_DIR # TODO: install it needs_sysroot = False # We don't need a complete sysroot default_build_type = BuildType.RELEASE - _default_install_dir_fn = ComputedDefaultValue(function=_morello_firmware_build_outputs_dir, - as_string="$MORELLO_SDK_ROOT/firmware/morello-fvp") + _default_install_dir_fn = ComputedDefaultValue( + function=_morello_firmware_build_outputs_dir, as_string="$MORELLO_SDK_ROOT/firmware/morello-fvp" + ) @property def optimization_flags(self): @@ -127,18 +140,26 @@ def setup(self): def process(self): if not self.CC.exists(): - self.fatal("Could not find", self.CC, - fixit_hint="Install the ARM GCC manually or use " - "`cheribuild.py " + ArmNoneEabiToolchain.target + "`") + self.fatal( + "Could not find", + self.CC, + fixit_hint="Install the ARM GCC manually or use " "`cheribuild.py " + ArmNoneEabiToolchain.target + "`", + ) super().process() def install(self, **kwargs): binaries_dir = self.build_dir / "build/product/morello" for i in ("mcp_ramfw_fvp", "scp_ramfw_fvp", "mcp_romfw", "scp_romfw"): - self.install_file(binaries_dir / i / self.build_mode / "bin" / (i + ".bin"), - self.install_dir / (i + ".bin"), print_verbose_only=False) - self.install_file(binaries_dir / i / self.build_mode / "bin" / (i + ".elf"), - self.install_dir / (i + ".elf"), print_verbose_only=False) + self.install_file( + binaries_dir / i / self.build_mode / "bin" / (i + ".bin"), + self.install_dir / (i + ".bin"), + print_verbose_only=False, + ) + self.install_file( + binaries_dir / i / self.build_mode / "bin" / (i + ".elf"), + self.install_dir / (i + ".elf"), + print_verbose_only=False, + ) def run_tests(self): self.run_make(make_target="test") # XXX: doesn't work yet, needs a read/write/isatty() @@ -157,10 +178,14 @@ class BuildMorelloTrustedFirmware(MorelloFirmwareBase): default_directory_basename = "morello-trusted-firmware-a" repository = GitRepository( "https://git.morello-project.org/morello/trusted-firmware-a.git", - force_branch=True, default_branch="morello/master", - old_urls=[b"git@git.morello-project.org:morello/trusted-firmware-a.git", - b"git@git.morello-project.org:university-of-cambridge/trusted-firmware-a.git", - b"https://git.morello-project.org/university-of-cambridge/trusted-firmware-a.git"]) + force_branch=True, + default_branch="morello/master", + old_urls=[ + b"git@git.morello-project.org:morello/trusted-firmware-a.git", + b"git@git.morello-project.org:university-of-cambridge/trusted-firmware-a.git", + b"https://git.morello-project.org/university-of-cambridge/trusted-firmware-a.git", + ], + ) set_commands_on_cmdline = True # Need to override this on the command line since the makefile uses := def check_system_dependencies(self) -> None: @@ -169,16 +194,18 @@ def check_system_dependencies(self) -> None: def setup(self): super().setup() - self.make_args.set(ENABLE_MORELLO_CAP=1, PLAT="morello", ARCH="aarch64", - DEBUG=1 if self.build_type.is_debug else 0, - CSS_USE_SCMI_SDS_DRIVER=1, - E=0, # disable -Werror since there are some unused functions - V=1, # verbose - ) + self.make_args.set( + ENABLE_MORELLO_CAP=1, + PLAT="morello", + ARCH="aarch64", + DEBUG=1 if self.build_type.is_debug else 0, + CSS_USE_SCMI_SDS_DRIVER=1, + E=0, # disable -Werror since there are some unused functions + V=1, # verbose + ) self.make_args.set_env(CROSS_COMPILE=str(self.sdk_bindir) + "/") # Need to override this on the command line, not just the environment) - self.make_args.set(LD=self.target_info.linker, - LINKER=self.target_info.linker) + self.make_args.set(LD=self.target_info.linker, LINKER=self.target_info.linker) # Uses raw linker -> don't set LDFLAGS self.make_args.set_env(LDFLAGS="-verbose") self.make_args.set(HOSTCC=self.host_CC) @@ -190,9 +217,11 @@ def compile(self, **kwargs): if OSInfo.IS_MAC: # TODO: should handle non-homebrew too openssl_prefix = self.get_homebrew_prefix("openssl") - fip_make.set_env(HOSTLDFLAGS="-L" + str(openssl_prefix / "lib"), - HOSTCCFLAGS="-I" + str(openssl_prefix / "include"), - CPPFLAGS="-I" + str(openssl_prefix / "include")) + fip_make.set_env( + HOSTLDFLAGS="-L" + str(openssl_prefix / "lib"), + HOSTCCFLAGS="-I" + str(openssl_prefix / "include"), + CPPFLAGS="-I" + str(openssl_prefix / "include"), + ) # FIXME: Makefile doesn't add HOSTLDFLAGS fip_make.set(HOSTCC=str(self.host_CC) + " -Qunused-arguments " + fip_make.env_vars["HOSTLDFLAGS"]) self.run_make(make_target="all", cwd=self.source_dir / "tools/fiptool", options=fip_make) @@ -200,10 +229,14 @@ def compile(self, **kwargs): def install(self, **kwargs): output_dir = self.build_dir / "build/morello" / ("debug" if self.build_type.is_debug else "release") self.install_file(output_dir / "bl31.bin", self.install_dir / "tf-bl31.bin", print_verbose_only=False) - self.install_file(output_dir / "fdts/morello-fvp.dtb", self.install_dir / "morello-fvp.dtb", - print_verbose_only=False) - self.install_file(self.build_dir / "tools/fiptool/fiptool", self.config.morello_sdk_dir / "bin/fiptool", - print_verbose_only=False) + self.install_file( + output_dir / "fdts/morello-fvp.dtb", self.install_dir / "morello-fvp.dtb", print_verbose_only=False + ) + self.install_file( + self.build_dir / "tools/fiptool/fiptool", + self.config.morello_sdk_dir / "bin/fiptool", + print_verbose_only=False, + ) class BuildMorelloACPICA(MakefileProject): @@ -221,18 +254,23 @@ def is_toolchain_target(cls): def setup(self): super().setup() # Seems unhappy if you use clang on Linux - self.make_args.set(CC="/usr/bin/cc", CPP="/usr/bin/cpp", CXX="/usr/bin/c++", CCLD="/usr/bin/cc", - CXXLD="/usr/bin/c++") + self.make_args.set( + CC="/usr/bin/cc", CPP="/usr/bin/cpp", CXX="/usr/bin/c++", CCLD="/usr/bin/cc", CXXLD="/usr/bin/c++" + ) class BuildMorelloUEFI(MorelloFirmwareBase): repository = GitRepository("https://git.morello-project.org/morello/edk2.git") morello_platforms_repository = GitRepository( "https://git.morello-project.org/morello/edk2-platforms.git", - force_branch=True, default_branch="morello/master", - old_urls=[b"git@git.morello-project.org:morello/edk2-platforms.git", - b"git@git.morello-project.org:university-of-cambridge/edk2-platforms.git", - b"https://git.morello-project.org/university-of-cambridge/edk2-platforms.git"]) + force_branch=True, + default_branch="morello/master", + old_urls=[ + b"git@git.morello-project.org:morello/edk2-platforms.git", + b"git@git.morello-project.org:university-of-cambridge/edk2-platforms.git", + b"https://git.morello-project.org/university-of-cambridge/edk2-platforms.git", + ], + ) dependencies = ("gdb-native", "morello-acpica") # To get ld.bfd target = "morello-uefi" default_directory_basename = "morello-edk2" @@ -242,13 +280,17 @@ class BuildMorelloUEFI(MorelloFirmwareBase): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.edk2_platforms_rev = cls.add_config_option( - "edk2-platforms-git-revision", kind=str, metavar="REVISION", help="The git revision for edk2-platforms") + "edk2-platforms-git-revision", kind=str, metavar="REVISION", help="The git revision for edk2-platforms" + ) def update(self): super().update() - self.morello_platforms_repository.update(self, src_dir=self.source_dir / "edk2-platforms", - skip_submodules=self.skip_git_submodules, - revision=self.edk2_platforms_rev) + self.morello_platforms_repository.update( + self, + src_dir=self.source_dir / "edk2-platforms", + skip_submodules=self.skip_git_submodules, + revision=self.edk2_platforms_rev, + ) def clean(self): super().clean() @@ -270,7 +312,9 @@ def _compile(self, fake_compiler_dir: Path): self.fatal("Missing iasl tool, run the", BuildMorelloACPICA.target, "first.") # Create the fake compiler directory with the tools and a clang wrapper script that forces bfd # Also disable lto since we don't install the LLVM LTO plugin - self.write_file(fake_compiler_dir / "clang", contents="""#!/usr/bin/env python3 + self.write_file( + fake_compiler_dir / "clang", + contents="""#!/usr/bin/env python3 import subprocess import sys @@ -281,10 +325,25 @@ def _compile(self, fake_compiler_dir: Path): continue args.append(arg) subprocess.check_call(["{real_clang}", "-B{fake_dir}"] + args + ["-fuse-ld=bfd", "-fno-lto", "-Qunused-arguments"]) -""".format(real_clang=self.CC, fake_dir=fake_compiler_dir), overwrite=True, mode=0o755) +""".format(real_clang=self.CC, fake_dir=fake_compiler_dir), + overwrite=True, + mode=0o755, + ) self.run_cmd(fake_compiler_dir / "clang", "-v") # check that the script works - for i in ("llvm-objcopy", "llvm-objdump", "llvm-ar", "llvm-ranlib", "objcopy", "objdump", "ar", "ranlib", - "nm", "llvm-nm", "size", "llvm-size"): + for i in ( + "llvm-objcopy", + "llvm-objdump", + "llvm-ar", + "llvm-ranlib", + "objcopy", + "objdump", + "ar", + "ranlib", + "nm", + "llvm-nm", + "size", + "llvm-size", + ): self.create_symlink(self.sdk_bindir / i, fake_compiler_dir / i, relative=False) # EDK2 needs bfd until the lld target is merged @@ -293,18 +352,31 @@ def _compile(self, fake_compiler_dir: Path): self.fatal("Missing ld.bfd, please run `cheribuild.py gdb-native --reconfigure`") self.create_symlink(bfd_path, fake_compiler_dir / "ld", relative=False) self.create_symlink(bfd_path, fake_compiler_dir / "ld.bfd", relative=False) - firmware_ver = self.run_cmd("git", "-C", self.source_dir, "rev-parse", "--short", "HEAD", - run_in_pretend_mode=shutil.which("git") is not None, - capture_output=True).stdout.decode("utf-8").strip() + firmware_ver = ( + self.run_cmd( + "git", + "-C", + self.source_dir, + "rev-parse", + "--short", + "HEAD", + run_in_pretend_mode=shutil.which("git") is not None, + capture_output=True, + ) + .stdout.decode("utf-8") + .strip() + ) # if ! git diff-index --quiet HEAD --; then # FIRMWARE_VER="${FIRMWARE_VER}-dirty" # fi - with self.set_env(CROSS_COMPILE=str(fake_compiler_dir) + "/", - CLANG_BIN=fake_compiler_dir, - EDK2_TOOLCHAIN="CLANG38", - VERBOSE=1, - IASL_PREFIX=str(iasl.parent) + "/", - PATH=str(fake_compiler_dir) + ":" + os.getenv("PATH")): + with self.set_env( + CROSS_COMPILE=str(fake_compiler_dir) + "/", + CLANG_BIN=fake_compiler_dir, + EDK2_TOOLCHAIN="CLANG38", + VERBOSE=1, + IASL_PREFIX=str(iasl.parent) + "/", + PATH=str(fake_compiler_dir) + ":" + os.getenv("PATH"), + ): platform_desc = "Platform/ARM/Morello/MorelloPlatformFvp.dsc" if not (self.source_dir / "edk2-platforms" / platform_desc).exists(): self.fatal("Could not find", self.source_dir / "edk2-platforms" / platform_desc) @@ -320,8 +392,11 @@ def _compile(self, fake_compiler_dir: Path): self.run_shell_script(script, shell="bash", cwd=self.source_dir) def install(self, **kwargs): - self.install_file(self.build_dir / "Build/morellofvp" / (self.build_mode + "_CLANG38") / "FV/BL33_AP_UEFI.fd", - self.install_dir / "uefi.bin", print_verbose_only=False) + self.install_file( + self.build_dir / "Build/morellofvp" / (self.build_mode + "_CLANG38") / "FV/BL33_AP_UEFI.fd", + self.install_dir / "uefi.bin", + print_verbose_only=False, + ) @classmethod def uefi_bin(cls, caller): @@ -331,21 +406,31 @@ def uefi_bin(cls, caller): class BuildMorelloFlashImages(Project): target = "morello-flash-images" dependencies = ("morello-scp-firmware", "morello-trusted-firmware") - _default_install_dir_fn = ComputedDefaultValue(function=_morello_firmware_build_outputs_dir, - as_string="$MORELLO_SDK_ROOT/fvp-firmware/morello/build-outputs") + _default_install_dir_fn = ComputedDefaultValue( + function=_morello_firmware_build_outputs_dir, as_string="$MORELLO_SDK_ROOT/fvp-firmware/morello/build-outputs" + ) repository = ReuseOtherProjectDefaultTargetRepository(source_project=BuildMorelloScpFirmware) def process(self): fw_dir = _morello_firmware_build_outputs_dir(self.config, self) self.info("Building combined SCP and AP flash image") - self.run_cmd(self.config.morello_sdk_dir / "bin/fiptool", "create", - "--scp-fw", fw_dir / "scp_ramfw_fvp.bin", - "--soc-fw", fw_dir / "tf-bl31.bin", - self.scp_ap_ram_firmware_image) + self.run_cmd( + self.config.morello_sdk_dir / "bin/fiptool", + "create", + "--scp-fw", + fw_dir / "scp_ramfw_fvp.bin", + "--soc-fw", + fw_dir / "tf-bl31.bin", + self.scp_ap_ram_firmware_image, + ) self.info("Building MCP flash image") - self.run_cmd(self.config.morello_sdk_dir / "bin/fiptool", "create", - "--blob", "uuid=54464222-a4cf-4bf8-b1b6-cee7dade539e,file=" + str(fw_dir / "mcp_ramfw_fvp.bin"), - self.mcp_ram_firmware_image) + self.run_cmd( + self.config.morello_sdk_dir / "bin/fiptool", + "create", + "--blob", + "uuid=54464222-a4cf-4bf8-b1b6-cee7dade539e,file=" + str(fw_dir / "mcp_ramfw_fvp.bin"), + self.mcp_ram_firmware_image, + ) @property def scp_ap_ram_firmware_image(self): @@ -378,6 +463,9 @@ def process(self): filename = "morello-fvp-firmware-2020.10.tar.xz" fvp_firmware_dir = _morello_firmware_build_outputs_dir(self.config, self) firmware_archive = fvp_firmware_dir.parent / filename - self.download_file(firmware_archive, url=download_url_base + filename, - sha256="440f08a05f2a8e6475e81d2527cc169f0491a6cd177da290e75b0e51363f1412") + self.download_file( + firmware_archive, + url=download_url_base + filename, + sha256="440f08a05f2a8e6475e81d2527cc169f0491a6cd177da290e75b0e51363f1412", + ) self.run_cmd("tar", "xf", firmware_archive, "-C", fvp_firmware_dir.parent) diff --git a/pycheribuild/projects/go.py b/pycheribuild/projects/go.py index 2361ef3c3..ee8495f7b 100644 --- a/pycheribuild/projects/go.py +++ b/pycheribuild/projects/go.py @@ -47,8 +47,9 @@ class BuildGo(Project): @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.go_bootstrap = cls.add_optional_path_option("bootstrap-toolchain", show_help=False, - help="Path to alternate go bootstrap toolchain.") + cls.go_bootstrap = cls.add_optional_path_option( + "bootstrap-toolchain", show_help=False, help="Path to alternate go bootstrap toolchain." + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/pycheribuild/projects/kdevelop.py b/pycheribuild/projects/kdevelop.py index 03b853c68..017396338 100644 --- a/pycheribuild/projects/kdevelop.py +++ b/pycheribuild/projects/kdevelop.py @@ -70,8 +70,9 @@ class StartKDevelop(SimpleProject): def check_system_dependencies(self) -> None: super().check_system_dependencies() - self.check_required_system_tool("cmake", default="cmake", homebrew="cmake", zypper="cmake", apt="cmake", - freebsd="cmake") + self.check_required_system_tool( + "cmake", default="cmake", homebrew="cmake", zypper="cmake", apt="cmake", freebsd="cmake" + ) self.check_required_system_tool("qtpaths") def process(self): diff --git a/pycheribuild/projects/meson_project.py b/pycheribuild/projects/meson_project.py index b8e0d8568..9b2883543 100644 --- a/pycheribuild/projects/meson_project.py +++ b/pycheribuild/projects/meson_project.py @@ -63,14 +63,22 @@ def set_minimum_meson_version(self, major: int, minor: int, patch: int = 0) -> N def _configure_tool_install_instructions(self) -> InstallInstructions: return OSInfo.install_instructions( - "meson", False, default="meson", homebrew="meson", zypper="meson", freebsd="meson", apt="meson", - alternative="run `pip3 install --upgrade --user meson` to install the latest version") + "meson", + False, + default="meson", + homebrew="meson", + zypper="meson", + freebsd="meson", + apt="meson", + alternative="run `pip3 install --upgrade --user meson` to install the latest version", + ) @classmethod def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) - cls.meson_options = cls.add_list_option("meson-options", metavar="OPTIONS", - help="Additional command line options to pass to Meson") + cls.meson_options = cls.add_list_option( + "meson-options", metavar="OPTIONS", help="Additional command line options to pass to Meson" + ) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -87,8 +95,12 @@ def _native_toolchain_file(self) -> Path: return self.build_dir / "meson-native-file.ini" def add_meson_options(self, _include_empty_vars=False, _replace=True, **kwargs) -> None: - return self._add_configure_options(_config_file_options=self.meson_options, _replace=_replace, - _include_empty_vars=_include_empty_vars, **kwargs) + return self._add_configure_options( + _config_file_options=self.meson_options, + _replace=_replace, + _include_empty_vars=_include_empty_vars, + **kwargs, + ) def add_asan_flags(self): self.add_meson_options(b_sanitize="address,undefined", b_lundef=False) @@ -120,8 +132,11 @@ def setup(self) -> None: self.configure_args.append("--wrap-mode=nofallback") self.add_meson_options(**self.build_type.to_meson_args()) if self.use_lto: - self.add_meson_options(b_lto=True, b_lto_threads=self.config.make_jobs, - b_lto_mode="thin" if self.get_compiler_info(self.CC).is_clang else "default") + self.add_meson_options( + b_lto=True, + b_lto_threads=self.config.make_jobs, + b_lto_mode="thin" if self.get_compiler_info(self.CC).is_clang else "default", + ) # Unlike CMake, Meson does not set the DT_RUNPATH entry automatically: # See https://github.com/mesonbuild/meson/issues/6220, https://github.com/mesonbuild/meson/issues/6541, etc. @@ -140,8 +155,7 @@ def needs_configure(self) -> bool: def _toolchain_file_list_to_str(self, values: "list[Union[str, Path]]") -> str: # The meson toolchain file uses python-style lists - assert all(isinstance(x, (str, Path)) for x in values), \ - "All values should be strings/Paths: " + str(values) + assert all(isinstance(x, (str, Path)) for x in values), "All values should be strings/Paths: " + str(values) return str(list(map(str, values))) def _bool_to_str(self, value: bool) -> str: @@ -172,16 +186,21 @@ def configure(self, **kwargs) -> None: host_target_info = NativeTargetInfo(BasicCompilationTargets.NATIVE, None) # pytype: disable=wrong-arg-types host_prefixes = self.host_dependency_prefixes assert self.config.other_tools_dir in host_prefixes - host_pkg_config_dirs = list(itertools.chain.from_iterable( - host_target_info.pkgconfig_candidates(x) for x in host_prefixes)) + host_pkg_config_dirs = list( + itertools.chain.from_iterable(host_target_info.pkgconfig_candidates(x) for x in host_prefixes) + ) self._replace_values_in_toolchain_file( - native_toolchain_template, self._native_toolchain_file, - NATIVE_C_COMPILER=self.host_CC, NATIVE_CXX_COMPILER=self.host_CXX, - TOOLCHAIN_PKGCONFIG_BINARY=pkg_config_bin, TOOLCHAIN_CMAKE_BINARY=cmake_bin, + native_toolchain_template, + self._native_toolchain_file, + NATIVE_C_COMPILER=self.host_CC, + NATIVE_CXX_COMPILER=self.host_CXX, + TOOLCHAIN_PKGCONFIG_BINARY=pkg_config_bin, + TOOLCHAIN_CMAKE_BINARY=cmake_bin, # To find native packages we have to add the bootstrap tools to PKG_CONFIG_PATH and CMAKE_PREFIX_PATH. NATIVE_PKG_CONFIG_PATH=remove_duplicates(host_pkg_config_dirs), NATIVE_CMAKE_PREFIX_PATH=remove_duplicates( - host_prefixes + host_target_info.cmake_prefix_paths(self.config)), + host_prefixes + host_target_info.cmake_prefix_paths(self.config) + ), MESON_EXTRA_BINARIES=self._meson_extra_binaries, ) @@ -199,16 +218,23 @@ def configure(self, **kwargs) -> None: self.configure_args.append(str(self.build_dir)) super().configure(**kwargs) if self.config.copy_compilation_db_to_source_dir and (self.build_dir / "compile_commands.json").exists(): - self.install_file(self.build_dir / "compile_commands.json", self.source_dir / "compile_commands.json", - force=True) + self.install_file( + self.build_dir / "compile_commands.json", self.source_dir / "compile_commands.json", force=True + ) def run_tests(self) -> None: if self.compiling_for_host(): self.run_cmd(self.configure_command, "test", "--print-errorlogs", cwd=self.build_dir) elif self.target_info.is_cheribsd(): - self.target_info.run_cheribsd_test_script("run_meson_tests.py", *self.meson_test_script_extra_args, - mount_builddir=True, mount_sysroot=True, mount_sourcedir=True, - use_full_disk_image=self.tests_need_full_disk_image) + self.target_info.run_cheribsd_test_script( + "run_meson_tests.py", + *self.meson_test_script_extra_args, + mount_builddir=True, + mount_sysroot=True, + mount_sourcedir=True, + use_full_disk_image=self.tests_need_full_disk_image, + ) else: - self.info("Don't know how to run tests for", self.target, "when cross-compiling for", - self.crosscompile_target) + self.info( + "Don't know how to run tests for", self.target, "when cross-compiling for", self.crosscompile_target + ) diff --git a/pycheribuild/projects/project.py b/pycheribuild/projects/project.py index 81e0ecde3..a6b739c24 100644 --- a/pycheribuild/projects/project.py +++ b/pycheribuild/projects/project.py @@ -107,7 +107,8 @@ "TargetBranchInfo", "TargetInfo", "commandline_to_str", - "default_source_dir_in_subdir"] + "default_source_dir_in_subdir", +] def install_dir_not_specified(_: CheriConfig, project: "Project"): @@ -233,14 +234,15 @@ def __infer_command(self) -> str: self.__project.check_required_system_tool("ninja", homebrew="ninja", apt="ninja-build") return "ninja" elif self.kind == MakeCommandKind.CMake: - self.__project.check_required_system_tool("cmake", default="cmake", homebrew="cmake", zypper="cmake", - apt="cmake", freebsd="cmake") + self.__project.check_required_system_tool( + "cmake", default="cmake", homebrew="cmake", zypper="cmake", apt="cmake", freebsd="cmake" + ) assert self.subkind is not None return "cmake" else: if self.__command is not None: return self.__command - self.__project.fatal("Cannot infer path from CustomMakeTool. Set self.make_args.set_command(\"tool\")") + self.__project.fatal('Cannot infer path from CustomMakeTool. Set self.make_args.set_command("tool")') raise RuntimeError() def set_command(self, value, can_pass_j_flag=True, early_args: "Optional[list[str]]" = None): @@ -256,8 +258,15 @@ def set_command(self, value, can_pass_j_flag=True, early_args: "Optional[list[st def all_commandline_args(self, config) -> "list[str]": return self.get_commandline_args(config=config) - def get_commandline_args(self, *, targets: "Optional[list[str]]" = None, jobs: "Optional[int]" = None, - verbose=False, continue_on_error=False, config: CheriConfig) -> "list[str]": + def get_commandline_args( + self, + *, + targets: "Optional[list[str]]" = None, + jobs: "Optional[int]" = None, + verbose=False, + continue_on_error=False, + config: CheriConfig, + ) -> "list[str]": assert self.kind result = list(self.__command_args) actual_build_tool = self.kind @@ -396,7 +405,8 @@ def default_source_dir_in_subdir(subdir: Path) -> ComputedDefaultValue[Path]: """ return ComputedDefaultValue( function=lambda config, project: _default_source_dir(config, project, subdir), - as_string=lambda cls: f"$SOURCE_ROOT/{subdir}/{(cls.default_directory_basename or cls.target)}") + as_string=lambda cls: f"$SOURCE_ROOT/{subdir}/{(cls.default_directory_basename or cls.target)}", + ) class Project(SimpleProject): @@ -413,7 +423,8 @@ class Project(SimpleProject): set_pkg_config_path: bool = True # set the PKG_CONFIG_* environment variables when building can_run_parallel_install: bool = False # Most projects don't work well with parallel installation default_source_dir: ComputedDefaultValue[Optional[Path]] = ComputedDefaultValue( - function=_default_source_dir, as_string=lambda cls: "$SOURCE_ROOT/" + cls.default_directory_basename) + function=_default_source_dir, as_string=lambda cls: "$SOURCE_ROOT/" + cls.default_directory_basename + ) # Some projects (e.g. python) need a native build for build tools, etc. needs_native_build_for_crosscompile: bool = False # Some projects build docbook xml files and in order to do so we need to set certain env vars to skip the @@ -446,7 +457,8 @@ def project_build_dir_help(cls) -> str: return result default_build_dir: ComputedDefaultValue[Path] = ComputedDefaultValue( - function=_default_build_dir, as_string=lambda cls: cls.project_build_dir_help()) + function=_default_build_dir, as_string=lambda cls: cls.project_build_dir_help() + ) make_kind: MakeCommandKind = MakeCommandKind.DefaultMake """ @@ -473,7 +485,8 @@ def get_install_dir(cls, caller: AbstractProject, cross_target: "Optional[CrossC def build_dir_for_target(self, target: CrossCompileTarget) -> Path: return self.config.build_root / ( - self.default_directory_basename + self.build_configuration_suffix(target) + "-build") + self.default_directory_basename + self.build_configuration_suffix(target) + "-build" + ) default_use_asan: bool = False @@ -505,10 +518,12 @@ def get_default_install_dir_kind(cls) -> DefaultInstallDir: install_dir = cls.cross_install_dir if install_dir is None and cls._default_install_dir_fn is Project._default_install_dir_fn: raise RuntimeError( - "native_install_dir/cross_install_dir/_default_install_dir_fn not specified for " + cls.target) + "native_install_dir/cross_install_dir/_default_install_dir_fn not specified for " + cls.target + ) if install_dir == DefaultInstallDir.SYSROOT_FOR_BAREMETAL_ROOTFS_OTHERWISE: if cls._xtarget is not None and ( - cls._xtarget.target_info_cls.is_baremetal() or cls._xtarget.target_info_cls.is_rtems()): + cls._xtarget.target_info_cls.is_baremetal() or cls._xtarget.target_info_cls.is_rtems() + ): install_dir = DefaultInstallDir.ROOTFS_LOCALBASE else: install_dir = DefaultInstallDir.ROOTFS_OPTBASE @@ -519,8 +534,9 @@ def get_default_install_dir_kind(cls) -> DefaultInstallDir: native_install_dir: Optional[DefaultInstallDir] = None cross_install_dir: Optional[DefaultInstallDir] = None # For more precise control over the install dir it is possible to provide a callback function - _default_install_dir_fn: ComputedDefaultValue[Path] = ComputedDefaultValue(function=_default_install_dir_handler, - as_string=_default_install_dir_str) + _default_install_dir_fn: ComputedDefaultValue[Path] = ComputedDefaultValue( + function=_default_install_dir_handler, as_string=_default_install_dir_str + ) """ The default installation directory """ @property @@ -535,8 +551,15 @@ def _rootfs_install_dir_name(self) -> str: __can_use_lld_map: "dict[str, bool]" = dict() def can_use_lld(self, compiler: Path) -> bool: - command = [str(compiler), *self.essential_compiler_and_linker_flags, - "-fuse-ld=lld", "-xc", "-o", "/dev/null", "-"] + command = [ + str(compiler), + *self.essential_compiler_and_linker_flags, + "-fuse-ld=lld", + "-xc", + "-o", + "/dev/null", + "-", + ] command_str = commandline_to_str(command) if command_str not in Project.__can_use_lld_map: assert compiler.is_absolute(), compiler @@ -544,9 +567,15 @@ def can_use_lld(self, compiler: Path) -> bool: if not compiler.exists(): return False try: - self.run_cmd(command, run_in_pretend_mode=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, raise_in_pretend_mode=True, - input="int main() { return 0; }\n", print_verbose_only=True) + self.run_cmd( + command, + run_in_pretend_mode=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + raise_in_pretend_mode=True, + input="int main() { return 0; }\n", + print_verbose_only=True, + ) status_update(compiler, "supports -fuse-ld=lld, linking should be much faster!") Project.__can_use_lld_map[command_str] = True except subprocess.CalledProcessError: @@ -556,14 +585,19 @@ def can_use_lld(self, compiler: Path) -> bool: def can_run_binaries_on_remote_morello_board(self) -> bool: morello_ssh_hostname = self.config.remote_morello_board - return morello_ssh_hostname and self.target_info.is_cheribsd() and self.compiling_for_aarch64( - include_purecap=True) and ssh_host_accessible_cached(morello_ssh_hostname, ssh_args=(), config=self.config) + return ( + morello_ssh_hostname + and self.target_info.is_cheribsd() + and self.compiling_for_aarch64(include_purecap=True) + and ssh_host_accessible_cached(morello_ssh_hostname, ssh_args=(), config=self.config) + ) def can_use_lto(self, ccinfo: CompilerInfo) -> bool: if ccinfo.compiler == "apple-clang": return True elif ccinfo.compiler == "clang" and ( - not self.compiling_for_host() or (ccinfo.version >= (4, 0, 0) and self.can_use_lld(ccinfo.path))): + not self.compiling_for_host() or (ccinfo.version >= (4, 0, 0) and self.can_use_lld(ccinfo.path)) + ): return True return self.compiling_for_host() and ccinfo.compiler == "gcc" @@ -578,8 +612,9 @@ def check_system_dependencies(self) -> None: if self.config.create_compilation_db and self.compile_db_requires_bear: if self.make_args.is_gnu_make and False: # use compiledb instead of bear for gnu make - self.check_required_system_tool("compiledb", - instructions=InstallInstructions("Run `pip install --user compiledb``")) + self.check_required_system_tool( + "compiledb", instructions=InstallInstructions("Run `pip install --user compiledb``") + ) else: self.check_required_system_tool("bear", homebrew="bear", cheribuild_target="bear") self._compiledb_tool = "bear" @@ -597,22 +632,32 @@ def setup_config_options(cls, install_directory_help="", **kwargs) -> None: super().setup_config_options(**kwargs) if cls.source_dir is None: cls._initial_source_dir = cls.add_optional_path_option( - "source-directory", metavar="DIR", default=cls.default_source_dir, - help="Override default source directory for " + cls.target) + "source-directory", + metavar="DIR", + default=cls.default_source_dir, + help="Override default source directory for " + cls.target, + ) # ---/build-directory is not inherited from the unsuffixed target (unless there is only one # supported target). default_xtarget = cls.default_architecture if cls._xtarget is not None or default_xtarget is not None: - cls.build_dir = cls.add_path_option("build-directory", metavar="DIR", default=cls.default_build_dir, - help="Override default source directory for " + cls.target, - use_default_fallback_config_names=cls._xtarget == default_xtarget) + cls.build_dir = cls.add_path_option( + "build-directory", + metavar="DIR", + default=cls.default_build_dir, + help="Override default source directory for " + cls.target, + use_default_fallback_config_names=cls._xtarget == default_xtarget, + ) if cls.can_build_with_asan: asan_default = ComputedDefaultValue( function=lambda config, proj: ( - False if proj.crosscompile_target.is_cheri_purecap() else proj.default_use_asan), - as_string=str(cls.default_use_asan)) - cls.use_asan = cls.add_bool_option("use-asan", default=asan_default, - help="Build with AddressSanitizer enabled") + False if proj.crosscompile_target.is_cheri_purecap() else proj.default_use_asan + ), + as_string=str(cls.default_use_asan), + ) + cls.use_asan = cls.add_bool_option( + "use-asan", default=asan_default, help="Build with AddressSanitizer enabled" + ) else: cls.use_asan = False if cls.can_build_with_msan: @@ -621,8 +666,7 @@ def setup_config_options(cls, install_directory_help="", **kwargs) -> None: cls.use_msan = False if cls.can_build_with_ccache: - cls.use_ccache = cls.add_bool_option("use-ccache", default=False, - help="Build with CCache") + cls.use_ccache = cls.add_bool_option("use-ccache", default=False, help="Build with CCache") else: cls.use_ccache = False cls.auto_var_init = cls.add_config_option( @@ -636,28 +680,40 @@ def setup_config_options(cls, install_directory_help="", **kwargs) -> None: ), help="Whether to initialize all local variables (currently only supported when compiling with clang)", ) - cls.skip_update = cls.add_bool_option("skip-update", - default=ComputedDefaultValue(lambda config, proj: config.skip_update, - "the value of the global --skip-update " - "option"), - help="Override --skip-update/--no-skip-update for this target only ") - cls.force_configure = cls.add_bool_option("reconfigure", altname="force-configure", - default=ComputedDefaultValue( - lambda config, proj: config.force_configure, - "the value of the global --reconfigure/--force-configure option"), - help="Override --(no-)reconfigure/--(no-)force-configure for this " - "target only") + cls.skip_update = cls.add_bool_option( + "skip-update", + default=ComputedDefaultValue( + lambda config, proj: config.skip_update, "the value of the global --skip-update " "option" + ), + help="Override --skip-update/--no-skip-update for this target only ", + ) + cls.force_configure = cls.add_bool_option( + "reconfigure", + altname="force-configure", + default=ComputedDefaultValue( + lambda config, proj: config.force_configure, + "the value of the global --reconfigure/--force-configure option", + ), + help="Override --(no-)reconfigure/--(no-)force-configure for this " "target only", + ) if not install_directory_help: install_directory_help = "Override default install directory for " + cls.target - cls._install_dir = cls.add_path_option("install-directory", metavar="DIR", help=install_directory_help, - default=cls._default_install_dir_fn) - if "repository" in dir(cls) and isinstance(cls.repository, GitRepository) and \ - "git_revision" not in cls.__dict__: - cls.git_revision = cls.add_config_option("git-revision", metavar="REVISION", - help="The git revision to checkout prior to building. Useful if " - "HEAD is broken for one " - "project but you still want to update the other projects.") + cls._install_dir = cls.add_path_option( + "install-directory", metavar="DIR", help=install_directory_help, default=cls._default_install_dir_fn + ) + if ( + "repository" in dir(cls) + and isinstance(cls.repository, GitRepository) + and "git_revision" not in cls.__dict__ + ): + cls.git_revision = cls.add_config_option( + "git-revision", + metavar="REVISION", + help="The git revision to checkout prior to building. Useful if " + "HEAD is broken for one " + "project but you still want to update the other projects.", + ) # TODO: can argparse action be used to store to the class member directly? # seems like I can create a new action a pass a reference to the repository: # class FooAction(argparse.Action): @@ -668,27 +724,46 @@ def setup_config_options(cls, install_directory_help="", **kwargs) -> None: # def __call__(self, parser, namespace, values, option_string=None): # print('%r %r %r' % (namespace, values, option_string)) # setattr(namespace, self.dest, values) - cls._repository_url = cls.add_config_option("repository", kind=str, help="The URL of the git repository", - default=cls.repository.url, metavar="REPOSITORY") - cls.use_lto = cls.add_bool_option("use-lto", help="Build with link-time optimization (LTO)", - default=cls.lto_by_default) + cls._repository_url = cls.add_config_option( + "repository", + kind=str, + help="The URL of the git repository", + default=cls.repository.url, + metavar="REPOSITORY", + ) + cls.use_lto = cls.add_bool_option( + "use-lto", help="Build with link-time optimization (LTO)", default=cls.lto_by_default + ) if cls.can_build_with_cfi: cls.use_cfi = cls.add_bool_option("use-cfi", help="Build with LLVM CFI (requires LTO)", default=False) else: cls.use_cfi = False - cls._linkage = cls.add_config_option("linkage", default=Linkage.DEFAULT, kind=Linkage, - help="Build static or dynamic (or use the project default)") + cls._linkage = cls.add_config_option( + "linkage", + default=Linkage.DEFAULT, + kind=Linkage, + help="Build static or dynamic (or use the project default)", + ) - cls.build_type = typing.cast(BuildType, cls.add_config_option( - "build-type", default=cls.default_build_type, kind=BuildType, - enum_choice_strings=supported_build_type_strings, - help="Optimization+debuginfo defaults (supports the same values as CMake (as well as 'DEFAULT' which" - " does not pass any additional flags to the configure command).")) + cls.build_type = typing.cast( + BuildType, + cls.add_config_option( + "build-type", + default=cls.default_build_type, + kind=BuildType, + enum_choice_strings=supported_build_type_strings, + help="Optimization+debuginfo defaults (supports the same values as CMake (as well as 'DEFAULT' which" + " does not pass any additional flags to the configure command).", + ), + ) if cls.has_optional_tests and "build_tests" not in cls.__dict__: - cls.build_tests = cls.add_bool_option("build-tests", help="Build the tests", - default=cls.default_build_tests, - show_help=cls.show_optional_tests_in_help) + cls.build_tests = cls.add_bool_option( + "build-tests", + help="Build the tests", + default=cls.default_build_tests, + show_help=cls.show_optional_tests_in_help, + ) def linkage(self) -> Linkage: if self.target_info.must_link_statically: @@ -783,10 +858,15 @@ def default_compiler_flags(self) -> "list[str]": if self.compiling_for_host(): return result if self.config.csetbounds_stats: - result.extend(["-mllvm", "-collect-csetbounds-output=" + str(self.csetbounds_stats_file), - "-mllvm", "-collect-csetbounds-stats=csv", - # "-Xclang", "-cheri-bounds=everywhere-unsafe"]) - ]) + result.extend( + [ + "-mllvm", + "-collect-csetbounds-output=" + str(self.csetbounds_stats_file), + "-mllvm", + "-collect-csetbounds-stats=csv", + # "-Xclang", "-cheri-bounds=everywhere-unsafe"]) + ] + ) return result @property @@ -846,8 +926,9 @@ def __init__(self, *args, **kwargs) -> None: initial_source_dir = inspect.getattr_static(self, "_initial_source_dir") assert isinstance(initial_source_dir, ConfigOptionHandle) # noinspection PyProtectedMember - assert initial_source_dir._get_default_value(self.config, self) is None, \ - "initial source dir != None for ReuseOtherProjectRepository" + assert ( + initial_source_dir._get_default_value(self.config, self) is None + ), "initial source dir != None for ReuseOtherProjectRepository" if self.source_dir is None: self.source_dir = self.repository.get_real_source_dir(self, self._initial_source_dir) else: @@ -888,15 +969,20 @@ def __init__(self, *args, **kwargs) -> None: elif install_dir_kind in (DefaultInstallDir.ROOTFS_OPTBASE, DefaultInstallDir.KDE_PREFIX): relative_to_rootfs = os.path.relpath(str(self._install_dir), str(self.rootfs_dir)) if relative_to_rootfs.startswith(os.path.pardir): - self.verbose_print("Custom install dir", self._install_dir, - "-> using / as install prefix for", self.target) + self.verbose_print( + "Custom install dir", self._install_dir, "-> using / as install prefix for", self.target + ) self._install_prefix = Path("/") self.destdir = self._install_dir else: self._install_prefix = Path("/", relative_to_rootfs) self.destdir = self.rootfs_dir - elif install_dir_kind in (None, DefaultInstallDir.DO_NOT_INSTALL, DefaultInstallDir.IN_BUILD_DIRECTORY, - DefaultInstallDir.CUSTOM_INSTALL_DIR): + elif install_dir_kind in ( + None, + DefaultInstallDir.DO_NOT_INSTALL, + DefaultInstallDir.IN_BUILD_DIRECTORY, + DefaultInstallDir.CUSTOM_INSTALL_DIR, + ): self._install_prefix = self._install_dir self.destdir = None else: @@ -905,8 +991,11 @@ def __init__(self, *args, **kwargs) -> None: # convert the tuples into mutable lists (this is needed to avoid modifying class variables) # See https://github.com/CTSRD-CHERI/cheribuild/issues/33 # FIXME: this should move to target_info - self.cross_warning_flags = ["-Werror=implicit-function-declaration", - "-Werror=format", "-Werror=incompatible-pointer-types"] + self.cross_warning_flags = [ + "-Werror=implicit-function-declaration", + "-Werror=format", + "-Werror=incompatible-pointer-types", + ] self.host_warning_flags = [] self.common_warning_flags = [] target_arch = self.crosscompile_target @@ -979,7 +1068,9 @@ def host_dependency_prefixes(self) -> "list[Path]": def setup(self): super().setup() self.verbose_print( - self.target, f"INSTALLDIR={self._install_dir}", f"INSTALL_PREFIX={self._install_prefix}", + self.target, + f"INSTALLDIR={self._install_dir}", + f"INSTALL_PREFIX={self._install_prefix}", f"DESTDIR={self.destdir}", ) if self.use_asan: @@ -995,7 +1086,8 @@ def setup(self): # We have to add the boostrap tools pkgconfig directory to PKG_CONFIG_PATH so that it is searched in # addition to the default paths. Note: We do not set PKG_CONFIG_LIBDIR since that overrides the default. pkg_config_args = dict( - PKG_CONFIG_PATH=":".join([*self.pkgconfig_dirs, os.getenv("PKG_CONFIG_PATH", "")])) + PKG_CONFIG_PATH=":".join([*self.pkgconfig_dirs, os.getenv("PKG_CONFIG_PATH", "")]) + ) if self.target_info.pkg_config_libdir_override is not None: pkg_config_args["PKG_CONFIG_LIBDIR"] = self.target_info.pkg_config_libdir_override elif self.needs_sysroot: @@ -1022,8 +1114,9 @@ def setup(self): self.cross_warning_flags += ["-Werror=cheri-capability-misuse", "-Werror=cheri-bitwise-operations"] # The morello compiler still uses the old flag name supports_new_flag = cc_info.supports_warning_flag("-Werror=cheri-prototypes") - self.cross_warning_flags.append("-Werror=cheri-prototypes" if supports_new_flag else - "-Werror=mips-cheri-prototypes") + self.cross_warning_flags.append( + "-Werror=cheri-prototypes" if supports_new_flag else "-Werror=mips-cheri-prototypes" + ) # Make underaligned capability loads/stores an error and require an explicit cast: self.cross_warning_flags.append("-Werror=pass-failed") if self.CC.exists() and cc_info.is_clang: @@ -1053,7 +1146,7 @@ def add_lto_build_options(self, ccinfo: CompilerInfo) -> bool: # For non apple-clang compilers we need to use llvm binutils: version_suffix = "" if compiler.name.startswith("clang"): - version_suffix = compiler.name[len("clang"):] + version_suffix = compiler.name[len("clang") :] llvm_ar = ccinfo.get_matching_binutil("llvm-ar") llvm_ranlib = ccinfo.get_matching_binutil("llvm-ranlib") llvm_nm = ccinfo.get_matching_binutil("llvm-nm") @@ -1061,8 +1154,10 @@ def add_lto_build_options(self, ccinfo: CompilerInfo) -> bool: # Find lld with the correct version (it must match the version of clang otherwise it breaks!) self._lto_linker_flags.extend(ccinfo.linker_override_flags(lld, linker_type="lld")) if not llvm_ar or not llvm_ranlib or not llvm_nm: - self.warning("Could not find llvm-{ar,ranlib,nm}" + version_suffix, - "-> disabling LTO (resulting binary will be a bit slower)") + self.warning( + "Could not find llvm-{ar,ranlib,nm}" + version_suffix, + "-> disabling LTO (resulting binary will be a bit slower)", + ) return False ld = lld if self.lto_set_ld else None self.set_lto_binutils(ar=llvm_ar, ranlib=llvm_ranlib, nm=llvm_nm, ld=ld) @@ -1102,17 +1197,26 @@ def __setattr__(self, name, value) -> None: if self.__dict__.get("_prevent_assign"): # assert name not in ("source_dir", "build_dir", "install_dir") assert name != "install_dir", "install_dir should not be modified, only _install_dir or _install_prefix" - assert name != "install_prefix", "install_prefix should not be modified, only _install_dir or " \ - "_install_prefix" + assert name != "install_prefix", ( + "install_prefix should not be modified, only _install_dir or " "_install_prefix" + ) if name in self._no_overwrite_allowed: import traceback + traceback.print_stack() - raise RuntimeError(self.__class__.__name__ + "." + name + " mustn't be set. Called from" + - self.__class__.__name__) + raise RuntimeError( + self.__class__.__name__ + "." + name + " mustn't be set. Called from" + self.__class__.__name__ + ) self.__dict__[name] = value - def _get_make_commandline(self, make_target: "Optional[Union[str, list[str]]]", make_command, - options: MakeOptions, parallel: bool = True, compilation_db_name: "Optional[str]" = None): + def _get_make_commandline( + self, + make_target: "Optional[Union[str, list[str]]]", + make_command, + options: MakeOptions, + parallel: bool = True, + compilation_db_name: "Optional[str]" = None, + ): assert options is not None assert make_command is not None options = options.copy() @@ -1128,41 +1232,62 @@ def _get_make_commandline(self, make_target: "Optional[Union[str, list[str]]]", tool_path = shutil.which(self._compiledb_tool) if not tool_path: self.dependency_error( - "Cannot find '" + self._compiledb_tool + "' which is needed to create a compilation DB") + "Cannot find '" + self._compiledb_tool + "' which is needed to create a compilation DB" + ) tool_path = self._compiledb_tool options.set_command(tool_path, can_pass_j_flag=options.can_pass_jflag, early_args=compdb_extra_args) # Ensure that recursive make invocations reuse the compilation DB tool options.set(MAKE=commandline_to_str([options.command, *compdb_extra_args])) make_command = options.command - all_args = [make_command, *options.get_commandline_args( - targets=[make_target] if isinstance(make_target, str) and make_target else make_target, - jobs=self.config.make_jobs if parallel else None, config=self.config, verbose=self.config.verbose, - continue_on_error=self.config.pass_dash_k_to_make)] + all_args = [ + make_command, + *options.get_commandline_args( + targets=[make_target] if isinstance(make_target, str) and make_target else make_target, + jobs=self.config.make_jobs if parallel else None, + config=self.config, + verbose=self.config.verbose, + continue_on_error=self.config.pass_dash_k_to_make, + ), + ] if not self.config.make_without_nice: all_args = ["nice", *all_args] return all_args - def get_make_commandline(self, make_target: "Union[str, list[str]]", make_command: "Optional[str]" = None, - options: "Optional[MakeOptions]" = None, parallel: bool = True, - compilation_db_name: "Optional[str]" = None) -> "list[str]": + def get_make_commandline( + self, + make_target: "Union[str, list[str]]", + make_command: "Optional[str]" = None, + options: "Optional[MakeOptions]" = None, + parallel: bool = True, + compilation_db_name: "Optional[str]" = None, + ) -> "list[str]": if not options: options = self.make_args if not make_command: make_command = self.make_args.command return self._get_make_commandline(make_target, make_command, options, parallel, compilation_db_name) - def run_make(self, make_target: "Optional[Union[str, list[str]]]" = None, *, - make_command: "Optional[str]" = None, options: "Optional[MakeOptions]" = None, - logfile_name: "Optional[str]" = None, cwd: "Optional[Path]" = None, - append_to_logfile=False, compilation_db_name="compile_commands.json", parallel: bool = True, - stdout_filter: "Optional[Callable[[bytes], None]]" = _default_stdout_filter) -> None: + def run_make( + self, + make_target: "Optional[Union[str, list[str]]]" = None, + *, + make_command: "Optional[str]" = None, + options: "Optional[MakeOptions]" = None, + logfile_name: "Optional[str]" = None, + cwd: "Optional[Path]" = None, + append_to_logfile=False, + compilation_db_name="compile_commands.json", + parallel: bool = True, + stdout_filter: "Optional[Callable[[bytes], None]]" = _default_stdout_filter, + ) -> None: if not options: options = self.make_args if not make_command: make_command = options.command - all_args = self._get_make_commandline(make_target, make_command, options, parallel=parallel, - compilation_db_name=compilation_db_name) + all_args = self._get_make_commandline( + make_target, make_command, options, parallel=parallel, compilation_db_name=compilation_db_name + ) if not cwd: cwd = self.build_dir if not logfile_name: @@ -1179,8 +1304,14 @@ def run_make(self, make_target: "Optional[Union[str, list[str]]]" = None, *, if stdout_filter is _default_stdout_filter: stdout_filter = self._stdout_filter env = options.env_vars - self.run_with_logfile(all_args, logfile_name=logfile_name, stdout_filter=stdout_filter, cwd=cwd, env=env, - append_to_logfile=append_to_logfile) + self.run_with_logfile( + all_args, + logfile_name=logfile_name, + stdout_filter=stdout_filter, + cwd=cwd, + env=env, + append_to_logfile=append_to_logfile, + ) # if we create a compilation db, copy it to the source dir: if self.config.copy_compilation_db_to_source_dir and (self.build_dir / compilation_db_name).exists(): self.install_file(self.build_dir / compilation_db_name, self.source_dir / compilation_db_name, force=True) @@ -1189,15 +1320,20 @@ def run_make(self, make_target: "Optional[Union[str, list[str]]]" = None, *, def update(self) -> None: if not self.repository and not self.skip_update: - self.fatal("Cannot update", self.target, "as it is missing a repository source", - fatal_when_pretending=True) - self.repository.update(self, src_dir=self.source_dir, base_project_source_dir=self._initial_source_dir, - revision=self.git_revision, skip_submodules=self.skip_git_submodules) + self.fatal("Cannot update", self.target, "as it is missing a repository source", fatal_when_pretending=True) + self.repository.update( + self, + src_dir=self.source_dir, + base_project_source_dir=self._initial_source_dir, + revision=self.git_revision, + skip_submodules=self.skip_git_submodules, + ) if self.is_large_source_repository and (self.source_dir / ".git").exists(): # This is a large repository, tell git to do whatever it can to speed up operations (new in 2.24): # https://git-scm.com/docs/git-config#Documentation/git-config.txt-featuremanyFiles - self.run_cmd("git", "config", "--local", "feature.manyFiles", "true", cwd=self.source_dir, - print_verbose_only=True) + self.run_cmd( + "git", "config", "--local", "feature.manyFiles", "true", cwd=self.source_dir, print_verbose_only=True + ) _extra_git_clean_excludes: "list[str]" = [] @@ -1216,8 +1352,10 @@ def clean(self) -> ThreadJoiner: # will have to check how well binutils and qemu work there if (self.build_dir / ".git").is_dir(): if ( - self.build_dir / "GNUmakefile").is_file() and self.make_kind != MakeCommandKind.BsdMake and \ - self.target != "elftoolchain": + (self.build_dir / "GNUmakefile").is_file() + and self.make_kind != MakeCommandKind.BsdMake + and self.target != "elftoolchain" + ): self.run_cmd(self.make_args.command, "distclean", cwd=self.build_dir) else: assert self.source_dir == self.build_dir @@ -1278,8 +1416,12 @@ def configure(self, *, cwd: "Optional[Path]" = None, configure_path: "Optional[P assert configure_path, "configure_command should not be empty!" if not Path(configure_path).exists(): self.fatal("Configure command ", configure_path, "does not exist!") - self.run_with_logfile([str(configure_path), *self.configure_args], logfile_name="configure", cwd=cwd, - env=self.configure_environment) + self.run_with_logfile( + [str(configure_path), *self.configure_args], + logfile_name="configure", + cwd=cwd, + env=self.configure_environment, + ) def compile(self, *, cwd: "Optional[Path]" = None, parallel: bool = True) -> None: if cwd is None: @@ -1318,9 +1460,17 @@ def install_prefix(self) -> Path: return self._install_prefix return self._install_dir - def run_make_install(self, *, options: "Optional[MakeOptions]" = None, _stdout_filter=_default_stdout_filter, - cwd: "Optional[Path]" = None, parallel: Optional[bool] = None, - target: "Union[str, list[str]]" = "install", make_install_env=None, **kwargs): + def run_make_install( + self, + *, + options: "Optional[MakeOptions]" = None, + _stdout_filter=_default_stdout_filter, + cwd: "Optional[Path]" = None, + parallel: Optional[bool] = None, + target: "Union[str, list[str]]" = "install", + make_install_env=None, + **kwargs, + ): if parallel is None: parallel = self.can_run_parallel_install if options is None: @@ -1330,8 +1480,9 @@ def run_make_install(self, *, options: "Optional[MakeOptions]" = None, _stdout_f if make_install_env is None: make_install_env = self.make_install_env options.env_vars.update(make_install_env) - self.run_make(make_target=target, options=options, stdout_filter=_stdout_filter, cwd=cwd, - parallel=parallel, **kwargs) + self.run_make( + make_target=target, options=options, stdout_filter=_stdout_filter, cwd=cwd, parallel=parallel, **kwargs + ) def install(self, *, _stdout_filter=_default_stdout_filter) -> None: self.run_make_install(_stdout_filter=_stdout_filter) @@ -1361,8 +1512,11 @@ def _do_generate_cmakelists(self) -> None: if existing_code == cmakelists: create = False elif "Generated by cheribuild.py" not in existing_code: - print("A different CMakeLists.txt already exists. Contents:\n", - coloured(AnsiColour.green, existing_code), end="") + print( + "A different CMakeLists.txt already exists. Contents:\n", + coloured(AnsiColour.green, existing_code), + end="", + ) if not self.query_yes_no("Overwrite?", force_result=False): create = False if create: @@ -1407,8 +1561,7 @@ def default_statcounters_csv_name(self) -> str: suffix += "-dynamic" if self.config.benchmark_lazy_binding: suffix += "-lazybinding" - return self.target + "-statcounters{}-{}.csv".format( - suffix, datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) + return self.target + "-statcounters{}-{}.csv".format(suffix, datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) def copy_asan_dependencies(self, dest_libdir) -> None: # ASAN depends on libraries that are not included in the benchmark image by default: @@ -1416,8 +1569,9 @@ def copy_asan_dependencies(self, dest_libdir) -> None: self.info("Adding ASAN library dependencies to", dest_libdir) self.makedirs(dest_libdir) for lib in ("usr/lib/librt.so.1", "usr/lib/libexecinfo.so.1", "lib/libgcc_s.so.1", "lib/libelf.so.2"): - self.install_file(self.sdk_sysroot / lib, dest_libdir / Path(lib).name, force=True, - print_verbose_only=False) + self.install_file( + self.sdk_sysroot / lib, dest_libdir / Path(lib).name, force=True, print_verbose_only=False + ) _check_install_dir_conflict: bool = True @@ -1436,15 +1590,23 @@ def _parse_require_clean_build_counter(self) -> Optional[int]: for i, line in enumerate(f.readlines()): # Remove comments while "#" in line: - line = line[:line.index('#')] + line = line[: line.index("#")] line = line.strip() if not line: continue try: parsed = int(line) if latest_counter is not None and parsed < latest_counter: - self.warning(require_clean_path, ":", i + 1, ": parsed counter ", parsed, - " is smaller than previous one: ", latest_counter, sep="") + self.warning( + require_clean_path, + ":", + i + 1, + ": parsed counter ", + parsed, + " is smaller than previous one: ", + latest_counter, + sep="", + ) else: latest_counter = parsed except ValueError as e: @@ -1458,8 +1620,16 @@ def process(self) -> None: if self.generate_cmakelists: self._do_generate_cmakelists() if self.config.verbose: - print(self.target, " directories: source=", self.source_dir, " build=", self.build_dir, " install=", - self.install_dir, sep="") + print( + self.target, + " directories: source=", + self.source_dir, + " build=", + self.build_dir, + " install=", + self.install_dir, + sep="", + ) if self.use_asan and self.compiling_for_mips(include_purecap=False): # copy the ASAN lib into the right directory: @@ -1471,6 +1641,7 @@ def process(self) -> None: # Find the newest ASAN runtime library versions from the FreeBSD sysroot found_asan_lib = None from distutils.version import StrictVersion + libname = "libclang_rt.asan-mips64.a" for version in reversed(sorted(versions, key=StrictVersion)): asan_libs = self.sdk_sysroot / "usr/lib/clang" / version / "lib/freebsd" @@ -1478,16 +1649,22 @@ def process(self) -> None: found_asan_lib = asan_libs / libname break if not found_asan_lib: - self.fatal("Cannot find", libname, "library in sysroot dirs", asan_libdir_candidates, - "-- Compilation will fail!") + self.fatal( + "Cannot find", + libname, + "library in sysroot dirs", + asan_libdir_candidates, + "-- Compilation will fail!", + ) found_asan_lib = Path("/some/invalid/path/to/lib") self.makedirs(expected_path) self.run_cmd("cp", "-av", found_asan_lib.parent, expected_path.parent) # For some reason they are 644 so we can't overwrite for the next build unless we chmod first self.run_cmd("chmod", "-R", "u+w", expected_path.parent) if not (expected_path / libname).exists(): - self.fatal("Cannot find", libname, "library in compiler dir", expected_path, - "-- Compilation will fail!") + self.fatal( + "Cannot find", libname, "library in compiler dir", expected_path, "-- Compilation will fail!" + ) install_dir_kind = self.get_default_install_dir_kind() if install_dir_kind != DefaultInstallDir.DO_NOT_INSTALL and self._check_install_dir_conflict: xtarget: CrossCompileTarget = self._xtarget @@ -1497,22 +1674,28 @@ def process(self) -> None: base = getattr(self, "synthetic_base", None) assert base is not None assert issubclass(base, SimpleProject) - other_instance = base.get_instance_for_cross_target(xtarget.check_conflict_with, self.config, - caller=self) + other_instance = base.get_instance_for_cross_target( + xtarget.check_conflict_with, self.config, caller=self + ) if self.config.verbose: self.info(self.target, "install dir for", xtarget.name, "is", self.install_dir) other_xtarget = other_instance.crosscompile_target self.info(self.target, "install dir for", other_xtarget.name, "is", self.install_dir) - assert other_instance.install_dir != self.install_dir, \ - other_instance.target + " reuses the same install prefix! This will cause conflicts: " + str( - other_instance.install_dir) + assert other_instance.install_dir != self.install_dir, ( + other_instance.target + + " reuses the same install prefix! This will cause conflicts: " + + str(other_instance.install_dir) + ) if self.skip_update: # When --skip-update is set (or we don't have working internet) only check that the repository exists if self.repository: - self.repository.ensure_cloned(self, src_dir=self.source_dir, - base_project_source_dir=self._initial_source_dir, - skip_submodules=self.skip_git_submodules) + self.repository.ensure_cloned( + self, + src_dir=self.source_dir, + base_project_source_dir=self._initial_source_dir, + skip_submodules=self.skip_git_submodules, + ) else: self.update() if not self._system_deps_checked: @@ -1526,10 +1709,15 @@ def process(self) -> None: else: last_build_kind = self.read_file(last_build_file) if last_build_kind != self.build_configuration_suffix(): - if not self.query_yes_no("Last build was for configuration" + last_build_kind + - " but currently building" + self.build_configuration_suffix() + - ". Will clean before build. Continue?", force_result=True, - default_result=True): + if not self.query_yes_no( + "Last build was for configuration" + + last_build_kind + + " but currently building" + + self.build_configuration_suffix() + + ". Will clean before build. Continue?", + force_result=True, + default_result=True, + ): self.fatal("Cannot continue") return self._force_clean = True @@ -1541,19 +1729,29 @@ def process(self) -> None: # Check if the last clean build had a smaller counter than the current required on and if so perform a clean # build and increment the value in the build directory. if not last_clean_counter_path.is_file(): - self.verbose_print("Forcing full rebuild since clean counter", last_clean_counter_path, - "does not exist yet") + self.verbose_print( + "Forcing full rebuild since clean counter", last_clean_counter_path, "does not exist yet" + ) self._force_clean = True else: try: clean_counter_in_build_dir = int(last_clean_counter_path.read_text().strip()) if clean_counter_in_build_dir < required_clean_counter: - self.info("Forcing full rebuild since clean counter in build dir (", clean_counter_in_build_dir, - ") is less than required minimum ", required_clean_counter, sep="") + self.info( + "Forcing full rebuild since clean counter in build dir (", + clean_counter_in_build_dir, + ") is less than required minimum ", + required_clean_counter, + sep="", + ) self._force_clean = True else: - self.verbose_print("Not forcing clean build since clean counter in build dir", - clean_counter_in_build_dir, "is >= required minimum", required_clean_counter) + self.verbose_print( + "Not forcing clean build since clean counter in build dir", + clean_counter_in_build_dir, + "is >= required minimum", + required_clean_counter, + ) except Exception as e: self.warning("Could not parse", last_clean_counter_path, "-> assuming clean build is required.", e) self._force_clean = True @@ -1585,9 +1783,11 @@ def process(self) -> None: # Build step if not self.config.skip_build: if self.config.csetbounds_stats and (self.csetbounds_stats_file.exists() or self.config.pretend): - self.move_file(self.csetbounds_stats_file, - self.csetbounds_stats_file.with_suffix(".from-configure.csv"), - force=True) + self.move_file( + self.csetbounds_stats_file, + self.csetbounds_stats_file.with_suffix(".from-configure.csv"), + force=True, + ) # move any csetbounds stats from configuration (since they are not useful) status_update("Building", self.display_name, "... ") self.compile() @@ -1655,7 +1855,7 @@ def _replace_value(self, template: str, required: bool, key: str, value: str) -> # CMake calling `clang -target;foo;--sysroot=...". We have to use a space-separated list instead, so we # also expand @{KEY}_STR@ (but don't make it an error if it doesn't exist in the toolchain file). # Feature request: https://github.com/mesonbuild/meson/issues/8534 - result = self._replace_value(result, required=False, key=key + '_STR', value=commandline_to_str(value.args)) + result = self._replace_value(result, required=False, key=key + "_STR", value=commandline_to_str(value.args)) strval = self._toolchain_file_command_args_to_str(value) elif isinstance(value, _CMakeAndMesonSharedLogic.EnvVarPathList): strval = self._toolchain_file_env_var_path_list_to_str(value) @@ -1682,8 +1882,13 @@ def _replace_values_in_toolchain_file(self, template: str, file: Path, **kwargs) result = self._replace_value(result, required=True, key=key, value=value) not_substituted = re.search(r"@[\w_\d]+@", result) if not_substituted: - self.fatal("Did not replace all keys, found", not_substituted.group(0), "at offset", not_substituted.span(), - fatal_when_pretending=True) + self.fatal( + "Did not replace all keys, found", + not_substituted.group(0), + "at offset", + not_substituted.span(), + fatal_when_pretending=True, + ) self.write_file(contents=result, file=file, overwrite=True) def _prepare_toolchain_file_common(self, output_file: "Optional[Path]" = None, **kwargs) -> None: @@ -1699,7 +1904,8 @@ def _prepare_toolchain_file_common(self, output_file: "Optional[Path]" = None, * # https://mesonbuild.com/Reference-tables.html#operating-system-names system_name = system_name.lower() self._replace_values_in_toolchain_file( - self._toolchain_template, output_file, + self._toolchain_template, + output_file, TOOLCHAIN_SDK_BINDIR=sdk_bindir, TOOLCHAIN_COMPILER_BINDIR=self.CC.parent, TOOLCHAIN_TARGET_TRIPLE=self.target_info.target_triple, @@ -1726,15 +1932,23 @@ def _prepare_toolchain_file_common(self, output_file: "Optional[Path]" = None, * TOOLCHAIN_CMAKE_PREFIX_PATH=self.cmake_prefix_paths, TOOLCHAIN_PKGCONFIG_DIRS=_CMakeAndMesonSharedLogic.EnvVarPathList(self.pkgconfig_dirs), COMMENT_IF_NATIVE="#" if self.compiling_for_host() else "", - **kwargs) + **kwargs, + ) - def _add_configure_options(self, *, _include_empty_vars=False, _replace=True, _config_file_options: "list[str]", - **kwargs) -> None: + def _add_configure_options( + self, *, _include_empty_vars=False, _replace=True, _config_file_options: "list[str]", **kwargs + ) -> None: for option, value in kwargs.items(): existing_option = next((x for x in self.configure_args if x.startswith("-D" + option + "=")), None) if any(x.startswith("-D" + option) for x in _config_file_options): - self.info("Not using default value of '", value, "' for configure option '", option, - "' since it is explicitly overwritten in the configuration", sep="") + self.info( + "Not using default value of '", + value, + "' for configure option '", + option, + "' since it is explicitly overwritten in the configuration", + sep="", + ) continue if existing_option is not None: if _replace: @@ -1785,10 +1999,16 @@ def process(self) -> None: expected_str = ".".join(map(str, self._minimum_cmake_or_meson_version)) tool = self._configure_tool_name install_instrs = self._configure_tool_install_instructions() - self.dependency_error(tool, "version", version_str, "is too old (need at least", expected_str + ")", - install_instructions=install_instrs, - cheribuild_target=install_instrs.cheribuild_target, - cheribuild_xtarget=BasicCompilationTargets.NATIVE) + self.dependency_error( + tool, + "version", + version_str, + "is too old (need at least", + expected_str + ")", + install_instructions=install_instrs, + cheribuild_target=install_instrs.cheribuild_target, + cheribuild_xtarget=BasicCompilationTargets.NATIVE, + ) class AutotoolsProject(Project): @@ -1801,13 +2021,15 @@ class AutotoolsProject(Project): @classmethod def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) - cls.extra_configure_flags = cls.add_list_option("configure-options", metavar="OPTIONS", - help="Additional command line options to pass to configure") + cls.extra_configure_flags = cls.add_list_option( + "configure-options", metavar="OPTIONS", help="Additional command line options to pass to configure" + ) """ Like Project but automatically sets up the defaults for autotools like projects Sets configure command to ./configure, adds --prefix=installdir """ + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.configure_command = self.source_dir / "configure" @@ -1822,8 +2044,9 @@ def setup(self) -> None: autotools_triple = autotools_triple.replace("-purecap", "") # TODO: do we have to remove these too? # autotools_triple = autotools_triple.replace("mips64c128-", "cheri-") - self.configure_args.extend(["--host=" + autotools_triple, "--target=" + autotools_triple, - "--build=" + buildhost]) + self.configure_args.extend( + ["--host=" + autotools_triple, "--target=" + autotools_triple, "--build=" + buildhost] + ) elif self.crosscompile_target.is_hybrid_or_purecap_cheri(): # When compiling natively on CheriBSD, most autotools projects don't like the inferred config.guess # value of aarch64c-unknown-freebsd14.0. Override it to make this work in most cases. @@ -1877,9 +2100,13 @@ def set_lto_binutils(self, ar, ranlib, nm, ld) -> None: def run_tests(self) -> None: # Most autotools projects have a "check" target that we can use. try: - self.run_cmd(self.make_args.command, - *self.make_args.get_commandline_args(targets=["-n", "check"], jobs=1, config=self.config), - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.build_dir) + self.run_cmd( + self.make_args.command, + *self.make_args.get_commandline_args(targets=["-n", "check"], jobs=1, config=self.config), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + cwd=self.build_dir, + ) except subprocess.CalledProcessError: # If make -n check fails, assume there are no tests. return super().run_tests() @@ -1888,6 +2115,7 @@ def run_tests(self) -> None: class MakefileProject(Project): """A very simple project that just set some defualt variables such as CC/CXX, etc""" + do_not_add_to_targets: bool = True build_in_source_dir: bool = True # Most makefile projects don't support out-of-source builds # Default to GNU make since that's what most makefile projects require. diff --git a/pycheribuild/projects/repository.py b/pycheribuild/projects/repository.py index 6e76feb64..2483e4622 100644 --- a/pycheribuild/projects/repository.py +++ b/pycheribuild/projects/repository.py @@ -39,7 +39,7 @@ from ..processutils import get_program_version, run_command from ..utils import AnsiColour, ConfigBase, coloured, remove_prefix, status_update -if typing.TYPE_CHECKING: # no-combine +if typing.TYPE_CHECKING: # no-combine from .project import Project # no-combine __all__ = [ @@ -56,7 +56,12 @@ class SourceRepository: def ensure_cloned( - self, current_project: "Project", *, src_dir: Path, base_project_source_dir: Path, skip_submodules=False, + self, + current_project: "Project", + *, + src_dir: Path, + base_project_source_dir: Path, + skip_submodules=False, ) -> None: raise NotImplementedError @@ -121,7 +126,10 @@ def update(self, current_project: "Project", *, src_dir: Path, **kwargs): src_proj.update() else: current_project.info( - "Not updating", src_dir, "since it reuses the repository for ", self.source_project.target, + "Not updating", + src_dir, + "since it reuses the repository for ", + self.source_project.target, ) @@ -243,7 +251,9 @@ def get_branch_info(src_dir: Path, config: ConfigBase) -> "Optional[GitBranchInf upstream = headers.get("branch.upstream", "") remote_name, remote_branch = upstream.split("/", maxsplit=1) if "/" in upstream else (None, None) return GitBranchInfo( - local_branch=headers.get("branch.head", ""), remote_name=remote_name, upstream_branch=remote_branch, + local_branch=headers.get("branch.head", ""), + remote_name=remote_name, + upstream_branch=remote_branch, ) except subprocess.CalledProcessError as e: if isinstance(e.__cause__, FileNotFoundError): @@ -315,7 +325,13 @@ def contains_commit( error_message = remove_prefix(is_ancestor.stderr.decode("utf-8"), "fatal: ").strip() current_project.verbose_print( coloured( - AnsiColour.blue, "Could not determine if ", expected_branch, " contains ", commit, ":", sep="", + AnsiColour.blue, + "Could not determine if ", + expected_branch, + " contains ", + commit, + ":", + sep="", ), coloured(AnsiColour.yellow, error_message), ) @@ -324,11 +340,19 @@ def contains_commit( current_project.warning("Unknown return code", is_ancestor) # some other error -> raise so that I can see what went wrong raise subprocess.CalledProcessError( - is_ancestor.returncode, is_ancestor.args, output=is_ancestor.stdout, stderr=is_ancestor.stderr, + is_ancestor.returncode, + is_ancestor.args, + output=is_ancestor.stdout, + stderr=is_ancestor.stderr, ) def ensure_cloned( - self, current_project: "Project", *, src_dir: Path, base_project_source_dir: Path, skip_submodules=False, + self, + current_project: "Project", + *, + src_dir: Path, + base_project_source_dir: Path, + skip_submodules=False, ) -> None: if current_project.config.skip_clone: if not (src_dir / ".git").exists(): @@ -422,7 +446,8 @@ def ensure_cloned( matching_remote = new_remote # Fetch from the remote to ensure that the target ref exists (otherwise git worktree add fails) current_project.run_cmd( - ["git", "-C", base_project_source_dir, "fetch", matching_remote], print_verbose_only=False, + ["git", "-C", base_project_source_dir, "fetch", matching_remote], + print_verbose_only=False, ) while True: try: @@ -439,9 +464,7 @@ def ensure_cloned( url = None if url == self.url: break - current_project.info( - "URL '", url, "' for remote ", matching_remote, " does not match expected url '", self.url, "'", sep="", - ) + current_project.info(f"URL '{url}' for remote {matching_remote} does not match expected url '{self.url}'") if current_project.query_yes_no("Use this remote?"): break matching_remote = input("Please enter the correct remote: ") @@ -570,7 +593,7 @@ def update( default_branch = self.old_branches.get(current_branch) if default_branch and current_branch != default_branch: current_project.warning( - "You are trying to build the", current_branch, "branch. You should be using", default_branch, + f"You are trying to build the {current_branch} branch. You should be using {default_branch}" ) if current_project.query_yes_no("Would you like to change to the " + default_branch + " branch?"): try: @@ -599,7 +622,10 @@ def update( # When checking if we are up to date, we treat a missing @{upstream} reference (no upstream branch # configured) as success to avoid getting an error from git pull. up_to_date = self.contains_commit( - current_project, "@{upstream}", src_dir=src_dir, invalid_commit_ref_result="invalid", + current_project, + "@{upstream}", + src_dir=src_dir, + invalid_commit_ref_result="invalid", ) if up_to_date is True: current_project.info("Skipping update: Current HEAD is up-to-date or ahead of upstream.") @@ -637,7 +663,9 @@ def update( if current_project.config.force_update: status_update("Updating", src_dir, "with autostash due to --force-update") elif not current_project.query_yes_no( - "Stash the changes, update and reapply?", default_result=True, force_result=True, + "Stash the changes, update and reapply?", + default_result=True, + force_result=True, ): status_update("Skipping update of", src_dir) return @@ -738,7 +766,12 @@ def contains_commit(current_project: "Project", commit: str, *, src_dir: Path, e return False def ensure_cloned( - self, current_project: "Project", *, src_dir: Path, base_project_source_dir: Path, skip_submodules=False, + self, + current_project: "Project", + *, + src_dir: Path, + base_project_source_dir: Path, + skip_submodules=False, ) -> None: if current_project.config.skip_clone: if not (src_dir / ".hg").exists(): @@ -783,7 +816,11 @@ def update( # handle repositories that have moved if src_dir.exists() and self.old_urls: remote_url = self.run_hg( - src_dir, "paths", "default", capture_output=True, project=current_project, + src_dir, + "paths", + "default", + capture_output=True, + project=current_project, ).stdout.strip() # Update from the old url: for old_url in self.old_urls: @@ -807,12 +844,19 @@ def update( if src_dir.exists() and self.force_branch: assert self.default_branch, "default_branch must be set if force_branch is true!" branch = self.run_hg( - src_dir, "branch", capture_output=True, print_verbose_only=True, project=current_project, + src_dir, + "branch", + capture_output=True, + print_verbose_only=True, + project=current_project, ) current_branch = branch.stdout.decode("utf-8") if current_branch != self.force_branch: current_project.warning( - "You are trying to build the", current_branch, "branch. You should be using", self.default_branch, + "You are trying to build the", + current_branch, + "branch. You should be using", + self.default_branch, ) if current_project.query_yes_no("Would you like to change to the " + self.default_branch + " branch?"): self.run_hg(src_dir, "update", "--merge", self.default_branch, project=current_project) @@ -835,7 +879,12 @@ def update( has_changes = ( len( self.run_hg( - src_dir, "diff", "--stat", capture_output=True, print_verbose_only=True, project=current_project, + src_dir, + "diff", + "--stat", + capture_output=True, + print_verbose_only=True, + project=current_project, ).stdout, ) > 1 @@ -846,7 +895,9 @@ def update( if current_project.config.force_update: status_update("Updating", src_dir, "with merge due to --force-update") elif not current_project.query_yes_no( - "Update and merge the changes?", default_result=True, force_result=True, + "Update and merge the changes?", + default_result=True, + force_result=True, ): status_update("Skipping update of", src_dir) return @@ -872,7 +923,8 @@ def ensure_cloned(self, current_project: "Project", src_dir: Path, **kwargs): if self._default_branch: checkout_url = checkout_url + "/" + self._default_branch if current_project.config.confirm_clone and not current_project.query_yes_no( - str(src_dir) + " is not a subversion checkout. Checkout from '" + checkout_url + "'?", default_result=True, + str(src_dir) + " is not a subversion checkout. Checkout from '" + checkout_url + "'?", + default_result=True, ): current_project.fatal("Sources for", str(src_dir), " missing!") return diff --git a/pycheribuild/projects/run_fvp.py b/pycheribuild/projects/run_fvp.py index ca81b8dd2..9b2758033 100644 --- a/pycheribuild/projects/run_fvp.py +++ b/pycheribuild/projects/run_fvp.py @@ -57,8 +57,9 @@ class InstallMorelloFVP(SimpleProject): min_ram_mb = 4900 warn_ram_mb = 7900 # We can run the FVP on macOS by using docker. FreeBSD might be able to use Linux emulation. - use_docker_container = BoolConfigOption("use-docker-container", default=OSInfo.IS_MAC, - help="Run the FVP inside a docker container") + use_docker_container = BoolConfigOption( + "use-docker-container", default=OSInfo.IS_MAC, help="Run the FVP inside a docker container" + ) i_agree_to_the_contained_eula = BoolConfigOption("agree-to-the-contained-eula", help="Accept the EULA") def check_system_dependencies(self) -> None: @@ -72,8 +73,9 @@ def check_system_dependencies(self) -> None: @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - cls.installer_path = cls.add_optional_path_option("installer-path", - help="Path to the FVP installer.sh or installer.tgz") + cls.installer_path = cls.add_optional_path_option( + "installer-path", help="Path to the FVP installer.sh or installer.tgz" + ) @property def install_dir(self): @@ -89,7 +91,8 @@ def process(self): # noinspection PyAttributeOutsideInit self.installer_path = self.install_dir.parent / self.installer_filename downloaded_new_file = self.download_file( - self.installer_path, url=self.base_url + self.installer_filename, sha256=self.installer_sha256) + self.installer_path, url=self.base_url + self.installer_filename, sha256=self.installer_sha256 + ) # Return early if we didn't download a new file or are running without --clean if not downloaded_new_file and not self.with_clean: @@ -126,14 +129,15 @@ def process(self): # to avoid prompts from the installer) self.clean_directory(self.install_dir, ensure_dir_exists=False) # Even when using docker, we extract on the host first to show the EULA and install the documentation - self.run_cmd([installer_sh, "--destination", self.install_dir, *eula_args], - print_verbose_only=False) + self.run_cmd([installer_sh, "--destination", self.install_dir, *eula_args], print_verbose_only=False) if self.use_docker_container: if installer_sh.parent != Path(td): self.install_file(installer_sh, Path(td, installer_sh.name)) # When building the docker container we have to pass --i-agree-to-the-contained-eula since it does # not seem possible to allow interactive prompts - self.write_file(Path(td, "Dockerfile"), contents=f""" + self.write_file( + Path(td, "Dockerfile"), + contents=f""" FROM opensuse/leap:15.2 RUN zypper in -y xterm gzip tar libdbus-1-3 libatomic1 telnet socat COPY {installer_sh.name} . @@ -143,20 +147,29 @@ def process(self): RUN useradd fvp-user USER fvp-user VOLUME /diskimg -""", overwrite=True) +""", + overwrite=True, + ) build_flags = [] if not self.config.skip_update: build_flags.append("--pull") if self.with_clean: build_flags.append("--no-cache") image_latest = self.container_name + ":latest" - self.run_cmd(["docker", "build", *build_flags, "-t", image_latest, "."], cwd=td, - print_verbose_only=False) + self.run_cmd( + ["docker", "build", *build_flags, "-t", image_latest, "."], cwd=td, print_verbose_only=False + ) # get the version from the newly-built image (don't use the cached_property) version = self._get_version(docker_image=image_latest, result_if_invalid=None) # Also create a morello-fvp:0.11.3 tag to allow running speicific versions - self.run_cmd("docker", "image", "tag", image_latest, - self.container_name + ":" + ".".join(map(str, version)), print_verbose_only=False) + self.run_cmd( + "docker", + "image", + "tag", + image_latest, + self.container_name + ":" + ".".join(map(str, version)), + print_verbose_only=False, + ) def _plugin_args(self): if self.fvp_revision >= (0, 10, 312): @@ -171,14 +184,23 @@ def _fvp_base_command(self, need_tty=True, docker_image=None) -> typing.Tuple[li if self.use_docker_container: if docker_image is None: docker_image = self.container_name + ":latest" - return (["docker", "run"] + (["-it"] if need_tty else []) + ["--rm", docker_image], - Path("/opt/FVP_Morello", model_relpath)) + return ( + ["docker", "run"] + (["-it"] if need_tty else []) + ["--rm", docker_image], + Path("/opt/FVP_Morello", model_relpath), + ) else: return [], self.install_dir / model_relpath - def execute_fvp(self, args: "list[str]", disk_image_path: "Optional[Path]" = None, - firmware_path: "Optional[Path]" = None, x11=True, tcp_ports: "Optional[list[int]]" = None, - interactive=True, **kwargs) -> CompletedProcess: + def execute_fvp( + self, + args: "list[str]", + disk_image_path: "Optional[Path]" = None, + firmware_path: "Optional[Path]" = None, + x11=True, + tcp_ports: "Optional[list[int]]" = None, + interactive=True, + **kwargs, + ) -> CompletedProcess: if tcp_ports is None: tcp_ports = [] display = os.getenv("DISPLAY", None) @@ -204,18 +226,30 @@ def execute_fvp(self, args: "list[str]", disk_image_path: "Optional[Path]" = Non pre_cmd += ["-v", str(disk_image_path) + ":" + str(disk_image_path)] docker_settings_fixit = "" if OSInfo.IS_MAC: - docker_settings_fixit = " This setting can be changed under \"Preferences > Resources > Advanced\"." + docker_settings_fixit = ' This setting can be changed under "Preferences > Resources > Advanced".' # If we are actually running a disk image, check the docker memory size first if self.docker_memory_size < self.min_ram_mb * 1024 * 1024: fixit = "Change the docker settings to allow at least 5GB (8GB recommended) of RAM for containers." - self.fatal("Docker container has less than ", self.min_ram_mb, "MB of RAM (", - self.docker_memory_size // 1024 // 1024, "MB), this is not enough to run the FVP!", - sep="", fixit_hint=fixit + docker_settings_fixit) + self.fatal( + "Docker container has less than ", + self.min_ram_mb, + "MB of RAM (", + self.docker_memory_size // 1024 // 1024, + "MB), this is not enough to run the FVP!", + sep="", + fixit_hint=fixit + docker_settings_fixit, + ) elif self.docker_memory_size < self.warn_ram_mb * 1024 * 1024: fixit = "Change the docker settings to allow at least 8GB of RAM for containers." - self.warning("Docker container has less than ", self.warn_ram_mb, "MB of RAM (", - self.docker_memory_size // 1024 // 1024, "MB), this may not enough to run the FVP", - sep="", fixit_hint=fixit + docker_settings_fixit) + self.warning( + "Docker container has less than ", + self.warn_ram_mb, + "MB of RAM (", + self.docker_memory_size // 1024 // 1024, + "MB), this may not enough to run the FVP", + sep="", + fixit_hint=fixit + docker_settings_fixit, + ) if firmware_path is not None: pre_cmd += ["-v", str(firmware_path) + ":" + str(firmware_path)] @@ -227,7 +261,7 @@ def execute_fvp(self, args: "list[str]", disk_image_path: "Optional[Path]" = Non bg_processes = [] if self.use_docker_container and x11 and OSInfo.IS_MAC: # To use X11 via docker on macos we need to run socat on port 6000 - socat_cmd = ["socat", "TCP-LISTEN:6000,reuseaddr,fork", "UNIX-CLIENT:\"" + display + "\""] + socat_cmd = ["socat", "TCP-LISTEN:6000,reuseaddr,fork", 'UNIX-CLIENT:"' + display + '"'] socat = popen(socat_cmd, stdin=subprocess.DEVNULL, config=self.config) bg_processes.append((socat, False)) try: @@ -240,15 +274,21 @@ def fvp_cmdline(): return self.run_cmd(fvp_cmdline(), **kwargs) else: if self.use_docker_container and not self.docker_has_socat: - self.fatal("Docker container needs updating to include socat for headless operation.", - fixit_hint="Re-run `cheribuild.py --clean " + self.target + "`") + self.fatal( + "Docker container needs updating to include socat for headless operation.", + fixit_hint="Re-run `cheribuild.py --clean " + self.target + "`", + ) def disable_uart(param_base): nonlocal extra_args - extra_args.extend([ - "-C", param_base + ".start_telnet=0", - "-C", param_base + ".quiet=1", - ]) + extra_args.extend( + [ + "-C", + param_base + ".start_telnet=0", + "-C", + param_base + ".quiet=1", + ] + ) disable_uart("board.terminal_uart0_board") disable_uart("board.terminal_uart1_board") @@ -258,11 +298,15 @@ def disable_uart(param_base): disable_uart("css.terminal_sec_uart_ap") disable_uart("css.terminal_uart1_ap") - extra_args.extend([ - "-q", - "-C", "disable_visualisation=true", - "-C", "css.terminal_uart_ap.quiet=1", - ]) + extra_args.extend( + [ + "-q", + "-C", + "disable_visualisation=true", + "-C", + "css.terminal_uart_ap.quiet=1", + ] + ) # Although we know nothing else is using ports in the container # and thus know what port will be allocated, we still need to @@ -287,9 +331,12 @@ def disable_uart(param_base): ap_servsock.socket.listen(1) socat_cmd = "socat - TCP:host.docker.internal:" + str(ap_servsock.port) - extra_args.extend([ - "-C", "css.terminal_uart_ap.terminal_command=echo %port | " + socat_cmd, - ]) + extra_args.extend( + [ + "-C", + "css.terminal_uart_ap.terminal_command=echo %port | " + socat_cmd, + ] + ) def get_ap_port(): (ap_sock, _) = ap_servsock.socket.accept() @@ -302,10 +349,13 @@ def get_ap_port(): return docker_host_ap_port else: ap_pipe_rfd, ap_pipe_wfd = os.pipe() - extra_args.extend([ - "-C", "css.terminal_uart_ap.terminal_command=echo %port >&" + str(ap_pipe_wfd), - ]) - fvp_kwargs['pass_fds'] = [ap_pipe_wfd] + extra_args.extend( + [ + "-C", + "css.terminal_uart_ap.terminal_command=echo %port >&" + str(ap_pipe_wfd), + ] + ) + fvp_kwargs["pass_fds"] = [ap_pipe_wfd] def get_ap_port(): with open(ap_pipe_rfd) as f: @@ -314,8 +364,9 @@ def get_ap_port(): # Pass os.setsid to create a new process group so signals # are passed to children; docker exec does not seem to want # to behave so we have to signal its child too. - fvp = popen(fvp_cmdline(), stdin=subprocess.DEVNULL, preexec_fn=os.setsid, config=self.config, - **fvp_kwargs) + fvp = popen( + fvp_cmdline(), stdin=subprocess.DEVNULL, preexec_fn=os.setsid, config=self.config, **fvp_kwargs + ) bg_processes.append((fvp, True)) self.info("Waiting for FVP to start...") # Don't call get_ap_port() in --pretend mode since it will hang forever @@ -327,9 +378,13 @@ def get_ap_port(): finally: pass - self.info("Connecting to the FVP using telnet. Press", coloured(AnsiColour.yellow, "CTRL+]"), - coloured(AnsiColour.cyan, "followed by"), coloured(AnsiColour.yellow, "q"), - coloured(AnsiColour.cyan, "to exit telnet and kill the FVP.")) + self.info( + "Connecting to the FVP using telnet. Press", + coloured(AnsiColour.yellow, "CTRL+]"), + coloured(AnsiColour.cyan, "followed by"), + coloured(AnsiColour.yellow, "q"), + coloured(AnsiColour.cyan, "to exit telnet and kill the FVP."), + ) # FVP only seems to listen on IPv4 so specify manually to avoid # messages about first trying to connect to ::1. @@ -379,8 +434,9 @@ def _get_version(self, docker_image=None, *, result_if_invalid) -> "tuple[int, . pre_cmd, fvp_path = self._fvp_base_command(need_tty=False, docker_image=docker_image) try: version_out = self.run_cmd([*pre_cmd, fvp_path, "--version"], capture_output=True, run_in_pretend_mode=True) - result = extract_version(version_out.stdout, - regex=re.compile(rb"Fast Models \[(\d+)\.(\d+)\.?(\d+)? \(.+\)]")) + result = extract_version( + version_out.stdout, regex=re.compile(rb"Fast Models \[(\d+)\.(\d+)\.?(\d+)? \(.+\)]") + ) self.info("Morello FVP version detected as", result) return result except Exception as e: @@ -394,16 +450,26 @@ def _get_version(self, docker_image=None, *, result_if_invalid) -> "tuple[int, . def docker_has_socat(self): assert self.use_docker_container try: - has_socat = self.run_cmd(["docker", "run", "--rm", self.container_name, "sh", "-c", - "command -v socat >/dev/null 2>&1 && printf true || printf false"], - capture_output=True, run_in_pretend_mode=True).stdout + has_socat = self.run_cmd( + [ + "docker", + "run", + "--rm", + self.container_name, + "sh", + "-c", + "command -v socat >/dev/null 2>&1 && printf true || printf false", + ], + capture_output=True, + run_in_pretend_mode=True, + ).stdout except Exception as e: self.fatal("Could not determine whether container has socat:", e) return True self.verbose_print("Has socat:", has_socat) - if has_socat == b'true': + if has_socat == b"true": return True - elif has_socat == b'false': + elif has_socat == b"false": return False else: self.fatal("Could not determine whether container has socat:", has_socat) @@ -415,14 +481,18 @@ def docker_memory_size(self): # try docker info first, and if that fails read /proc/meminfo in the container try: try: - memtotal = self.run_cmd(["docker", "info", "-f", "{{json .MemTotal}}"], capture_output=True, - run_in_pretend_mode=True).stdout + memtotal = self.run_cmd( + ["docker", "info", "-f", "{{json .MemTotal}}"], capture_output=True, run_in_pretend_mode=True + ).stdout self.verbose_print("Docker memory total:", memtotal.strip()) return int(memtotal.strip()) except Exception as e: self.warning("docker info failed:", e) - memtotal = self.run_cmd(["docker", "run", "--rm", self.container_name, "grep", "MemTotal:", - "/proc/meminfo"], capture_output=True, run_in_pretend_mode=True).stdout + memtotal = self.run_cmd( + ["docker", "run", "--rm", self.container_name, "grep", "MemTotal:", "/proc/meminfo"], + capture_output=True, + run_in_pretend_mode=True, + ).stdout self.verbose_print("Docker memory total:", memtotal) return int(memtotal.split()[1]) * 1024 except Exception as e: @@ -466,8 +536,9 @@ def setup(self): ssh_port = IntConfigOption("ssh-port", default=_default_fvp_ssh_port(), help="SSH port to use to connect to guest") force_headless = BoolConfigOption("force-headless", default=False, help="Force headless use of the FVP") # Allow using the architectural FVP: - use_architectureal_fvp = BoolConfigOption("use-architectural-fvp", - help="Use the architectural FVP that requires a license.") + use_architectureal_fvp = BoolConfigOption( + "use-architectural-fvp", help="Use the architectural FVP that requires a license." + ) fvp_trace_unbuffered = BoolConfigOption("trace-unbuffered", help="Don't buffer FVP trace output") fvp_trace_mmu = BoolConfigOption("trace-mmu", default=False, help="Emit FVP MMU trace events") smp = BoolConfigOption("smp", help="Simulate multiple CPU cores in the FVP", default=True) @@ -476,23 +547,31 @@ def setup(self): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) - fw_default = ComputedDefaultValue(function=lambda c, _: c.morello_sdk_dir / "firmware/morello-fvp", - as_string="/firmware/morello-fvp") - cls.firmware_path = cls.add_path_option("firmware-path", default=fw_default, - help="Path to the UEFI firmware binaries") - cls.remote_disk_image_path = cls.add_config_option("remote-disk-image-path", - help="When set rsync will be used to update the image from " - "the remote server prior to running it.") + fw_default = ComputedDefaultValue( + function=lambda c, _: c.morello_sdk_dir / "firmware/morello-fvp", + as_string="/firmware/morello-fvp", + ) + cls.firmware_path = cls.add_path_option( + "firmware-path", default=fw_default, help="Path to the UEFI firmware binaries" + ) + cls.remote_disk_image_path = cls.add_config_option( + "remote-disk-image-path", + help="When set rsync will be used to update the image from " "the remote server prior to running it.", + ) cls.extra_tcp_forwarding = cls.add_list_option( "extra-tcp-forwarding", - help="Additional TCP bridge ports beyond ssh/22; list of [hostip:]port=[guestip:]port") + help="Additional TCP bridge ports beyond ssh/22; list of [hostip:]port=[guestip:]port", + ) cls.license_server = cls.add_config_option("license-server", help="License server to use for the model") - cls.arch_model_path = cls.add_path_option("simulator-path", help="Path to the FVP Model", - default=Path("/usr/local/FVP_Base_RevC-Rainier")) - cls.fvp_trace = cls.add_optional_path_option("trace", - help="Enable FVP tracing plugin to output to the given file") - cls.fvp_trace_icount = cls.add_config_option("trace-start-icount", - help="Instruction count from which to start Tarmac trace") + cls.arch_model_path = cls.add_path_option( + "simulator-path", help="Path to the FVP Model", default=Path("/usr/local/FVP_Base_RevC-Rainier") + ) + cls.fvp_trace = cls.add_optional_path_option( + "trace", help="Enable FVP tracing plugin to output to the given file" + ) + cls.fvp_trace_icount = cls.add_config_option( + "trace-start-icount", help="Instruction count from which to start Tarmac trace" + ) @property def use_virtio_net(self): @@ -502,13 +581,21 @@ def use_virtio_net(self): # noinspection PyAttributeOutsideInit def process(self): if not self.firmware_path.exists(): - self.fatal("Firmware path", self.firmware_path, " is invalid, set the", - "--" + self.get_config_option_name("firmware_path"), "config option!") + self.fatal( + "Firmware path", + self.firmware_path, + " is invalid, set the", + "--" + self.get_config_option_name("firmware_path"), + "config option!", + ) disk_image_project = self._source_class.get_instance(self) if self.remote_disk_image_path: self.copy_remote_file(self.remote_disk_image_path, disk_image_project.disk_image_path) - disk_image = self.ensure_file_exists("disk image", disk_image_project.disk_image_path, - fixit_hint="Run `cheribuild.py " + disk_image_project.target + "`") + disk_image = self.ensure_file_exists( + "disk image", + disk_image_project.disk_image_path, + fixit_hint="Run `cheribuild.py " + disk_image_project.target + "`", + ) model_params = [] def add_board_params(*params): @@ -526,32 +613,41 @@ def add_hostbridge_params(*params): add_board_params("smsc_91c111.enabled=1") add_hostbridge_params( - "userNetworking=true", - "userNetPorts=" + ",".join([str(self.ssh_port) + "=22", *self.extra_tcp_forwarding])) + "userNetworking=true", "userNetPorts=" + ",".join([str(self.ssh_port) + "=22", *self.extra_tcp_forwarding]) + ) # NB: Set transport even if virtio_net is disabled since it still shows # up and is detected, just doesn't have any queues. add_board_params( "virtio_net.transport=legacy", "virtio_rng.transport=legacy", - "virtioblockdevice.image_path=" + str(disk_image)) + "virtioblockdevice.image_path=" + str(disk_image), + ) if self.use_architectureal_fvp: if not self.license_server: self.license_server = "unknown.license.server" # for --pretend - self.fatal("License server info unknown, set the", - "--" + self.get_config_option_name("license_server"), - "config option!") + self.fatal( + "License server info unknown, set the", + "--" + self.get_config_option_name("license_server"), + "config option!", + ) if not self.arch_model_path.is_dir(): - self.fatal("FVP path", self.arch_model_path, "does not exist, set the", - "--" + self.get_config_option_name("simulator_path"), "config option!") + self.fatal( + "FVP path", + self.arch_model_path, + "does not exist, set the", + "--" + self.get_config_option_name("simulator_path"), + "config option!", + ) with self.set_env(ARMLMD_LICENSE_FILE=self.license_server, print_verbose_only=False): - sim_binary = self.ensure_file_exists("Model binary", - self.arch_model_path / - "models/Linux64_GCC-6.4/FVP_Base_RevC-Rainier") - plugin = self.ensure_file_exists("Morello FVP plugin", - self.arch_model_path / "plugins/Linux64_GCC-6.4/MorelloPlugin.so") + sim_binary = self.ensure_file_exists( + "Model binary", self.arch_model_path / "models/Linux64_GCC-6.4/FVP_Base_RevC-Rainier" + ) + plugin = self.ensure_file_exists( + "Morello FVP plugin", self.arch_model_path / "plugins/Linux64_GCC-6.4/MorelloPlugin.so" + ) # prepend -C to each of the parameters: bl1_bin = self.ensure_file_exists("bl1.bin firmware", self.firmware_path / "bl1.bin") fip_bin = self.ensure_file_exists("fip.bin firmware", self.firmware_path / "fip.bin") @@ -567,17 +663,27 @@ def add_hostbridge_params(*params): self.run_cmd([sim_binary, "--plugin", plugin, "--print-port-number", *fvp_args]) else: if self.fvp_project.fvp_revision < self.required_fvp_version: - self.dependency_error("FVP is too old, please update to the latest version", - problem="outdated", cheribuild_target="install-morello-fvp", - cheribuild_action="update") + self.dependency_error( + "FVP is too old, please update to the latest version", + problem="outdated", + cheribuild_target="install-morello-fvp", + cheribuild_action="update", + ) del self.fvp_project.fvp_revision # reset cached value if self.fvp_project.fvp_revision < self.required_fvp_version: - self.fatal("FVP update failed, version is reported as", self.fvp_project.fvp_revision, - "but needs to be at least", self.required_fvp_version) + self.fatal( + "FVP update failed, version is reported as", + self.fvp_project.fvp_revision, + "but needs to be at least", + self.required_fvp_version, + ) elif self.fvp_project.fvp_revision < self.fvp_project.latest_known_fvp: - self.dependency_warning("FVP is old, it is recommended to update to the latest version", - problem="outdated", cheribuild_target="install-morello-fvp", - cheribuild_action="update") + self.dependency_warning( + "FVP is old, it is recommended to update to the latest version", + problem="outdated", + cheribuild_target="install-morello-fvp", + cheribuild_action="update", + ) del self.fvp_project.fvp_revision # reset cached value self.fvp_project.check_system_dependencies() # warn if socat/docker is missing @@ -601,18 +707,24 @@ def add_hostbridge_params(*params): # scp_romfw_elf = self.ensure_file_exists("SCP ROM ELF firmware", # self.firmware_path / "morello/components/morello/scp_romfw.elf") firmware_fixit = "Run `cheribuild.py morello-fvp-firmware`" - mcp_rom_bin = self.ensure_file_exists("MCP ROM ELF firmware", BuildMorelloScpFirmware.mcp_rom_bin(self), - fixit_hint=firmware_fixit) - scp_rom_bin = self.ensure_file_exists("SCP ROM ELF firmware", BuildMorelloScpFirmware.scp_rom_bin(self), - fixit_hint=firmware_fixit) - uefi_bin = self.ensure_file_exists("UEFI firmware", BuildMorelloUEFI.uefi_bin(self), - fixit_hint=firmware_fixit) + mcp_rom_bin = self.ensure_file_exists( + "MCP ROM ELF firmware", BuildMorelloScpFirmware.mcp_rom_bin(self), fixit_hint=firmware_fixit + ) + scp_rom_bin = self.ensure_file_exists( + "SCP ROM ELF firmware", BuildMorelloScpFirmware.scp_rom_bin(self), fixit_hint=firmware_fixit + ) + uefi_bin = self.ensure_file_exists( + "UEFI firmware", BuildMorelloUEFI.uefi_bin(self), fixit_hint=firmware_fixit + ) flash_images = BuildMorelloFlashImages.get_instance(self, cross_target=CompilationTargets.NATIVE) - scp_fw_bin = self.ensure_file_exists("SCP/AP firmware", flash_images.scp_ap_ram_firmware_image, - fixit_hint=firmware_fixit) - mcp_fw_bin = self.ensure_file_exists("MCP firmware", flash_images.mcp_ram_firmware_image, - fixit_hint=firmware_fixit) + scp_fw_bin = self.ensure_file_exists( + "SCP/AP firmware", flash_images.scp_ap_ram_firmware_image, fixit_hint=firmware_fixit + ) + mcp_fw_bin = self.ensure_file_exists( + "MCP firmware", flash_images.mcp_ram_firmware_image, fixit_hint=firmware_fixit + ) assert uefi_bin.parent == scp_fw_bin.parent and scp_fw_bin.parent == scp_rom_bin.parent, "Different dirs?" + # fmt: off fvp_args += [ # "-a", "Morello_Top.css.scp.armcortexm7ct=" + str(scp_romfw_elf), # "-a", "Morello_Top.css.mcp.armcortexm7ct=" + str(mcp_romfw_elf), @@ -631,18 +743,21 @@ def add_hostbridge_params(*params): # "-C", "css.nonTrustedROMloader.fname=" + str(uefi_bin), # "-C", "css.trustedBootROMloader.fname=" + str(trusted_fw), "-C", "css.pl011_uart_ap.unbuffered_output=1", - ] + ] # fmt: on gdb_port = None if self.config.wait_for_debugger: gdb_port = find_free_port(preferred_port=1234).port if self.config.gdb_random_port else 1234 + # fmt: off fvp_args += [ "--allow-debug-plugin", "--plugin", self.fvp_project.plugin_dir / "GDBRemoteConnection.so", "-C", "REMOTE_CONNECTION.GDBRemoteConnection.listen_address=127.0.0.1", - "-C", f"REMOTE_CONNECTION.GDBRemoteConnection.port={gdb_port}"] + "-C", f"REMOTE_CONNECTION.GDBRemoteConnection.port={gdb_port}" + ] # fmt: on if self.fvp_trace: + # fmt: off fvp_args += [ "--plugin", self.fvp_project.plugin_dir / "TarmacTrace.so", "-C", f"TRACE.TarmacTrace.trace-file={self.fvp_trace}", @@ -670,7 +785,7 @@ def add_hostbridge_params(*params): "-C", "css.cluster0.cpu1.trace_special_hlt_imm16=0xbeef", "-C", "css.cluster1.cpu0.trace_special_hlt_imm16=0xbeef", "-C", "css.cluster1.cpu1.trace_special_hlt_imm16=0xbeef", - ] + ] # fmt: on if self.fvp_trace_unbuffered: fvp_args += ["-C", "TRACE.TarmacTrace.unbuffered=true"] if self.fvp_trace_icount: @@ -707,8 +822,13 @@ def add_hostbridge_params(*params): continue tcp_ports += gaddrport[-1] # either just port or last in "host:port".split(":") - self.fvp_project.execute_fvp(fvp_args, disk_image_path=disk_image, firmware_path=uefi_bin.parent, - x11=not self.force_headless, tcp_ports=tcp_ports) + self.fvp_project.execute_fvp( + fvp_args, + disk_image_path=disk_image, + firmware_path=uefi_bin.parent, + x11=not self.force_headless, + tcp_ports=tcp_ports, + ) class LaunchFVPCheriBSD(LaunchFVPBase): diff --git a/pycheribuild/projects/run_qemu.py b/pycheribuild/projects/run_qemu.py index f701429d6..dbbfe60b7 100644 --- a/pycheribuild/projects/run_qemu.py +++ b/pycheribuild/projects/run_qemu.py @@ -71,8 +71,9 @@ class QEMUType(Enum): class ChosenQEMU: - def __init__(self, cls: "Optional[type[BuildQEMUBase]]", binary: Optional[Path], - can_provide_src_via_smb: Optional[bool]): + def __init__( + self, cls: "Optional[type[BuildQEMUBase]]", binary: Optional[Path], can_provide_src_via_smb: Optional[bool] + ): self.cls = cls self._binary = binary self._can_provide_src_via_smb = can_provide_src_via_smb @@ -106,8 +107,9 @@ def setup(self, launch): # either explicitly or as the default. If using the default QEMU we # prefer CHERI QEMU's corresponding non-CHERI binary if it exists, in # part due to its SMBD support - assert launch.use_qemu == QEMUType.SYSTEM or launch.use_qemu == QEMUType.DEFAULT, \ - "Unexpected use_qemu for lazy binary location: " + str(launch.use_qemu) + assert launch.use_qemu == QEMUType.SYSTEM or launch.use_qemu == QEMUType.DEFAULT, ( + "Unexpected use_qemu for lazy binary location: " + str(launch.use_qemu) + ) binary_name = "qemu-system-" + launch.qemu_options.qemu_arch_sufffix if (launch.config.qemu_bindir / binary_name).is_file() and launch.use_qemu != QEMUType.SYSTEM: # Only CHERI QEMU supports more than one SMB share @@ -142,54 +144,82 @@ class LaunchQEMUBase(SimpleProject): disk_image_project: Optional[Project] = None _uses_disk_image = True - use_uboot = BoolConfigOption("use-u-boot", default=False, - help="Boot using U-Boot for UEFI if supported (only RISC-V)") + use_uboot = BoolConfigOption( + "use-u-boot", default=False, help="Boot using U-Boot for UEFI if supported (only RISC-V)" + ) cvtrace = BoolConfigOption("cvtrace", help="Use binary trace output instead of textual") @classmethod def setup_config_options(cls, default_ssh_port: "Optional[int]" = None, **kwargs): super().setup_config_options(**kwargs) - cls.use_qemu = typing.cast(QEMUType, cls.add_config_option( - "use-qemu", kind=QEMUType, default=QEMUType.DEFAULT, enum_choice_strings=[t.value for t in QEMUType], - help="The QEMU type to run with. When set to 'custom', the 'custom-qemu-path' option must also be set.")) + cls.use_qemu = typing.cast( + QEMUType, + cls.add_config_option( + "use-qemu", + kind=QEMUType, + default=QEMUType.DEFAULT, + enum_choice_strings=[t.value for t in QEMUType], + help="The QEMU type to run with. When set to 'custom', the 'custom-qemu-path' option must also be set.", + ), + ) cls.custom_qemu_path = cls.add_optional_path_option("custom-qemu-path", help="Path to the custom QEMU binary") - cls.extra_qemu_options = cls.add_list_option("extra-options", metavar="QEMU_OPTIONS", - help="Additional command line flags to pass to qemu-system") - cls.logfile = cls.add_optional_path_option("logfile", metavar="LOGFILE", - help="The logfile that QEMU should use.") + cls.extra_qemu_options = cls.add_list_option( + "extra-options", metavar="QEMU_OPTIONS", help="Additional command line flags to pass to qemu-system" + ) + cls.logfile = cls.add_optional_path_option( + "logfile", metavar="LOGFILE", help="The logfile that QEMU should use." + ) cls.log_directory = cls.add_optional_path_option( - "log-directory", metavar="DIR", + "log-directory", + metavar="DIR", help="If set QEMU will log to a timestamped file in this directory. " - "Will be ignored if the 'logfile' option is set") - cls.use_telnet = cls.add_config_option("monitor-over-telnet", kind=int, metavar="PORT", show_help=False, - help="If set, the QEMU monitor will be reachable by connecting to " - "localhost at $PORT via telnet instead of using CTRL+A,C") + "Will be ignored if the 'logfile' option is set", + ) + cls.use_telnet = cls.add_config_option( + "monitor-over-telnet", + kind=int, + metavar="PORT", + show_help=False, + help="If set, the QEMU monitor will be reachable by connecting to " + "localhost at $PORT via telnet instead of using CTRL+A,C", + ) cls.custom_qemu_smb_mount = cls.add_optional_path_option( - "smb-host-directory", metavar="DIR", + "smb-host-directory", + metavar="DIR", help="If set QEMU will provide this directory over smb with the name //10.0.2.4/qemu for use with " - "mount_smbfs") + "mount_smbfs", + ) # TODO: -s will no longer work, not sure anyone uses it though if cls.forward_ssh_port: - default_ssh_port_computed = ComputedDefaultValue(function=lambda p, _: default_ssh_port, - as_string=str(default_ssh_port), - as_readme_string="") - cls.ssh_forwarding_port = cls.add_config_option("ssh-forwarding-port", kind=int, - default=default_ssh_port_computed, metavar="PORT", - show_help=True, - help="The port on localhost to forward to the QEMU ssh " - "port. You can then use `ssh root@localhost -p $PORT` " - "to connect to the VM") + default_ssh_port_computed = ComputedDefaultValue( + function=lambda p, _: default_ssh_port, + as_string=str(default_ssh_port), + as_readme_string="", + ) + cls.ssh_forwarding_port = cls.add_config_option( + "ssh-forwarding-port", + kind=int, + default=default_ssh_port_computed, + metavar="PORT", + show_help=True, + help="The port on localhost to forward to the QEMU ssh " + "port. You can then use `ssh root@localhost -p $PORT` " + "to connect to the VM", + ) cls.ephemeral = False if cls._uses_disk_image: cls.ephemeral = cls.add_bool_option( - "ephemeral", show_help=True, - help="Run qemu in 'snapshot' mode, changes to the disk image are non-persistent") + "ephemeral", + show_help=True, + help="Run qemu in 'snapshot' mode, changes to the disk image are non-persistent", + ) # TODO: add a shortcut for vnc? cls.extra_tcp_forwarding = cls.add_list_option( "extra-tcp-forwarding", - help="Additional TCP bridge ports beyond ssh/22; list of [hostip:]port=[guestip:]port") + help="Additional TCP bridge ports beyond ssh/22; list of [hostip:]port=[guestip:]port", + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -243,8 +273,10 @@ def get_chosen_qemu(cls, config: CheriConfig): # guess what kind of QEMU this is can_provide_src_via_smb = xtarget.is_hybrid_or_purecap_cheri() if not cls.custom_qemu_path: - fatal_error("Must specify path to custom QEMU with --" + cls.target + "/custom-qemu-path", - pretend=config.pretend) + fatal_error( + "Must specify path to custom QEMU with --" + cls.target + "/custom-qemu-path", + pretend=config.pretend, + ) qemu_binary = Path("/no/custom/path/to/qemu") else: qemu_binary = Path(cls.custom_qemu_path) @@ -261,8 +293,13 @@ def get_chosen_qemu(cls, config: CheriConfig): QEMUType.UPSTREAM: BuildUpstreamQEMU, }[cls.use_qemu] if qemu_class not in supported_qemu_classes: - fatal_error("Cannot use", cls.use_qemu.value, "QEMU with target", xtarget.generic_target_suffix, - pretend=config.pretend) + fatal_error( + "Cannot use", + cls.use_qemu.value, + "QEMU with target", + xtarget.generic_target_suffix, + pretend=config.pretend, + ) qemu_class = None qemu_binary = Path("/target/not/supported/with/this/qemu") else: @@ -301,9 +338,12 @@ def setup(self): def process(self): if not self.chosen_qemu.binary.exists(): - self.dependency_error("QEMU is missing:", self.chosen_qemu.binary, - cheribuild_target=self.chosen_qemu.cls.target if self.chosen_qemu.cls else None, - cheribuild_xtarget=CompilationTargets.NATIVE) + self.dependency_error( + "QEMU is missing:", + self.chosen_qemu.binary, + cheribuild_target=self.chosen_qemu.cls.target if self.chosen_qemu.cls else None, + cheribuild_xtarget=CompilationTargets.NATIVE, + ) qemu_loader_or_kernel = self.current_kernel if self.use_uboot: @@ -318,20 +358,29 @@ def process(self): if uboot_xtarget is not None: qemu_loader_or_kernel = BuildUBoot.get_firmware_path(self, self.config, cross_target=uboot_xtarget) else: - self.warning("Unsupported U-Boot QEMU target", xtarget.generic_target_suffix, - "- falling back on kernel") + self.warning( + "Unsupported U-Boot QEMU target", xtarget.generic_target_suffix, "- falling back on kernel" + ) if qemu_loader_or_kernel is not None and not qemu_loader_or_kernel.exists(): kernel_target_name = self.kernel_project.target if self.kernel_project is not None else None kernel_xtarget = self.kernel_project.crosscompile_target if self.kernel_project is not None else None - self.dependency_error("Loader/kernel is missing:", qemu_loader_or_kernel, - cheribuild_target=kernel_target_name, cheribuild_xtarget=kernel_xtarget) + self.dependency_error( + "Loader/kernel is missing:", + qemu_loader_or_kernel, + cheribuild_target=kernel_target_name, + cheribuild_xtarget=kernel_xtarget, + ) if self.forward_ssh_port and not self.is_port_available(self.ssh_forwarding_port): self.print_port_usage(self.ssh_forwarding_port) - self.fatal("SSH forwarding port", self.ssh_forwarding_port, "is already in use! Make sure you don't " - "already have a QEMU instance running or change the chosen port by setting the config option", - self.get_config_option_name("ssh_forwarding_port")) + self.fatal( + "SSH forwarding port", + self.ssh_forwarding_port, + "is already in use! Make sure you don't " + "already have a QEMU instance running or change the chosen port by setting the config option", + self.get_config_option_name("ssh_forwarding_port"), + ) monitor_options = [] if self.use_telnet: @@ -356,8 +405,9 @@ def process(self): if latest_symlink.is_symlink(): latest_symlink.unlink() if not latest_symlink.exists(): - self.create_symlink(self.log_directory / filename, latest_symlink, relative=True, - cwd=self.log_directory) + self.create_symlink( + self.log_directory / filename, latest_symlink, relative=True, cwd=self.log_directory + ) logfile_options = ["-D", self.log_directory / filename] if self.cvtrace: @@ -365,16 +415,23 @@ def process(self): if self.disk_image is not None and not self.disk_image.exists(): disk_image_target_name = self.disk_image_project.target if self.disk_image_project is not None else None disk_image_xtarget = ( - self.disk_image_project.crosscompile_target if self.disk_image_project is not None else None) - self.dependency_error("Disk image is missing:", self.disk_image, cheribuild_target=disk_image_target_name, - cheribuild_xtarget=disk_image_xtarget) + self.disk_image_project.crosscompile_target if self.disk_image_project is not None else None + ) + self.dependency_error( + "Disk image is missing:", + self.disk_image, + cheribuild_target=disk_image_target_name, + cheribuild_xtarget=disk_image_xtarget, + ) user_network_options = "" smb_dir_count = 0 # TODO: use 9pfs once kernel support is merged - have_9pfs_support = False and ( - self.crosscompile_target.is_native() or self.crosscompile_target.is_any_x86() - ) and qemu_supports_9pfs(self.chosen_qemu.binary, config=self.config) + have_9pfs_support = ( + False + and (self.crosscompile_target.is_native() or self.crosscompile_target.is_any_x86()) + and qemu_supports_9pfs(self.chosen_qemu.binary, config=self.config) + ) # Only default to providing the smb mount if smbd exists have_smbfs_support = self.chosen_qemu.can_provide_src_via_smb and shutil.which("smbd") @@ -398,25 +455,40 @@ def add_smb_or_9p_dir(directory, target, share_name=None, readonly=False): else: share_name = f"qemu{smb_dir_count}" user_network_options += str(directory) + share_name_option + ("@ro" if readonly else "") - guest_cmd = coloured(AnsiColour.yellow, - f"mkdir -p {target} && mount_smbfs -I 10.0.2.4 -N //10.0.2.4/{share_name}" - f" {target}") - self.info("Providing ", coloured(AnsiColour.green, str(directory)), - coloured(AnsiColour.cyan, " over SMB to the guest. Use `"), guest_cmd, - coloured(AnsiColour.cyan, "` to mount it"), sep="") + guest_cmd = coloured( + AnsiColour.yellow, + f"mkdir -p {target} && mount_smbfs -I 10.0.2.4 -N //10.0.2.4/{share_name}" f" {target}", + ) + self.info( + "Providing ", + coloured(AnsiColour.green, str(directory)), + coloured(AnsiColour.cyan, " over SMB to the guest. Use `"), + guest_cmd, + coloured(AnsiColour.cyan, "` to mount it"), + sep="", + ) if have_9pfs_support: if smb_dir_count > 1: return # FIXME: 9pfs panics if there is more than one device # Also provide it via virtfs: virtfs_args.append("-virtfs") - virtfs_args.append("local,id=virtfs{n},mount_tag={tag},path={path},security_model=none{ro}".format( - n=smb_dir_count, path=directory, tag=share_name, ro=",readonly" if readonly else "")) - guest_cmd = coloured(AnsiColour.yellow, - "mkdir -p {tgt} && mount -t virtfs -o trans=virtio,version=9p2000.L {share_name} " - "{tgt}".format(tgt=target, share_name=share_name)) - self.info("Providing ", coloured(AnsiColour.green, str(directory)), - coloured(AnsiColour.cyan, " over 9pfs to the guest. Use `"), guest_cmd, - coloured(AnsiColour.cyan, "` to mount it"), sep="") + virtfs_args.append( + "local,id=virtfs{n},mount_tag={tag},path={path},security_model=none{ro}".format( + n=smb_dir_count, path=directory, tag=share_name, ro=",readonly" if readonly else "" + ) + ) + guest_cmd = coloured( + AnsiColour.yellow, + f"mkdir -p {target} && mount -t virtfs -o trans=virtio,version=9p2000.L {share_name} " "{tgt}", + ) + self.info( + "Providing ", + coloured(AnsiColour.green, str(directory)), + coloured(AnsiColour.cyan, " over 9pfs to the guest. Use `"), + guest_cmd, + coloured(AnsiColour.cyan, "` to mount it"), + sep="", + ) virtfs_args = [] if have_smbfs_support or have_9pfs_support: # for running CheriBSD + FreeBSD @@ -430,19 +502,22 @@ def add_smb_or_9p_dir(directory, target, share_name=None, readonly=False): user_network_options += ",hostfwd=tcp::" + str(self.ssh_forwarding_port) + "-:22" # bind the qemu ssh port to the hosts port # qemu_command += ["-redir", "tcp:" + str(self.ssh_forwarding_port) + "::22"] - print(coloured(AnsiColour.green, "\nListening for SSH connections on localhost:", self.ssh_forwarding_port, - sep="")) + print( + coloured( + AnsiColour.green, "\nListening for SSH connections on localhost:", self.ssh_forwarding_port, sep="" + ) + ) for x in self.extra_tcp_forwarding: # QEMU insists on having : field delimeters; add if not given - hg = x.split('=') + hg = x.split("=") if len(hg) != 2: self.fatal("Bad extra-tcp-forwarding (not just one '=' in '%s')" % x) (h, g) = hg - if ':' not in h: - h = ':' + h - if ':' not in g: - g = ':' + g + if ":" not in h: + h = ":" + h + if ":" not in g: + g = ":" + g user_network_options += ",hostfwd=tcp:" + h + "-" + g @@ -450,16 +525,18 @@ def add_smb_or_9p_dir(directory, target, share_name=None, readonly=False): self._after_disk_options += ["-snapshot"] # input("Press enter to continue") - qemu_command = self.qemu_options.get_commandline(qemu_command=self.chosen_qemu.binary, - kernel_file=qemu_loader_or_kernel, - disk_image=self.disk_image, - disk_image_format=self.disk_image_format, - add_network_device=self.qemu_user_networking, - bios_args=self.bios_flags, - user_network_args=user_network_options, - trap_on_unrepresentable=self.config.trap_on_unrepresentable, - debugger_on_cheri_trap=self.config.debugger_on_cheri_trap, - add_virtio_rng=self._add_virtio_rng) + qemu_command = self.qemu_options.get_commandline( + qemu_command=self.chosen_qemu.binary, + kernel_file=qemu_loader_or_kernel, + disk_image=self.disk_image, + disk_image_format=self.disk_image_format, + add_network_device=self.qemu_user_networking, + bios_args=self.bios_flags, + user_network_args=user_network_options, + trap_on_unrepresentable=self.config.trap_on_unrepresentable, + debugger_on_cheri_trap=self.config.debugger_on_cheri_trap, + add_virtio_rng=self._add_virtio_rng, + ) qemu_command += self._project_specific_options + self._after_disk_options + monitor_options qemu_command += logfile_options + self.extra_qemu_options + virtfs_args if self.disk_image is None: @@ -471,8 +548,10 @@ def add_smb_or_9p_dir(directory, target, share_name=None, readonly=False): if self.config.wait_for_debugger or self.config.debugger_in_tmux_pane: gdb_socket_placeholder = find_free_port(preferred_port=1234) gdb_port = gdb_socket_placeholder.port if self.config.gdb_random_port else 1234 - self.info(f"QEMU is waiting for GDB to attach (using `target remote :{gdb_port}`)." - " Once connected enter 'continue\\n' to continue booting") + self.info( + f"QEMU is waiting for GDB to attach (using `target remote :{gdb_port}`)." + " Once connected enter 'continue\\n' to continue booting" + ) def gdb_command(main_binary, bp=None, extra_binary=None) -> str: gdb_cmd = BuildGDB.get_install_dir(self, cross_target=CompilationTargets.NATIVE) / "bin/gdb" @@ -482,10 +561,13 @@ def gdb_command(main_binary, bp=None, extra_binary=None) -> str: # It seems this does not always work as expected, so also set substitute-path and # debug-file-directory. assert self.rootfs_path is not None - result.extend(["--init-eval-command=set sysroot " + str(self.rootfs_path), - "--init-eval-command=set substitute-path " + str(self.rootfs_path) + " /", - "--init-eval-command=set debug-file-directory " + str( - self.rootfs_path / "usr/lib/debug")]) + result.extend( + [ + "--init-eval-command=set sysroot " + str(self.rootfs_path), + "--init-eval-command=set substitute-path " + str(self.rootfs_path) + " /", + "--init-eval-command=set debug-file-directory " + str(self.rootfs_path / "usr/lib/debug"), + ] + ) # Once the file has been loaded set a breakpoint on panic() and connect to the remote host if bp: result.append("--eval-command=break " + bp) @@ -510,20 +592,31 @@ def gdb_command(main_binary, bp=None, extra_binary=None) -> str: else: self.info("\t", coloured(AnsiColour.red, gdb_command(path_to_kernel, "panic")), sep="") if self.rootfs_path is not None: - self.info("If you would like to debug /sbin/init (or any other statically linked program) run this" - " inside GDB:") + self.info( + "If you would like to debug /sbin/init (or any other statically linked program) run this" + " inside GDB:" + ) self.info(coloured(AnsiColour.red, "\tadd-symbol-file -o 0", str(self.rootfs_path / "sbin/init"))) - self.info("For dynamically linked programs you will have to add libraries at the correct offset." - " For example:") - self.info(coloured(AnsiColour.red, "\tadd-symbol-file -o 0x40212000", - str(self.rootfs_path / "lib/libc.so.7"))) + self.info( + "For dynamically linked programs you will have to add libraries at the correct offset." + " For example:" + ) + self.info( + coloured( + AnsiColour.red, "\tadd-symbol-file -o 0x40212000", str(self.rootfs_path / "lib/libc.so.7") + ) + ) self.info("If you would like to debug a userspace program (e.g. sbin/init):") - self.info("\t", coloured(AnsiColour.red, gdb_command(self.rootfs_path / "sbin/init", "main", - path_to_kernel)), sep="") + self.info( + "\t", + coloured(AnsiColour.red, gdb_command(self.rootfs_path / "sbin/init", "main", path_to_kernel)), + sep="", + ) self.info("Launching QEMU in suspended state...") def start_gdb_in_tmux_pane(command): import libtmux + server = libtmux.Server() if server is None: raise Exception("Tmux server not found") @@ -555,9 +648,11 @@ def start_gdb_in_tmux_pane(command): self.info(coloured(AnsiColour.red, f"Unable to start gdb in tmux: {e}")) gdb_socket_placeholder.socket.close() # the port is now available for qemu - qemu_command += ["-gdb", f"tcp::{gdb_port}", # wait for gdb on localhost:1234 - "-S", # freeze CPU at startup (use 'c' to start execution) - ] + qemu_command += [ + "-gdb", + f"tcp::{gdb_port}", # wait for gdb on localhost:1234 + "-S", # freeze CPU at startup (use 'c' to start execution) + ] # We want stdout/stderr here even when running with --quiet # FIXME: it seems like QEMU often breaks the line wrapping state: https://bugs.launchpad.net/qemu/+bug/1857449 self.run_cmd(qemu_command, stdout=sys.stdout, stderr=sys.stderr, give_tty_control=True) @@ -568,9 +663,9 @@ def print_port_usage(self, port: int): self.run_cmd("sockstat", "-P", "tcp", "-p", str(port)) elif OSInfo.IS_LINUX: if shutil.which("ss"): - self.run_cmd("sh", "-c", "ss -tulpne | grep \":" + str(port) + "\"") + self.run_cmd("sh", "-c", 'ss -tulpne | grep ":' + str(port) + '"') elif shutil.which("netstat"): - self.run_cmd("sh", "-c", "netstat -tulpne | grep \":" + str(port) + "\"") + self.run_cmd("sh", "-c", 'netstat -tulpne | grep ":' + str(port) + '"') else: self.info(coloured(AnsiColour.yellow, "Missing ss and netstat; unable to report port usage")) elif OSInfo.IS_MAC: @@ -599,20 +694,33 @@ class AbstractLaunchFreeBSD(LaunchQEMUBase, LaunchFreeBSDInterface): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.remote_kernel_path = cls.add_config_option( - "remote-kernel-path", show_help=True, + "remote-kernel-path", + show_help=True, help="When set rsync will be used to update the kernel image from a remote host before launching QEMU. " - "Useful when building and running on separate machines.") + "Useful when building and running on separate machines.", + ) cls.kernel_config = cls.add_config_option( - "alternative-kernel", show_help=True, + "alternative-kernel", + show_help=True, help="Select the kernel to run by specifying the kernel build configuration name." - "The list of available kernel configurations is given by --list-kernels") + "The list of available kernel configurations is given by --list-kernels", + ) cls.kernel_abi = cls.add_config_option( - "kernel-abi", show_help=True, - kind=KernelABI, enum_choices=[KernelABI.HYBRID, KernelABI.PURECAP], - help="Select extra kernel variant with the given ABI to run.") - - def __init__(self, config: CheriConfig, *, freebsd_class: "Optional[type[BuildFreeBSD]]" = None, - disk_image_class: "Optional[type[BuildDiskImageBase]]" = None, **kwargs): + "kernel-abi", + show_help=True, + kind=KernelABI, + enum_choices=[KernelABI.HYBRID, KernelABI.PURECAP], + help="Select extra kernel variant with the given ABI to run.", + ) + + def __init__( + self, + config: CheriConfig, + *, + freebsd_class: "Optional[type[BuildFreeBSD]]" = None, + disk_image_class: "Optional[type[BuildDiskImageBase]]" = None, + **kwargs, + ): super().__init__(config, **kwargs) self.freebsd_class = freebsd_class self.disk_image_class = disk_image_class @@ -693,7 +801,7 @@ def supported_architectures(self) -> "tuple[CrossCompileTarget, ...]": @classmethod def get_cross_target_index(cls, **kwargs): - xtarget = kwargs.get('xtarget', cls._xtarget) + xtarget = kwargs.get("xtarget", cls._xtarget) for idx, value in enumerate(cls.supported_architectures): if xtarget is value: return idx @@ -738,9 +846,13 @@ def run_tests(self): extra_args.append(f"--test-output-dir={tests_dir}") if self.kernel_abi is not None and self.crosscompile_target.is_hybrid_or_purecap_cheri(): extra_args.append(f"--expected-kernel-abi={self.kernel_abi.value}") - self.target_info.run_cheribsd_test_script("run_cheribsd_tests.py", *extra_args, - disk_image_path=self.disk_image, kernel_path=self.current_kernel, - rootfs_alternate_kernel_dir=rootfs_kernel_bootdir) + self.target_info.run_cheribsd_test_script( + "run_cheribsd_tests.py", + *extra_args, + disk_image_path=self.disk_image, + kernel_path=self.current_kernel, + rootfs_alternate_kernel_dir=rootfs_kernel_bootdir, + ) class LaunchCheriBSD(_RunMultiArchFreeBSDImage): @@ -838,8 +950,10 @@ class LaunchCheriBsdMfsRoot(LaunchMinimalCheriBSD): # XXX: Existing code isn't reqdy to run these but we want to support building them @classproperty def supported_architectures(self) -> "tuple[CrossCompileTarget, ...]": - return tuple(set(super().supported_architectures) - - {CompilationTargets.CHERIBSD_AARCH64, *CompilationTargets.ALL_CHERIBSD_MORELLO_TARGETS}) + return tuple( + set(super().supported_architectures) + - {CompilationTargets.CHERIBSD_AARCH64, *CompilationTargets.ALL_CHERIBSD_MORELLO_TARGETS} + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/pycheribuild/projects/sail.py b/pycheribuild/projects/sail.py index a97c0b301..599ceb80d 100644 --- a/pycheribuild/projects/sail.py +++ b/pycheribuild/projects/sail.py @@ -65,14 +65,19 @@ def process(self): self.check_required_system_tool("opam", homebrew="opam", apt="opam", cheribuild_target="opam-2.0") opam_path = shutil.which("opam") if opam_path: - opam_version = get_program_version(Path(opam_path), regex=b"(\\d+)\\.(\\d+)\\.?(\\d+)?", - config=self.config) + opam_version = get_program_version(Path(opam_path), regex=b"(\\d+)\\.(\\d+)\\.?(\\d+)?", config=self.config) min_version = (2, 0, 8) if opam_version < min_version: - install_inst = OSInfo.install_instructions("opam", False, apt="opam", - cheribuild_target="opam-2.0" if OSInfo.IS_LINUX else None) - self.dependency_error("Opam version", ".".join(map(str, opam_version)), "is too old. Need at least", - ".".join(map(str, min_version)), install_instructions=install_inst) + install_inst = OSInfo.install_instructions( + "opam", False, apt="opam", cheribuild_target="opam-2.0" if OSInfo.IS_LINUX else None + ) + self.dependency_error( + "Opam version", + ".".join(map(str, opam_version)), + "is too old. Need at least", + ".".join(map(str, min_version)), + install_instructions=install_inst, + ) @property def opam_binary(self): @@ -95,8 +100,9 @@ def _ensure_correct_switch(self): self.run_opam_cmd("switch", self.required_ocaml_version, _add_switch=False) except CalledProcessError: # create the switch if it doesn't exist - self.run_opam_cmd("switch", "--verbose", "--debug", "create", self.required_ocaml_version, - _add_switch=False) + self.run_opam_cmd( + "switch", "--verbose", "--debug", "create", self.required_ocaml_version, _add_switch=False + ) finally: self.__ignore_switch_version = False self.__using_correct_switch = True @@ -109,8 +115,10 @@ def run_opam_cmd(self, command, *args, ignore_errors=False, _add_switch=True, ** except CalledProcessError: if ignore_errors: # noinspection PyUnresolvedReferences - self.verbose_print("Ignoring non-zero exit code from", - coloured(AnsiColour.yellow, self.commandline_to_str(command_list))) + self.verbose_print( + "Ignoring non-zero exit code from", + coloured(AnsiColour.yellow, self.commandline_to_str(command_list)), + ) else: raise @@ -119,14 +127,25 @@ def _run_in_ocaml_env_prepare(self, cwd=None) -> "Tuple[Dict[Any, Union[Union[st cwd = self.source_dir if getattr(self, "source_dir") else "/" self._ensure_correct_switch() - opam_env = dict(GIT_TEMPLATE_DIR="", # see https://github.com/ocaml/opam/issues/3493 - OPAMROOT=self.opamroot, CCACHE_DISABLE=1, # https://github.com/ocaml/opam/issues/3395 - PATH=self.config.dollar_path_with_other_tools) + opam_env = dict( + GIT_TEMPLATE_DIR="", # see https://github.com/ocaml/opam/issues/3493 + OPAMROOT=self.opamroot, + CCACHE_DISABLE=1, # https://github.com/ocaml/opam/issues/3395 + PATH=self.config.dollar_path_with_other_tools, + ) if Path(self.opam_binary).is_absolute(): opam_env["OPAM_USER_PATH_RO"] = Path(self.opam_binary).parent if not (self.opamroot / "opam-init").exists(): - self.run_cmd(self.opam_binary, "init", "--cli=2.1", "--disable-sandboxing", "--root=" + str(self.opamroot), - "--no-setup", cwd="/", env=opam_env) + self.run_cmd( + self.opam_binary, + "init", + "--cli=2.1", + "--disable-sandboxing", + "--root=" + str(self.opamroot), + "--no-setup", + cwd="/", + env=opam_env, + ) return opam_env, cwd def run_in_ocaml_env(self, command: str, cwd=None, print_verbose_only=False, **kwargs): @@ -157,14 +176,24 @@ def process(self): self.makedirs(self.config.other_tools_dir / "bin") with tempfile.TemporaryDirectory() as td: base_url = "https://github.com/ocaml/opam/releases/download/" - self.download_file(Path(td, "opam"), url=base_url + "2.0.8/opam-2.0.8-x86_64-linux", - sha256="95365a873d9e3ae6fb48e6109b5fc5df3b4e526c9d65d20652a78e263f745a35") - self.install_file(Path(td, "opam"), self.config.other_tools_dir / "bin/opam", force=True, - print_verbose_only=False, mode=0o755) + self.download_file( + Path(td, "opam"), + url=base_url + "2.0.8/opam-2.0.8-x86_64-linux", + sha256="95365a873d9e3ae6fb48e6109b5fc5df3b4e526c9d65d20652a78e263f745a35", + ) + self.install_file( + Path(td, "opam"), + self.config.other_tools_dir / "bin/opam", + force=True, + print_verbose_only=False, + mode=0o755, + ) self.delete_file(self.config.other_tools_dir / "bin/opam.downloaded", print_verbose_only=False) else: - self.fatal("This target is only implement for Linux x86_64, for others operating systems you will have" - " to install opam 2.0 manually") + self.fatal( + "This target is only implement for Linux x86_64, for others operating systems you will have" + " to install opam 2.0 manually" + ) class BuildBubbleWrap(AutotoolsProject): @@ -191,8 +220,9 @@ class BuildSailFromOpam(ProjectUsingOpam): native_install_dir = DefaultInstallDir.CHERI_SDK build_in_source_dir = True # Cannot build out-of-source make_kind = MakeCommandKind.GnuMake - use_git_version = BoolConfigOption("use-git-version", - help="Install sail from github instead of using the latest released version") + use_git_version = BoolConfigOption( + "use-git-version", help="Install sail from github instead of using the latest released version" + ) def check_system_dependencies(self): super().check_system_dependencies() @@ -258,8 +288,12 @@ def check_system_dependencies(self): def compile(self, **kwargs): if self.with_trace_support: self.make_args.set(TRACE="yes") - cmd = [self.make_args.command, self.config.make_j_flag, "all", - *self.make_args.all_commandline_args(self.config)] + cmd = [ + self.make_args.command, + self.config.make_j_flag, + "all", + *self.make_args.all_commandline_args(self.config), + ] self.run_command_in_ocaml_env(cmd, cwd=self.source_dir) def install(self, **kwargs): @@ -277,6 +311,7 @@ def process(self): shell = os.getenv("SHELL", "bash") self.info(f"Starting sail shell (using {shell})... ") import subprocess + try: prompt_env = {} if "_P9K_TTY" in os.environ or "P9K_TTY" in os.environ: @@ -290,7 +325,8 @@ def process(self): with self.set_env(**prompt_env): self.run_command_in_ocaml_env( [shell, "-c", f"echo 'Entering sail environment, send CTRL+D to exit'; exec {shell} -i"], - cwd=os.getcwd()) + cwd=os.getcwd(), + ) except subprocess.CalledProcessError as e: if e.returncode == 130: return # User pressed Ctrl+D to exit shell, don't print an error @@ -311,8 +347,15 @@ def check_system_dependencies(self): def compile(self, **kwargs): for arch in ("RV64", "RV32"): - cmd = [self.make_args.command, self.config.make_j_flag, "ARCH=" + arch, "csim", "osim", "rvfi", - *self.make_args.all_commandline_args(self.config)] + cmd = [ + self.make_args.command, + self.config.make_j_flag, + "ARCH=" + arch, + "csim", + "osim", + "rvfi", + *self.make_args.all_commandline_args(self.config), + ] self.run_command_in_ocaml_env(cmd, cwd=self.source_dir) def install(self, **kwargs): @@ -335,8 +378,15 @@ def check_system_dependencies(self): def compile(self, **kwargs): for arch in ("RV64", "RV32"): - cmd = [self.make_args.command, self.config.make_j_flag, "ARCH=" + arch, "csim", "osim", "rvfi", - *self.make_args.all_commandline_args(self.config)] + cmd = [ + self.make_args.command, + self.config.make_j_flag, + "ARCH=" + arch, + "csim", + "osim", + "rvfi", + *self.make_args.all_commandline_args(self.config), + ] self.run_command_in_ocaml_env(cmd, cwd=self.source_dir) def install(self, **kwargs): @@ -358,6 +408,11 @@ def check_system_dependencies(self): self.check_required_system_header("gmp.h", homebrew="gmp", apt="libgmp-dev") def compile(self, **kwargs): - cmd = [self.make_args.command, self.config.make_j_flag, "gen_c", "check_sail", - *self.make_args.all_commandline_args(self.config)] + cmd = [ + self.make_args.command, + self.config.make_j_flag, + "gen_c", + "check_sail", + *self.make_args.all_commandline_args(self.config), + ] self.run_command_in_ocaml_env(cmd, cwd=self.source_dir) diff --git a/pycheribuild/projects/samba.py b/pycheribuild/projects/samba.py index 157523dd3..4ad593cd1 100644 --- a/pycheribuild/projects/samba.py +++ b/pycheribuild/projects/samba.py @@ -45,9 +45,12 @@ class BuildSamba(Project): else: build_in_source_dir = True # NB: We can't update beyond 4.13 due to https://bugzilla.samba.org/show_bug.cgi?id=15024 - repository = GitRepository("https://github.com/CTSRD-CHERI/samba.git", - old_urls=[b"https://github.com/samba-team/samba.git"], - default_branch="v4-13-stable", force_branch=True) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/samba.git", + old_urls=[b"https://github.com/samba-team/samba.git"], + default_branch="v4-13-stable", + force_branch=True, + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -65,25 +68,27 @@ def setup(self): super().setup() # Based on https://willhaley.com/blog/compile-samba-macos/ # Also try to disable everything that is not needed for QEMU user shares - self.configure_args.extend([ - "--disable-cephfs", - "--disable-cups", - "--disable-iprint", - "--disable-glusterfs", - "--disable-python", - "--without-acl-support", - "--without-ad-dc", - "--without-ads", - "--without-ldap", - "--without-pam", - "--without-quotas", - "--without-regedit", - "--without-syslog", - "--without-utmp", - "--without-winbind", - # "--without-json-audit", "--without-ldb-lmdb", (only needed in master not 4.8 stable) - "--prefix=" + str(self.install_dir), - ]) + self.configure_args.extend( + [ + "--disable-cephfs", + "--disable-cups", + "--disable-iprint", + "--disable-glusterfs", + "--disable-python", + "--without-acl-support", + "--without-ad-dc", + "--without-ads", + "--without-ldap", + "--without-pam", + "--without-quotas", + "--without-regedit", + "--without-syslog", + "--without-utmp", + "--without-winbind", + # "--without-json-audit", "--without-ldb-lmdb", (only needed in master not 4.8 stable) + "--prefix=" + str(self.install_dir), + ] + ) # version 4.9 "--without-json-audit", self.configure_args.append("--without-json") @@ -94,17 +99,24 @@ def configure(self, **kwargs): # XXX: Can't call contains_commit inside setup() since we might not have cloned the repo yet. if self.repository.contains_commit(self, "91c024dfd8ecf909f23ab8ee3816ae6a4c9b881c", src_dir=self.source_dir): # current master branch doesn't need as many workarounds - self.configure_args.extend([ - # Avoid depending on libraries from the build tree: - "--bundled-libraries=ALL", "--with-static-modules=ALL", - "--enable-debug", - ]) + self.configure_args.extend( + [ + # Avoid depending on libraries from the build tree: + "--bundled-libraries=ALL", + "--with-static-modules=ALL", + "--enable-debug", + ] + ) else: - self.configure_args.extend([ - "--without-ntvfs-fileserver", "--without-dnsupdate", - # Avoid depending on libraries from the build tree: - "--bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent", "--with-static-modules=ALL", - ]) + self.configure_args.extend( + [ + "--without-ntvfs-fileserver", + "--without-dnsupdate", + # Avoid depending on libraries from the build tree: + "--bundled-libraries=talloc,tdb,pytdb,ldb,pyldb,tevent,pytevent", + "--with-static-modules=ALL", + ] + ) super().configure(cwd=self.source_dir, **kwargs) def compile(self, **kwargs): @@ -123,14 +135,19 @@ def process(self): if OSInfo.IS_MAC: # We need icu4c, libarchive, readline and krb5 from homebrew: homebrew_dirs = [str(self.get_homebrew_prefix(pkg)) for pkg in ("krb5", "libarchive", "readline", "icu4c")] - with self.set_env(PATH=':'.join([x + "/bin" for x in homebrew_dirs]) + ':' + - ':'.join([x + "/sbin" for x in homebrew_dirs]) + ':' + - os.getenv("PATH", ""), - PKG_CONFIG_PATH=':'.join([x + "/lib/pkgconfig" for x in homebrew_dirs]) + ':' + - os.getenv("PKG_CONFIG_PATH", ""), - LDFLAGS=' '.join(["-L" + x + "/lib" for x in homebrew_dirs]), - CPPFLAGS=' '.join(["-I" + x + "/include" for x in homebrew_dirs]), - CFLAGS=' '.join(["-I" + x + "/include" for x in homebrew_dirs])): + with self.set_env( + PATH=":".join([x + "/bin" for x in homebrew_dirs]) + + ":" + + ":".join([x + "/sbin" for x in homebrew_dirs]) + + ":" + + os.getenv("PATH", ""), + PKG_CONFIG_PATH=":".join([x + "/lib/pkgconfig" for x in homebrew_dirs]) + + ":" + + os.getenv("PKG_CONFIG_PATH", ""), + LDFLAGS=" ".join(["-L" + x + "/lib" for x in homebrew_dirs]), + CPPFLAGS=" ".join(["-I" + x + "/include" for x in homebrew_dirs]), + CFLAGS=" ".join(["-I" + x + "/include" for x in homebrew_dirs]), + ): super().process() else: super().process() diff --git a/pycheribuild/projects/sdk.py b/pycheribuild/projects/sdk.py index 4e1d79452..6979080fc 100644 --- a/pycheribuild/projects/sdk.py +++ b/pycheribuild/projects/sdk.py @@ -93,8 +93,16 @@ def build_cheridis(self): # Compile the cheridis helper (TODO: add it to the LLVM repo instead?) cheridis_src = include_local_file("files/cheridis.c") self.makedirs(self.config.cheri_sdk_bindir) - self.run_cmd("cc", "-DLLVM_PATH=\"%s/\"" % str(self.config.cheri_sdk_bindir), "-x", "c", "-", - "-o", self.config.cheri_sdk_bindir / "cheridis", input=cheridis_src) + self.run_cmd( + "cc", + '-DLLVM_PATH="%s/"' % str(self.config.cheri_sdk_bindir), + "-x", + "c", + "-", + "-o", + self.config.cheri_sdk_bindir / "cheridis", + input=cheridis_src, + ) def process(self): self.install_cmake_config() diff --git a/pycheribuild/projects/simple_project.py b/pycheribuild/projects/simple_project.py index 883afc07d..b7e61ea3d 100644 --- a/pycheribuild/projects/simple_project.py +++ b/pycheribuild/projects/simple_project.py @@ -127,21 +127,30 @@ def __init__(cls, name: str, bases, clsdict) -> None: if clsdict.get("do_not_add_to_targets") is True: return # if do_not_add_to_targets is defined within the class we skip it elif name.endswith("Base"): - fatal_error("Found class name ending in Base (", name, ") but do_not_add_to_targets was not defined", - sep="", pretend=False) + fatal_error( + "Found class name ending in Base (", + name, + ") but do_not_add_to_targets was not defined", + sep="", + pretend=False, + ) def die(msg): sys.exit(inspect.getfile(cls) + ":" + str(inspect.findsource(cls)[1] + 1) + ": error: " + msg) + # load "target" field first then use that to infer the default source/build/install dir names target_name = None if "target" in clsdict: target_name = clsdict["target"] elif name.startswith("Build"): - target_name = name[len("Build"):].replace("_", "-").lower() + target_name = name[len("Build") :].replace("_", "-").lower() cls.target = target_name if not target_name: - die("target name is not set and cannot infer from class " + name + - " -- set target= or do_not_add_to_targets=True") + die( + "target name is not set and cannot infer from class " + + name + + " -- set target= or do_not_add_to_targets=True" + ) # Make the local config options dictionary read-only cls._local_config_options = MappingProxyType(cls._local_config_options) @@ -150,16 +159,20 @@ def die(msg): cls.default_directory_basename = target_name if "project_name" in clsdict: - die("project_name should no longer be used, change the definition of class " + name + - " to include target and/or default_directory_basename") + die( + "project_name should no longer be used, change the definition of class " + + name + + " to include target and/or default_directory_basename" + ) if cls.__dict__.get("dependencies_must_be_built") and not cls.dependencies: sys.exit("PseudoTarget with no dependencies should not exist!! Target name = " + target_name) supported_archs = cls.supported_architectures assert supported_archs, "Must not be empty: " + str(supported_archs) assert isinstance(supported_archs, tuple) - assert len(set(supported_archs)) == len( - supported_archs), "Duplicates in supported archs for " + cls.__name__ + ": " + str(supported_archs) + assert len(set(supported_archs)) == len(supported_archs), ( + "Duplicates in supported archs for " + cls.__name__ + ": " + str(supported_archs) + ) # TODO: if len(cls.supported_architectures) > 1: if cls._always_add_suffixed_targets or len(supported_archs) > 1: # Add a the target for the default architecture @@ -208,8 +221,10 @@ def die(msg): if arch.is_mips(include_purecap=False): new_cls._config_file_aliases += (replace_one(new_name, "-mips64", "-mips"),) elif arch.is_x86_64(include_purecap=False): - new_cls._config_file_aliases += (replace_one(new_name, "-amd64", "-x86"), - replace_one(new_name, "-amd64", "-x86_64")) + new_cls._config_file_aliases += ( + replace_one(new_name, "-amd64", "-x86"), + replace_one(new_name, "-amd64", "-x86_64"), + ) if len(set(new_cls._config_file_aliases)) != len(new_cls._config_file_aliases): raise ValueError(f"Duplicate aliases for {new_name}: {new_cls._config_file_aliases}") else: @@ -240,50 +255,71 @@ def __get__(self, instance: "SimpleProject", owner: "type[SimpleProject]"): if typing.TYPE_CHECKING: - # noinspection PyPep8Naming - def BoolConfigOption(name: str, help: str, # noqa: N802 - default: "typing.Union[bool, ComputedDefaultValue[bool]]" = False, **kwargs) -> bool: + + def BoolConfigOption( # noqa: N802 + name: str, + help: str, + default: "typing.Union[bool, ComputedDefaultValue[bool]]" = False, + **kwargs, + ) -> bool: return typing.cast(bool, default) # noinspection PyPep8Naming - def IntConfigOption(name: str, help: str, # noqa: N802 - default: "typing.Union[int, ComputedDefaultValue[int]]", **kwargs) -> int: + def IntConfigOption( # noqa: N802 + name: str, + help: str, + default: "typing.Union[int, ComputedDefaultValue[int]]", + **kwargs, + ) -> int: return typing.cast(int, default) # noinspection PyPep8Naming - def OptionalIntConfigOption(name: str, help: str, # noqa: N802 - default: "typing.Union[Optional[int], ComputedDefaultValue[Optional[int]]]" = None, - **kwargs) -> "Optional[int]": + def OptionalIntConfigOption( # noqa: N802 + name: str, + help: str, + default: "typing.Union[Optional[int], ComputedDefaultValue[Optional[int]]]" = None, + **kwargs, + ) -> "Optional[int]": return typing.cast(Optional[int], default) else: + class BoolConfigOption(PerProjectConfigOption): - def __init__(self, name: str, help: str, default: "typing.Union[bool, ComputedDefaultValue[bool]]" = False, - **kwargs): + def __init__( + self, name: str, help: str, default: "typing.Union[bool, ComputedDefaultValue[bool]]" = False, **kwargs + ): super().__init__(name, help, default, **kwargs) def register_config_option(self, owner: "type[SimpleProject]") -> ConfigOptionHandle: - return typing.cast(ConfigOptionHandle, - owner.add_bool_option(self._name, default=self._default, help=self._help, - **self._kwargs)) + return typing.cast( + ConfigOptionHandle, + owner.add_bool_option(self._name, default=self._default, help=self._help, **self._kwargs), + ) class IntConfigOption(PerProjectConfigOption): def __init__(self, name: str, help: str, default: "typing.Union[int, ComputedDefaultValue[int]]", **kwargs): super().__init__(name, help, default, **kwargs) def register_config_option(self, owner: "type[SimpleProject]") -> ConfigOptionHandle: - return typing.cast(ConfigOptionHandle, - owner.add_config_option(self._name, default=self._default, help=self._help, kind=int, - **self._kwargs)) + return typing.cast( + ConfigOptionHandle, + owner.add_config_option(self._name, default=self._default, help=self._help, kind=int, **self._kwargs), + ) class OptionalIntConfigOption(PerProjectConfigOption): - def __init__(self, name: str, help: str, - default: "typing.Union[Optional[int], ComputedDefaultValue[Optional[int]]]" = None, **kwargs): + def __init__( + self, + name: str, + help: str, + default: "typing.Union[Optional[int], ComputedDefaultValue[Optional[int]]]" = None, + **kwargs, + ): super().__init__(name, help, default, **kwargs) def register_config_option(self, owner: "type[SimpleProject]") -> ConfigOptionHandle: - return typing.cast(ConfigOptionHandle, - owner.add_config_option(self._name, default=self._default, help=self._help, kind=int, - **self._kwargs)) + return typing.cast( + ConfigOptionHandle, + owner.add_config_option(self._name, default=self._default, help=self._help, kind=int, **self._kwargs), + ) @functools.lru_cache(maxsize=20) @@ -294,8 +330,11 @@ def _cached_get_homebrew_prefix(package: "Optional[str]", config: CheriConfig): command.append(package) prefix = None try: - prefix_str = run_command(command, capture_output=True, run_in_pretend_mode=True, - print_verbose_only=False, config=config).stdout.decode("utf-8").strip() + prefix_str = ( + run_command(command, capture_output=True, run_in_pretend_mode=True, print_verbose_only=False, config=config) + .stdout.decode("utf-8") + .strip() + ) prefix = Path(prefix_str) if not prefix.exists(): prefix = None @@ -374,7 +413,7 @@ def is_toolchain_target(cls) -> bool: @property def _no_overwrite_allowed(self) -> "tuple[str]": - return ("_xtarget", ) + return ("_xtarget",) @classmethod def all_dependency_names(cls, config: CheriConfig) -> "list[str]": @@ -385,9 +424,14 @@ def all_dependency_names(cls, config: CheriConfig) -> "list[str]": # noinspection PyCallingNonCallable @classmethod - def _direct_dependencies(cls, config: CheriConfig, *, include_toolchain_dependencies: bool, - include_sdk_dependencies: bool, - explicit_dependencies_only: bool) -> "typing.Iterator[Target]": + def _direct_dependencies( + cls, + config: CheriConfig, + *, + include_toolchain_dependencies: bool, + include_sdk_dependencies: bool, + explicit_dependencies_only: bool, + ) -> "typing.Iterator[Target]": if not include_sdk_dependencies: include_toolchain_dependencies = False # --skip-sdk means skip toolchain and skip sysroot assert cls._xtarget is not None @@ -422,31 +466,50 @@ def _direct_dependencies(cls, config: CheriConfig, *, include_toolchain_dependen ) dependencies.append(dep_target.name) except KeyError: - fatal_error("Could not find sysroot target '", dep_name, "' for ", cls.__name__, sep="", - pretend=config.pretend, fatal_when_pretending=True) + fatal_error( + "Could not find sysroot target '", + dep_name, + "' for ", + cls.__name__, + sep="", + pretend=config.pretend, + fatal_when_pretending=True, + ) raise # Try to resovle the target names to actual targets and potentially add recursive depdencies for dep_name in dependencies: try: dep_target = target_manager.get_target( - dep_name, arch_for_unqualified_targets=expected_build_arch, config=config, caller=cls.target, + dep_name, + arch_for_unqualified_targets=expected_build_arch, + config=config, + caller=cls.target, ) except KeyError: - fatal_error("Could not find target '", dep_name, "' for ", cls.__name__, sep="", - pretend=config.pretend, fatal_when_pretending=True) + fatal_error( + "Could not find target '", + dep_name, + "' for ", + cls.__name__, + sep="", + pretend=config.pretend, + fatal_when_pretending=True, + ) raise # Handle --include-dependencies when --skip-sdk/--no-include-toolchain-dependencies is passed if explicit_dependencies_only: pass # add all explicit direct dependencies elif not include_sdk_dependencies and dep_target.project_class.is_sdk_target: if config.verbose: - status_update("Not adding ", cls.target, "dependency", dep_target.name, - "since it is an SDK target.") + status_update( + "Not adding ", cls.target, "dependency", dep_target.name, "since it is an SDK target." + ) continue elif not include_toolchain_dependencies and dep_target.project_class.is_toolchain_target(): if config.verbose: - status_update("Not adding ", cls.target, "dependency", dep_target.name, - "since it is a toolchain target.") + status_update( + "Not adding ", cls.target, "dependency", dep_target.name, "since it is a toolchain target." + ) continue # Now find the actual crosscompile targets for target aliases: if isinstance(dep_target, MultiArchTargetAlias): @@ -457,8 +520,9 @@ def _direct_dependencies(cls, config: CheriConfig, *, include_toolchain_dependen dep_target = tgt # print("Overriding with", tgt.name) break - assert not isinstance(dep_target, MultiArchTargetAlias), "All targets should be fully resolved but got " \ - + str(dep_target) + " in " + cls.__name__ + assert not isinstance(dep_target, MultiArchTargetAlias), ( + "All targets should be fully resolved but got " + str(dep_target) + " in " + cls.__name__ + ) if dep_target.project_class is cls: # assert False, "Found self as dependency:" + str(cls) continue @@ -484,23 +548,31 @@ def recursive_dependencies(cls, config: CheriConfig) -> "list[Target]": with_toolchain_deps = config.include_toolchain_dependencies and not cls.skip_toolchain_dependencies with_sdk_deps = not config.skip_sdk result = cls._recursive_dependencies_impl( - config, include_dependencies=config.include_dependencies or cls.dependencies_must_be_built, - include_toolchain_dependencies=with_toolchain_deps, include_sdk_dependencies=with_sdk_deps) + config, + include_dependencies=config.include_dependencies or cls.dependencies_must_be_built, + include_toolchain_dependencies=with_toolchain_deps, + include_sdk_dependencies=with_sdk_deps, + ) cls._cached_filtered_deps = result return result @classmethod - def _recursive_dependencies_impl(cls, config: CheriConfig, *, include_dependencies: bool, - include_toolchain_dependencies: bool, - dependency_chain: "Optional[list[type[SimpleProject]]]" = None, - include_sdk_dependencies: bool) -> "list[Target]": + def _recursive_dependencies_impl( + cls, + config: CheriConfig, + *, + include_dependencies: bool, + include_toolchain_dependencies: bool, + dependency_chain: "Optional[list[type[SimpleProject]]]" = None, + include_sdk_dependencies: bool, + ) -> "list[Target]": assert cls._xtarget is not None, cls if not include_dependencies: return [] if dependency_chain: new_dependency_chain = [*dependency_chain, cls] if cls in dependency_chain: - cycle = new_dependency_chain[new_dependency_chain.index(cls):] + cycle = new_dependency_chain[new_dependency_chain.index(cls) :] fatal_error("Cyclic dependency found:", " -> ".join(map(lambda c: c.target, cycle)), pretend=False) else: new_dependency_chain = [cls] @@ -511,9 +583,12 @@ def _recursive_dependencies_impl(cls, config: CheriConfig, *, include_dependenci if cached_result is not None: return cached_result result = [] - for target in cls._direct_dependencies(config, include_toolchain_dependencies=include_toolchain_dependencies, - include_sdk_dependencies=include_sdk_dependencies, - explicit_dependencies_only=cls.direct_dependencies_only): + for target in cls._direct_dependencies( + config, + include_toolchain_dependencies=include_toolchain_dependencies, + include_sdk_dependencies=include_sdk_dependencies, + explicit_dependencies_only=cls.direct_dependencies_only, + ): if config.should_skip_dependency(target.name, cls.target): continue @@ -523,10 +598,12 @@ def _recursive_dependencies_impl(cls, config: CheriConfig, *, include_dependenci continue # don't add recursive dependencies for e.g. "build-and-run" # now recursively add the other deps: recursive_deps = target.project_class._recursive_dependencies_impl( - config, include_dependencies=include_dependencies, + config, + include_dependencies=include_dependencies, include_toolchain_dependencies=include_toolchain_dependencies, include_sdk_dependencies=include_sdk_dependencies, - dependency_chain=new_dependency_chain) + dependency_chain=new_dependency_chain, + ) for r in recursive_deps: if r not in result: result.append(r) @@ -546,13 +623,17 @@ def cached_full_dependencies(cls) -> "list[Target]": @classmethod def _cache_full_dependencies(cls, config, *, allow_already_cached=False) -> None: assert allow_already_cached or cls.__dict__.get("_cached_full_deps", None) is None, "Already cached??" - cls._cached_full_deps = cls._recursive_dependencies_impl(config, include_dependencies=True, - include_toolchain_dependencies=True, - include_sdk_dependencies=True) + cls._cached_full_deps = cls._recursive_dependencies_impl( + config, include_dependencies=True, include_toolchain_dependencies=True, include_sdk_dependencies=True + ) @classmethod - def get_instance(cls: "type[T]", caller: "Optional[AbstractProject]", config: "Optional[CheriConfig]" = None, - cross_target: Optional[CrossCompileTarget] = None) -> T: + def get_instance( + cls: "type[T]", + caller: "Optional[AbstractProject]", + config: "Optional[CheriConfig]" = None, + cross_target: Optional[CrossCompileTarget] = None, + ) -> T: # TODO: assert that target manager has been initialized if caller is not None: if config is None: @@ -566,8 +647,9 @@ def get_instance(cls: "type[T]", caller: "Optional[AbstractProject]", config: "O return cls.get_instance_for_cross_target(cross_target, config, caller=caller) @classmethod - def _get_instance_no_setup(cls: "type[T]", caller: AbstractProject, - cross_target: Optional[CrossCompileTarget] = None) -> T: + def _get_instance_no_setup( + cls: "type[T]", caller: AbstractProject, cross_target: Optional[CrossCompileTarget] = None + ) -> T: if cross_target is None: cross_target = caller.crosscompile_target target_name = cls.target @@ -582,8 +664,12 @@ def _get_instance_no_setup(cls: "type[T]", caller: AbstractProject, return result @staticmethod - def get_instance_for_target_name(target_name: str, cross_target: CrossCompileTarget, config: CheriConfig, - caller: "Optional[AbstractProject]" = None) -> "SimpleProject": + def get_instance_for_target_name( + target_name: str, + cross_target: CrossCompileTarget, + config: CheriConfig, + caller: "Optional[AbstractProject]" = None, + ) -> "SimpleProject": if caller is not None: assert caller._init_called, "Cannot call this inside __init__()" assert cross_target is not None @@ -592,13 +678,18 @@ def get_instance_for_target_name(target_name: str, cross_target: CrossCompileTar assert isinstance(result, SimpleProject) found_target = result.get_crosscompile_target() # XXX: FIXME: add cross target to every call - assert found_target is cross_target, (f"Didn't find right instance of {target_name}: {found_target} vs " - f"{cross_target}, caller was {caller!r}") + assert found_target is cross_target, ( + f"Didn't find right instance of {target_name}: {found_target} vs " f"{cross_target}, caller was {caller!r}" + ) return result @classmethod - def get_instance_for_cross_target(cls: "type[T]", cross_target: CrossCompileTarget, config: CheriConfig, - caller: "Optional[AbstractProject]" = None) -> T: + def get_instance_for_cross_target( + cls: "type[T]", + cross_target: CrossCompileTarget, + config: CheriConfig, + caller: "Optional[AbstractProject]" = None, + ) -> T: # Also need to handle calling self.get_instance_for_cross_target() on a target-specific instance # In that case cls.target returns e.g. foo-mips, etc. and target_manager will always return the MIPS version # which is not what we want if there is an explicit cross_target @@ -696,7 +787,7 @@ def build_configuration_suffix(self, target: Optional[CrossCompileTarget] = None @property def triple_arch(self) -> str: target_triple = self.target_info.target_triple - return target_triple[:target_triple.find("-")] + return target_triple[: target_triple.find("-")] @property def sdk_sysroot(self) -> Path: @@ -714,8 +805,12 @@ def sdk_bindir(self) -> Path: def display_name(self) -> str: if self._xtarget is None: return self.target + " (target alias)" - return self.target + " (" + self._xtarget.build_suffix(self.config, - include_os=self.include_os_in_target_suffix) + ")" + return ( + self.target + + " (" + + self._xtarget.build_suffix(self.config, include_os=self.include_os_in_target_suffix) + + ")" + ) @classmethod def get_class_for_target(cls: "type[T]", arch: CrossCompileTarget) -> "type[T]": @@ -753,14 +848,36 @@ def cross_sysroot_path(self) -> Path: # Duplicate all arguments instead of using **kwargs to get sensible code completion # noinspection PyShadowingBuiltins - def run_cmd(self, *args, capture_output=False, capture_error=False, input: "Optional[Union[str, bytes]]" = None, - timeout=None, print_verbose_only=False, run_in_pretend_mode=False, raise_in_pretend_mode=False, - no_print=False, replace_env=False, give_tty_control=False, - **kwargs) -> "subprocess.CompletedProcess[bytes]": - return run_command(*args, capture_output=capture_output, capture_error=capture_error, input=input, - timeout=timeout, config=self.config, print_verbose_only=print_verbose_only, - run_in_pretend_mode=run_in_pretend_mode, raise_in_pretend_mode=raise_in_pretend_mode, - no_print=no_print, replace_env=replace_env, give_tty_control=give_tty_control, **kwargs) + def run_cmd( + self, + *args, + capture_output=False, + capture_error=False, + input: "Optional[Union[str, bytes]]" = None, + timeout=None, + print_verbose_only=False, + run_in_pretend_mode=False, + raise_in_pretend_mode=False, + no_print=False, + replace_env=False, + give_tty_control=False, + **kwargs, + ) -> "subprocess.CompletedProcess[bytes]": + return run_command( + *args, + capture_output=capture_output, + capture_error=capture_error, + input=input, + timeout=timeout, + config=self.config, + print_verbose_only=print_verbose_only, + run_in_pretend_mode=run_in_pretend_mode, + raise_in_pretend_mode=raise_in_pretend_mode, + no_print=no_print, + replace_env=replace_env, + give_tty_control=give_tty_control, + **kwargs, + ) def set_env(self, *, print_verbose_only=True, **environ) -> "typing.ContextManager": return set_env(print_verbose_only=print_verbose_only, config=self.config, **environ) @@ -776,12 +893,20 @@ def get_config_option_name(cls, option: str) -> str: return option.full_option_name @classmethod - def add_config_option(cls, name: str, *, show_help=False, altname: "Optional[str]" = None, - kind: "Union[type[T], Callable[[str], T]]" = str, - default: "Union[ComputedDefaultValue[T], Callable[[CheriConfig, SimpleProject], T], T, None]" - = None, only_add_for_targets: "Optional[tuple[CrossCompileTarget, ...]]" = None, - extra_fallback_config_names: "Optional[list[str]]" = None, _allow_unknown_targets=False, - use_default_fallback_config_names=True, **kwargs) -> Optional[T]: + def add_config_option( + cls, + name: str, + *, + show_help=False, + altname: "Optional[str]" = None, + kind: "Union[type[T], Callable[[str], T]]" = str, + default: "Union[ComputedDefaultValue[T], Callable[[CheriConfig, SimpleProject], T], T, None]" = None, + only_add_for_targets: "Optional[tuple[CrossCompileTarget, ...]]" = None, + extra_fallback_config_names: "Optional[list[str]]" = None, + _allow_unknown_targets=False, + use_default_fallback_config_names=True, + **kwargs, + ) -> Optional[T]: fullname = cls.target + "/" + name # We abuse shortname to implement altname if altname is not None: @@ -802,7 +927,8 @@ def add_config_option(cls, name: str, *, show_help=False, altname: "Optional[str if hasattr(cls._config_loader, "_parser"): # XXX: Use hasattr instead of isinstance to avoid imports. # noinspection PyProtectedMember cls._commandline_option_group = cls._config_loader._parser.add_argument_group( - "Options for target '" + cls.target + "'") + "Options for target '" + cls.target + "'" + ) else: cls._commandline_option_group = None if cls.hide_options_from_help: @@ -821,9 +947,13 @@ def add_config_option(cls, name: str, *, show_help=False, altname: "Optional[str # If we are adding to the base class or the target is not in the list, emit a warning if not _allow_unknown_targets: for t in only_add_for_targets: - assert t in cls.supported_architectures, \ - cls.__name__ + ": some of " + str(only_add_for_targets) + " not in " + str( - cls.supported_architectures) + assert t in cls.supported_architectures, ( + cls.__name__ + + ": some of " + + str(only_add_for_targets) + + " not in " + + str(cls.supported_architectures) + ) if target is not None and target not in only_add_for_targets and not typing.TYPE_CHECKING: kwargs["option_cls"] = DefaultValueOnlyConfigOption kwargs["fallback_replaceable"] = True @@ -850,22 +980,42 @@ def add_config_option(cls, name: str, *, show_help=False, altname: "Optional[str if extra_fallback_config_names: fallback_config_names.extend(extra_fallback_config_names) legacy_alias_target_names = [tgt + "/" + name for tgt in cls.__dict__.get("_config_file_aliases", tuple())] - return cls._config_loader.add_option(fullname, shortname, default=default, type=kind, - _owning_class=cls, group=cls._commandline_option_group, - help_hidden=help_hidden, _fallback_names=fallback_config_names, - _legacy_alias_names=legacy_alias_target_names, **kwargs) + return cls._config_loader.add_option( + fullname, + shortname, + default=default, + type=kind, + _owning_class=cls, + group=cls._commandline_option_group, + help_hidden=help_hidden, + _fallback_names=fallback_config_names, + _legacy_alias_names=legacy_alias_target_names, + **kwargs, + ) @classmethod - def add_bool_option(cls, name: str, *, altname=None, - only_add_for_targets: "Optional[tuple[CrossCompileTarget, ...]]" = None, - default: "Union[bool, ComputedDefaultValue[bool]]" = False, **kwargs) -> bool: - return typing.cast(bool, cls.add_config_option(name, default=default, kind=bool, altname=altname, - only_add_for_targets=only_add_for_targets, **kwargs)) + def add_bool_option( + cls, + name: str, + *, + altname=None, + only_add_for_targets: "Optional[tuple[CrossCompileTarget, ...]]" = None, + default: "Union[bool, ComputedDefaultValue[bool]]" = False, + **kwargs, + ) -> bool: + return typing.cast( + bool, + cls.add_config_option( + name, default=default, kind=bool, altname=altname, only_add_for_targets=only_add_for_targets, **kwargs + ), + ) @classmethod def add_list_option(cls, name: str, *, default=None, **kwargs) -> "list[str]": - return typing.cast(typing.List[str], - cls.add_config_option(name, kind=list, default=[] if default is None else default, **kwargs)) + return typing.cast( + typing.List[str], + cls.add_config_option(name, kind=list, default=[] if default is None else default, **kwargs), + ) @classmethod def add_optional_path_option(cls, name: str, **kwargs) -> Optional[Path]: @@ -873,8 +1023,11 @@ def add_optional_path_option(cls, name: str, **kwargs) -> Optional[Path]: @classmethod def add_path_option( - cls, name: str, *, - default: "Union[ComputedDefaultValue[Path], Callable[[CheriConfig, SimpleProject], Path], Path]", **kwargs, + cls, + name: str, + *, + default: "Union[ComputedDefaultValue[Path], Callable[[CheriConfig, SimpleProject], Path], Path]", + **kwargs, ) -> Path: return typing.cast(Path, cls.add_config_option(name, kind=Path, default=default, **kwargs)) @@ -904,7 +1057,8 @@ def __init__(self, config: CheriConfig, *, crosscompile_target: CrossCompileTarg assert self._xtarget == crosscompile_target, "Failed to update all callers?" assert not self._should_not_be_instantiated, "Should not have instantiated " + self.__class__.__name__ assert self.__class__ in self.__config_options_set, "Forgot to call super().setup_config_options()? " + str( - self.__class__) + self.__class__ + ) self._system_deps_checked = False self._setup_called = False self._setup_late_called = False @@ -934,21 +1088,42 @@ def _validate_cheribuild_target_for_system_deps(self, cheribuild_target: "Option tgt = target_manager.get_target(cheribuild_target, config=self.config, caller=self) # And check that it's a native target: if not tgt.project_class.get_crosscompile_target().is_native(): - self.fatal("add_required_*() should use a native cheribuild target and not ", cheribuild_target, - "- found while processing", self.target, fatal_when_pretending=True) - - def check_required_system_tool(self, executable: str, instructions: "Optional[InstallInstructions]" = None, - default: "Optional[str]" = None, freebsd: "Optional[str]" = None, - apt: "Optional[str]" = None, zypper: "Optional[str]" = None, - homebrew: "Optional[str]" = None, cheribuild_target: "Optional[str]" = None, - alternative_instructions: "Optional[str]" = None, - compat_abi: "Optional[bool]" = None): + self.fatal( + "add_required_*() should use a native cheribuild target and not ", + cheribuild_target, + "- found while processing", + self.target, + fatal_when_pretending=True, + ) + + def check_required_system_tool( + self, + executable: str, + instructions: "Optional[InstallInstructions]" = None, + default: "Optional[str]" = None, + freebsd: "Optional[str]" = None, + apt: "Optional[str]" = None, + zypper: "Optional[str]" = None, + homebrew: "Optional[str]" = None, + cheribuild_target: "Optional[str]" = None, + alternative_instructions: "Optional[str]" = None, + compat_abi: "Optional[bool]" = None, + ): if instructions is None: if compat_abi is None: compat_abi = self.compiling_for_host() and self.compiling_for_cheri_hybrid() instructions = OSInfo.install_instructions( - executable, is_lib=False, default=default, freebsd=freebsd, zypper=zypper, apt=apt, homebrew=homebrew, - cheribuild_target=cheribuild_target, alternative=alternative_instructions, compat_abi=compat_abi) + executable, + is_lib=False, + default=default, + freebsd=freebsd, + zypper=zypper, + apt=apt, + homebrew=homebrew, + cheribuild_target=cheribuild_target, + alternative=alternative_instructions, + compat_abi=compat_abi, + ) if executable in self.__checked_system_tools: # If we already checked for this tool, the install instructions should match assert instructions.fixit_hint() == self.__checked_system_tools[executable].fixit_hint(), executable @@ -957,22 +1132,42 @@ def check_required_system_tool(self, executable: str, instructions: "Optional[In assert instructions.alternative == alternative_instructions self._validate_cheribuild_target_for_system_deps(instructions.cheribuild_target) if not shutil.which(str(executable)): - self.dependency_error("Required program", executable, "is missing!", install_instructions=instructions, - cheribuild_target=instructions.cheribuild_target, - cheribuild_xtarget=BasicCompilationTargets.NATIVE) + self.dependency_error( + "Required program", + executable, + "is missing!", + install_instructions=instructions, + cheribuild_target=instructions.cheribuild_target, + cheribuild_xtarget=BasicCompilationTargets.NATIVE, + ) self.__checked_system_tools[executable] = instructions - def check_required_pkg_config(self, package: str, instructions: "Optional[InstallInstructions]" = None, - default: "Optional[str]" = None, freebsd: "Optional[str]" = None, - apt: "Optional[str]" = None, zypper: "Optional[str]" = None, - homebrew: "Optional[str]" = None, cheribuild_target: "Optional[str]" = None, - alternative_instructions: "Optional[str]" = None) -> None: + def check_required_pkg_config( + self, + package: str, + instructions: "Optional[InstallInstructions]" = None, + default: "Optional[str]" = None, + freebsd: "Optional[str]" = None, + apt: "Optional[str]" = None, + zypper: "Optional[str]" = None, + homebrew: "Optional[str]" = None, + cheribuild_target: "Optional[str]" = None, + alternative_instructions: "Optional[str]" = None, + ) -> None: if "pkg-config" not in self.__checked_system_tools: self.check_required_system_tool("pkg-config", freebsd="pkgconf", homebrew="pkg-config", apt="pkg-config") if instructions is None: instructions = OSInfo.install_instructions( - package, is_lib=False, default=default, freebsd=freebsd, zypper=zypper, apt=apt, homebrew=homebrew, - cheribuild_target=cheribuild_target, alternative=alternative_instructions) + package, + is_lib=False, + default=default, + freebsd=freebsd, + zypper=zypper, + apt=apt, + homebrew=homebrew, + cheribuild_target=cheribuild_target, + alternative=alternative_instructions, + ) if package in self.__checked_pkg_config: # If we already checked for this pkg-config .pc file, the install instructions should match assert instructions.fixit_hint() == self.__checked_pkg_config[package].fixit_hint(), package @@ -991,21 +1186,41 @@ def check_required_pkg_config(self, package: str, instructions: "Optional[Instal with self.set_env(**env): self.run_cmd(["pkg-config", "--modversion", package], capture_output=True) except subprocess.CalledProcessError as e: - self.dependency_error("Required pkg-config file for", package, "is missing:", e, - install_instructions=instructions, - cheribuild_target=instructions.cheribuild_target, - cheribuild_xtarget=BasicCompilationTargets.NATIVE) + self.dependency_error( + "Required pkg-config file for", + package, + "is missing:", + e, + install_instructions=instructions, + cheribuild_target=instructions.cheribuild_target, + cheribuild_xtarget=BasicCompilationTargets.NATIVE, + ) self.__checked_pkg_config[package] = instructions - def check_required_system_header(self, header: str, instructions: "Optional[InstallInstructions]" = None, - default: "Optional[str]" = None, freebsd: "Optional[str]" = None, - apt: "Optional[str]" = None, zypper: "Optional[str]" = None, - homebrew: "Optional[str]" = None, cheribuild_target: "Optional[str]" = None, - alternative_instructions: "Optional[str]" = None) -> None: + def check_required_system_header( + self, + header: str, + instructions: "Optional[InstallInstructions]" = None, + default: "Optional[str]" = None, + freebsd: "Optional[str]" = None, + apt: "Optional[str]" = None, + zypper: "Optional[str]" = None, + homebrew: "Optional[str]" = None, + cheribuild_target: "Optional[str]" = None, + alternative_instructions: "Optional[str]" = None, + ) -> None: if instructions is None: instructions = OSInfo.install_instructions( - header, is_lib=False, default=default, freebsd=freebsd, zypper=zypper, apt=apt, homebrew=homebrew, - cheribuild_target=cheribuild_target, alternative=alternative_instructions) + header, + is_lib=False, + default=default, + freebsd=freebsd, + zypper=zypper, + apt=apt, + homebrew=homebrew, + cheribuild_target=cheribuild_target, + alternative=alternative_instructions, + ) if header in self.__checked_system_headers: # If we already checked for this header file, the install instructions should match assert instructions.fixit_hint() == self.__checked_system_headers[header].fixit_hint(), header @@ -1013,18 +1228,26 @@ def check_required_system_header(self, header: str, instructions: "Optional[Inst self._validate_cheribuild_target_for_system_deps(instructions.cheribuild_target) include_dirs = self.get_compiler_info(self.CC).get_include_dirs(self.essential_compiler_and_linker_flags) if not any(Path(d, header).exists() for d in include_dirs): - self.dependency_error("Required C header", header, "is missing!", install_instructions=instructions, - cheribuild_target=instructions.cheribuild_target, - cheribuild_xtarget=BasicCompilationTargets.NATIVE) + self.dependency_error( + "Required C header", + header, + "is missing!", + install_instructions=instructions, + cheribuild_target=instructions.cheribuild_target, + cheribuild_xtarget=BasicCompilationTargets.NATIVE, + ) self.__checked_system_headers[header] = instructions - def query_yes_no(self, message: str = "", *, default_result=False, force_result=True, - yes_no_str: "Optional[str]" = None) -> bool: - return query_yes_no(self.config, message, default_result=default_result, force_result=force_result, - yes_no_str=yes_no_str) + def query_yes_no( + self, message: str = "", *, default_result=False, force_result=True, yes_no_str: "Optional[str]" = None + ) -> bool: + return query_yes_no( + self.config, message, default_result=default_result, force_result=force_result, yes_no_str=yes_no_str + ) - def ask_for_confirmation(self, message: str, error_message="Cannot continue.", default_result=True, - **kwargs) -> None: + def ask_for_confirmation( + self, message: str, error_message="Cannot continue.", default_result=True, **kwargs + ) -> None: if not self.query_yes_no(message, default_result=default_result, **kwargs): self.fatal(error_message) @@ -1069,9 +1292,17 @@ def _show_line_stdout_filter(self, line: bytes) -> None: # def _stdout_filter(self, line: bytes): # self._line_not_important_stdout_filter(line) - def run_with_logfile(self, args: "typing.Sequence[str]", logfile_name: str, *, stdout_filter=None, - cwd: "Optional[Path]" = None, env: "Optional[dict[str, Optional[str]]]" = None, - append_to_logfile=False, stdin=subprocess.DEVNULL) -> None: + def run_with_logfile( + self, + args: "typing.Sequence[str]", + logfile_name: str, + *, + stdout_filter=None, + cwd: "Optional[Path]" = None, + env: "Optional[dict[str, Optional[str]]]" = None, + append_to_logfile=False, + stdin=subprocess.DEVNULL, + ) -> None: """ Runs make and logs the output config.quiet doesn't display anything, normal only status updates and config.verbose everything @@ -1128,19 +1359,25 @@ def run_with_logfile(self, args: "typing.Sequence[str]", logfile_name: str, *, s check_call_handle_noexec(args, cwd=str(cwd), stdout=logfile, stderr=logfile, stdin=stdin, env=new_env) return with keep_terminal_sane(command=args): - make = popen_handle_noexec(args, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=stdin, env=new_env) + make = popen_handle_noexec( + args, cwd=str(cwd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=stdin, env=new_env + ) self.__run_process_with_filtered_output(make, logfile, stdout_filter, args) - def __run_process_with_filtered_output(self, proc: subprocess.Popen, logfile: "Optional[typing.IO]", - stdout_filter: "Callable[[bytes], None]", - args: "list[str]"): + def __run_process_with_filtered_output( + self, + proc: subprocess.Popen, + logfile: "Optional[typing.IO]", + stdout_filter: "Callable[[bytes], None]", + args: "list[str]", + ): logfile_lock = threading.Lock() # we need a mutex so the logfile line buffer doesn't get messed up stderr_thread = None if logfile: # use a thread to print stderr output and write it to logfile (not using a thread would block) - stderr_thread = threading.Thread(target=self._handle_stderr, - args=(logfile, proc.stderr, logfile_lock, self)) + stderr_thread = threading.Thread( + target=self._handle_stderr, args=(logfile, proc.stderr, logfile_lock, self) + ) stderr_thread.start() for line in proc.stdout: with logfile_lock: # make sure we don't interleave stdout and stderr lines @@ -1173,8 +1410,9 @@ def __run_process_with_filtered_output(self, proc: subprocess.Popen, logfile: "O message = ("See " + logfile.name + " for details.").encode("utf-8") if logfile else None raise subprocess.CalledProcessError(retcode, args, None, stderr=message) - def maybe_strip_elf_file(self, file: Path, *, output_path: "Optional[Path]" = None, - print_verbose_only=True) -> bool: + def maybe_strip_elf_file( + self, file: Path, *, output_path: "Optional[Path]" = None, print_verbose_only=True + ) -> bool: """Runs llvm-strip on the file if it is an ELF file and it is safe to do so.""" # NB: Must check if it's a symlink first; if it refers to a path we # don't have permission to stat on the host (e.g. a symlink into /root) @@ -1224,8 +1462,13 @@ def _cleanup_renamed_files(self, current_path: Path, current_suffix: str, old_su if not old_suffixes: return if current_suffix not in current_path.name: - self.info("Warning:", current_path.name, "does not include expected suffix", current_suffix, - "-- either it was set in the config file or this is a logic error.") + self.info( + "Warning:", + current_path.name, + "does not include expected suffix", + current_suffix, + "-- either it was set in the config file or this is a logic error.", + ) # raise ValueError((current_path, current_suffix)) for old_suffix in old_suffixes: old_name = current_path.name.replace(current_suffix, old_suffix) @@ -1241,11 +1484,16 @@ def _cleanup_renamed_files(self, current_path: Path, current_suffix: str, old_su if self.query_yes_no("Would you like to remove the old directory " + str(old_path)): self._delete_directories(old_path) - def _dependency_message(self, *args, problem="missing", - install_instructions: "Optional[InstallInstructions]" = None, - cheribuild_target: "Optional[str]" = None, - cheribuild_xtarget: "Optional[CrossCompileTarget]" = None, - cheribuild_action: str = "install", fatal: bool): + def _dependency_message( + self, + *args, + problem="missing", + install_instructions: "Optional[InstallInstructions]" = None, + cheribuild_target: "Optional[str]" = None, + cheribuild_xtarget: "Optional[CrossCompileTarget]" = None, + cheribuild_action: str = "install", + fatal: bool, + ): self._system_deps_checked = True # make sure this is always set if install_instructions is not None and isinstance(install_instructions, InstallInstructions): install_instructions = install_instructions.fixit_hint() @@ -1253,15 +1501,24 @@ def _dependency_message(self, *args, problem="missing", self.warning("Dependency for", self.target, problem + ":", *args, fixit_hint=install_instructions) if not self._setup_late_called: # TODO: make this a fatal error - self.warning("TODO: Should not call dependency_error() with a cheribuild target fixit before " - "setup() has completed. Move the call to process() instead.") + self.warning( + "TODO: Should not call dependency_error() with a cheribuild target fixit before " + "setup() has completed. Move the call to process() instead." + ) self.fatal("Dependency for", self.target, problem + ":", *args, fixit_hint=install_instructions) return - if self.query_yes_no("Would you like to " + cheribuild_action + " the dependency (" + cheribuild_target + - ") using cheribuild?", force_result=False if is_jenkins_build() else True): + if self.query_yes_no( + "Would you like to " + + cheribuild_action + + " the dependency (" + + cheribuild_target + + ") using cheribuild?", + force_result=False if is_jenkins_build() else True, + ): xtarget = cheribuild_xtarget if cheribuild_xtarget is not None else self.crosscompile_target - dep_target = target_manager.get_target(cheribuild_target, required_arch=xtarget, config=self.config, - caller=self) + dep_target = target_manager.get_target( + cheribuild_target, required_arch=xtarget, config=self.config, caller=self + ) dep_target.check_system_deps(self.config) assert dep_target.get_or_create_project(None, self.config, caller=self).crosscompile_target == xtarget dep_target.execute(self.config) @@ -1269,21 +1526,43 @@ def _dependency_message(self, *args, problem="missing", if fatal: self.fatal("Dependency for", self.target, problem + ":", *args, fixit_hint=install_instructions) - def dependency_error(self, *args, problem="missing", install_instructions: "Optional[InstallInstructions]" = None, - cheribuild_target: "Optional[str]" = None, - cheribuild_xtarget: "Optional[CrossCompileTarget]" = None, cheribuild_action: str = "install"): - self._dependency_message(*args, problem=problem, install_instructions=install_instructions, - cheribuild_target=cheribuild_target, cheribuild_action=cheribuild_action, - cheribuild_xtarget=cheribuild_xtarget, fatal=True) - - def dependency_warning(self, *args, problem="missing", - install_instructions: "Optional[InstallInstructions]" = None, - cheribuild_target: "Optional[str]" = None, - cheribuild_xtarget: "Optional[CrossCompileTarget]" = None, - cheribuild_action: str = "install"): - self._dependency_message(*args, problem=problem, install_instructions=install_instructions, - cheribuild_target=cheribuild_target, cheribuild_xtarget=cheribuild_xtarget, - cheribuild_action=cheribuild_action, fatal=False) + def dependency_error( + self, + *args, + problem="missing", + install_instructions: "Optional[InstallInstructions]" = None, + cheribuild_target: "Optional[str]" = None, + cheribuild_xtarget: "Optional[CrossCompileTarget]" = None, + cheribuild_action: str = "install", + ): + self._dependency_message( + *args, + problem=problem, + install_instructions=install_instructions, + cheribuild_target=cheribuild_target, + cheribuild_action=cheribuild_action, + cheribuild_xtarget=cheribuild_xtarget, + fatal=True, + ) + + def dependency_warning( + self, + *args, + problem="missing", + install_instructions: "Optional[InstallInstructions]" = None, + cheribuild_target: "Optional[str]" = None, + cheribuild_xtarget: "Optional[CrossCompileTarget]" = None, + cheribuild_action: str = "install", + ): + self._dependency_message( + *args, + problem=problem, + install_instructions=install_instructions, + cheribuild_target=cheribuild_target, + cheribuild_xtarget=cheribuild_xtarget, + cheribuild_action=cheribuild_action, + fatal=False, + ) def check_system_dependencies(self) -> None: """ @@ -1297,8 +1576,11 @@ def get_homebrew_prefix(self, package: "Optional[str]" = None, optional: bool = if not prefix and not optional: prefix = Path("/fake/homebrew/prefix/when/pretending") if package: - self.dependency_error("Could not find homebrew package", package, - install_instructions=InstallInstructions(f"Try running `brew install {package}`")) + self.dependency_error( + "Could not find homebrew package", + package, + install_instructions=InstallInstructions(f"Try running `brew install {package}`"), + ) prefix = prefix / "opt" / package else: self.dependency_error("Could not find homebrew") @@ -1367,8 +1649,10 @@ def download_file(self, dest: Path, url: str, sha256: "Optional[str]" = None) -> elif shutil.which("fetch"): # pre-installed on FreeBSD/CheriBSD self.run_cmd("fetch", "-o", dest, url) else: - self.dependency_error("Cannot find a tool to download target URL.", - install_instructions=InstallInstructions("Please install wget or curl")) + self.dependency_error( + "Cannot find a tool to download target URL.", + install_instructions=InstallInstructions("Please install wget or curl"), + ) downloaded_sha256 = self.sha256sum(dest) self.verbose_print("Downloaded", url, "with SHA256 hash", downloaded_sha256) if sha256 is not None and downloaded_sha256 != sha256: diff --git a/pycheribuild/projects/soaap.py b/pycheribuild/projects/soaap.py index 8ca7aca10..0cc79337c 100644 --- a/pycheribuild/projects/soaap.py +++ b/pycheribuild/projects/soaap.py @@ -31,8 +31,9 @@ from .cross.llvm import BuildLLVMSplitRepoBase from .project import ComputedDefaultValue, GitRepository -install_to_soaap_dir = ComputedDefaultValue(function=lambda config, project: config.output_root / "soaap", - as_string="$INSTALL_ROOT/soaap") +install_to_soaap_dir = ComputedDefaultValue( + function=lambda config, project: config.output_root / "soaap", as_string="$INSTALL_ROOT/soaap" +) class BuildSoaapLLVM(BuildLLVMSplitRepoBase): diff --git a/pycheribuild/projects/spike.py b/pycheribuild/projects/spike.py index 85afb9979..f4c959035 100644 --- a/pycheribuild/projects/spike.py +++ b/pycheribuild/projects/spike.py @@ -38,8 +38,9 @@ class BuildCheriSpike(AutotoolsProject): target = "spike" - repository = GitRepository("https://github.com/CTSRD-CHERI/riscv-isa-sim", - default_branch="cheri", force_branch=True) + repository = GitRepository( + "https://github.com/CTSRD-CHERI/riscv-isa-sim", default_branch="cheri", force_branch=True + ) native_install_dir = DefaultInstallDir.CHERI_SDK default_build_type = BuildType.RELEASE lto_by_default = True @@ -80,13 +81,23 @@ def process(self): kernel_config = kernel_project.default_kernel_config(ConfigPlatform.QEMU) kernel = kernel_project.get_kernel_install_path(kernel_config) # We always want output even with --quiet - self.run_cmd([BuildCheriSpike.get_simulator_binary(self), "+payload=" + str(kernel), - self._bbl_class.get_installed_kernel_path(self, cross_target=self._bbl_xtarget)], - give_tty_control=True, stdout=sys.stdout, stderr=sys.stderr) + self.run_cmd( + [ + BuildCheriSpike.get_simulator_binary(self), + "+payload=" + str(kernel), + self._bbl_class.get_installed_kernel_path(self, cross_target=self._bbl_xtarget), + ], + give_tty_control=True, + stdout=sys.stdout, + stderr=sys.stderr, + ) class RunCheriBsdSpike(RunCheriSpikeBase): target = "run-spike" _source_class = BuildCheriBsdMfsKernel - supported_architectures = (CompilationTargets.CHERIBSD_RISCV_PURECAP, CompilationTargets.CHERIBSD_RISCV_NO_CHERI, - CompilationTargets.CHERIBSD_RISCV_HYBRID) + supported_architectures = ( + CompilationTargets.CHERIBSD_RISCV_PURECAP, + CompilationTargets.CHERIBSD_RISCV_NO_CHERI, + CompilationTargets.CHERIBSD_RISCV_HYBRID, + ) diff --git a/pycheribuild/projects/syzkaller.py b/pycheribuild/projects/syzkaller.py index 4d558304f..1569e8c9b 100644 --- a/pycheribuild/projects/syzkaller.py +++ b/pycheribuild/projects/syzkaller.py @@ -48,8 +48,9 @@ class BuildSyzkaller(CrossCompileProject): dependencies = ("go",) target = "cheri-syzkaller" - repository = GitRepository("https://github.com/CTSRD-CHERI/cheri-syzkaller.git", force_branch=True, - default_branch="morello-syzkaller") + repository = GitRepository( + "https://github.com/CTSRD-CHERI/cheri-syzkaller.git", force_branch=True, default_branch="morello-syzkaller" + ) # no_default_sysroot = None // probably useless?? # skip_cheri_symlinks = True // llvm target only, useless here make_kind = MakeCommandKind.GnuMake @@ -162,7 +163,8 @@ class RunSyzkaller(SimpleProject): def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) cls.syz_config = cls.add_optional_path_option( - "syz-config", help="Path to the syzkaller configuration file to use.", show_help=True) + "syz-config", help="Path to the syzkaller configuration file to use.", show_help=True + ) cls.syz_ssh_key = cls.add_path_option( "ssh-privkey", show_help=True, @@ -186,16 +188,16 @@ def setup_config_options(cls, **kwargs): ) def syzkaller_config(self, syzkaller: BuildSyzkaller): - """ Get path of syzkaller configuration file to use. """ + """Get path of syzkaller configuration file to use.""" if self.syz_config: return self.syz_config else: xtarget = syzkaller.crosscompile_target.get_cheri_purecap_target() qemu_binary = BuildQEMU.qemu_binary(self, xtarget=xtarget) kernel_project = BuildCHERIBSD.get_instance(self, cross_target=xtarget) - kernel_config = CheriBSDConfigTable.get_configs(xtarget, platform=ConfigPlatform.QEMU, - kernel_abi=kernel_project.get_default_kernel_abi(), - fuzzing=True) + kernel_config = CheriBSDConfigTable.get_configs( + xtarget, platform=ConfigPlatform.QEMU, kernel_abi=kernel_project.get_default_kernel_abi(), fuzzing=True + ) if len(kernel_config) == 0: self.fatal("No kcov kernel configuration found") return diff --git a/pycheribuild/projects/testrig.py b/pycheribuild/projects/testrig.py index 94c210bc5..755f947b3 100644 --- a/pycheribuild/projects/testrig.py +++ b/pycheribuild/projects/testrig.py @@ -239,7 +239,8 @@ def setup_config_options(cls, **kwargs) -> None: super().setup_config_options(**kwargs) if getattr(cls, "_replay_trace_path", None) is None: cls._replay_trace_path = cls.add_optional_path_option( - "replay-trace", help="Run QCV trace from file/directory") + "replay-trace", help="Run QCV trace from file/directory" + ) @cached_property def run_implementations_with_tracing(self) -> bool: diff --git a/pycheribuild/projects/valgrind.py b/pycheribuild/projects/valgrind.py index 4a7ecd92e..27f62f699 100644 --- a/pycheribuild/projects/valgrind.py +++ b/pycheribuild/projects/valgrind.py @@ -35,8 +35,9 @@ # Install latest version of valgrind from source class BuildValgrind(AutotoolsProject): native_install_dir = DefaultInstallDir.BOOTSTRAP_TOOLS - repository = GitRepository("git://sourceware.org/git/valgrind.git", - default_branch="VALGRIND_3_14_BRANCH", force_branch=True) + repository = GitRepository( + "git://sourceware.org/git/valgrind.git", default_branch="VALGRIND_3_14_BRANCH", force_branch=True + ) def check_system_dependencies(self): super().check_system_dependencies() diff --git a/pycheribuild/qemu_utils.py b/pycheribuild/qemu_utils.py index dbb1b71dd..60e7f129d 100644 --- a/pycheribuild/qemu_utils.py +++ b/pycheribuild/qemu_utils.py @@ -96,9 +96,9 @@ def disk_image_args(self, image: Path, image_format: str) -> "list[str]": if image_format is None: image_format = "qcow2" if image.name.endswith(".qcow2") else "raw" else: - with image.open('rb') as imgf: + with image.open("rb") as imgf: magic = imgf.read(4) - is_qcow2 = magic == b'QFI\xfb' + is_qcow2 = magic == b"QFI\xfb" if image_format is None: image_format = "qcow2" if is_qcow2 else "raw" @@ -113,8 +113,12 @@ def disk_image_args(self, image: Path, image_format: str) -> "list[str]": device_kind = "virtio-blk-device" else: device_kind = "virtio-blk-pci" - return ["-drive", "if=none,file=" + str(image) + ",id=drv,format=" + image_format, - "-device", device_kind + ",drive=drv"] + return [ + "-drive", + "if=none,file=" + str(image) + ",id=drv,format=" + image_format, + "-device", + device_kind + ",drive=drv", + ] else: return ["-drive", "file=" + str(image) + ",format=" + image_format + ",index=0,media=disk"] @@ -152,11 +156,22 @@ def get_qemu_binary(self) -> "Optional[Path]": found_in_path = shutil.which("qemu-system-" + self.qemu_arch_sufffix) return Path(found_in_path) if found_in_path is not None else None - def get_commandline(self, *, qemu_command=None, kernel_file: "Optional[Path]" = None, - disk_image: "Optional[Path]" = None, disk_image_format: str = "raw", - user_network_args: str = "", add_network_device=True, bios_args: "Optional[list[str]]" = None, - trap_on_unrepresentable=False, debugger_on_cheri_trap=False, add_virtio_rng=False, - write_disk_image_changes=True, gui_options: "Optional[list[str]]" = None) -> "list[str]": + def get_commandline( + self, + *, + qemu_command=None, + kernel_file: "Optional[Path]" = None, + disk_image: "Optional[Path]" = None, + disk_image_format: str = "raw", + user_network_args: str = "", + add_network_device=True, + bios_args: "Optional[list[str]]" = None, + trap_on_unrepresentable=False, + debugger_on_cheri_trap=False, + add_virtio_rng=False, + write_disk_image_changes=True, + gui_options: "Optional[list[str]]" = None, + ) -> "list[str]": if kernel_file is None and disk_image is None: raise ValueError("Must pass kernel and/or disk image path when launching QEMU") if qemu_command is None: @@ -194,8 +209,16 @@ def get_commandline(self, *, qemu_command=None, kernel_file: "Optional[Path]" = def qemu_supports_9pfs(qemu: Path, *, config: ConfigBase) -> bool: if not qemu.is_file(): return False - prog = run_command([str(qemu), "-virtfs", "?"], stdin=subprocess.DEVNULL, capture_output=True, capture_error=True, - run_in_pretend_mode=True, expected_exit_code=1, print_verbose_only=True, config=config) + prog = run_command( + [str(qemu), "-virtfs", "?"], + stdin=subprocess.DEVNULL, + capture_output=True, + capture_error=True, + run_in_pretend_mode=True, + expected_exit_code=1, + print_verbose_only=True, + config=config, + ) return b"-virtfs ?: Usage: -virtfs" in prog.stderr diff --git a/pycheribuild/targets.py b/pycheribuild/targets.py index f9a380c90..3fb0dcdcc 100644 --- a/pycheribuild/targets.py +++ b/pycheribuild/targets.py @@ -78,12 +78,14 @@ def xtarget(self): assert result._xtarget is not None return result._xtarget - def get_real_target(self, cross_target: Optional[CrossCompileTarget], config, - caller: "Union[SimpleProject, str]" = "") -> "Target": + def get_real_target( + self, cross_target: Optional[CrossCompileTarget], config, caller: "Union[SimpleProject, str]" = "" + ) -> "Target": return self - def _get_or_create_project_no_setup(self, _: Optional[CrossCompileTarget], config, - caller: "Optional[AbstractProject]") -> "SimpleProject": + def _get_or_create_project_no_setup( + self, _: Optional[CrossCompileTarget], config, caller: "Optional[AbstractProject]" + ) -> "SimpleProject": # Note: MultiArchTarget uses cross_target to select the right project (e.g. libcxxrt-native needs # libunwind-native path) if self.__project is None: @@ -94,8 +96,9 @@ def _get_or_create_project_no_setup(self, _: Optional[CrossCompileTarget], confi # noinspection PyProtectedMember @final - def get_or_create_project(self, cross_target: Optional[CrossCompileTarget], config, - caller: "Optional[SimpleProject]") -> "SimpleProject": + def get_or_create_project( + self, cross_target: Optional[CrossCompileTarget], config, caller: "Optional[SimpleProject]" + ) -> "SimpleProject": if caller is not None: # noinspection PyProtectedMember assert caller._init_called, "Cannot call this inside __init__()" @@ -225,8 +228,13 @@ def __repr__(self) -> str: # XXX: can't call this CrossCompileTarget since that is already the name of the enum class MultiArchTarget(Target): - def __init__(self, name, project_class: "type[SimpleProject]", target_arch: "CrossCompileTarget", - base_target: "MultiArchTargetAlias"): + def __init__( + self, + name, + project_class: "type[SimpleProject]", + target_arch: "CrossCompileTarget", + base_target: "MultiArchTargetAlias", + ): super().__init__(name, project_class) assert target_arch is not None self.target_arch = target_arch @@ -263,12 +271,14 @@ def xtarget(self): def _create_project(self, config: CheriConfig) -> "SimpleProject": raise ValueError("Should not be called!") - def get_real_target(self, cross_target: Optional[CrossCompileTarget], config, - caller: "Union[SimpleProject, str]" = "") -> Target: + def get_real_target( + self, cross_target: Optional[CrossCompileTarget], config, caller: "Union[SimpleProject, str]" = "" + ) -> Target: raise NotImplementedError() - def _get_or_create_project_no_setup(self, cross_target: Optional[CrossCompileTarget], config, - caller: "Optional[AbstractProject]") -> "SimpleProject": + def _get_or_create_project_no_setup( + self, cross_target: Optional[CrossCompileTarget], config, caller: "Optional[AbstractProject]" + ) -> "SimpleProject": if caller is not None: # noinspection PyProtectedMember assert caller._init_called, "Cannot call this inside __init__()" @@ -309,8 +319,9 @@ def xtarget(self): raise ValueError("ERROR:", self.name, "does not have a default_architecture value!") return cross_target - def get_real_target(self, cross_target: "Optional[CrossCompileTarget]", config, - caller: "Union[SimpleProject, str]" = "") -> Target: + def get_real_target( + self, cross_target: "Optional[CrossCompileTarget]", config, caller: "Union[SimpleProject, str]" = "" + ) -> Target: assert self.derived_targets, "derived targets must not be empty" if cross_target is None: # Use the default target: @@ -322,7 +333,8 @@ def get_real_target(self, cross_target: "Optional[CrossCompileTarget]", config, if tgt.target_arch is cross_target: return tgt raise LookupError( - "Could not find '" + self.name + "' target for " + str(cross_target) + ", caller was " + str(caller)) + "Could not find '" + self.name + "' target for " + str(cross_target) + ", caller was " + str(caller) + ) class SimpleTargetAlias(_TargetAliasBase): @@ -330,8 +342,9 @@ class SimpleTargetAlias(_TargetAliasBase): def __init__(self, name, real_target_name: str, t: "TargetManager"): self._real_target = t.get_target_raw(real_target_name) real_cls = self._real_target.project_class - assert not isinstance(self._real_target, - _TargetAliasBase), "Target aliases must reference a real target not another alias" + assert not isinstance( + self._real_target, _TargetAliasBase + ), "Target aliases must reference a real target not another alias" super().__init__(name, real_cls) self.real_target_name = real_target_name # Add the alias name for config lookups so that old configs remain valid @@ -346,8 +359,9 @@ def __init__(self, name, real_target_name: str, t: "TargetManager"): def xtarget(self): return self._real_target.xtarget - def get_real_target(self, cross_target: Optional[CrossCompileTarget], config, - caller: "Union[SimpleProject, str]" = "") -> Target: + def get_real_target( + self, cross_target: Optional[CrossCompileTarget], config, caller: "Union[SimpleProject, str]" = "" + ) -> Target: return self._real_target def __repr__(self) -> str: @@ -355,12 +369,20 @@ def __repr__(self) -> str: class DeprecatedTargetAlias(SimpleTargetAlias): - def get_real_target(self, cross_target: Optional[CrossCompileTarget], config: "CheriConfig", - caller: "Union[SimpleProject, str]" = "") -> Target: - warning_message("Using deprecated target ", coloured(AnsiColour.red, self.name), - coloured(AnsiColour.magenta, ". Please use "), - coloured(AnsiColour.yellow, self.real_target_name), - coloured(AnsiColour.magenta, " instead."), sep="") + def get_real_target( + self, + cross_target: Optional[CrossCompileTarget], + config: "CheriConfig", + caller: "Union[SimpleProject, str]" = "", + ) -> Target: + warning_message( + "Using deprecated target ", + coloured(AnsiColour.red, self.name), + coloured(AnsiColour.magenta, ". Please use "), + coloured(AnsiColour.yellow, self.real_target_name), + coloured(AnsiColour.magenta, " instead."), + sep="", + ) if not query_yes_no(config, "Continue?", default_result=True): fatal_error("Cannot continue.", pretend=config.pretend, fatal_when_pretending=True) return self._real_target @@ -407,7 +429,7 @@ def target_disabled_reason(target: Target, config: CheriConfig) -> Optional[str] ): return ( f"{target.name} is a hybrid target, which should not be used unless you know what you're doing. " - f"If you are still sure you want to build this, use --enable-hybrid-targets." + "If you are still sure you want to build this, use --enable-hybrid-targets." ) return None @@ -442,9 +464,15 @@ def get_target_raw(self, name: str) -> Target: except KeyError: return self._targets_for_command_line_options_only[name] - def get_target(self, name: str, *, required_arch: Optional[CrossCompileTarget] = None, - arch_for_unqualified_targets: Optional[CrossCompileTarget] = None, config: CheriConfig, - caller: "Union[AbstractProject, str]") -> Target: + def get_target( + self, + name: str, + *, + required_arch: Optional[CrossCompileTarget] = None, + arch_for_unqualified_targets: Optional[CrossCompileTarget] = None, + config: CheriConfig, + caller: "Union[AbstractProject, str]", + ) -> Target: target = self.get_target_raw(name) # print("get_target", name, arch, end="") if isinstance(target, MultiArchTargetAlias): @@ -453,8 +481,9 @@ def get_target(self, name: str, *, required_arch: Optional[CrossCompileTarget] = arch_for_unqualified_targets = required_arch target = target.get_real_target(arch_for_unqualified_targets, config, caller=caller) if required_arch is not None and target.xtarget != required_arch: - raise LookupError(f"Target {target.name} has wrong architecture:" - f"{target.xtarget} but expected {required_arch}") + raise LookupError( + f"Target {target.name} has wrong architecture: {target.xtarget} but expected {required_arch}" + ) # print(" ->", target) return target @@ -494,7 +523,7 @@ def get_all_targets(self, explicit_targets: "list[Target]", config: CheriConfig) if config.start_with: sort = sort[found_index:] elif config.start_after: - sort = sort[found_index + 1:] + sort = sort[found_index + 1 :] if not sort: raise ValueError("selected target list is empty after --start-after/--start-with filtering") return sort @@ -502,9 +531,11 @@ def get_all_targets(self, explicit_targets: "list[Target]", config: CheriConfig) def run(self, config: CheriConfig, chosen_targets=None) -> None: if chosen_targets is None: chosen_targets = self.get_all_chosen_targets(config) - with set_env(PATH=config.dollar_path_with_other_tools, - CLANG_FORCE_COLOR_DIAGNOSTICS="always" if config.clang_colour_diags else None, - config=config): + with set_env( + PATH=config.dollar_path_with_other_tools, + CLANG_FORCE_COLOR_DIAGNOSTICS="always" if config.clang_colour_diags else None, + config=config, + ): for target in chosen_targets: target.check_system_deps(config) # all dependencies exist -> run the targets @@ -536,13 +567,17 @@ def get_all_chosen_targets(self, config) -> "list[Target]": suggestions = sorted([tgt.name for tgt in alias.derived_targets]) else: import difflib + errmsg = coloured(AnsiColour.red, "Target", target_name, "does not exist.") suggestions = difflib.get_close_matches(target_name, self.target_names(config)) if suggestions: errmsg += " Did you mean " + " or ".join(coloured(AnsiColour.blue, s) for s in suggestions) + "?" else: - errmsg += " See " + coloured(AnsiColour.yellow, os.path.basename(sys.argv[0]), "--list-targets") + \ - " for the list of available targets." + errmsg += ( + " See " + + coloured(AnsiColour.yellow, os.path.basename(sys.argv[0]), "--list-targets") + + " for the list of available targets." + ) sys.exit(errmsg) explicitly_chosen_targets.append(self.get_target(target_name, config=config, caller="cmdline parsing")) chosen_targets = self.get_all_targets(explicitly_chosen_targets, config) diff --git a/pycheribuild/utils.py b/pycheribuild/utils.py index d4c716876..91a871fdb 100644 --- a/pycheribuild/utils.py +++ b/pycheribuild/utils.py @@ -77,7 +77,8 @@ "replace_one", "status_update", "typing", - "warning_message"] + "warning_message", +] if sys.version_info < (3, 6, 0): sys.exit("This script requires at least Python 3.6.0") @@ -87,6 +88,7 @@ try: from typing import final except ImportError: + def final(f: Type_T) -> Type_T: return f @@ -104,6 +106,7 @@ def __get__(self, obj, owner) -> Type_T: if typing.TYPE_CHECKING: DoNotUseInIfStmt = bool else: + class DoNotUseInIfStmt: def __bool__(self) -> "typing.NoReturn": raise ValueError("Should not be used") @@ -125,8 +128,9 @@ def __init__(self, *, pretend: bool, verbose: bool, quiet: bool, force: bool) -> self.internet_connection_last_check_result = False -GlobalConfig: ConfigBase = ConfigBase(pretend=DoNotUseInIfStmt(), verbose=DoNotUseInIfStmt(), quiet=DoNotUseInIfStmt(), - force=DoNotUseInIfStmt()) +GlobalConfig: ConfigBase = ConfigBase( + pretend=DoNotUseInIfStmt(), verbose=DoNotUseInIfStmt(), quiet=DoNotUseInIfStmt(), force=DoNotUseInIfStmt() +) def init_global_config(config: ConfigBase, *, test_mode: bool = False) -> None: @@ -160,8 +164,9 @@ def __set_name__(self, _, name) -> None: # XXX: requires python 3.6 if self.attrname is None: self.attrname = name elif name != self.attrname: - raise TypeError("Cannot assign the same cached_property to two different names " - f"({self.attrname} and {name}).") + raise TypeError( + f"Cannot assign the same cached_property to two different names ({self.attrname} and {name})." + ) def __get__(self, instance, owner=None) -> Type_T: if instance is None: @@ -171,8 +176,9 @@ def __get__(self, instance, owner=None) -> Type_T: try: cache = instance.__dict__ except AttributeError: # not all objects have __dict__ (e.g. class defines slots) - msg = ("No '__dict__' attribute on {} instance to cache {} property.".format(type(instance).__name__, - self.attrname)) + msg = "No '__dict__' attribute on {} instance to cache {} property.".format( + type(instance).__name__, self.attrname + ) raise TypeError(msg) from None val = cache.get(self.attrname, _NOT_FOUND) if val is _NOT_FOUND: @@ -184,8 +190,10 @@ def __get__(self, instance, owner=None) -> Type_T: try: cache[self.attrname] = val except TypeError: - msg = ("The '__dict__' attribute on {} instance does not support item assignment for" - " caching {} property.".format(type(instance).__name__, self.attrname)) + msg = ( + "The '__dict__' attribute on {} instance does not support item assignment for" + " caching {} property.".format(type(instance).__name__, self.attrname) + ) raise TypeError(msg) from None return val @@ -208,10 +216,11 @@ def find_free_port(preferred_port: "Optional[int]" = None) -> SocketAndPort: return SocketAndPort(s, s.getsockname()[1]) except OSError as e: import errno + if e.errno != errno.EADDRINUSE: warning_message("Got unexpected error when checking whether port", preferred_port, "is free:", e) status_update("Port", preferred_port, "is not available, falling back to using a random port") - s.bind(('localhost', 0)) + s.bind(("localhost", 0)) return SocketAndPort(s, s.getsockname()[1]) @@ -227,7 +236,7 @@ def default_make_jobs_count() -> Optional[int]: def maybe_add_space(msg: str, sep: str) -> "tuple[str, ...]": if sep == "": return msg, " " - return (msg, ) + return (msg,) def status_update(*args, sep=" ", **kwargs) -> None: @@ -235,8 +244,11 @@ def status_update(*args, sep=" ", **kwargs) -> None: def fixit_message(*args, sep=" ") -> None: - print(coloured(AnsiColour.blue, maybe_add_space("Possible solution:", sep) + args, sep=sep), file=sys.stderr, - flush=True) + print( + coloured(AnsiColour.blue, maybe_add_space("Possible solution:", sep) + args, sep=sep), + file=sys.stderr, + flush=True, + ) def warning_message(*args, sep=" ", fixit_hint=None) -> None: @@ -265,8 +277,11 @@ def add_error_context(context: str): def _add_error_context(prefix, args, sep) -> "str": if _ERROR_CONTEXT: # _ERROR_CONTEXT might contain escape sequences so we have to reset to red afterwards - return coloured(AnsiColour.red, maybe_add_space(prefix + " " + _ERROR_CONTEXT[-1] + - AnsiColour.red.escape_sequence() + ":", sep) + args, sep=sep) + return coloured( + AnsiColour.red, + maybe_add_space(prefix + " " + _ERROR_CONTEXT[-1] + AnsiColour.red.escape_sequence() + ":", sep) + args, + sep=sep, + ) return coloured(AnsiColour.red, maybe_add_space(prefix + ":", sep) + args, sep=sep) @@ -293,8 +308,14 @@ def fatal_error(*args, sep=" ", fixit_hint=None, fatal_when_pretending=False, ex sys.exit(exit_code) -def query_yes_no(config: ConfigBase, message: str = "", *, default_result=False, force_result=True, - yes_no_str: "Optional[str]" = None) -> bool: +def query_yes_no( + config: ConfigBase, + message: str = "", + *, + default_result=False, + force_result=True, + yes_no_str: "Optional[str]" = None, +) -> bool: if yes_no_str is None: yes_no_str = " [Y]/n " if default_result else " y/[N] " if config.pretend: @@ -378,8 +399,12 @@ def is_case_sensitive_dir(d: Path) -> bool: class InstallInstructions: - def __init__(self, message: "Union[str, Callable[[], str]]", - cheribuild_target: "Optional[str]" = None, alternative: "Optional[str]" = None): + def __init__( + self, + message: "Union[str, Callable[[], str]]", + cheribuild_target: "Optional[str]" = None, + alternative: "Optional[str]" = None, + ): self._message = message self.cheribuild_target = cheribuild_target self.alternative = alternative @@ -443,7 +468,7 @@ def __parse_etc_os_release() -> "dict[str, str]": d = {} for line in f: line = line.strip() - if line == '' or line[0] == '#': + if line == "" or line[0] == "#": continue k, v = line.split("=", maxsplit=1) # .strip('"') will remove if there or else do nothing @@ -466,8 +491,19 @@ def package_manager(cls, compat_abi=False) -> str: return "" @classmethod - def install_instructions(cls, name, is_lib, default=None, homebrew=None, apt=None, zypper=None, freebsd=None, - cheribuild_target=None, alternative=None, compat_abi=False) -> InstallInstructions: + def install_instructions( + cls, + name, + is_lib, + default=None, + homebrew=None, + apt=None, + zypper=None, + freebsd=None, + cheribuild_target=None, + alternative=None, + compat_abi=False, + ) -> InstallInstructions: guessed_package = False if cls.IS_MAC and homebrew: install_name = homebrew @@ -493,8 +529,8 @@ def command_not_found(): if msg_start: hint = hint[msg_start:] return hint - return "Could not find package for program " + name + ". " \ - "Maybe `zypper in " + name + "` will work." + return f"Could not find package for program {name}. Maybe `zypper in {name}` will work." + return InstallInstructions(command_not_found, cheribuild_target, alternative) guessed_package = True install_name = "lib" + name + "-devel" if is_lib else name @@ -507,12 +543,16 @@ def command_not_found(): if guessed_package: # not sure if the package name is correct: - return InstallInstructions(f"Possibly running `{cls.package_manager(compat_abi)} install {install_name}" - f"` fixes this. Note: package name may not be correct.", cheribuild_target, - alternative) + return InstallInstructions( + f"Possibly running `{cls.package_manager(compat_abi)} install {install_name}" + "` fixes this. Note: package name may not be correct.", + cheribuild_target, + alternative, + ) else: - return InstallInstructions(f"Run `{cls.package_manager(compat_abi)} install " + install_name + "`", - cheribuild_target, alternative) + return InstallInstructions( + f"Run `{cls.package_manager(compat_abi)} install " + install_name + "`", cheribuild_target, alternative + ) @classmethod def uses_apt(cls) -> bool: @@ -561,7 +601,7 @@ def remove_prefix(s: str, prefix: str, prefix_required=False) -> str: if prefix_required: raise ValueError(s + " does not start with " + prefix) return s - return s[len(prefix):] + return s[len(prefix) :] # A dictionary for string formatting (format_map) that preserves values not diff --git a/pyproject.toml b/pyproject.toml index cf98522cf..4a8d1485f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ ignore = [ "UP036", # 'Version block is outdated for minimum Python version' to be removed when we set the minimum to 3.8 "PLW2901", # `for` loop variable `value` overwritten by assignment target "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` (cannot be fixed due to bug) + "COM812", # Incompatible with `ruff format` ] [tool.ruff.lint.isort] diff --git a/tests/test_argument_parsing.py b/tests/test_argument_parsing.py index b7f6a01a9..6da3e8fcd 100644 --- a/tests/test_argument_parsing.py +++ b/tests/test_argument_parsing.py @@ -937,8 +937,12 @@ def test_kernel_configs(target, config_options: "list[str]", expected_kernels: " pytest.param( "cheribsd-mfs-root-kernel-riscv64-purecap", ["--cheribsd/build-nocaprevoke-kernel"], - ["CHERI-QEMU-MFS-ROOT", "CHERI-NOCAPREVOKE-QEMU-MFS-ROOT", "CHERI-PURECAP-QEMU-MFS-ROOT", - "CHERI-PURECAP-NOCAPREVOKE-QEMU-MFS-ROOT"], + [ + "CHERI-QEMU-MFS-ROOT", + "CHERI-NOCAPREVOKE-QEMU-MFS-ROOT", + "CHERI-PURECAP-QEMU-MFS-ROOT", + "CHERI-PURECAP-NOCAPREVOKE-QEMU-MFS-ROOT", + ], ), ], ) @@ -1404,7 +1408,9 @@ def test_jenkins_hack_disk_image(): config = _parse_arguments(args) jenkins_override_install_dirs_hack(config, Path("/rootfs")) disk_image = _get_target_instance( - "disk-image-aarch64", config, BuildCheriBSDDiskImage, + "disk-image-aarch64", + config, + BuildCheriBSDDiskImage, ) assert disk_image.disk_image_path == Path("/tmp/tarball/cheribsd-aarch64.img") assert disk_image.rootfs_dir == Path("/tmp/tarball/rootfs") diff --git a/tests/test_metalog.py b/tests/test_metalog.py index 45609c265..19a121d9c 100644 --- a/tests/test_metalog.py +++ b/tests/test_metalog.py @@ -207,13 +207,16 @@ def test_contents_root(): # END """ mtree = MtreeFile(file=io.StringIO(file), contents_root=Path("/path/to/rootfs"), verbose=False) - assert _get_as_str(mtree) == """#mtree 2.0 + assert ( + _get_as_str(mtree) + == """#mtree 2.0 . type=dir uname=root gname=wheel mode=0755 ./bin type=dir uname=root gname=wheel mode=0755 ./bin/cat type=file uname=root gname=wheel mode=0755 contents=/path/to/rootfs/bin/cheribsdbox ./bin/cheribsdbox type=file uname=root gname=wheel mode=0755 contents=/path/to/rootfs/bin/cheribsdbox # END """ + ) def test_add_file():