From c89a4c17917282d0fb6c7554b99ca3c314284353 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Thu, 10 Aug 2023 08:32:07 -0700 Subject: [PATCH] Re-land "Jenkins: change installation directories before calling setup()"" The underlying issue with not calling setup() should hopefully be fixed. This reverts commit 2edc5e4592497d5cf19fb450c3eee0f6de3876ff. This reverts commit f88311cc8b7a07b6eda59a510afa7f062f0da1ad. --- pycheribuild/jenkins.py | 69 +++++++++++++--------- pycheribuild/projects/bluespec_compiler.py | 3 + pycheribuild/projects/cross/cheribsd.py | 8 ++- pycheribuild/projects/cross/compiler_rt.py | 1 - pycheribuild/projects/project.py | 12 +++- tests/test_async_delete.py | 26 +++++--- 6 files changed, 76 insertions(+), 43 deletions(-) diff --git a/pycheribuild/jenkins.py b/pycheribuild/jenkins.py index 44b68b678..c9aa6af4a 100644 --- a/pycheribuild/jenkins.py +++ b/pycheribuild/jenkins.py @@ -49,7 +49,6 @@ # noinspection PyUnresolvedReferences from .projects.cross import * # noqa: F401, F403, RUF100 -from .projects.cross.crosscompileproject import CrossCompileMixin from .projects.project import Project from .projects.simple_project import SimpleProject from .targets import MultiArchTargetAlias, SimpleTargetAlias, Target, target_manager @@ -244,46 +243,58 @@ def _jenkins_main() -> None: fatal_error("More than one target is not supported yet.", pretend=False) sys.exit() - if JenkinsAction.BUILD in cheri_config.action or JenkinsAction.TEST in cheri_config.action: - # Ugly workaround to override all install dirs to go to the tarball - for tgt in target_manager.targets(cheri_config): - if isinstance(tgt, SimpleTargetAlias): - continue - cls = tgt.project_class - if issubclass(cls, Project) and not isinstance(tgt, MultiArchTargetAlias): - cls._default_install_dir_fn = Path( - str(cheri_config.output_root) + str(cheri_config.installation_prefix)) - i = inspect.getattr_static(cls, "_install_dir") - assert isinstance(i, CommandLineConfigOption) - # But don't change it if it was specified on the command line. Note: This also does the config - # inheritance: i.e. setting --cheribsd/install-dir will also affect cheribsd-cheri/cheribsd-mips - # noinspection PyTypeChecker - from_cmdline = i.load_option(cheri_config, cls, cls, return_none_if_default=True) - if from_cmdline is not None: - status_update("Install directory for", cls.target, "was specified on commandline:", from_cmdline) - else: - cls._install_dir = Path(str(cheri_config.output_root) + str(cheri_config.installation_prefix)) - cls._check_install_dir_conflict = False + expected_install_path = Path(f"{cheri_config.output_root}{cheri_config.installation_prefix}") + # Ugly workaround to override all install dirs to go to the tarball + for tgt in target_manager.targets(cheri_config): + if isinstance(tgt, SimpleTargetAlias): + continue + cls = tgt.project_class + if issubclass(cls, Project) and not isinstance(tgt, MultiArchTargetAlias): + cls._default_install_dir_fn = Path(expected_install_path) + i = inspect.getattr_static(cls, "_install_dir") + assert isinstance(i, CommandLineConfigOption) + # But don't change it if it was specified on the command line. Note: This also does the config + # inheritance: i.e. setting --cheribsd/install-dir will also affect cheribsd-cheri/cheribsd-mips + # noinspection PyTypeChecker + from_cmdline = i.load_option(cheri_config, cls, cls, return_none_if_default=True) + if from_cmdline is not None: + status_update("Install directory for", cls.target, "was specified on commandline:", from_cmdline) + else: + cls._install_dir = cheri_config.output_root + cls._check_install_dir_conflict = False Target.instantiating_targets_should_warn = False - for target in cheri_config.targets: - build_target(cheri_config, target_manager.get_target_raw(target)) + # Override the installation directory for all enabled targets + for name in cheri_config.targets: + target = target_manager.get_target_raw(name) + # noinspection PyProtectedMember + project = target._get_or_create_project_no_setup(None, cheri_config, caller=None) + if isinstance(project, Project): + # Using "/" as the install prefix results inconsistently prefixing some paths with '/usr/'. + # To avoid this, just use the full install path as the prefix. + if cheri_config.installation_prefix == Path("/"): + project._install_prefix = expected_install_path + project.destdir = Path("/") + else: + project._install_prefix = cheri_config.installation_prefix + project.destdir = cheri_config.output_root + assert project.real_install_root_dir == expected_install_path + + if JenkinsAction.BUILD in cheri_config.action or JenkinsAction.TEST in cheri_config.action: + for tgt in cheri_config.targets: + build_target(cheri_config, target_manager.get_target_raw(tgt)) if JenkinsAction.CREATE_TARBALL in cheri_config.action: create_tarball(cheri_config) def build_target(cheri_config, target: Target) -> None: - # Note: This if exists for now to avoid a large diff. target.check_system_deps(cheri_config) - # need to set destdir after check_system_deps: project = target.get_or_create_project(None, cheri_config, caller=None) assert project _ = project.all_dependency_names(cheri_config) # Ensure dependencies are cached. - if isinstance(project, CrossCompileMixin): - project.destdir = cheri_config.output_root - project._install_prefix = cheri_config.installation_prefix - project._install_dir = cheri_config.output_root + if isinstance(project, Project): + assert project.real_install_root_dir == Path(f"{cheri_config.output_root}{cheri_config.installation_prefix}") if cheri_config.debug_output: status_update("Configuration options for building", project.target, file=sys.stderr) diff --git a/pycheribuild/projects/bluespec_compiler.py b/pycheribuild/projects/bluespec_compiler.py index e29e500c8..6e4794003 100644 --- a/pycheribuild/projects/bluespec_compiler.py +++ b/pycheribuild/projects/bluespec_compiler.py @@ -46,6 +46,9 @@ def check_system_dependencies(self): self.check_required_system_tool("gperf", homebrew="gperf", apt="gperf") for i in ("autoconf", "bison", "flex"): self.check_required_system_tool(i, homebrew=i) + + def setup(self): + super().setup() self.make_args.set(PREFIX=self.install_dir) def compile(self, **kwargs): diff --git a/pycheribuild/projects/cross/cheribsd.py b/pycheribuild/projects/cross/cheribsd.py index d86517fd6..1085e1ada 100644 --- a/pycheribuild/projects/cross/cheribsd.py +++ b/pycheribuild/projects/cross/cheribsd.py @@ -767,8 +767,6 @@ def _try_find_compatible_system_clang(self) -> "tuple[Optional[Path], Optional[s def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._setup_make_args_called = False - self.destdir = self.install_dir - self._install_prefix = Path("/") self.kernel_toolchain_exists: bool = False self.cross_toolchain_config = MakeOptions(MakeCommandKind.BsdMake, self) if self.has_default_buildkernel_kernel_config: @@ -776,6 +774,12 @@ def __init__(self, *args, **kwargs) -> None: self.make_args.set(**self.arch_build_flags) self.extra_kernels = [] + def setup(self) -> None: + super().setup() + self.destdir = self.install_dir + self._install_prefix = Path("/") + assert self.real_install_root_dir == self.destdir + @cached_property def build_toolchain_root_dir(self) -> "Optional[Path]": if self.build_toolchain == FreeBSDToolchainKind.BOOTSTRAPPED: diff --git a/pycheribuild/projects/cross/compiler_rt.py b/pycheribuild/projects/cross/compiler_rt.py index e02cce558..4efac8370 100644 --- a/pycheribuild/projects/cross/compiler_rt.py +++ b/pycheribuild/projects/cross/compiler_rt.py @@ -177,7 +177,6 @@ def setup(self): def install(self, **kwargs): super().install(**kwargs) - libname = "libclang_rt.builtins-" + self.triple_arch + ".a" if self.target_info.is_rtems(): self.move_file(self.install_dir / "lib/rtems5" / libname, self.install_dir / "lib" / libname) diff --git a/pycheribuild/projects/project.py b/pycheribuild/projects/project.py index 19dd69910..d8e732ff8 100644 --- a/pycheribuild/projects/project.py +++ b/pycheribuild/projects/project.py @@ -821,7 +821,8 @@ def __init__(self, *args, **kwargs) -> None: if self.build_in_source_dir: assert not self.build_via_symlink_farm, "Using a symlink farm only makes sense with a separate build dir" - self.verbose_print("Cannot build", self.target, "in a separate build dir, will build in", self.source_dir) + if self.config.debug_output: + self.info("Cannot build", self.target, "in a separate build dir, will build in", self.source_dir) self.build_dir = self.source_dir self.configure_command = None @@ -851,7 +852,8 @@ def __init__(self, *args, **kwargs) -> None: elif install_dir_kind in (DefaultInstallDir.ROOTFS_OPTBASE, DefaultInstallDir.KDE_PREFIX): relative_to_rootfs = os.path.relpath(str(self._install_dir), str(self.rootfs_dir)) if relative_to_rootfs.startswith(os.path.pardir): - self.verbose_print("Custom install dir", self._install_dir, "-> using / as install prefix") + self.verbose_print("Custom install dir", self._install_dir, + "-> using / as install prefix for", self.target) self._install_prefix = Path("/") self.destdir = self._install_dir else: @@ -944,6 +946,10 @@ def host_dependency_prefixes(self) -> "list[Path]": def setup(self): super().setup() + self.verbose_print( + self.target, f"INSTALLDIR={self._install_dir}", f"INSTALL_PREFIX={self._install_prefix}", + f"DESTDIR={self.destdir}", + ) if self.set_pkg_config_path: pkg_config_args = dict() if self.compiling_for_host(): @@ -1263,10 +1269,12 @@ def real_install_root_dir(self) -> Path: @property def install_dir(self) -> Path: + assert self._setup_called, "Should be called after base class setup()" return self.real_install_root_dir @property def install_prefix(self) -> Path: + assert self._setup_called, "Should be called after base class setup()" if self._install_prefix is not None: return self._install_prefix return self._install_dir diff --git a/tests/test_async_delete.py b/tests/test_async_delete.py index 7c1538db3..79fea0192 100644 --- a/tests/test_async_delete.py +++ b/tests/test_async_delete.py @@ -23,18 +23,25 @@ class MockProject(Project): def __init__(self, config: MockConfig, name: str): self.target = name self.default_directory_basename = name - expected_src = config.source_root / "sources" / name - self._initial_source_dir = expected_src - expected_install = config.source_root / "install" / name - self._install_dir = expected_install - expected_build = Path(config.source_root, "build", name + "-build") - self.build_dir = expected_build + self.expected_src = config.source_root / "sources" / name + self._initial_source_dir = self.expected_src + self.expected_install = config.source_root / "install" / name + self._install_dir = self.expected_install + self.expected_build = Path(config.source_root, "build", name + "-build") + self.build_dir = self.expected_build super().__init__(config, crosscompile_target=CompilationTargets.NATIVE) - assert self.source_dir == expected_src - assert self.build_dir == expected_build - assert self.install_dir == expected_install + + def setup(self): + super().setup() + assert self.source_dir == self.expected_src + assert self.build_dir == self.expected_build + assert self.install_dir == self.expected_install self.source_dir.mkdir(parents=True) + @classmethod + def cached_full_dependencies(cls): + return [] + def _delete_directories(self, *dirs): if self.config.sleep_before_delete: print("SLEEPING") @@ -57,6 +64,7 @@ def setUp(self): assert self.tempRoot == self.config.source_root assert self.tempRoot / "build" == self.config.build_root self.project = MockProject(self.config, "foo") + self.project.setup() assert self.project.source_dir.exists(), self.project.source_dir def tearDown(self):