diff --git a/src/west/app/project.py b/src/west/app/project.py index a046b65c..0ffe1e5a 100644 --- a/src/west/app/project.py +++ b/src/west/app/project.py @@ -321,7 +321,7 @@ def bootstrap(self, args) -> Path: # Parse the manifest to get "self: path:", if it declares one. # Otherwise, use the URL. Ignore imports -- all we really # want to know is if there's a "self: path:" or not. - manifest = Manifest.from_data(temp_manifest.read_text(), + manifest = Manifest.from_data(temp_manifest.read_text(encoding=Manifest.encoding), import_flags=ImportFlag.IGNORE) if manifest.yaml_path: manifest_path = manifest.yaml_path @@ -1049,7 +1049,7 @@ def update_importer(self, project, path): self.updated.add(project.name) try: - return _manifest_content_at(project, path) + return _manifest_content_at(project, path, Manifest.encoding) except FileNotFoundError: # FIXME we need each project to have back-pointers # to the manifest file where it was defined, so we can diff --git a/src/west/manifest.py b/src/west/manifest.py index e28faded..4cabef51 100644 --- a/src/west/manifest.py +++ b/src/west/manifest.py @@ -243,7 +243,7 @@ def _west_commands_merge(wc1: List[str], wc2: List[str]) -> List[str]: def _default_importer(project: 'Project', file: str) -> NoReturn: raise ManifestImportFailed(project, file) -def _manifest_content_at(project: 'Project', path: PathType, +def _manifest_content_at(project: 'Project', path: PathType, mf_encoding: str, rev: str = QUAL_MANIFEST_REV_BRANCH) \ -> ImportedContentType: # Get a list of manifest data from project at path @@ -272,19 +272,22 @@ def _manifest_content_at(project: 'Project', path: PathType, # enough! raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), path) - ptype = out.decode('utf-8').split()[1] + # TODO: does git always output utf-8? Test git config 'core.precomposeunicode' etc. + git_filenames_encoding = 'utf-8' + ptype = out.decode(git_filenames_encoding).split()[1] if ptype == 'blob': # Importing a file: just return its content. - return project.read_at(path, rev=rev).decode('utf-8') + return project.read_at(path, rev=rev).decode(mf_encoding) elif ptype == 'tree': # Importing a tree: return the content of the YAML files inside it. ret = [] # Use a PurePosixPath because that's the form git seems to # store internally, even on Windows. pathobj = PurePosixPath(path) - for f in filter(_is_yml, project.listdir_at(path, rev=rev)): - ret.append(project.read_at(pathobj / f, rev=rev).decode('utf-8')) + for f in filter(_is_yml, project.listdir_at(path, rev=rev, + encoding=git_filenames_encoding)): + ret.append(project.read_at(pathobj / f, rev=rev).decode(mf_encoding)) return ret else: raise MalformedManifest(f"can't decipher project {project.name} " @@ -1184,6 +1187,9 @@ class Manifest: '''The parsed contents of a west manifest file. ''' + # TODO: make this a new west config: 'manifest.encoding' + encoding: str = 'utf-8' + @staticmethod def from_topdir(topdir: Optional[PathType] = None, config: Optional[Configuration] = None, @@ -1902,7 +1908,7 @@ def get_option(option, default=None): current_relpath = manifest_path / manifest_file current_abspath = topdir_abspath / current_relpath try: - current_data = current_abspath.read_text(encoding='utf-8') + current_data = current_abspath.read_text(encoding=Manifest.encoding) except FileNotFoundError: raise MalformedConfig( f'file not found: manifest file {current_abspath} ' @@ -2201,7 +2207,7 @@ def _import_pathobj_from_self(self, pathobj_abs: Path, child_ctx = self._ctx._replace( current_abspath=pathobj_abs, current_relpath=pathobj, - current_data=pathobj_abs.read_text(encoding='utf-8') + current_data=pathobj_abs.read_text(encoding=Manifest.encoding) ) try: Manifest(topdir=self.topdir, internal_import_ctx=child_ctx) @@ -2239,7 +2245,7 @@ def _import_map_from_self(self, imp: Dict) -> None: path_prefix=path_prefix, current_abspath=import_abs, current_relpath=pathobj / import_abs.name, - current_data=import_abs.read_text(encoding='utf-8') + current_data=import_abs.read_text(encoding=Manifest.encoding) ) try: Manifest(topdir=self.topdir, internal_import_ctx=child_ctx) @@ -2572,7 +2578,7 @@ def _import_content_from_project(self, project: Project, if not (self._ctx.import_flags & ImportFlag.FORCE_PROJECTS) and \ project.is_cloned(): try: - content = _manifest_content_at(project, path) + content = _manifest_content_at(project, path, Manifest.encoding) except MalformedManifest as mm: self._malformed(mm.args[0]) except FileNotFoundError: