Always decode manifests with utf-8; not just most of the time

src/west/app/project.py

@@ -321,7 +321,7 @@ def bootstrap(self, args) -> Path:
  # Parse the manifest to get "self: path:", if it declares one.
  # Otherwise, use the URL. Ignore imports -- all we really
  # want to know is if there's a "self: path:" or not.
- manifest = Manifest.from_data(temp_manifest.read_text(),
+ manifest = Manifest.from_data(temp_manifest.read_text(encoding=Manifest.encoding),
  import_flags=ImportFlag.IGNORE)
  if manifest.yaml_path:
  manifest_path = manifest.yaml_path
@@ -1049,7 +1049,7 @@ def update_importer(self, project, path):
  self.updated.add(project.name)
 
  try:
- return _manifest_content_at(project, path)
+ return _manifest_content_at(project, path, Manifest.encoding)
  except FileNotFoundError:
  # FIXME we need each project to have back-pointers
  # to the manifest file where it was defined, so we can

src/west/manifest.py

@@ -243,7 +243,7 @@ def _west_commands_merge(wc1: List[str], wc2: List[str]) -> List[str]:
 def _default_importer(project: 'Project', file: str) -> NoReturn:
  raise ManifestImportFailed(project, file)
 
-def _manifest_content_at(project: 'Project', path: PathType,
+def _manifest_content_at(project: 'Project', path: PathType, mf_encoding: str,
  rev: str = QUAL_MANIFEST_REV_BRANCH) \
  -> ImportedContentType:
  # Get a list of manifest data from project at path
@@ -272,19 +272,22 @@ def _manifest_content_at(project: 'Project', path: PathType,
  # enough!
  raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), path)
 
- ptype = out.decode('utf-8').split()[1]
+ # TODO: does git always output utf-8? Test git config 'core.precomposeunicode' etc.
+ git_filenames_encoding = 'utf-8'
+ ptype = out.decode(git_filenames_encoding).split()[1]
 
  if ptype == 'blob':
  # Importing a file: just return its content.
- return project.read_at(path, rev=rev).decode('utf-8')
+ return project.read_at(path, rev=rev).decode(mf_encoding)
  elif ptype == 'tree':
  # Importing a tree: return the content of the YAML files inside it.
  ret = []
  # Use a PurePosixPath because that's the form git seems to
  # store internally, even on Windows.
  pathobj = PurePosixPath(path)
- for f in filter(_is_yml, project.listdir_at(path, rev=rev)):
- ret.append(project.read_at(pathobj / f, rev=rev).decode('utf-8'))
+ for f in filter(_is_yml, project.listdir_at(path, rev=rev,
+ encoding=git_filenames_encoding)):
+ ret.append(project.read_at(pathobj / f, rev=rev).decode(mf_encoding))
  return ret
  else:
  raise MalformedManifest(f"can't decipher project {project.name} "
@@ -1184,6 +1187,9 @@ class Manifest:
  '''The parsed contents of a west manifest file.
  '''
 
+ # TODO: make this a new west config: 'manifest.encoding'
+ encoding: str = 'utf-8'
+
  @staticmethod
  def from_topdir(topdir: Optional[PathType] = None,
  config: Optional[Configuration] = None,
@@ -1902,7 +1908,7 @@ def get_option(option, default=None):
  current_relpath = manifest_path / manifest_file
  current_abspath = topdir_abspath / current_relpath
  try:
- current_data = current_abspath.read_text(encoding='utf-8')
+ current_data = current_abspath.read_text(encoding=Manifest.encoding)
  except FileNotFoundError:
  raise MalformedConfig(
  f'file not found: manifest file {current_abspath} '
@@ -2201,7 +2207,7 @@ def _import_pathobj_from_self(self, pathobj_abs: Path,
  child_ctx = self._ctx._replace(
  current_abspath=pathobj_abs,
  current_relpath=pathobj,
- current_data=pathobj_abs.read_text(encoding='utf-8')
+ current_data=pathobj_abs.read_text(encoding=Manifest.encoding)
  )
  try:
  Manifest(topdir=self.topdir, internal_import_ctx=child_ctx)
@@ -2239,7 +2245,7 @@ def _import_map_from_self(self, imp: Dict) -> None:
  path_prefix=path_prefix,
  current_abspath=import_abs,
  current_relpath=pathobj / import_abs.name,
- current_data=import_abs.read_text(encoding='utf-8')
+ current_data=import_abs.read_text(encoding=Manifest.encoding)
  )
  try:
  Manifest(topdir=self.topdir, internal_import_ctx=child_ctx)
@@ -2572,7 +2578,7 @@ def _import_content_from_project(self, project: Project,
  if not (self._ctx.import_flags & ImportFlag.FORCE_PROJECTS) and \
  project.is_cloned():
  try:
- content = _manifest_content_at(project, path)
+ content = _manifest_content_at(project, path, Manifest.encoding)
  except MalformedManifest as mm:
  self._malformed(mm.args[0])
  except FileNotFoundError: