project.py: extend cache support for submodules

If cache is used try to also use it for submodules.

Signed-off-by: Kari Hamalainen <kari.hamalainen@nordicsemi.no>

src/west/app/project.py

@@ -26,6 +26,7 @@
  ManifestProject, _manifest_content_at, ManifestImportFailed
 from west.manifest import is_group as is_project_group
 from west.manifest import MANIFEST_REV_BRANCH as MANIFEST_REV
+from west.manifest import Submodule
 from west.manifest import QUAL_MANIFEST_REV_BRANCH as QUAL_MANIFEST_REV
 from west.manifest import QUAL_REFS_WEST as QUAL_REFS
 
@@ -1160,23 +1161,43 @@ def update_submodules(self, project):
  for config_opt in self.args.submodule_init_config:
  config_opts.extend(['-c', config_opt])
 
- # For the list type, update given list of submodules.
- if isinstance(submodules, list):
- for submodule in submodules:
+ cache_dir = self.project_cache(project)
+ # For the boolean type, update all the submodules.
+ if isinstance(submodules, bool):
+ if cache_dir is None:
  if self.sync_submodules:
- project.git(['submodule', 'sync', '--recursive',
- '--', submodule.path])
+ project.git(['submodule', 'sync', '--recursive'])
  project.git(config_opts +
- ['submodule', 'update',
- '--init', submodules_update_strategy,
- '--recursive', submodule.path])
- # For the bool type, update all project submodules
- elif isinstance(submodules, bool):
+ ['submodule', 'update', '--init',
+ submodules_update_strategy, '--recursive'])
+ return
+ else:
+ # Cache used so convert to a list so that --reference can be used.
+ res = project.git(['submodule', 'status'], capture_stdout=True)
+ if not res.stdout or res.returncode:
+ self.die(
+ f"Submodule status failed for project: {project.name}.")
+ mod_list = [s.strip() for s in res.stdout.decode('utf-8').split('\n') if s]
+ submodules = [Submodule(line.split(' ')[1]) for line in mod_list]
+
+ # For the list type, update given list of submodules.
+ for submodule in submodules:
  if self.sync_submodules:
- project.git(['submodule', 'sync', '--recursive'])
+ project.git(['submodule', 'sync', '--recursive',
+ '--', submodule.path])
+ ref = []
+ if (cache_dir):
+ submodule_ref = Path(cache_dir, submodule.path)
+ if any(os.scandir(submodule_ref)):
+ ref = ['--reference', os.fspath(submodule_ref)]
+ self.small_banner(f'using reference from: {submodule_ref}')
+ self.dbg(
+ f'found {submodule.path} in --path-cache {submodule_ref}',
+ level=Verbosity.DBG_MORE)
  project.git(config_opts +
- ['submodule', 'update', '--init',
- submodules_update_strategy, '--recursive'])
+ ['submodule', 'update',
+ '--init', submodules_update_strategy,
+ '--recursive'] + ref + [submodule.path])
 
  def update(self, project):
  if self.args.stats:

tests/test_project.py

@@ -1262,7 +1262,7 @@ def test_update_recovery(tmpdir):
  assert prev == rgood
 
 
-def setup_cache_workspace(workspace, foo_head, bar_head):
+def setup_cache_workspace(workspace, remote, foo_head, bar_head):
  # Shared helper code that sets up a workspace used to test the
  # 'west update --foo-cache' options.
 
@@ -1275,10 +1275,10 @@ def setup_cache_workspace(workspace, foo_head, bar_head):
  projects:
  - name: foo
  path: subdir/foo
- url: should-not-be-fetched
+ url: file://{remote}
  revision: {foo_head}
  - name: bar
- url: should-not-be-fetched
+ url: file://{remote}
  revision: {bar_head}
  ''')
 
@@ -1287,14 +1287,16 @@ def test_update_name_cache(tmpdir):
  # Test that 'west update --name-cache' works and doesn't hit the
  # network if it doesn't have to.
 
+ remote = tmpdir / 'remote'
+ create_repo(remote)
  name_cache_dir = tmpdir / 'name_cache'
  create_repo(name_cache_dir / 'foo')
  create_repo(name_cache_dir / 'bar')
  foo_head = rev_parse(name_cache_dir / 'foo', 'HEAD')
  bar_head = rev_parse(name_cache_dir / 'bar', 'HEAD')
 
  workspace = tmpdir / 'workspace'
- setup_cache_workspace(workspace, foo_head, bar_head)
+ setup_cache_workspace(workspace, remote, foo_head, bar_head)
  workspace.chdir()
  foo = workspace / 'subdir' / 'foo'
  bar = workspace / 'bar'
@@ -1322,14 +1324,16 @@ def test_update_path_cache(tmpdir):
  # Test that 'west update --path-cache' works and doesn't hit the
  # network if it doesn't have to.
 
+ remote = tmpdir / 'remote'
+ create_repo(remote)
  path_cache_dir = tmpdir / 'path_cache_dir'
  create_repo(path_cache_dir / 'subdir' / 'foo')
  create_repo(path_cache_dir / 'bar')
  foo_head = rev_parse(path_cache_dir / 'subdir' / 'foo', 'HEAD')
  bar_head = rev_parse(path_cache_dir / 'bar', 'HEAD')
 
  workspace = tmpdir / 'workspace'
- setup_cache_workspace(workspace, foo_head, bar_head)
+ setup_cache_workspace(workspace, remote, foo_head, bar_head)
  workspace.chdir()
  foo = workspace / 'subdir' / 'foo'
  bar = workspace / 'bar'