extend cache for submodules

This switches cache to use --reference and extends cache
support for submodules.

Signed-off-by: Kari Hamalainen <kari.hamalainen@nordicsemi.no>

src/west/app/project.py

@@ -26,9 +26,11 @@
     ManifestProject, _manifest_content_at, ManifestImportFailed
 from west.manifest import is_group as is_project_group
 from west.manifest import MANIFEST_REV_BRANCH as MANIFEST_REV
+from west.manifest import Submodule
 from west.manifest import QUAL_MANIFEST_REV_BRANCH as QUAL_MANIFEST_REV
 from west.manifest import QUAL_REFS_WEST as QUAL_REFS
 
+
 #
 # Project-related or multi-repo commands, like "init", "update",
 # "diff", etc.
@@ -1150,23 +1152,43 @@ def update_submodules(self, project):
         for config_opt in self.args.submodule_init_config:
             config_opts.extend(['-c', config_opt])
 
-        # For the list type, update given list of submodules.
-        if isinstance(submodules, list):
-            for submodule in submodules:
+        cache_dir = self.project_cache(project)
+        # For the boolean type, update all the submodules.
+        if isinstance(submodules, bool):
+            if cache_dir is None:
                 if self.sync_submodules:
-                    project.git(['submodule', 'sync', '--recursive',
-                                 '--', submodule.path])
+                    project.git(['submodule', 'sync', '--recursive'])
                 project.git(config_opts +
-                            ['submodule', 'update',
-                             '--init', submodules_update_strategy,
-                             '--recursive', submodule.path])
-        # For the bool type, update all project submodules
-        elif isinstance(submodules, bool):
+                            ['submodule', 'update', '--init',
+                                submodules_update_strategy, '--recursive'])
+                return
+            else:
+                # Cache used so convert to a list so that --reference can be used
+                res = project.git(['submodule', 'status'], capture_stdout=True)
+                if not res.stdout or res.returncode:
+                    self.die(
+                        f"Submodule status failed for project: {project.name}.")
+                mod_list = [s.strip() for s in res.stdout.decode('utf-8').split('\n') if s]
+                submodules = [Submodule(line.split(' ')[1]) for line in mod_list]
+
+        # For the list type, update given list of submodules.
+        for submodule in submodules:
             if self.sync_submodules:
-                project.git(['submodule', 'sync', '--recursive'])
+                project.git(['submodule', 'sync', '--recursive',
+                             '--', submodule.path])
+            ref = []
+            if (cache_dir):
+                submodule_ref = Path(cache_dir, submodule.path)
+                if submodule_ref.is_dir():
+                    ref = ['--reference', os.fspath(submodule_ref)]
+                    self.small_banner(f'using reference from: {submodule_ref}')
+                    self.dbg(
+                        f'found {submodule.path} in --path-cache {submodule_ref}',
+                        level=Verbosity.DBG_MORE)
             project.git(config_opts +
-                        ['submodule', 'update', '--init',
-                         submodules_update_strategy, '--recursive'])
+                        ['submodule', 'update',
+                            '--init', submodules_update_strategy,
+                            '--recursive'] + ref + [submodule.path])
 
     def update(self, project):
         if self.args.stats:
@@ -1322,15 +1344,9 @@ def init_project(self, project):
             # The user is therefore free to change the URL of this remote.
             project.git(['remote', 'add', '--', project.remote_name, project.url])
         else:
-            self.small_banner(f'{project.name}: cloning from {cache_dir}')
-            # Clone the project from a local cache repository. Set the
-            # remote name to the value that would be used without a
-            # cache.
-            project.git(['clone', '--origin', project.remote_name,
-                         cache_dir, project.abspath], cwd=self.topdir)
-            # Reset the remote's URL to the project's fetch URL.
-            project.git(['remote', 'set-url', project.remote_name,
-                         project.url])
+            self.small_banner(f'{project.name}: using reference from {cache_dir}')
+            project.git(['clone', '--origin', project.remote_name, '--reference',
+                         cache_dir, project.url, project.abspath], cwd=self.topdir)
             # Make sure we have a detached HEAD so we can delete the
             # local branch created by git clone.
             project.git('checkout --quiet --detach HEAD')

tests/test_project.py

@@ -1262,7 +1262,7 @@ def test_update_recovery(tmpdir):
     assert prev == rgood
 
 
-def setup_cache_workspace(workspace, foo_head, bar_head):
+def setup_cache_workspace(workspace, remote, foo_head, bar_head):
     # Shared helper code that sets up a workspace used to test the
     # 'west update --foo-cache' options.
 
@@ -1275,10 +1275,10 @@ def setup_cache_workspace(workspace, foo_head, bar_head):
           projects:
           - name: foo
             path: subdir/foo
-            url: should-not-be-fetched
+            url: file://{remote}
             revision: {foo_head}
           - name: bar
-            url: should-not-be-fetched
+            url: file://{remote}
             revision: {bar_head}
         ''')
 
@@ -1287,14 +1287,16 @@ def test_update_name_cache(tmpdir):
     # Test that 'west update --name-cache' works and doesn't hit the
     # network if it doesn't have to.
 
+    remote = tmpdir / 'remote'
+    create_repo(remote)
     name_cache_dir = tmpdir / 'name_cache'
     create_repo(name_cache_dir / 'foo')
     create_repo(name_cache_dir / 'bar')
     foo_head = rev_parse(name_cache_dir / 'foo', 'HEAD')
     bar_head = rev_parse(name_cache_dir / 'bar', 'HEAD')
 
     workspace = tmpdir / 'workspace'
-    setup_cache_workspace(workspace, foo_head, bar_head)
+    setup_cache_workspace(workspace, remote, foo_head, bar_head)
     workspace.chdir()
     foo = workspace / 'subdir' / 'foo'
     bar = workspace / 'bar'
@@ -1322,14 +1324,16 @@ def test_update_path_cache(tmpdir):
     # Test that 'west update --path-cache' works and doesn't hit the
     # network if it doesn't have to.
 
+    remote = tmpdir / 'remote'
+    create_repo(remote)
     path_cache_dir = tmpdir / 'path_cache_dir'
     create_repo(path_cache_dir / 'subdir' / 'foo')
     create_repo(path_cache_dir / 'bar')
     foo_head = rev_parse(path_cache_dir / 'subdir' / 'foo', 'HEAD')
     bar_head = rev_parse(path_cache_dir / 'bar', 'HEAD')
 
     workspace = tmpdir / 'workspace'
-    setup_cache_workspace(workspace, foo_head, bar_head)
+    setup_cache_workspace(workspace, remote, foo_head, bar_head)
     workspace.chdir()
     foo = workspace / 'subdir' / 'foo'
     bar = workspace / 'bar'