Merge pull request #1817 from ogayot/mantic-2023-10-02

Mantic merge 2023-10-02

.github/workflows/snap.yaml

@@ -1,4 +1,4 @@
-name: CI
+name: Snap
 
 on: [push, pull_request]
 

examples/autoinstall/fallback-offline.yaml

@@ -0,0 +1,21 @@
+version: 1
+locale: en_GB.UTF-8
+network:
+  version: 2
+  ethernets:
+    all-eth:
+      match:
+        name: "en*"
+      dhcp6: yes
+apt:
+  mirror-selection:
+    primary:
+    - uri: http://localhost/failed
+  fallback: offline-install
+codecs:
+  install: true
+identity:
+  realname: ''
+  username: ubuntu
+  password: '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1'
+  hostname: ubuntu

scripts/runtests.sh

@@ -68,6 +68,9 @@ validate () {
                     apt.security[1].uri='"http://ports.ubuntu.com/ubuntu-ports"'
                 ;;
         esac
+        if [ "$testname" == autoinstall-fallback-offline ]; then
+            grep -F -- 'skipping installation of package ubuntu-restricted-addons' "$tmpdir"/subiquity-server-debug.log
+        fi
         netplan generate --root $tmpdir
     elif [ "${mode}" = "system_setup" ]; then
         setup_mode="$2"
@@ -319,6 +322,18 @@ LANG=C.UTF-8 timeout --foreground 60 \
     --source-catalog examples/sources/install.yaml
 validate install
 
+clean
+testname=autoinstall-fallback-offline
+LANG=C.UTF-8 timeout --foreground 60 \
+    python3 -m subiquity.cmd.tui \
+    --dry-run \
+    --output-base "$tmpdir" \
+    --machine-config examples/machines/simple.json \
+    --autoinstall examples/autoinstall/fallback-offline.yaml \
+    --kernel-cmdline autoinstall \
+    --source-catalog examples/sources/install.yaml
+validate install
+
 # The OOBE doesn't exist in WSL < 20.04
 if [ "${RELEASE%.*}" -ge 20 ]; then
     # Test TCP connectivity (system_setup only)

subiquity/common/pkg.py

@@ -0,0 +1,25 @@
+# Copyright 2023 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import attr
+
+
+@attr.s(auto_attribs=True)
+class TargetPkg:
+    name: str
+    # Some packages are not present in the pool and require a working network
+    # connection to be downloaded. By marking them with "skip_when_offline", we
+    # can skip them when running an offline install.
+    skip_when_offline: bool

subiquity/models/ad.py

@@ -14,8 +14,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
-from typing import Optional
+from typing import List, Optional
 
+from subiquity.common.pkg import TargetPkg
 from subiquity.common.types import AdConnectionInfo
 
 log = logging.getLogger("subiquity.models.ad")
@@ -42,10 +43,14 @@ def set_domain(self, domain: str):
         else:
             self.conn_info = AdConnectionInfo(domain_name=domain)
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         # NOTE Those packages must be present in the target system to allow
         # joining to a domain.
         if self.do_join:
-            return ["adcli", "realmd", "sssd"]
+            return [
+                TargetPkg(name="adcli", skip_when_offline=False),
+                TargetPkg(name="realmd", skip_when_offline=False),
+                TargetPkg(name="sssd", skip_when_offline=False),
+            ]
 
         return []

subiquity/models/codecs.py

@@ -14,16 +14,22 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+from typing import List
+
+from subiquity.common.pkg import TargetPkg
 
 log = logging.getLogger("subiquity.models.codecs")
 
 
 class CodecsModel:
     do_install = False
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         # NOTE currently, ubuntu-restricted-addons is an empty package that
         # pulls relevant packages through Recommends: Ideally, we should make
         # sure to run the APT command for this package with the
         # --install-recommends option.
-        return ["ubuntu-restricted-addons"] if self.do_install else []
+        if not self.do_install:
+            return []
+
+        return [TargetPkg(name="ubuntu-restricted-addons", skip_when_offline=True)]

subiquity/models/locale.py

@@ -16,7 +16,9 @@
 import locale
 import logging
 import subprocess
+from typing import List
 
+from subiquity.common.pkg import TargetPkg
 from subiquitycore.utils import arun_command, split_cmd_output
 
 log = logging.getLogger("subiquity.models.locale")
@@ -60,7 +62,7 @@ def make_cloudconfig(self):
             locale += ".UTF-8"
         return {"locale": locale}
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         if self.selected_language is None:
             return []
         if self.locale_support != "langpack":
@@ -69,6 +71,7 @@ async def target_packages(self):
         if lang == "C":
             return []
 
-        return await split_cmd_output(
+        pkgs = await split_cmd_output(
             self.chroot_prefix + ["check-language-support", "-l", lang], None
         )
+        return [TargetPkg(name=pkg, skip_when_offline=False) for pkg in pkgs]

subiquity/models/network.py

@@ -15,8 +15,10 @@
 
 import logging
 import subprocess
+from typing import List
 
 from subiquity import cloudinit
+from subiquity.common.pkg import TargetPkg
 from subiquitycore.models.network import NetworkModel as CoreNetworkModel
 from subiquitycore.utils import arun_command
 
@@ -93,12 +95,12 @@ def render(self):
                 }
         return r
 
-    async def target_packages(self):
-        if self.needs_wpasupplicant:
-            return ["wpasupplicant"]
-        else:
+    async def target_packages(self) -> List[TargetPkg]:
+        if not self.needs_wpasupplicant:
             return []
 
+        return [TargetPkg(name="wpasupplicant", skip_when_offline=False)]
+
     async def is_nm_enabled(self):
         try:
             cp = await arun_command(("nmcli", "networking"), check=True)

subiquity/models/ssh.py

@@ -16,6 +16,8 @@
 import logging
 from typing import List
 
+from subiquity.common.pkg import TargetPkg
+
 log = logging.getLogger("subiquity.models.ssh")
 
 
@@ -29,8 +31,8 @@ def __init__(self):
         # we go back to it.
         self.ssh_import_id = ""
 
-    async def target_packages(self):
-        if self.install_server:
-            return ["openssh-server"]
-        else:
+    async def target_packages(self) -> List[TargetPkg]:
+        if not self.install_server:
             return []
+
+        return [TargetPkg(name="openssh-server", skip_when_offline=False)]

subiquity/models/subiquity.py

@@ -20,7 +20,7 @@
 import os
 import uuid
 from collections import OrderedDict
-from typing import Any, Dict, Set, Tuple
+from typing import Any, Dict, List, Set, Tuple
 
 import yaml
 from cloudinit.config.schema import (
@@ -39,6 +39,7 @@ def SchemaProblem(x, y):
 
 from curtin.config import merge_config
 
+from subiquity.common.pkg import TargetPkg
 from subiquity.common.resources import get_users_and_groups
 from subiquity.server.types import InstallerChannels
 from subiquitycore.file_util import generate_timestamped_header, write_file
@@ -177,6 +178,9 @@ def __init__(self, root, hub, install_model_names, postinstall_model_names):
             self.chroot_prefix = []
 
         self.active_directory = AdModel()
+        # List of packages that will be installed using cloud-init on first
+        # boot.
+        self.cloud_init_packages: List[str] = []
         self.codecs = CodecsModel()
         self.debconf_selections = DebconfSelectionsModel()
         self.drivers = DriversModel()
@@ -189,7 +193,7 @@ def __init__(self, root, hub, install_model_names, postinstall_model_names):
         self.mirror = MirrorModel()
         self.network = NetworkModel()
         self.oem = OEMModel()
-        self.packages = []
+        self.packages: List[TargetPkg] = []
         self.proxy = ProxyModel()
         self.snaplist = SnapListModel()
         self.ssh = SSHModel()
@@ -376,13 +380,15 @@ def _cloud_init_config(self):
             model = getattr(self, model_name)
             if getattr(model, "make_cloudconfig", None):
                 merge_config(config, model.make_cloudconfig())
+        for package in self.cloud_init_packages:
+            merge_config(config, {"packages": list(self.cloud_init_packages)})
         merge_cloud_init_config(config, self.userdata)
         if lsb_release()["release"] not in ("20.04", "22.04"):
             config.setdefault("write_files", []).append(CLOUDINIT_DISABLE_AFTER_INSTALL)
         self.validate_cloudconfig_schema(data=config, data_source="system install")
         return config
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         packages = list(self.packages)
         for model_name in self._postinstall_model_names.all():
             meth = getattr(getattr(self, model_name), "target_packages", None)

subiquity/server/controllers/filesystem.py

@@ -20,6 +20,7 @@
 import logging
 import os
 import pathlib
+import subprocess
 import time
 from typing import Any, Callable, Dict, List, Optional, Union
 
@@ -1532,6 +1533,15 @@ def make_autoinstall(self):
 
     async def _pre_shutdown(self):
         if not self.reset_partition_only:
-            await self.app.command_runner.run(["umount", "--recursive", "/target"])
+            # /target is mounted only if the installation was actually started.
+            try:
+                await self.app.command_runner.run(["mountpoint", "/target"])
+            except subprocess.CalledProcessError:
+                log.debug(
+                    "/target does not exist or is not mounted,"
+                    " skipping call to umount --recursive"
+                )
+            else:
+                await self.app.command_runner.run(["umount", "--recursive", "/target"])
         if len(self.model._all(type="zpool")) > 0:
             await self.app.command_runner.run(["zpool", "export", "-a"])

subiquity/server/controllers/install.py

@@ -34,6 +34,7 @@
 from curtin.util import get_efibootmgr, is_uefi_bootable
 
 from subiquity.common.errorreport import ErrorReportKind
+from subiquity.common.pkg import TargetPkg
 from subiquity.common.types import ApplicationState, PackageInstallState
 from subiquity.journald import journald_listen
 from subiquity.models.filesystem import ActionRenderMode, Partition
@@ -689,11 +690,22 @@ async def postinstall(self, *, context):
             {"autoinstall": self.app.make_autoinstall()}
         )
         write_file(autoinstall_path, autoinstall_config)
-        await self.configure_cloud_init(context=context)
+        try:
+            if self.supports_apt():
+                packages = await self.get_target_packages(context=context)
+                for package in packages:
+                    if package.skip_when_offline and not self.model.network.has_network:
+                        log.warning(
+                            "skipping installation of package %s when"
+                            " performing an offline install.",
+                            package.name,
+                        )
+                        continue
+                    await self.install_package(context=context, package=package.name)
+        finally:
+            await self.configure_cloud_init(context=context)
+
         if self.supports_apt():
-            packages = await self.get_target_packages(context=context)
-            for package in packages:
-                await self.install_package(context=context, package=package)
             if self.model.drivers.do_install:
                 with context.child(
                     "ubuntu-drivers-install", "installing third-party drivers"
@@ -720,7 +732,7 @@ async def configure_cloud_init(self, context):
         await run_in_thread(self.model.configure_cloud_init)
 
     @with_context(description="calculating extra packages to install")
-    async def get_target_packages(self, context):
+    async def get_target_packages(self, context) -> List[TargetPkg]:
         return await self.app.base_model.target_packages()
 
     @with_context(name="install_{package}", description="installing {package}")

subiquity/server/controllers/package.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from subiquity.common.pkg import TargetPkg
 from subiquity.server.controller import NonInteractiveController
 
 
@@ -25,7 +26,7 @@ class PackageController(NonInteractiveController):
     }
 
     def load_autoinstall_data(self, data):
-        self.model[:] = data
+        self.model[:] = [TargetPkg(name=pkg, skip_when_offline=False) for pkg in data]
 
     def make_autoinstall(self):
-        return self.model
+        return [pkg.name for pkg in self.model]