Merge branch 'oamg:main' into bugs/rhelc-1124

.devcontainer/alma9/devcontainer.json

@@ -31,8 +31,8 @@
 	"customizations": {
 		"vscode": {
 			"settings": {
-				"extensions.autoUpdate": false,
-				"extensions.autoCheckUpdates": false,
+				"extensions.autoUpdate": true,
+				"extensions.autoCheckUpdates": true,
 				"python.languageServer": "Pylance"
 			},
 			"extensions": [
@@ -41,7 +41,7 @@
 				"GitHub.vscode-pull-request-github",
 				"Cameron.vscode-pytest",
 				"njpwerner.autodocstring",
-				"ms-python.python@2022.8.1"
+				"ms-python.python"
 			]
 		}
 	}

.pre-commit-config.yaml

@@ -9,18 +9,14 @@ repos:
         language: python
         language_version: "python3"
         additional_dependencies: [click==7.1.2, black==21.12b0]
-  - repo: local
+  - repo: https://github.com/pylint-dev/pylint
+    rev: v3.1.0
     hooks:
       - id: pylint
-        name: pylint
-        entry: pylint
-        types: [python]
-        language: python
-        additional_dependencies: [pylint==2.15.10]
         args: [
-            "-sn", # Don't display the score
-            "--rcfile=.pylintrc", # Link to your config file
-          ]
+          "-sn", # Don't display the score
+          "--rcfile=.pylintrc", # Link to your config file
+        ]
   - repo: "https://github.com/timothycrosley/isort"
     rev: 5.13.2
     hooks:

.pylintrc

@@ -15,6 +15,7 @@ disable=
 # on the versions of python we support.
  redundant-u-string-prefix,  # Python 3.0+
  consider-using-f-string,  # Python 3.6+
+ use-yield-from, # Python 3+
 # These are our current failures 2023-01-11.  We can go through them and either
 # fix them or add a comment to say why we are leaving it disabled.
  anomalous-backslash-in-string,

CONTRIBUTING.md

@@ -76,12 +76,6 @@ Dev Containers: Open Folder in Container
 You are now inside a container as indicated by the bottom left box. There is an extension for tests in the left
 navigation that auto-discovers tests that you can run. This will be much faster than developing using `make tests`
 
-#### Known issues
-##### Debugging just loads and does nothing
-Versions newer than `2022.8.1` of extension `ms-python.python` will not work when debugging code. We have pinned the version but VS Code might auto-update the extension.
-
-To work around this, go to extensions tab and select Ignore Updates on the extension, thereafter select install a new version and install `2022.8.1`. It will prompt to reload the window, make sure the right version is installed after reload. Now debugging should work
-
 ### Dependencies for local development
 
 We have some required dependencies you should have installed on your system

convert2rhel/actions/pre_ponr_changes/handle_packages.py

@@ -19,10 +19,11 @@
 import logging
 import os
 
-from convert2rhel import actions, pkghandler
+from convert2rhel import actions, pkghandler, utils
+from convert2rhel.backup import BACKUP_DIR, backup_control
+from convert2rhel.backup.packages import RestorablePackage
 from convert2rhel.repo import DEFAULT_YUM_REPOFILE_DIR
 from convert2rhel.systeminfo import system_info
-from convert2rhel.utils import BACKUP_DIR
 
 
 logger = logging.getLogger(__name__)
@@ -65,149 +66,122 @@ def extract_packages(self, pkg):
         return pkg.nevra.name
 
 
-class RemoveExcludedPackages(actions.Action):
-    id = "REMOVE_EXCLUDED_PACKAGES"
+class RemoveSpecialPackages(actions.Action):
+    id = "REMOVE_SPECIAL_PACKAGES"
     dependencies = (
+        # We use the backed up repos in remove_pkgs_unless_from_redhat()
         "BACKUP_REPOSITORY",
         "BACKUP_PACKAGE_FILES",
-    )  # We use the backed up repos in remove_pkgs_unless_from_redhat()
+        "BACKUP_REDHAT_RELEASE",
+    )
 
     def run(self):
+        """Remove a set of special packages from the system.
+
+        The packages marked for exclusion here are the excluded_pkgs and
+        repofile_pkgs that comes from the system_info singleton. This class
+        substitute the old RemoveExcludedPackages and RemoveRepofilePackages as
+        both of them depends on the RestorablePackage class, in which case, the
+        RestorablePackage was designed to handle a set of packages in the
+        moment of instantiation of the class, but since both removal classes
+        executed separately, we couldn't properly reinstall some packages that
+        got excluded and backed up, as they had to be reinstalled in the system
+        in the same RPM transaction call. To not redo how the RestorablePackage
+        works, both RemoveExcludedPackages and RemoveRepofilePackages classes
+        got merged together into this one, making possible to remove and back
+        up all the packages in a single transaction.
         """
-        Certain packages need to be removed before the system conversion,
-        depending on the system to be converted.
-        """
-        super(RemoveExcludedPackages, self).run()
-
-        logger.task("Convert: Remove excluded packages")
-        logger.info("Searching for the following excluded packages:\n")
-
+        all_pkgs = []
         pkgs_removed = []
-        # Since the MD5 checksum of original path is used in backup path to avoid
-        # conflicts in backup folder, preparing the path is needed.
-        backedup_reposdir = os.path.join(BACKUP_DIR, hashlib.md5(DEFAULT_YUM_REPOFILE_DIR.encode()).hexdigest())
-
         try:
-            pkgs_to_remove = sorted(pkghandler.get_packages_to_remove(system_info.excluded_pkgs))
-            # this call can return None, which is not ideal to use with sorted.
-            pkgs_removed = sorted(pkghandler.remove_pkgs_unless_from_redhat(pkgs_to_remove, backedup_reposdir) or [])
+            logger.task("Convert: Searching for the following excluded packages")
+            excluded_pkgs = sorted(pkghandler.get_packages_to_remove(system_info.excluded_pkgs))
 
-            # TODO: Handling SystemExit here as way to speedup exception
-            # handling and not refactor contents of the underlying function.
+            logger.task(
+                "Convert: Searching for packages containing .repo files or affecting variables in the .repo files"
+            )
+            repofile_pkgs = sorted(pkghandler.get_packages_to_remove(system_info.repofile_pkgs))
+
+            logger.info("\n")
+
+            all_pkgs = excluded_pkgs + repofile_pkgs
+            if not all_pkgs:
+                logger.info("No packages to backup and remove.")
+                return
+
+            # We're using the backed up yum repositories to prevent the following:
+            # - the system was registered to RHSM prior to the conversion and the system didn't have the redhat.repo generated
+            #   for the lack of the RHSM product certificate
+            # - at this point convert2rhel has installed the RHSM product cert (e.g. /etc/pki/product-default/69.pem)
+            # - this function might be performing the first yum call convert2rhel does after cleaning yum metadata
+            # - the "subscription-manager" yum plugin spots that there's a new RHSM product cert and generates
+            #   /etc/yum.repos.d/redhat.repo
+            # - the suddenly enabled RHEL repos cause a package backup failure
+            # Since the MD5 checksum of original path is used in backup path to avoid
+            # conflicts in backup folder, preparing the path is needed.
+            backedup_reposdir = os.path.join(BACKUP_DIR, hashlib.md5(DEFAULT_YUM_REPOFILE_DIR.encode()).hexdigest())
+            backup_control.push(RestorablePackage(pkgs=pkghandler.get_pkg_nevras(all_pkgs), reposdir=backedup_reposdir))
+
+            logger.info("\nRemoving special packages from the system.")
+            pkgs_removed = _remove_packages_unless_from_redhat(pkgs_list=all_pkgs)
         except SystemExit as e:
             # TODO(r0x0d): Places where we raise SystemExit and need to be
             # changed to something more specific.
             #   - When we can't remove a package.
             self.set_result(
                 level="ERROR",
-                id="EXCLUDED_PACKAGE_REMOVAL_FAILED",
-                title="Failed to remove excluded package",
+                id="SPECIAL_PACKAGE_REMOVAL_FAILED",
+                title="Failed to remove some packages necessary for the conversion.",
                 description="The cause of this error is unknown, please look at the diagnosis for more information.",
                 diagnosis=str(e),
             )
             return
 
         # shows which packages were not removed, if false, all packages were removed
-        pkgs_not_removed = sorted(frozenset(pkghandler.get_pkg_nevras(pkgs_to_remove)).difference(pkgs_removed))
+        pkgs_not_removed = sorted(frozenset(pkghandler.get_pkg_nevras(all_pkgs)).difference(pkgs_removed))
         if pkgs_not_removed:
             message = "The following packages were not removed: %s" % ", ".join(pkgs_not_removed)
             logger.warning(message)
             self.add_message(
                 level="WARNING",
-                id="EXCLUDED_PACKAGES_NOT_REMOVED",
-                title="Excluded packages not removed",
-                description="Excluded packages which could not be removed",
+                id="SPECIAL_PACKAGES_NOT_REMOVED",
+                title="Special packages not removed",
+                description="Special packages which could not be removed",
                 diagnosis=message,
             )
+
         if pkgs_removed:
             message = "The following packages will be removed during the conversion: %s" % ", ".join(pkgs_removed)
             logger.info(message)
             self.add_message(
                 level="INFO",
-                id="EXCLUDED_PACKAGES_REMOVED",
-                title="Excluded packages to be removed",
-                description="We have identified installed packages that match a pre-defined list of packages that are"
-                " to be removed during the conversion",
+                id="SPECIAL_PACKAGES_REMOVED",
+                title="Special packages to be removed",
+                description=(
+                    "We have identified installed packages that match a pre-defined list of packages that are"
+                    " to be removed during the conversion"
+                ),
                 diagnosis=message,
             )
 
+        super(RemoveSpecialPackages, self).run()
 
-class RemoveRepositoryFilesPackages(actions.Action):
-    id = "REMOVE_REPOSITORY_FILES_PACKAGES"
-    dependencies = (
-        "BACKUP_REDHAT_RELEASE",
-        # We use the backed up repos in remove_pkgs_unless_from_redhat()
-        "BACKUP_REPOSITORY",
-        # The installation of sub-man pkgs needs access to the original repofiles to get the sub-man deps from there
-        "PRE_SUBSCRIPTION",
-        "BACKUP_PACKAGE_FILES",
-    )
 
-    def run(self):
-        """
-        Remove those non-RHEL packages that contain YUM/DNF repofiles
-        (/etc/yum.repos.d/*.repo) or affect variables in the repofiles (e.g.
-        $releasever).
-
-        Red Hat cannot automatically remove these non-RHEL packages with other
-        excluded packages. While other excluded packages must be removed before
-        installing subscription-manager to prevent package conflicts, these
-        non-RHEL packages must be present on the system during
-        subscription-manager installation so that the system can access and
-        install subscription-manager dependencies. As a result, these non-RHEL
-        packages must be manually removed after subscription-manager
-        installation.
-        """
-        super(RemoveRepositoryFilesPackages, self).run()
+def _remove_packages_unless_from_redhat(pkgs_list):
+    """Remove packages from the system that are not RHEL.
 
-        logger.task("Convert: Remove packages containing .repo files")
-        logger.info("Searching for packages containing .repo files or affecting variables in the .repo files:\n")
+    :param pkgs_list list[str]: Packages that will be removed.
+    :return list[str]: Packages removed from the system.
+    """
+    if not pkgs_list:
+        logger.info("\nNothing to do.")
+        return []
 
-        pkgs_removed = []
-        # Since the MD5 checksum of original path is used in backup path to avoid
-        # conflicts in backup folder, preparing the path is needed.
-        backedup_reposdir = os.path.join(BACKUP_DIR, hashlib.md5(DEFAULT_YUM_REPOFILE_DIR.encode()).hexdigest())
-
-        try:
-            pkgs_to_remove = sorted(pkghandler.get_packages_to_remove(system_info.repofile_pkgs))
-            # this call can return None, which is not ideal to use with sorted.
-            pkgs_removed = sorted(pkghandler.remove_pkgs_unless_from_redhat(pkgs_to_remove, backedup_reposdir) or [])
+    # this call can return None, which is not ideal to use with sorted.
+    logger.warning("Removing the following %s packages:\n" % len(pkgs_list))
+    pkghandler.print_pkg_info(pkgs_list)
 
-            # TODO: Handling SystemExit here as way to speedup exception
-            # handling and not refactor contents of the underlying function.
-        except SystemExit as e:
-            # TODO(r0x0d): Places where we raise SystemExit and need to be
-            # changed to something more specific.
-            #   - When we can't remove a package.
-            self.set_result(
-                level="ERROR",
-                id="REPOSITORY_FILE_PACKAGE_REMOVAL_FAILED",
-                title="Repository file package removal failure",
-                description="The cause of this error is unknown, please look at the diagnosis for more information.",
-                diagnosis=str(e),
-            )
-            return
+    pkgs_removed = utils.remove_pkgs(pkghandler.get_pkg_nevras(pkgs_list))
+    logger.debug("Successfully removed %s packages" % len(pkgs_list))
 
-        # shows which packages were not removed, if false, all packages were removed
-        pkgs_not_removed = sorted(frozenset(pkghandler.get_pkg_nevras(pkgs_to_remove)).difference(pkgs_removed))
-        if pkgs_not_removed:
-            message = "The following packages were not removed: %s" % ", ".join(pkgs_not_removed)
-            logger.warning(message)
-            self.add_message(
-                level="WARNING",
-                id="REPOSITORY_FILE_PACKAGES_NOT_REMOVED",
-                title="Repository file packages not removed",
-                description="Repository file packages which could not be removed",
-                diagnosis=message,
-            )
-        if pkgs_removed:
-            message = "The following packages will be removed during the conversion: %s" % ", ".join(pkgs_removed)
-            logger.info(message)
-            self.add_message(
-                level="INFO",
-                id="REPOSITORY_FILE_PACKAGES_REMOVED",
-                title="Repository file packages to be removed",
-                description="We have identified installed packages that match a pre-defined list of packages that are"
-                " to be removed during the conversion",
-                diagnosis=message,
-            )
+    return pkgs_removed

convert2rhel/actions/pre_ponr_changes/subscription.py

@@ -68,9 +68,9 @@ def run(self):
 class PreSubscription(actions.Action):
     id = "PRE_SUBSCRIPTION"
     dependencies = (
+        "REMOVE_SPECIAL_PACKAGES",
         "INSTALL_RED_HAT_CERT_FOR_YUM",
         "INSTALL_RED_HAT_GPG_KEY",
-        "REMOVE_EXCLUDED_PACKAGES",
     )
 
     def run(self):
@@ -159,7 +159,6 @@ class SubscribeSystem(actions.Action):
     id = "SUBSCRIBE_SYSTEM"
     dependencies = (
         # Implicit dependency for `BACKUP_REDHAT_RELEASE`
-        "REMOVE_REPOSITORY_FILES_PACKAGES",
         "PRE_SUBSCRIPTION",
         "EUS_SYSTEM_CHECK",
     )

convert2rhel/actions/pre_ponr_changes/transaction.py

@@ -27,7 +27,6 @@ class ValidatePackageManagerTransaction(actions.Action):
     id = "VALIDATE_PACKAGE_MANAGER_TRANSACTION"
     dependencies = (
         "INSTALL_RED_HAT_GPG_KEY",
-        "REMOVE_EXCLUDED_PACKAGES",
         # This package can cause problems during the validation. Since no one
         # is depending on this action, it may run whenever it wants to, which
         # can cause problems.

convert2rhel/actions/system_checks/custom_repos_are_valid.py

@@ -18,7 +18,7 @@
 import logging
 
 from convert2rhel import actions
-from convert2rhel.pkghandler import call_yum_cmd
+from convert2rhel.pkgmanager import call_yum_cmd
 from convert2rhel.toolopts import tool_opts
 
 

convert2rhel/actions/system_checks/rhel_compatible_kernel.py

@@ -99,19 +99,19 @@ def _bad_kernel_version(kernel_release):
         raise KernelIncompatibleError(
             "UNEXPECTED_VERSION",
             "Unexpected OS major version. Expected: {compatible_version}",
-            dict(compatible_version=COMPATIBLE_KERNELS_VERS.keys()),
+            {"compatible_version": COMPATIBLE_KERNELS_VERS.keys()},
         )
 
     if incompatible_version:
         raise KernelIncompatibleError(
             "INCOMPATIBLE_VERSION",
             "Booted kernel version '{kernel_version}' does not correspond to the version "
             "'{compatible_version}' available in RHEL {rhel_major_version}",
-            dict(
-                kernel_version=kernel_version,
-                compatible_version=COMPATIBLE_KERNELS_VERS[system_info.version.major],
-                rhel_major_version=system_info.version.major,
-            ),
+            {
+                "kernel_version": kernel_version,
+                "compatible_version": COMPATIBLE_KERNELS_VERS[system_info.version.major],
+                "rhel_major_version": system_info.version.major,
+            },
         )
 
     logger.debug(
@@ -134,7 +134,7 @@ def _bad_kernel_package_signature(kernel_release):
             "UNSIGNED_PACKAGE",
             "The booted kernel {vmlinuz_path} is not owned by any installed package."
             " It needs to be owned by a package signed by {os_vendor}.",
-            dict(vmlinuz_path=vmlinuz_path, os_vendor=os_vendor),
+            {"vmlinuz_path": vmlinuz_path, "os_vendor": os_vendor},
         )
 
     kernel_pkg_obj = get_installed_pkg_information(pkg_name=kernel_pkg)
@@ -143,7 +143,7 @@ def _bad_kernel_package_signature(kernel_release):
         raise KernelIncompatibleError(
             "INVALID_KERNEL_PACKAGE_SIGNATURE",
             "Custom kernel detected. The booted kernel needs to be signed by {os_vendor}.",
-            dict(os_vendor=os_vendor),
+            {"os_vendor": os_vendor},
         )
 
     logger.debug("The booted kernel is signed by %s." % os_vendor)
@@ -158,6 +158,6 @@ def _bad_kernel_substring(kernel_release):
             "INVALID_PACKAGE_SUBSTRING",
             "The booted kernel '{kernel_release}' contains one of the disallowed "
             "substrings: {bad_kernel_release_substrings}",
-            dict(kernel_release=kernel_release, bad_kernel_release_substrings=BAD_KERNEL_RELEASE_SUBSTRINGS),
+            {"kernel_release": kernel_release, "bad_kernel_release_substrings": BAD_KERNEL_RELEASE_SUBSTRINGS},
         )
     return False