project-installcheck.py

#!/usr/bin/python3

import difflib
import hashlib
import logging
import os
import os.path
import subprocess
import sys
import tempfile
import cmdln
import dateutil.parser
from datetime import datetime, timedelta
from urllib.parse import urlencode

import yaml
from lxml import etree as ET
from osc import conf
from osc.core import http_request

import ToolBase
from osclib.conf import Config
from osclib.core import (http_DELETE, http_GET, makeurl,
                         repository_path_expand, repository_path_search,
                         target_archs, source_file_load, source_file_ensure)
from osclib.repochecks import mirror, parsed_installcheck, CorruptRepos
from osclib.comments import CommentAPI


class RepoChecker():
    def __init__(self):
        self.logger = logging.getLogger('RepoChecker')
        self.store_project = None
        self.store_package = None
        self.rebuild = None
        self.comment = None

    def parse_store(self, project_package):
        if project_package:
            self.store_project, self.store_package = project_package.split('/')

    def check(self, project, repository):
        self.project = project
        if not repository:
            repository = self.project_repository(project)
        if not repository:
            self.logger.error(f'a repository must be specified via OSRT:Config main-repo for {project}')
            return
        self.repository = repository

        archs = target_archs(self.apiurl, project, repository)
        if not len(archs):
            self.logger.debug(f'{project} has no relevant architectures')
            return None

        for arch in archs:
            self.arch = arch
            state = self.check_pra(project, repository, arch)

        if self.comment:
            self.create_comments(state)

    def create_comments(self, state):
        comments = dict()
        for source, details in state['check'].items():
            rebuild = dateutil.parser.parse(details["rebuild"])
            if datetime.now() - rebuild < timedelta(days=2):
                self.logger.debug(f"Ignore {source} - problem too recent")
                continue
            _, _, arch, rpm = source.split('/')
            rpm = rpm.split(':')[0]
            comments.setdefault(rpm, {})
            comments[rpm][arch] = details['problem']

        url = makeurl(self.apiurl, ['comments', 'user'])
        root = ET.parse(http_GET(url)).getroot()
        for comment in root.findall('.//comment'):
            if comment.get('project') != self.project:
                continue
            if comment.get('package') in comments:
                continue
            self.logger.info(f"Removing comment for package {comment.get('package')}")
            url = makeurl(self.apiurl, ['comment', comment.get('id')])
            http_DELETE(url)

        commentapi = CommentAPI(self.apiurl)
        MARKER = 'Installcheck'

        for package in comments:
            newcomment = ''
            for arch in sorted(comments[package]):
                newcomment += f"\n\n**Installcheck problems for {arch}**\n\n"
                for problem in sorted(comments[package][arch]):
                    newcomment += "+ " + problem + "\n"

            newcomment = commentapi.add_marker(newcomment.strip(), MARKER)
            oldcomments = commentapi.get_comments(project_name=self.project, package_name=package)
            oldcomment, _ = commentapi.comment_find(oldcomments, MARKER)
            if oldcomment and oldcomment['comment'] == newcomment:
                continue

            if oldcomment:
                commentapi.delete(oldcomment['id'])
            self.logger.debug(f"Adding comment to {self.project}/{package}")
            commentapi.add_comment(project_name=self.project, package_name=package, comment=newcomment)

    def _split_and_filter(self, output):
        output = output.split("\n")
        for lnr, line in enumerate(output):
            if line.startswith('FOLLOWUP'):
                # there can be multiple lines with missing providers
                while lnr >= 0 and output[lnr - 1].endswith('none of the providers can be installed'):
                    output.pop()
                    lnr = lnr - 1
        for lnr in reversed(range(len(output))):
            # those lines are hardly interesting for us
            if output[lnr].find('(we have') >= 0:
                del output[lnr]
            else:
                output[lnr] = output[lnr]
        return output

    def project_repository(self, project):
        repository = Config.get(self.apiurl, project).get('main-repo')
        if not repository:
            self.logger.debug(f'no main-repo defined for {project}')

            search_project = 'openSUSE:Factory'
            for search_repository in ('snapshot', 'standard'):
                repository = repository_path_search(
                    self.apiurl, project, search_project, search_repository)

                if repository:
                    self.logger.debug('found chain to {}/{} via {}'.format(
                        search_project, search_repository, repository))
                    break

        return repository

    def store_yaml(self, state):
        if not self.store_project or not self.store_package:
            return

        state_yaml = yaml.dump(state, default_flow_style=False)
        comment = f'Updated rebuild infos for {self.project}/{self.repository}/{self.arch}'
        source_file_ensure(self.apiurl, self.store_project, self.store_package,
                           self.store_filename, state_yaml, comment=comment)

    def check_buildstate(self, oldstate, buildresult, code):
        oldstate.setdefault(code, {})
        for source in list(oldstate[code]):
            project, repo, arch, rpm = source.split('/')
            if project != self.project or repo != self.repository or arch != self.arch:
                continue
            if buildresult.get(rpm, 'gone') != code:
                del oldstate[code][source]
        for rpm, rcode in buildresult.items():
            if rcode != code:
                continue
            source = f"{self.project}/{self.repository}/{self.arch}/{rpm}"
            if source not in oldstate[code]:
                oldstate[code][source] = str(datetime.now())

    def check_pra(self, project, repository, arch):
        config = Config.get(self.apiurl, project)

        oldstate = None
        self.store_filename = f'rebuildpacs.{project}-{repository}.yaml'
        if self.store_project and self.store_package:
            state_yaml = source_file_load(self.apiurl, self.store_project, self.store_package,
                                          self.store_filename)
            if state_yaml:
                oldstate = yaml.safe_load(state_yaml)

        oldstate = oldstate or {}
        oldstate.setdefault('check', {})
        if not isinstance(oldstate['check'], dict):
            oldstate['check'] = {}
        oldstate.setdefault('leafs', {})
        if not isinstance(oldstate['leafs'], dict):
            oldstate['leafs'] = {}

        repository_pairs = repository_path_expand(self.apiurl, project, repository)
        directories = []
        primaryxmls = []
        for pair_project, pair_repository in repository_pairs:
            mirrored = mirror(self.apiurl, pair_project, pair_repository, arch)
            if os.path.isdir(mirrored):
                directories.append(mirrored)
            else:
                primaryxmls.append(mirrored)

        parsed = dict()
        with tempfile.TemporaryDirectory(prefix='repochecker') as dir:
            pfile = os.path.join(dir, 'packages')

            SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
            script = os.path.join(SCRIPT_PATH, 'write_repo_susetags_file.pl')
            parts = ['perl', script, dir] + directories

            p = subprocess.run(parts)
            if p.returncode:
                # technically only 126, but there is no other value atm -
                # so if some other perl error happens, we don't continue
                raise CorruptRepos

            target_packages = {}
            with open(os.path.join(dir, 'catalog.yml')) as file:
                catalog = yaml.safe_load(file)
                if catalog is not None:
                    target_packages = catalog.get(directories[0], [])

            parsed = parsed_installcheck([pfile] + primaryxmls, arch, target_packages, [])

        for package in parsed:
            parsed[package]['output'] = "\n".join(parsed[package]['output'])

        # let's risk a N*N algorithm in the hope that we have a limited N
        for package1 in parsed:
            output = parsed[package1]['output']
            for package2 in parsed:
                if package1 == package2:
                    continue
                output = output.replace(parsed[package2]['output'], 'FOLLOWUP(' + package2 + ')')
            parsed[package1]['output'] = output

        for package in parsed:
            parsed[package]['output'] = self._split_and_filter(parsed[package]['output'])

        url = makeurl(self.apiurl, ['build', project, '_result'], {'repository': repository, 'arch': arch})
        root = ET.parse(http_GET(url)).getroot()
        buildresult = dict()
        for p in root.findall('.//status'):
            buildresult[p.get('package')] = p.get('code')

        repo_state = root.find('result').get('state')
        if repo_state in ['published', 'unpublished']:
            self.check_buildstate(oldstate, buildresult, 'unresolvable')
            self.check_buildstate(oldstate, buildresult, 'failed')

        per_source = dict()

        ignore_conflicts = config.get('installcheck-ignore-conflicts', '').split() + \
            config.get(f'installcheck-ignore-conflicts-{arch}', '').split()

        for package, entry in parsed.items():
            source = f"{project}/{repository}/{arch}/{entry['source']}"
            per_source.setdefault(source, {'output': [], 'buildresult': buildresult.get(entry['source'], 'gone'), 'ignored': True})
            per_source[source]['output'].extend(entry['output'])
            if package not in ignore_conflicts:
                per_source[source]['ignored'] = False

        rebuilds = set()

        for source in sorted(per_source):
            if not len(per_source[source]['output']):
                continue
            self.logger.debug(f"{source} builds: {per_source[source]['buildresult']}")
            self.logger.debug("  " + "\n  ".join(per_source[source]['output']))
            if per_source[source]['buildresult'] != 'succeeded':  # nothing we can do
                continue
            if per_source[source]['ignored']:
                self.logger.debug("All binaries failing the check are ignored")
                if source in oldstate['check']:
                    del oldstate['check'][source]
                continue
            old_output = oldstate['check'].get(source, {}).get('problem', [])
            if sorted(old_output) == sorted(per_source[source]['output']):
                self.logger.debug("unchanged problem")
                continue
            self.logger.info("rebuild %s", source)
            rebuilds.add(os.path.basename(source))
            for line in difflib.unified_diff(old_output, per_source[source]['output'], 'before', 'now'):
                self.logger.debug(line.strip())
            oldstate['check'][source] = {'problem': per_source[source]['output'],
                                         'rebuild': str(datetime.now())}

        for source in list(oldstate['check']):
            if not source.startswith(f'{project}/{repository}/{arch}/'):
                continue
            code = buildresult.get(os.path.basename(source), 'gone')
            if code == 'gone' or code == 'excluded':
                del oldstate['check'][source]
            if code != 'succeeded':
                self.logger.debug(f"Skipping build result for {source} {code}")
                continue
            if source not in per_source:
                self.logger.info("No known problem, erasing %s", source)
                del oldstate['check'][source]

        packages = config.get('rebuildpacs-leafs', '').split()
        if not self.rebuild:  # ignore in this case
            packages = []

        # first round: collect all infos from obs
        infos = dict()
        for package in packages:
            subpacks, build_deps = self.check_leaf_package(project, repository, arch, package)
            infos[package] = {'subpacks': subpacks, 'deps': build_deps}

        # calculate rebuild triggers
        rebuild_triggers = dict()
        for package1 in packages:
            for package2 in packages:
                if package1 == package2:
                    continue
                for subpack in infos[package1]['subpacks']:
                    if subpack in infos[package2]['deps']:
                        rebuild_triggers.setdefault(package1, set())
                        rebuild_triggers[package1].add(package2)
                        # ignore this depencency. we already trigger both of them
                        del infos[package2]['deps'][subpack]

        # calculate build info hashes
        for package in packages:
            if buildresult[package] != 'succeeded':
                self.logger.debug("Ignore %s for the moment, %s", package, buildresult[package])
                continue
            m = hashlib.sha256()
            for bdep in sorted(infos[package]['deps']):
                m.update(bytes(bdep + '-' + infos[package]['deps'][bdep], 'utf-8'))
            state_key = f'{project}/{repository}/{arch}/{package}'
            olddigest = oldstate['leafs'].get(state_key, {}).get('buildinfo')
            if olddigest == m.hexdigest():
                continue
            self.logger.info("rebuild leaf package %s (%s vs %s)", package, olddigest, m.hexdigest())
            rebuilds.add(package)
            oldstate['leafs'][state_key] = {'buildinfo': m.hexdigest(),
                                            'rebuild': str(datetime.now())}

        if self.dryrun:
            if self.rebuild:
                self.logger.info("To rebuild: %s", ' '.join(rebuilds))
            return oldstate

        if not self.rebuild or not len(rebuilds):
            self.logger.debug("Nothing to rebuild")
            # in case we do rebuild, wait for it to succeed before saving
            self.store_yaml(oldstate)
            return oldstate

        query = {'cmd': 'rebuild', 'repository': repository, 'arch': arch, 'package': rebuilds}
        url = makeurl(self.apiurl, ['build', project])
        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        http_request('POST', url, headers, data=urlencode(query, doseq=True))

        self.store_yaml(oldstate)
        return oldstate

    def check_leaf_package(self, project, repository, arch, package):
        url = makeurl(self.apiurl, ['build', project, repository, arch, package, '_buildinfo'])
        root = ET.parse(http_GET(url)).getroot()
        subpacks = set()
        for sp in root.findall('subpack'):
            subpacks.add(sp.text)
        build_deps = dict()
        for bd in root.findall('bdep'):
            if bd.get('notmeta') == '1':
                continue
            build_deps[bd.get('name')] = bd.get('version') + '-' + bd.get('release')
        return subpacks, build_deps


class CommandLineInterface(ToolBase.CommandLineInterface):

    def __init__(self, *args, **kwargs):
        ToolBase.CommandLineInterface.__init__(self, args, kwargs)

    def setup_tool(self):
        return RepoChecker()

    @cmdln.option('--store', help='Project/Package to store the rebuild infos in')
    @cmdln.option('-r', '--repo', dest='repo', help='Repository to check')
    @cmdln.option('--add-comments', dest='comments', action='store_true', help='Create comments about issues')
    @cmdln.option('--no-rebuild', dest='norebuild', action='store_true', help='Only track issues, do not rebuild')
    def do_check(self, subcmd, opts, project):
        """${cmd_name}: Rebuild packages in rebuild=local projects

        ${cmd_usage}
        ${cmd_option_list}
        """
        self.tool.rebuild = not opts.norebuild
        self.tool.comment = opts.comments
        self.tool.parse_store(opts.store)
        self.tool.apiurl = conf.config['apiurl']
        self.tool.check(project, opts.repo)


if __name__ == '__main__':
    app = CommandLineInterface()
    sys.exit(app.main())