check.py

#!/usr/bin/python3
# -*- encoding: utf-8 -*-

import koji as brew
import bugzilla as bz
import logging as log
import mysql.connector as mydb
import configparser
import optparse
import string
import random
import sys
import git
import os
import re

LAST_NVR = None
COMPLETION_TIME = None
VOLUME_ID = None
IS_CONTAINER = False
LOGFILE = "Log\n"
BAD_BZS = []
BZRE = re.compile(r"(?:(?:bug|bz|rhbz)\s*#?|#)\s*(\d+)", re.IGNORECASE)
CVERE = re.compile(r"(CVE-[0-9]{4}-[0-9]{4,})", re.IGNORECASE)

def add_to_log(message):
    global LOGFILE
    if connection != None:
        LOGFILE += message
        connection.execute("UPDATE results SET log='" + LOGFILE + "' WHERE id = " + str(taskId))
        dtb.commit()

def update_state(state_current):
    if connection != None:
        if state_current == "error":
            connection.execute("UPDATE results SET state=" + state[state_current] + ", end = now(), result=1 WHERE id = " + str(taskId))
        else:
            connection.execute("UPDATE results SET state=" + state[state_current] + " WHERE id = " + str(taskId))
        dtb.commit()

def update_db(column, value):
    if connection != None:
        if column == 'end':
            connection.execute("UPDATE results SET end=now() WHERE id = %d" % (taskId))
        else:
            connection.execute("UPDATE results SET %s='%s' WHERE id = %d" % (column, value, taskId))
        dtb.commit()

def randomString(stringLength):
    letters = string.ascii_letters
    return "".join(random.choice(letters) for i in range(stringLength))

def split_rpm_name(name):
    m = re.search(r'(.*)-(.*)-(.*?)\.(el.*)', name)
    if m:
        (name, version, release, platform) = m.groups()
    else:
        (name, version, release, platform) = split_module_name(name)

    return (name, version, release, platform)

def split_module_name(name):
    m = re.search(r'(.*)-(.*)-(.*?)\+(el.*)', name)
    if m:
        (name, version, release, platform) = m.groups()
        return (name, version, release, platform)
    else:
        (name, version, release, platform) = split_container_name(name)

    return (name, version, release, platform)

def split_container_name(name):
    global IS_CONTAINER

    m = re.search(r'(.*)-container-(.*)-(.*)', name)
    if m:
        (name, version, release) = m.groups()
        IS_CONTAINER = True
        return (name, version, release, "el8")
    else:
        log.error('Invalid name: %s' % name)
        add_to_log("ERROR: Invalid name: %s\n" % name)
        update_state("error")
        log.shutdown()
        sys.exit(2)

def atoi(text):
    return int(text) if text.isdigit() else text

def natural_keys(text):
    '''
    alist.sort(key=natural_keys) sorts in human order
    http://nedbatchelder.com/blog/200712/human_sorting.html
    (See Toothy's implementation in the comments)
    '''
    return [ atoi(c) for c in re.split(r'(\d+)', text) ]

def get_info_from_brew(check_pkg):
    global LAST_NVR, COMPLETION_TIME, VOLUME_ID, IS_CONTAINER

    log.info("Pulling info about " + check_pkg + " from brew.")
    add_to_log("INFO: Pulling info about " + check_pkg + " from brew.\n")

    # Get info about build
    try:
        checked_build = brew_proxy.getBuild(check_pkg)
    except Exception as e:
        log.error("NVR - " + str(e))
        add_to_log("ERROR: NVR - " + str(e) + "\n")
        update_state("error")
        sys.exit(2)

    # Check for output of brew query
    if checked_build == None:
        log.error("Invalid NVR")
        add_to_log("ERROR: Invalid NVR\n")
        update_state("error")
        sys.exit(2)

    if checked_build['state'] != 1:
        log.error("Build for this NVR is running, has been canceled or failed.")
        add_to_log("ERROR: Build for this NVR is running, has been canceled or failed.\n")
        update_state("error")
        sys.exit(2)

    update_db("nvr", check_pkg)
    update_db("package", checked_build['package_name'])

    if IS_CONTAINER == False:
        if "source" not in checked_build['extra']:
            log.error("No repo URL to check.")
            add_to_log("ERROR: No repo URL to check.\n")
            update_state("error")
            sys.exit(2)

        repo_name = re.findall(r'rpms/(.+)#', checked_build['extra']['source']['original_url'])[0]
    else:
        if "source" not in checked_build:
            log.error("No repo URL to check.")
            add_to_log("ERROR: No repo URL to check.\n")
            update_state("error")
            sys.exit(2)

        repo_name = re.findall(r'containers/(.+)#', checked_build['source'])[0]
            

    last_commit = None

    repo_name = repo_name.replace("?", "")

    # Get info about released versions
    log.info("Pulling info about latest build for " + checked_build['name'] + " from brew.")
    add_to_log("INFO: Pulling info about latest build for " + checked_build['name'] + " from brew.\n")

    for entry in brew_proxy.getLatestBuilds('rhel-' + pkg_platform[2] + '-latest-build', package=checked_build['name']):
        VOLUME_ID = entry['volume_id']
        LAST_NVR = entry['nvr']
        COMPLETION_TIME = entry['completion_time']
        if IS_CONTAINER == True:
            last_commit = brew_proxy.getBuild(LAST_NVR)['source']
        else:
            last_commit = brew_proxy.getBuild(LAST_NVR)['extra']['source']['original_url']

    if last_commit == None:
        log.info("No last NVR.")
        add_to_log("INFO: No last NVR.\n")
        if IS_CONTAINER == True:
            return "", checked_build['source'].split("#")[1], checked_build['package_id'], repo_name
        else:
            return "", checked_build['extra']['source']['original_url'].split("#")[1], checked_build['package_id'], repo_name

    if IS_CONTAINER == True:
        return last_commit.split("#")[1], checked_build['source'].split("#")[1], checked_build['package_id'], repo_name
    else:
        return last_commit.split("#")[1], checked_build['extra']['source']['original_url'].split("#")[1], checked_build['package_id'], repo_name

def pull_git_for_pkg(pkg_name, commit_from, commit_to, git_dir=None):
    global IS_CONTAINER

    if git_dir == None:
        log.info("Making temporary clone of " + pkg_name + " repository. This can take a while.")
        add_to_log("INFO: Making temporary clone of " + pkg_name + " repository. This can take a while.\n")

        uid = randomString(10)
        if IS_CONTAINER == True:
            repo = git.Repo.clone_from("git://pkgs.devel.redhat.com/containers/" + pkg_name + ".git", "/tmp/" + pkg_name + "-" + uid)
        else:
            repo = git.Repo.clone_from("git://pkgs.devel.redhat.com/rpms/" + pkg_name + ".git", "/tmp/" + pkg_name + "-" + uid)
    else:
        log.info("Checking older commits for " + pkg_name + " repository.")
        add_to_log("INFO: Checking older commits for " + pkg_name + " repository.\n")
        repo = git.Repo("/tmp/" + git_dir)

    bzs = {}
    lineREres = re.compile(r"^\s*-?\s*%s" % "Resolves?:", re.IGNORECASE)
    lineRErel = re.compile(r"^\s*-?\s*%s" % "Related?:", re.IGNORECASE)
    lineRErev = re.compile(r"^\s*-?\s*%s" % "Reverts?:", re.IGNORECASE)
    log.info("Checking commits ...")
    add_to_log("INFO: Checking commits ...\n")
    for commit in repo.iter_commits(commit_from+".."+commit_to):
        for line in list(commit.message.split("\n")):
            if lineREres.search(line) or lineRErel.search(line) or lineRErev.search(line):
                for bzID in BZRE.findall(line):
                    try:
                        bzID = int(bzID)
                        bzs[bzID] = 1
                    except ValueError:
                        pass

    log.info("Checking commits: DONE")
    add_to_log("INFO: Checking commits: DONE\n")

    if git_dir == None:
        return list(bzs.keys()), pkg_name + "-" + uid
    else:
        return list(bzs.keys()), git_dir

def find_previous_released(check_pkg, git_folder, pkg_id, base_commit, repo_name):
    global BAD_BZS, LAST_NVR

    builds = {}
    for bld in brew_proxy.listBuilds(pkg_id, state=1, volumeID=VOLUME_ID, completeAfter=COMPLETION_TIME):
        builds[bld['nvr']] = bld
    
    keys = list(builds.keys())
    keys.sort(key=natural_keys)

    prev_index = keys.index(check_pkg)-1 if check_pkg in keys else keys.index(keys[-1])
    if prev_index < 0:
        log.error("No previous suitable build found. Use baseline.")
        add_to_log("ERROR: No previous suitable build found. Use baseline.\n")
        update_db("safe_nvr", LAST_NVR)
        update_db("result", "1")
        update_db("end","now()")
        update_state("done")
        print(LAST_NVR)
        return 1

    previous_nvr = keys[prev_index]

    previous_build = builds[previous_nvr]

    if VOLUME_ID == None:
        for tag in brew_proxy.listTags(previous_build['build_id']):
            if "-released" in tag['name']:
                log.info("Following NVR has been released: " + previous_build['nvr'])
                add_to_log("INFO: Following NVR has been released: " + previous_build['nvr'] +"\n")
                update_db("safe_nvr", previous_build['nvr'])
                update_db("result", "1")
                update_db("end","now()")
                update_state("done")
                print(previous_build['nvr'])
                return 1

    bz_to_check, git_dir = pull_git_for_pkg(repo_name, base_commit, previous_build['extra']['source']['original_url'].split("#")[1], git_folder)

    skip = False
    for bbz in BAD_BZS:
        if bbz in bz_to_check:
            skip = True
            break;

    if skip == True:
        return find_previous_released(previous_nvr, git_folder, pkg_id, base_commit, repo_name)

    return check_bz(bz_to_check, previous_nvr, git_folder, pkg_id, base_commit, repo_name, False)

def check_bz(bzid_to_check, check_pkg, git_folder, pkg_id, base_commit, repo_name, parent=True):
    global BAD_BZS

    log.info("Checking related bug reports ...")
    add_to_log("INFO: Checking related bug reports ... \n")

    for bug in bzid_to_check:
        toContinue = False
        while True:
            try:
                bToCheck = bugs.getbug(bug)
                break
            except Exception as e:
                if "502 Server" not in str(e):
                    log.warning("#" + str(bug) + " - " + str(e))
                    add_to_log("WARNING: #" + str(bug) + " - " + str(e).replace("'","\"") + "\n")
                    toContinue = True
                    break

        if toContinue == True:
            continue

        # Check if CVE- is in summary of the bug report
        if "CVE-" in bToCheck.summary and bToCheck.resolution != "ERRATA":

            # Check severity and priority to be high or urgent
            if bToCheck.severity == "high" or bToCheck.severity == "urgent":

                # Is current bug for y-stream?
                if (hasattr(bToCheck, 'cf_zstream_target_release') == True and bToCheck.cf_zstream_target_release == "---") or hasattr(bToCheck, 'cf_zstream_target_release') == False:

                    # Separate akk CVE IDs and search for z-stream version of the bug
                    for cveID in CVERE.findall(bToCheck.summary):
                        oneOut = False
                        if IS_CONTAINER == True:
                            for otherBug in bugs.query(
                                {'f1': 'cf_zstream_target_release',
                                 'o1': 'isnotempty',
                                 'product': 'Red Hat Enterprise Linux ' + pkg_platform[2],
                                 'query_format': 'advanced',
                                 'short_desc': cveID,
                                 'short_desc_type': 'substring',
                                 'component': pkg_name + '-container' }):

                                if otherBug.id != bToCheck.id:
                                    if otherBug.resolution == "ERRATA":
                                        oneOut = True
                        else:
                            for otherBug in bugs.query(
                                {'f1': 'cf_zstream_target_release',
                                 'o1': 'isnotempty',
                                 'product': 'Red Hat Enterprise Linux ' + pkg_platform[2],
                                 'query_format': 'advanced',
                                 'short_desc': cveID,
                                 'short_desc_type': 'substring',
                                 'component': pkg_name }):

                                if otherBug.id != bToCheck.id:
                                    if otherBug.resolution == "ERRATA":
                                        oneOut = True

                        if oneOut == False:
                            log.error("No z-stream errata has been released for this CVE (" + cveID + ").")
                            add_to_log("ERROR: No z-stream errata has been released for this CVE (" + cveID + ").\n")
                            BAD_BZS.append(bug)
                            find_previous_released(check_pkg, git_folder, pkg_id, base_commit, repo_name)
                            if parent == True:
                                log.info("Removing temporary clone of repository.")
                                add_to_log("INFO: Removing temporary clone of repository.\n")
                                os.system("rm -rf /tmp/" + git_folder)
                                log.shutdown()
                            return 1
                else:
                    for cveID in CVERE.findall(bToCheck.summary):
                        oneOut = False
                        if IS_CONTAINER == True:
                            for otherBug in bugs.query(
                                {'f1': 'cf_zstream_target_release',
                                 'o1': 'isnotempty',
                                 'product': 'Red Hat Enterprise Linux ' + pkg_platform[2],
                                 'query_format': 'advanced',
                                 'short_desc': cveID,
                                 'short_desc_type': 'substring',
                                 'component': pkg_name }):

                                if otherBug.resolution == "ERRATA":
                                    oneOut = True
                        else:
                            for otherBug in bugs.query(
                                {'f1': 'cf_zstream_target_release',
                                 'o1': 'isnotempty',
                                 'product': 'Red Hat Enterprise Linux ' + pkg_platform[2],
                                 'query_format': 'advanced',
                                 'short_desc': cveID,
                                 'short_desc_type': 'substring',
                                 'component': pkg_name + '-container'}):
    
                                if otherBug.resolution == "ERRATA":
                                    oneOut = True

                        if oneOut == False:
                            log.error("No z-stream errata has been released for this CVE (" + cveID + ").")
                            add_to_log("ERROR: No z-stream errata has been released for this CVE (" + cveID + ").\n")
                            BAD_BZS.append(bug)
                            find_previous_released(check_pkg, git_folder, pkg_id, base_commit, repo_name)
                            if parent == True:
                                log.info("Removing temporary clone of repository.")
                                add_to_log("INFO: Removing temporary clone of repository.\n")
                                os.system("rm -rf /tmp/" + git_folder)
                                log.shutdown()
                            return 1

    log.info("Checking related bug reports: DONE")
    log.info("NVR " + check_pkg + " allowed.")
    add_to_log("INFO: Checking related bug reports: DONE\nINFO: NVR " + check_pkg + " allowed.\n")
    update_db("safe_nvr", check_pkg)
    update_db("result", "0")
    update_db("end","now()")
    update_state("done")
    print(check_pkg)
    if parent == True:
       log.info("Removing temporary clone of repository.")
       add_to_log("INFO: Removing temporary clone of repository.\n")
       os.system("rm -rf /tmp/" + git_folder)
       log.shutdown()
    return 0

def check_nvr(check_pkg):
    commit_from, commit_to, pkg_id, repo_name = get_info_from_brew(check_pkg)

    if commit_from == commit_to:
        log.info("Current NVR is identical with released baseline.")
        add_to_log("INFO: Current NVR is identical with released baseline.\n")
        update_db("safe_nvr", check_pkg)
        update_db("result", "0")
        update_db("end","now()")
        update_state("done")
        print(check_pkg)
        log.shutdown()
        return 0

    bzid_to_check, git_folder = pull_git_for_pkg(repo_name, commit_from, commit_to)

    return check_bz(bzid_to_check, check_pkg, git_folder, pkg_id, commit_from, repo_name)

if __name__ == '__main__':

    state = {"init": "0", "running": "1", "done": "2", "error": "3"}

    usage = "usage: python3 %prog NVR"
    parser = optparse.OptionParser(usage)
    parser.add_option("-v", help="Enable log output (INFO)", dest="info", action="store_true", default=False)
    parser.add_option("-d", help="Enable log output (DEBUG)", dest="debug", action="store_true", default=False)
    parser.add_option("-l", help="Disable communication with database", dest="database", action="store_false", default=True)
    parser.add_option("-t", help="Results will be stored under this task id", dest="taskid", type="int", default=-1)

    options, args = parser.parse_args()

    if options.database == True:
        dbconfig = configparser.ConfigParser()
        dbconfig.read('database.conf')
        dtb = mydb.connect(
            host=os.environ.get("MYSQL_SERVICE_HOST", dbconfig['mysql']['host']),
            user=os.environ.get("DATABASE_USER", dbconfig['mysql']['user']),
            passwd=os.environ.get("DATABASE_PASSWORD", dbconfig['mysql']['password']),
            database=os.environ.get("DATABASE_NAME", dbconfig['mysql']['database'])
        )
        connection = dtb.cursor()
        if options.taskid < 0:
            connection.execute("INSERT INTO results (state) VALUES(" + state['init'] + ")")
            dtb.commit()
            connection.execute("SELECT LAST_INSERT_ID()")
            taskId = connection.fetchone()[0]
        else:
            taskId = options.taskid
    else:
        connection = None

    if options.debug == True:
        log.basicConfig(format="%(levelname)s: %(message)s", level=log.DEBUG)
        log.info("Debug output enabled")
        add_to_log("INFO: Debug output enabled\n")
    elif options.info == True:
        log.basicConfig(format="%(levelname)s: %(message)s", level=log.INFO)
        log.info("Info output enabled")
        add_to_log("INFO: Info output enabled\n")
    else:
        log.basicConfig(format="%(levelname)s: %(message)s")



    # Brew instance
    brew_proxy = brew.ClientSession("http://brewhub.engineering.redhat.com/brewhub/")

    # Check if argument passed
    if len(args) > 0:
        check_pkg = args[0]
    else:
        parser.error("No NVR passed")
        add_to_log("ERROR: No NVR passed")
        update_state(states['error'])
        log.shutdown()
        sys.exit(2)

    pkg_name, pkg_version, pkg_release, pkg_platform = split_rpm_name(check_pkg)

    bugs = bz.Bugzilla(url="https://bugzilla.redhat.com", api_key=os.environ.get('BZ_API_KEY', None))

    update_state("running")
    sys.exit(check_nvr(check_pkg))