check-ebuild-update

#!/bin/bash
# License: MIT or GPL2+
# Checks if a newer vanilla ebuild from various sources so this overlay can
# apply modifications to them.  For example we check the Gentoo overlay
# to see if there is a latest stable and apply multilib modifications to
# them.

# This script was created to eliminate some human error and bias.
# More recent package versions will have up to date security fixes if any.

# The results will clearly print which packages are outdated.  This
# can be used to compare the version differential.  Large differences
# may indicate the package may need to be pruned or it requires special
# attention for vulnerability checks, or it may indicate that the
# project has transfered ownership or to a new place.

# The pattern is that the longer the package has not updated, the more
# critical or high vulnerabilities it may accumulate over time.  This
# script will help catch these packages.

# Still work in progress

# _pYYYMMDD is broken if using timestamp because there is no standard timezone
# for YYMMDD in the gentoo spec, so there could be off by one errors.

# The overlay assumes local time PDT/PST for _YYMMDD.

# The technical spec: https://wiki.gentoo.org/wiki/Package_Manager_Specification

if [[ -f .check-ebuild-update.config ]] ; then
source .check-ebuild-update.config
fi

PORTAGE_DIR=${PORTAGE_DIR:-"/usr/portage"}
STABLE_ARCH=${STABLE_ARCH:-"amd64"}
SCRAPERS_FILE=${SCRAPERS_FILE:-"check-ebuild-update-scrapers"}
DEBUG=${DEBUG:-"0"}
REPO_TZ=${REPO_TZ:-"America/Los_Angeles"}
CACHE_EXPIRE=${CACHE_EXPIRE:-86400} # 1 day
CACHE_DIR=${CACHE_DIR:-"/tmp/check-ebuild-update"}

setup_cache() {
	mkdir -p "${CACHE_DIR}"
}

check_prereqs() {
	if [[ ! -d "${PORTAGE_DIR}" ]] ; then
		echo "Your PORTAGE_DIR cannot be found"
		exit 1
	fi

	if [[ "${PORTAGE_DIR}" != "/usr/portage" ]] ; then
		echo \
"Your PORTAGE_DIR is not /usr/portage.  You must check the cut commands are\n\
working properly."
		exit 1
	fi

	if grep --help | grep -q -P -e "--perl-regexp" ; then
		:;
	else
		echo "You need grep compiled with Perl regular expressions."
		exit 1
	fi

	if ! which git 2>/dev/null 1>/dev/null ; then
		echo "You need git"
		exit 1
	fi

	if ! which html2text 2>/dev/null 1>/dev/null ; then
		echo "You need html2text"
		exit 1
	fi

	if ! which wget 2>/dev/null 1>/dev/null ; then
		echo "You need wget"
		exit 1
	fi

	if ! which xmlstarlet 2>/dev/null 1>/dev/null ; then
		echo "You need xmlstarlet"
		exit 1
	fi
}

# version tests
# x < y
is_x_lt_y() {
	local x="${1}"
	local y="${2}"
	local result=$(echo -e "${x}\n${y}" | sort -V | tr "\n" " " \
		| cut -f 1 -d $' ')
	if [[ "${result}" == "${x}" ]] ; then
		return 0
	else
		return 1
	fi
}
is_x_le_y() {
	local x="${1}"
	local y="${2}"
	local result=$(echo -e "${x}\n${y}" | sort -V | tr "\n" " " \
		| cut -f 1 -d $' ')
	if [[ "${result}" == "${x}" || "${x}" == "${y}" ]] ; then
		return 0
	else
		return 1
	fi
}
is_x_gt_y() {
	local x="${1}"
	local y="${2}"
	local result=$(echo -e "${x}\n${y}" | sort -V | tr "\n" " " \
		| cut -f 1 -d $' ')
	if [[ "${result}" != "${x}" ]] ; then
		return 0
	else
		return 1
	fi
}
is_x_ge_y() {
	local x="${1}"
	local y="${2}"
	local result=$(echo -e "${x}\n${y}" | sort -V | tr "\n" " " \
		| cut -f 1 -d $' ')
	if [[ "${result}" != "${x}" || "${x}" == "${y}" ]] ; then
		return 0
	else
		return 1
	fi
}
is_x_eq_y() {
	local x="${1}"
	local y="${2}"
	if [[ "${x}" == "${y}" ]] ; then
		return 0
	else
		return 1
	fi
}
is_x_ne_y() {
	local x="${1}"
	local y="${2}"
	if [[ "${x}" != "${y}" ]] ; then
		return 0
	else
		return 1
	fi
}

is_x_op_y() {
	local x="${1}"
	local op="${2}"
	local y="${3}"

	if [[ "${op}" == "-lt" ]] ; then
		is_x_lt_y "${x}" "${y}"
	elif [[ "${op}" == "-le" ]] ; then
		is_x_le_y "${x}" "${y}"
	elif [[ "${op}" == "-gt" ]] ; then
		is_x_gt_y "${x}" "${y}"
	elif [[ "${op}" == "-ge" ]] ; then
		is_x_ge_y "${x}" "${y}"
	elif [[ "${op}" == "-eq" ]] ; then
		is_x_eq_y "${x}" "${y}"
	elif [[ "${op}" == "-ne" ]] ; then
		is_x_ne_y "${x}" "${y}"
	fi
}

get_stable_versions() {
	arch="${1}"
	for ebuild in $(find "${PORTAGE_DIR}/${category}/${pn}" \
			-name "*.ebuild" 2>/dev/null \
			| tr " " "\n" ) ; do
		if grep -q -P -e "KEYWORDS=\".*(?<!\~)${arch}" "${ebuild}"
		then
			echo "${ebuild}"
		fi
	done
}

get_newest_pv() {
	local uri="${1}"
	local r=$(git ls-remote --tags "${uri}" \
		| sed -e "s|refs/tags/|;|g" \
		| cut -f 2 -d ";" \
		| sed -e "s|^v||g" -e "s|[\^\{\}]+||g" \
		| grep -v -G -e "[\^]" \
		| sed -r -e "s#-(alpha|beta|rc|pre)\.?#_\1#g" \
		| sed -e "s|_preview|_pre|" \
		| esort \
		| tr " " "\n" \
		| tail -n 1)
	if echo "${r}" | grep -G -e "^[^0-9]" ; then
		echo "${category}/${pn} has tag that is not a version r=${r}"
		exit 1
	fi
	if [[ -z "${r}" ]] ; then
		echo "${category}/${pn} has an empty pv"
		exit 1
	fi
	echo "${r}"
}

# gets all tags without prefixed v
# used when more variations of alpha/beta/pre or non standard versioning encountered
get_tags() {
	local uri="${1}"
	local r=$(git ls-remote --tags "${uri}" \
		| sed -e "s|refs/tags/|;|g" \
		| cut -f 2 -d ";" \
		| grep -v -G -e "[\^]" \
		| sed -r -e "s|^v||g" -e "s|[\^\{\}]+||g" \
		| sed -r -e "s#-(alpha|beta|rc|pre)\.?#_\1#g" \
		| sed -e "s|_preview|_pre|")
	if [[ -z "${r}" ]] ; then
		echo "${category}/${pn} has no tags"
		exit 1
	fi
	echo "${r}"
}

# Demote _beta, _alpha, _rc, _pre, etc for sort -V
# You need to call `| tr " " "\n"` immediately afterwards.
demote_prereleases() {
      	ARG=$(</dev/stdin)
        echo -e $(echo -e "${ARG}" \
		| sed -r -e 's#([0-9]+)$#\1___#g' \
		| sed -r -e "s#_(rc|beta|alpha|pre|p)#z\1#")
}

# strip hinter
strip_hinter() {
      	ARG=$(</dev/stdin)
        echo -e $(echo -e "${ARG}" \
		| sed -r -e 's#([0-9]+)___$#\1#g' \
		| sed -r -e "s#z(rc|beta|alpha|pre|p)#_\1#")
}

# You need to call `| tr " " "\n"` immediately afterwards.
esort() {
      	ARG=$(</dev/stdin)
	echo -e $(echo -e "${ARG}" \
		| demote_prereleases \
		| tr " " "\n" \
		| sort -V \
		| strip_hinter)
}

get_head_suffix() {
	local domain="${1}"
	local org_project="${2}"
	local branch="${3}"
	# github reports local timestamp on www (Firefox) but on wget it is UTC/zulu
	echo $(TZ="America/Los_Angeles" date -d "$(wget -q -O - \
https://${domain}/${org_project}/commit/$(git ls-remote --heads \
https://${domain}/${org_project}.git \
			| grep -F -e "${branch}" \
			| cut -f 1 -d $'\t') \
		| grep -F -e "relative-time" \
		| grep -G -o -e "datetime=\".*\" " \
		| sed -e "s|datetime=\"||g" -e "s|\" ||g")" +"%Y%m%d")
}

get_head_suffix_rss() {
	local url="${1}"
	wget -q -O - "${1}" \
		| grep -F -e "pubDate" \
		| sed -r -e "s|[</]+pubDate[>]||g" \
		| head -n 1 \
		| sed -r -e "s|^[ ]+||g"
}

source "${SCRAPERS_FILE}"

edebug() {
	local message="${1}"
	if [[ -n "${DEBUG}" && "${DEBUG}" == "1" ]] ; then
		echo "${message}"
	fi
}

gen_save_pv() {
	mkdir -p "${CACHE_DIR}/${type}/${category}/${pn}"
	rm -rf "${CACHE_DIR}/${type}/${category}/${pn}/${oiledmachine_pv}"
		"${fn}" \
		> "${CACHE_DIR}/${type}/${category}/${pn}/${oiledmachine_pv}"
	cat "${CACHE_DIR}/${type}/${category}/${pn}/${oiledmachine_pv}"
}

get_pv() {
	local fn="${1}"
	local type="${2}" # can be gentoo or repos
	# can't use SLOT because it uses ver_cut and variables
	if [[ -f "${CACHE_DIR}/${type}/${category}/${pn}/${oiledmachine_pv}" ]] ; then
		local datec=$(stat -c "%W" \
			"${CACHE_DIR}/${type}/${category}/${pn}/${oiledmachine_pv}")
		local now=$(date +"%s")
		if (( ${now} < ${datec} + ${CACHE_EXPIRE} )) ; then
			cat "${CACHE_DIR}/${type}/${category}/${pn}/${oiledmachine_pv}"
		else
			gen_save_pv
		fi
	else
		gen_save_pv
	fi
}

scan_repos() {
	echo "Checking projects"
	for ebuild in $(find . -name "*.ebuild") ; do
		local category=$(echo "${ebuild}" | cut -f 2 -d "/")
		local ebuild_filename=$(basename "${ebuild}")
		local pn=$(echo "${ebuild}" | cut -f 3 -d "/")
		local oiledmachine_pv=$(echo "${ebuild_filename}" \
			| sed -r -e "s|${pn}-||g" -e "s|\.ebuild$||")
		local repo_pv
		edebug "Inspecting ${category}/${pn}"
		for p in $(grep -E -e "^repo_latest_version_get_(${category}-)?" \
					"${SCRAPERS_FILE}" \
				| sed -r -e "s|\(\) \{||g" \
					-e "s|repo_latest_version_get_(${category}-)?||" \
					-e "s|_pv||g") ; do
			[[ "${p}" != "${pn}" ]] && continue

			if grep -q -F -e \
				"repo_latest_version_get_${category}-${pn}_pv" \
				"${SCRAPERS_FILE}" ; then
				repo_pv=$(get_pv repo_latest_version_get_${category}-${pn}_pv "repos")
				edebug "${repo_pv} > ${oiledmachine_pv}"
				if is_x_op_y ${repo_pv} "-gt" \
					"${oiledmachine_pv}"
				then
					echo \
"Found newer ${pn} repo_pv=${repo_pv} oiledmachine_pv=${oiledmachine_pv}"
				fi
			elif [[ "${pn}" == "tulua++" ]] ; then
				repo_pv=$(get_pv repo_latest_version_get_tuluaxx_pv)
				edebug "${repo_pv} > ${oiledmachine_pv}"
				if is_x_op_y ${repo_pv} "-gt" \
					"${oiledmachine_pv}"
				then
					echo \
"Found newer ${pn} repo_pv=${repo_pv} oiledmachine_pv=${oiledmachine_pv}"
				fi
			elif [[ "${pn}" == "noto-color-emoji" \
				|| "${pn}" == "noto-color-emoji-bin" ]] ; then
				repo_pv=$(get_pv repo_latest_version_get_${pn}_pv "repos")
				edebug "${repo_pv} > ${oiledmachine_pv}"
				oiledmachine_pv_date=$(echo "${oiledmachine_pv}" \
					| sed -r -e "s#_beta[0-9]+##g" \
					| sed -r -e "s#(_|-)#-#g" \
					| cut -f 2 -d "-" \
					| sed -r -e "s|[^0-9]||g")
				if is_x_op_y ${repo_pv} "-gt" \
					"${oiledmachine_pv_date}"
				then
					echo \
"Found newer ${pn} repo_pv=${repo_pv} oiledmachine_pv_date=${oiledmachine_pv_date}"
				fi
			elif [[ "${pn}" == "${p}" ]] ; then
				repo_pv=$(get_pv repo_latest_version_get_${pn}_pv "repos")
				edebug "${repo_pv} > ${oiledmachine_pv}"
				if is_x_op_y ${repo_pv} "-gt" "${oiledmachine_pv}"
				then
					echo \
"Found newer ${pn} repo_pv=${repo_pv} oiledmachine_pv=${oiledmachine_pv}"
				fi
			else
				edebug "Skipped ${category}/${pn}"
			fi
		done
	done
}

filter_by_slot() {
	local stable_versions=$(get_stable_versions "${STABLE_ARCH}")
	if [[ "${category}/${pn}" == "dev-lang/lua" ]] ; then
		echo "${stable_versions}" \
			| grep -F -e "^"$(echo "${oiledmachine_pv}" \
						| cut -f 1-2 -d ".")
	elif [[ "${category}/${pn}" == "dev-util/premake" ]] ; then
		echo "${stable_versions}" \
			| grep -F -e "^"$(echo "${oiledmachine_pv}" \
						| cut -f 1 -d ".")
	elif [[ "${category}/${pn}" == "media-gfx/blender" ]] ; then
		echo "${stable_versions}" \
			| grep -F -e "^"$(echo "${oiledmachine_pv}" \
						| cut -f 1-2 -d ".")
	elif [[ "${category}/${pn}" == "net-libs/nodejs" ]] ; then
		echo "${stable_versions}" \
			| grep -F -e "^"$(echo "${oiledmachine_pv}" \
						| cut -f 1 -d ".")
	else
		echo -e "${stable_versions}"
	fi
}

gen_gentoo_ver() {
	echo "${bn_newest}" | sed -r -e "s|${pn}-||g" -e "s|\.ebuild$||"
}

scan_gentoo_overlay() {
	echo "Checking the Gentoo overlay"
	for ebuild in $(find . -name "*.ebuild") ; do
		local category=$(echo "${ebuild}" | cut -f 2 -d "/")
		local ebuild_filename=$(basename "${ebuild}")
		local pn=$(echo "${ebuild}" | cut -f 3 -d "/")
		local oiledmachine_pv=$(echo "${ebuild_filename}" \
			| sed -r -e "s|${pn}-||g" -e "s|\.ebuild$||")
		local versions=$(filter_by_slot $(get_stable_versions \
							"${STABLE_ARCH}"))
		local newest=$(echo -e "${versions}" | tail -n 1)
		if (( ${#newest} > 0 )) ; then
			edebug "Inspecting ${category}/${pn}"
			local bn_newest=$(basename "${newest}")
			local gentoo_pv=$(get_pv gen_gentoo_ver "gentoo")

			edebug "${gentoo_pv} > ${oiledmachine_pv}"
			if is_x_op_y $(echo "${gentoo_pv}" | cut -f 1 -d "-") \
				"-gt" \
				$(echo "${oiledmachine_pv}" | cut -f 1 -d "-")
			then
				echo -e \
"Found ${pn} is newer on the Gentoo overlay oiledmachine_pv=${oiledmachine_pv} \
gentoo_pv=${gentoo_pv}"
			fi
		else
			edebug "Skipped ${category}/${pn}"
		fi
	done
}

main() {
	check_prereqs
	setup_cache
	scan_gentoo_overlay
	scan_repos
}

main