diff --git a/scripts/check-kernel-fix b/scripts/check-kernel-fix index 8ea73fe..6429f27 100755 --- a/scripts/check-kernel-fix +++ b/scripts/check-kernel-fix @@ -1,6 +1,9 @@ #!/bin/bash # vim: sw=4:sts=4:et +# TODO: Error handling is really not great. Especially failures from nested shells +# sucks. + . $(dirname "$0")/common-functions usage() @@ -10,6 +13,7 @@ usage() echo "Expect upstream kernel tree sha or CVE number as the parameter." echo "The script checks whether the commit is already in the upstream" echo "baseline or backported in kernel-source tree." + echo "Requires LINUX_GIT pointing to Linus git tree clone." echo echo "If backported, checks for CVE/bsc references and recommends adding these" echo "if they are missing. (Requires VULNS_GIT pointing to" @@ -40,6 +44,10 @@ usage() echo " -v: verbose mode: show state of each branch and even NOP actions" echo " -r: refresh any cached data. Use if cve->sha or cve->cvss fails" echo " (git pull VULNS_GIT, cve, bsc medata)" + echo " -f: flat mode. Do not filter output based on cvss scoring or common" + echo " ancestors." + echo " -s CVSS: override the CVSS score if known. This can be useful when" + echo " the CVE->CVSS DB is not synced yet." } branch= @@ -235,7 +243,8 @@ cvss_affects_branch() find_and_print_toplevel_actions() { local branch="$1" - local cvss="${2%%.*}" + local flat_mode=$2 + local cvss="${3%%.*}" local action_parameters= local merge_branch= local mb_line= @@ -245,11 +254,17 @@ find_and_print_toplevel_actions() local mb_state= [ -z "$branch" ] && fail "find_and_print_toplevel_actions: No branch provided" + [ ! -f $branch_state_file ] && fail "Bailing out" grep "^$branch:" "$branch_state_file" | \ while read line ; do - state=$(echo $line | cut -d: -f3) + if [ "$flat_mode" -eq 1 ] + then + print_action ${line//:/ } + continue + fi + state=$(echo $line | cut -d: -f3) # We only want to print branches which really need CVE fix backported # CVSS 9+ EB branches # CVSS 7+ LTSS branches @@ -306,8 +321,9 @@ find_and_print_toplevel_actions() verbose_mode= quiet_mode= +flat_mode=0 -while getopts "hvrq" OPT +while getopts "hvrqfs:" OPT do case $OPT in h) @@ -323,6 +339,11 @@ do q) quiet_mode=1 ;; + f) + flat_mode=1 + ;; + s) + cvss=$OPTARG esac done @@ -353,14 +374,15 @@ bsc= if [ -n "$cve" ] then bsc=$(cve2bugzilla $cve $refresh) - cvss=$(cve2cvss $cve $refresh) - echo "Security fix for $cve $bsc with CVSS ${cvss:-unknown}" + [ -z "$cvss" ] && cvss=$(cve2cvss $cve $refresh) + echo "Security fix for $cve $bsc with CVSS ${cvss:-unknown (assuming high impact), re-check later with -r}" else # emulate no CVE fix as CVSS==0. This will typically happen # for upstream commit with Fixes: which we want to target to # less conservative branches only - cvss=0 + [ -z "$cvss" ] && ccvss=0 fi + references="$cve $bsc" branches_conf="$(fetch_branches $refresh)" @@ -371,9 +393,5 @@ for_each_build_branch "$branches_conf" check_branch_state $sha $references # Newline after the dots showing progress [ -z "$quiet_mode" ] && echo -for_each_build_branch "$branches_conf" find_and_print_toplevel_actions $cvss - -if [ ! -e "$actions_file" ] ; then - echo "EVERYTHING IS OK!" -fi +for_each_build_branch "$branches_conf" find_and_print_toplevel_actions $flat_mode $cvss diff --git a/scripts/common-functions b/scripts/common-functions index 2cd4a78..161007f 100644 --- a/scripts/common-functions +++ b/scripts/common-functions @@ -8,7 +8,7 @@ fetch_cache() local EXPIRE=$3 local REFRESH=$4 - [ -n "$REFRESH" ] && rm "$CACHE_FILE" + [ -n "$REFRESH" ] && rm "$CACHE_FILE" 2>/dev/null if [[ $(find "$CACHE_FILE" -mtime -${EXPIRE:-7} -print 2>/dev/null) \ && -s "$CACHE_FILE" ]]; then echo $CACHE_FILE @@ -142,7 +142,7 @@ fail() branch_base_ver() { local branch="origin/$1" - git show-ref --verify --quiet "refs/remotes/${branch}" || fail "$branch invalid branch" + git show-ref --verify --quiet "refs/remotes/${branch}" || fail "$branch invalid branch. Please git fetch origin." local base_ver="v$(git grep SRCVERSION $branch -- rpm/config.sh | sed 's@.*=@@')" @@ -173,13 +173,20 @@ print_upstream_sha_summary() { local sha=$1 local upstream_git=${2:-$LINUX_GIT} + local has_fixes=0 print_upstream_sha_info $sha $upstream_git for fix in $(sha_get_upstream_git_fixes $1 $upstream_git) do echo -n "Fixes: " print_upstream_sha_info $fix $upstream_git + has_fixes=1 done + + if [ $has_fixes -eq 0 ] + then + echo "No Fixes tag. Requires manual review for affected branches." + fi } sha_merged_in_upstream_tag() @@ -201,7 +208,7 @@ sha_in_upstream() local upstream_git=${2:-$LINUX_GIT} [ -z "$sha" ] && fail "sha_in_upstream: No sha provided" - [ -z "$upstream_git" ] && fail "sha_in_upstream: No upstream git tree" + [ -z "$upstream_git" ] && fail "sha_in_upstream: No upstream git tree. LINUX_GIT should point to Linus git tree clone." sha_merged_in_upstream_tag $sha origin/master $upstream_git } @@ -274,7 +281,7 @@ sha_to_patch_in_branch() [ -z "$sha" ] && fail "sha_to_patch_in_branch: No sha provided" [ -z "$branch" ] && fail "sha_to_patch_in_branch: No branch provided" - branch_file=$(git --no-pager grep -l -i "^git-commit[[:space:]]*:[[:space:]]*$sha" "origin/$branch") + branch_file=$(git --no-pager grep -l -i "^git-commit[[:space:]]*:[[:space:]]*$sha" "origin/$branch" 2>/dev/null ) echo "${branch_file#origin/$branch:}" } diff --git a/scripts/cve_tools/.gitignore b/scripts/cve_tools/.gitignore new file mode 100644 index 0000000..773a6df --- /dev/null +++ b/scripts/cve_tools/.gitignore @@ -0,0 +1 @@ +*.dat diff --git a/scripts/cve_tools/Makefile b/scripts/cve_tools/Makefile new file mode 100644 index 0000000..d96676e --- /dev/null +++ b/scripts/cve_tools/Makefile @@ -0,0 +1,97 @@ +ifndef VULNS_GIT +$(error Please set VULNS_GIT to a clone of https://git.kernel.org/pub/scm/linux/security/vulns.git) +endif +ifndef KSOURCE_GIT +$(error Please set KSOURCE_GIT to a clone of kernel-source) +endif +mk_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + +# URL of upstream vulns +VULNS_GIT_URL=https://git.kernel.org/pub/scm/linux/security/vulns.git + +# URL of CVE +CVE2BUG_URL=https://gitlab.suse.de/security/cve-database/-/raw/master/data/cve2bugzilla + +# cached data expiration in days +EXPIRE=3 + +ADD_REF=$(mk_dir)/add-missing-reference + +# oldest stable 4.19 when kernel.org CNA started is from 2018 +FIRST_YEAR=2018 +# Outer parameter, can be overriden +YEAR=$(shell date +%Y) +# Outer parameter, can be overriden +BRANCH=cve/linux-5.14 +branch=$(subst /,_,$(BRANCH)) + +.PHONY: check_cache update_refs_history update_refs_$(branch)_$(YEAR) + +CACHESTAMP=cachestamp + +check_cache: + @test -n "$$(find "$(CACHESTAMP)" -mtime -$(EXPIRE) -print 2>/dev/null)" || touch "$(CACHESTAMP)" + +$(CACHESTAMP): check_cache + +cve2bugzilla: $(CACHESTAMP) + curl "$(CVE2BUG_URL)" >"$@" + +hash_cve_$(YEAR).dat: $(wildcard $(VULNS_GIT)/cve/published/$(YEAR)/*.sha1) $(VULNS_GIT)/.git/refs/heads/master + @for f in $^ ; do \ + [[ $$f == *.sha1 ]] && \ + echo $$(head -n1 $$f) $$(basename $${f%.sha1}) ; \ + done | sort -k1 >"$@" + +$(wildcard $(VULNS_GIT)/cve/published/$(YEAR)/*.sha1): $(VULNS_GIT)/.git/refs/heads/master + +$(VULNS_GIT)/.git/refs/heads/master: $(CACHESTAMP) + test -d $(VULNS_GIT) || git clone "$(VULNS_GIT_URL)" "$(VULNS_GIT)" + git -C "$(VULNS_GIT)" pull + touch "$@" + +# cve2bugzilla contains multiple bugs for single CVE, use the heuristics of +# lowest numerical bug being the primary bug +cve_bug_$(YEAR).dat: cve2bugzilla + @sed -n '/^CVE-$(YEAR)-.*BUGZILLA:/{s/^\(CVE-[^,]*\),.*BUGZILLA:\([0-9]*\).*$$/\1 \2/;p}' <"$<" | \ + sort -n -k 2 | sort -k 1b,1 -s | uniq | \ + awk '{ primary_bug=$$1 != cve; cve=$$1; print $$0, primary_bug; }' >"$@" + +hash_file_$(branch).dat: $(KSOURCE_GIT)/.git/refs/remotes/origin/$(BRANCH) + @git -C "$(KSOURCE_GIT)" \ + grep -i "^git-commit[[:space:]]*:[[:space:]]*[0-9a-f]*[[:space:]]*$$" origin/$(BRANCH) -- "$(KSOURCE_GIT)/patches.suse" |\ + awk -vFS=":" '{gsub(" ", "", $$4); print $$4, $$2}' | sort -k1 >"$@" + +hash_cve_bug_$(YEAR).dat: hash_cve_$(YEAR).dat cve_bug_$(YEAR).dat + @sort -k 2b,2 hash_cve_$(YEAR).dat | \ + join -1 2 -2 1 -o 1.1,1.2,2.2,2.3 - cve_bug_$(YEAR).dat | \ + sort -k 1 >"$@" + +update_refs: update_refs_$(branch)_$(YEAR) + +update_refs_history: + set -e; pushd "$(KSOURCE_GIT)" >/dev/null ; \ + git checkout -f -B users/$$USER/$(BRANCH)/cve-refs origin/$(BRANCH) 2>/dev/null ; \ + popd >/dev/null + for y in $$(seq $(FIRST_YEAR) $(YEAR)) ; do \ + make --no-print-directory -f $(mk_dir)/Makefile BRANCH=$(BRANCH) YEAR=$$y update_refs ; \ + done + +update_refs_$(branch)_$(YEAR): hash_file_$(branch).dat hash_cve_bug_$(YEAR).dat + @echo -e "\n[ $(YEAR) ] processing..." + @set -e ; \ + join hash_file_$(branch).dat hash_cve_bug_$(YEAR).dat | \ + while read sha file cve bug primary; do \ + [ "$$primary" -eq 0 ] && continue ; \ + [ -z "$$bug" ] && echo "Unknown bug for $$cve" && continue ; \ + $(ADD_REF) -r $$cve -r "bsc#"$$bug "$(KSOURCE_GIT)/$$file" ; \ + done + set -e ; pushd "$(KSOURCE_GIT)" >/dev/null ; \ + scripts/log2 --no-edit || true ; \ + popd >/dev/null + +clean: + rm -f *_$(branch).dat + for y in $$(seq $(FIRST_YEAR) $(YEAR)) ; do \ + rm -f *_$$y.dat ; \ + done diff --git a/scripts/cve_tools/README.md b/scripts/cve_tools/README.md new file mode 100644 index 0000000..12493c0 --- /dev/null +++ b/scripts/cve_tools/README.md @@ -0,0 +1,28 @@ +## Usage + +* Set `VULNS_GIT` environment variable to a clone of https://git.kernel.org/pub/scm/linux/security/vulns.git +* Set `KSOURCE_GIT` environment variable to a clone of kernel-source + * Fetch the repo to base work on up-to-date branches +* Pick a working directory `WD` (will store working data) + +* Run as +``` +cd $WD +make -f path/to/scripts/cve_tools/Makefile BRANCH=cve/linux-5.14-LTSS update_refs_history +``` + +* that will create a new git branch in `KSOURCE_GIT` and add commits with new + references +* it is recommended that `KSOURCE_GIT` is not same directory where + scripts/cve_tools/Makefile resides (e.g. use git worktrees) + * conversely `KSOURCE_GIT` cannot be a worktree (implementation issue) +* it will store processed data files in the `WD` + +## TODO + +* move working data from CWD to `XDG_CACHE_HOME` so that they can be used by + other utils +* integrate with branches.conf so that list of "root" branches is extracted +* integrate with branches.conf so that non-root branches are handled too (easy + if we allow multiplicities of RPM changelog messages) +* `git --git-dir="$(VULNS_GIT)/.git" pull` is broken, it adds files to $WD when fresh pull diff --git a/scripts/cve_tools/add-missing-reference b/scripts/cve_tools/add-missing-reference index 72af708..6655168 100755 --- a/scripts/cve_tools/add-missing-reference +++ b/scripts/cve_tools/add-missing-reference @@ -23,7 +23,8 @@ if __name__ == "__main__": with Patch(open(f, "r+b")) as patch: refs = "".join(patch.get("References")) refs = list(refs.replace(",", " ").split()) - new_refs = refs + [r for r in added_refs if not r in refs] + refs_lower = [r.lower() for r in refs] + new_refs = refs + [r for r in added_refs if not r.lower() in refs_lower] if new_refs == refs: continue patch.change("References", " ".join(new_refs))