diff options
Diffstat (limited to 'contrib/scripts')
| -rwxr-xr-x | contrib/scripts/checkpatch-feature-branch.sh | 21 | ||||
| -rwxr-xr-x | contrib/scripts/checkpatch.pl | 4 | ||||
| -rwxr-xr-x | contrib/scripts/find-backports | 336 | ||||
| -rwxr-xr-x | contrib/scripts/nm-ci-run.sh | 12 |
4 files changed, 354 insertions, 19 deletions
diff --git a/contrib/scripts/checkpatch-feature-branch.sh b/contrib/scripts/checkpatch-feature-branch.sh index 72e5903dae..c46fcd4b5a 100755 --- a/contrib/scripts/checkpatch-feature-branch.sh +++ b/contrib/scripts/checkpatch-feature-branch.sh @@ -13,19 +13,26 @@ if printf '%s' "$HEAD" | grep -q '\.\.'; then # Check the explicitly specified range from the argument. REFS=( $(git log --reverse --format='%H' "$HEAD") ) || die "not a valid range (HEAD is $HEAD)" else - BASE_REF="refs/remotes/origin/" + BASE_REF="refs/remotes/origin" + NM_UPSTREAM_REMOTE= if [ "$NM_CHECKPATCH_FETCH_UPSTREAM" == 1 ]; then - git remote add nm-upstream https://gitlab.freedesktop.org/NetworkManager/NetworkManager.git - git fetch nm-upstream || die "failure to fetch from https://gitlab.freedesktop.org/NetworkManager/NetworkManager.git" - BASE_REF="refs/remotes/nm-upstream/" + NM_UPSTREAM_REMOTE="nm-upstream-$(date '+%Y%m%d-%H%M%S')-$RANDOM" + git remote add "$NM_UPSTREAM_REMOTE" https://gitlab.freedesktop.org/NetworkManager/NetworkManager.git + BASE_REF="refs/remotes/$NM_UPSTREAM_REMOTE" + git fetch origin "$(git rev-parse "$HEAD")" --no-tags --unshallow + git fetch "$NM_UPSTREAM_REMOTE" \ + --no-tags \ + "refs/heads/master:$BASE_REF/master" \ + "refs/heads/nm-*:$BASE_REF/nm-*" \ + || die "failure to fetch from https://gitlab.freedesktop.org/NetworkManager/NetworkManager.git" fi # the argument is only a single ref (or the default "HEAD"). # Find all commits that branch off one of the stable branches or master # and lead to $HEAD. These are the commits of the feature branch. - RANGES=( $(git show-ref | sed 's#^\(.*\) '"$BASE_REF"'\(master\|nm-1-[0-9]\+\)$#\1..'"$HEAD"'#p' -n) ) + RANGES=( $(git show-ref | sed 's#^\(.*\) '"$BASE_REF/"'\(master\|nm-1-[0-9]\+\)$#\1..'"$HEAD"'#p' -n) ) [ "${#RANGES[@]}" != 0 ] || die "cannot detect git-ranges (HEAD is $(git rev-parse "$HEAD"))" @@ -36,6 +43,10 @@ else # stable nm-1-* branches. Just check the patch itself. REFS=( "$HEAD" ) fi + + if [ -n "$NM_UPSTREAM_REMOTE" ]; then + git remote remove "$NM_UPSTREAM_REMOTE" + fi fi SUCCESS=0 diff --git a/contrib/scripts/checkpatch.pl b/contrib/scripts/checkpatch.pl index 40518c8e72..c702810e22 100755 --- a/contrib/scripts/checkpatch.pl +++ b/contrib/scripts/checkpatch.pl @@ -137,7 +137,7 @@ if ($is_patch) { $check_line = 1; $line = $_; /^---$/ and $is_commit_message = 0; - /^Fixes: *(.*)/ and check_commit ($1, 1); + /^(Reverts|Fixes): *(.*)/ and check_commit ($2, 1); /This reverts commit/ and next; /cherry picked from/ and next; /\bcommit (.*)/ and check_commit ($1, 0); @@ -188,7 +188,7 @@ complain ("Don't use space inside elvis operator ?:") if $line =~ /\?[\t ]+:/; complain ("Don't add Emacs editor formatting hints to source files") if $line_no == 1 and $line =~ /-\*-.+-\*-/; complain ("XXX marker are reserved for development while work-in-progress. Use TODO or FIXME comment instead?") if $line =~ /\bXXX\b/; complain ("This gtk-doc annotation looks wrong") if $line =~ /\*.*\( *(transfer-(none|container|full)|allow none) *\) *(:|\()/; -complain ("Prefer nm_assert() or g_return*() to g_assert*()") if $line =~ /g_assert/ and not $filename =~ /\/tests\//; +complain ("Prefer nm_assert() or g_return*() to g_assert*()") if $line =~ /g_assert/ and (not $filename =~ /\/tests\//) and (not $filename =~ /\/nm-test-/); complain ("Use gs_free_error with GError variables") if $line =~ /\bgs_free\b +GError *\*/; #complain ("Use spaces instead of tabs") if $line =~ /\t/; diff --git a/contrib/scripts/find-backports b/contrib/scripts/find-backports new file mode 100755 index 0000000000..7f1b8ab8dd --- /dev/null +++ b/contrib/scripts/find-backports @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 + +import subprocess +import collections +import os +import sys +import re +import pprint + + +FNULL = open(os.devnull, "w") +pp = pprint.PrettyPrinter(indent=4) + + +def print_err(s): + print(s, file=sys.stderr) + + +def die(s): + print_err(s) + sys.exit(1) + + +def memoize(f): + memo = {} + + def helper(x): + if x not in memo: + memo[x] = f(x) + return memo[x] + + return helper + + +def re_bin(r): + return r.encode("utf8") + + +def _keys_to_dict(itr): + d = collections.OrderedDict() + for c in itr: + d[c] = None + return d + + +@memoize +def git_ref_exists(ref): + try: + out = subprocess.check_output( + ["git", "rev-parse", "--verify", str(ref) + "^{commit}"], stderr=FNULL + ) + except subprocess.CalledProcessError: + return None + try: + o = out.decode("ascii").strip() + if len(o) == 40: + return o + except Exception: + pass + raise Exception("git-rev-parse for '%s' returned unexpected output %s" % (ref, out)) + + +@memoize +def git_get_head_name(ref): + out = subprocess.check_output( + ["git", "rev-parse", "--symbolic-full-name", str(ref)], stderr=FNULL + ) + return out.decode("utf-8").strip() + + +def git_merge_base(a, b): + out = subprocess.check_output(["git", "merge-base", str(a), str(b)], stderr=FNULL) + out = out.decode("ascii").strip() + assert git_ref_exists(out) + return out + + +def git_all_commits_grep(rnge, grep=None): + if grep: + grep = ["--grep=%s" % (str(grep))] + notes = ["-c", "notes.displayref=refs/notes/bugs"] + else: + grep = [] + notes = [] + out = subprocess.check_output( + ["git"] + + notes + + ["log", "--pretty=%H", "--notes", "--reverse"] + + grep + + [str(rnge)], + stderr=FNULL, + ) + return list([x for x in out.decode("ascii").split("\n") if x]) + + +def git_logg(commits): + commits = list(commits) + if not commits: + return "" + out = subprocess.check_output( + [ + "git", + "log", + "--no-show-signature", + "--no-walk", + "--pretty=format:%Cred%h%Creset - %Cgreen(%ci)%Creset [%C(yellow)%an%Creset] %s%C(yellow)%d%Creset", + "--abbrev-commit", + "--date=local", + ] + + list([str(c) for c in commits]), + stderr=FNULL, + ) + return out.decode("utf-8").strip() + + +@memoize +def git_all_commits(rnge): + return git_all_commits_grep(rnge) + + +def git_commit_sorted(commits): + commits = list(commits) + if not commits: + return [] + out = subprocess.check_output( + ["git", "log", "--no-walk", "--pretty=%H", "--reverse"] + + list([str(x) for x in commits]), + stderr=FNULL, + ) + out = out.decode("ascii") + return list([x for x in out.split("\n") if x]) + + +@memoize +def git_ref_commit_body(ref): + return subprocess.check_output( + [ + "git", + "-c", + "notes.displayref=refs/notes/bugs", + "log", + "-n1", + "--pretty=%B%n%N", + str(ref), + ], + stderr=FNULL, + ) + + +@memoize +def git_ref_commit_body_get_fixes(ref): + body = git_ref_commit_body(ref) + result = [] + for mo in re.finditer(re_bin("\\b[fF]ixes: *([0-9a-z]+)\\b"), body): + c = mo.group(1).decode("ascii") + h = git_ref_exists(c) + if h: + result.append(h) + return result + + +@memoize +def git_ref_commit_body_get_cherry_picked_one(ref): + ref = git_ref_exists(ref) + if not ref: + return None + body = git_ref_commit_body(ref) + result = None + for r in [ + re_bin("\(cherry picked from commit ([0-9a-z]+)\)"), + re_bin("\\bIgnore-Backport: *([0-9a-z]+)\\b"), + ]: + for mo in re.finditer(r, body): + c = mo.group(1).decode("ascii") + h = git_ref_exists(c) + if h: + if not result: + result = [h] + else: + result.append(h) + return result + + +@memoize +def git_ref_commit_body_get_cherry_picked_recurse(ref): + ref = git_ref_exists(ref) + if not ref: + return None + + def do_recurse(result, ref): + result2 = git_ref_commit_body_get_cherry_picked_one(ref) + if result2: + extra = list([h2 for h2 in result2 if h2 not in result]) + if extra: + result.extend(extra) + for h2 in extra: + do_recurse(result, h2) + + result = [] + do_recurse(result, ref) + return result + + +def git_commits_annotate_fixes(rnge): + commits = git_all_commits(rnge) + c_dict = _keys_to_dict(commits) + for c in git_all_commits_grep(rnge, grep="[Ff]ixes:"): + ff = git_ref_commit_body_get_fixes(c) + if ff: + c_dict[c] = ff + return c_dict + + +def git_commits_annotate_cherry_picked(rnge): + commits = git_all_commits(rnge) + c_dict = _keys_to_dict(commits) + for c in git_all_commits_grep(ref_head, grep="cherry picked from commit"): + ff = git_ref_commit_body_get_cherry_picked_recurse(c) + if ff: + c_dict[c] = ff + return c_dict + + +if __name__ == "__main__": + if len(sys.argv) <= 1: + ref_head0 = "HEAD" + else: + ref_head0 = sys.argv[1] + + ref_head = git_ref_exists(ref_head0) + if not ref_head: + die('Ref "%s" does not exist' % (ref_head0)) + + ref_upstreams = [] + if len(sys.argv) <= 2: + head_name = git_get_head_name(ref_head0) + match = False + if head_name: + match = re.match("^refs/(heads|remotes/[^/]*)/nm-1-([0-9]+)$", head_name) + if match: + i = int(match.group(2)) + while True: + i += 2 + r = "nm-1-" + str(i) + if not git_ref_exists(r): + r = "refs/remotes/origin/nm-1-" + str(i) + if not git_ref_exists(r): + break + ref_upstreams.append(r) + ref_upstreams.append("master") + + if not ref_upstreams: + if len(sys.argv) <= 2: + ref_upstreams = ["master"] + else: + ref_upstreams = list(sys.argv[2:]) + + for h in ref_upstreams: + if not git_ref_exists(h): + die('Upstream ref "%s" does not exist' % (h)) + + print_err("Check %s (%s)" % (ref_head0, ref_head)) + print_err("Upstream refs: %s" % (ref_upstreams)) + + print_err('Check patches of "%s"...' % (ref_head)) + own_commits_list = git_all_commits(ref_head) + own_commits_cherry_picked = git_commits_annotate_cherry_picked(ref_head) + + cherry_picks_all = collections.OrderedDict() + for c, cherry_picked in own_commits_cherry_picked.items(): + if cherry_picked: + for c2 in cherry_picked: + l = cherry_picks_all.get(c2) + if not l: + cherry_picks_all[c2] = [c] + else: + l.append(c) + + own_commits_cherry_picked_flat = set() + for c, p in own_commits_cherry_picked.items(): + own_commits_cherry_picked_flat.add(c) + if p: + own_commits_cherry_picked_flat.update(p) + + # print(">>> own_commits_cherry_picked") + # pp.pprint(own_commits_cherry_picked) + + # print(">>> cherry_picks_all") + # pp.pprint(cherry_picks_all) + + # find all commits on the upstream branches that fix another commit. + fixing_commits = {} + for ref_upstream in ref_upstreams: + ref_str = ref_head + ".." + ref_upstream + print_err('Check upstream patches "%s"...' % (ref_str)) + for c, fixes in git_commits_annotate_fixes(ref_str).items(): + # print(">>> test %s ==> %s" % (c, fixes)) + if not fixes: + # print(">>> test %s ==> SKIP" % (c)) + continue + if c in cherry_picks_all: + # commit 'c' is already backported. Skip it. + # print(">>> in cherry_picks_all") + continue + for f in fixes: + if f not in own_commits_cherry_picked_flat: + # commit "c" fixes commit "f", but this is not one of our own commits + # and not interesting. + # print(">>> fixes %s not in own_commits_cherry_picked" % (f)) + continue + # print(">>> take %s (fixes %s)" % (c, fixes)) + fixing_commits[c] = fixes + break + + extra = collections.OrderedDict( + [(c, git_ref_commit_body_get_cherry_picked_recurse(c)) for c in fixing_commits] + ) + extra2 = [] + for c in extra: + is_back = False + for e_v in extra.values(): + if c in e_v: + is_back = True + break + if not is_back: + extra2.append(c) + + commits_good = list(fixing_commits) + commits_good = extra2 + + commits_good = git_commit_sorted(commits_good) + + print_err(git_logg(commits_good)) + + for c in reversed(commits_good): + print("%s" % (c)) diff --git a/contrib/scripts/nm-ci-run.sh b/contrib/scripts/nm-ci-run.sh index 52dffe2672..4d46ddb018 100755 --- a/contrib/scripts/nm-ci-run.sh +++ b/contrib/scripts/nm-ci-run.sh @@ -67,12 +67,6 @@ case "$CI" in ""|"true"|"default"|"gitlab") CI=default ;; - "travis") - _WITH_WERROR=0 - _WITH_LIBTEAM="$_FALSE" - _WITH_DOCS="$_FALSE" - _WITH_SYSTEMD_LOGIND="$_FALSE" - ;; *) die "invalid \$CI \"$CI\"" ;; @@ -172,12 +166,6 @@ run_autotools() { export NM_TEST_CLIENT_CHECK_L10N=1 - if [ "$CI" == travis ]; then - # travis is known to generate the settings doc differently. - # Don't compare. - export NMTST_NO_CHECK_SETTINGS_DOCS=yes - fi - if ! make check -j 6 -k ; then _print_test_logs "first-test" |