Git hook for renamed/added/deleted files + flake8 print() + tidelift (#1704)

16 files changed, 230 insertions, 173 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 41c58a93..a0b8f1f0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -1,4 +1,4 @@
-freebsd_12_1_py3_task:
+freebsd_13_py3_task:
   freebsd_instance:
     image: freebsd-12-1-release-amd64
   env:
@@ -11,10 +11,10 @@ freebsd_12_1_py3_task:
     - make install
     - make test
     - make test-memleaks
-    - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_access_denied.py
-    - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_api_speed.py
+    - make print-access-denied
+    - make print-api-speed
 
-freebsd_12_1_py2_task:
+freebsd_11_py2_task:
   freebsd_instance:
     image: freebsd-12-1-release-amd64
   env:
@@ -27,5 +27,5 @@ freebsd_12_1_py2_task:
     - make install
     - make test
     - make test-memleaks
-    - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_access_denied.py
-    - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_api_speed.py
+    - make print-access-denied
+    - make print-api-speed
diff --git a/.flake8 b/.flake8
new file mode 100644
index 00000000..6581552a
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,10 @@
+# Configuration file for flake 8. This is used by "make lint" and by the
+# GIT commit hook script.
+# T001 = print() statement
+
+[flake8]
+per-file-ignores =
+   setup.py:T001
+   scripts/*:T001
+   psutil/tests/runner.py:T001
+   psutil/tests/test_memory_leaks.py:T001
diff --git a/MANIFEST.in b/MANIFEST.in
index 801139b2..380a4fa2 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,6 @@
 include .cirrus.yml
 include .coveragerc
+include .flake8
 include .gitignore
 include CREDITS
 include HISTORY.rst
@@ -110,7 +111,6 @@ include scripts/disk_usage.py
 include scripts/fans.py
 include scripts/free.py
 include scripts/ifconfig.py
-include scripts/internal/.git-pre-commit
 include scripts/internal/README
 include scripts/internal/bench_oneshot.py
 include scripts/internal/bench_oneshot_2.py
@@ -118,11 +118,13 @@ include scripts/internal/check_broken_links.py
 include scripts/internal/clinter.py
 include scripts/internal/fix_flake8.py
 include scripts/internal/generate_manifest.py
+include scripts/internal/git_pre_commit.py
 include scripts/internal/print_access_denied.py
 include scripts/internal/print_announce.py
 include scripts/internal/print_api_speed.py
 include scripts/internal/print_timeline.py
 include scripts/internal/purge_installation.py
+include scripts/internal/tidelift.py
 include scripts/internal/win_download_wheels.py
 include scripts/internal/winmake.py
 include scripts/iotop.py
diff --git a/Makefile b/Makefile
index b15aa5d9..5c4d5b70 100644
--- a/Makefile
+++ b/Makefile
@@ -11,6 +11,7 @@ DEPS = \
 	check-manifest \
 	coverage \
 	flake8 \
+	flake8-print \
 	pyperf \
 	requests \
 	setuptools \
@@ -170,7 +171,7 @@ test-coverage:  ## Run test coverage.
 # ===================================================================
 
 lint-py:  ## Run Python (flake8) linter.
-	@git ls-files '*.py' | xargs $(PYTHON) -m flake8
+	@git ls-files '*.py' | xargs $(PYTHON) -m flake8 --config=.flake8
 
 lint-c:  ## Run  C linter.
 	@git ls-files '*.c' '*.h' | xargs $(PYTHON) scripts/internal/clinter.py
@@ -186,18 +187,18 @@ fix-lint:  ## Attempt to automatically fix some Python lint issues.
 # GIT
 # ===================================================================
 
-git-tag-release:  ## Git-tag a new release.
-	git tag -a release-`python -c "import setup; print(setup.get_version())"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD`
-	git push --follow-tags
-
 install-git-hooks:  ## Install GIT pre-commit hook.
-	ln -sf ../../scripts/internal/.git-pre-commit .git/hooks/pre-commit
+	ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit
 	chmod +x .git/hooks/pre-commit
 
 # ===================================================================
 # Distribution
 # ===================================================================
 
+git-tag-release:  ## Git-tag a new release.
+	git tag -a release-`python -c "import setup; print(setup.get_version())"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD`
+	git push --follow-tags
+
 sdist:  ## Create tar.gz source distribution.
 	${MAKE} generate-manifest
 	$(PYTHON) setup.py sdist
@@ -225,6 +226,9 @@ check-sdist:  ## Create source distribution and checks its sanity (MANIFEST)
 	build/venv/bin/python -m pip install -v --isolated --quiet dist/*.tar.gz
 	build/venv/bin/python -c "import os; os.chdir('build/venv'); import psutil"
 
+tidelift-relnotes:  ## upload release notes from HISTORY
+	$(PYTHON) scripts/internal/tidelift.py
+
 pre-release:  ## Check if we're ready to produce a new release.
 	${MAKE} check-sdist
 	${MAKE} install
@@ -245,6 +249,7 @@ release:  ## Create a release (down/uploads tar.gz, wheels, git tag release).
 	${MAKE} pre-release
 	$(PYTHON) -m twine upload dist/*  # upload tar.gz and Windows wheels on PyPI
 	${MAKE} git-tag-release
+	${MAKE} tidelift-relnotes
 
 check-manifest:  ## Inspect MANIFEST.in file.
 	$(PYTHON) -m check_manifest -v $(ARGS)
diff --git a/docs/index.rst b/docs/index.rst
index c6b9a8cc..7233793f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -2492,8 +2492,7 @@ Running tests
 Development guide
 =================
 
-If you want to hacking on psutil (e.g. want to add a new feature or fix a bug)
-take a look at the `development guide`_.
+If you want to develop psutil take a look at the `development guide`_.
 
 Platforms support history
 =========================
diff --git a/psutil/__init__.py b/psutil/__init__.py
index 22bb46f3..18fb1f13 100644
--- a/psutil/__init__.py
+++ b/psutil/__init__.py
@@ -224,7 +224,6 @@ __all__ = [
     "users", "boot_time",                                           # others
 ]
 
-
 __all__.extend(_psplatform.__extra__all__)
 __author__ = "Giampaolo Rodola'"
 __version__ = "5.7.0"
@@ -2348,7 +2347,7 @@ def test():  # pragma: no cover
     templ = "%-10s %5s %5s %7s %7s %5s %6s %6s %6s  %s"
     attrs = ['pid', 'memory_percent', 'name', 'cmdline', 'cpu_times',
              'create_time', 'memory_info', 'status', 'nice', 'username']
-    print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "NICE",
+    print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "NICE",  # NOQA
                    "STATUS", "START", "TIME", "CMDLINE"))
     for p in process_iter(attrs, ad_value=None):
         if p.info['create_time']:
@@ -2398,7 +2397,7 @@ def test():  # pragma: no cover
             ctime,
             cputime,
             cmdline)
-        print(line[:get_terminal_size()[0]])
+        print(line[:get_terminal_size()[0]])  # NOQA
 
 
 del memoize, memoize_when_activated, division, deprecated_method
diff --git a/psutil/_common.py b/psutil/_common.py
index 17b6eeb3..8a39de7f 100644
--- a/psutil/_common.py
+++ b/psutil/_common.py
@@ -801,9 +801,9 @@ def hilite(s, color="green", bold=False):
 def print_color(s, color="green", bold=False, file=sys.stdout):
     """Print a colorized version of string."""
     if not term_supports_colors():
-        print(s, file=file)
+        print(s, file=file)  # NOQA
     elif POSIX:
-        print(hilite(s, color, bold), file=file)
+        print(hilite(s, color, bold), file=file)  # NOQA
     else:
         import ctypes
 
@@ -827,7 +827,7 @@ def print_color(s, color="green", bold=False, file=sys.stdout):
         handle = GetStdHandle(handle_id)
         SetConsoleTextAttribute(handle, color)
         try:
-            print(s, file=file)
+            print(s, file=file)    # NOQA
         finally:
             SetConsoleTextAttribute(handle, DEFAULT_COLOR)
 
@@ -839,7 +839,7 @@ if bool(os.getenv('PSUTIL_DEBUG', 0)):
         """If PSUTIL_DEBUG env var is set, print a debug message to stderr."""
         fname, lineno, func_name, lines, index = inspect.getframeinfo(
             inspect.currentframe().f_back)
-        print("psutil-debug [%s:%s]> %s" % (fname, lineno, msg),
+        print("psutil-debug [%s:%s]> %s" % (fname, lineno, msg),  # NOQA
               file=sys.stderr)
 else:
     def debug(msg):
diff --git a/psutil/_compat.py b/psutil/_compat.py
index a9371382..2965fd1b 100644
--- a/psutil/_compat.py
+++ b/psutil/_compat.py
@@ -117,7 +117,7 @@ else:
             pass
         except OSError:
             raise RuntimeError(
-                "broken / incompatible Python implementation, see: "
+                "broken or incompatible Python implementation, see: "
                 "https://github.com/giampaolo/psutil/issues/1659")
 
 
diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c
index 07578eda..d63b4d9c 100644
--- a/psutil/_psutil_common.c
+++ b/psutil/_psutil_common.c
@@ -391,7 +391,6 @@ double
 psutil_FiletimeToUnixTime(FILETIME ft) {
     return _to_unix_time((ULONGLONG)ft.dwHighDateTime,
                          (ULONGLONG)ft.dwLowDateTime);
-
 }
 
 
diff --git a/psutil/arch/netbsd/socks.c b/psutil/arch/netbsd/socks.c
index f370f094..08b0b7e6 100644
--- a/psutil/arch/netbsd/socks.c
+++ b/psutil/arch/netbsd/socks.c
@@ -154,7 +154,7 @@ psutil_get_files(void) {
     // debug
     struct kif *k;
     SLIST_FOREACH(k, &kihead, kifs) {
-            printf("%d\n", k->kif->ki_pid);
+            printf("%d\n", k->kif->ki_pid);  // NOQA
     }
     */
 
@@ -206,17 +206,6 @@ psutil_get_sockets(const char *name) {
         kpcb->kpcb = &kp[j];
         SLIST_INSERT_HEAD(&kpcbhead, kpcb, kpcbs);
     }
-
-    /*
-    // debug
-    struct kif *k;
-    struct kpcb *k;
-    SLIST_FOREACH(k, &kpcbhead, kpcbs) {
-            printf("ki_type: %d\n", k->kpcb->ki_type);
-            printf("ki_family: %d\n", k->kpcb->ki_family);
-    }
-    */
-
     return 0;
 }
 
diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py
index 3e4dc880..cd78fae6 100644
--- a/psutil/tests/__init__.py
+++ b/psutil/tests/__init__.py
@@ -819,7 +819,7 @@ def retry_on_failure(retries=NO_RETRIES):
     actually failing.
     """
     def logfun(exc):
-        print("%r, retrying" % exc, file=sys.stderr)
+        print("%r, retrying" % exc, file=sys.stderr)  # NOQA
 
     return retry(exception=AssertionError, timeout=None, retries=retries,
                  logfun=logfun)
diff --git a/scripts/internal/.git-pre-commit b/scripts/internal/.git-pre-commit
deleted file mode 100755
index e6009253..00000000
--- a/scripts/internal/.git-pre-commit
+++ /dev/null
@@ -1,131 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""
-This gets executed on 'git commit' and rejects the commit in case the
-submitted code does not pass validation. Validation is run only against
-the *.py files which were modified in the commit. Checks:
-
-- assert no space at EOLs
-- assert not pdb.set_trace in code
-- assert no bare except clause ("except:") in code
-- assert "flake8" returns no warnings
-
-Install this with "make install-git-hooks".
-"""
-
-from __future__ import print_function
-import os
-import subprocess
-import sys
-
-
-def term_supports_colors():
-    try:
-        import curses
-        assert sys.stderr.isatty()
-        curses.setupterm()
-        assert curses.tigetnum("colors") > 0
-    except Exception:
-        return False
-    else:
-        return True
-
-
-def hilite(s, ok=True, bold=False):
-    """Return an highlighted version of 'string'."""
-    if not term_supports_colors():
-        return s
-    attr = []
-    if ok is None:  # no color
-        pass
-    elif ok:   # green
-        attr.append('32')
-    else:   # red
-        attr.append('31')
-    if bold:
-        attr.append('1')
-    return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s)
-
-
-def exit(msg):
-    msg = hilite(msg, ok=False)
-    print(msg, file=sys.stderr)
-    sys.exit(1)
-
-
-def sh(cmd):
-    """run cmd in a subprocess and return its output.
-    raises RuntimeError on error.
-    """
-    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE, universal_newlines=True)
-    stdout, stderr = p.communicate()
-    if p.returncode != 0:
-        raise RuntimeError(stderr)
-    if stderr:
-        print(stderr, file=sys.stderr)
-    if stdout.endswith('\n'):
-        stdout = stdout[:-1]
-    return stdout
-
-
-def main():
-    out = sh("git diff --cached --name-only")
-    py_files = [x for x in out.split('\n') if x.endswith('.py') and
-                os.path.exists(x)]
-    c_files = [x for x in out.split('\n') if x.endswith(('.c', '.h')) and
-               os.path.exists(x)]
-
-    lineno = 0
-    kw = {'encoding': 'utf8'} if sys.version_info[0] == 3 else {}
-    for path in py_files:
-        with open(path, 'rt', **kw) as f:
-            for line in f:
-                lineno += 1
-                # space at end of line
-                if line.endswith(' '):
-                    print("%s:%s %r" % (path, lineno, line))
-                    return exit(
-                        "commit aborted: space at end of line")
-                line = line.rstrip()
-                # pdb
-                if "pdb.set_trace" in line:
-                    print("%s:%s %s" % (path, lineno, line))
-                    return exit(
-                        "commit aborted: you forgot a pdb in your python code")
-                # bare except clause
-                if "except:" in line and not line.endswith("# NOQA"):
-                    print("%s:%s %s" % (path, lineno, line))
-                    return exit("commit aborted: bare except clause")
-
-    # Python linter
-    if py_files:
-        try:
-            import flake8  # NOQA
-        except ImportError:
-            return exit("commit aborted: flake8 is not installed; "
-                        "run 'make setup-dev-env'")
-
-        # XXX: we should escape spaces and possibly other amenities here
-        ret = subprocess.call(
-            "%s -m flake8 %s" % (sys.executable, " ".join(py_files)),
-            shell=True)
-        if ret != 0:
-            return exit("commit aborted: python code is not flake8 compliant")
-
-    # C linter
-    if c_files:
-        # XXX: we should escape spaces and possibly other amenities here
-        cmd = "%s scripts/internal/clinter.py %s" % (
-            sys.executable, " ".join(c_files))
-        print(cmd)
-        ret = subprocess.call(cmd, shell=True)
-        if ret != 0:
-            return exit("commit aborted: C code didn't pass style check")
-
-
-main()
diff --git a/scripts/internal/clinter.py b/scripts/internal/clinter.py
index 1d4ba9b1..fde1a3f2 100755
--- a/scripts/internal/clinter.py
+++ b/scripts/internal/clinter.py
@@ -43,16 +43,19 @@ def check_line(path, line, idx, lines):
     sls = s.lstrip()
     if sls.startswith('//') and sls[2] != ' ' and line.strip() != '//':
         warn(path, line, lineno, "no space after // comment")
-
     # e.g. "if(..." after keywords
     keywords = ("if", "else", "while", "do", "enum", "for")
     for kw in keywords:
         if sls.startswith(kw + '('):
             warn(path, line, lineno, "missing space between %r and '('" % kw)
     # eof
-    if eof:
-        if not line.endswith('\n'):
-            warn(path, line, lineno, "no blank line at EOF")
+    if eof and not line.endswith('\n'):
+        warn(path, line, lineno, "no blank line at EOF")
+
+    ss = s.strip()
+    if ss.startswith(("printf(", "printf (", )):
+        if not ss.endswith(("// NOQA", "//  NOQA")):
+            warn(path, line, lineno, "printf() statement")
 
 
 def process(path):
diff --git a/scripts/internal/git_pre_commit.py b/scripts/internal/git_pre_commit.py
new file mode 100755
index 00000000..2ec4303d
--- /dev/null
+++ b/scripts/internal/git_pre_commit.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This gets executed on 'git commit' and rejects the commit in case the
+submitted code does not pass validation. Validation is run only against
+the files which were modified in the commit. Checks:
+
+- assert no space at EOLs
+- assert not pdb.set_trace in code
+- assert no bare except clause ("except:") in code
+- assert "flake8" checks pass
+- assert C linter checks pass
+- abort if files were added/renamed/removed and MANIFEST.in was not updated
+
+Install this with "make install-git-hooks".
+"""
+
+from __future__ import print_function
+import os
+import subprocess
+import sys
+
+
+PYTHON = sys.executable
+PY3 = sys.version_info[0] == 3
+THIS_SCRIPT = os.path.realpath(__file__)
+
+
+def term_supports_colors():
+    try:
+        import curses
+        assert sys.stderr.isatty()
+        curses.setupterm()
+        assert curses.tigetnum("colors") > 0
+    except Exception:
+        return False
+    return True
+
+
+def hilite(s, ok=True, bold=False):
+    """Return an highlighted version of 'string'."""
+    if not term_supports_colors():
+        return s
+    attr = []
+    if ok is None:  # no color
+        pass
+    elif ok:   # green
+        attr.append('32')
+    else:   # red
+        attr.append('31')
+    if bold:
+        attr.append('1')
+    return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s)
+
+
+def exit(msg):
+    print(hilite("commit aborted: " + msg, ok=False), file=sys.stderr)
+    sys.exit(1)
+
+
+def sh(cmd):
+    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE, universal_newlines=True)
+    stdout, stderr = p.communicate()
+    if p.returncode != 0:
+        raise RuntimeError(stderr)
+    if stderr:
+        print(stderr, file=sys.stderr)
+    if stdout.endswith('\n'):
+        stdout = stdout[:-1]
+    return stdout
+
+
+def open_text(path):
+    kw = {'encoding': 'utf8'} if PY3 else {}
+    return open(path, 'rt', **kw)
+
+
+def git_commit_files():
+    out = sh("git diff --cached --name-only")
+    py_files = [x for x in out.split('\n') if x.endswith('.py') and
+                os.path.exists(x)]
+    c_files = [x for x in out.split('\n') if x.endswith(('.c', '.h')) and
+               os.path.exists(x)]
+    new_rm_mv = sh("git diff --name-only --diff-filter=ADR --cached")
+    # XXX: we should escape spaces and possibly other amenities here
+    new_rm_mv = new_rm_mv.split()
+    return (py_files, c_files, new_rm_mv)
+
+
+def main():
+    py_files, c_files, new_rm_mv = git_commit_files()
+    # Check file content.
+    for path in py_files:
+        if os.path.realpath(path) == THIS_SCRIPT:
+            continue
+        with open_text(path) as f:
+            lines = f.readlines()
+        for lineno, line in enumerate(lines, 1):
+            # space at end of line
+            if line.endswith(' '):
+                print("%s:%s %r" % (path, lineno, line))
+                return exit("space at end of line")
+            line = line.rstrip()
+            # pdb
+            if "pdb.set_trace" in line:
+                print("%s:%s %s" % (path, lineno, line))
+                return exit("you forgot a pdb in your python code")
+            # bare except clause
+            if "except:" in line and not line.endswith("# NOQA"):
+                print("%s:%s %s" % (path, lineno, line))
+                return exit("bare except clause")
+
+    # Python linter
+    if py_files:
+        assert os.path.exists('.flake8')
+        # XXX: we should escape spaces and possibly other amenities here
+        cmd = "%s -m flake8 --config=.flake8 %s" % (PYTHON, " ".join(py_files))
+        ret = subprocess.call(cmd, shell=True)
+        if ret != 0:
+            return exit("python code is not flake8 compliant")
+    # C linter
+    if c_files:
+        # XXX: we should escape spaces and possibly other amenities here
+        cmd = "%s scripts/internal/clinter.py %s" % (PYTHON, " ".join(c_files))
+        ret = subprocess.call(cmd, shell=True)
+        if ret != 0:
+            return exit("C code didn't pass style check")
+    if new_rm_mv:
+        out = sh("%s scripts/internal/generate_manifest.py" % PYTHON)
+        with open_text('MANIFEST.in') as f:
+            if out.strip() != f.read().strip():
+                exit("some files were added, deleted or renamed; "
+                     "run 'make generate-manifest' and commit again")
+
+
+main()
diff --git a/scripts/internal/tidelift.py b/scripts/internal/tidelift.py
new file mode 100644
index 00000000..fcba3e61
--- /dev/null
+++ b/scripts/internal/tidelift.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Update news entry of Tidelift with latest HISTORY.rst section.
+Put your Tidelift API token in a file first:
+~/.tidelift.token
+"""
+
+from __future__ import print_function
+import os
+import requests
+import psutil
+from psutil.tests import import_module_by_path
+
+
+def upload_relnotes(package, version, text, token):
+    url = "https://api.tidelift.com/external-api/" + \
+          "lifting/pypi/%s/release-notes/%s" % (package, version)
+    res = requests.put(
+        url=url,
+        data=text.encode('utf8'),
+        headers={"Authorization": "Bearer: %s" % token})
+    print(version, res.status_code, res.text)
+    res.raise_for_status()
+
+
+def main():
+    here = os.path.abspath(os.path.dirname(__file__))
+    path = os.path.join(here, "print_announce.py")
+    get_changes = import_module_by_path(path).get_changes
+    with open(os.path.expanduser("~/.tidelift.token")) as f:
+        token = f.read().strip()
+    upload_relnotes('psutil', psutil.__version__, get_changes(), token)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py
index 4d3fa318..c9aa2952 100755
--- a/scripts/internal/winmake.py
+++ b/scripts/internal/winmake.py
@@ -490,7 +490,8 @@ def test_memleaks():
 def install_git_hooks():
     """Install GIT pre-commit hook."""
     if os.path.isdir('.git'):
-        src = os.path.join(ROOT_DIR, "scripts", "internal", ".git-pre-commit")
+        src = os.path.join(
+            ROOT_DIR, "scripts", "internal", "git_pre_commit.py")
         dst = os.path.realpath(
             os.path.join(ROOT_DIR, ".git", "hooks", "pre-commit"))
         with open(src, "rt") as s: