From 1f53b4fbd4600d03dcaae01ad1485d1dd0825f6f Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sat, 17 Sep 2016 12:36:12 +0200
Subject: skip test failing on travis

---
 psutil/tests/test_linux.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index 9f7c25fb..dffacc72 100644
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -370,6 +370,7 @@ class TestSystemCPU(unittest.TestCase):
 @unittest.skipUnless(LINUX, "not a Linux system")
 class TestSystemCPUStats(unittest.TestCase):
 
+    @unittest.skipIf(TRAVIS, "fails on Travis")
     def test_ctx_switches(self):
         vmstat_value = vmstat("context switches")
         psutil_value = psutil.cpu_stats().ctx_switches
-- 
cgit v1.2.1


From ea08d4cf941c23e1c16bba1e84e7f6e80207c5b5 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 15:12:44 +0200
Subject: linux test refactoring

---
 psutil/tests/test_linux.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index dffacc72..f9a1478f 100644
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -117,8 +117,9 @@ def free_physmem():
         if line.startswith('Mem'):
             total, used, free, shared = \
                 [int(x) for x in line.split()[1:5]]
-            nt = collections.namedtuple('free', 'total used free shared')
-            return nt(total, used, free, shared)
+            nt = collections.namedtuple(
+                'free', 'total used free shared output')
+            return nt(total, used, free, shared, out)
     raise ValueError(
         "can't find 'Mem' in 'free' output:\n%s" % '\n'.join(lines))
 
@@ -150,10 +151,12 @@ class TestSystemVirtualMemory(unittest.TestCase):
 
     @retry_before_failing()
     def test_used(self):
-        free_value = free_physmem().used
+        free = free_physmem()
+        free_value = free.used
         psutil_value = psutil.virtual_memory().used
         self.assertAlmostEqual(
-            free_value, psutil_value, delta=MEMORY_TOLERANCE)
+            free_value, psutil_value, delta=MEMORY_TOLERANCE,
+            msg='%s %s \n%s' % (free_value, psutil_value, free.output))
 
     @retry_before_failing()
     def test_free(self):
@@ -190,12 +193,14 @@ class TestSystemVirtualMemory(unittest.TestCase):
     @retry_before_failing()
     @unittest.skipIf(TRAVIS, "fails on travis")
     def test_shared(self):
-        free_value = free_physmem().shared
+        free = free_physmem()
+        free_value = free.shared
         if free_value == 0:
             raise unittest.SkipTest("free does not support 'shared' column")
         psutil_value = psutil.virtual_memory().shared
         self.assertAlmostEqual(
-            free_value, psutil_value, delta=MEMORY_TOLERANCE)
+            free_value, psutil_value, delta=MEMORY_TOLERANCE,
+            msg='%s %s \n%s' % (free_value, psutil_value, free.output))
 
     # --- mocked tests
 
-- 
cgit v1.2.1


From 4ef75441af929750309fecbe6a0e49a2a6f03b05 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 16:01:37 +0200
Subject: #887: add test to matche 'avail' column of 'free'

---
 psutil/tests/test_linux.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index f9a1478f..87765107 100644
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -202,22 +202,18 @@ class TestSystemVirtualMemory(unittest.TestCase):
             free_value, psutil_value, delta=MEMORY_TOLERANCE,
             msg='%s %s \n%s' % (free_value, psutil_value, free.output))
 
-    # --- mocked tests
-
-    def test_warnings_mocked(self):
-        with mock.patch('psutil._pslinux.open', create=True) as m:
-            with warnings.catch_warnings(record=True) as ws:
-                warnings.simplefilter("always")
-                ret = psutil._pslinux.virtual_memory()
-                assert m.called
-                self.assertEqual(len(ws), 1)
-                w = ws[0]
-                self.assertTrue(w.filename.endswith('psutil/_pslinux.py'))
-                self.assertIn(
-                    "memory stats couldn't be determined", str(w.message))
-                self.assertEqual(ret.cached, 0)
-                self.assertEqual(ret.active, 0)
-                self.assertEqual(ret.inactive, 0)
+    def test_available(self):
+        # "free" output format has changed at some point:
+        # https://github.com/giampaolo/psutil/issues/538#issuecomment-147192098
+        out = sh("free -b")
+        lines = out.split('\n')
+        if 'available' in lines[0]:
+            raise unittest.SkipTest("free does not support 'available' column")
+        free_value = int(lines[1].split()[-1])
+        psutil_value = psutil.virtual_memory().available
+        self.assertAlmostEqual(
+            free_value, psutil_value, delta=MEMORY_TOLERANCE,
+            msg='%s %s \n%s' % (free_value, psutil_value, out))
 
 
 # =====================================================================
-- 
cgit v1.2.1


From e67cf876f4e987d7f3ceebe3bd6caecf20086fa3 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 16:04:51 +0200
Subject: #887: match 'available' column of 'free'; also calculate 'cached' mem
 the same way free does (includes reclaimable mem)

---
 psutil/__init__.py |  2 +-
 psutil/_pslinux.py | 70 ++++++++++++++++++++++++------------------------------
 2 files changed, 32 insertions(+), 40 deletions(-)

diff --git a/psutil/__init__.py b/psutil/__init__.py
index 0a6f3ec6..020a0bdd 100644
--- a/psutil/__init__.py
+++ b/psutil/__init__.py
@@ -187,7 +187,7 @@ __all__ = [
 ]
 __all__.extend(_psplatform.__extra__all__)
 __author__ = "Giampaolo Rodola'"
-__version__ = "4.3.2"
+__version__ = "4.4.0"
 version_info = tuple([int(num) for num in __version__.split('.')])
 AF_LINK = _psplatform.AF_LINK
 _TOTAL_PHYMEM = None
diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 7f6e0405..82f79074 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -299,47 +299,39 @@ def virtual_memory():
     if shared == 0:
         shared = None
 
-    cached = active = inactive = None
+    mems = {}
     with open_binary('%s/meminfo' % get_procfs_path()) as f:
         for line in f:
-            if cached is None and line.startswith(b"Cached:"):
-                cached = int(line.split()[1]) * 1024
-            elif active is None and line.startswith(b"Active:"):
-                active = int(line.split()[1]) * 1024
-            elif inactive is None and line.startswith(b"Inactive:"):
-                inactive = int(line.split()[1]) * 1024
-            # From "man free":
-            # The shared memory column represents either the MemShared
-            # value (2.4 kernels) or the Shmem value (2.6+ kernels) taken
-            # from the /proc/meminfo file. The value is zero if none of
-            # the entries is exported by the kernel.
-            elif shared is None and \
-                    line.startswith(b"MemShared:") or \
-                    line.startswith(b"Shmem:"):
-                shared = int(line.split()[1]) * 1024
-
-    missing = []
-    if cached is None:
-        missing.append('cached')
-        cached = 0
-    if active is None:
-        missing.append('active')
-        active = 0
-    if inactive is None:
-        missing.append('inactive')
-        inactive = 0
-    if shared is None:
-        missing.append('shared')
-        shared = 0
-    if missing:
-        msg = "%s memory stats couldn't be determined and %s set to 0" % (
-            ", ".join(missing),
-            "was" if len(missing) == 1 else "were")
-        warnings.warn(msg, RuntimeWarning)
-
-    # Note: this value matches "htop" perfectly.
-    avail = free + buffers + cached
-    # Note: this value matches "free", but not all the time, see:
+            fields = line.split()
+            mems[fields[0]] = int(fields[1]) * 1024
+
+    # Match "free" cmdline utility:
+    # https://gitlab.com/procps-ng/procps/
+    #     blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L761
+    cached = mems.get(b"Cached:", 0) + mems.get(b"SReclaimable:", 0)
+
+    active = mems.get(b"Active:", 0)
+
+    # Match "free" cmdline utility in case "Inactive:" is not there:
+    # https://gitlab.com/procps-ng/procps/
+    #   blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L758
+    inactive = mems.get(b"Inactive:", 0)
+    if inactive == 0:
+        inactive = \
+            mems.get(b"Inact_dirty:", 0) + \
+            mems.get(b"Inact_clean:", 0) + \
+            mems.get(b"Inact_laundry:", 0)
+
+    # Note: starting from 4.4.0 we match "free" "available" column.
+    # Before 4.4.0 we calculated it as:
+    # >>> avail = free + buffers + cached
+    # ...which matched htop.
+    # free and htop available memory differs as per:
+    # http://askubuntu.com/a/369589
+    # http://unix.stackexchange.com/a/65852/168884
+    avail = mems['MemAvailable:']
+
+    # XXX: this value matches "free", but not all the time, see:
     # https://github.com/giampaolo/psutil/issues/685#issuecomment-202914057
     used = total - free
     # Note: this value matches "htop" perfectly.
-- 
cgit v1.2.1


From 75aecfefea66f66d3524d74ea15ea7210a278fec Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 16:34:48 +0200
Subject: #887: MemAvailable column may not be there; in that case calculate
 available mem the same way htop does

---
 psutil/_pslinux.py         | 21 ++++++++++++++++++++-
 psutil/tests/test_linux.py |  2 +-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 82f79074..04ed4eb2 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -289,6 +289,16 @@ except Exception:
 
 
 def virtual_memory():
+    """Report memory stats trying to match "free" and "vmstat -s" cmdline
+    utility values as much as possible.
+
+    This implementation uses procps-ng-3.3.12 as a reference (2016-09-18):
+    https://gitlab.com/procps-ng/procps/blob/
+        24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c
+
+    ...or "free" / procps-ng-3.3.10 version which is available in Ubuntu
+    16.04 and which should report the same numbers.
+    """
     total, free, buffers, shared, _, _, unit_multiplier = cext.linux_sysinfo()
     total *= unit_multiplier
     free *= unit_multiplier
@@ -329,7 +339,16 @@ def virtual_memory():
     # free and htop available memory differs as per:
     # http://askubuntu.com/a/369589
     # http://unix.stackexchange.com/a/65852/168884
-    avail = mems['MemAvailable:']
+    try:
+        avail = mems['MemAvailable:']
+    except KeyError:
+        # Column is not there; it's likely this is an older kernel.
+        # In this case "free" won't show an "available" column.
+        # Also, procps does some hacky things:
+        # https://gitlab.com/procps-ng/procps/blob/
+        #     /24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L774
+        # We won't. Like this we'll match "htop".
+        avail = free + buffers + cached
 
     # XXX: this value matches "free", but not all the time, see:
     # https://github.com/giampaolo/psutil/issues/685#issuecomment-202914057
diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index 87765107..520ae8e3 100644
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -191,7 +191,6 @@ class TestSystemVirtualMemory(unittest.TestCase):
             vmstat_value, psutil_value, delta=MEMORY_TOLERANCE)
 
     @retry_before_failing()
-    @unittest.skipIf(TRAVIS, "fails on travis")
     def test_shared(self):
         free = free_physmem()
         free_value = free.shared
@@ -202,6 +201,7 @@ class TestSystemVirtualMemory(unittest.TestCase):
             free_value, psutil_value, delta=MEMORY_TOLERANCE,
             msg='%s %s \n%s' % (free_value, psutil_value, free.output))
 
+    @retry_before_failing()
     def test_available(self):
         # "free" output format has changed at some point:
         # https://github.com/giampaolo/psutil/issues/538#issuecomment-147192098
-- 
cgit v1.2.1


From 1ad19c4e969a40580215f5923139c9dadc73c25a Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 16:49:05 +0200
Subject: reintroduce warnings for missing fields

---
 psutil/_pslinux.py | 45 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 04ed4eb2..bf8f6baf 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -299,6 +299,7 @@ def virtual_memory():
     ...or "free" / procps-ng-3.3.10 version which is available in Ubuntu
     16.04 and which should report the same numbers.
     """
+    missing_fields = []
     total, free, buffers, shared, _, _, unit_multiplier = cext.linux_sysinfo()
     total *= unit_multiplier
     free *= unit_multiplier
@@ -315,22 +316,38 @@ def virtual_memory():
             fields = line.split()
             mems[fields[0]] = int(fields[1]) * 1024
 
-    # Match "free" cmdline utility:
+    # "free" cmdline utility sums cached + reclamaible:
     # https://gitlab.com/procps-ng/procps/
     #     blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L761
-    cached = mems.get(b"Cached:", 0) + mems.get(b"SReclaimable:", 0)
+    try:
+        cached = mems[b"Cached:"]
+    except KeyError:
+        cached = 0
+        missing_fields.append('cached')
+    else:
+        cached += mems.get(b"SReclaimable:", 0)
 
-    active = mems.get(b"Active:", 0)
+    # active
+    try:
+        active = mems["Active:"]
+    except KeyError:
+        active = 0
+        missing_fields.append('active')
 
-    # Match "free" cmdline utility in case "Inactive:" is not there:
+    # inactive
     # https://gitlab.com/procps-ng/procps/
     #   blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L758
-    inactive = mems.get(b"Inactive:", 0)
-    if inactive == 0:
-        inactive = \
-            mems.get(b"Inact_dirty:", 0) + \
-            mems.get(b"Inact_clean:", 0) + \
-            mems.get(b"Inact_laundry:", 0)
+    try:
+        inactive = mems[b"Inactive:"]
+    except KeyError:
+        try:
+            inactive = \
+                mems[b"Inact_dirty:"] + \
+                mems[b"Inact_clean:"] + \
+                mems[b"Inact_laundry:"]
+        except KeyError:
+            inactive = 0
+            missing_fields.append('inactive')
 
     # Note: starting from 4.4.0 we match "free" "available" column.
     # Before 4.4.0 we calculated it as:
@@ -355,6 +372,14 @@ def virtual_memory():
     used = total - free
     # Note: this value matches "htop" perfectly.
     percent = usage_percent((total - avail), total, _round=1)
+
+    # Warn about missing metrics which are set to 0.
+    if missing_fields:
+        msg = "%s memory stats couldn't be determined and %s set to 0" % (
+            ", ".join(missing_fields),
+            "was" if len(missing_fields) == 1 else "were")
+        warnings.warn(msg, RuntimeWarning)
+
     return svmem(total, avail, percent, used, free,
                  active, inactive, buffers, cached, shared)
 
-- 
cgit v1.2.1


From a956b5fe2aa270b48ae04c41e7b105205488ef5b Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 16:55:51 +0200
Subject: #887 correctly calculate shared mem

---
 psutil/_pslinux.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index bf8f6baf..8208f75a 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -306,9 +306,7 @@ def virtual_memory():
     buffers *= unit_multiplier
     # Note: this (on my Ubuntu 14.04, kernel 3.13 at least) may be 0.
     # If so, it will be determined from /proc/meminfo.
-    shared *= unit_multiplier or None
-    if shared == 0:
-        shared = None
+    shared *= unit_multiplier
 
     mems = {}
     with open_binary('%s/meminfo' % get_procfs_path()) as f:
@@ -316,6 +314,19 @@ def virtual_memory():
             fields = line.split()
             mems[fields[0]] = int(fields[1]) * 1024
 
+    # shared
+    if shared == 0:
+        # Note: if 0 (e.g. my Ubuntu 14.04, kernel 3.13 at least)
+        # this can be determined from /proc/meminfo.
+        try:
+            shared = mems['Shmem:']  # kernel 2.6.32
+        except KeyError:
+            try:
+                shared = mems['MemShared:']  # kernels 2.4
+            except KeyError:
+                shared = 0
+                missing_fields.append('shared')
+
     # "free" cmdline utility sums cached + reclamaible:
     # https://gitlab.com/procps-ng/procps/
     #     blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L761
-- 
cgit v1.2.1


From 6e4f0d3e53adfb8af826aa77f55f7db080a5cac3 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 17:14:11 +0200
Subject: #877: properly calculate used mem

---
 psutil/_pslinux.py         | 13 ++++++++++---
 psutil/tests/test_linux.py | 13 +++++++------
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 8208f75a..6bfa4f1d 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -330,6 +330,10 @@ def virtual_memory():
     # "free" cmdline utility sums cached + reclamaible:
     # https://gitlab.com/procps-ng/procps/
     #     blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L761
+    # Older versions of procps added slab memory instead.
+    # This got changed in:
+    # https://gitlab.com/procps-ng/procps/commit/
+    #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
     try:
         cached = mems[b"Cached:"]
     except KeyError:
@@ -378,9 +382,12 @@ def virtual_memory():
         # We won't. Like this we'll match "htop".
         avail = free + buffers + cached
 
-    # XXX: this value matches "free", but not all the time, see:
-    # https://github.com/giampaolo/psutil/issues/685#issuecomment-202914057
-    used = total - free
+    # https://gitlab.com/procps-ng/procps/blob/
+    #     24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L769
+    used = total - free - cached - buffers
+    if used < 0:
+        used = total - free
+
     # Note: this value matches "htop" perfectly.
     percent = usage_percent((total - avail), total, _round=1)
 
diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index 520ae8e3..f7e8f6e0 100644
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -207,13 +207,14 @@ class TestSystemVirtualMemory(unittest.TestCase):
         # https://github.com/giampaolo/psutil/issues/538#issuecomment-147192098
         out = sh("free -b")
         lines = out.split('\n')
-        if 'available' in lines[0]:
+        if 'available' not in lines[0]:
             raise unittest.SkipTest("free does not support 'available' column")
-        free_value = int(lines[1].split()[-1])
-        psutil_value = psutil.virtual_memory().available
-        self.assertAlmostEqual(
-            free_value, psutil_value, delta=MEMORY_TOLERANCE,
-            msg='%s %s \n%s' % (free_value, psutil_value, out))
+        else:
+            free_value = int(lines[1].split()[-1])
+            psutil_value = psutil.virtual_memory().available
+            self.assertAlmostEqual(
+                free_value, psutil_value, delta=MEMORY_TOLERANCE,
+                msg='%s %s \n%s' % (free_value, psutil_value, out))
 
 
 # =====================================================================
-- 
cgit v1.2.1


From 9f044a84c4ac8c4180ae308be04b9cd287706e07 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 17:20:48 +0200
Subject: #87: skip used mem test in case of older free version

---
 psutil/_pslinux.py         |  1 -
 psutil/tests/test_linux.py | 11 +++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 6bfa4f1d..bdf01c33 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -388,7 +388,6 @@ def virtual_memory():
     if used < 0:
         used = total - free
 
-    # Note: this value matches "htop" perfectly.
     percent = usage_percent((total - avail), total, _round=1)
 
     # Warn about missing metrics which are set to 0.
diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index f7e8f6e0..9fa72320 100644
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -133,6 +133,11 @@ def vmstat(stat):
     raise ValueError("can't find %r in 'vmstat' output" % stat)
 
 
+def get_free_version_info():
+    out = sh("free -V").strip()
+    return tuple(map(int, out.split()[-1].split('.')))
+
+
 # =====================================================================
 # system virtual memory
 # =====================================================================
@@ -149,6 +154,12 @@ class TestSystemVirtualMemory(unittest.TestCase):
         psutil_value = psutil.virtual_memory().total
         self.assertAlmostEqual(vmstat_value, psutil_value)
 
+    # Older versions of procps used slab memory to calculate used memory.
+    # This got changed in:
+    # https://gitlab.com/procps-ng/procps/commit/
+    #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
+    @unittest.skipUnless(
+        get_free_version_info() >= (3, 3, 12), "old free version")
     @retry_before_failing()
     def test_used(self):
         free = free_physmem()
-- 
cgit v1.2.1


From a76ba8f2e415ced5ca500484e9be1da48c3633da Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 17:23:37 +0200
Subject: fix typos

---
 psutil/_pslinux.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index bdf01c33..c4a5a07e 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -316,8 +316,8 @@ def virtual_memory():
 
     # shared
     if shared == 0:
-        # Note: if 0 (e.g. my Ubuntu 14.04, kernel 3.13 at least)
-        # this can be determined from /proc/meminfo.
+        # Note: if 0 (e.g. Ubuntu 14.04, kernel 3.13) this can be
+        # determined from /proc/meminfo.
         try:
             shared = mems['Shmem:']  # kernel 2.6.32
         except KeyError:
@@ -372,7 +372,7 @@ def virtual_memory():
     # http://askubuntu.com/a/369589
     # http://unix.stackexchange.com/a/65852/168884
     try:
-        avail = mems['MemAvailable:']
+        avail = mems[b'MemAvailable:']
     except KeyError:
         # Column is not there; it's likely this is an older kernel.
         # In this case "free" won't show an "available" column.
-- 
cgit v1.2.1


From ec6ee5560f815e4cfdcf672464d7b1851b209f59 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 17:28:00 +0200
Subject: use bytes

---
 psutil/_pslinux.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index c4a5a07e..7d2935a3 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -344,7 +344,7 @@ def virtual_memory():
 
     # active
     try:
-        active = mems["Active:"]
+        active = mems[b"Active:"]
     except KeyError:
         active = 0
         missing_fields.append('active')
-- 
cgit v1.2.1


From 4b4e8282ea60840f48f0fe797da76d7d5598b9da Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 17:36:25 +0200
Subject: refactoring

---
 psutil/_pslinux.py | 30 +++++++++++-------------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 7d2935a3..1a6a6ba0 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -314,12 +314,21 @@ def virtual_memory():
             fields = line.split()
             mems[fields[0]] = int(fields[1]) * 1024
 
-    # shared
+    cached = mems[b"Cached:"]
+    # "free" cmdline utility sums cached + reclamaible:
+    # https://gitlab.com/procps-ng/procps/
+    #     blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L761
+    # Older versions of procps added slab memory instead.
+    # This got changed in:
+    # https://gitlab.com/procps-ng/procps/commit/
+    #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
+    cached += mems.get(b"SReclaimable:", 0)  # kernel 2.6.19
+
     if shared == 0:
         # Note: if 0 (e.g. Ubuntu 14.04, kernel 3.13) this can be
         # determined from /proc/meminfo.
         try:
-            shared = mems['Shmem:']  # kernel 2.6.32
+            shared = mems['Shmem:']  # since kernel 2.6.32
         except KeyError:
             try:
                 shared = mems['MemShared:']  # kernels 2.4
@@ -327,29 +336,12 @@ def virtual_memory():
                 shared = 0
                 missing_fields.append('shared')
 
-    # "free" cmdline utility sums cached + reclamaible:
-    # https://gitlab.com/procps-ng/procps/
-    #     blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L761
-    # Older versions of procps added slab memory instead.
-    # This got changed in:
-    # https://gitlab.com/procps-ng/procps/commit/
-    #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
-    try:
-        cached = mems[b"Cached:"]
-    except KeyError:
-        cached = 0
-        missing_fields.append('cached')
-    else:
-        cached += mems.get(b"SReclaimable:", 0)
-
-    # active
     try:
         active = mems[b"Active:"]
     except KeyError:
         active = 0
         missing_fields.append('active')
 
-    # inactive
     # https://gitlab.com/procps-ng/procps/
     #   blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L758
     try:
-- 
cgit v1.2.1


From 559270e5d7d36707995e2ddf99e3b9b80fba9f85 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 17:47:51 +0200
Subject: #887: do not use sysinfo() syscall; use /proc/meminfo to calculate
 all stats

---
 psutil/_pslinux.py | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 1a6a6ba0..88e6a939 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -300,20 +300,16 @@ def virtual_memory():
     16.04 and which should report the same numbers.
     """
     missing_fields = []
-    total, free, buffers, shared, _, _, unit_multiplier = cext.linux_sysinfo()
-    total *= unit_multiplier
-    free *= unit_multiplier
-    buffers *= unit_multiplier
-    # Note: this (on my Ubuntu 14.04, kernel 3.13 at least) may be 0.
-    # If so, it will be determined from /proc/meminfo.
-    shared *= unit_multiplier
-
     mems = {}
     with open_binary('%s/meminfo' % get_procfs_path()) as f:
         for line in f:
             fields = line.split()
             mems[fields[0]] = int(fields[1]) * 1024
 
+    # Note: these info are available also as cext.linux_sysinfo().
+    total = mems[b'MemTotal:']
+    buffers = mems[b'Buffers:']
+    free = mems[b'MemFree:']
     cached = mems[b"Cached:"]
     # "free" cmdline utility sums cached + reclamaible:
     # https://gitlab.com/procps-ng/procps/
@@ -324,17 +320,16 @@ def virtual_memory():
     #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
     cached += mems.get(b"SReclaimable:", 0)  # kernel 2.6.19
 
-    if shared == 0:
-        # Note: if 0 (e.g. Ubuntu 14.04, kernel 3.13) this can be
-        # determined from /proc/meminfo.
+    # Note: if 0 (e.g. Ubuntu 14.04, kernel 3.13) this can be
+    # determined from /proc/meminfo.
+    try:
+        shared = mems[b'Shmem:']  # since kernel 2.6.32
+    except KeyError:
         try:
-            shared = mems['Shmem:']  # since kernel 2.6.32
+            shared = mems[b'MemShared:']  # kernels 2.4
         except KeyError:
-            try:
-                shared = mems['MemShared:']  # kernels 2.4
-            except KeyError:
-                shared = 0
-                missing_fields.append('shared')
+            shared = 0
+            missing_fields.append('shared')
 
     try:
         active = mems[b"Active:"]
-- 
cgit v1.2.1


From bd36e0b4c4d8f8eb8073e9e164744355a5f06ccf Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 18:04:44 +0200
Subject: update doc

---
 HISTORY.rst                | 4 +++-
 docs/index.rst             | 7 +++++--
 psutil/tests/test_linux.py | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index b5775bfd..8104e174 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -1,7 +1,7 @@
 Bug tracker at https://github.com/giampaolo/psutil/issues
 
 
-4.3.2 - XXXX-XX-XX
+4.4.0 - XXXX-XX-XX
 ==================
 
 **Bug fixes**
@@ -10,6 +10,8 @@ Bug tracker at https://github.com/giampaolo/psutil/issues
 - #880: [Windows] Handle race condition inside psutil_net_connections.
 - #885: ValueError is raised if a negative integer is passed to cpu_percent()
   functions.
+- #887: [Linux] free, available and used fields are more precise and match
+  "free" cmdline utility.
 
 
 4.3.1 - 2016-09-01
diff --git a/docs/index.rst b/docs/index.rst
index fa83aefa..d64ab43a 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -178,8 +178,8 @@ Memory
   - **available**: the actual amount of available memory that can be given
     instantly to processes that request more memory in bytes; this is
     calculated by summing different memory values depending on the platform
-    (e.g. ``(free + buffers + cached)`` on Linux) and it is supposed to be used
-    to monitor actual memory usage in a cross platform fashion.
+    and it is supposed to be used to monitor actual memory usage in a cross
+    platform fashion.
   - **percent**: the percentage usage calculated as
     ``(total - available) / total * 100``.
   - **used**: memory used, calculated differently depending on the platform and
@@ -221,6 +221,9 @@ Memory
 
   .. versionchanged:: 4.2.0 added *shared* metrics on Linux.
 
+  .. versionchanged:: 4.4.0 on Linux, *free*, *available* and *used* fields
+    are more precise and match "free" cmdline utility.
+
 .. function:: swap_memory()
 
   Return system swap memory statistics as a namedtuple including the following
diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py
index 9fa72320..a6b5d3f3 100644
--- a/psutil/tests/test_linux.py
+++ b/psutil/tests/test_linux.py
@@ -159,7 +159,7 @@ class TestSystemVirtualMemory(unittest.TestCase):
     # https://gitlab.com/procps-ng/procps/commit/
     #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
     @unittest.skipUnless(
-        get_free_version_info() >= (3, 3, 12), "old free version")
+        LINUX and get_free_version_info() >= (3, 3, 12), "old free version")
     @retry_before_failing()
     def test_used(self):
         free = free_physmem()
-- 
cgit v1.2.1


From 9068be71df0bbd41127c309afcf2917d0634a088 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 18:15:30 +0200
Subject: #877 - takes LCX containers into account and prevent 'avail' to
 overflow 'total'

---
 HISTORY.rst        |  3 ++-
 psutil/_pslinux.py | 25 +++++++++++++++----------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index 8104e174..34a34165 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -11,7 +11,8 @@ Bug tracker at https://github.com/giampaolo/psutil/issues
 - #885: ValueError is raised if a negative integer is passed to cpu_percent()
   functions.
 - #887: [Linux] free, available and used fields are more precise and match
-  "free" cmdline utility.
+  "free" cmdline utility. It also takes into account LCX containers preventing
+  "avail" to overflow "total".
 
 
 4.3.1 - 2016-09-01
diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 88e6a939..4d8f3db3 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -296,8 +296,8 @@ def virtual_memory():
     https://gitlab.com/procps-ng/procps/blob/
         24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c
 
-    ...or "free" / procps-ng-3.3.10 version which is available in Ubuntu
-    16.04 and which should report the same numbers.
+    For reference, procps-ng-3.3.10 is the version available on Ubuntu
+    16.04.
     """
     missing_fields = []
     mems = {}
@@ -308,8 +308,8 @@ def virtual_memory():
 
     # Note: these info are available also as cext.linux_sysinfo().
     total = mems[b'MemTotal:']
-    buffers = mems[b'Buffers:']
     free = mems[b'MemFree:']
+    buffers = mems[b'Buffers:']
     cached = mems[b"Cached:"]
     # "free" cmdline utility sums cached + reclamaible:
     # https://gitlab.com/procps-ng/procps/
@@ -320,8 +320,6 @@ def virtual_memory():
     #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
     cached += mems.get(b"SReclaimable:", 0)  # kernel 2.6.19
 
-    # Note: if 0 (e.g. Ubuntu 14.04, kernel 3.13) this can be
-    # determined from /proc/meminfo.
     try:
         shared = mems[b'Shmem:']  # since kernel 2.6.32
     except KeyError:
@@ -351,6 +349,12 @@ def virtual_memory():
             inactive = 0
             missing_fields.append('inactive')
 
+    # https://gitlab.com/procps-ng/procps/blob/
+    #     24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L769
+    used = total - free - cached - buffers
+    if used < 0:
+        used = total - free
+
     # Note: starting from 4.4.0 we match "free" "available" column.
     # Before 4.4.0 we calculated it as:
     # >>> avail = free + buffers + cached
@@ -368,12 +372,13 @@ def virtual_memory():
         #     /24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L774
         # We won't. Like this we'll match "htop".
         avail = free + buffers + cached
-
+    # If avail is greater than total or our calculation overflows,
+    # that's symptomatic of running within a LCX container where such
+    # values will be dramatically distorted over those of the host.
     # https://gitlab.com/procps-ng/procps/blob/
-    #     24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L769
-    used = total - free - cached - buffers
-    if used < 0:
-        used = total - free
+    #     24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L764
+    if avail > total:
+        avail = free
 
     percent = usage_percent((total - avail), total, _round=1)
 
-- 
cgit v1.2.1


From c4cad5a96f1d4c6d383d51e0da2344cedcf7028e Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Sun, 18 Sep 2016 18:25:03 +0200
Subject: update comments

---
 psutil/_pslinux.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py
index 4d8f3db3..1a0b57bc 100644
--- a/psutil/_pslinux.py
+++ b/psutil/_pslinux.py
@@ -311,14 +311,15 @@ def virtual_memory():
     free = mems[b'MemFree:']
     buffers = mems[b'Buffers:']
     cached = mems[b"Cached:"]
-    # "free" cmdline utility sums cached + reclamaible:
+    # "free" cmdline utility sums cached + reclamaible (available
+    # since kernel 2.6.19):
     # https://gitlab.com/procps-ng/procps/
     #     blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L761
     # Older versions of procps added slab memory instead.
     # This got changed in:
     # https://gitlab.com/procps-ng/procps/commit/
     #     05d751c4f076a2f0118b914c5e51cfbb4762ad8e
-    cached += mems.get(b"SReclaimable:", 0)  # kernel 2.6.19
+    cached += mems.get(b"SReclaimable:", 0)
 
     try:
         shared = mems[b'Shmem:']  # since kernel 2.6.32
@@ -335,11 +336,11 @@ def virtual_memory():
         active = 0
         missing_fields.append('active')
 
-    # https://gitlab.com/procps-ng/procps/
-    #   blob/195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L758
     try:
         inactive = mems[b"Inactive:"]
     except KeyError:
+        # https://gitlab.com/procps-ng/procps/blob/
+        #   195565746136d09333ded280cf3ba93853e855b8/proc/sysinfo.c#L758
         try:
             inactive = \
                 mems[b"Inact_dirty:"] + \
@@ -353,6 +354,8 @@ def virtual_memory():
     #     24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L769
     used = total - free - cached - buffers
     if used < 0:
+        # May be symptomatic of running within a LCX container where such
+        # values will be dramatically distorted over those of the host.
         used = total - free
 
     # Note: starting from 4.4.0 we match "free" "available" column.
-- 
cgit v1.2.1