From a0b096c88421548593ecebe93bbe369385087f3b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 15 Apr 2023 13:39:02 +0200 Subject: [NetBSD] virtual_memory() metrics are completely wrong (#2235), fixes #2234 Match values shown by **htop**. This is before. In here: * available mem is almost the same as total (unrealistic) * used is higher than total; there's also a failing test: ``` MEMORY ------ Total : 972.7M Available : 959.1M Percent : 1.4 Used : 1.1G Free : 173.6M Active : 434.3M Inactive : 258.4M Buffers : 509.4M Cached : 692.9M Shared : 0.0B Wired : 280.0K ``` Now: ``` MEMORY ------ Total : 972.7M Available : 538.1M Percent : 44.7 Used : 434.5M Free : 173.6M Active : 434.2M Inactive : 258.4M Buffers : 509.4M Cached : 692.9M Shared : 0.0B Wired : 280.0K ``` --- HISTORY.rst | 2 ++ docs/index.rst | 15 +++++++++------ psutil/_psbsd.py | 41 +++++++++++++++++++++++------------------ psutil/arch/netbsd/mem.c | 24 ++++++++++++------------ 4 files changed, 46 insertions(+), 36 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 63b43c69..d214e61f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,8 @@ - 2229_, [OpenBSD]: unable to properly recognize zombie processes. `NoSuchProcess`_ may be raised instead of `ZombieProcess`_. - 2231_, [NetBSD]: *available* `virtual_memory()`_ is higher than *total*. +- 2234_, [NetBSD]: `virtual_memory()`_ metrics are wrong: *available* and + *used* are too high. We now match values shown by *htop* CLI utility. 5.9.4 ===== diff --git a/docs/index.rst b/docs/index.rst index dfca444d..2a1c5b6a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -334,18 +334,20 @@ Memory .. function:: virtual_memory() Return statistics about system memory usage as a named tuple including the - following fields, expressed in bytes. Main metrics: + following fields, expressed in bytes. + + Main metrics: - **total**: total physical memory (exclusive swap). - **available**: the memory that can be given instantly to processes without the system going into swap. - This is calculated by summing different memory values depending on the - platform and it is supposed to be used to monitor actual memory usage in a - cross platform fashion. + This is calculated by summing different memory metrics that vary depending + on the platform. It is supposed to be used to monitor actual memory usage + in a cross platform fashion. + - **percent**: the percentage usage calculated as ``(total - available) / total * 100``. Other metrics: - - **percent**: the percentage usage calculated as ``(total - available) / total * 100`` - **used**: memory used, calculated differently depending on the platform and designed for informational purposes only. **total - free** does not necessarily match **used**. @@ -370,7 +372,8 @@ Memory human readable form. .. note:: if you just want to know how much physical memory is left in a - cross platform fashion simply rely on the **available** field. + cross platform fashion simply rely on **available** and **percent** + fields. >>> import psutil >>> mem = psutil.virtual_memory() diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index c1cc1c03..9dd8420d 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -176,11 +176,10 @@ else: # ===================================================================== -if NETBSD: - def virtual_memory(): - """System virtual memory as a namedtuple.""" - mem = cext.virtual_mem() - total, free, active, inactive, wired, cached, avail = mem +def virtual_memory(): + mem = cext.virtual_mem() + if NETBSD: + total, free, active, inactive, wired, cached = mem # On NetBSD buffers and shared mem is determined via /proc. # The C ext set them to 0. with open('/proc/meminfo', 'rb') as f: @@ -189,22 +188,28 @@ if NETBSD: buffers = int(line.split()[1]) * 1024 elif line.startswith(b'MemShared:'): shared = int(line.split()[1]) * 1024 - elif line.startswith(b'Cached:'): - cached = int(line.split()[1]) * 1024 - used = active + wired + cached - percent = usage_percent((total - avail), total, round_=1) - return svmem(total, avail, percent, used, free, - active, inactive, buffers, cached, shared, wired) -else: - def virtual_memory(): - """System virtual memory as a namedtuple.""" - mem = cext.virtual_mem() + # Before avail was calculated as (inactive + cached + free), + # same as zabbix, but it turned out it could exceed total (see + # #2233), so zabbix seems to be wrong. Htop calculates it + # differently, and the used value seem more realistic, so let's + # match htop. + # https://github.com/htop-dev/htop/blob/e7f447b/netbsd/NetBSDProcessList.c#L162 # noqa + # https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/netbsd/memory.c#L135 # noqa + used = active + wired + avail = total - used + else: total, free, active, inactive, wired, cached, buffers, shared = mem + # matches freebsd-memory CLI: + # * https://people.freebsd.org/~rse/dist/freebsd-memory + # * https://www.cyberciti.biz/files/scripts/freebsd-memory.pl.txt + # matches zabbix: + # * https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/freebsd/memory.c#L143 # noqa avail = inactive + cached + free used = active + wired + cached - percent = usage_percent((total - avail), total, round_=1) - return svmem(total, avail, percent, used, free, - active, inactive, buffers, cached, shared, wired) + + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, used, free, + active, inactive, buffers, cached, shared, wired) def swap_memory(): diff --git a/psutil/arch/netbsd/mem.c b/psutil/arch/netbsd/mem.c index 26ab8e69..456479ba 100644 --- a/psutil/arch/netbsd/mem.c +++ b/psutil/arch/netbsd/mem.c @@ -31,8 +31,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { size_t size; struct uvmexp_sysctl uv; int mib[] = {CTL_VM, VM_UVMEXP2}; - long pagesize = psutil_getpagesize(); - unsigned long long available; + long long cached; size = sizeof(uv); if (sysctl(mib, 2, &uv, &size, NULL, 0) < 0) { @@ -40,17 +39,18 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { return NULL; } - // follow zabbix - available = uv.inactive + uv.execpages + uv.filepages + uv.free; + // Note: zabbix does not include anonpages, but that doesn't match the + // "Cached" value in /proc/meminfo. + // https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/netbsd/memory.c#L182 + cached = (uv.filepages + uv.execpages + uv.anonpages) << uv.pageshift; return Py_BuildValue( - "KKKKKKK", - (unsigned long long) uv.npages << uv.pageshift, // total - (unsigned long long) uv.free << uv.pageshift, // free - (unsigned long long) uv.active << uv.pageshift, // active - (unsigned long long) uv.inactive << uv.pageshift, // inactive - (unsigned long long) uv.wired << uv.pageshift, // wired - (unsigned long long) (uv.filepages + uv.execpages) * pagesize, // cached - available << uv.pageshift // available + "LLLLLL", + (long long) uv.npages << uv.pageshift, // total + (long long) uv.free << uv.pageshift, // free + (long long) uv.active << uv.pageshift, // active + (long long) uv.inactive << uv.pageshift, // inactive + (long long) uv.wired << uv.pageshift, // wired + cached // cached ); } -- cgit v1.2.1