Merge branch 'master' of github.com:giampaolo/psutil

.github/workflows/build.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
         # os: [ubuntu-latest, macos-latest, windows-latest]
-        os: [ubuntu-latest, macos-10.15]
+        os: [ubuntu-latest, macos-12]
         include:
           - {name: Linux, python: '3.9', os: ubuntu-latest}
     env:
@@ -86,7 +86,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-10.15]
+        os: [ubuntu-latest, macos-12]
         include:
           - {name: Linux, python: '3.9', os: ubuntu-latest}
     env:

CREDITS

@@ -781,7 +781,7 @@ I: 1956
 
 N: Matthieu Darbois
 W: https://github.com/mayeut
-I: 2039, 2142
+I: 2039, 2142, 2147, 2153
 
 N: Hugo van Kemenade
 W: https://github.com/hugovk

HISTORY.rst

@@ -14,8 +14,11 @@ XXXX-XX-XX
   undefined ``ethtool_cmd_speed`` symbol.
 - 2142_, [POSIX]: `net_if_stats()`_ 's ``flags`` on Python 2 returned unicode
   instead of str.  (patch by Matthieu Darbois)
+- 2147_, [macOS] Fix disk usage report on macOS 12+.  (patch by Matthieu Darbois)
 - 2150_, [Linux] `Process.threads()`_ may raise ``NoSuchProcess``. Fix race
   condition.  (patch by Daniel Li)
+- 2153_, [macOS] Fix race condition in test_posix.TestProcess.test_cmdline.
+  (patch by Matthieu Darbois)
 
 5.9.2
 =====

psutil/_psposix.py

@@ -14,6 +14,7 @@
 from ._common import memoize
 from ._common import sdiskusage
 from ._common import usage_percent
+from ._common import MACOS
 from ._compat import PY3
 from ._compat import ChildProcessError
 from ._compat import FileNotFoundError
@@ -22,6 +23,9 @@
 from ._compat import ProcessLookupError
 from ._compat import unicode
 
+if MACOS:
+    from . import _psutil_osx
+
 
 if sys.version_info >= (3, 4):
     import enum
@@ -193,6 +197,9 @@ def disk_usage(path):
     avail_to_user = (st.f_bavail * st.f_frsize)
     # Total space being used in general.
     used = (total - avail_to_root)
+    if MACOS:
+        # see: https://github.com/giampaolo/psutil/pull/2152
+        used = _psutil_osx.disk_usage_used(path, used)
     # Total space which is available to user (same as 'total' but
     # for the user).
     total_user = used + avail_to_user

psutil/_psutil_osx.c

@@ -836,6 +836,52 @@ psutil_disk_partitions(PyObject *self, PyObject *args) {
 }
 
 
+static PyObject *
+psutil_disk_usage_used(PyObject *self, PyObject *args) {
+    PyObject *py_default_value;
+    PyObject *py_mount_point_bytes = NULL;
+    char* mount_point;
+
+#if PY_MAJOR_VERSION >= 3
+    if (!PyArg_ParseTuple(args, "O&O", PyUnicode_FSConverter, &py_mount_point_bytes, &py_default_value)) {
+        return NULL;
+    }
+    mount_point = PyBytes_AsString(py_mount_point_bytes);
+    if (NULL == mount_point) {
+        Py_XDECREF(py_mount_point_bytes);
+        return NULL;
+    }
+#else
+    if (!PyArg_ParseTuple(args, "sO", &mount_point, &py_default_value)) {
+        return NULL;
+    }
+#endif
+
+#ifdef ATTR_VOL_SPACEUSED
+    /* Call getattrlist(ATTR_VOL_SPACEUSED) to get used space info. */
+    int ret;
+    struct {
+        uint32_t size;
+        uint64_t spaceused;
+    } __attribute__((aligned(4), packed)) attrbuf = {0};
+    struct attrlist attrs = {0};
+
+    attrs.bitmapcount = ATTR_BIT_MAP_COUNT;
+    attrs.volattr = ATTR_VOL_INFO | ATTR_VOL_SPACEUSED;
+    Py_BEGIN_ALLOW_THREADS
+    ret = getattrlist(mount_point, &attrs, &attrbuf, sizeof(attrbuf), 0);
+    Py_END_ALLOW_THREADS
+    if (ret == 0) {
+        Py_XDECREF(py_mount_point_bytes);
+        return PyLong_FromUnsignedLongLong(attrbuf.spaceused);
+    }
+    psutil_debug("getattrlist(ATTR_VOL_SPACEUSED) failed, fall-back to default value");
+#endif
+    Py_XDECREF(py_mount_point_bytes);
+    Py_INCREF(py_default_value);
+    return py_default_value;
+}
+
 /*
  * Return process threads
  */
@@ -1681,6 +1727,7 @@ static PyMethodDef mod_methods[] = {
     {"cpu_times", psutil_cpu_times, METH_VARARGS},
     {"disk_io_counters", psutil_disk_io_counters, METH_VARARGS},
     {"disk_partitions", psutil_disk_partitions, METH_VARARGS},
+    {"disk_usage_used", psutil_disk_usage_used, METH_VARARGS},
     {"net_io_counters", psutil_net_io_counters, METH_VARARGS},
     {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS},
     {"pids", psutil_pids, METH_VARARGS},

psutil/tests/__init__.py

@@ -18,6 +18,7 @@
 import gc
 import inspect
 import os
+import platform
 import random
 import re
 import select
@@ -47,6 +48,7 @@
 from psutil import SUNOS
 from psutil import WINDOWS
 from psutil._common import bytes2human
+from psutil._common import memoize
 from psutil._common import print_color
 from psutil._common import supports_ipv6
 from psutil._compat import PY3
@@ -84,7 +86,8 @@
     "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS",
     "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT",
     "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS",
-    "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO",
+    "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", "MACOS_11PLUS",
+    "MACOS_12PLUS",
     # subprocesses
     'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie',
     'spawn_children_pair',
@@ -129,6 +132,35 @@
 IS_64BIT = sys.maxsize > 2 ** 32
 
 
+@memoize
+def macos_version():
+    version_str = platform.mac_ver()[0]
+    version = tuple(map(int, version_str.split(".")[:2]))
+    if version == (10, 16):
+        # When built against an older macOS SDK, Python will report
+        # macOS 10.16 instead of the real version.
+        version_str = subprocess.check_output(
+            [
+                sys.executable,
+                "-sS",
+                "-c",
+                "import platform; print(platform.mac_ver()[0])",
+            ],
+            env={"SYSTEM_VERSION_COMPAT": "0"},
+            universal_newlines=True,
+        )
+        version = tuple(map(int, version_str.split(".")[:2]))
+    return version
+
+
+if MACOS:
+    MACOS_11PLUS = macos_version() > (10, 15)
+    MACOS_12PLUS = macos_version() >= (12, 0)
+else:
+    MACOS_11PLUS = False
+    MACOS_12PLUS = False
+
+
 # --- configurable defaults
 
 # how many times retry_on_failure() decorator will retry

psutil/tests/test_posix.py

@@ -271,6 +271,12 @@ def test_exe(self):
             adjusted_ps_pathname = ps_pathname[:len(ps_pathname)]
             self.assertEqual(ps_pathname, adjusted_ps_pathname)
 
+    # On macOS the official python installer exposes a python wrapper that
+    # executes a python executable hidden inside an application bundle inside
+    # the Python framework.
+    # There's a race condition between the ps call & the psutil call below
+    # depending on the completion of the execve call so let's retry on failure
+    @retry_on_failure()
     def test_cmdline(self):
         ps_cmdline = ps_args(self.pid)
         psutil_cmdline = " ".join(psutil.Process(self.pid).cmdline())

psutil/tests/test_process.py

@@ -49,6 +49,7 @@
 from psutil.tests import HAS_PROC_IO_COUNTERS
 from psutil.tests import HAS_RLIMIT
 from psutil.tests import HAS_THREADS
+from psutil.tests import MACOS_11PLUS
 from psutil.tests import PYPY
 from psutil.tests import PYTHON_EXE
 from psutil.tests import PsutilTestCase
@@ -1426,6 +1427,10 @@ def clean_dict(d):
 
     @unittest.skipIf(not HAS_ENVIRON, "not supported")
     @unittest.skipIf(not POSIX, "POSIX only")
+    @unittest.skipIf(
+        MACOS_11PLUS,
+        "macOS 11+ can't get another process environment, issue #2084"
+    )
     def test_weird_environ(self):
         # environment variables can contain values without an equals sign
         code = textwrap.dedent("""

psutil/tests/test_system.py

@@ -44,6 +44,7 @@
 from psutil.tests import HAS_SENSORS_FANS
 from psutil.tests import HAS_SENSORS_TEMPERATURES
 from psutil.tests import IS_64BIT
+from psutil.tests import MACOS_12PLUS
 from psutil.tests import PYPY
 from psutil.tests import UNICODE_SUFFIX
 from psutil.tests import PsutilTestCase
@@ -561,8 +562,10 @@ def test_disk_usage(self):
             self.assertEqual(usage.total, shutil_usage.total)
             self.assertAlmostEqual(usage.free, shutil_usage.free,
                                    delta=tolerance)
-            self.assertAlmostEqual(usage.used, shutil_usage.used,
-                                   delta=tolerance)
+            if not MACOS_12PLUS:
+                # see https://github.com/giampaolo/psutil/issues/2147
+                self.assertAlmostEqual(usage.used, shutil_usage.used,
+                                       delta=tolerance)
 
         # if path does not exist OSError ENOENT is expected across
         # all platforms