Dependencies: remove simplejson (aiidateam#5391)

This library was used as a drop-in replacement for the `json` module of
the standard library to provide consistency of JSON (de)serializing
between Python 2 and Python 3. Since we have long since dropped support
for Python 2, we can now simply use the standard library.

The module `aiida.common.json` provided an interface to `simplejson` and
was used throughout `aiida-core` instead of `json`. This module is now
deprecated and `aiida-core` just uses `json` directly.

There are three significant changes that needed to be taken into account:

 * `simplejson` provided automatic serialization for `decimal.Decimal`
   but `json` does not. The support for serializing these types to the
   database is maintained by performing the serialization manually in
   the `aiida.orm.implementation.utils.clean_value` method. All instances
   of `decimal.Decimal` are serialized as `numbers.Real`, e.g. floats,
   so the behavior should remain the same.

 * The `aiida.common.json` wrapper functions `dump` and `load` accepted
   file objects in text and binary mode. However, the `json` analogues
   only support text files. The wrapper functions therefore had to be
   adapted to decode and encode, respectively, the contents of the file
   handle before passing it to the `json` function.

 * The code that calls `json.dump` passing a handle from a temporary file
   generated with `tempfile.NamedTemporaryFile` had to be updated to
   wrap the handle in `codecs.getwriter('utf-8')` since the default mode
   for `NamedTemporaryFile` is `w+b` and `json.dump` requires a text
   file handle.

Finally, the `aiida.common.json` module is deprecated and will emit a
deprecation warning when it is imported.

aiida/cmdline/utils/echo.py

@@ -10,6 +10,7 @@
 """Convenience functions for logging output from ``verdi`` commands."""
 import collections
 import enum
+import json
 import sys
 
 import click
@@ -207,7 +208,6 @@ def echo_formatted_list(collection, attributes, sort=None, highlight=None, hide=
 
 def _format_dictionary_json_date(dictionary, sort_keys=True):
     """Return a dictionary formatted as a string using the json format and converting dates to strings."""
-    from aiida.common import json
 
     def default_jsondump(data):
         """Function needed to decode datetimes, that would otherwise not be JSON-decodable."""

aiida/common/json.py

@@ -7,65 +7,73 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-"""
-Abstracts JSON usage to ensure compatibility with Python2 and Python3.
+"""Abstracts JSON usage to ensure compatibility with Python2 and Python3.
 
 Use this module prefentially over standard json to ensure compatibility.
 
+.. deprecated:: This module is deprecated in v2.0.0 and should no longer be used. Python 2 support has long since been
+    dropped and for Python 3, one should simply use the ``json`` module of the standard library directly.
+
 """
-import simplejson
+import codecs
+import json
+import warnings
 
-JSONEncoder = simplejson.JSONEncoder
+from aiida.common.warnings import AiidaDeprecationWarning
 
+warnings.warn(
+    'This module has been deprecated and should no longer be used. Use the `json` standard library instead.',
+    AiidaDeprecationWarning
+)
 
-def dump(data, fhandle, **kwargs):
-    """
-    Write JSON encoded 'data' to a file-like object, fhandle
-    Use open(filename, 'wb') to write.
-    The utf8write object is used to ensure that the resulting serialised data is
-    encoding as UTF8.
-    Any strings with non-ASCII characters need to be unicode strings.
-    We use ensure_ascii=False to write unicode characters specifically
-    as this improves the readability of the json and reduces the file size.
-    """
-    import codecs
-    utf8writer = codecs.getwriter('utf8')
-    simplejson.dump(data, utf8writer(fhandle), ensure_ascii=False, encoding='utf8', **kwargs)
 
+def dump(data, handle, **kwargs):
+    """Serialize ``data`` as a JSON formatted stream to ``handle``.
 
-def dumps(data, **kwargs):
-    """
-    Write JSON encoded 'data' to a string.
-    simplejson is useful here as it always returns unicode if ensure_ascii=False is used,
-    unlike the standard library json, rather than being dependant on the input.
-    We use also ensure_ascii=False to write unicode characters specifically
-    as this improves the readability of the json and reduces the file size.
-    When writing to file, use open(filename, 'w', encoding='utf8')
+    We use ``ensure_ascii=False`` to write unicode characters specifically as this improves the readability of the json
+    and reduces the file size.
     """
-    return simplejson.dumps(data, ensure_ascii=False, encoding='utf8', **kwargs)
+    try:
+        if 'b' in handle.mode:
+            handle = codecs.getwriter('utf-8')(handle)
+    except AttributeError:
+        pass
 
+    return json.dump(data, handle, ensure_ascii=False, **kwargs)
 
-def load(fhandle, **kwargs):
+
+def dumps(data, **kwargs):
+    """Serialize ``data`` as a JSON formatted string.
+
+    We use ``ensure_ascii=False`` to write unicode characters specifically as this improves the readability of the json
+    and reduces the file size.
     """
-    Deserialise a JSON file.
+    return json.dumps(data, ensure_ascii=False, **kwargs)
+
 
-    For encoding consistency, open(filename, 'r', encoding='utf8') should be used.
+def load(handle, **kwargs):
+    """Deserialize ``handle`` text or binary file containing a JSON document to a Python object.
 
-    :raises ValueError: if no valid JSON object could be decoded
+    :raises ValueError: if no valid JSON object could be decoded.
     """
+    if 'b' in handle.mode:
+        handle = codecs.getreader('utf-8')(handle)
+
     try:
-        return simplejson.load(fhandle, encoding='utf8', **kwargs)
-    except simplejson.errors.JSONDecodeError:
-        raise ValueError
+        return json.load(handle, **kwargs)
+    except json.JSONDecodeError as exc:
+        raise ValueError from exc
 
 
-def loads(json_string, **kwargs):
-    """
-    Deserialise a JSON string.
+def loads(string, **kwargs):
+    """Deserialize text or binary ``string`` containing a JSON document to a Python object.
 
-    :raises ValueError: if no valid JSON object could be decoded
+    :raises ValueError: if no valid JSON object could be decoded.
     """
+    if isinstance(string, bytes):
+        string = string.decode('utf-8')
+
     try:
-        return simplejson.loads(json_string, encoding='utf8', **kwargs)
-    except simplejson.errors.JSONDecodeError:
-        raise ValueError
+        return json.loads(string, **kwargs)
+    except json.JSONDecodeError as exc:
+        raise ValueError from exc

aiida/engine/processes/calcjobs/calcjob.py

@@ -9,6 +9,7 @@
 ###########################################################################
 """Implementation of the CalcJob process."""
 import io
+import json
 import os
 import shutil
 from typing import Any, Dict, Hashable, Optional, Type, Union
@@ -582,7 +583,6 @@ def presubmit(self, folder: Folder) -> CalcInfo:
 
         """
         # pylint: disable=too-many-locals,too-many-statements,too-many-branches
-        from aiida.common import json
         from aiida.common.datastructures import CodeInfo, CodeRunMode
         from aiida.common.exceptions import InputValidationError, InvalidOperation, PluginInternalError, ValidationError
         from aiida.common.utils import validate_list_of_string_tuples

aiida/manage/configuration/config.py

@@ -8,16 +8,17 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """Module that defines the configuration file of an AiiDA instance and functions to create and load it."""
+import codecs
 from functools import lru_cache
 from importlib import resources
+import json
 import os
 import shutil
 import tempfile
 from typing import Any, Dict, Optional, Sequence, Tuple
 
 import jsonschema
 
-from aiida.common import json
 from aiida.common.exceptions import ConfigurationError
 
 from . import schema as schema_module
@@ -81,7 +82,7 @@ def from_file(cls, filepath):
         from .migrations import check_and_migrate_config, config_needs_migrating
 
         try:
-            with open(filepath, 'r', encoding='utf8') as handle:
+            with open(filepath, 'rb') as handle:
                 config = json.load(handle)
         except FileNotFoundError:
             config = Config(filepath, check_and_migrate_config({}))
@@ -492,7 +493,7 @@ def store(self):
         # Otherwise, we write the content to a temporary file and compare its md5 checksum with the current config on
         # disk. When the checksums differ, we first create a backup and only then overwrite the existing file.
         with tempfile.NamedTemporaryFile() as handle:
-            json.dump(self.dictionary, handle, indent=DEFAULT_CONFIG_INDENT_SIZE)
+            json.dump(self.dictionary, codecs.getwriter('utf-8')(handle), indent=DEFAULT_CONFIG_INDENT_SIZE)
             handle.seek(0)
 
             if md5_from_filelike(handle) != md5_file(self.filepath):
@@ -522,7 +523,7 @@ def _atomic_write(self, filepath=None):
         # Create a temporary file in the same directory as the target filepath, which guarantees that the temporary
         # file is on the same filesystem, which is necessary to be able to use ``os.rename``. Since we are moving the
         # temporary file, we should also tell the tempfile to not be automatically deleted as that will raise.
-        with tempfile.NamedTemporaryFile(dir=os.path.dirname(filepath), delete=False) as handle:
+        with tempfile.NamedTemporaryFile(dir=os.path.dirname(filepath), delete=False, mode='w') as handle:
             try:
                 json.dump(self.dictionary, handle, indent=DEFAULT_CONFIG_INDENT_SIZE)
             finally:

aiida/orm/implementation/utils.py

@@ -63,7 +63,7 @@ def clean_builtin(val):
         It mainly checks that we don't store NaN or Inf.
         """
         # This is a whitelist of all the things we understand currently
-        if val is None or isinstance(val, (bool, str, Decimal)):
+        if val is None or isinstance(val, (bool, str)):
             return val
 
         # This fixes #2773 - in python3, ``numpy.int64(-1)`` cannot be json-serialized
@@ -77,7 +77,7 @@ def clean_builtin(val):
 
         # This is for float-like types, like ``numpy.float128`` that are not json-serializable
         # Note that `numbers.Real` also match booleans but they are already returned above
-        if isinstance(val, numbers.Real):
+        if isinstance(val, (numbers.Real, Decimal)):
             string_representation = f'{{:.{AIIDA_FLOAT_PRECISION}g}}'.format(val)
             new_val = float(string_representation)
             if 'e' in string_representation and new_val.is_integer():

aiida/orm/nodes/data/array/bands.py

@@ -12,6 +12,7 @@
 This module defines the classes related to band structures or dispersions
 in a Brillouin zone, and how to operate on them.
 """
+import json
 from string import Template
 
 import numpy
@@ -811,8 +812,6 @@ def _prepare_mpl_singlefile(self, *args, **kwargs):
         For the possible parameters, see documentation of
         :py:meth:`~aiida.orm.nodes.data.array.bands.BandsData._matplotlib_get_dict`
         """
-        from aiida.common import json
-
         all_data = self._matplotlib_get_dict(*args, **kwargs)
 
         s_header = MATPLOTLIB_HEADER_TEMPLATE.substitute()
@@ -834,8 +833,6 @@ def _prepare_mpl_withjson(self, main_file_name='', *args, **kwargs):  # pylint:
         """
         import os
 
-        from aiida.common import json
-
         all_data = self._matplotlib_get_dict(*args, main_file_name=main_file_name, **kwargs)
 
         json_fname = os.path.splitext(main_file_name)[0] + '_data.json'
@@ -866,8 +863,6 @@ def _prepare_mpl_pdf(self, main_file_name='', *args, **kwargs):  # pylint: disab
         import sys
         import tempfile
 
-        from aiida.common import json
-
         all_data = self._matplotlib_get_dict(*args, **kwargs)
 
         # Use the Agg backend
@@ -911,7 +906,6 @@ def _prepare_mpl_png(self, main_file_name='', *args, **kwargs):  # pylint: disab
         For the possible parameters, see documentation of
         :py:meth:`~aiida.orm.nodes.data.array.bands.BandsData._matplotlib_get_dict`
         """
-        import json
         import os
         import subprocess
         import sys
@@ -1237,7 +1231,6 @@ def _prepare_json(self, main_file_name='', comments=True):  # pylint: disable=un
             format)
         """
         from aiida import get_file_header
-        from aiida.common import json
 
         json_dict = self._get_band_segments(cartesian=True)
         json_dict['original_uuid'] = self.uuid

aiida/orm/nodes/data/structure.py

@@ -15,6 +15,7 @@
 import copy
 import functools
 import itertools
+import json
 
 from aiida.common.constants import elements
 from aiida.common.exceptions import UnsupportedSpeciesError
@@ -1016,8 +1017,6 @@ def _prepare_chemdoodle(self, main_file_name=''):  # pylint: disable=unused-argu
 
         import numpy as np
 
-        from aiida.common import json
-
         supercell_factors = [1, 1, 1]
 
         # Get cell vectors and atomic position

aiida/restapi/translator/nodes/data/array/bands.py

@@ -10,6 +10,7 @@
 """
 Translator for bands data
 """
+import json
 
 from aiida.restapi.translator.nodes.data import DataTranslator
 
@@ -44,7 +45,6 @@ def get_derived_properties(node):
         """
         response = {}
 
-        from aiida.common import json
         json_string = node._exportcontent('json', comments=False)  # pylint: disable=protected-access
         json_content = json.loads(json_string[0])
         response['bands'] = json_content

aiida/schedulers/datastructures.py

@@ -16,6 +16,7 @@
 """
 import abc
 import enum
+import json
 
 from aiida.common import AIIDA_LOGGER
 from aiida.common.extendeddicts import AttributeDict, DefaultFieldsAttributeDict
@@ -543,8 +544,6 @@ def serialize(self):
 
         :return: A string with serialised representation of the current data.
         """
-        from aiida.common import json
-
         return json.dumps(self.get_dict())
 
     def get_dict(self):
@@ -574,6 +573,4 @@ def load_from_serialized(cls, data):
 
         :param data: The string with the JSON-serialised data to load from
         """
-        from aiida.common import json
-
         return cls.load_from_dict(json.loads(data))

aiida/storage/psql_dos/migrations/utils/create_dbattribute.py

@@ -14,8 +14,8 @@
 from __future__ import annotations
 
 import datetime
+import json
 
-from aiida.common import json
 from aiida.common.exceptions import ValidationError
 from aiida.common.timezone import get_current_timezone, is_naive, make_aware
 

aiida/storage/psql_dos/migrations/utils/legacy_workflows.py

@@ -9,13 +9,14 @@
 ###########################################################################
 # pylint: disable=invalid-name
 """Utilities for removing legacy workflows."""
+import codecs
+import json
 import sys
 
 import click
 from sqlalchemy.sql import func, select, table
 
 from aiida.cmdline.utils import echo
-from aiida.common import json
 
 
 def json_serializer(obj):
@@ -70,7 +71,7 @@ def export_workflow_data(connection, profile):
         prefix='legacy-workflows', suffix='.json', dir='.', delete=delete_on_close, mode='wb'
     ) as handle:
         filename = handle.name
-        json.dump(data, handle, default=json_serializer)
+        json.dump(data, codecs.getwriter('utf-8')(handle), default=json_serializer)
 
     # If delete_on_close is False, we are running for the user and add additional message of file location
     if not delete_on_close:

aiida/storage/psql_dos/migrations/utils/utils.py

@@ -12,6 +12,7 @@
 import datetime
 import functools
 import io
+import json
 import os
 import pathlib
 import re
@@ -21,7 +22,7 @@
 from disk_objectstore.utils import LazyOpener
 import numpy
 
-from aiida.common import exceptions, json
+from aiida.common import exceptions
 from aiida.repository.backend import AbstractRepositoryBackend
 from aiida.repository.common import File, FileType
 from aiida.repository.repository import Repository

aiida/storage/psql_dos/utils.py

@@ -9,6 +9,7 @@
 ###########################################################################
 # pylint: disable=import-error,no-name-in-module
 """Utility functions specific to the SqlAlchemy backend."""
+import json
 from typing import TypedDict
 
 
@@ -33,8 +34,6 @@ def create_sqlalchemy_engine(config: PsqlConfig):
     """
     from sqlalchemy import create_engine
 
-    from aiida.common import json
-
     # The hostname may be `None`, which is a valid value in the case of peer authentication for example. In this case
     # it should be converted to an empty string, because otherwise the `None` will be converted to string literal "None"
     hostname = config['database_hostname'] or ''