Merge pull request nicolargo#2938 from nicolargo/feature/json-serialization-fallbacks

JSON serialization fallbacks on ujson and builtin json lib

Author: RazCrimson

glances/exports/glances_graph/__init__.py

@@ -17,7 +17,7 @@
 from pygal import DateTimeLine
 
 from glances.exports.export import GlancesExport
-from glances.globals import iteritems, time_serie_subsample
+from glances.globals import iteritems, time_series_subsample
 from glances.logger import logger
 from glances.timer import Timer
 
@@ -120,7 +120,7 @@ def export(self, title, data):
             x_label_rotation=20,
             x_value_formatter=lambda dt: dt.strftime('%Y/%m/%d %H:%M:%S'),
         )
-        for k, v in iteritems(time_serie_subsample(data, self.width)):
+        for k, v in iteritems(time_series_subsample(data, self.width)):
             chart.add(k, v)
         chart.render_to_file(os.path.join(self.path, title + '.svg'))
         return True

glances/exports/glances_json/__init__.py

@@ -47,8 +47,8 @@ def export(self, name, columns, points):
             logger.debug(f"Exporting stats ({listkeys(self.buffer)}) to JSON file ({self.json_filename})")
 
             # Export stats to JSON file
-            with open(self.json_filename, "w") as self.json_file:
-                self.json_file.write(f"{json_dumps(self.buffer)}\n")
+            with open(self.json_filename, "wb") as self.json_file:
+                self.json_file.write(json_dumps(self.buffer) + b'\n')
 
             # Reset buffer
             self.buffer = {}

glances/exports/glances_kafka/__init__.py

@@ -52,7 +52,7 @@ def init(self):
         try:
             s = KafkaProducer(
                 bootstrap_servers=server_uri,
-                value_serializer=lambda v: json_dumps(v).encode('utf-8'),
+                value_serializer=lambda v: json_dumps(v),
                 compression_type=self.compression,
             )
         except Exception as e:

glances/exports/glances_zeromq/__init__.py

@@ -11,7 +11,6 @@
 import sys
 
 import zmq
-from zmq.utils.strtypes import asbytes
 
 from glances.exports.export import GlancesExport
 from glances.globals import b, json_dumps
@@ -81,7 +80,7 @@ def export(self, name, columns, points):
         # - First frame containing the following prefix (STRING)
         # - Second frame with the Glances plugin name (STRING)
         # - Third frame with the Glances plugin stats (JSON)
-        message = [b(self.prefix), b(name), asbytes(json_dumps(data))]
+        message = [b(self.prefix), b(name), json_dumps(data)]
 
         # Write data to the ZeroMQ bus
         # Result can be view: tcp://host:port

glances/globals.py

@@ -16,6 +16,7 @@
 import base64
 import errno
 import functools
+import importlib
 import os
 import platform
 import queue
@@ -27,7 +28,7 @@
 from datetime import datetime
 from operator import itemgetter, methodcaller
 from statistics import mean
-from typing import Dict, List, Union
+from typing import Any, Dict, List, Union
 from urllib.error import HTTPError, URLError
 from urllib.parse import urlparse
 from urllib.request import Request, urlopen
@@ -36,14 +37,34 @@
 
 from defusedxml.xmlrpc import monkey_patch
 
-# Optionally use orjson if available
+# Correct issue #1025 by monkey path the xmlrpc lib
+monkey_patch()
+
+# Prefer faster libs for JSON (de)serialization
+# Preference Order: orjson > ujson > json (builtin)
 try:
     import orjson as json
+
+    json.dumps = functools.partial(json.dumps, option=json.OPT_NON_STR_KEYS)
 except ImportError:
-    import json
+    # Need to log info but importing logger will cause cyclic imports
+    pass
 
-# Correct issue #1025 by monkey path the xmlrpc lib
-monkey_patch()
+if 'json' not in globals():
+    try:
+        # Note: ujson is not officially supported
+        # Available as a fallback to allow orjson's unsupported platforms to use a faster serialization lib
+        import ujson as json
+    except ImportError:
+        import json
+
+    # To allow ujson & json dumps to serialize datetime
+    def _json_default(v: Any) -> Any:
+        if isinstance(v, datetime):
+            return v.isoformat()
+        return v
+
+    json.dumps = functools.partial(json.dumps, default=_json_default)
 
 ##############
 # GLOBALS VARS
@@ -166,7 +187,7 @@ def subsample(data, sampling):
     return [mean(data[s * sampling_length : (s + 1) * sampling_length]) for s in range(0, sampling)]
 
 
-def time_serie_subsample(data, sampling):
+def time_series_subsample(data, sampling):
     """Compute a simple mean subsampling.
 
     Data should be a list of set (time, value)
@@ -303,20 +324,22 @@ def urlopen_auth(url, username, password):
     return urlopen(
         Request(
             url,
-            headers={'Authorization': 'Basic ' + base64.b64encode((f'{username}:{password}').encode()).decode()},
+            headers={'Authorization': 'Basic ' + base64.b64encode(f'{username}:{password}'.encode()).decode()},
         )
     )
 
 
-def json_dumps(data) -> str:
+def json_dumps(data) -> bytes:
     """Return the object data in a JSON format.
 
     Manage the issue #815 for Windows OS with UnicodeDecodeError catching.
     """
     try:
-        return json.dumps(data)
+        res = json.dumps(data)
     except UnicodeDecodeError:
-        return json.dumps(data, ensure_ascii=False)
+        res = json.dumps(data, ensure_ascii=False)
+    # ujson & json libs return strings, but our contract expects bytes
+    return b(res)
 
 
 def json_loads(data: Union[str, bytes, bytearray]) -> Union[Dict, List]: