fix json utilty methods, and add tests to cover

Author: JJ11teen

README.md

@@ -69,8 +69,8 @@ Each `cloud-mapping` keeps an internal dict of [etags](https://en.wikipedia.org/
 
 ### Serialisation
 
-If you don't call `.with_pickle()` and instead pass your providers configuration directly to the `cloud-mapping` class, you will get a "raw" `cloud-mapping` which only accepts byte-likes as values. You may build your own serialisation either using [zict](https://zict.readthedocs.io/en/latest/); or by calling `.with_buffers([dumps_1, loads_1, dumps_2, loads_2, ...])` where `dumps` and `loads` are the ordered functions to serialise and deserialise your data respectively.
+If you don't call `.with_pickle()` and instead pass your providers configuration directly to the `CloudMapping` class, you will get a "raw" `cloud-mapping` which accepts only byte-likes as values. Along with the `.with_pickle()` serialisation utility, `.with_json()` and `.with_json_zlib()` also exist.
 
-The following exist as common starting points: `.with_pickle()`, `.with_json()`, `.with_json_zlib()`.
+You may build your own serialisation either using [zict](https://zict.readthedocs.io/en/latest/); or by calling `.with_buffers([dumps_1, dumps_2, ..., dumps_N], [loads_1, loads_2, ..., loads_N])`, where `dumps` and `loads` are the ordered functions to serialise and parse your data respectively.
 
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)

setup.cfg

@@ -1,7 +1,7 @@
 [metadata]
 # replace with your username:
 name = cloud-mappings
-version = 0.7.5
+version = 0.7.6
 author = Lucas Sargent
 author_email = lucas.sargent@eliiza.com.au
 description = MutableMapping interfaces for common cloud storage providers

src/cloudmappings/cloudstoragemapping.py

@@ -1,3 +1,4 @@
+from functools import partial
 from typing import MutableMapping, Dict
 from urllib.parse import quote, unquote
 
@@ -15,90 +16,103 @@ def _unsafe_key(key: str) -> str:
 
 
 class CloudMapping(MutableMapping):
-    etags: Dict[str, str]
+    _etags: Dict[str, str]
 
     def __init__(
         self,
         storageprovider: StorageProvider,
         sync_initially: bool = True,
     ) -> None:
         self._storageprovider = storageprovider
-        self.etags = {}
+        self._etags = {}
         if self._storageprovider.create_if_not_exists() and sync_initially:
             self.sync_with_cloud()
 
     def sync_with_cloud(self, key: str = None) -> None:
         prefix_key = _safe_key(key) if key is not None else None
-        self.etags.update({_unsafe_key(k): i for k, i in self._storageprovider.list_keys_and_etags(prefix_key).items()})
+        self._etags.update(
+            {_unsafe_key(k): i for k, i in self._storageprovider.list_keys_and_etags(prefix_key).items()}
+        )
+
+    @property
+    def etags(self):
+        return self._etags
 
     def __getitem__(self, key: str) -> bytes:
-        if key not in self.etags:
+        if key not in self._etags:
             raise KeyError(key)
-        return self._storageprovider.download_data(key=_safe_key(key), etag=self.etags[key])
+        return self._storageprovider.download_data(key=_safe_key(key), etag=self._etags[key])
 
     def __setitem__(self, key: str, value: bytes) -> None:
         if not isinstance(value, bytes):
             raise ValueError("Value must be bytes like")
-        self.etags[key] = self._storageprovider.upload_data(
+        self._etags[key] = self._storageprovider.upload_data(
             key=_safe_key(key),
-            etag=self.etags.get(key, None),
+            etag=self._etags.get(key, None),
             data=value,
         )
 
     def __delitem__(self, key: str) -> None:
-        if key not in self.etags:
+        if key not in self._etags:
             raise KeyError(key)
-        self._storageprovider.delete_data(key=_safe_key(key), etag=self.etags[key])
-        del self.etags[key]
+        self._storageprovider.delete_data(key=_safe_key(key), etag=self._etags[key])
+        del self._etags[key]
 
     def __contains__(self, key: str) -> bool:
-        return key in self.etags
+        return key in self._etags
 
     def keys(self):
-        return iter(self.etags.keys())
+        return iter(self._etags.keys())
 
     __iter__ = keys
 
     def __len__(self) -> int:
-        return len(self.etags)
+        return len(self._etags)
 
     def __repr__(self) -> str:
         return f"cloudmapping<{self._storageprovider.safe_name()}>"
 
     @classmethod
-    def with_buffers(cls, io_buffers, *args, **kwargs) -> MutableMapping:
+    def with_buffers(cls, input_buffers, output_buffers, *args, **kwargs) -> "CloudMapping":
         from zict import Func
 
-        if len(io_buffers) % 2 != 0:
+        if len(input_buffers) != len(output_buffers):
             raise ValueError("Must have an equal number of input buffers as output buffers")
 
         raw_mapping = cls(*args, **kwargs)
         mapping = raw_mapping
 
-        for dump, load in zip(io_buffers[::2], io_buffers[1::2]):
+        for dump, load in zip(input_buffers[::-1], output_buffers):
             mapping = Func(dump, load, mapping)
 
         mapping.sync_with_cloud = raw_mapping.sync_with_cloud
+        mapping.etags = raw_mapping.etags
         return mapping
 
     @classmethod
-    def with_pickle(cls, *args, **kwargs) -> MutableMapping:
+    def with_pickle(cls, *args, **kwargs) -> "CloudMapping":
         import pickle
 
-        return cls.with_buffers([pickle.dumps, pickle.loads], *args, **kwargs)
+        return cls.with_buffers([pickle.dumps], [pickle.loads], *args, **kwargs)
 
     @classmethod
-    def with_json(cls, *args, **kwargs) -> MutableMapping:
+    def with_json(cls, encoding="utf-8", *args, **kwargs) -> "CloudMapping":
         import json
 
-        return cls.with_buffers([json.dumps, json.loads], *args, **kwargs)
+        return cls.with_buffers(
+            [json.dumps, partial(bytes, encoding=encoding)],
+            [partial(str, encoding=encoding), json.loads],
+            *args,
+            **kwargs,
+        )
 
     @classmethod
-    def with_json_zlib(cls, *args, **kwargs) -> MutableMapping:
+    def with_json_zlib(cls, encoding="utf-8", *args, **kwargs) -> "CloudMapping":
         import json, zlib
 
         return cls.with_buffers(
-            [json.dumps, json.loads, zlib.compress, zlib.decompress],
+            [json.dumps, partial(bytes, encoding=encoding), zlib.compress],
+            [zlib.decompress, partial(str, encoding=encoding), json.loads],
             *args,
             **kwargs,
         )

tests/tests/4_cloudmappingutilities.py

@@ -0,0 +1,54 @@
+import pytest
+import pickle
+import zlib
+
+from cloudmappings.cloudstoragemapping import CloudMapping
+from cloudmappings.errors import KeySyncError
+from cloudmappings.storageproviders.storageprovider import StorageProvider
+
+
+class CloudMappingUtilityTests:
+    def test_with_buffers_fails_with_uneven_buffers(self, storage_provider: StorageProvider, test_id: str):
+        with pytest.raises(ValueError, match="equal number of input buffers as output buffers"):
+            CloudMapping.with_buffers(
+                [lambda i: i, lambda i: i],
+                [lambda i: i],
+                storageprovider=storage_provider,
+                sync_initially=False,
+            )
+
+    def test_with_pickle(self, storage_provider: StorageProvider, test_id: str):
+        cm = CloudMapping.with_pickle(storageprovider=storage_provider, sync_initially=False)
+
+        key = test_id + "with-pickle"
+        data = {"picklable": True, "number": 10.01}
+
+        cm[key] = data
+        assert cm[key] == data
+        # Manual download and deserialisation:
+        assert pickle.loads(storage_provider.download_data(key, cm.etags[key])) == data
+
+    def test_with_json(self, storage_provider: StorageProvider, test_id: str):
+        cm = CloudMapping.with_json(storageprovider=storage_provider, sync_initially=False)
+
+        key = test_id + "with-json"
+        data = [10, "json-encodable"]
+        json = b'[10, "json-encodable"]'
+
+        cm[key] = data
+        assert cm[key] == data
+        # Manual download:
+        assert storage_provider.download_data(key, cm.etags[key]) == json
+
+    def test_with_compressed_json(self, storage_provider: StorageProvider, test_id: str):
+        cm = CloudMapping.with_json_zlib(storageprovider=storage_provider, sync_initially=False)
+
+        key = test_id + "with-compressed-json"
+        data = {"a": True, "b": {}, "c": 3}
+        json = b'{"a": true, "b": {}, "c": 3}'
+
+        cm[key] = data
+        assert cm[key] == data
+        # Manual download:
+        raw_bytes = storage_provider.download_data(key, cm.etags[key])
+        assert zlib.decompress(raw_bytes) == json