Remove table backup/restore functionality. (#858)

Author: jpinner-lyft

README.rst

@@ -213,25 +213,13 @@ the type of data you'd like to stream.
         name = UnicodeAttribute(range_key=True)
         id = UnicodeAttribute(hash_key=True)
 
-Want to backup and restore a table? No problem.
-
-.. code-block:: python
-
-    # Backup the table
-    UserModel.dump("usermodel_backup.json")
-
-    # Restore the table
-    UserModel.load("usermodel_backup.json")
-
-
 Features
 ========
 
 * Python >= 3.6 support
 * An ORM-like interface with query and scan filters
 * Compatible with DynamoDB Local
 * Supports the entire DynamoDB API
-* Full table backup/restore
 * Support for Unicode, Binary, JSON, Number, Set, and UTC Datetime attributes
 * Support for Global and Local Secondary Indexes
 * Provides iterators for working with queries, scans, that are automatically paginated

docs/backup_restore.rst

@@ -37,7 +37,6 @@ Topics
    optimistic_locking
    rate_limited_operations
    local
-   backup_restore
    signals
    examples
    settings

docs/index.rst

@@ -19,9 +19,14 @@ The UTCDateTimeAttribute now strictly requires the date string format '%Y-%m-%dT
 PynamoDB has always written values with this format but previously would accept reading other formats.
 Items written using other formats must be rewritten before upgrading.
 
+** Model Serialization **
+
+The ``Model._serialize`` method has changed and now only returns a dictionary of the DynamoDB attribute values.
+
 Other changes in this release:
 
 * Python 2 is no longer supported. Python 3.6 or greater is now required.
+* Table backup functionality (``Model.dump[s]`` and ``Model.load[s]``) has been removed.
 * ``Model.query`` no longer demotes invalid range key conditions to be filter conditions to avoid surprising behaviors:
   where what's intended to be a cheap and fast condition ends up being expensive and slow. Since filter conditions
   cannot contain range keys, this had limited utility to begin with, and would sometimes cause confusing

docs/release_notes.rst

@@ -206,21 +206,8 @@ class Meta:
 # Print the size of the table
 print("Table size: {}".format(Thread.describe_table().get('ItemCount')))
 
-# Dump the entire table to a file
-Thread.dump('thread.json')
-
 # Optionally Delete all table items
 # Commented out for safety
 # for item in Thread.scan():
 #     item.delete()
 print("Table size: {}".format(Thread.describe_table().get('ItemCount')))
-
-# Restore table from a file
-Thread.load('thread.json')
-print("Table size: {}".format(Thread.describe_table().get('ItemCount')))
-
-# Dump the entire table to a string
-serialized = Thread.dumps()
-
-# Load the entire table from a string
-Thread.loads(serialized)

examples/model.py

@@ -1,7 +1,6 @@
 """
 DynamoDB Models for PynamoDB
 """
-import json
 import random
 import time
 import logging
@@ -13,7 +12,7 @@
 from pynamodb.expressions.update import Action
 from pynamodb.exceptions import DoesNotExist, TableDoesNotExist, TableError, InvalidStateError, PutError
 from pynamodb.attributes import (
-    Attribute, AttributeContainer, AttributeContainerMeta, MapAttribute, TTLAttribute, VersionAttribute
+    Attribute, AttributeContainer, AttributeContainerMeta, TTLAttribute, VersionAttribute
 )
 from pynamodb.connection.table import TableConnection
 from pynamodb.expressions.condition import Condition
@@ -119,10 +118,9 @@ def commit(self) -> None:
         log.debug("%s committing batch operation", self.model)
         put_items = []
         delete_items = []
-        attrs_name = snake_to_camel_case(ATTRIBUTES)
         for item in self.pending_operations:
             if item['action'] == PUT:
-                put_items.append(item['item']._serialize(attr_map=True)[attrs_name])
+                put_items.append(item['item']._serialize())
             elif item['action'] == DELETE:
                 delete_items.append(item['item']._get_keys())
         self.pending_operations = []
@@ -380,12 +378,11 @@ def batch_write(cls: Type[_T], auto_commit: bool = True) -> BatchWrite[_T]:
         return BatchWrite(cls, auto_commit=auto_commit)
 
     def __repr__(self) -> str:
-        table_name = self.Meta.table_name if self.Meta.table_name else 'unknown'
-        serialized = self._serialize(null_check=False)
+        hash_key, range_key = self._get_serialized_keys()
         if self._range_keyname:
-            msg = "{}<{}, {}>".format(self.Meta.table_name, serialized.get(HASH), serialized.get(RANGE))
+            msg = "{}<{}, {}>".format(self.Meta.table_name, hash_key, range_key)
         else:
-            msg = "{}<{}>".format(self.Meta.table_name, serialized.get(HASH))
+            msg = "{}<{}>".format(self.Meta.table_name, hash_key)
         return msg
 
     def delete(self, condition: Optional[Condition] = None) -> Any:
@@ -824,59 +821,7 @@ def update_ttl(cls, ignore_update_ttl_errors: bool) -> None:
                 else:
                     raise
 
-    @classmethod
-    def dumps(cls) -> Any:
-        """
-        Returns a JSON representation of this model's table
-        """
-        return json.dumps([item._get_json() for item in cls.scan()])
-
-    @classmethod
-    def dump(cls, filename: str) -> None:
-        """
-        Writes the contents of this model's table as JSON to the given filename
-        """
-        with open(filename, 'w') as out:
-            out.write(cls.dumps())
-
-    @classmethod
-    def loads(cls, data: str) -> None:
-        content = json.loads(data)
-        with cls.batch_write() as batch:
-            for item_data in content:
-                item = cls._from_data(item_data)
-                batch.save(item)
-
-    @classmethod
-    def load(cls, filename: str) -> None:
-        with open(filename, 'r') as inf:
-            cls.loads(inf.read())
-
     # Private API below
-    @classmethod
-    def _from_data(cls, data):
-        """
-        Reconstructs a model object from JSON.
-        """
-        hash_key, attrs = data
-        range_key = attrs.pop('range_key', None)
-        attributes = attrs.pop(snake_to_camel_case(ATTRIBUTES))
-        hash_key_attribute = cls._hash_key_attribute()
-        hash_keyname = hash_key_attribute.attr_name
-        hash_keytype = hash_key_attribute.attr_type
-        attributes[hash_keyname] = {
-            hash_keytype: hash_key
-        }
-        if range_key is not None:
-            range_key_attribute = cls._range_key_attribute()
-            range_keyname = range_key_attribute.attr_name
-            range_keytype = range_key_attribute.attr_type
-            attributes[range_keyname] = {
-                range_keytype: range_key
-            }
-        item = cls(_user_instantiated=False)
-        item._deserialize(attributes)
-        return item
 
     @classmethod
     def _get_schema(cls):
@@ -943,19 +888,6 @@ def _get_indexes(cls):
                     cls._indexes[snake_to_camel_case(LOCAL_SECONDARY_INDEXES)].append(idx)
         return cls._indexes
 
-    def _get_json(self):
-        """
-        Returns a Python object suitable for serialization
-        """
-        kwargs = {}
-        serialized = self._serialize(null_check=False)
-        hash_key = serialized.get(HASH)
-        range_key = serialized.get(RANGE, None)
-        if range_key is not None:
-            kwargs[snake_to_camel_case(RANGE_KEY)] = range_key
-        kwargs[snake_to_camel_case(ATTRIBUTES)] = serialized[snake_to_camel_case(ATTRIBUTES)]
-        return hash_key, kwargs
-
     def _get_save_args(self, attributes=True, null_check=True):
         """
         Gets the proper *args, **kwargs for saving and retrieving this object
@@ -966,14 +898,18 @@ def _get_save_args(self, attributes=True, null_check=True):
         :param null_check: If True, then attributes are checked for null.
         """
         kwargs = {}
-        serialized = self._serialize(null_check=null_check)
-        hash_key = serialized.get(HASH)
-        range_key = serialized.get(RANGE, None)
+        attribute_values = self._serialize(null_check)
+        hash_key_attribute = self._hash_key_attribute()
+        hash_key = attribute_values.pop(hash_key_attribute.attr_name, {}).get(hash_key_attribute.attr_type)
+        range_key = None
+        range_key_attribute = self._range_key_attribute()
+        if range_key_attribute:
+            range_key = attribute_values.pop(range_key_attribute.attr_name, {}).get(range_key_attribute.attr_type)
         args = (hash_key, )
         if range_key is not None:
             kwargs[snake_to_camel_case(RANGE_KEY)] = range_key
         if attributes:
-            kwargs[snake_to_camel_case(ATTRIBUTES)] = serialized[snake_to_camel_case(ATTRIBUTES)]
+            kwargs[snake_to_camel_case(ATTRIBUTES)] = attribute_values
         return args, kwargs
 
     def _handle_version_attribute(self, serialized_attributes, actions=None):
@@ -1042,17 +978,21 @@ def _get_keys(self):
         """
         Returns the proper arguments for deleting
         """
-        serialized = self._serialize(null_check=False)
-        hash_key = serialized.get(HASH)
-        range_key = serialized.get(RANGE, None)
-        attrs = {
-            self._hash_key_attribute().attr_name: hash_key,
-        }
-        if self._range_keyname is not None:
-            range_keyname = self._range_key_attribute().attr_name
-            attrs[range_keyname] = range_key
+        hash_key, range_key = self._get_serialized_keys()
+        hash_key_attribute = self._hash_key_attribute()
+        range_key_attribute = self._range_key_attribute()
+        attrs = {}
+        if hash_key_attribute:
+            attrs[hash_key_attribute.attr_name] = hash_key
+        if range_key_attribute:
+            attrs[range_key_attribute.attr_name] = range_key
         return attrs
 
+    def _get_serialized_keys(self) -> Tuple[_KeyType, _KeyType]:
+        hash_key = getattr(self, self._hash_keyname) if self._hash_keyname else None
+        range_key = getattr(self, self._range_keyname) if self._range_keyname else None
+        return self._serialize_keys(hash_key, range_key)
+
     @classmethod
     def _batch_get_page(cls, keys_to_get, consistent_read, attributes_to_get):
         """
@@ -1107,27 +1047,6 @@ def _get_connection(cls) -> TableConnection:
                                               aws_session_token=cls.Meta.aws_session_token)
         return cls._connection
 
-    def _serialize(self, null_check=True, attr_map=False) -> Dict[str, Dict[str, Any]]:
-        """
-        Serializes all model attributes for use with DynamoDB
-
-        :param null_check: If True, then attributes are checked for null
-        :param attr_map: If True, then attributes are returned
-        """
-        attributes = snake_to_camel_case(ATTRIBUTES)
-        attrs: Dict[str, Dict] = {attributes: super()._serialize(null_check)}
-        if not attr_map:
-            hash_key_attribute = self._hash_key_attribute()
-            hash_key_attribute_value = attrs[attributes].pop(hash_key_attribute.attr_name, None)
-            if hash_key_attribute_value is not None:
-                attrs[HASH] = hash_key_attribute_value[hash_key_attribute.attr_type]
-            range_key_attribute = self._range_key_attribute()
-            if range_key_attribute:
-                range_key_attribute_value = attrs[attributes].pop(range_key_attribute.attr_name, None)
-                if range_key_attribute_value is not None:
-                    attrs[RANGE] = range_key_attribute_value[range_key_attribute.attr_type]
-        return attrs
-
     @classmethod
     def _serialize_value(cls, attr, value):
         """
@@ -1153,7 +1072,8 @@ def _serialize_keys(cls, hash_key, range_key=None) -> Tuple[_KeyType, _KeyType]:
         :param hash_key: The hash key value
         :param range_key: The range key value
         """
-        hash_key = cls._hash_key_attribute().serialize(hash_key)
+        if hash_key is not None:
+            hash_key = cls._hash_key_attribute().serialize(hash_key)
         if range_key is not None:
             range_key = cls._range_key_attribute().serialize(range_key)
         return hash_key, range_key