From 4929aee27a81b7389fd7a38e3af19fff2d7c4b59 Mon Sep 17 00:00:00 2001 From: jpinner-lyft Date: Wed, 1 Dec 2021 13:04:27 -0800 Subject: [PATCH] Add support for JSON serialization. (#857) --- docs/release_notes.rst | 1 + pynamodb/attributes.py | 50 ++++++++++++++-- pynamodb/models.py | 12 +++- pynamodb/util.py | 50 ++++++++++++++++ tests/test_attributes.py | 122 ++++++++++++++++++++++++++++++++++++++- tests/test_model.py | 34 +++++++++++ 6 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 pynamodb/util.py diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 4f6be0732..2b23f51d1 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -4,6 +4,7 @@ Release Notes Unreleased ---------- * The ``IndexMeta`` class has been removed. Now ``type(Index) == type``. +* JSON serialization support (``Model.to_json`` and ``Model.from_json``) has been added. v5.1.0 diff --git a/pynamodb/attributes.py b/pynamodb/attributes.py index 6c99519bb..39a756123 100644 --- a/pynamodb/attributes.py +++ b/pynamodb/attributes.py @@ -17,11 +17,20 @@ from typing import TYPE_CHECKING from pynamodb._compat import GenericMeta -from pynamodb.constants import ( - BINARY, BINARY_SET, BOOLEAN, DATETIME_FORMAT, DEFAULT_ENCODING, - LIST, MAP, NULL, NUMBER, NUMBER_SET, STRING, STRING_SET -) -from pynamodb.exceptions import AttributeDeserializationError, AttributeNullError +from pynamodb.constants import BINARY +from pynamodb.constants import BINARY_SET +from pynamodb.constants import BOOLEAN +from pynamodb.constants import DATETIME_FORMAT +from pynamodb.constants import DEFAULT_ENCODING +from pynamodb.constants import LIST +from pynamodb.constants import MAP +from pynamodb.constants import NULL +from pynamodb.constants import NUMBER +from pynamodb.constants import NUMBER_SET +from pynamodb.constants import STRING +from pynamodb.constants import STRING_SET +from pynamodb.exceptions import AttributeDeserializationError +from pynamodb.exceptions import AttributeNullError from pynamodb.expressions.operand import Path @@ -369,6 +378,36 @@ def _container_deserialize(self, attribute_values: Dict[str, Dict[str, Any]]) -> value = attr.deserialize(attr.get_value(attribute_value)) setattr(self, name, value) + @classmethod + def _update_attribute_types(cls, attribute_values: Dict[str, Dict[str, Any]]): + """ + Update the attribute types in the attribute values dictionary to disambiguate json string and array types + """ + for attr in cls.get_attributes().values(): + attribute_value = attribute_values.get(attr.attr_name) + if attribute_value: + AttributeContainer._coerce_attribute_type(attr.attr_type, attribute_value) + if isinstance(attr, ListAttribute) and attr.element_type and LIST in attribute_value: + if issubclass(attr.element_type, AttributeContainer): + for element in attribute_value[LIST]: + if MAP in element: + attr.element_type._update_attribute_types(element[MAP]) + else: + for element in attribute_value[LIST]: + AttributeContainer._coerce_attribute_type(attr.element_type.attr_type, element) + if isinstance(attr, AttributeContainer) and MAP in attribute_value: + attr._update_attribute_types(attribute_value[MAP]) + + @staticmethod + def _coerce_attribute_type(attr_type: str, attribute_value: Dict[str, Any]): + # coerce attribute types to disambiguate json string and array types + if attr_type == BINARY and STRING in attribute_value: + attribute_value[BINARY] = attribute_value.pop(STRING) + if attr_type in {BINARY_SET, NUMBER_SET, STRING_SET} and LIST in attribute_value: + json_type = NUMBER if attr_type == NUMBER_SET else STRING + if all(next(iter(v)) == json_type for v in attribute_value[LIST]): + attribute_value[attr_type] = [value[json_type] for value in attribute_value.pop(LIST)] + @classmethod def _get_discriminator_class(cls, attribute_values: Dict[str, Dict[str, Any]]) -> Optional[Type]: discriminator_attr = cls._get_discriminator_attribute() @@ -1174,4 +1213,5 @@ def _get_serialize_class(self, value): float: NumberAttribute(), int: NumberAttribute(), str: UnicodeAttribute(), + bytes: BinaryAttribute(), } diff --git a/pynamodb/models.py b/pynamodb/models.py index 809be7b39..c8c78724b 100644 --- a/pynamodb/models.py +++ b/pynamodb/models.py @@ -1,6 +1,7 @@ """ DynamoDB Models for PynamoDB """ +import json import random import time import logging @@ -54,6 +55,8 @@ COUNT, ITEM_COUNT, KEY, UNPROCESSED_ITEMS, STREAM_VIEW_TYPE, STREAM_SPECIFICATION, STREAM_ENABLED, BILLING_MODE, PAY_PER_REQUEST_BILLING_MODE, TAGS ) +from pynamodb.util import attribute_value_to_json +from pynamodb.util import json_to_attribute_value _T = TypeVar('_T', bound='Model') _KeyType = Any @@ -846,7 +849,6 @@ def update_ttl(cls, ignore_update_ttl_errors: bool) -> None: raise # Private API below - @classmethod def _get_schema(cls) -> Dict[str, Any]: """ @@ -1110,6 +1112,14 @@ def deserialize(self, attribute_values: Dict[str, Dict[str, Any]]) -> None: """ return self._container_deserialize(attribute_values=attribute_values) + def to_json(self) -> str: + return json.dumps({k: attribute_value_to_json(v) for k, v in self.serialize().items()}) + + def from_json(self, s: str) -> None: + attribute_values = {k: json_to_attribute_value(v) for k, v in json.loads(s).items()} + self._update_attribute_types(attribute_values) + self.deserialize(attribute_values) + class _ModelFuture(Generic[_T]): """ diff --git a/pynamodb/util.py b/pynamodb/util.py new file mode 100644 index 000000000..cc96e209c --- /dev/null +++ b/pynamodb/util.py @@ -0,0 +1,50 @@ +""" +Utils +""" +import json +from typing import Any +from typing import Dict + +from pynamodb.constants import BINARY +from pynamodb.constants import BINARY_SET +from pynamodb.constants import BOOLEAN +from pynamodb.constants import LIST +from pynamodb.constants import MAP +from pynamodb.constants import NULL +from pynamodb.constants import NUMBER +from pynamodb.constants import NUMBER_SET +from pynamodb.constants import STRING +from pynamodb.constants import STRING_SET + + +def attribute_value_to_json(attribute_value: Dict[str, Any]) -> Any: + attr_type, attr_value = next(iter(attribute_value.items())) + if attr_type == LIST: + return [attribute_value_to_json(v) for v in attr_value] + if attr_type == MAP: + return {k: attribute_value_to_json(v) for k, v in attr_value.items()} + if attr_type == NULL: + return None + if attr_type in {BINARY, BINARY_SET, BOOLEAN, STRING, STRING_SET}: + return attr_value + if attr_type == NUMBER: + return json.loads(attr_value) + if attr_type == NUMBER_SET: + return [json.loads(v) for v in attr_value] + raise ValueError("Unknown attribute type: {}".format(attr_type)) + + +def json_to_attribute_value(value: Any) -> Dict[str, Any]: + if value is None: + return {NULL: True} + if value is True or value is False: + return {BOOLEAN: value} + if isinstance(value, (int, float)): + return {NUMBER: json.dumps(value)} + if isinstance(value, str): + return {STRING: value} + if isinstance(value, list): + return {LIST: [json_to_attribute_value(v) for v in value]} + if isinstance(value, dict): + return {MAP: {k: json_to_attribute_value(v) for k, v in value.items()}} + raise ValueError("Unknown value type: {}".format(type(value).__name__)) diff --git a/tests/test_attributes.py b/tests/test_attributes.py index a6db1e01f..366536185 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -1,6 +1,7 @@ """ pynamodb attributes tests """ +import calendar import json from base64 import b64encode @@ -13,10 +14,10 @@ from pynamodb.attributes import ( BinarySetAttribute, BinaryAttribute, DynamicMapAttribute, NumberSetAttribute, NumberAttribute, - UnicodeAttribute, UnicodeSetAttribute, UTCDateTimeAttribute, BooleanAttribute, MapAttribute, + UnicodeAttribute, UnicodeSetAttribute, UTCDateTimeAttribute, BooleanAttribute, MapAttribute, NullAttribute, ListAttribute, JSONAttribute, TTLAttribute, VersionAttribute) from pynamodb.constants import ( - DEFAULT_ENCODING, NUMBER, STRING, STRING_SET, NUMBER_SET, BINARY_SET, + DATETIME_FORMAT, DEFAULT_ENCODING, NUMBER, STRING, STRING_SET, NUMBER_SET, BINARY_SET, BINARY, BOOLEAN, ) from pynamodb.models import Model @@ -39,6 +40,7 @@ class Meta: json_attr = JSONAttribute() map_attr = MapAttribute() ttl_attr = TTLAttribute() + null_attr = NullAttribute(null=True) class CustomAttrMap(MapAttribute): @@ -1062,3 +1064,119 @@ def test_deserialize(self): assert attr.deserialize('1') == 1 assert attr.deserialize('3.141') == 3 assert attr.deserialize('12345678909876543211234234324234') == 12345678909876543211234234324234 + + +class TestAttributeContainer: + def test_to_json(self): + now = datetime.now(tz=timezone.utc) + now_formatted = now.strftime(DATETIME_FORMAT) + now_unix_ts = calendar.timegm(now.utctimetuple()) + test_model = AttributeTestModel() + test_model.binary_attr = b'foo' + test_model.binary_set_attr = {b'bar'} + test_model.number_attr = 1 + test_model.number_set_attr = {0, 0.5, 1} + test_model.unicode_attr = 'foo' + test_model.unicode_set_attr = {'baz'} + test_model.datetime_attr = now + test_model.bool_attr = True + test_model.json_attr = {'foo': 'bar'} + test_model.map_attr = {'foo': 'bar'} + test_model.ttl_attr = now + test_model.null_attr = True + assert test_model.to_json() == ( + '{' + '"binary_attr": "Zm9v", ' + '"binary_set_attr": ["YmFy"], ' + '"bool_attr": true, ' + '"datetime_attr": "' + now_formatted + '", ' + '"json_attr": "{\\"foo\\": \\"bar\\"}", ' + '"map_attr": {"foo": "bar"}, ' + '"null_attr": null, ' + '"number_attr": 1, ' + '"number_set_attr": [0, 0.5, 1], ' + '"ttl_attr": ' + str(now_unix_ts) + ', ' + '"unicode_attr": "foo", ' + '"unicode_set_attr": ["baz"]' + '}') + + def test_from_json(self): + now = datetime.now(tz=timezone.utc) + now_formatted = now.strftime(DATETIME_FORMAT) + now_unix_ts = calendar.timegm(now.utctimetuple()) + json_string = ( + '{' + '"binary_attr": "Zm9v", ' + '"binary_set_attr": ["YmFy"], ' + '"bool_attr": true, ' + '"datetime_attr": "' + now_formatted + '", ' + '"json_attr": "{\\"foo\\": \\"bar\\"}", ' + '"map_attr": {"foo": "bar"}, ' + '"null_attr": null, ' + '"number_attr": 1, ' + '"number_set_attr": [0, 0.5, 1], ' + '"ttl_attr": ' + str(now_unix_ts) + ', ' + '"unicode_attr": "foo", ' + '"unicode_set_attr": ["baz"]' + '}') + test_model = AttributeTestModel() + test_model.from_json(json_string) + assert test_model.binary_attr == b'foo' + assert test_model.binary_set_attr == {b'bar'} + assert test_model.number_attr == 1 + assert test_model.number_set_attr == {0, 0.5, 1} + assert test_model.unicode_attr == 'foo' + assert test_model.unicode_set_attr == {'baz'} + assert test_model.datetime_attr == now + assert test_model.bool_attr is True + assert test_model.json_attr == {'foo': 'bar'} + assert test_model.map_attr.foo == 'bar' + assert test_model.ttl_attr == now.replace(microsecond=0) + assert test_model.null_attr is None + + def test_to_json_complex(self): + class MyMap(MapAttribute): + foo = UnicodeSetAttribute(attr_name='bar') + + class ListTestModel(Model): + class Meta: + host = 'http://localhost:8000' + table_name = 'test' + unicode_attr = UnicodeAttribute(hash_key=True) + list_attr = ListAttribute(of=NumberSetAttribute) + list_map_attr = ListAttribute(of=MyMap) + + list_test_model = ListTestModel() + list_test_model.unicode_attr = 'foo' + list_test_model.list_attr = [{0, 1, 2}] + list_test_model.list_map_attr = [MyMap(foo={'baz'})] + assert list_test_model.to_json() == ( + '{' + '"list_attr": [[0, 1, 2]], ' + '"list_map_attr": [{"bar": ["baz"]}], ' + '"unicode_attr": "foo"' + '}') + + def test_from_json_complex(self): + class MyMap(MapAttribute): + foo = UnicodeSetAttribute(attr_name='bar') + + class ListTestModel(Model): + class Meta: + host = 'http://localhost:8000' + table_name = 'test' + unicode_attr = UnicodeAttribute(hash_key=True) + list_attr = ListAttribute(of=NumberSetAttribute) + list_map_attr = ListAttribute(of=MyMap) + + json_string = ( + '{' + '"list_attr": [[0, 1, 2]], ' + '"list_map_attr": [{"bar": ["baz"]}], ' + '"unicode_attr": "foo"' + '}') + list_test_model = ListTestModel() + list_test_model.from_json(json_string) + assert list_test_model.unicode_attr == 'foo' + assert list_test_model.list_attr == [{0, 1, 2}] + assert list_test_model.list_map_attr[0].foo == {'baz'} diff --git a/tests/test_model.py b/tests/test_model.py index 3daf14fb4..357fb157c 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -2528,6 +2528,40 @@ class Meta: with self.assertRaises(AttributeError): MissingTableNameModel.exists() + def test_to_json(self): + """ + Model.to_json + """ + user = UserModel() + user.custom_user_name = 'foo' + user.user_id = 'bar' + user.picture = base64.b64decode(BINARY_ATTR_DATA) + user.zip_code = 88030 + json_user = json.loads(user.to_json()) + self.assertEqual(json_user['user_name'], user.custom_user_name) # uses custom attribute name + self.assertEqual(json_user['user_id'], user.user_id) + self.assertEqual(json_user['picture'], BINARY_ATTR_DATA) + self.assertEqual(json_user['zip_code'], user.zip_code) + self.assertEqual(json_user['email'], 'needs_email') # set to default value + + def test_from_json(self): + """ + Model.from_json + """ + json_user = { + 'user_name': 'foo', + 'user_id': 'bar', + 'picture': BINARY_ATTR_DATA, + 'zip_code': 88030, + } + user = UserModel() + user.from_json(json.dumps(json_user)) + self.assertEqual(user.custom_user_name, json_user['user_name']) # uses custom attribute name + self.assertEqual(user.user_id, json_user['user_id']) + self.assertEqual(user.picture, base64.b64decode(json_user['picture'])) + self.assertEqual(user.zip_code, json_user['zip_code']) + self.assertEqual(user.email, 'needs_email') # set to default value + def _get_office_employee(self): justin = Person( fname='Justin',