From d75f6a82da1542b4713c3b07277f6d3828e407ed Mon Sep 17 00:00:00 2001 From: schetvertkov Date: Mon, 17 Jun 2024 18:34:27 +0300 Subject: [PATCH 1/5] Fixes #691 - when decoding json payload using decoder derived from a schema, the missing fields in are populated using their default values (for case classes with more than 22 fields) --- .../zio/schema/codec/JsonCodecJVMSpec.scala | 75 +++++++++++++++++++ .../scala/zio/schema/codec/JsonCodec.scala | 13 +++- .../zio/schema/codec/JsonCodecSpec.scala | 55 ++++++++++++-- 3 files changed, 136 insertions(+), 7 deletions(-) diff --git a/zio-schema-json/jvm/src/test/scala-2/zio/schema/codec/JsonCodecJVMSpec.scala b/zio-schema-json/jvm/src/test/scala-2/zio/schema/codec/JsonCodecJVMSpec.scala index 01f1da1c2..a1d3ac230 100644 --- a/zio-schema-json/jvm/src/test/scala-2/zio/schema/codec/JsonCodecJVMSpec.scala +++ b/zio-schema-json/jvm/src/test/scala-2/zio/schema/codec/JsonCodecJVMSpec.scala @@ -20,6 +20,18 @@ object JsonCodecJVMSpec extends ZIOSpecDefault { ) @@ TestAspect.jvmOnly @@ timeout(180.seconds) private val decoderSuite = suite("decoding")( + suite("decode record with more than 22 fields")( + test("missing fields in the json payload are populated with their default values") { + val exampleSchema = zio.schema.DeriveSchema.gen[RecordExample] + val string = """{"f1": "test"}""" + assertDecodesJson(exampleSchema, RecordExample(Some("test")), string) + }, + test("fail if a field with no default value is missing in the json payload") { + val exampleSchema = zio.schema.DeriveSchema.gen[RecordExample2] + val string = """{"f1": "test"}""" + assertDecodesJsonFailure(exampleSchema, string) + } + ), suite("case class")( test("case class with empty option field is decoded by stream") { val names = Gen.option(Gen.elements("John", "Jane", "Jermaine", "Jasmine")) @@ -50,6 +62,16 @@ object JsonCodecJVMSpec extends ZIOSpecDefault { ) ) + private def assertDecodesJson[A](schema: Schema[A], value: A, jsonString: String) = { + val either = JsonCodec.jsonDecoder(schema).decodeJson(jsonString) + zio.test.assert(either)(isRight(equalTo(value))) + } + + private def assertDecodesJsonFailure[A](schema: Schema[A], jsonString: String) = { + val either = JsonCodec.jsonDecoder(schema).decodeJson(jsonString) + zio.test.assertTrue(either.isLeft) + } + private def assertDecodesJsonStream[A]( schema: Schema[A], value: Chunk[A], @@ -64,4 +86,57 @@ object JsonCodecJVMSpec extends ZIOSpecDefault { .either assertZIO(result)(isRight(equalTo(value))) } + + case class RecordExample( + f1: Option[String], + f2: Option[String] = None, + f3: Option[String] = None, + f4: Option[String] = None, + f5: Option[String] = None, + f6: Option[String] = None, + f7: Option[String] = None, + f8: Option[String] = None, + f9: Option[String] = None, + f10: Option[String] = None, + f11: Option[String] = None, + f12: Option[String] = None, + f13: Option[String] = None, + f14: Option[String] = None, + f15: Option[String] = None, + f16: Option[String] = None, + f17: Option[String] = None, + f18: Option[String] = None, + f19: Option[String] = None, + f20: Option[String] = None, + f21: Option[String] = None, + f22: Option[String] = None, + f23: Option[String] = None + ) + + case class RecordExample2( + f1: Option[String], + f2: Option[String], + f3: Option[String] = None, + f4: Option[String] = None, + f5: Option[String] = None, + f6: Option[String] = None, + f7: Option[String] = None, + f8: Option[String] = None, + f9: Option[String] = None, + f10: Option[String] = None, + f11: Option[String] = None, + f12: Option[String] = None, + f13: Option[String] = None, + f14: Option[String] = None, + f15: Option[String] = None, + f16: Option[String] = None, + f17: Option[String] = None, + f18: Option[String] = None, + f19: Option[String] = None, + f20: Option[String] = None, + f21: Option[String] = None, + f22: Option[String] = None, + f23: Option[String] = None + ) + } diff --git a/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala b/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala index 8aacace25..c11bea9f6 100644 --- a/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala +++ b/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala @@ -830,7 +830,18 @@ object JsonCodec { () } } - (ListMap.newBuilder[String, Any] ++= builder.result()).result() + val tuples = builder.result() + val collectedFields: Set[String] = tuples.map { case (fieldName, _) => fieldName }.toSet + val resultBuilder = ListMap.newBuilder[String, Any] + + // add fields with default values if they are not present in the JSON + structure.foreach { field => + if (!collectedFields.contains(field.name) && field.optional && field.defaultValue.isDefined) { + val value = field.name -> field.defaultValue.get + resultBuilder += value + } + } + (resultBuilder ++= tuples).result() } } diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index 0393c8a7a..ff1611bf6 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -1,21 +1,19 @@ package zio.schema.codec -import java.time.{ ZoneId, ZoneOffset } - +import java.time.{ZoneId, ZoneOffset} import scala.collection.immutable.ListMap - import zio.Console._ import zio._ import zio.json.JsonDecoder.JsonError import zio.json.ast.Json -import zio.json.{ DeriveJsonEncoder, JsonEncoder } +import zio.json.{DeriveJsonEncoder, JsonEncoder} import zio.schema.CaseSet._ import zio.schema._ import zio.schema.annotation._ import zio.schema.codec.DecodeError.ReadError import zio.schema.codec.JsonCodec.JsonEncoder.charSequenceToByteChunk -import zio.schema.codec.JsonCodecSpec.PaymentMethod.{ CreditCard, PayPal, WireTransfer } -import zio.schema.codec.JsonCodecSpec.Subscription.{ OneTime, Recurring } +import zio.schema.codec.JsonCodecSpec.PaymentMethod.{CreditCard, PayPal, WireTransfer} +import zio.schema.codec.JsonCodecSpec.Subscription.{OneTime, Recurring} import zio.schema.meta.MetaSchema import zio.stream.ZStream import zio.test.Assertion._ @@ -219,6 +217,13 @@ object JsonCodecSpec extends ZIOSpecDefault { } ), suite("record")( + test("missing fields should be replaced by default values") { + assertDecodes( + recordSchema, + ListMap[String, Any]("foo" -> "s", "bar" -> 0), + charSequenceToByteChunk("""{"foo":"s"}""") + ) + }, test("of primitives") { assertEncodes( recordSchema, @@ -435,6 +440,13 @@ object JsonCodecSpec extends ZIOSpecDefault { ListMap[String, Any]("foo" -> "s", "bar" -> 1), charSequenceToByteChunk("""{"foo":"s","bar":1,"baz":2}""") ) + }, + test("with missing fields") { + assertDecodes( + RecordExample.schema, + RecordExample(f1 = Some("test"), f2 = None), + charSequenceToByteChunk("""{"f1":"test"}""") + ) } ), suite("transform")( @@ -1867,4 +1879,35 @@ object JsonCodecSpec extends ZIOSpecDefault { object AllOptionalFields { implicit lazy val schema: Schema[AllOptionalFields] = DeriveSchema.gen[AllOptionalFields] } + + case class RecordExample( + f1: Option[String], + f2: Option[String], + f3: Option[String] = None, + f4: Option[String] = None, + f5: Option[String] = None, + f6: Option[String] = None, + f7: Option[String] = None, + f8: Option[String] = None, + f9: Option[String] = None, + f10: Option[String] = None, + f11: Option[String] = None, + f12: Option[String] = None, + f13: Option[String] = None, + f14: Option[String] = None, + f15: Option[String] = None, + f16: Option[String] = None, + f17: Option[String] = None, + f18: Option[String] = None, + f19: Option[String] = None, + f20: Option[String] = None, + f21: Option[String] = None, + f22: Option[String] = None, + f23: Option[String] = None + ) + + object RecordExample { + implicit lazy val schema: Schema[RecordExample] = DeriveSchema.gen[RecordExample] + } + } From 5ac988945c04455dbf25c3e9a84fd63601dced0d Mon Sep 17 00:00:00 2001 From: schetvertkov Date: Mon, 17 Jun 2024 18:34:27 +0300 Subject: [PATCH 2/5] Fixes #691 - when decoding json payload using decoder derived from a schema, the missing fields in are populated using their default values (for case classes with more than 22 fields) --- .../zio/schema/codec/JsonCodecSpec.scala | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index ff1611bf6..2a0db54ed 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -1881,30 +1881,30 @@ object JsonCodecSpec extends ZIOSpecDefault { } case class RecordExample( - f1: Option[String], - f2: Option[String], - f3: Option[String] = None, - f4: Option[String] = None, - f5: Option[String] = None, - f6: Option[String] = None, - f7: Option[String] = None, - f8: Option[String] = None, - f9: Option[String] = None, - f10: Option[String] = None, - f11: Option[String] = None, - f12: Option[String] = None, - f13: Option[String] = None, - f14: Option[String] = None, - f15: Option[String] = None, - f16: Option[String] = None, - f17: Option[String] = None, - f18: Option[String] = None, - f19: Option[String] = None, - f20: Option[String] = None, - f21: Option[String] = None, - f22: Option[String] = None, - f23: Option[String] = None - ) + f1: Option[String], + f2: Option[String], + f3: Option[String] = None, + f4: Option[String] = None, + f5: Option[String] = None, + f6: Option[String] = None, + f7: Option[String] = None, + f8: Option[String] = None, + f9: Option[String] = None, + f10: Option[String] = None, + f11: Option[String] = None, + f12: Option[String] = None, + f13: Option[String] = None, + f14: Option[String] = None, + f15: Option[String] = None, + f16: Option[String] = None, + f17: Option[String] = None, + f18: Option[String] = None, + f19: Option[String] = None, + f20: Option[String] = None, + f21: Option[String] = None, + f22: Option[String] = None, + f23: Option[String] = None + ) object RecordExample { implicit lazy val schema: Schema[RecordExample] = DeriveSchema.gen[RecordExample] From f7f7aa4a87b9697147bf0239576c9938fa432f13 Mon Sep 17 00:00:00 2001 From: schetvertkov Date: Wed, 26 Jun 2024 09:46:25 +0300 Subject: [PATCH 3/5] #691 - formatting --- .../test/scala-2/zio/schema/codec/JsonCodecSpec.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index 2a0db54ed..a695a9bd7 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -1,19 +1,21 @@ package zio.schema.codec -import java.time.{ZoneId, ZoneOffset} +import java.time.{ ZoneId, ZoneOffset } + import scala.collection.immutable.ListMap + import zio.Console._ import zio._ import zio.json.JsonDecoder.JsonError import zio.json.ast.Json -import zio.json.{DeriveJsonEncoder, JsonEncoder} +import zio.json.{ DeriveJsonEncoder, JsonEncoder } import zio.schema.CaseSet._ import zio.schema._ import zio.schema.annotation._ import zio.schema.codec.DecodeError.ReadError import zio.schema.codec.JsonCodec.JsonEncoder.charSequenceToByteChunk -import zio.schema.codec.JsonCodecSpec.PaymentMethod.{CreditCard, PayPal, WireTransfer} -import zio.schema.codec.JsonCodecSpec.Subscription.{OneTime, Recurring} +import zio.schema.codec.JsonCodecSpec.PaymentMethod.{ CreditCard, PayPal, WireTransfer } +import zio.schema.codec.JsonCodecSpec.Subscription.{ OneTime, Recurring } import zio.schema.meta.MetaSchema import zio.stream.ZStream import zio.test.Assertion._ From 9509110911481bf82b5b1b65650f385268cd9fbb Mon Sep 17 00:00:00 2001 From: schetvertkov Date: Wed, 26 Jun 2024 10:56:44 +0300 Subject: [PATCH 4/5] #691 - formatting --- .../src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index a695a9bd7..15edcb80e 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -1883,8 +1883,8 @@ object JsonCodecSpec extends ZIOSpecDefault { } case class RecordExample( - f1: Option[String], - f2: Option[String], + f1: Option[String], // the only field that does not have a default value + f2: Option[String] = None, f3: Option[String] = None, f4: Option[String] = None, f5: Option[String] = None, From c1be0367fb007b28aec8aa57a645c944e38a104f Mon Sep 17 00:00:00 2001 From: schetvertkov Date: Wed, 26 Jun 2024 11:24:18 +0300 Subject: [PATCH 5/5] #691 - fix tests in JsonCodecSpec --- .../src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index 15edcb80e..aeb062fb9 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -222,7 +222,7 @@ object JsonCodecSpec extends ZIOSpecDefault { test("missing fields should be replaced by default values") { assertDecodes( recordSchema, - ListMap[String, Any]("foo" -> "s", "bar" -> 0), + ListMap[String, Any]("foo" -> "s", "bar" -> 1), charSequenceToByteChunk("""{"foo":"s"}""") ) }, @@ -1650,6 +1650,7 @@ object JsonCodecSpec extends ZIOSpecDefault { .Field( "bar", Schema.Primitive(StandardType.IntType), + annotations0 = Chunk(fieldDefaultValue(1)), get0 = (p: ListMap[String, _]) => p("bar").asInstanceOf[Int], set0 = (p: ListMap[String, _], v: Int) => p.updated("bar", v) )