From 26bb289a166d23d346abe09939eeb7fa31397ea6 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Fri, 21 Jul 2023 16:42:23 +0100 Subject: [PATCH 1/3] Fix Json codec of maps with lazy key schema (#586) * Fix Json codec of maps with lazy key schema * readme --- README.md | 10 +++++----- .../scala/zio/schema/codec/JsonCodec.scala | 2 ++ .../zio/schema/codec/JsonCodecSpec.scala | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 243bf44ee..243558c74 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,13 @@ _ZIO Schema_ is used by a growing number of ZIO libraries, including _ZIO Flow_, In order to use this library, we need to add the following lines in our `build.sbt` file: ```scala -libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.11" -libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.11" +libraryDependencies += "dev.zio" %% "zio-schema" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-bson" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-json" % "0.4.12" +libraryDependencies += "dev.zio" %% "zio-schema-protobuf" % "0.4.12" // Required for automatic generic derivation of schemas -libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.11", +libraryDependencies += "dev.zio" %% "zio-schema-derivation" % "0.4.12", libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided" ``` diff --git a/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala b/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala index 8c22480fa..fb56ec666 100644 --- a/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala +++ b/zio-schema-json/shared/src/main/scala/zio/schema/codec/JsonCodec.scala @@ -273,6 +273,7 @@ object JsonCodec { case Schema.Primitive(StandardType.StringType, _) => Option(JsonFieldEncoder.string) case Schema.Primitive(StandardType.LongType, _) => Option(JsonFieldEncoder.long) case Schema.Primitive(StandardType.IntType, _) => Option(JsonFieldEncoder.int) + case Schema.Lazy(inner) => jsonFieldEncoder(inner()) case _ => None } @@ -598,6 +599,7 @@ object JsonCodec { case Schema.Primitive(StandardType.StringType, _) => Option(JsonFieldDecoder.string) case Schema.Primitive(StandardType.LongType, _) => Option(JsonFieldDecoder.long) case Schema.Primitive(StandardType.IntType, _) => Option(JsonFieldDecoder.int) + case Schema.Lazy(inner) => jsonFieldDecoder(inner()) case _ => None } diff --git a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala index d35adb72d..933c6ccb3 100644 --- a/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala +++ b/zio-schema-json/shared/src/test/scala-2/zio/schema/codec/JsonCodecSpec.scala @@ -96,6 +96,15 @@ object JsonCodecSpec extends ZIOSpecDefault { """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" ) ) + }, + test("of simple keys and values where the key's schema is lazy") { + assertEncodes( + Schema.map[Int, Value](Schema.defer(Schema[Int]), Schema[Value]), + Map(0 -> Value(0, true), 1 -> Value(1, false)), + charSequenceToByteChunk( + """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" + ) + ) } ), suite("Set")( @@ -469,6 +478,15 @@ object JsonCodecSpec extends ZIOSpecDefault { """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" ) ) + }, + test("of simple keys and values where the key schema is lazy") { + assertDecodes( + Schema.map[Int, Value](Schema.defer(Schema[Int]), Schema[Value]), + Map(0 -> Value(0, true), 1 -> Value(1, false)), + charSequenceToByteChunk( + """{"0":{"first":0,"second":true},"1":{"first":1,"second":false}}""" + ) + ) } ) ) From 8122cbccd3f8688b71c13c6de76e8e6445af0e0e Mon Sep 17 00:00:00 2001 From: Gabriel Ciuloaica Date: Mon, 12 Jun 2023 16:50:12 +0300 Subject: [PATCH 2/3] added API to allow encoding and decoding to/from Generic record, to allow integration with Schema Registry (AWS Glue) --- .../main/scala/zio/schema/codec/AvroCodec.scala | 15 +++++++++++++-- .../scala-2/zio/schema/codec/AvroCodecSpec.scala | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala index 97a499e7d..532ebc734 100644 --- a/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala +++ b/zio-schema-avro/shared/src/main/scala/zio/schema/codec/AvroCodec.scala @@ -25,8 +25,13 @@ import zio.{ Chunk, Unsafe, ZIO } object AvroCodec { - implicit def schemaBasedBinaryCodec[A](implicit schema: Schema[A]): BinaryCodec[A] = - new BinaryCodec[A] { + trait ExtendedBinaryCodec[A] extends BinaryCodec[A] { + def encodeGenericRecord(value: A)(implicit schema: Schema[A]): GenericData.Record + def decodeGenericRecord(value: GenericRecord)(implicit schema: Schema[A]): Either[DecodeError, A] + } + + implicit def schemaBasedBinaryCodec[A](implicit schema: Schema[A]): ExtendedBinaryCodec[A] = + new ExtendedBinaryCodec[A] { val avroSchema: SchemaAvro = AvroSchemaCodec.encodeToApacheAvro(schema).getOrElse(throw new Exception("Avro schema could not be generated.")) @@ -59,6 +64,12 @@ object AvroCodec { decode(chunk).map(Chunk(_)) ) } + + override def encodeGenericRecord(value: A)(implicit schema: Schema[A]): GenericData.Record = + encodeValue(value, schema).asInstanceOf[GenericData.Record] + + override def decodeGenericRecord(value: GenericRecord)(implicit schema: Schema[A]): Either[DecodeError, A] = + decodeValue(value, schema) } private def decodeValue[A](raw: Any, schema: Schema[A]): Either[DecodeError, A] = schema match { diff --git a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala index 5790916c8..033aa7369 100644 --- a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala +++ b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala @@ -1,5 +1,7 @@ package zio.schema.codec +import org.apache.avro.generic.GenericData + import java.math.BigInteger import java.time.{ DayOfWeek, @@ -19,7 +21,6 @@ import java.time.{ ZonedDateTime } import java.util.UUID - import zio._ import zio.schema.{ DeriveSchema, Schema } import zio.stream.ZStream @@ -138,7 +139,8 @@ object AvroCodecSpec extends ZIOSpecDefault { sequenceDecoderSpec, genericRecordDecoderSpec, enumDecoderSpec, - streamEncodingDecodingSpec + streamEncodingDecodingSpec, + genericRecordEncodeDecodeSpec ) private val primitiveEncoderSpec = suite("Avro Codec - Encoder primitive spec")( @@ -687,4 +689,13 @@ object AvroCodecSpec extends ZIOSpecDefault { }) + private val genericRecordEncodeDecodeSpec = suite("AvroCodec - encode/decode Generic Record")( + test("Encode/Decode") { + val codec = AvroCodec.schemaBasedBinaryCodec[Record] + val generic: GenericData.Record = codec.encodeGenericRecord(Record("John", 42)) + val result = codec.decodeGenericRecord(generic) + assertTrue(result == Right(Record("John", 42))) + } + ) + } From 453b1ab4a562fe4d8f8ea7b42ed07b13bd8fe7fa Mon Sep 17 00:00:00 2001 From: Gabriel Ciuloaica Date: Mon, 12 Jun 2023 17:02:18 +0300 Subject: [PATCH 3/3] fixed imports --- .../src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala index 033aa7369..db4f48129 100644 --- a/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala +++ b/zio-schema-avro/shared/src/test/scala-2/zio/schema/codec/AvroCodecSpec.scala @@ -1,7 +1,5 @@ package zio.schema.codec -import org.apache.avro.generic.GenericData - import java.math.BigInteger import java.time.{ DayOfWeek, @@ -21,6 +19,9 @@ import java.time.{ ZonedDateTime } import java.util.UUID + +import org.apache.avro.generic.GenericData + import zio._ import zio.schema.{ DeriveSchema, Schema } import zio.stream.ZStream