Skip to content

Commit

Permalink
Make schema val @transient in ParquetType (#619)
Browse files Browse the repository at this point in the history
* Make schema val @transient in ParquetType

* fix test

* Fixing the Avro compat error in serialization test (#620)

Co-authored-by: Claire McGinty <clairem@spotify.com>

Co-authored-by: Andrew Kabas <kabas.andrew@gmail.com>
  • Loading branch information
clairemcginty and shnapz authored Oct 18, 2022
1 parent ad42b3f commit a8c5b67
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 3 deletions.
4 changes: 2 additions & 2 deletions parquet/src/main/scala/magnolify/parquet/ParquetType.scala
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ object ParquetType {
)(implicit f: ParquetField[T], pa: ParquetArray): ParquetType[T] = f match {
case r: ParquetField.Record[_] =>
new ParquetType[T] {
override lazy val schema: MessageType = Schema.message(r.schema(cm))
override lazy val avroSchema: AvroSchema = {
@transient override lazy val schema: MessageType = Schema.message(r.schema(cm))
@transient override lazy val avroSchema: AvroSchema = {
val s = new AvroSchemaConverter().convert(schema)
// add doc to avro schema
SchemaUtil.deepCopy(s, f.typeDoc, f.fieldDocs.get)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,14 @@ class AvroParquetSuite extends MagnolifySuite {
schemaErrors: List[String] = List.empty
)(implicit
at: AvroType[T],
pt: ParquetType[T],
tpe: ParquetType[T],
eq: Eq[T]
): Unit = {
// Ensure serializable even after evaluation of `schema` and `avroSchema`
val parquetSchema = tpe.schema
val avroSchema = tpe.avroSchema
val pt = ensureSerializable(tpe)

val name = className[T]

property(s"$name.avro2parquet") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,10 @@ class ParquetTypeSuite extends MagnolifySuite {
t: ParquetType[T],
eq: Eq[T]
): Unit = {
// Ensure serializable even after evaluation of `schema`
val parquetSchema = t.schema
val tpe = ensureSerializable(t)

property(className[T]) {
Prop.forAll { t: T =>
val out = new TestOutputFile
Expand Down

0 comments on commit a8c5b67

Please sign in to comment.