From da3232646350c3964ef95fadbac7b7a3b62ebb48 Mon Sep 17 00:00:00 2001 From: Gregor Rayman Date: Tue, 10 Sep 2024 19:28:21 +0200 Subject: [PATCH 01/11] Changes the default response encoding of `Endpoint.outStream[X]` for the MIME type `application/json` so that is produces a valid JSON array. Before it would simply produce concatenated JSON representations of the elements. For numeric `X` the response would not be parsable. --- .../zio/http/endpoint/RoundtripSpec.scala | 37 +++++ .../zio/http/codec/HttpContentCodec.scala | 153 +++++++++++++++++- 2 files changed, 185 insertions(+), 5 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala index 00062ede9a..3b6fa237f6 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala @@ -369,6 +369,43 @@ object RoundtripSpec extends ZIOHttpSpec { (stream: ZStream[Any, Nothing, Byte]) => stream.runCount.map(c => assert(c)(equalTo(1024L * 1024L))), ) }, + test("string stream output") { + val api = Endpoint(GET / "download").query(HttpCodec.query[Int]("count")).outStream[String] + val route = api.implementHandler { + Handler.fromFunctionZIO { count => + ZIO.succeed(ZStream.fromIterable((0 until count).map(_.toString))) + } + } + + testEndpointZIO( + api, + Routes(route), + 1024 * 1024, + (stream: ZStream[Any, Nothing, String]) => + stream.zipWithIndex + .runFold((true, 0)) { case ((allOk, count), (str, idx)) => + (allOk && str == idx.toString, count + 1) + } + .map { case (allOk, c) => + assertTrue(allOk && c == 1024 * 1024) + }, + ) + }, + test("string output") { + val api = Endpoint(GET / "download").query(HttpCodec.query[String]("param")).out[String] + val route = api.implementHandler { + Handler.fromFunctionZIO { param => + ZIO.succeed(param) + } + } + + testEndpointZIO( + api, + Routes(route), + "test", + (str: String) => assertTrue(str == "test"), + ) + }, test("multi-part input") { val api = Endpoint(POST / "test") .in[String]("name") diff --git a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala index 0c92a45d89..db90f2c1f3 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala @@ -1,11 +1,15 @@ package zio.http.codec +import java.nio.charset.StandardCharsets + import scala.collection.immutable.ListMap import zio._ -import zio.stream.ZPipeline +import zio.stream.{ZChannel, ZPipeline} +import zio.schema.codec.DecodeError.ReadError +import zio.schema.codec.JsonCodec.{JsonDecoder, JsonEncoder} import zio.schema.codec._ import zio.schema.{DeriveSchema, Schema} @@ -273,6 +277,118 @@ object HttpContentCodec { } object json { + + val splitJsonArrayElements: ZPipeline[Any, Nothing, String, String] = { + val validNumChars = Set('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'E', 'e', '-', '+', '.') + val ContextJson = 'j' + val ContextString = 's' + val ContextBoolean = 'b' + val ContextNull = 'u' + val ContextNullAfterFirstL = 'x' + val ContextNumber = 'n' + val ContextEscape = 'e' + val ContextDone = 'd' + + ZPipeline.suspend { + val stringBuilder = new StringBuilder + var depth = -1 + var context = ContextJson + + def fetchChunk(chunk: Chunk[String]): Chunk[String] = { + val chunkBuilder = ChunkBuilder.make[String]() + for { + string <- chunk + c <- string + } { + var valueEnded = false + context match { + case ContextEscape => + context = 's' + case ContextString => + c match { + case '\\' => context = ContextEscape + case '"' => + context = ContextJson + valueEnded = true + case _ => + } + case ContextBoolean => + if (c == 'e') { + context = ContextJson + valueEnded = true + } + case ContextNull => + if (c == 'l') { + context = ContextNullAfterFirstL + } + case ContextNullAfterFirstL => + if (c == 'l') { + context = ContextJson + valueEnded = true + } + case ContextNumber => + c match { + case '}' | ']' => + depth -= 1 + context = if (depth < 0) ContextDone else ContextJson + valueEnded = true + case _ if !validNumChars(c) => + context = ContextJson + valueEnded = true + case _ => + } + case ContextDone => // no more values, ignore everything + case _ => + c match { + case '{' | '[' => + depth += 1 + case '}' | ']' => + depth -= 1 + valueEnded = true + if (depth == -1) context = ContextDone + case '"' => + context = ContextString + case 't' | 'f' => + context = ContextBoolean + case 'n' => + context = ContextNull + case x if validNumChars(x) => + context = ContextNumber + case _ => + } + } + if (context != ContextDone && (depth > 0 || context != ContextJson || valueEnded)) + stringBuilder.append(c) + + if (valueEnded && depth == 0) { + val str = stringBuilder.result() + if (!str.forall(_.isWhitespace)) { + chunkBuilder += str + } + stringBuilder.clear() + } + } + chunkBuilder.result() + } + + lazy val loop: ZChannel[Any, ZNothing, Chunk[String], Any, Nothing, Chunk[String], Any] = + ZChannel.readWithCause( + in => { + val out = fetchChunk(in) + if (out.isEmpty) loop else ZChannel.write(out) *> loop + }, + err => + if (stringBuilder.isEmpty) ZChannel.refailCause(err) + else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.refailCause(err), + done => + if (stringBuilder.isEmpty) ZChannel.succeed(done) + else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.succeed(done), + ) + + ZPipeline.fromChannel(loop) + } + } + private var jsonCodecCache: Map[Schema[_], HttpContentCodec[_]] = Map.empty def only[A](implicit schema: Schema[A]): HttpContentCodec[A] = if (jsonCodecCache.contains(schema)) { @@ -282,10 +398,37 @@ object HttpContentCodec { ListMap( MediaType.application.`json` -> BinaryCodecWithSchema( - config => - JsonCodec.schemaBasedBinaryCodec[A]( - JsonCodec.Config(ignoreEmptyCollections = config.ignoreEmptyCollections), - )(schema), + config => { + val codecConfig = JsonCodec.Config(ignoreEmptyCollections = config.ignoreEmptyCollections) + + new BinaryCodec[A] { + override def decode(whole: Chunk[Byte]): Either[DecodeError, A] = + JsonDecoder.decode( + schema, + new String(whole.toArray, StandardCharsets.UTF_8), + ) + + override def streamDecoder: ZPipeline[Any, DecodeError, Byte, A] = + ZPipeline.utfDecode.mapError(cce => ReadError(Cause.fail(cce), cce.getMessage)) >>> + splitJsonArrayElements >>> + ZPipeline.mapZIO { (s: String) => + ZIO.fromEither(JsonDecoder.decode(schema, s)) + } + + override def encode(value: A): Chunk[Byte] = + JsonEncoder.encode(schema, value, codecConfig) + + override def streamEncoder: ZPipeline[Any, Nothing, A, Byte] = { + val interspersed: ZPipeline[Any, Nothing, A, Byte] = ZPipeline + .mapChunks[A, Chunk[Byte]](_.map(encode)) + .intersperse(Chunk.single(','.toByte)) + .flattenChunks + val prepended: ZPipeline[Any, Nothing, A, Byte] = + interspersed >>> ZPipeline.prepend(Chunk.single('['.toByte)) + prepended >>> ZPipeline.append(Chunk.single(']'.toByte)) + } + } + }, schema, ), ), From f25d3d3e479e14ab1a3aaca6e9502170ce17fef3 Mon Sep 17 00:00:00 2001 From: Gregor Rayman Date: Tue, 10 Sep 2024 21:08:30 +0200 Subject: [PATCH 02/11] The change to the default response encoding of `Endpoint.outStream[X]` for the MIME type `application/json` to JSON array reflected in the generated OpenAPI. --- .../endpoint/openapi/OpenAPIGenSpec.scala | 115 +++++++++++++++++- .../http/endpoint/openapi/OpenAPIGen.scala | 21 +++- 2 files changed, 133 insertions(+), 3 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala index 99faac7f3d..06e00af622 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala @@ -11,7 +11,7 @@ import zio.schema.{DeriveSchema, Schema} import zio.http.Method.{GET, POST} import zio.http._ import zio.http.codec.PathCodec.string -import zio.http.codec.{ContentCodec, Doc, HttpCodec, HttpContentCodec, QueryCodec} +import zio.http.codec.{ContentCodec, Doc, HttpCodec} import zio.http.endpoint._ object OpenAPIGenSpec extends ZIOSpecDefault { @@ -2854,6 +2854,119 @@ object OpenAPIGenSpec extends ZIOSpecDefault { |""".stripMargin assertTrue(json == toJsonAst(expectedJson)) }, + test("Stream schema") { + val endpoint = Endpoint(RoutePattern.POST / "folder") + .outStream[Int] + val openApi = OpenAPIGen.fromEndpoints(endpoint) + val json = toJsonAst(openApi) + val expectedJson = + """ + |{ + | "openapi" : "3.1.0", + | "info" : { + | "title" : "", + | "version" : "" + | }, + | "paths" : { + | "/folder" : { + | "post" : { + | "responses" : { + | "200" : + | { + | "content" : { + | "application/json" : { + | "schema" : + | { + | "type" : + | "array", + | "items" : { + | "type" : + | "integer", + | "format" : "int32" + | } + | } + | } + | } + | } + | } + | } + | } + | }, + | "components" : { + | + | } + |} + |""".stripMargin + assertTrue(json == toJsonAst(expectedJson)) + }, + test("Stream schema multipart") { + val endpoint = Endpoint(RoutePattern.POST / "folder") + .outCodec( + HttpCodec.contentStream[String]("strings") ++ + HttpCodec.contentStream[Int]("ints"), + ) + val openApi = OpenAPIGen.fromEndpoints(endpoint) + val json = toJsonAst(openApi) + val expectedJson = + """ + |{ + | "openapi" : "3.1.0", + | "info" : { + | "title" : "", + | "version" : "" + | }, + | "paths" : { + | "/folder" : { + | "post" : { + | "responses" : { + | "default" : + | { + | "content" : { + | "multipart/form-data" : { + | "schema" : + | { + | "type" : + | "object", + | "properties" : { + | "strings" : { + | "type" : + | "array", + | "items" : { + | "type" : + | "string" + | } + | }, + | "ints" : { + | "type" : + | "array", + | "items" : { + | "type" : + | "integer", + | "format" : "int32" + | } + | } + | }, + | "additionalProperties" : + | false, + | "required" : [ + | "strings", + | "ints" + | ] + | } + | } + | } + | } + | } + | } + | } + | }, + | "components" : { + | + | } + |} + |""".stripMargin + assertTrue(json == toJsonAst(expectedJson)) + }, test("Lazy schema") { val endpoint = Endpoint(RoutePattern.POST / "lazy") .in[Lazy.A] diff --git a/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala b/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala index 8506980560..9b12066770 100644 --- a/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala +++ b/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala @@ -314,7 +314,17 @@ object OpenAPIGen { findName(metadata).orElse(maybeName).getOrElse(throw new Exception("Multipart content without name")) JsonSchema.obj( name -> JsonSchema - .fromZSchema(codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), referenceType) + .ArrayType( + Some( + JsonSchema + .fromZSchema( + codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), + referenceType, + ), + ), + None, + uniqueItems = false, + ) .description(descriptionFromMeta) .deprecated(deprecated(metadata)) .nullable(optional(metadata)), @@ -327,7 +337,14 @@ object OpenAPIGen { .nullable(optional(metadata)) case HttpCodec.ContentStream(codec, _, _) => JsonSchema - .fromZSchema(codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), referenceType) + .ArrayType( + Some( + JsonSchema + .fromZSchema(codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), referenceType), + ), + None, + uniqueItems = false, + ) .description(descriptionFromMeta) .deprecated(deprecated(metadata)) .nullable(optional(metadata)) From d3a926dd1a7900a3bc49ef5491b941107e90efd9 Mon Sep 17 00:00:00 2001 From: Gregor Rayman Date: Mon, 16 Sep 2024 07:56:59 +0200 Subject: [PATCH 03/11] Uses the snaphsot version 1.4.1+7-41892d53-SNAPSHOT of zio-schema, for the JsonCodec.Config.treatStreamsAsArrays --- build.sbt | 2 + project/Dependencies.scala | 2 +- .../scala/zio/http/BodySchemaOpsSpec.scala | 4 +- .../zio/http/codec/HttpContentCodec.scala | 144 +----------------- 4 files changed, 10 insertions(+), 142 deletions(-) diff --git a/build.sbt b/build.sbt index cc48774a24..e2ad47a064 100644 --- a/build.sbt +++ b/build.sbt @@ -12,6 +12,8 @@ val _ = sys.props += ("ZIOHttpLogLevel" -> Debug.ZIOHttpLogLevel) ThisBuild / githubWorkflowEnv += ("JDK_JAVA_OPTIONS" -> "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8") ThisBuild / githubWorkflowEnv += ("SBT_OPTS" -> "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8") +ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshots") + ThisBuild / githubWorkflowJavaVersions := Seq( JavaSpec.graalvm(Graalvm.Distribution("graalvm"), "17"), JavaSpec.graalvm(Graalvm.Distribution("graalvm"), "21"), diff --git a/project/Dependencies.scala b/project/Dependencies.scala index abca59886e..47d32a3ae9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,7 +9,7 @@ object Dependencies { val ZioCliVersion = "0.5.0" val ZioJsonVersion = "0.7.1" val ZioParserVersion = "0.1.10" - val ZioSchemaVersion = "1.4.1" + val ZioSchemaVersion = "1.4.1+7-41892d53-SNAPSHOT" val SttpVersion = "3.3.18" val ZioConfigVersion = "4.0.2" diff --git a/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala b/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala index 8dee92abb6..5bbc0a915d 100644 --- a/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala @@ -39,7 +39,9 @@ object BodySchemaOpsSpec extends ZIOHttpSpec { }, test("Body.fromStream") { val body = Body.fromStream(persons) - val expected = """{"name":"John","age":42}{"name":"Jane","age":43}""" + val expected = + """{"name":"John","age":42} + |{"name":"Jane","age":43}""".stripMargin body.asString.map(s => assertTrue(s == expected)) }, test("Body#to") { diff --git a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala index db90f2c1f3..c0223e4fe6 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala @@ -278,117 +278,6 @@ object HttpContentCodec { object json { - val splitJsonArrayElements: ZPipeline[Any, Nothing, String, String] = { - val validNumChars = Set('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'E', 'e', '-', '+', '.') - val ContextJson = 'j' - val ContextString = 's' - val ContextBoolean = 'b' - val ContextNull = 'u' - val ContextNullAfterFirstL = 'x' - val ContextNumber = 'n' - val ContextEscape = 'e' - val ContextDone = 'd' - - ZPipeline.suspend { - val stringBuilder = new StringBuilder - var depth = -1 - var context = ContextJson - - def fetchChunk(chunk: Chunk[String]): Chunk[String] = { - val chunkBuilder = ChunkBuilder.make[String]() - for { - string <- chunk - c <- string - } { - var valueEnded = false - context match { - case ContextEscape => - context = 's' - case ContextString => - c match { - case '\\' => context = ContextEscape - case '"' => - context = ContextJson - valueEnded = true - case _ => - } - case ContextBoolean => - if (c == 'e') { - context = ContextJson - valueEnded = true - } - case ContextNull => - if (c == 'l') { - context = ContextNullAfterFirstL - } - case ContextNullAfterFirstL => - if (c == 'l') { - context = ContextJson - valueEnded = true - } - case ContextNumber => - c match { - case '}' | ']' => - depth -= 1 - context = if (depth < 0) ContextDone else ContextJson - valueEnded = true - case _ if !validNumChars(c) => - context = ContextJson - valueEnded = true - case _ => - } - case ContextDone => // no more values, ignore everything - case _ => - c match { - case '{' | '[' => - depth += 1 - case '}' | ']' => - depth -= 1 - valueEnded = true - if (depth == -1) context = ContextDone - case '"' => - context = ContextString - case 't' | 'f' => - context = ContextBoolean - case 'n' => - context = ContextNull - case x if validNumChars(x) => - context = ContextNumber - case _ => - } - } - if (context != ContextDone && (depth > 0 || context != ContextJson || valueEnded)) - stringBuilder.append(c) - - if (valueEnded && depth == 0) { - val str = stringBuilder.result() - if (!str.forall(_.isWhitespace)) { - chunkBuilder += str - } - stringBuilder.clear() - } - } - chunkBuilder.result() - } - - lazy val loop: ZChannel[Any, ZNothing, Chunk[String], Any, Nothing, Chunk[String], Any] = - ZChannel.readWithCause( - in => { - val out = fetchChunk(in) - if (out.isEmpty) loop else ZChannel.write(out) *> loop - }, - err => - if (stringBuilder.isEmpty) ZChannel.refailCause(err) - else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.refailCause(err), - done => - if (stringBuilder.isEmpty) ZChannel.succeed(done) - else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.succeed(done), - ) - - ZPipeline.fromChannel(loop) - } - } - private var jsonCodecCache: Map[Schema[_], HttpContentCodec[_]] = Map.empty def only[A](implicit schema: Schema[A]): HttpContentCodec[A] = if (jsonCodecCache.contains(schema)) { @@ -399,35 +288,10 @@ object HttpContentCodec { MediaType.application.`json` -> BinaryCodecWithSchema( config => { - val codecConfig = JsonCodec.Config(ignoreEmptyCollections = config.ignoreEmptyCollections) - - new BinaryCodec[A] { - override def decode(whole: Chunk[Byte]): Either[DecodeError, A] = - JsonDecoder.decode( - schema, - new String(whole.toArray, StandardCharsets.UTF_8), - ) - - override def streamDecoder: ZPipeline[Any, DecodeError, Byte, A] = - ZPipeline.utfDecode.mapError(cce => ReadError(Cause.fail(cce), cce.getMessage)) >>> - splitJsonArrayElements >>> - ZPipeline.mapZIO { (s: String) => - ZIO.fromEither(JsonDecoder.decode(schema, s)) - } - - override def encode(value: A): Chunk[Byte] = - JsonEncoder.encode(schema, value, codecConfig) - - override def streamEncoder: ZPipeline[Any, Nothing, A, Byte] = { - val interspersed: ZPipeline[Any, Nothing, A, Byte] = ZPipeline - .mapChunks[A, Chunk[Byte]](_.map(encode)) - .intersperse(Chunk.single(','.toByte)) - .flattenChunks - val prepended: ZPipeline[Any, Nothing, A, Byte] = - interspersed >>> ZPipeline.prepend(Chunk.single('['.toByte)) - prepended >>> ZPipeline.append(Chunk.single(']'.toByte)) - } - } + JsonCodec.schemaBasedBinaryCodec( + JsonCodec + .Config(ignoreEmptyCollections = config.ignoreEmptyCollections, treatStreamsAsArrays = true), + )(schema) }, schema, ), From a52f6f68a6a3285ed2577d955f7177d516b034ca Mon Sep 17 00:00:00 2001 From: Aleksandr Klimov <2767789+geeeezmo@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:19:24 +0300 Subject: [PATCH 04/11] fix OpenAPI code gen not quoting arbitrary header names (#3136) --- zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala | 2 +- zio-http-gen/src/test/resources/EndpointWithHeaders.scala | 1 + .../src/test/scala/zio/http/gen/scala/CodeGenSpec.scala | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala b/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala index 1f4830e0db..4a797b83f7 100644 --- a/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala +++ b/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala @@ -371,7 +371,7 @@ object CodeGen { case "www-authenticate" => "HeaderCodec.wwwAuthenticate" case "x-frame-options" => "HeaderCodec.xFrameOptions" case "x-requested-with" => "HeaderCodec.xRequestedWith" - case name => s"HeaderCodec.name[String]($name)" + case name => s"""HeaderCodec.name[String]("$name")""" } s""".header($headerSelector)""" } diff --git a/zio-http-gen/src/test/resources/EndpointWithHeaders.scala b/zio-http-gen/src/test/resources/EndpointWithHeaders.scala index 677c72cf24..32d117c859 100644 --- a/zio-http-gen/src/test/resources/EndpointWithHeaders.scala +++ b/zio-http-gen/src/test/resources/EndpointWithHeaders.scala @@ -9,6 +9,7 @@ object Users { val get = Endpoint(Method.GET / "api" / "v1" / "users") .header(HeaderCodec.accept) .header(HeaderCodec.contentType) + .header(HeaderCodec.name[String]("token")) .in[Unit] } diff --git a/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala b/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala index 6fc4910a98..df89ae8bec 100644 --- a/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala +++ b/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala @@ -151,7 +151,10 @@ object CodeGenSpec extends ZIOSpecDefault { }, test("Endpoint with headers") { val endpoint = - Endpoint(Method.GET / "api" / "v1" / "users").header(HeaderCodec.accept).header(HeaderCodec.contentType) + Endpoint(Method.GET / "api" / "v1" / "users") + .header(HeaderCodec.accept) + .header(HeaderCodec.contentType) + .header(HeaderCodec.name[String]("Token")) val openAPI = OpenAPIGen.fromEndpoints(endpoint) codeGenFromOpenAPI(openAPI) { testDir => From 46980fd01365e336b879fcf65151f908df62efc3 Mon Sep 17 00:00:00 2001 From: kyri-petrou <67301607+kyri-petrou@users.noreply.github.com> Date: Wed, 18 Sep 2024 18:18:07 +0300 Subject: [PATCH 05/11] Optimizations for request execution happy path (#3143) Optimize for request happy path --- .../netty/server/ServerInboundHandler.scala | 8 +++--- .../main/scala/zio/http/RoutePattern.scala | 15 ++++++----- .../src/main/scala/zio/http/Routes.scala | 25 +++++++++++-------- .../main/scala/zio/http/codec/PathCodec.scala | 14 +++++------ 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index 5c11953cbf..74340d825e 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -48,8 +48,8 @@ private[zio] final case class ServerInboundHandler( implicit private val unsafe: Unsafe = Unsafe.unsafe - private var routes: Routes[Any, Response] = _ - private var runtime: NettyRuntime = _ + private var handler: Handler[Any, Nothing, Request, Response] = _ + private var runtime: NettyRuntime = _ val inFlightRequests: LongAdder = new LongAdder() private val readClientCert = config.sslConfig.exists(_.includeClientCert) @@ -58,7 +58,7 @@ private[zio] final case class ServerInboundHandler( def refreshApp(): Unit = { val pair = appRef.get() - this.routes = pair._1 + this.handler = pair._1.toHandler this.runtime = new NettyRuntime(pair._2) } @@ -88,7 +88,7 @@ private[zio] final case class ServerInboundHandler( releaseRequest() } else { val req = makeZioRequest(ctx, jReq) - val exit = routes(req) + val exit = handler(req) if (attemptImmediateWrite(ctx, req.method, exit)) { releaseRequest() } else { diff --git a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala index f0adad7730..b73f5b09b4 100644 --- a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala +++ b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala @@ -172,16 +172,15 @@ object RoutePattern { tree.add(p, v) } + private val wildcardsTree = roots.getOrElse(Method.ANY, null) + def get(method: Method, path: Path): Chunk[A] = { - val wildcards = roots.get(Method.ANY) match { - case None => Chunk.empty - case Some(value) => value.get(path) + val forMethod = roots.getOrElse(method, null) match { + case null => Chunk.empty + case value => value.get(path) } - - (roots.get(method) match { - case None => Chunk.empty - case Some(value) => value.get(path) - }) ++ wildcards + if (wildcardsTree eq null) forMethod + else forMethod ++ wildcardsTree.get(path) } def map[B](f: A => B): Tree[B] = diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 720a959683..5847d9524e 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -245,20 +245,25 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s */ def toHandler(implicit ev: Err <:< Response): Handler[Env, Nothing, Request, Response] = { implicit val trace: Trace = Trace.empty + val tree = self.tree Handler .fromFunctionHandler[Request] { req => val chunk = tree.get(req.method, req.path) - - if (chunk.length == 0) Handler.notFound - else if (chunk.length == 1) chunk(0) - else { - // TODO: Support precomputed fallback among all chunk elements: - chunk.tail.foldLeft(chunk.head) { (acc, h) => - acc.catchAll { response => - if (response.status == Status.NotFound) h - else Handler.fail(response) + chunk.length match { + case 0 => Handler.notFound + case 1 => chunk(0) + case n => // TODO: Support precomputed fallback among all chunk elements + var acc = chunk(0) + var i = 1 + while (i < n) { + val h = chunk(i) + acc = acc.catchAll { response => + if (response.status == Status.NotFound) h + else Handler.fail(response) + } + i += 1 } - } + acc } } .merge diff --git a/zio-http/shared/src/main/scala/zio/http/codec/PathCodec.scala b/zio-http/shared/src/main/scala/zio/http/codec/PathCodec.scala index d9b3a9384d..ead97b5da7 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/PathCodec.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/PathCodec.scala @@ -788,7 +788,7 @@ object PathCodec { def get(path: Path): Chunk[A] = get(path, 0) - private def get(path: Path, from: Int, skipLiteralsFor: Set[Int] = Set.empty): Chunk[A] = { + private def get(path: Path, from: Int, skipLiteralsFor: Set[Int] = null): Chunk[A] = { val segments = path.segments val nSegments = segments.length var subtree = self @@ -801,7 +801,7 @@ object PathCodec { val segment = segments(i) // Fast path, jump down the tree: - if (!skipLiteralsFor.contains(i) && subtree.literals.contains(segment)) { + if ((skipLiteralsFor.eq(null) || !skipLiteralsFor.contains(i)) && subtree.literals.contains(segment)) { // this subtree segment have conflict with others // will try others if result was empty @@ -875,19 +875,19 @@ object PathCodec { // Might be some other matches because trailing matches everything: if (subtree ne null) { - subtree.others.get(SegmentCodec.trailing) match { - case Some(subtree) => - result = result ++ subtree.value - case None => + subtree.others.getOrElse(SegmentCodec.Trailing, null) match { + case null => () + case subtree => result = result ++ subtree.value } } if (trySkipLiteralIdx.nonEmpty && result.isEmpty) { trySkipLiteralIdx = trySkipLiteralIdx.reverse + val skipLiteralsFor0 = if (skipLiteralsFor eq null) Set.empty[Int] else skipLiteralsFor while (trySkipLiteralIdx.nonEmpty && result.isEmpty) { val skipIdx = trySkipLiteralIdx.head trySkipLiteralIdx = trySkipLiteralIdx.tail - result = get(path, from, skipLiteralsFor + skipIdx) + result = get(path, from, skipLiteralsFor0 + skipIdx) } result } else result From 087ee9456f07b4c1f33d842f4955b550855e86e5 Mon Sep 17 00:00:00 2001 From: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> Date: Thu, 19 Sep 2024 00:33:53 +0200 Subject: [PATCH 06/11] Deactivate broken test for now to make CI more reliable --- zio-http/jvm/src/test/scala/zio/http/ClientSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ClientSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ClientSpec.scala index 0bd5350539..51695c4c0d 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ClientSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ClientSpec.scala @@ -105,7 +105,7 @@ object ClientSpec extends RoutesRunnableSpec { val url = URL.decode("https://test.com").toOption.get val resp = ZClient.batched(Request.get(url)).timeout(500.millis) assertZIO(resp)(isNone) - } @@ timeout(5.seconds) @@ flaky(20), + } @@ timeout(5.seconds) @@ flaky(20) @@ TestAspect.ignore, // annoying in CI test("authorization header without scheme") { val app = Handler From 635603e22d6c97bb14c7a4c51d0cc322bc53da92 Mon Sep 17 00:00:00 2001 From: Naftoli Gugenheim <98384+nafg@users.noreply.github.com> Date: Thu, 19 Sep 2024 01:32:19 -0400 Subject: [PATCH 07/11] Fix #3103 Only last response is generated into Endpoint code (#3151) * gen: failing test: code is only generated for last response * Fix #3103 Only last response is generated into Endpoint code --- .../zio/http/gen/openapi/EndpointGen.scala | 2 +- .../zio/http/gen/scala/CodeGenSpec.scala | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala b/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala index 6c9209b6b0..f22afb38b8 100644 --- a/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala +++ b/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala @@ -460,7 +460,7 @@ final case class EndpointGen(config: Config) { val (outImports: Iterable[List[Code.Import]], outCodes: Iterable[Code.OutCode]) = // TODO: ignore default for now. Not sure how to handle it - op.responses.collect { + op.responses.toSeq.collect { case (OpenAPI.StatusOrDefault.StatusValue(status), OpenAPI.ReferenceOr.Reference(ResponseRef(key), _, _)) => val response = resolveResponseRef(openAPI, key) val (imports, code) = diff --git a/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala b/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala index df89ae8bec..e9a9e3a940 100644 --- a/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala +++ b/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala @@ -3,6 +3,7 @@ package zio.http.gen.scala import java.nio.file._ import scala.annotation.nowarn +import scala.collection.immutable.ListMap import scala.jdk.CollectionConverters._ import scala.meta._ import scala.meta.parsers._ @@ -975,5 +976,72 @@ object CodeGenSpec extends ZIOSpecDefault { } } } @@ TestAspect.exceptScala3, + test("Generate all responses") { + val oapi = + OpenAPI( + openapi = "3.0.0", + info = OpenAPI.Info( + title = "XXX", + description = None, + termsOfService = None, + contact = None, + license = None, + version = "1.0.0", + ), + paths = ListMap( + OpenAPI.Path + .fromString(name = "/api/a/b") + .map { path => + path -> OpenAPI.PathItem( + ref = None, + summary = None, + description = None, + get = None, + put = None, + post = Some( + OpenAPI.Operation( + summary = None, + description = None, + externalDocs = None, + operationId = None, + requestBody = None, + responses = Map( + OpenAPI.StatusOrDefault.StatusValue(status = Status.Ok) -> + OpenAPI.ReferenceOr.Or(value = OpenAPI.Response()), + OpenAPI.StatusOrDefault.StatusValue(Status.BadRequest) -> + OpenAPI.ReferenceOr.Or(OpenAPI.Response()), + OpenAPI.StatusOrDefault.StatusValue(Status.Unauthorized) -> + OpenAPI.ReferenceOr.Or(OpenAPI.Response()), + ), + ), + ), + delete = None, + options = None, + head = None, + patch = None, + trace = None, + ) + } + .toSeq: _*, + ), + components = None, + externalDocs = None, + ) + + val maybeEndpointCode = + EndpointGen + .fromOpenAPI(oapi, Config.default) + .files + .flatMap(_.objects) + .flatMap(_.endpoints) + .collectFirst { + case (field, code) if field.name == "post" => code + } + + assertTrue( + maybeEndpointCode.is(_.some).outCodes.length == 1 && + maybeEndpointCode.is(_.some).errorsCode.length == 2, + ) + }, ) @@ java11OrNewer @@ flaky @@ blocking // Downloading scalafmt on CI is flaky } From 25a272d87e1f58160354e80842436195428bc3b5 Mon Sep 17 00:00:00 2001 From: Gregor Rayman Date: Tue, 10 Sep 2024 19:28:21 +0200 Subject: [PATCH 08/11] Changes the default response encoding of `Endpoint.outStream[X]` for the MIME type `application/json` so that is produces a valid JSON array. Before it would simply produce concatenated JSON representations of the elements. For numeric `X` the response would not be parsable. --- .../zio/http/endpoint/RoundtripSpec.scala | 37 +++++ .../zio/http/codec/HttpContentCodec.scala | 153 +++++++++++++++++- 2 files changed, 185 insertions(+), 5 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala index 00062ede9a..3b6fa237f6 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala @@ -369,6 +369,43 @@ object RoundtripSpec extends ZIOHttpSpec { (stream: ZStream[Any, Nothing, Byte]) => stream.runCount.map(c => assert(c)(equalTo(1024L * 1024L))), ) }, + test("string stream output") { + val api = Endpoint(GET / "download").query(HttpCodec.query[Int]("count")).outStream[String] + val route = api.implementHandler { + Handler.fromFunctionZIO { count => + ZIO.succeed(ZStream.fromIterable((0 until count).map(_.toString))) + } + } + + testEndpointZIO( + api, + Routes(route), + 1024 * 1024, + (stream: ZStream[Any, Nothing, String]) => + stream.zipWithIndex + .runFold((true, 0)) { case ((allOk, count), (str, idx)) => + (allOk && str == idx.toString, count + 1) + } + .map { case (allOk, c) => + assertTrue(allOk && c == 1024 * 1024) + }, + ) + }, + test("string output") { + val api = Endpoint(GET / "download").query(HttpCodec.query[String]("param")).out[String] + val route = api.implementHandler { + Handler.fromFunctionZIO { param => + ZIO.succeed(param) + } + } + + testEndpointZIO( + api, + Routes(route), + "test", + (str: String) => assertTrue(str == "test"), + ) + }, test("multi-part input") { val api = Endpoint(POST / "test") .in[String]("name") diff --git a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala index 0c92a45d89..db90f2c1f3 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala @@ -1,11 +1,15 @@ package zio.http.codec +import java.nio.charset.StandardCharsets + import scala.collection.immutable.ListMap import zio._ -import zio.stream.ZPipeline +import zio.stream.{ZChannel, ZPipeline} +import zio.schema.codec.DecodeError.ReadError +import zio.schema.codec.JsonCodec.{JsonDecoder, JsonEncoder} import zio.schema.codec._ import zio.schema.{DeriveSchema, Schema} @@ -273,6 +277,118 @@ object HttpContentCodec { } object json { + + val splitJsonArrayElements: ZPipeline[Any, Nothing, String, String] = { + val validNumChars = Set('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'E', 'e', '-', '+', '.') + val ContextJson = 'j' + val ContextString = 's' + val ContextBoolean = 'b' + val ContextNull = 'u' + val ContextNullAfterFirstL = 'x' + val ContextNumber = 'n' + val ContextEscape = 'e' + val ContextDone = 'd' + + ZPipeline.suspend { + val stringBuilder = new StringBuilder + var depth = -1 + var context = ContextJson + + def fetchChunk(chunk: Chunk[String]): Chunk[String] = { + val chunkBuilder = ChunkBuilder.make[String]() + for { + string <- chunk + c <- string + } { + var valueEnded = false + context match { + case ContextEscape => + context = 's' + case ContextString => + c match { + case '\\' => context = ContextEscape + case '"' => + context = ContextJson + valueEnded = true + case _ => + } + case ContextBoolean => + if (c == 'e') { + context = ContextJson + valueEnded = true + } + case ContextNull => + if (c == 'l') { + context = ContextNullAfterFirstL + } + case ContextNullAfterFirstL => + if (c == 'l') { + context = ContextJson + valueEnded = true + } + case ContextNumber => + c match { + case '}' | ']' => + depth -= 1 + context = if (depth < 0) ContextDone else ContextJson + valueEnded = true + case _ if !validNumChars(c) => + context = ContextJson + valueEnded = true + case _ => + } + case ContextDone => // no more values, ignore everything + case _ => + c match { + case '{' | '[' => + depth += 1 + case '}' | ']' => + depth -= 1 + valueEnded = true + if (depth == -1) context = ContextDone + case '"' => + context = ContextString + case 't' | 'f' => + context = ContextBoolean + case 'n' => + context = ContextNull + case x if validNumChars(x) => + context = ContextNumber + case _ => + } + } + if (context != ContextDone && (depth > 0 || context != ContextJson || valueEnded)) + stringBuilder.append(c) + + if (valueEnded && depth == 0) { + val str = stringBuilder.result() + if (!str.forall(_.isWhitespace)) { + chunkBuilder += str + } + stringBuilder.clear() + } + } + chunkBuilder.result() + } + + lazy val loop: ZChannel[Any, ZNothing, Chunk[String], Any, Nothing, Chunk[String], Any] = + ZChannel.readWithCause( + in => { + val out = fetchChunk(in) + if (out.isEmpty) loop else ZChannel.write(out) *> loop + }, + err => + if (stringBuilder.isEmpty) ZChannel.refailCause(err) + else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.refailCause(err), + done => + if (stringBuilder.isEmpty) ZChannel.succeed(done) + else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.succeed(done), + ) + + ZPipeline.fromChannel(loop) + } + } + private var jsonCodecCache: Map[Schema[_], HttpContentCodec[_]] = Map.empty def only[A](implicit schema: Schema[A]): HttpContentCodec[A] = if (jsonCodecCache.contains(schema)) { @@ -282,10 +398,37 @@ object HttpContentCodec { ListMap( MediaType.application.`json` -> BinaryCodecWithSchema( - config => - JsonCodec.schemaBasedBinaryCodec[A]( - JsonCodec.Config(ignoreEmptyCollections = config.ignoreEmptyCollections), - )(schema), + config => { + val codecConfig = JsonCodec.Config(ignoreEmptyCollections = config.ignoreEmptyCollections) + + new BinaryCodec[A] { + override def decode(whole: Chunk[Byte]): Either[DecodeError, A] = + JsonDecoder.decode( + schema, + new String(whole.toArray, StandardCharsets.UTF_8), + ) + + override def streamDecoder: ZPipeline[Any, DecodeError, Byte, A] = + ZPipeline.utfDecode.mapError(cce => ReadError(Cause.fail(cce), cce.getMessage)) >>> + splitJsonArrayElements >>> + ZPipeline.mapZIO { (s: String) => + ZIO.fromEither(JsonDecoder.decode(schema, s)) + } + + override def encode(value: A): Chunk[Byte] = + JsonEncoder.encode(schema, value, codecConfig) + + override def streamEncoder: ZPipeline[Any, Nothing, A, Byte] = { + val interspersed: ZPipeline[Any, Nothing, A, Byte] = ZPipeline + .mapChunks[A, Chunk[Byte]](_.map(encode)) + .intersperse(Chunk.single(','.toByte)) + .flattenChunks + val prepended: ZPipeline[Any, Nothing, A, Byte] = + interspersed >>> ZPipeline.prepend(Chunk.single('['.toByte)) + prepended >>> ZPipeline.append(Chunk.single(']'.toByte)) + } + } + }, schema, ), ), From 05fd7c1baa484e033c8c9e88708c37c57ec50be7 Mon Sep 17 00:00:00 2001 From: Gregor Rayman Date: Tue, 10 Sep 2024 21:08:30 +0200 Subject: [PATCH 09/11] The change to the default response encoding of `Endpoint.outStream[X]` for the MIME type `application/json` to JSON array reflected in the generated OpenAPI. --- .../endpoint/openapi/OpenAPIGenSpec.scala | 115 +++++++++++++++++- .../http/endpoint/openapi/OpenAPIGen.scala | 21 +++- 2 files changed, 133 insertions(+), 3 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala index 99faac7f3d..06e00af622 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/openapi/OpenAPIGenSpec.scala @@ -11,7 +11,7 @@ import zio.schema.{DeriveSchema, Schema} import zio.http.Method.{GET, POST} import zio.http._ import zio.http.codec.PathCodec.string -import zio.http.codec.{ContentCodec, Doc, HttpCodec, HttpContentCodec, QueryCodec} +import zio.http.codec.{ContentCodec, Doc, HttpCodec} import zio.http.endpoint._ object OpenAPIGenSpec extends ZIOSpecDefault { @@ -2854,6 +2854,119 @@ object OpenAPIGenSpec extends ZIOSpecDefault { |""".stripMargin assertTrue(json == toJsonAst(expectedJson)) }, + test("Stream schema") { + val endpoint = Endpoint(RoutePattern.POST / "folder") + .outStream[Int] + val openApi = OpenAPIGen.fromEndpoints(endpoint) + val json = toJsonAst(openApi) + val expectedJson = + """ + |{ + | "openapi" : "3.1.0", + | "info" : { + | "title" : "", + | "version" : "" + | }, + | "paths" : { + | "/folder" : { + | "post" : { + | "responses" : { + | "200" : + | { + | "content" : { + | "application/json" : { + | "schema" : + | { + | "type" : + | "array", + | "items" : { + | "type" : + | "integer", + | "format" : "int32" + | } + | } + | } + | } + | } + | } + | } + | } + | }, + | "components" : { + | + | } + |} + |""".stripMargin + assertTrue(json == toJsonAst(expectedJson)) + }, + test("Stream schema multipart") { + val endpoint = Endpoint(RoutePattern.POST / "folder") + .outCodec( + HttpCodec.contentStream[String]("strings") ++ + HttpCodec.contentStream[Int]("ints"), + ) + val openApi = OpenAPIGen.fromEndpoints(endpoint) + val json = toJsonAst(openApi) + val expectedJson = + """ + |{ + | "openapi" : "3.1.0", + | "info" : { + | "title" : "", + | "version" : "" + | }, + | "paths" : { + | "/folder" : { + | "post" : { + | "responses" : { + | "default" : + | { + | "content" : { + | "multipart/form-data" : { + | "schema" : + | { + | "type" : + | "object", + | "properties" : { + | "strings" : { + | "type" : + | "array", + | "items" : { + | "type" : + | "string" + | } + | }, + | "ints" : { + | "type" : + | "array", + | "items" : { + | "type" : + | "integer", + | "format" : "int32" + | } + | } + | }, + | "additionalProperties" : + | false, + | "required" : [ + | "strings", + | "ints" + | ] + | } + | } + | } + | } + | } + | } + | } + | }, + | "components" : { + | + | } + |} + |""".stripMargin + assertTrue(json == toJsonAst(expectedJson)) + }, test("Lazy schema") { val endpoint = Endpoint(RoutePattern.POST / "lazy") .in[Lazy.A] diff --git a/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala b/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala index 8506980560..9b12066770 100644 --- a/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala +++ b/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/OpenAPIGen.scala @@ -314,7 +314,17 @@ object OpenAPIGen { findName(metadata).orElse(maybeName).getOrElse(throw new Exception("Multipart content without name")) JsonSchema.obj( name -> JsonSchema - .fromZSchema(codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), referenceType) + .ArrayType( + Some( + JsonSchema + .fromZSchema( + codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), + referenceType, + ), + ), + None, + uniqueItems = false, + ) .description(descriptionFromMeta) .deprecated(deprecated(metadata)) .nullable(optional(metadata)), @@ -327,7 +337,14 @@ object OpenAPIGen { .nullable(optional(metadata)) case HttpCodec.ContentStream(codec, _, _) => JsonSchema - .fromZSchema(codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), referenceType) + .ArrayType( + Some( + JsonSchema + .fromZSchema(codec.lookup(mediaType).map(_._2.schema).getOrElse(codec.defaultSchema), referenceType), + ), + None, + uniqueItems = false, + ) .description(descriptionFromMeta) .deprecated(deprecated(metadata)) .nullable(optional(metadata)) From a6ca02fac54de86480bbc711563750704bdf4cd3 Mon Sep 17 00:00:00 2001 From: Gregor Rayman Date: Mon, 16 Sep 2024 07:56:59 +0200 Subject: [PATCH 10/11] Uses the snaphsot version 1.4.1+7-41892d53-SNAPSHOT of zio-schema, for the JsonCodec.Config.treatStreamsAsArrays --- build.sbt | 2 + project/Dependencies.scala | 2 +- .../scala/zio/http/BodySchemaOpsSpec.scala | 4 +- .../zio/http/codec/HttpContentCodec.scala | 144 +----------------- 4 files changed, 10 insertions(+), 142 deletions(-) diff --git a/build.sbt b/build.sbt index cc48774a24..e2ad47a064 100644 --- a/build.sbt +++ b/build.sbt @@ -12,6 +12,8 @@ val _ = sys.props += ("ZIOHttpLogLevel" -> Debug.ZIOHttpLogLevel) ThisBuild / githubWorkflowEnv += ("JDK_JAVA_OPTIONS" -> "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8") ThisBuild / githubWorkflowEnv += ("SBT_OPTS" -> "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8") +ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshots") + ThisBuild / githubWorkflowJavaVersions := Seq( JavaSpec.graalvm(Graalvm.Distribution("graalvm"), "17"), JavaSpec.graalvm(Graalvm.Distribution("graalvm"), "21"), diff --git a/project/Dependencies.scala b/project/Dependencies.scala index abca59886e..47d32a3ae9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,7 +9,7 @@ object Dependencies { val ZioCliVersion = "0.5.0" val ZioJsonVersion = "0.7.1" val ZioParserVersion = "0.1.10" - val ZioSchemaVersion = "1.4.1" + val ZioSchemaVersion = "1.4.1+7-41892d53-SNAPSHOT" val SttpVersion = "3.3.18" val ZioConfigVersion = "4.0.2" diff --git a/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala b/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala index 8dee92abb6..5bbc0a915d 100644 --- a/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/BodySchemaOpsSpec.scala @@ -39,7 +39,9 @@ object BodySchemaOpsSpec extends ZIOHttpSpec { }, test("Body.fromStream") { val body = Body.fromStream(persons) - val expected = """{"name":"John","age":42}{"name":"Jane","age":43}""" + val expected = + """{"name":"John","age":42} + |{"name":"Jane","age":43}""".stripMargin body.asString.map(s => assertTrue(s == expected)) }, test("Body#to") { diff --git a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala index db90f2c1f3..c0223e4fe6 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala @@ -278,117 +278,6 @@ object HttpContentCodec { object json { - val splitJsonArrayElements: ZPipeline[Any, Nothing, String, String] = { - val validNumChars = Set('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'E', 'e', '-', '+', '.') - val ContextJson = 'j' - val ContextString = 's' - val ContextBoolean = 'b' - val ContextNull = 'u' - val ContextNullAfterFirstL = 'x' - val ContextNumber = 'n' - val ContextEscape = 'e' - val ContextDone = 'd' - - ZPipeline.suspend { - val stringBuilder = new StringBuilder - var depth = -1 - var context = ContextJson - - def fetchChunk(chunk: Chunk[String]): Chunk[String] = { - val chunkBuilder = ChunkBuilder.make[String]() - for { - string <- chunk - c <- string - } { - var valueEnded = false - context match { - case ContextEscape => - context = 's' - case ContextString => - c match { - case '\\' => context = ContextEscape - case '"' => - context = ContextJson - valueEnded = true - case _ => - } - case ContextBoolean => - if (c == 'e') { - context = ContextJson - valueEnded = true - } - case ContextNull => - if (c == 'l') { - context = ContextNullAfterFirstL - } - case ContextNullAfterFirstL => - if (c == 'l') { - context = ContextJson - valueEnded = true - } - case ContextNumber => - c match { - case '}' | ']' => - depth -= 1 - context = if (depth < 0) ContextDone else ContextJson - valueEnded = true - case _ if !validNumChars(c) => - context = ContextJson - valueEnded = true - case _ => - } - case ContextDone => // no more values, ignore everything - case _ => - c match { - case '{' | '[' => - depth += 1 - case '}' | ']' => - depth -= 1 - valueEnded = true - if (depth == -1) context = ContextDone - case '"' => - context = ContextString - case 't' | 'f' => - context = ContextBoolean - case 'n' => - context = ContextNull - case x if validNumChars(x) => - context = ContextNumber - case _ => - } - } - if (context != ContextDone && (depth > 0 || context != ContextJson || valueEnded)) - stringBuilder.append(c) - - if (valueEnded && depth == 0) { - val str = stringBuilder.result() - if (!str.forall(_.isWhitespace)) { - chunkBuilder += str - } - stringBuilder.clear() - } - } - chunkBuilder.result() - } - - lazy val loop: ZChannel[Any, ZNothing, Chunk[String], Any, Nothing, Chunk[String], Any] = - ZChannel.readWithCause( - in => { - val out = fetchChunk(in) - if (out.isEmpty) loop else ZChannel.write(out) *> loop - }, - err => - if (stringBuilder.isEmpty) ZChannel.refailCause(err) - else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.refailCause(err), - done => - if (stringBuilder.isEmpty) ZChannel.succeed(done) - else ZChannel.write(Chunk.single(stringBuilder.result())) *> ZChannel.succeed(done), - ) - - ZPipeline.fromChannel(loop) - } - } - private var jsonCodecCache: Map[Schema[_], HttpContentCodec[_]] = Map.empty def only[A](implicit schema: Schema[A]): HttpContentCodec[A] = if (jsonCodecCache.contains(schema)) { @@ -399,35 +288,10 @@ object HttpContentCodec { MediaType.application.`json` -> BinaryCodecWithSchema( config => { - val codecConfig = JsonCodec.Config(ignoreEmptyCollections = config.ignoreEmptyCollections) - - new BinaryCodec[A] { - override def decode(whole: Chunk[Byte]): Either[DecodeError, A] = - JsonDecoder.decode( - schema, - new String(whole.toArray, StandardCharsets.UTF_8), - ) - - override def streamDecoder: ZPipeline[Any, DecodeError, Byte, A] = - ZPipeline.utfDecode.mapError(cce => ReadError(Cause.fail(cce), cce.getMessage)) >>> - splitJsonArrayElements >>> - ZPipeline.mapZIO { (s: String) => - ZIO.fromEither(JsonDecoder.decode(schema, s)) - } - - override def encode(value: A): Chunk[Byte] = - JsonEncoder.encode(schema, value, codecConfig) - - override def streamEncoder: ZPipeline[Any, Nothing, A, Byte] = { - val interspersed: ZPipeline[Any, Nothing, A, Byte] = ZPipeline - .mapChunks[A, Chunk[Byte]](_.map(encode)) - .intersperse(Chunk.single(','.toByte)) - .flattenChunks - val prepended: ZPipeline[Any, Nothing, A, Byte] = - interspersed >>> ZPipeline.prepend(Chunk.single('['.toByte)) - prepended >>> ZPipeline.append(Chunk.single(']'.toByte)) - } - } + JsonCodec.schemaBasedBinaryCodec( + JsonCodec + .Config(ignoreEmptyCollections = config.ignoreEmptyCollections, treatStreamsAsArrays = true), + )(schema) }, schema, ), From f7784202393ae1a2585e78c0bb4efb103b93c9c6 Mon Sep 17 00:00:00 2001 From: Gregor Rayman Date: Thu, 19 Sep 2024 16:17:53 +0200 Subject: [PATCH 11/11] Uses the version 1.5.0 of zio-schema, for the JsonCodec.Config.treatStreamsAsArrays --- build.sbt | 2 -- project/Dependencies.scala | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index e2ad47a064..cc48774a24 100644 --- a/build.sbt +++ b/build.sbt @@ -12,8 +12,6 @@ val _ = sys.props += ("ZIOHttpLogLevel" -> Debug.ZIOHttpLogLevel) ThisBuild / githubWorkflowEnv += ("JDK_JAVA_OPTIONS" -> "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8") ThisBuild / githubWorkflowEnv += ("SBT_OPTS" -> "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8") -ThisBuild / resolvers ++= Resolver.sonatypeOssRepos("snapshots") - ThisBuild / githubWorkflowJavaVersions := Seq( JavaSpec.graalvm(Graalvm.Distribution("graalvm"), "17"), JavaSpec.graalvm(Graalvm.Distribution("graalvm"), "21"), diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 47d32a3ae9..65007e599b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,7 +9,7 @@ object Dependencies { val ZioCliVersion = "0.5.0" val ZioJsonVersion = "0.7.1" val ZioParserVersion = "0.1.10" - val ZioSchemaVersion = "1.4.1+7-41892d53-SNAPSHOT" + val ZioSchemaVersion = "1.5.0" val SttpVersion = "3.3.18" val ZioConfigVersion = "4.0.2"