Merge pull request #516 from gnieh/fix/issue-515

Fix output duplication on escaped chunk boundaries

json/src/main/scala/fs2/data/json/internal/JsonTokenParser.scala

@@ -105,6 +105,7 @@ private[json] class JsonTokenParser[F[_], T, Res](
           } else if (c == '\\') {
             T.appendMarked(context, acc)
             T.advance(context)
+            T.mark(context)
             slowString_(key, StringState.SeenBackslash, 0, acc)
           } else if (c >= 0x20 && c <= 0x10ffff) {
             T.advance(context)

json/src/test/scala/fs2/data/json/Issue515Spec.scala

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2023 Lucas Satabin
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs2
+package data.json
+
+import cats.effect.IO
+import fs2.data.text.utf8._
+import weaver.SimpleIOSuite
+
+import java.nio.charset.StandardCharsets
+
+object Issue515Spec extends SimpleIOSuite {
+
+  // Tests the fix for https://github.com/gnieh/fs2-data/issues/515
+  // Problem was an escape sequence on a chunk boundary leading to outputting parts of the output buffer twice.
+  test("not duplicate tokens for escaped chars on chunk boundaries") {
+    val chunkSize = 55
+    val input =
+      """{"FilePaths":["\\\\path\\to\\some\\file.txt","\\\\path\\to\\some\\other\\file.txt","\\\\path\\to\\another\\file.txt"]}"""
+    val stream: fs2.Stream[IO, Byte] = Stream.emits(input.getBytes(StandardCharsets.UTF_8))
+    stream
+      .chunkLimit(chunkSize)
+      .flatMap(Stream.chunk)
+      .through(tokens[IO, Byte])
+      .through(render.compact)
+      .compile
+      .foldMonoid
+      .map(out => expect.eql(input, out))
+  }
+
+}