fix(yieldable-parser): parseAsync sometimes blocks for a *long* time #36

test/async-index-of.js

@@ -0,0 +1,35 @@
+const tap = require('tap');
+const { asyncIndexOf } = require('../utils/async-index-of');
+
+const stringAsArray = ['s', 'o', 'm', 'e'];
+const text = stringAsArray.join('');
+
+tap.test('should return -1 if not found due to initialIndex', async () => {
+ const index = await asyncIndexOf(text, 's', 1);
+ const syncIndexOf = text.indexOf('s', 1);
+ tap.equal(index, syncIndexOf);
+})
+
+tap.test('should return proper index', async () => {
+ const index = await asyncIndexOf(text, 'e');
+ const syncIndexOf = text.indexOf('e');
+ tap.equal(index, syncIndexOf);
+})
+
+tap.test('should return proper test with initialIndex', async () => {
+ const index = await asyncIndexOf(text, 'e', 2);
+ const syncIndexOf = text.indexOf('e', 2);
+ tap.equal(index, syncIndexOf); 
+})
+
+tap.test('should return correct indexOf if initialIndex is same', async () => {
+ const index = await asyncIndexOf(text, 'e', 3);
+ const syncIndexOf = text.indexOf('e', 3);
+ tap.equal(index, syncIndexOf); 
+})
+
+tap.test('should return -1 if not found', async () => {
+ const index = await asyncIndexOf(text, 'x');
+ const syncIndexOf = text.indexOf('x');
+ tap.equal(index, syncIndexOf); 
+})

test/big-json-with-string copy.txt

@@ -0,0 +1 @@
+{"titles":["É9075840612JËÍ "],"text":"Your operator's manuals \n\nDigital in the vehicle Familiarize yourself with the contents of the operator's manual directly via the vehicle's multimedia system (Menu item \"Vehicle\"). Start with the quick guide or deepen your knowledge with practical tips. \n\nVehicle document wallet in the vehicle Here you can find information on operation, service work and the warranty for your vehicle in printed form. \n\nSprinter \n\nOperating Instructions \n\n9075840612 \n\nOrder no. T907 0455 13 Part no. 907 584 06 12 Edition A-2023-2 \n\nFront passenger air bag warning \n\nAir bag warning sticker for USA and Canada \n\n& WARNING Risk of injury or fatal injuries if the front passenger air bag is enabled \n\nIf the front passenger air bag is enabled, a child on the front passenger seat may be struck by the front passenger air bag in the event of an accident. # NEVER use a rearward-facing child \n\nrestraint system on a seat with an ENA- BLED FRONT AIR BAG. This can result in the DEATH of or SERIOUS INJURY to the CHILD. \n\nObserve the chapter entitled \"Children in the vehi- cle\". \n\nPublication details \n\nInternet Further information about Mercedes-Benz vehicles and about Mercedes‑Benz AG can be found on the following websites: https://www.mercedes-benz.com https://www.mbusa.com (USA only) https://www.mercedes-benz.ca (Canada only) \n\nDocumentation team ©Daimler VANS USA, LLC ©Mercedes‑Benz AG: Not to be reprinted, transla- ted or otherwise reproduced, in whole or in part, without","unique":0,"loc":1,"id":"sec_76f277dd-8730-4376-a281-2b195c8930f4","sumid":"sum_b977530e-7227-49e2-900d-b2c68de38734"}

test/test-parse-big-escaped-string.js

@@ -0,0 +1,16 @@
+'use strict';
+
+const yj = require('../index');
+const tap = require('tap');
+const { readFileSync } = require('fs');
+//const str = readFileSync('./test/big-json-with-string.txt');
+const str = readFileSync('./test/big-json-with-string copy.txt');
+const nodeJSNativeParse = JSON.parse(str);
+
+// Make sure the API works well without the optional parameters.
+yj.parseAsync(str, (err, obj) => {
+ if (!err)
+ tap.equal(obj.text, nodeJSNativeParse.text, 'Failed to parse big string message - omitting logs', { diagnostic: true });
+ else
+ tap.fail(err);
+});

utils/async-index-of.js

@@ -0,0 +1,21 @@
+module.exports.indexOfGenerator = function* indexOfGenerator(stringInput, searchInput, initialIndex = 0) {
+ if (!stringInput || !stringInput.length) {
+ return -1;
+ }
+ stringInput = stringInput.slice(initialIndex);
+ let innerAt = 0;
+ let index = -1;
+ while (true) {
+ const at = Math.max(innerAt - 1, 0);
+ yield chunk = stringInput.slice(at, innerAt + 5000);
+ if (!chunk.length) {
+ return index;
+ }
+ const indexOf = chunk.indexOf(searchInput);
+ if (indexOf !== -1) {
+ yield index = indexOf + at + initialIndex;
+ return index;
+ }
+ innerAt += chunk.length + 1;
+ }
+}

yieldable-parser.js

@@ -13,6 +13,7 @@
  * Multiple authors (IBM Corp.) - initial implementation and documentation
  ***************************************************************************/
 'use strict';
+const { indexOfGenerator } = require('./utils/async-index-of');
 
 /**
  * This method parses a JSON text to produce an object or array.
@@ -64,14 +65,16 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  };
 
  // Process strings specially.
- let normalizeUnicodedString = () => {
+ //not sure how i'll make it async
+ let normalizeUnicodedString = function*() {
  let inQuotes = ' ';
  let tempIndex = at;
  let index = 0;
  let slash = 0;
  let c = '"';
  while (c) {
- index = parseStr.indexOf('"', tempIndex + 1);
+ //index = parseStr.indexOf('"', tempIndex + 1);
+ index = yield * indexOfGenerator(parseStr, '"', tempIndex + 1);
  tempIndex = index;
  ch = parseStr.charAt(tempIndex - 1);
  while (ch === '\\') {
@@ -88,7 +91,8 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  }
 
  // When parsing string values, look for " and \ characters.
- index = inQuotes.indexOf('\\');
+ //index = inQuotes.indexOf('\\');
+ index = yield * indexOfGenerator(inQuotes, '\\');
  while (index >= 0) {
  let escapee = {
  '"': '"',
@@ -124,7 +128,8 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  at = index + 1;
  } else
  break;
- index = inQuotes.indexOf('\\', at);
+ //index = inQuotes.indexOf('\\', at);
+ index = yield * indexOfGenerator(inQuotes, '\\', at);
  }
  at = 0;
  return inQuotes;
@@ -225,7 +230,7 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  return inQuotes;
  } else {
  seek();
- return normalizeUnicodedString();
+ return yield * normalizeUnicodedString();
  }
  case '0':
  case '1':
@@ -302,16 +307,15 @@ let parseWrapper = (text, reviver, intensity, cb) => {
 
  let yielding = '';
  // To hold 'parseYield' genarator function
- function * yieldBridge() {
- yielding = yield* parseYield();
+ async function * yieldBridge() {
+ yielding = yield* await parseYield();
  }
  let rs = yieldBridge();
- let gen = rs.next();
+ rs.next().then((gen) => {
 
- // Main yield control logic.
  let yieldCPU = () => {
- setImmediate(() => {
- gen = rs.next();
+ setImmediate(async () => {
+ gen = await rs.next();
 
  if (gen && gen.done === true) {
  let isEmpty = (value) => {
@@ -339,6 +343,9 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  });
  };
  return yieldCPU();
+ });
+
+ // Main yield control logic.
 };
 
 exports.parseWrapper = parseWrapper;