fix(yieldable-parser): parseAsync sometimes blocks for a *long* time #36

package.json

@@ -39,7 +39,8 @@
  "index.js",
  "yieldable-stringify.js",
  "yieldable-parser.js",
- "test/"
+ "test/",
+ "utils/"
  ],
  "devDependencies": {
  "eslint": "^6.6.0",

test/test-index-of-generator.js

@@ -0,0 +1,54 @@
+const tap = require('tap');
+const { indexOfGenerator } = require('../utils/index-of-generator');
+
+function promisifyGenerator(generator){
+ return (...args) => {
+ const gen = generator(...args);
+
+ function handleNext(resolvedValue){
+ if(resolvedValue.done){
+ return Promise.resolve(resolvedValue.value);
+ }
+ return Promise.resolve(gen.next(resolvedValue.value)).then(handleNext);
+ }
+
+
+ return Promise.resolve(gen.next())
+ .then((resolvedValue) => handleNext(resolvedValue), (error) => Promise.reject(gen.throw(error)));
+ }
+}
+
+const asyncIndexOf = promisifyGenerator(indexOfGenerator);
+
+const stringAsArray = ['s', 'o', 'm', 'e'];
+const text = stringAsArray.join('');
+
+tap.test('should return -1 if not found due to initialIndex', async () => {
+ const index = await asyncIndexOf(text, 's', 1);
+ const syncIndexOf = text.indexOf('s', 1);
+ tap.equal(index, syncIndexOf);
+})
+
+tap.test('should return proper index', async () => {
+ const index = await asyncIndexOf(text, 'e');
+ const syncIndexOf = text.indexOf('e');
+ tap.equal(index, syncIndexOf);
+})
+
+tap.test('should return proper test with initialIndex', async () => {
+ const index = await asyncIndexOf(text, 'e', 2);
+ const syncIndexOf = text.indexOf('e', 2);
+ tap.equal(index, syncIndexOf); 
+})
+
+tap.test('should return correct indexOf if initialIndex is same', async () => {
+ const index = await asyncIndexOf(text, 'e', 3);
+ const syncIndexOf = text.indexOf('e', 3);
+ tap.equal(index, syncIndexOf); 
+})
+
+tap.test('should return -1 if not found', async () => {
+ const index = await asyncIndexOf(text, 'x');
+ const syncIndexOf = text.indexOf('x');
+ tap.equal(index, syncIndexOf); 
+})

test/test-parse-big-escaped-string.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const yj = require('../index');
+const tap = require('tap');
+const { readFileSync } = require('fs');
+const str = readFileSync('./test/big-json-with-string.txt');
+const nodeJSNativeParse = JSON.parse(str);
+
+// Make sure the API works well without the optional parameters.
+yj.parseAsync(str, (err, obj) => {
+ if (!err)
+ tap.equal(obj.text, nodeJSNativeParse.text, 'Failed to parse big string message - omitting logs', { diagnostic: false });
+ else
+ tap.fail(err);
+});

utils/index-of-generator.js

@@ -0,0 +1,26 @@
+//This number may require proper tuning - selecting 10K since any string bigger than that may benefit from specific algorithms.
+//Like if more than 50k characters, use BHM Algorithm
+//We are considering a big string to be ~10k characters
+const MAX_STRING_SIZE = 10000;
+
+module.exports.indexOfGenerator = function* indexOfGenerator(stringInput, searchInput, initialIndex = 0) {
+ if (!stringInput || !stringInput.length) {
+ return -1;
+ }
+ stringInput = stringInput.slice(initialIndex);
+ let innerAt = 0;
+ let index = -1;
+ while (true) {
+ const at = Math.max(innerAt - 1, 0);
+ yield chunk = stringInput.slice(at, innerAt + MAX_STRING_SIZE);
+ if (!chunk.length) {
+ return index;
+ }
+ const indexOf = chunk.indexOf(searchInput);
+ if (indexOf !== -1) {
+ yield index = indexOf + at + initialIndex;
+ return index;
+ }
+ innerAt += chunk.length + 1;
+ }
+}

yieldable-parser.js

@@ -13,6 +13,7 @@
  * Multiple authors (IBM Corp.) - initial implementation and documentation
  ***************************************************************************/
 'use strict';
+const { indexOfGenerator } = require('./utils/index-of-generator');
 
 /**
  * This method parses a JSON text to produce an object or array.
@@ -64,14 +65,14 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  };
 
  // Process strings specially.
- let normalizeUnicodedString = () => {
+ let normalizeUnicodedString = function*() {
  let inQuotes = ' ';
  let tempIndex = at;
  let index = 0;
  let slash = 0;
  let c = '"';
  while (c) {
- index = parseStr.indexOf('"', tempIndex + 1);
+ index = yield * indexOfGenerator(parseStr, '"', tempIndex + 1);
  tempIndex = index;
  ch = parseStr.charAt(tempIndex - 1);
  while (ch === '\\') {
@@ -88,7 +89,7 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  }
 
  // When parsing string values, look for " and \ characters.
- index = inQuotes.indexOf('\\');
+ index = yield * indexOfGenerator(inQuotes, '\\');
  while (index >= 0) {
  let escapee = {
  '"': '"',
@@ -124,7 +125,7 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  at = index + 1;
  } else
  break;
- index = inQuotes.indexOf('\\', at);
+ index = yield * indexOfGenerator(inQuotes, '\\', at);
  }
  at = 0;
  return inQuotes;
@@ -225,7 +226,7 @@ let parseWrapper = (text, reviver, intensity, cb) => {
  return inQuotes;
  } else {
  seek();
- return normalizeUnicodedString();
+ return yield * normalizeUnicodedString();
  }
  case '0':
  case '1':