Merge pull request #164 from bettyblocks/fix/replace-special-quotes-i…

…n-chunk-DT-4221

feat: replace special quotes in chunk 1.1

__tests__/chunk/1.1/index.test.js

@@ -0,0 +1,30 @@
+import chunk from '../../../functions/chunk/1.1/index';
+
+describe('chunk', () => {
+  it('should split the input into chunks of specified size', async () => {
+    const input =
+      'Homeworld is a real-time strategy video game developed by Relic Entertainment and published by Sierra Studios on September 28, 1999, for Microsoft Windows. Set in space, the science fiction game follows the Kushan exiles of the planet Kharak after their home planet is destroyed by the Taiidan Empire in retaliation for developing hyperspace jump technology. ';
+    const chunkSize = 1;
+    const expectedChunks = [
+      'Homeworld is a real-time strategy video game developed by Relic Entertainment and published by Sierra Studios on September 28, 1999, for Microsoft Windows.',
+      'Set in space, the science fiction game follows the Kushan exiles of the planet Kharak after their home planet is destroyed by the Taiidan Empire in retaliation for developing hyperspace jump technology.',
+    ];
+
+    const result = await chunk({ input, chunkSize });
+
+    expect(result.result).toEqual(expectedChunks);
+  });
+  it('should still split the input into chunks of specified size why there are special quotes', async () => {
+    const input =
+      '“Homeworld" is a real-time strategy video game developed by Relic Entertainment and published by Sierra Studios on September 28, 1999, for Microsoft Windows. Set in space, the science fiction game follows the Kushan exiles of the planet Kharak after their home planet is destroyed by the Taiidan Empire in retaliation for developing hyperspace jump technology. ';
+    const chunkSize = 1;
+    const expectedChunks = [
+      '"Homeworld" is a real-time strategy video game developed by Relic Entertainment and published by Sierra Studios on September 28, 1999, for Microsoft Windows.',
+      'Set in space, the science fiction game follows the Kushan exiles of the planet Kharak after their home planet is destroyed by the Taiidan Empire in retaliation for developing hyperspace jump technology.',
+    ];
+
+    const result = await chunk({ input, chunkSize });
+
+    expect(result.result).toEqual(expectedChunks);
+  });
+});

blocks/ai-search.json

@@ -1,5 +1,5 @@
 {
   "dependencies": ["sentence-splitter"],
-  "functions": ["collectionSearch 1.0", "aiQueryGenerator 1.0", "chunk 1.0"],
+  "functions": ["collectionSearch 1.0", "aiQueryGenerator 1.0", "chunk 1.1"],
   "includes": []
 }

functions/chunk/1.1/function.json

@@ -0,0 +1,46 @@
+{
+  "description": "Chop a text into smaller pieces.",
+  "label": "Chunk",
+  "category": "AI Search",
+  "icon": {
+    "name": "AlignSpaceBetweenIcon",
+    "color": "Pink"
+  },
+  "options": [
+    {
+      "meta": {
+        "type": "Text",
+        "validations": {
+          "required": true
+        }
+      },
+      "name": "input",
+      "label": "Input Text",
+      "info": "The text to chunk"
+    },
+    {
+      "meta": {
+        "type": "Number",
+        "default": 5,
+        "validations": {
+          "required": true
+        }
+      },
+      "name": "chunkSize",
+      "label": "Sentences in a chunk",
+      "info": "The amount of sentences in a chunk"
+    },
+    {
+      "name": "result",
+      "label": "Result",
+      "meta": {
+        "type": "Output",
+        "output": {
+          "type": "Array",
+          "dataType": "STRING"
+        }
+      }
+    }
+  ],
+  "yields": "NONE"
+}

functions/chunk/1.1/index.js

@@ -0,0 +1,24 @@
+import { split } from 'sentence-splitter';
+
+const chunk = async ({ input, chunkSize }) => {
+  const updatedInput = input
+    .replace(/(“|”)/g, '"')
+    .replace(/(‘|’)/g, "'")
+    .replace(/•/g, '.');
+
+  const sentences = split(updatedInput)
+    .filter((c) => c.raw.trim() !== '')
+    .map((c) => c.raw);
+
+  const chunks = [];
+  for (let i = 0; i < sentences.length; i += chunkSize) {
+    chunks.push(sentences.slice(i, i + chunkSize));
+  }
+  const result = chunks.map((c) => c.join(' '));
+
+  return {
+    result,
+  };
+};
+
+export default chunk;