Move tensor to array functions to utils (#7810)

* Move tensor to array functions to utils

* fix lint errors

tfjs-layers/src/layers/nlp/tokenizers.ts

@@ -24,7 +24,8 @@ import { Tensor, serialization, tensor, tidy} from '@tensorflow/tfjs-core';
 
 import { Layer, LayerArgs } from '../../engine/topology';
 import { NotImplementedError, ValueError } from '../../errors';
-import { BytePairTokenizerCache, StaticHashTable, bytesToUnicode, createStaticHashtable, removeStringsFromInputs, splitStringsForBpe, tensorArrTo2DArr, tensorToArr } from './tokenizers_utils';
+import { BytePairTokenizerCache, StaticHashTable, bytesToUnicode, createStaticHashtable, removeStringsFromInputs, splitStringsForBpe } from './tokenizers_utils';
+import { tensorToArr, tensorArrTo2DArr } from './utils';
 
 export declare interface TokenizerOptions {
   mode?: 'tokenize' | 'detokenize';

tfjs-layers/src/layers/nlp/tokenizers_utils.ts

@@ -20,6 +20,7 @@
 import { Tensor, tensor } from '@tensorflow/tfjs-core';
 import { ValueError } from '../../errors';
 import { matchAll } from './match_all_polyfill';
+import { tensorArrTo2DArr, tensorToArr } from './utils';
 
 export function bytesToUnicode(): [Uint8Array, string[]] {
   const inclusiveRange = (start: number, end: number) =>
@@ -249,14 +250,6 @@ export function regexSplit(
   });
 }
 
-export function tensorToArr(input: Tensor): unknown[] {
-  return input.dataSync() as unknown as unknown[];
-}
-
-export function tensorArrTo2DArr(inputs: Tensor[]): unknown[][] {
-  return inputs.map(input => tensorToArr(input));
-}
-
 export function splitStringsForBpe(
   inputs: Tensor, unsplittableTokens?: string[]): Tensor[] {
 

tfjs-layers/src/layers/nlp/tokenizers_utils_test.ts

@@ -19,8 +19,7 @@ import { tensor, test_util } from '@tensorflow/tfjs-core';
 
 import { BytePairTokenizerCache, SPLIT_PATTERN_1, bytesToUnicode,
   createAltsForUnsplittableTokens, createStaticHashtable, regexSplit,
-  removeStringsFromInputs, splitStringsForBpe, tensorArrTo2DArr,
-  tensorToArr } from './tokenizers_utils';
+  removeStringsFromInputs, splitStringsForBpe } from './tokenizers_utils';
 import { expectTensorsClose } from '../../utils/test_utils';
 
 describe('bytesToUnicode', () => {
@@ -231,24 +230,3 @@ describe('splitStringsForBpe', () => {
     expectTensorsClose(result[1], tensor(['black', '.']));
   });
 });
-
-describe('tensor to array functions', () => {
-  it('tensorToArr', () => {
-    const inputStr = tensor(['these', 'are', 'strings', '.']);
-    const inputNum = tensor([2, 11, 15]);
-
-    test_util.expectArraysEqual(
-      tensorToArr(inputStr) as string[], ['these', 'are', 'strings', '.']);
-    test_util.expectArraysEqual(tensorToArr(inputNum) as number[], [2, 11, 15]);
-  });
-
-  it('tensorArrTo2DArr', () => {
-    const inputStr = [tensor(['these', 'are']), tensor(['strings', '.'])];
-    const inputNum = [tensor([2, 11]), tensor([15])];
-
-    test_util.expectArraysEqual(
-      tensorArrTo2DArr(inputStr) as string[][], [['these', 'are'], ['strings', '.']]);
-    test_util.expectArraysEqual(
-      tensorArrTo2DArr(inputNum) as number[][], [[2, 11], [15]]);
-  });
-});

tfjs-layers/src/layers/nlp/utils.ts

@@ -0,0 +1,26 @@
+/**
+ * @license
+ * Copyright 2023 Google LLC.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import { Tensor } from '@tensorflow/tfjs-core';
+
+export function tensorToArr(input: Tensor): unknown[] {
+  return Array.from(input.dataSync()) as unknown as unknown[];
+}
+
+export function tensorArrTo2DArr(inputs: Tensor[]): unknown[][] {
+  return inputs.map(input => tensorToArr(input));
+}

tfjs-layers/src/layers/nlp/utils_test.ts

@@ -0,0 +1,42 @@
+/**
+ * @license
+ * Copyright 2023 Google LLC.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import { tensor, test_util } from '@tensorflow/tfjs-core';
+import { tensorArrTo2DArr, tensorToArr } from './utils';
+
+describe('tensor to array functions', () => {
+  it('tensorToArr', () => {
+    const inputStr = tensor(['these', 'are', 'strings', '.']);
+    const inputNum = tensor([2, 11, 15]);
+
+    test_util.expectArraysEqual(
+      tensorToArr(inputStr) as string[], ['these', 'are', 'strings', '.']);
+    test_util.expectArraysEqual(tensorToArr(inputNum) as number[], [2, 11, 15]);
+  });
+
+  it('tensorArrTo2DArr', () => {
+    const inputStr = [tensor(['these', 'are']), tensor(['strings', '.'])];
+    const inputNum = [tensor([2, 11]), tensor([15])];
+
+    test_util.expectArraysEqual(
+      tensorArrTo2DArr(inputStr) as string[][],
+      [['these', 'are'], ['strings', '.']]
+    );
+    test_util.expectArraysEqual(
+      tensorArrTo2DArr(inputNum) as number[][], [[2, 11], [15]]);
+  });
+});