Add NumPy test files (#711)

source/base.js

@@ -585,7 +585,7 @@ base.Tensor = class {
  this.type = tensor.type;
  this.layout = tensor.type.layout;
  this.stride = tensor.stride;
- base.Tensor.dataTypes = base.Tensor.dataTypeSizes || new Map([
+ base.Tensor._dataTypes = base.Tensor._dataTypes || new Map([
  ['boolean', 1],
  ['qint8', 1], ['qint16', 2], ['qint32', 4],
  ['quint8', 1], ['quint16', 2], ['quint32', 4],
@@ -732,8 +732,8 @@ base.Tensor = class {
  case '>': {
  context.data = (this.data instanceof Uint8Array || this.data instanceof Int8Array) ? this.data : this.data.peek();
  context.view = new DataView(context.data.buffer, context.data.byteOffset, context.data.byteLength);
- if (base.Tensor.dataTypes.has(dataType)) {
- const itemsize = base.Tensor.dataTypes.get(dataType);
+ if (base.Tensor._dataTypes.has(dataType)) {
+ const itemsize = base.Tensor._dataTypes.get(dataType);
  const length = context.data.length;
  const stride = context.stride;
  if (length < (itemsize * shape.reduce((a, v) => a * v, 1))) {
@@ -759,7 +759,7 @@ base.Tensor = class {
  }
  case '|': {
  context.data = this.values;
- if (!base.Tensor.dataTypes.has(dataType) && dataType !== 'string' && dataType !== 'object') {
+ if (!base.Tensor._dataTypes.has(dataType) && dataType !== 'string' && dataType !== 'object' && dataType !== 'void') {
  throw new Error(`Tensor data type '${dataType}' is not implemented.`);
  }
  const size = context.dimensions.reduce((a, v) => a * v, 1);
@@ -1027,6 +1027,14 @@ base.Tensor = class {
  case 'bigint':
  return indentation + value.toString();
  default:
+ if (value instanceof Uint8Array) {
+ let content = '';
+ for (let i = 0; i < value.length; i++) {
+ const x = value[i];
+ content += x >= 32 && x <= 126 ? String.fromCharCode(x) : `\\x${x.toString(16).padStart(2, '0')}`;
+ }
+ return `${indentation}"${content}"`;
+ }
  if (value && value.toString) {
  return indentation + value.toString();
  }

source/hailo.js

@@ -197,7 +197,7 @@ hailo.Tensor = class {
  if (array) {
  this.stride = array.strides.map((stride) => stride / array.itemsize);
  this.layout = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder;
- this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.tolist() : array.tobytes();
+ this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.tolist() : array.tobytes();
  }
  }
 };

source/numpy.js

@@ -44,11 +44,17 @@ numpy.ModelFactory = class {
  switch (context.type) {
  case 'npy': {
  format = 'NumPy Array';
+ const unresolved = new Set();
  const execution = new python.Execution();
+ execution.on('resolve', (_, name) => unresolved.add(name));
  const stream = context.stream;
  const buffer = stream.peek();
  const bytes = execution.invoke('io.BytesIO', [buffer]);
  const array = execution.invoke('numpy.load', [bytes]);
+ if (unresolved.size > 0) {
+ const name = unresolved.values().next().value;
+ throw new numpy.Error(`Unknown type name '${name}'.`);
+ }
  const layer = { type: 'numpy.ndarray', parameters: [{ name: 'value', tensor: { name: '', array } }] };
  graphs.push({ layers: [layer] });
  break;
@@ -209,7 +215,7 @@ numpy.Tensor = class {
  constructor(array) {
  this.type = new numpy.TensorType(array.dtype.__name__, new numpy.TensorShape(array.shape));
  this.stride = array.strides.map((stride) => stride / array.itemsize);
- this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.flatten().tolist() : array.tobytes();
+ this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.flatten().tolist() : array.tobytes();
  this.encoding = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder;
  }
 };

source/pickle.js

@@ -191,7 +191,7 @@ pickle.Tensor = class {
  this.type = new pickle.TensorType(array.dtype.__name__, new pickle.TensorShape(array.shape));
  this.stride = array.strides.map((stride) => stride / array.itemsize);
  this.encoding = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder;
- this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.flatten().tolist() : array.tobytes();
+ this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.flatten().tolist() : array.tobytes();
  }
 };
 

source/python.js

@@ -1914,7 +1914,7 @@ python.Execution = class {
  });
  this.registerType('numpy.dtype', class {
  constructor(obj, align, copy) {
- if (typeof obj === 'string' && (obj.startsWith('<') || obj.startsWith('>'))) {
+ if (typeof obj === 'string' && (obj.startsWith('<') || obj.startsWith('>') || obj.startsWith('|'))) {
  this.byteorder = obj.substring(0, 1);
  obj = obj.substring(1);
  } else {
@@ -1937,6 +1937,7 @@ python.Execution = class {
  case 'c8': case 'complex64': this.itemsize = 8; this.kind = 'c'; break;
  case 'c16': case 'complex128': case 'complex': this.itemsize = 16; this.kind = 'c'; break;
  case 'M8': case 'M': this.itemsize = 8; this.kind = 'M'; break;
+ case 'V': case 'void': this.itemsize = 0; this.kind = 'V'; break;
  default:
  if (obj.startsWith('V')) {
  this.itemsize = parseInt(obj.substring(1), 10);
@@ -1950,6 +1951,9 @@ python.Execution = class {
  } else if (obj.startsWith('U')) { // Unicode string
  this.kind = 'U';
  this.itemsize = 4 * parseInt(obj.substring(1), 10);
+ } else if (obj.startsWith('T')) {
+ this.kind = 'T';
+ this.itemsize = parseInt(obj.substring(1), 10);
  } else {
  throw new python.Error(`Unsupported dtype '${obj}'.`);
  }
@@ -1970,6 +1974,7 @@ python.Execution = class {
  case 'V': return `void${this.itemsize === 0 ? '' : (this.itemsize * 8)}`;
  case 'S': return `bytes${this.itemsize === 0 ? '' : (this.itemsize * 8)}`;
  case 'U': return `str${this.itemsize === 0 ? '' : (this.itemsize * 8)}`;
+ case 'T': return `StringDType${this.itemsize === 0 ? '' : (this.itemsize * 8)}`;
  case 'M': return 'datetime64';
  case 'b': return 'bool';
  default: return this.__name__;
@@ -2032,6 +2037,8 @@ python.Execution = class {
  default: throw new python.Error(`Unsupported complex itemsize '${this.itemsize}'.`);
  }
  case 'S':
+ case 'T':
+ return 'string';
  case 'U':
  return 'string';
  case 'M':
@@ -2065,6 +2072,11 @@ python.Execution = class {
  this.registerType('numpy.uint32', class extends numpy.unsignedinteger {});
  this.registerType('numpy.uint64', class extends numpy.unsignedinteger {});
  this.registerType('numpy.datetime64', class extends numpy.generic {});
+ this.registerType('numpy.dtypes.StringDType', class extends numpy.dtype {
+ constructor() {
+ super('|T16');
+ }
+ });
  this.registerType('gensim.models.doc2vec.Doctag', class {});
  this.registerType('gensim.models.doc2vec.Doc2Vec', class {});
  this.registerType('gensim.models.doc2vec.Doc2VecTrainables', class {});
@@ -2438,6 +2450,19 @@ python.Execution = class {
  }
  return list;
  }
+ case 'V': {
+ const data = this.data;
+ const itemsize = this.dtype.itemsize;
+ let offset = 0;
+ for (let i = 0; i < size; i++) {
+ list[i] = data.slice(offset, offset + itemsize);
+ offset += itemsize;
+ }
+ return list;
+ }
+ case 'T': {
+ return this.data;
+ }
  case 'O': {
  return this.data;
  }
@@ -3821,6 +3846,12 @@ python.Execution = class {
  this.registerFunction('numpy.core.multiarray._reconstruct', (subtype, shape, dtype) => {
  return numpy.ndarray.__new__(subtype, shape, dtype);
  });
+ this.registerFunction('numpy._core.multiarray._reconstruct', (subtype, shape, dtype) => {
+ return numpy.ndarray.__new__(subtype, shape, dtype);
+ });
+ this.registerFunction('numpy._core._internal._convert_to_stringdtype_kwargs', () => {
+ return new numpy.dtypes.StringDType();
+ });
  numpy.core._multiarray_umath._reconstruct = numpy.core.multiarray._reconstruct;
  this.registerFunction('numpy.core.multiarray.scalar', (dtype, rawData) => {
  let data = rawData;
@@ -3911,7 +3942,6 @@ python.Execution = class {
  throw new python.Error(`Unsupported scalar type '${dtype.__name__}'.`);
  }
  });
- numpy._core = numpy.core;
  this.registerFunction('numpy.load', (file) => {
  // https://github.com/numpy/numpy/blob/main/numpy/lib/format.py
  const signature = [0x93, 0x4E, 0x55, 0x4D, 0x50, 0x59];

source/sklearn.js

@@ -309,7 +309,7 @@ sklearn.Tensor = class {
  this.type = new sklearn.TensorType(array.dtype.__name__, new sklearn.TensorShape(array.shape));
  this.stride = array.strides.map((stride) => stride / array.itemsize);
  this.encoding = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder;
- this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.flatten().tolist() : array.tobytes();
+ this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.flatten().tolist() : array.tobytes();
  }
 };
 

test/models.json

@@ -3631,13 +3631,35 @@
  "format": "NumPy Array",
  "link": "https://github.com/lutzroeder/netron/issues/711"
  },
+ {
+ "type": "numpy",
+ "target": "StringDType.npy",
+ "source": "https://github.com/user-attachments/files/16061374/StringDType.npy.zip[StringDType.npy]",
+ "format": "NumPy Array",
+ "link": "https://github.com/lutzroeder/netron/issues/711"
+ },
+ {
+ "type": "numpy",
+ "target": "struct.npy",
+ "source": "https://github.com/user-attachments/files/16075809/struct.npy.zip[struct.npy]",
+ "format": "NumPy Array",
+ "error": "Unexpected token '(', ...\" \"<U10\"], (\"age\", \"<\"... is not valid JSON",
+ "link": "https://github.com/lutzroeder/netron/issues/711"
+ },
  {
  "type": "numpy",
  "target": "tensor.npy",
  "source": "https://github.com/lutzroeder/netron/files/6225732/tensor.npy.zip[tensor.npy]",
  "format": "NumPy Array",
  "link": "https://github.com/lutzroeder/netron/issues/711"
  },
+ {
+ "type": "numpy",
+ "target": "void.npy",
+ "source": "https://github.com/user-attachments/files/16075808/void.npy.zip[void.npy]",
+ "format": "NumPy Array",
+ "link": "https://github.com/lutzroeder/netron/issues/711"
+ },
  {
  "type": "onednn",
  "target": "f8_matmul.json",