diff --git a/source/base.js b/source/base.js index a703248840..377a4020b3 100644 --- a/source/base.js +++ b/source/base.js @@ -585,7 +585,7 @@ base.Tensor = class { this.type = tensor.type; this.layout = tensor.type.layout; this.stride = tensor.stride; - base.Tensor.dataTypes = base.Tensor.dataTypeSizes || new Map([ + base.Tensor._dataTypes = base.Tensor._dataTypes || new Map([ ['boolean', 1], ['qint8', 1], ['qint16', 2], ['qint32', 4], ['quint8', 1], ['quint16', 2], ['quint32', 4], @@ -732,8 +732,8 @@ base.Tensor = class { case '>': { context.data = (this.data instanceof Uint8Array || this.data instanceof Int8Array) ? this.data : this.data.peek(); context.view = new DataView(context.data.buffer, context.data.byteOffset, context.data.byteLength); - if (base.Tensor.dataTypes.has(dataType)) { - const itemsize = base.Tensor.dataTypes.get(dataType); + if (base.Tensor._dataTypes.has(dataType)) { + const itemsize = base.Tensor._dataTypes.get(dataType); const length = context.data.length; const stride = context.stride; if (length < (itemsize * shape.reduce((a, v) => a * v, 1))) { @@ -759,7 +759,7 @@ base.Tensor = class { } case '|': { context.data = this.values; - if (!base.Tensor.dataTypes.has(dataType) && dataType !== 'string' && dataType !== 'object') { + if (!base.Tensor._dataTypes.has(dataType) && dataType !== 'string' && dataType !== 'object' && dataType !== 'void') { throw new Error(`Tensor data type '${dataType}' is not implemented.`); } const size = context.dimensions.reduce((a, v) => a * v, 1); @@ -1027,6 +1027,14 @@ base.Tensor = class { case 'bigint': return indentation + value.toString(); default: + if (value instanceof Uint8Array) { + let content = ''; + for (let i = 0; i < value.length; i++) { + const x = value[i]; + content += x >= 32 && x <= 126 ? String.fromCharCode(x) : `\\x${x.toString(16).padStart(2, '0')}`; + } + return `${indentation}"${content}"`; + } if (value && value.toString) { return indentation + value.toString(); } diff --git a/source/hailo.js b/source/hailo.js index 034b13346f..dda043499e 100644 --- a/source/hailo.js +++ b/source/hailo.js @@ -197,7 +197,7 @@ hailo.Tensor = class { if (array) { this.stride = array.strides.map((stride) => stride / array.itemsize); this.layout = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder; - this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.tolist() : array.tobytes(); + this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.tolist() : array.tobytes(); } } }; diff --git a/source/numpy.js b/source/numpy.js index 563287ad8f..c0e2405a6c 100644 --- a/source/numpy.js +++ b/source/numpy.js @@ -44,11 +44,17 @@ numpy.ModelFactory = class { switch (context.type) { case 'npy': { format = 'NumPy Array'; + const unresolved = new Set(); const execution = new python.Execution(); + execution.on('resolve', (_, name) => unresolved.add(name)); const stream = context.stream; const buffer = stream.peek(); const bytes = execution.invoke('io.BytesIO', [buffer]); const array = execution.invoke('numpy.load', [bytes]); + if (unresolved.size > 0) { + const name = unresolved.values().next().value; + throw new numpy.Error(`Unknown type name '${name}'.`); + } const layer = { type: 'numpy.ndarray', parameters: [{ name: 'value', tensor: { name: '', array } }] }; graphs.push({ layers: [layer] }); break; @@ -209,7 +215,7 @@ numpy.Tensor = class { constructor(array) { this.type = new numpy.TensorType(array.dtype.__name__, new numpy.TensorShape(array.shape)); this.stride = array.strides.map((stride) => stride / array.itemsize); - this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.flatten().tolist() : array.tobytes(); + this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.flatten().tolist() : array.tobytes(); this.encoding = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder; } }; diff --git a/source/pickle.js b/source/pickle.js index 905b07a93e..425af107d2 100644 --- a/source/pickle.js +++ b/source/pickle.js @@ -191,7 +191,7 @@ pickle.Tensor = class { this.type = new pickle.TensorType(array.dtype.__name__, new pickle.TensorShape(array.shape)); this.stride = array.strides.map((stride) => stride / array.itemsize); this.encoding = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder; - this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.flatten().tolist() : array.tobytes(); + this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.flatten().tolist() : array.tobytes(); } }; diff --git a/source/python.js b/source/python.js index 268ecdd529..76c1cfeac3 100644 --- a/source/python.js +++ b/source/python.js @@ -1914,7 +1914,7 @@ python.Execution = class { }); this.registerType('numpy.dtype', class { constructor(obj, align, copy) { - if (typeof obj === 'string' && (obj.startsWith('<') || obj.startsWith('>'))) { + if (typeof obj === 'string' && (obj.startsWith('<') || obj.startsWith('>') || obj.startsWith('|'))) { this.byteorder = obj.substring(0, 1); obj = obj.substring(1); } else { @@ -1937,6 +1937,7 @@ python.Execution = class { case 'c8': case 'complex64': this.itemsize = 8; this.kind = 'c'; break; case 'c16': case 'complex128': case 'complex': this.itemsize = 16; this.kind = 'c'; break; case 'M8': case 'M': this.itemsize = 8; this.kind = 'M'; break; + case 'V': case 'void': this.itemsize = 0; this.kind = 'V'; break; default: if (obj.startsWith('V')) { this.itemsize = parseInt(obj.substring(1), 10); @@ -1950,6 +1951,9 @@ python.Execution = class { } else if (obj.startsWith('U')) { // Unicode string this.kind = 'U'; this.itemsize = 4 * parseInt(obj.substring(1), 10); + } else if (obj.startsWith('T')) { + this.kind = 'T'; + this.itemsize = parseInt(obj.substring(1), 10); } else { throw new python.Error(`Unsupported dtype '${obj}'.`); } @@ -1970,6 +1974,7 @@ python.Execution = class { case 'V': return `void${this.itemsize === 0 ? '' : (this.itemsize * 8)}`; case 'S': return `bytes${this.itemsize === 0 ? '' : (this.itemsize * 8)}`; case 'U': return `str${this.itemsize === 0 ? '' : (this.itemsize * 8)}`; + case 'T': return `StringDType${this.itemsize === 0 ? '' : (this.itemsize * 8)}`; case 'M': return 'datetime64'; case 'b': return 'bool'; default: return this.__name__; @@ -2032,6 +2037,8 @@ python.Execution = class { default: throw new python.Error(`Unsupported complex itemsize '${this.itemsize}'.`); } case 'S': + case 'T': + return 'string'; case 'U': return 'string'; case 'M': @@ -2065,6 +2072,11 @@ python.Execution = class { this.registerType('numpy.uint32', class extends numpy.unsignedinteger {}); this.registerType('numpy.uint64', class extends numpy.unsignedinteger {}); this.registerType('numpy.datetime64', class extends numpy.generic {}); + this.registerType('numpy.dtypes.StringDType', class extends numpy.dtype { + constructor() { + super('|T16'); + } + }); this.registerType('gensim.models.doc2vec.Doctag', class {}); this.registerType('gensim.models.doc2vec.Doc2Vec', class {}); this.registerType('gensim.models.doc2vec.Doc2VecTrainables', class {}); @@ -2438,6 +2450,19 @@ python.Execution = class { } return list; } + case 'V': { + const data = this.data; + const itemsize = this.dtype.itemsize; + let offset = 0; + for (let i = 0; i < size; i++) { + list[i] = data.slice(offset, offset + itemsize); + offset += itemsize; + } + return list; + } + case 'T': { + return this.data; + } case 'O': { return this.data; } @@ -3821,6 +3846,12 @@ python.Execution = class { this.registerFunction('numpy.core.multiarray._reconstruct', (subtype, shape, dtype) => { return numpy.ndarray.__new__(subtype, shape, dtype); }); + this.registerFunction('numpy._core.multiarray._reconstruct', (subtype, shape, dtype) => { + return numpy.ndarray.__new__(subtype, shape, dtype); + }); + this.registerFunction('numpy._core._internal._convert_to_stringdtype_kwargs', () => { + return new numpy.dtypes.StringDType(); + }); numpy.core._multiarray_umath._reconstruct = numpy.core.multiarray._reconstruct; this.registerFunction('numpy.core.multiarray.scalar', (dtype, rawData) => { let data = rawData; @@ -3911,7 +3942,6 @@ python.Execution = class { throw new python.Error(`Unsupported scalar type '${dtype.__name__}'.`); } }); - numpy._core = numpy.core; this.registerFunction('numpy.load', (file) => { // https://github.com/numpy/numpy/blob/main/numpy/lib/format.py const signature = [0x93, 0x4E, 0x55, 0x4D, 0x50, 0x59]; diff --git a/source/sklearn.js b/source/sklearn.js index c062c4e497..947975b351 100644 --- a/source/sklearn.js +++ b/source/sklearn.js @@ -309,7 +309,7 @@ sklearn.Tensor = class { this.type = new sklearn.TensorType(array.dtype.__name__, new sklearn.TensorShape(array.shape)); this.stride = array.strides.map((stride) => stride / array.itemsize); this.encoding = this.type.dataType === 'string' || this.type.dataType === 'object' ? '|' : array.dtype.byteorder; - this.values = this.type.dataType === 'string' || this.type.dataType === 'object' ? array.flatten().tolist() : array.tobytes(); + this.values = this.type.dataType === 'string' || this.type.dataType === 'object' || this.type.dataType === 'void' ? array.flatten().tolist() : array.tobytes(); } }; diff --git a/test/models.json b/test/models.json index 28eef7fbe1..a84a4406f7 100644 --- a/test/models.json +++ b/test/models.json @@ -3631,6 +3631,21 @@ "format": "NumPy Array", "link": "https://github.com/lutzroeder/netron/issues/711" }, + { + "type": "numpy", + "target": "StringDType.npy", + "source": "https://github.com/user-attachments/files/16061374/StringDType.npy.zip[StringDType.npy]", + "format": "NumPy Array", + "link": "https://github.com/lutzroeder/netron/issues/711" + }, + { + "type": "numpy", + "target": "struct.npy", + "source": "https://github.com/user-attachments/files/16075809/struct.npy.zip[struct.npy]", + "format": "NumPy Array", + "error": "Unexpected token '(', ...\" \"