Add HDF5 Weights test file (#467)

lutzroeder · Nov 10, 2024 · e4f2a85 · e4f2a85
1 parent a420f86
commit e4f2a85
Show file tree

Hide file tree

Showing 2 changed files with 81 additions and 6 deletions.
diff --git a/source/hdf5.js b/source/hdf5.js
@@ -4,6 +4,7 @@
 import * as zip from './zip.js';
 
 const hdf5 = {};
+const lzf = {};
 
 hdf5.File = class {
 
@@ -239,13 +240,13 @@ hdf5.Variable = class {
                 }
                 const data = new Uint8Array(data_size * item_size);
                 for (const node of tree.nodes) {
-                    if (node.filterMask !== 0) {
-                        return null;
-                    }
                     let chunk = node.data;
                     if (this._filterPipeline) {
-                        for (const filter of this._filterPipeline.filters) {
-                            chunk = filter.decode(chunk);
+                        for (let i = 0; i < this._filterPipeline.filters.length; i++) {
+                            if ((node.filterMask & (1 << i)) === 0) {
+                                const filter = this._filterPipeline.filters[i];
+                                chunk = filter.decode(chunk);
+                            }
                         }
                     }
                     const chunk_offset = node.fields.map((x) => x.toNumber());
@@ -1368,8 +1369,12 @@ hdf5.Filter = class {
                 const archive = zip.Archive.open(data);
                 return archive.entries.get('').peek();
             }
+            case 32000: { // lzf
+                const stream = new lzf.Stream(data);
+                return stream.read();
+            }
             default: {
-                throw new hdf5.Error(`Unsupported filter '${this.name}'.`);
+                throw new hdf5.Error(`Unsupported filter '${this.id}:${this.name}'.`);
             }
         }
     }
@@ -1656,6 +1661,69 @@ hdf5.GlobalHeapID = class {
     }
 };
 
+lzf.Stream = class {
+
+    constructor(buffer) {
+        this.buffer = buffer;
+    }
+
+    read() {
+        const input = this.buffer;
+        let i = 0;
+        let o = 0;
+        while (i < input.length) {
+            let c = input[i++];
+            if (c < (1 << 5)) {
+                c++;
+                i += c;
+                o += c;
+                if (i > input.length) {
+                    throw new Error('Invalid LZF compressed data.');
+                }
+            } else {
+                let length = c >> 5;
+                if (i >= input.length) {
+                    throw new Error('Invalid LZF compressed data.');
+                }
+                if (length === 7) {
+                    length += input[i++];
+                    if (i >= input.length) {
+                        throw new Error('Invalid LZF compressed data.');
+                    }
+                }
+                const ref =  (o - ((c & 0x1f) << 8) - 1) - input[i++];
+                if (ref < 0) {
+                    throw new Error('Invalid LZF compressed data.');
+                }
+                o += length + 2;
+            }
+        }
+        const output = new Uint8Array(o);
+        i = 0;
+        o = 0;
+        while (i < input.length) {
+            let c = input[i++];
+            if (c < (1 << 5)) {
+                c++;
+                output.set(input.subarray(i, i + c), o);
+                i += c;
+                o += c;
+            } else {
+                let length = c >> 5;
+                if (length === 7) {
+                    length += input[i++];
+                }
+                length += 2;
+                let ref =  o - ((c & 0x1f) << 8) - 1 - input[i++];
+                do {
+                    output[o++] = output[ref++];
+                } while (--length);
+            }
+        }
+        return output;
+    }
+};
+
 hdf5.Error = class extends Error {
 
     constructor(message) {

diff --git a/test/models.json b/test/models.json
@@ -2420,6 +2420,13 @@
     "format":   "Keras",
     "link":     "https://github.com/lutzroeder/netron/issues/57"
   },
+  {
+    "type":     "keras",
+    "target":   "lzf.h5",
+    "source":   "https://github.com/user-attachments/files/17692525/lzf.hdf5.zip[lzf.hdf5]",
+    "format":   "HDF5 Weights",
+    "link":     "https://github.com/lutzroeder/netron/issues/467"
+  },
   {
     "type":     "keras",
     "target":   "mlp.h5",