workflows: add float-tests (#3)

* workflows: add float-tests

Signed-off-by: William Woodruff <william@trailofbits.com>

* _impl: comments

Signed-off-by: William Woodruff <william@trailofbits.com>

* float-tests: typo

Signed-off-by: William Woodruff <william@trailofbits.com>

* add TODO

Signed-off-by: William Woodruff <william@trailofbits.com>

* fix test, update .gitignore

Signed-off-by: William Woodruff <william@trailofbits.com>

* float-tests: follow redirect

Signed-off-by: William Woodruff <william@trailofbits.com>

---------

Signed-off-by: William Woodruff <william@trailofbits.com>

.github/workflows/float-tests.yml

@@ -0,0 +1,35 @@
+name: Full ES6 floating point tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "src/rfc8785/_impl.py"
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: pyproject.toml
+
+      - name: download es6 test file
+        run: |
+          curl -L -o test/assets/es6testfile100m.txt.gz \
+            https://github.com/cyberphone/json-canonicalization/releases/download/es6testfile/es6testfile100m.txt.gz
+
+      - name: test
+        run: make test INSTALL_EXTRA=test

.gitignore

@@ -6,3 +6,6 @@ __pycache__/
 .idea
 html/
 dist/
+
+# ignore any local copy of the es6 floating tests
+test/assets/es6testfile100m.txt.gz

src/rfc8785/_impl.py

@@ -172,6 +172,7 @@ def dumps(obj: _Value) -> bytes:
     Perform JCS serialization of `obj`, returning the canonical serialization
     as `bytes`.
     """
+    # TODO: Optimize this?
     sink = BytesIO()
     dump(obj, sink)
     return sink.getvalue()
@@ -199,6 +200,7 @@ def dump(obj: _Value, sink: IO[bytes]) -> None:
             sink.write(b"null")
         case list() | tuple():
             if not obj:
+                # Optimization for empty lists.
                 sink.write(b"[]")
                 return
 
@@ -210,6 +212,7 @@ def dump(obj: _Value, sink: IO[bytes]) -> None:
             sink.write(b"]")
         case dict():
             if not obj:
+                # Optimization for empty dicts.
                 sink.write(b"{}")
                 return
 

test/test_impl.py

@@ -60,12 +60,16 @@ def test_es6_float_stringification_full(es6_test_file):
     if not es6_test_file.is_file():
         pytest.skip(f"no {es6_test_file}, skipping")
 
+    # TODO: Thread or otherwise chunk this; it's ridiculously slow for
+    # 100M testcases.
     with gzip.open(es6_test_file, mode="rt") as io:
         for line in io:
             line = line.rstrip()
             hex_ieee, expected = line.split(",", 1)
-            bytes_ieee = bytearray.fromhex(hex_ieee)
-            (float_ieee,) = struct.unpack(">d", bytes_ieee)
+            # `hex_ieee` is not consistently padded, so we have to do
+            # things the annoying way: convert it into an int, pack the int
+            # as u64be, and then unpack into a float64be.
+            (float_ieee,) = struct.unpack(">d", struct.pack(">Q", int(hex_ieee, 16)))
 
             sink = BytesIO()
             impl._serialize_float(float_ieee, sink)