support float_mode='decimal' (#116)

.github/workflows/ci.yml

@@ -165,9 +165,7 @@ jobs:
  env:
  RUST_BACKTRACE: 1
 
- - run: python crates/jiter-python/bench.py
- env:
- FAST: 1
+ - run: python crates/jiter-python/bench.py --fast
 
  - run: coverage-prepare lcov $(python -c 'import jiter.jiter;print(jiter.jiter.__file__)')
 

Makefile

@@ -28,6 +28,14 @@ python-dev:
 python-test: python-dev
  pytest crates/jiter-python/tests
 
+.PHONY: python-dev-release
+python-dev-release:
+ maturin develop -m crates/jiter-python/Cargo.toml --release
+
+.PHONY: python-bench
+python-bench: python-dev-release
+ python crates/jiter-python/bench.py
+
 .PHONY: bench
 bench:
  cargo bench -p jiter -F python

crates/jiter-python/README.md

@@ -18,7 +18,7 @@ def from_json(
  cache_mode: Literal[True, False, "all", "keys", "none"] = "all",
  partial_mode: Literal[True, False, "off", "on", "trailing-strings"] = False,
  catch_duplicate_keys: bool = False,
- lossless_floats: bool = False,
+ float_mode: Literal["float", "decimal", "lossless-float"] = False,
 ) -> Any:
  """
  Parse input bytes into a JSON object.
@@ -36,7 +36,7 @@ def from_json(
  - True / 'on' - allow incomplete JSON but discard the last string if it is incomplete
  - 'trailing-strings' - allow incomplete JSON, and include the last incomplete string in the output
  catch_duplicate_keys: if True, raise an exception if objects contain the same key multiple times
- lossless_floats: if True, preserve full detail on floats using `LosslessFloat`
+ float_mode: How to return floats: as a `float`, `Decimal` or `LosslessFloat`
 
  Returns:
  Python object built from the JSON input.

crates/jiter-python/bench.py

@@ -5,37 +5,28 @@
 
 import json
 
-FAST = bool(os.getenv('FAST'))
-THIS_DIR = Path(__file__).parent
-
-cases = [
- ("medium_response", (THIS_DIR / "../jiter/benches/medium_response.json").read_bytes()),
- (
- "massive_ints_array",
- (THIS_DIR / "../jiter/benches/massive_ints_array.json").read_bytes(),
- ),
- ("array_short_strings", "[{}]".format(", ".join('"123"' for _ in range(100_000)))),
- (
- "object_short_strings",
- "{%s}" % ", ".join(f'"{i}": "{i}x"' for i in range(100_000)),
- ),
- (
- "array_short_arrays",
- "[{}]".format(", ".join('["a", "b", "c", "d"]' for _ in range(10_000))),
- ),
- ("one_long_string", json.dumps("x" * 100)),
- ("one_short_string", b'"foobar"'),
- ("1m_strings", json.dumps([str(i) for i in range(1_000_000)])),
-]
-
-
-def run_bench(func, d):
+CASES = {
+ "array_short_strings": "[{}]".format(", ".join('"123"' for _ in range(100_000))),
+ "object_short_strings": "{%s}" % ", ".join(f'"{i}": "{i}x"' for i in range(100_000)),
+ "array_short_arrays": "[{}]".format(", ".join('["a", "b", "c", "d"]' for _ in range(10_000))),
+ "one_long_string": json.dumps("x" * 100),
+ "one_short_string": b'"foobar"',
+ "1m_strings": json.dumps([str(i) for i in range(1_000_000)]),
+}
+
+BENCHES_DIR = Path(__file__).parent.parent / "jiter/benches/"
+
+for p in BENCHES_DIR.glob('*.json'):
+ CASES[p.stem] = p.read_bytes()
+
+
+def run_bench(func, d, fast: bool):
  if isinstance(d, str):
  d = d.encode()
  timer = timeit.Timer(
  "func(json_data)", setup="", globals={"func": func, "json_data": d}
  )
- if FAST:
+ if fast:
  return timer.timeit(1)
  else:
  n, t = timer.autorange()
@@ -85,20 +76,21 @@ def setup_json():
 
 def main():
  parser = argparse.ArgumentParser()
+ parser.add_argument("--case", default="all", choices=[*CASES.keys(), "all"])
+ parser.add_argument("--fast", action="store_true", default=False)
  parser.add_argument(
  "parsers", nargs="*", default="all", choices=[*PARSERS.keys(), "all"]
  )
  args = parser.parse_args()
 
- if "all" in args.parsers:
-   args.parsers = [*PARSERS.keys()]
+ parsers = [*PARSERS.keys()] if "all" in args.parsers else args.parsers
+ cases = [*CASES.keys()] if args.case == "all" else [args.case]
 
- for name, json_data in cases:
+ for name in cases:
  print(f"Case: {name}")
 
- times = [
- (parser, run_bench(PARSERS[parser](), json_data)) for parser in args.parsers
- ]
+ json_data = CASES[name]
+ times = [(parser, run_bench(PARSERS[parser](), json_data, args.fast)) for parser in parsers]
 
  times.sort(key=lambda x: x[1])
  best = times[0][1]

crates/jiter-python/jiter.pyi

@@ -9,7 +9,7 @@ def from_json(
  cache_mode: Literal[True, False, "all", "keys", "none"] = "all",
  partial_mode: Literal[True, False, "off", "on", "trailing-strings"] = False,
  catch_duplicate_keys: bool = False,
- lossless_floats: bool = False,
+ float_mode: Literal["float", "decimal", "lossless-float"] = False,
 ) -> Any:
  """
  Parse input bytes into a JSON object.
@@ -27,7 +27,7 @@ def from_json(
  - True / 'on' - allow incomplete JSON but discard the last string if it is incomplete
  - 'trailing-strings' - allow incomplete JSON, and include the last incomplete string in the output
  catch_duplicate_keys: if True, raise an exception if objects contain the same key multiple times
- lossless_floats: if True, preserve full detail on floats using `LosslessFloat`
+ float_mode: How to return floats: as a `float`, `Decimal` or `LosslessFloat`
 
  Returns:
  Python object built from the JSON input.

crates/jiter-python/src/lib.rs

@@ -19,7 +19,7 @@ pub fn get_jiter_version() -> &'static str {
 mod jiter_python {
  use pyo3::prelude::*;
 
- use jiter::{map_json_error, LosslessFloat, PartialMode, PythonParse, StringCacheMode};
+ use jiter::{map_json_error, FloatMode, LosslessFloat, PartialMode, PythonParse, StringCacheMode};
 
  use super::get_jiter_version;
 
@@ -33,7 +33,7 @@ mod jiter_python {
  cache_mode=StringCacheMode::All,
  partial_mode=PartialMode::Off,
  catch_duplicate_keys=false,
- lossless_floats=false,
+ float_mode=FloatMode::Float,
  )
  )]
  pub fn from_json<'py>(
@@ -43,14 +43,14 @@ mod jiter_python {
  cache_mode: StringCacheMode,
  partial_mode: PartialMode,
  catch_duplicate_keys: bool,
- lossless_floats: bool,
+ float_mode: FloatMode,
  ) -> PyResult<Bound<'py, PyAny>> {
  let parse_builder = PythonParse {
  allow_inf_nan,
  cache_mode,
  partial_mode,
  catch_duplicate_keys,
- lossless_floats,
+ float_mode,
  };
  parse_builder
  .python_parse(py, json_data)

crates/jiter-python/tests/test_jiter.py

@@ -240,25 +240,58 @@ def test_lossless_floats():
  assert isinstance(f, float)
  assert f == 12.3
 
- f = jiter.from_json(b'12.3', lossless_floats=True)
+ f = jiter.from_json(b'12.3', float_mode='float')
+ assert isinstance(f, float)
+ assert f == 12.3
+
+ f = jiter.from_json(b'12.3', float_mode='lossless-float')
  assert isinstance(f, jiter.LosslessFloat)
  assert str(f) == '12.3'
  assert float(f) == 12.3
  assert f.as_decimal() == Decimal('12.3')
 
- f = jiter.from_json(b'123.456789123456789e45', lossless_floats=True)
+ f = jiter.from_json(b'123.456789123456789e45', float_mode='lossless-float')
  assert isinstance(f, jiter.LosslessFloat)
  assert 123e45 < float(f) < 124e45
  assert f.as_decimal() == Decimal('1.23456789123456789E+47')
  assert bytes(f) == b'123.456789123456789e45'
  assert str(f) == '123.456789123456789e45'
  assert repr(f) == 'LosslessFloat(123.456789123456789e45)'
 
+ f = jiter.from_json(b'123', float_mode='lossless-float')
+ assert isinstance(f, int)
+ assert f == 123
+
+ with pytest.raises(ValueError, match='expected value at line 1 column 1'):
+ jiter.from_json(b'wrong', float_mode='lossless-float')
+
+ with pytest.raises(ValueError, match='trailing characters at line 1 column 2'):
+ jiter.from_json(b'1wrong', float_mode='lossless-float')
+
+
+
+def test_decimal_floats():
+ f = jiter.from_json(b'12.3')
+ assert isinstance(f, float)
+ assert f == 12.3
+
+ f = jiter.from_json(b'12.3', float_mode='decimal')
+ assert isinstance(f, Decimal)
+ assert f == Decimal('12.3')
+
+ f = jiter.from_json(b'123.456789123456789e45', float_mode='decimal')
+ assert isinstance(f, Decimal)
+ assert f == Decimal('1.23456789123456789E+47')
+
+ f = jiter.from_json(b'123', float_mode='decimal')
+ assert isinstance(f, int)
+ assert f == 123
+
+ with pytest.raises(ValueError, match='expected value at line 1 column 1'):
+ jiter.from_json(b'wrong', float_mode='decimal')
 
-def test_lossless_floats_int():
- v = jiter.from_json(b'123', lossless_floats=True)
- assert isinstance(v, int)
- assert v == 123
+ with pytest.raises(ValueError, match='trailing characters at line 1 column 2'):
+ jiter.from_json(b'1wrong', float_mode='decimal')
 
 
 def test_unicode_roundtrip():

crates/jiter/src/errors.rs

@@ -10,6 +10,9 @@ pub enum JsonErrorType {
  /// duplicate keys in an object
  DuplicateKey(String),
 
+ /// happens when getting the `Decimal` type or constructing a decimal fails
+ InternalError(String),
+
  /// NOTE: all errors from here on are copied from serde_json
  /// [src/error.rs](https://github.com/serde-rs/json/blob/v1.0.107/src/error.rs#L236)
  /// with `Io` and `Message` removed
@@ -81,6 +84,7 @@ impl std::fmt::Display for JsonErrorType {
  match self {
  Self::FloatExpectingInt => f.write_str("float value was found where an int was expected"),
  Self::DuplicateKey(s) => write!(f, "Detected duplicate key {s:?}"),
+ Self::InternalError(s) => write!(f, "Internal error: {s:?}"),
  Self::EofWhileParsingList => f.write_str("EOF while parsing a list"),
  Self::EofWhileParsingObject => f.write_str("EOF while parsing an object"),
  Self::EofWhileParsingString => f.write_str("EOF while parsing a string"),

crates/jiter/src/lib.rs

@@ -24,7 +24,7 @@ pub use parse::Peek;
 pub use value::{JsonArray, JsonObject, JsonValue};
 
 #[cfg(feature = "python")]
-pub use py_lossless_float::LosslessFloat;
+pub use py_lossless_float::{FloatMode, LosslessFloat};
 #[cfg(feature = "python")]
 pub use py_string_cache::{cache_clear, cache_usage, cached_py_string, pystring_fast_new, StringCacheMode};
 #[cfg(feature = "python")]

crates/jiter/src/py_lossless_float.rs

@@ -1,10 +1,40 @@
-use pyo3::exceptions::PyValueError;
+use pyo3::exceptions::{PyTypeError, PyValueError};
 use pyo3::prelude::*;
 use pyo3::sync::GILOnceCell;
 use pyo3::types::PyType;
 
 use crate::Jiter;
 
+#[derive(Debug, Clone, Copy)]
+pub enum FloatMode {
+ Float,
+ Decimal,
+ LosslessFloat,
+}
+
+impl Default for FloatMode {
+ fn default() -> Self {
+ Self::Float
+ }
+}
+
+const FLOAT_ERROR: &str = "Invalid float mode, should be `'float'`, `'decimal'` or `'lossless-float'`";
+
+impl<'py> FromPyObject<'py> for FloatMode {
+ fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult<Self> {
+ if let Ok(str_mode) = ob.extract::<&str>() {
+ match str_mode {
+ "float" => Ok(Self::Float),
+ "decimal" => Ok(Self::Decimal),
+ "lossless-float" => Ok(Self::LosslessFloat),
+ _ => Err(PyValueError::new_err(FLOAT_ERROR)),
+ }
+ } else {
+ Err(PyTypeError::new_err(FLOAT_ERROR))
+ }
+ }
+}
+
 /// Represents a float from JSON, by holding the underlying bytes representing a float from JSON.
 #[derive(Debug, Clone)]
 #[pyclass(module = "jiter")]

crates/jiter/src/python.rs

@@ -12,6 +12,7 @@ use smallvec::SmallVec;
 use crate::errors::{json_err, json_error, JsonError, JsonResult, DEFAULT_RECURSION_LIMIT};
 use crate::number_decoder::{AbstractNumberDecoder, NumberAny, NumberRange};
 use crate::parse::{Parser, Peek};
+use crate::py_lossless_float::{get_decimal_type, FloatMode};
 use crate::py_string_cache::{StringCacheAll, StringCacheKeys, StringCacheMode, StringMaybeCache, StringNoCache};
 use crate::string_decoder::{StringDecoder, Tape};
 use crate::{JsonErrorType, LosslessFloat};
@@ -27,8 +28,9 @@ pub struct PythonParse {
  pub partial_mode: PartialMode,
  /// Whether to catch duplicate keys in objects.
  pub catch_duplicate_keys: bool,
- /// Whether to preserve full detail on floats using [`LosslessFloat`]
- pub lossless_floats: bool,
+ /// How to return floats: as a `float` (`'float'`), `Decimal` (`'decimal'`) or
+ /// [`LosslessFloat`] (`'lossless-float'`)
+ pub float_mode: FloatMode,
 }
 
 impl PythonParse {
@@ -56,11 +58,13 @@ impl PythonParse {
  }
  macro_rules! ppp_group {
  ($string_cache:ident) => {
- match (self.catch_duplicate_keys, self.lossless_floats) {
- (true, true) => ppp!($string_cache, DuplicateKeyCheck, ParseNumberLossless),
- (true, false) => ppp!($string_cache, DuplicateKeyCheck, ParseNumberLossy),
- (false, true) => ppp!($string_cache, NoopKeyCheck, ParseNumberLossless),
- (false, false) => ppp!($string_cache, NoopKeyCheck, ParseNumberLossy),
+ match (self.catch_duplicate_keys, self.float_mode) {
+ (true, FloatMode::Float) => ppp!($string_cache, DuplicateKeyCheck, ParseNumberLossy),
+ (true, FloatMode::Decimal) => ppp!($string_cache, DuplicateKeyCheck, ParseNumberDecimal),
+ (true, FloatMode::LosslessFloat) => ppp!($string_cache, DuplicateKeyCheck, ParseNumberLossless),
+ (false, FloatMode::Float) => ppp!($string_cache, NoopKeyCheck, ParseNumberLossy),
+ (false, FloatMode::Decimal) => ppp!($string_cache, NoopKeyCheck, ParseNumberDecimal),
+ (false, FloatMode::LosslessFloat) => ppp!($string_cache, NoopKeyCheck, ParseNumberLossless),
  }
  };
  }
@@ -378,3 +382,42 @@ impl MaybeParseNumber for ParseNumberLossless {
  }
  }
 }
+
+struct ParseNumberDecimal;
+
+impl MaybeParseNumber for ParseNumberDecimal {
+ fn parse_number<'py>(
+ py: Python<'py>,
+ parser: &mut Parser,
+ peek: Peek,
+ allow_inf_nan: bool,
+ ) -> JsonResult<Bound<'py, PyAny>> {
+ match parser.consume_number::<NumberRange>(peek.into_inner(), allow_inf_nan) {
+ Ok(number_range) => {
+ let bytes = parser.slice(number_range.range).unwrap();
+ if number_range.is_int {
+ let obj = NumberAny::decode(bytes, 0, peek.into_inner(), allow_inf_nan)?
+ .0
+ .to_object(py);
+ Ok(obj.into_bound(py))
+ } else {
+ let decimal_type = get_decimal_type(py)
+ .map_err(|e| JsonError::new(JsonErrorType::InternalError(e.to_string()), parser.index))?;
+ // SAFETY: NumberRange::decode has already confirmed that bytes are a valid JSON number,
+ // and therefore valid str
+ let float_str = unsafe { std::str::from_utf8_unchecked(bytes) };
+ decimal_type
+ .call1((float_str,))
+ .map_err(|e| JsonError::new(JsonErrorType::InternalError(e.to_string()), parser.index))
+ }
+ }
+ Err(e) => {
+ if !peek.is_num() {
+ Err(json_error!(ExpectedSomeValue, parser.index))
+ } else {
+ Err(e)
+ }
+ }
+ }
+ }
+}