diff --git a/extras/simple-bench/Cargo.toml b/extras/simple-bench/Cargo.toml index 5cd59c8..a42a314 100644 --- a/extras/simple-bench/Cargo.toml +++ b/extras/simple-bench/Cargo.toml @@ -14,3 +14,7 @@ anyhow = "1.0" lexical = "5.2" lexical-core = "0.7" fastrand = "1.4" + +[features] +default = [] +use_tokenized = [] diff --git a/extras/simple-bench/src/main.rs b/extras/simple-bench/src/main.rs index 9428fad..438f197 100644 --- a/extras/simple-bench/src/main.rs +++ b/extras/simple-bench/src/main.rs @@ -108,6 +108,8 @@ fn run_bench T>( #[derive(Debug, Copy, Clone, Eq, PartialEq)] enum Method { FastFloat, + #[cfg(feature = "use_tokenized")] + FastFloatTokenized, Lexical, FromStr, } @@ -120,10 +122,87 @@ fn type_str(float32: bool) -> &'static str { } } +#[inline] +#[cfg(feature = "use_tokenized")] +fn parse_sign<'a>(s: &'a str) -> (bool, &'a str) { + match s.as_bytes().get(0) { + Some(&b'+') => (false, &s[1..]), + Some(&b'-') => (true, &s[1..]), + _ => (false, s), + } +} + +#[inline] +#[cfg(feature = "use_tokenized")] +fn decimal_index(s: &str) -> Option { + s.as_bytes().iter().position(|&c| c == b'.') +} + +#[inline] +#[cfg(feature = "use_tokenized")] +fn exponent_index(s: &str) -> Option { + s.as_bytes().iter().position(|&c| c == b'e' || c == b'E') +} + +#[inline] +#[cfg(feature = "use_tokenized")] +fn split_index<'a>(s: &'a str, index: usize) -> (&'a str, &'a str) { + let (lead, trail) = s.as_bytes().split_at(index); + let trail = &trail[1..]; + use std::str; + unsafe { + (str::from_utf8_unchecked(lead), str::from_utf8_unchecked(trail)) + } +} + +#[inline] +#[cfg(feature = "use_tokenized")] +fn split_end<'a>(s: &'a str) -> (&'a str, &'a str) { + let (lead, trail) = s.as_bytes().split_at(s.len()); + use std::str; + unsafe { + (str::from_utf8_unchecked(lead), str::from_utf8_unchecked(trail)) + } +} + +#[inline] +#[cfg(feature = "use_tokenized")] +fn parse_exponent(s: &str) -> i64 { + s.parse::().unwrap() +} + +#[inline] +#[cfg(feature = "use_tokenized")] +fn tokenize<'a>(s: &'a str) -> (&'a str, &'a str, i64, bool) { + let (negative, s) = parse_sign(s); + if let Some(index) = decimal_index(s) { + let (i, rest) = split_index(s, index); + if let Some(index) = exponent_index(s) { + let (f, exp) = split_index(rest, index); + let exp = parse_exponent(exp); + (i, f, exp, negative) + } else { + (i, rest, 0, negative) + } + } else { + if let Some(index) = exponent_index(s) { + let (i, exp) = split_index(s, index); + let (i, f) = split_end(i); + let exp = parse_exponent(exp); + (i, f, exp, negative) + } else { + let (i, f) = split_end(s); + (i, f, 0, negative) + } + } +} + impl Method { pub fn name(&self) -> &'static str { match self { Self::FastFloat => "fast-float", + #[cfg(feature = "use_tokenized")] + Self::FastFloatTokenized => "fast-float-tokenized", Self::Lexical => "lexical", Self::FromStr => "from_str", } @@ -140,6 +219,11 @@ impl Method { Self::FastFloat => run_bench(data, repeat, |s: &str| { fast_float::parse_partial::(s).unwrap_or_default().0 }), + #[cfg(feature = "use_tokenized")] + Self::FastFloatTokenized => run_bench(data, repeat, |s: &str| { + let (i, f, e, n) = tokenize(s); + fast_float::parse_from_parts::(i, f, e, n) + }), Self::Lexical => run_bench(data, repeat, |s: &str| { lexical_core::parse_partial::(s.as_bytes()) .unwrap_or_default() @@ -165,7 +249,15 @@ impl Method { } pub fn all() -> &'static [Self] { - &[Method::FastFloat, Method::Lexical, Method::FromStr] + #[cfg(feature = "use_tokenized")] + { + &[Method::FastFloat, Method::FastFloatTokenized, Method::Lexical, Method::FromStr] + } + + #[cfg(not(feature = "use_tokenized"))] + { + &[Method::FastFloat, Method::Lexical, Method::FromStr] + } } } diff --git a/src/decimal.rs b/src/decimal.rs index c36d6d3..9d612e1 100644 --- a/src/decimal.rs +++ b/src/decimal.rs @@ -187,41 +187,37 @@ impl Decimal { } #[inline] -pub fn parse_decimal(mut s: &[u8]) -> Decimal { - // can't fail since it follows a call to parse_number - let mut d = Decimal::default(); - let start = s; - let c = s.get_first(); - d.negative = c == b'-'; - if c == b'-' || c == b'+' { - s = s.advance(1); +fn parse_fractional<'a>(mut s: &'a [u8], d: &mut Decimal) -> &'a [u8] { + let first = s; + if d.num_digits == 0 { + s = s.skip_chars(b'0'); } - s = s.skip_chars(b'0'); - parse_digits(&mut s, |digit| d.try_add_digit(digit)); - if s.check_first(b'.') { - s = s.advance(1); - let first = s; - if d.num_digits == 0 { - s = s.skip_chars(b'0'); - } - if cfg!(target_endian = "little") { - while s.len() >= 8 && d.num_digits + 8 < Decimal::MAX_DIGITS { - let v = s.read_u64(); - if !is_8digits_le(v) { - break; - } - d.digits[d.num_digits..].write_u64(v - 0x3030_3030_3030_3030); - d.num_digits += 8; - s = s.advance(8); + if cfg!(target_endian = "little") { + while s.len() >= 8 && d.num_digits + 8 < Decimal::MAX_DIGITS { + let v = s.read_u64(); + if !is_8digits_le(v) { + break; } + d.digits[d.num_digits..].write_u64(v - 0x3030_3030_3030_3030); + d.num_digits += 8; + s = s.advance(8); } - parse_digits(&mut s, |digit| d.try_add_digit(digit)); - d.decimal_point = s.len() as i32 - first.len() as i32; } + parse_digits(&mut s, |digit| d.try_add_digit(digit)); + d.decimal_point = s.len() as i32 - first.len() as i32; + + s +} + +#[inline] +fn trim_zeros<'a, Iter>(iter: Iter, d: &mut Decimal) +where + Iter: Iterator +{ if d.num_digits != 0 { // Ignore the trailing zeros if there are any let mut n_trailing_zeros = 0; - for &c in start[..(start.len() - s.len())].iter().rev() { + for &c in iter { if c == b'0' { n_trailing_zeros += 1; } else if c != b'.' { @@ -236,6 +232,51 @@ pub fn parse_decimal(mut s: &[u8]) -> Decimal { d.num_digits = Decimal::MAX_DIGITS; } } +} + +#[inline] +fn add_zero_digits(d: &mut Decimal) { + for i in d.num_digits..Decimal::MAX_DIGITS_WITHOUT_OVERFLOW { + d.digits[i] = 0; + } +} + +#[inline] +pub fn parse_decimal_from_parts(mut i: &[u8], f: &[u8], e: i64, negative: bool) -> Decimal { + // can't fail since it follows a call to parse_number + let mut d = Decimal::default(); + + // Integral + let i_start = i; + d.negative = negative; + i = i.skip_chars(b'0'); + parse_digits(&mut i, |digit| d.try_add_digit(digit)); + + parse_fractional(f, &mut d); + trim_zeros(i_start.iter().chain(f.iter()).rev(), &mut d); + d.decimal_point += e as i32; + add_zero_digits(&mut d); + + d +} + +#[inline] +pub fn parse_decimal(mut s: &[u8]) -> Decimal { + // can't fail since it follows a call to parse_number + let mut d = Decimal::default(); + let start = s; + let c = s.get_first(); + d.negative = c == b'-'; + if c == b'-' || c == b'+' { + s = s.advance(1); + } + s = s.skip_chars(b'0'); + parse_digits(&mut s, |digit| d.try_add_digit(digit)); + if s.check_first(b'.') { + s = s.advance(1); + s = parse_fractional(s, &mut d); + } + trim_zeros(start[..(start.len() - s.len())].iter().rev(), &mut d); if s.check_first2(b'e', b'E') { s = s.advance(1); let mut neg_exp = false; @@ -253,9 +294,7 @@ pub fn parse_decimal(mut s: &[u8]) -> Decimal { }); d.decimal_point += if neg_exp { -exp_num } else { exp_num }; } - for i in d.num_digits..Decimal::MAX_DIGITS_WITHOUT_OVERFLOW { - d.digits[i] = 0; - } + add_zero_digits(&mut d); d } diff --git a/src/lib.rs b/src/lib.rs index aef86be..5f30dd4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -105,6 +105,21 @@ pub trait FastFloat: float::Float { fn parse_float_partial>(s: S) -> Result<(Self, usize)> { parse::parse_float(s.as_ref()).ok_or(Error) } + + /// Parse a pre-tokenized decimal number from string into float. + /// + /// This assumes the float has already been tokenized into valid + /// integral and fractional components, and has parsed an optional + /// exponent notation. + /// + /// It is up to you to validate and tokenize the input: although + /// this will not error, this might truncate the significant + /// digits as soon as an invalid digit is found. This does not + /// handle special values, such as NaN, INF, or Infinity. + #[inline] + fn parse_from_parts>(integral: S, fractional: S, exponent: i64, negative: bool) -> Self { + parse::parse_from_parts(integral.as_ref(), fractional.as_ref(), exponent, negative) + } } impl FastFloat for f32 {} @@ -134,3 +149,18 @@ pub fn parse>(s: S) -> Result { pub fn parse_partial>(s: S) -> Result<(T, usize)> { T::parse_float_partial(s) } + +/// Parse a pre-tokenized decimal number from string into float. +/// +/// This assumes the float has already been tokenized into valid +/// integral and fractional components, and has parsed an optional +/// exponent notation. +/// +/// It is up to you to validate and tokenize the input: although +/// this will not error, this might truncate the significant +/// digits as soon as an invalid digit is found. This does not +/// handle special values, such as NaN, INF, or Infinity. +#[inline] +pub fn parse_from_parts>(integral: S, fractional: S, exponent: i64, negative: bool) -> T { + T::parse_from_parts(integral.as_ref(), fractional.as_ref(), exponent, negative) +} diff --git a/src/number.rs b/src/number.rs index 194cbf6..d40d657 100644 --- a/src/number.rs +++ b/src/number.rs @@ -148,6 +148,88 @@ fn parse_scientific(s: &mut AsciiStr<'_>) -> i64 { } } +#[inline] +fn parse_integral(s: &mut AsciiStr, start: &AsciiStr, mantissa: &mut u64) -> isize { + try_parse_digits(s, mantissa); + s.offset_from(start) +} + +#[inline] +fn parse_fractional(s: &mut AsciiStr, start: &AsciiStr, mantissa: &mut u64) -> isize { + try_parse_8digits_le(s, mantissa); + try_parse_digits(s, mantissa); + s.offset_from(start) +} + +#[inline] +fn trim_leading_zeros(mut n_digits: isize, mut p: AsciiStr) -> isize { + n_digits -= 19; + while p.check_first_either(b'0', b'.') { + n_digits -= p.first().saturating_sub(b'0' - 1) as isize; // '0' = b'.' + 2 + p.step(); + } + n_digits +} + +#[inline] +pub fn parse_number_from_parts(i: &[u8], f: &[u8], e: i64, negative: bool) -> Option { + + let mut mantissa = 0_u64; + let mut i = AsciiStr::new(i); + let i_start = i; + let mut n_digits = parse_integral(&mut i, &i_start, &mut mantissa); + + let mut f = AsciiStr::new(f); + let f_start = f; + let int_end = i; + let n_after_dot = parse_fractional(&mut f, &f_start, &mut mantissa); + n_digits += n_after_dot; + let mut exponent = e - n_after_dot as i64; + + if n_digits == 0 { + return None; + } + + // handle uncommon case with many digits + if n_digits <= 19 { + return Some( + Number { + exponent, + mantissa, + negative, + many_digits: false, + } + ); + } + + let mut many_digits = false; + n_digits = trim_leading_zeros(n_digits, i_start); + if n_digits > 0 { + // at this point we have more than 19 significant digits, let's try again + many_digits = true; + mantissa = 0; + let mut i = i_start; + try_parse_19digits(&mut i, &mut mantissa); + exponent = if mantissa >= MIN_19DIGIT_INT { + int_end.offset_from(&i) // big int + } else { + let mut f = f_start; + try_parse_19digits(&mut f, &mut mantissa); + -f.offset_from(&f_start) + } as i64; + exponent += e; // add back the explicit part + } + + Some( + Number { + exponent, + mantissa, + negative, + many_digits, + }, + ) +} + #[inline] pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> { debug_assert!(!s.is_empty()); @@ -170,8 +252,7 @@ pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> { // parse initial digits before dot let mut mantissa = 0_u64; let digits_start = s; - try_parse_digits(&mut s, &mut mantissa); - let mut n_digits = s.offset_from(&digits_start); + let mut n_digits = parse_integral(&mut s, &digits_start, &mut mantissa); // handle dot with the following digits let mut n_after_dot = 0; @@ -180,9 +261,7 @@ pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> { if s.check_first(b'.') { s.step(); let before = s; - try_parse_8digits_le(&mut s, &mut mantissa); - try_parse_digits(&mut s, &mut mantissa); - n_after_dot = s.offset_from(&before); + n_after_dot = parse_fractional(&mut s, &before, &mut mantissa); exponent = -n_after_dot as i64; } @@ -213,13 +292,8 @@ pub fn parse_number(s: &[u8]) -> Option<(Number, usize)> { )); } - n_digits -= 19; let mut many_digits = false; - let mut p = digits_start; - while p.check_first_either(b'0', b'.') { - n_digits -= p.first().saturating_sub(b'0' - 1) as isize; // '0' = b'.' + 2 - p.step(); - } + n_digits = trim_leading_zeros(n_digits, digits_start); if n_digits > 0 { // at this point we have more than 19 significant digits, let's try again many_digits = true; diff --git a/src/parse.rs b/src/parse.rs index 9c592d4..465d8fa 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,8 +1,52 @@ use crate::binary::compute_float; +use crate::common::AdjustedMantissa; +use crate::decimal::{parse_decimal, parse_decimal_from_parts}; use crate::float::Float; -use crate::number::{parse_inf_nan, parse_number}; +use crate::number::{Number, parse_inf_nan, parse_number, parse_number_from_parts}; use crate::simple::parse_long_mantissa; +#[inline] +fn lemire(num: &Number) -> AdjustedMantissa { + let mut am = compute_float::(num.exponent, num.mantissa); + if num.many_digits && am != compute_float::(num.exponent, num.mantissa + 1) { + am.power2 = -1; + } + am +} + +#[inline] +fn to_float(am: AdjustedMantissa, negative: bool) -> F { + let mut word = am.mantissa; + word |= (am.power2 as u64) << F::MANTISSA_EXPLICIT_BITS; + if negative { + word |= 1_u64 << F::SIGN_INDEX; + } + F::from_u64_bits(word) +} + +#[inline] +pub fn parse_from_parts(i: &[u8], f: &[u8], e: i64, negative: bool) -> F { + if i.is_empty() && f.is_empty() { + return F::from_u64(0); + } + + let num = match parse_number_from_parts(i, f, e, negative) { + Some(n) => n, + None => return F::from_u64(0), + }; + if let Some(value) = num.try_fast_path::() { + return value; + } + + let mut am = lemire::(&num); + if am.power2 < 0 { + am = parse_long_mantissa::(parse_decimal_from_parts(i, f, e, negative)); + } + + to_float(am, num.negative) +} + + #[inline] pub fn parse_float(s: &[u8]) -> Option<(F, usize)> { if s.is_empty() { @@ -17,18 +61,11 @@ pub fn parse_float(s: &[u8]) -> Option<(F, usize)> { return Some((value, rest)); } - let mut am = compute_float::(num.exponent, num.mantissa); - if num.many_digits && am != compute_float::(num.exponent, num.mantissa + 1) { - am.power2 = -1; - } + let mut am = lemire::(&num); if am.power2 < 0 { - am = parse_long_mantissa::(s); + am = parse_long_mantissa::(parse_decimal(s)); } - let mut word = am.mantissa; - word |= (am.power2 as u64) << F::MANTISSA_EXPLICIT_BITS; - if num.negative { - word |= 1_u64 << F::SIGN_INDEX; - } - Some((F::from_u64_bits(word), rest)) + let flt = to_float(am, num.negative); + Some((flt, rest)) } diff --git a/src/simple.rs b/src/simple.rs index cec1183..226d244 100644 --- a/src/simple.rs +++ b/src/simple.rs @@ -1,9 +1,9 @@ use crate::common::AdjustedMantissa; -use crate::decimal::{parse_decimal, Decimal}; +use crate::decimal::Decimal; use crate::float::Float; #[inline] -pub fn parse_long_mantissa(s: &[u8]) -> AdjustedMantissa { +pub fn parse_long_mantissa(mut d: Decimal) -> AdjustedMantissa { const MAX_SHIFT: usize = 60; const NUM_POWERS: usize = 19; const POWERS: [u8; 19] = [ @@ -21,8 +21,6 @@ pub fn parse_long_mantissa(s: &[u8]) -> AdjustedMantissa { let am_zero = AdjustedMantissa::zero_pow2(0); let am_inf = AdjustedMantissa::zero_pow2(F::INFINITE_POWER); - let mut d = parse_decimal(s); - if d.num_digits == 0 || d.decimal_point < -324 { return am_zero; } else if d.decimal_point >= 310 { diff --git a/tests/test_json.rs b/tests/test_json.rs new file mode 100644 index 0000000..cf8790c --- /dev/null +++ b/tests/test_json.rs @@ -0,0 +1,336 @@ +use std::string::ToString; + +macro_rules! check_float { + ($i:expr, $f:expr, $e:expr, $n:expr, $ty:ty) => {{ + let mut s = String::new(); + if $n { + s.push('-'); + } + s.push_str($i); + if $f.len() > 0 { + s.push('.'); + s.push_str($f); + } + if $e != 0 { + s.push('e'); + s.push_str(&$e.to_string()); + } + let expected = fast_float::parse::<$ty, _>(s).unwrap(); + let result = fast_float::parse_from_parts::<$ty, _>($i, $f, $e, $n); + assert_eq!(expected, result); + }}; +} + +macro_rules! check_f32 { + ($i:expr, $f:expr, $e:expr, $n:expr) => {{ + check_float!($i, $f, $e, $n, f32); + }}; +} + +macro_rules! check_f64 { + ($i:expr, $f:expr, $e:expr, $n:expr) => {{ + check_float!($i, $f, $e, $n, f64); + }}; +} + +fn append_zeros(s: impl AsRef, n: usize) -> String { + let mut s = String::from(s.as_ref()); + for _ in 0..n { + s.push('0'); + } + s +} + +#[test] +fn test_tokenized_f64() { + // Inf + check_f64!("1234456789012345678901234567890", "0", 2147483647, false); + check_f64!("1", "832312213213213232132132143451234453123412321321312", 308, false); + check_f64!("2", "0", 2147483647, false); + check_f64!("2", "0", 3000, false); + check_f64!("1", "8", 308, false); + check_f64!("1", "9", 308, false); + + // Negative inf + check_f64!("2139879401095466344511101915470454744", "9813888656856943", 272, true); + + // Long + check_f64!( + "9355950000000000000", + "\ + 000000000000000000000000000000000018446744073709551616000001\ + 84467440737095516161844674407370955161407370955161618446744073709551616000184467\ + 44073709551616600000184467440737095516161844674407370955161407370955161618446744\ + 07370955161600018446744073709551616018446744073709556744516161844674407370955161\ + 40737095516161844674407370955161600018446744073709551616018446744073709551611616\ + 00018446744073709500184467440737095516160018446744073709551616001844674407370955\ + 11681644674407370955161600018440737095516160184467440737095516161844674407370955\ + 16160001844674407536910751601611616000184467440737095001844674407370955161600184\ + 46744073709551616001844674407370955161618446744073709551616000184495516161844674\ + 4073709551616000184467440753691075160018446744073709", + 0, + false + ); + check_f64!( + "2", + "\ + 225073858507202124188701479202220329072405282794390378143031338374351073192441\ + 94686754406432563881851382188218502438069999947733013005649884107791928741341929\ + 29720097048195199306799329096904278406473168204156592672863293363047467012331685\ + 29834221527445172608358596545663192828352447877877998943107797838336991592885945\ + 55213714181128458251145584319223079897504395086859412457230891738946169368372321\ + 19137365897797772328669884035639025104444303545739673370658398105542045669382465\ + 84137476071559811765738776267476659123871999319040063173347090030127901881752034\ + 47190250028061277777916798391090578584006464715943810511489154282775041174682194\ + 13395246668250343130618158782937900420539237507208336669324158000275839111885418\ + 8641513168478436313080237596295773983001708984375", + -308, + false + ); + check_f64!( + "\ + 14384566631413902735261182076422355811832278452463312311626366537903681520913941\ + 96930365828634687637948157940776599182791387527135353034738357134110310609455693\ + 90082419354977279201654318268051974058035436546798544018359870131225762454556233\ + 13970183299286131961255902741877200739148180625308303165331580986249841188892982\ + 81371812288789537310599037529113415438738954894752124724983067241108764488346454\ + 37669901867307840475112141480493722424080599312381693232622368309077056159757045\ + 77939329858261626042558845291341263962822021265262533893834218067279545885255961\ + 14379801269094096329805054803089299736996870951258573010877404407451953846698609\ + 19821392688269207855703322826525930548119852605981316446918758669325733577952202\ + 04076454986842633399219052275566166981299674128912822316855046606712779271982900\ + 09824680186319750978665734576683784255802269708917361719466043175201158849097881\ + 37047711185017157986905601606166617302905958843377601564443970505037755427769614\ + 39282780934537928038462527159660167332226464423828921239400524413468224297215938\ + 84378212558701004356924243030059517489346646577724622498919752597382095222500311\ + 12418182351225107135618176937657765139002829779615620881537508915912839494571051\ + 58613344862671017974971111259092725051947928708896171797587034426080161433432621\ + 59998149700606597792535574457560429226974273443630323818747730771316763398572110\ + 87495998192373246307688452867739265415001026982223940199342748237651323138921235\ + 35835735663769155726509168665536123661873789595549835667127670933729060301889762\ + 20169058025354973622211666504549316958271880975697143546564469806791358707318873\ + 07570838334500409015197406832583817753126695417740666139222980134999469594150993\ + 5655355652985723782153570084089560139142231", + "\ + 738475042362596875449154552392299548\ + 94713816208169416867534067784380761312978044932336375902701297246698737092181681\ + 31626587547265451210905455072402670004565947865409496052607224619378706306348749\ + 91729398208026467698131898691830012167897399682179601734569071423681", + -733, + false + ); + check_f64!( + "0", + "\ + 000000000000000000000000000000000000000000000000000000000000000000000000000000\ + 00000000000000000000000000000000000000000000000000000000000000000000000000000000\ + 00000000000000000000000000000000000000000000000000000000000000000000000000000000\ + 00000000000000000000000000000000000000000000000000000000000000000000044501477170\ + 14402272114819593418263951869639092703291296046852219449644444042153891033059047\ + 81627017582829831782607924221374017287738918929105531441481564124348675997628212\ + 65346585071045737627442980259622449029037796981144446145705102663115100318287949\ + 52795966823603998647925096578034214163701381261333311989876551545144031526125381\ + 32666529513060001849177663286607555958373922409899478075565940981010216121988146\ + 05258742579179000071675999344145086087205681577915435923018910334964869420614052\ + 18289243144579760516365090360651414037721744226256159024466852576737244643007551\ + 33324500796506867194913776884780053099639677097589658441378944337966219939673169\ + 36280457084866613206797017728916080020698679408551343728867675409720757232455434\ + 770912461317493580281734466552734375", + 0, + false + ); + check_f64!( + "0", + "\ + 000000000000000000000000000000000000000000000000000000000000000000000000000000\ + 00000000000000000000000000000000000000000000000000000000000000000000000000000000\ + 00000000000000000000000000000000000000000000000000000000000000000000000000000000\ + 00000000000000000000000000000000000000000000000000000000000000000000022250738585\ + 07200889024586876085859887650423112240959465493524802562440009228235695178775888\ + 80375915526423097809504343120858773871583572918219930202943792242235598198275012\ + 42041788969571311791082261043971979604000454897391938079198936081525613113376149\ + 84204327175103362739154978273159414382813627511383860409424946494228631669542910\ + 50802018159266421349966065178030950759130587198464239060686371020051087232827846\ + 78843631944515866135041223479014792369585208321597621066375401613736583044193603\ + 71477835530668283453563400507407304013560296804637591858316312422452159926254649\ + 43008368518617194224176464551371354201322170313704965832101546540680353974179060\ + 22589503023501937519773030945763173210852507299305089761582519159720757232455434\ + 770912461317493580281734466552734375", + 0, + false + ); + + // General + check_f64!("9007199254740993", "0", 0, false); + check_f64!("9007199254740993", &append_zeros("0", 1000), 0, false); + check_f64!("10000000000000000000", "", 0, false); + check_f64!("10000000000000000000000000000001000000000000", "", 0, false); + check_f64!("10000000000000000000000000000000000000000001", "", 0, false); + check_f64!("1", "1920928955078125", -7, false); + check_f64!("1", "0000000000000006661338147750939242541790008544921875", 0, false); + check_f64!("1090544144181609348835077142190", "", 0, false); + check_f64!("2", "2250738585072013", -308, false); + check_f64!("92666518056446206563", "", 3, true); + check_f64!("42823146028335318693", "", -128, true); + check_f64!("90054602635948575728", "", 72, false); + check_f64!( + "1", + "\ + 000000000000001885589208702234638701745660206917535153946435506630705583683732\ + 21972569761144603605635692374830246134201063722058", + -309, + false + ); + check_f64!("0", "", 2147483647, false); + check_f64!("2402844368454405395", "2", 2147483647, true); + check_f64!("2402844368454405395", "2", 2147483647, false); + check_f64!("7", "0420557077594588669468784357561207962098443483187940792729600000", 59, false); + check_f64!("7", "0420557077594588669468784357561207962098443483187940792729600000", 42, false); + check_f64!("1", "7339253062092163730578609458683877051596800000000000000000000000", 42, true); + check_f64!("2", "0972622234386619214559824785284023792871122537545728000000000000", 52, true); + check_f64!("1", "0001803374372191849407179462120053338028379051879898808320000000", 57, true); + check_f64!("1", "8607245283054342363818436991534856973992070520151142825984000000", 58, true); + check_f64!("1", "9189205311132686907264385602245237137907390376574976000000000000", 52, true); + check_f64!("2", "8184483231688951563253238886553506793085187889855201280000000000", 54, true); + check_f64!("1", "7664960224650106892054063261344555646357024359107788800000000000", 53, true); + check_f64!("2", "1470977154320536489471030463761883783915110400000000000000000000", 45, true); + check_f64!("4", "4900312744003159009338275160799498340862630046359789166919680000", 61, true); + check_f64!("1", "", 0, false); + check_f64!("1", "797693134862315700000000000000001", 308, false); + check_f64!("3", "", -324, false); + check_f64!("1", "00000006", 9, false); + check_f64!("4", "9406564584124653", -324, false); + check_f64!("4", "9406564584124654", -324, false); + check_f64!("2", "2250738585072009", -308, false); + check_f64!("2", "2250738585072014", -308, false); + check_f64!("1", "7976931348623157", 308, false); + check_f64!("1", "7976931348623158", 308, false); + check_f64!("4503599627370496", "5", 0, false); + check_f64!("4503599627475352", "5", 0, false); + check_f64!("4503599627475353", "5", 0, false); + check_f64!("2251799813685248", "25", 0, false); + check_f64!("1125899906842624", "125", 0, false); + check_f64!("1125899906842901", "875", 0, false); + check_f64!("2251799813685803", "75", 0, false); + check_f64!("4503599627370497", "5", 0, false); + check_f64!("45035996", "273704995", 0, false); + check_f64!("45035996", "273704985", 0, false); + check_f64!("1", "2345", 30, false); +} + +#[test] +fn test_tokenized_f32() { + // Inf + check_f32!("1234456789012345678901234567890", "", 2147483647, false); + check_f32!("2", "", 3000, false); + check_f32!("3", "5028234666", 38, false); + + // General + let f1 = "\ + 175494140627517859246175898662808184331245864732796240031385942718174675986064\ + 7699724722770042717456817626953125"; + check_f32!("1", f1, 0, false); + check_f32!("1", &append_zeros(f1, 655), 0, false); + check_f32!("1", &append_zeros(f1, 656), 0, false); + check_f32!("1", &append_zeros(f1, 1000), 0, false); + check_f32!("1", "00000006", 9, false); + check_f32!("1", "4012984643", -45, false); + check_f32!("1", "1754942107", -38, false); + check_f32!("1", "1754943508", -45, false); + check_f32!("0", "", 0, true); + check_f32!("1090544144181609348835077142190", "", 0, false); + check_f32!("1", "1754943508", -38, false); + check_f32!("30219", "0830078125", 0, false); + check_f32!("16252921", "5", 0, false); + check_f32!("5322519", "25", 0, false); + check_f32!("3900245", "875", 0, false); + check_f32!("1510988", "3125", 0, false); + check_f32!("782262", "28125", 0, false); + check_f32!("328381", "484375", 0, false); + check_f32!("156782", "0703125", 0, false); + check_f32!("85003", "24609375", 0, false); + check_f32!("43827", "048828125", 0, false); + check_f32!("17419", "6494140625", 0, false); + check_f32!("15498", "36376953125", 0, false); + check_f32!("6318", "580322265625", 0, false); + check_f32!("2525", "2840576171875", 0, false); + check_f32!("1370", "9265747070312", 0, false); + check_f32!("936", "3702087402344", 0, false); + check_f32!("411", "88682556152344", 0, false); + check_f32!("206", "50310516357422", 0, false); + check_f32!("124", "16878890991211", 0, false); + check_f32!("50", "811574935913086", 0, false); + check_f32!("17", "486443519592285", 0, false); + check_f32!("13", "91745138168335", 0, false); + check_f32!("7", "5464513301849365", 0, false); + check_f32!("2", "687217116355896", 0, false); + check_f32!("1", "1877630352973938", 0, false); + check_f32!("0", "7622503340244293", 0, false); + check_f32!("0", "30531780421733856", 0, false); + check_f32!("0", "21791061013936996", 0, false); + check_f32!("0", "09289376810193062", 0, false); + check_f32!("0", "03706067614257336", 0, false); + check_f32!("0", "028068351559340954", 0, false); + check_f32!("0", "012114629615098238", 0, false); + check_f32!("0", "004221370676532388", 0, false); + check_f32!("0", "002153817447833717", 0, false); + check_f32!("0", "0015924838953651488", 0, false); + check_f32!("0", "0008602388261351734", 0, false); + check_f32!("0", "00036393293703440577", 0, false); + check_f32!("0", "00013746770127909258", 0, false); + check_f32!("16407", "9462890625", 0, false); + check_f32!("1", "1754947011469036", -38, false); + check_f32!("7", "0064923216240854", -46, false); + check_f32!("8388614", "5", 0, false); + check_f32!("0", "", 2147483647, false); + check_f32!("4", "7019774032891500318749461488889827112746622270883500860350068251", -38, false); + check_f32!( + "3", + "\ + 141592653589793238462643383279502884197169399375105820974944592307816406286\ + 2089986280348253421170679", + 0, + false + ); + check_f32!("2", "3509887016445750159374730744444913556373311135441750430175034126", -38, false); + check_f32!("1", "", 0, false); + check_f32!("7", "0060", -46, false); + check_f32!("7", "", -46, false); + check_f32!("3", "4028234664", 38, false); + check_f32!("3", "4028234665", 38, false); + check_f32!("3", "4028234666", 38, false); + + check_f32!( + "0", + "\ + 000000000000000000000000000000000000011754943508222875079687365372222456778186\ + 655567720875215087517062784172594547271728515625", + 0, + false + ); + check_f32!( + "0", + "\ + 000000000000000000000000000000000000000000001401298464324817070923729583289916\ + 13128026194187651577175706828388979108268586060148663818836212158203125", + 0, + false + ); + check_f32!( + "0", + "\ + 000000000000000000000000000000000000023509885615147285834557659820715330266457\ + 17985517980855365926236850006129930346077117064851336181163787841796875", + 0, + false + ); + check_f32!( + "0", + "\ + 000000000000000000000000000000000000011754942106924410754870294448492873488270\ + 52428745893333857174530571588870475618904265502351336181163787841796875", + 0, + false + ); + check_f32!("1", "2345", 15, false); +}