diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c4aecb246..6701cf7084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased +- BREAKING: Switch to using `half::f16` [#12](https://github.com/rerun-io/re_arrow2/pull/12) + + ## [v0.17.0](https://github.com/jorgecarleitao/arrow2/tree/v0.17.0) (2023-03-27) [Full Changelog](https://github.com/jorgecarleitao/arrow2/compare/v0.16.0...v0.17.0) diff --git a/Cargo.lock b/Cargo.lock index 072ceea0ee..a9b6bd5bbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1112,6 +1112,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" dependencies = [ + "bytemuck", "crunchy", "num-traits", ] @@ -2220,6 +2221,7 @@ dependencies = [ "foreign_vec", "futures", "getrandom 0.2.8", + "half 2.2.1", "hash_hasher", "hashbrown 0.14.3", "hex", diff --git a/Cargo.toml b/Cargo.toml index 1a0eefe17d..950eea397a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,6 +118,8 @@ arrow-schema = { version = ">=52", optional = true } arrow-data = { version = ">=52", optional = true } arrow-array = { version = ">=52", optional = true } +half = { version = "2.2", features = ["bytemuck"] } + [target.wasm32-unknown-unknown.dependencies] getrandom = { version = "0.2", features = ["js"] } diff --git a/src/lib.rs b/src/lib.rs index bef2e6e53c..37d83aa524 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(feature = "simd", feature(portable_simd))] #![cfg_attr(feature = "nightly_build", feature(build_hasher_simple_hash_one))] +#![allow(deprecated)] // chrono has a bunch of deprecated stuff #[macro_use] pub mod array; diff --git a/src/types/native.rs b/src/types/native.rs index 6e50a1454e..25fc27caec 100644 --- a/src/types/native.rs +++ b/src/types/native.rs @@ -4,6 +4,8 @@ use std::panic::RefUnwindSafe; use bytemuck::{Pod, Zeroable}; +pub use half::f16; + use super::PrimitiveType; /// Sealed trait implemented by all physical types that can be allocated, @@ -331,186 +333,27 @@ impl Neg for months_days_ns { } } -/// Type representation of the Float16 physical type -#[derive(Copy, Clone, Default, Zeroable, Pod)] -#[allow(non_camel_case_types)] -#[repr(C)] -pub struct f16(pub u16); - -impl PartialEq for f16 { - #[inline] - fn eq(&self, other: &f16) -> bool { - if self.is_nan() || other.is_nan() { - false - } else { - (self.0 == other.0) || ((self.0 | other.0) & 0x7FFFu16 == 0) - } - } -} - -// see https://github.com/starkat99/half-rs/blob/main/src/binary16.rs -impl f16 { - /// The difference between 1.0 and the next largest representable number. - pub const EPSILON: f16 = f16(0x1400u16); - - #[inline] - #[must_use] - pub(crate) const fn is_nan(self) -> bool { - self.0 & 0x7FFFu16 > 0x7C00u16 - } - - /// Casts from u16. - #[inline] - pub const fn from_bits(bits: u16) -> f16 { - f16(bits) - } - - /// Casts to u16. - #[inline] - pub const fn to_bits(self) -> u16 { - self.0 - } - - /// Casts this `f16` to `f32` - pub fn to_f32(self) -> f32 { - let i = self.0; - // Check for signed zero - if i & 0x7FFFu16 == 0 { - return f32::from_bits((i as u32) << 16); - } - - let half_sign = (i & 0x8000u16) as u32; - let half_exp = (i & 0x7C00u16) as u32; - let half_man = (i & 0x03FFu16) as u32; - - // Check for an infinity or NaN when all exponent bits set - if half_exp == 0x7C00u32 { - // Check for signed infinity if mantissa is zero - if half_man == 0 { - let number = (half_sign << 16) | 0x7F80_0000u32; - return f32::from_bits(number); - } else { - // NaN, keep current mantissa but also set most significiant mantissa bit - let number = (half_sign << 16) | 0x7FC0_0000u32 | (half_man << 13); - return f32::from_bits(number); - } - } - - // Calculate single-precision components with adjusted exponent - let sign = half_sign << 16; - // Unbias exponent - let unbiased_exp = ((half_exp as i32) >> 10) - 15; - - // Check for subnormals, which will be normalized by adjusting exponent - if half_exp == 0 { - // Calculate how much to adjust the exponent by - let e = (half_man as u16).leading_zeros() - 6; - - // Rebias and adjust exponent - let exp = (127 - 15 - e) << 23; - let man = (half_man << (14 + e)) & 0x7F_FF_FFu32; - return f32::from_bits(sign | exp | man); - } - - // Rebias exponent for a normalized normal - let exp = ((unbiased_exp + 127) as u32) << 23; - let man = (half_man & 0x03FFu32) << 13; - f32::from_bits(sign | exp | man) - } - - /// Casts an `f32` into `f16` - pub fn from_f32(value: f32) -> Self { - let x: u32 = value.to_bits(); - - // Extract IEEE754 components - let sign = x & 0x8000_0000u32; - let exp = x & 0x7F80_0000u32; - let man = x & 0x007F_FFFFu32; - - // Check for all exponent bits being set, which is Infinity or NaN - if exp == 0x7F80_0000u32 { - // Set mantissa MSB for NaN (and also keep shifted mantissa bits) - let nan_bit = if man == 0 { 0 } else { 0x0200u32 }; - return f16(((sign >> 16) | 0x7C00u32 | nan_bit | (man >> 13)) as u16); - } - - // The number is normalized, start assembling half precision version - let half_sign = sign >> 16; - // Unbias the exponent, then bias for half precision - let unbiased_exp = ((exp >> 23) as i32) - 127; - let half_exp = unbiased_exp + 15; - - // Check for exponent overflow, return +infinity - if half_exp >= 0x1F { - return f16((half_sign | 0x7C00u32) as u16); - } - - // Check for underflow - if half_exp <= 0 { - // Check mantissa for what we can do - if 14 - half_exp > 24 { - // No rounding possibility, so this is a full underflow, return signed zero - return f16(half_sign as u16); - } - // Don't forget about hidden leading mantissa bit when assembling mantissa - let man = man | 0x0080_0000u32; - let mut half_man = man >> (14 - half_exp); - // Check for rounding (see comment above functions) - let round_bit = 1 << (13 - half_exp); - if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 { - half_man += 1; - } - // No exponent for subnormals - return f16((half_sign | half_man) as u16); - } - - // Rebias the exponent - let half_exp = (half_exp as u32) << 10; - let half_man = man >> 13; - // Check for rounding (see comment above functions) - let round_bit = 0x0000_1000u32; - if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 { - // Round it - f16(((half_sign | half_exp | half_man) + 1) as u16) - } else { - f16((half_sign | half_exp | half_man) as u16) - } - } -} - -impl std::fmt::Debug for f16 { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.to_f32()) - } -} - -impl std::fmt::Display for f16 { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.to_f32()) - } -} - impl NativeType for f16 { const PRIMITIVE: PrimitiveType = PrimitiveType::Float16; type Bytes = [u8; 2]; #[inline] fn to_le_bytes(&self) -> Self::Bytes { - self.0.to_le_bytes() + f16::to_le_bytes(*self) } #[inline] fn to_be_bytes(&self) -> Self::Bytes { - self.0.to_be_bytes() + f16::to_be_bytes(*self) } #[inline] fn from_be_bytes(bytes: Self::Bytes) -> Self { - Self(u16::from_be_bytes(bytes)) + f16::from_be_bytes(bytes) } #[inline] fn from_le_bytes(bytes: Self::Bytes) -> Self { - Self(u16::from_le_bytes(bytes)) + f16::from_le_bytes(bytes) } } @@ -627,11 +470,14 @@ mod test { // diff must be <= 4 * EPSILON, as 7 has two more significant bits than 1 assert!(diff <= 4.0 * f16::EPSILON.to_f32()); - assert_eq!(f16(0x0000_0001).to_f32(), 2.0f32.powi(-24)); - assert_eq!(f16(0x0000_0005).to_f32(), 5.0 * 2.0f32.powi(-24)); + assert_eq!(f16::from_bits(0x0000_0001).to_f32(), 2.0f32.powi(-24)); + assert_eq!(f16::from_bits(0x0000_0005).to_f32(), 5.0 * 2.0f32.powi(-24)); - assert_eq!(f16(0x0000_0001), f16::from_f32(2.0f32.powi(-24))); - assert_eq!(f16(0x0000_0005), f16::from_f32(5.0 * 2.0f32.powi(-24))); + assert_eq!(f16::from_bits(0x0000_0001), f16::from_f32(2.0f32.powi(-24))); + assert_eq!( + f16::from_bits(0x0000_0005), + f16::from_f32(5.0 * 2.0f32.powi(-24)) + ); assert_eq!(format!("{}", f16::from_f32(7.0)), "7".to_string()); assert_eq!(format!("{:?}", f16::from_f32(7.0)), "7.0".to_string());