From f8d2fdf02b09c594534cc19a47162aed6eaa0b1c Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Thu, 21 Apr 2016 21:35:39 +0200 Subject: [PATCH 1/8] specialize zip: Introduce TrustedRandomAccess trait --- src/libcore/iter_private.rs | 27 +++++++++++++++++++++++++++ src/libcore/lib.rs | 1 + 2 files changed, 28 insertions(+) create mode 100644 src/libcore/iter_private.rs diff --git a/src/libcore/iter_private.rs b/src/libcore/iter_private.rs new file mode 100644 index 0000000000000..effe43cc67cea --- /dev/null +++ b/src/libcore/iter_private.rs @@ -0,0 +1,27 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + + +use iter::ExactSizeIterator; + +/// An iterator whose items are random accessible efficiently +/// +/// # Safety +/// +/// The iterator's .len() and size_hint() must be exact. +/// +/// .get_unchecked() must return distinct mutable references for distinct +/// indices (if applicable), and must return a valid reference if index is in +/// 0..self.len(). +#[doc(hidden)] +pub unsafe trait TrustedRandomAccess : ExactSizeIterator { + unsafe fn get_unchecked(&mut self, i: usize) -> Self::Item; +} + diff --git a/src/libcore/lib.rs b/src/libcore/lib.rs index f2a297b763028..db73f4759a5ff 100644 --- a/src/libcore/lib.rs +++ b/src/libcore/lib.rs @@ -156,4 +156,5 @@ pub mod hash; pub mod fmt; // note: does not need to be public +mod iter_private; mod tuple; From 592eaa5bb39eef95a06ba9ec75c708592287da07 Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Thu, 21 Apr 2016 21:35:39 +0200 Subject: [PATCH 2/8] specialize zip: Implement TrustedRandomAccess for slice iterators --- src/libcore/slice.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/libcore/slice.rs b/src/libcore/slice.rs index b6ae6fde1e35d..727c26ba9abd1 100644 --- a/src/libcore/slice.rs +++ b/src/libcore/slice.rs @@ -50,6 +50,7 @@ use result::Result::{Ok, Err}; use ptr; use mem; use marker::{Copy, Send, Sync, self}; +use iter_private::TrustedRandomAccess; #[repr(C)] struct Repr { @@ -1942,3 +1943,17 @@ macro_rules! impl_marker_for { impl_marker_for!(BytewiseEquality, u8 i8 u16 i16 u32 i32 u64 i64 usize isize char bool); + +#[doc(hidden)] +unsafe impl<'a, T> TrustedRandomAccess for Iter<'a, T> { + unsafe fn get_unchecked(&mut self, i: usize) -> &'a T { + &*self.ptr.offset(i as isize) + } +} + +#[doc(hidden)] +unsafe impl<'a, T> TrustedRandomAccess for IterMut<'a, T> { + unsafe fn get_unchecked(&mut self, i: usize) -> &'a mut T { + &mut *self.ptr.offset(i as isize) + } +} From a8f2e9b3597b4ff5dc6230c327b35a2b0e7122c1 Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Thu, 21 Apr 2016 21:35:39 +0200 Subject: [PATCH 3/8] specialize zip: Specialize .zip() for TrustedRandomAccess iterators This allows common iterator compositions like a.zip(b) where a, b are slice::{Iter, IterMut} compile to *much* better code. --- src/libcore/iter/iterator.rs | 3 +- src/libcore/iter/mod.rs | 139 ++++++++++++++++++++++++++++++----- 2 files changed, 121 insertions(+), 21 deletions(-) diff --git a/src/libcore/iter/iterator.rs b/src/libcore/iter/iterator.rs index 71ca5ccdc8dfb..8fb71295a88a6 100644 --- a/src/libcore/iter/iterator.rs +++ b/src/libcore/iter/iterator.rs @@ -23,6 +23,7 @@ use super::{Chain, Cycle, Cloned, Enumerate, Filter, FilterMap, FlatMap, Fuse, use super::ChainState; use super::{DoubleEndedIterator, ExactSizeIterator, Extend, FromIterator, IntoIterator}; +use super::ZipImpl; fn _assert_is_object_safe(_: &Iterator) {} @@ -383,7 +384,7 @@ pub trait Iterator { fn zip(self, other: U) -> Zip where Self: Sized, U: IntoIterator { - Zip{a: self, b: other.into_iter()} + Zip::new(self, other.into_iter()) } /// Takes a closure and creates an iterator which calls that closure on each diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index ae1e311682617..c07a15ebc06d6 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -302,6 +302,7 @@ use clone::Clone; use cmp; use fmt; +use iter_private::TrustedRandomAccess; use ops::FnMut; use option::Option::{self, Some, None}; use usize; @@ -622,7 +623,9 @@ impl DoubleEndedIterator for Chain where #[stable(feature = "rust1", since = "1.0.0")] pub struct Zip { a: A, - b: B + b: B, + index: usize, + len: usize, } #[stable(feature = "rust1", since = "1.0.0")] @@ -631,29 +634,13 @@ impl Iterator for Zip where A: Iterator, B: Iterator type Item = (A::Item, B::Item); #[inline] - fn next(&mut self) -> Option<(A::Item, B::Item)> { - self.a.next().and_then(|x| { - self.b.next().and_then(|y| { - Some((x, y)) - }) - }) + fn next(&mut self) -> Option { + ZipImpl::next(self) } #[inline] fn size_hint(&self) -> (usize, Option) { - let (a_lower, a_upper) = self.a.size_hint(); - let (b_lower, b_upper) = self.b.size_hint(); - - let lower = cmp::min(a_lower, b_lower); - - let upper = match (a_upper, b_upper) { - (Some(x), Some(y)) => Some(cmp::min(x,y)), - (Some(x), None) => Some(x), - (None, Some(y)) => Some(y), - (None, None) => None - }; - - (lower, upper) + ZipImpl::size_hint(self) } } @@ -664,6 +651,51 @@ impl DoubleEndedIterator for Zip where { #[inline] fn next_back(&mut self) -> Option<(A::Item, B::Item)> { + ZipImpl::next_back(self) + } +} + +// Zip specialization trait +#[doc(hidden)] +trait ZipImpl { + type Item; + fn new(a: A, b: B) -> Self; + fn next(&mut self) -> Option; + fn size_hint(&self) -> (usize, Option); + fn next_back(&mut self) -> Option + where A: DoubleEndedIterator + ExactSizeIterator, + B: DoubleEndedIterator + ExactSizeIterator; +} + +// General Zip impl +#[doc(hidden)] +impl ZipImpl for Zip + where A: Iterator, B: Iterator +{ + type Item = (A::Item, B::Item); + default fn new(a: A, b: B) -> Self { + Zip { + a: a, + b: b, + index: 0, // not used in general case + len: 0, + } + } + + #[inline] + default fn next(&mut self) -> Option<(A::Item, B::Item)> { + self.a.next().and_then(|x| { + self.b.next().and_then(|y| { + Some((x, y)) + }) + }) + } + + #[inline] + default fn next_back(&mut self) -> Option<(A::Item, B::Item)> + where A: DoubleEndedIterator + ExactSizeIterator, + B: DoubleEndedIterator + ExactSizeIterator + { let a_sz = self.a.len(); let b_sz = self.b.len(); if a_sz != b_sz { @@ -680,6 +712,73 @@ impl DoubleEndedIterator for Zip where _ => unreachable!(), } } + + #[inline] + default fn size_hint(&self) -> (usize, Option) { + let (a_lower, a_upper) = self.a.size_hint(); + let (b_lower, b_upper) = self.b.size_hint(); + + let lower = cmp::min(a_lower, b_lower); + + let upper = match (a_upper, b_upper) { + (Some(x), Some(y)) => Some(cmp::min(x,y)), + (Some(x), None) => Some(x), + (None, Some(y)) => Some(y), + (None, None) => None + }; + + (lower, upper) + } +} + +#[doc(hidden)] +impl ZipImpl for Zip + where A: TrustedRandomAccess, B: TrustedRandomAccess +{ + fn new(a: A, b: B) -> Self { + let len = cmp::min(a.len(), b.len()); + Zip { + a: a, + b: b, + index: 0, + len: len, + } + } + + #[inline] + fn next(&mut self) -> Option<(A::Item, B::Item)> { + if self.index < self.len { + let i = self.index; + self.index += 1; + unsafe { + Some((self.a.get_unchecked(i), self.b.get_unchecked(i))) + } + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len - self.index; + (len, Some(len)) + } + + #[inline] + fn next_back(&mut self) -> Option<(A::Item, B::Item)> + where A: DoubleEndedIterator + ExactSizeIterator, + B: DoubleEndedIterator + ExactSizeIterator + { + if self.index < self.len { + self.len -= 1; + let i = self.len; + unsafe { + Some((self.a.get_unchecked(i), self.b.get_unchecked(i))) + } + } else { + None + } + } } #[stable(feature = "rust1", since = "1.0.0")] From 515c4d3c1e6c38ff2d8bd3b3b3e5bd8b6b4c2322 Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Thu, 21 Apr 2016 21:35:39 +0200 Subject: [PATCH 4/8] specialize zip: TrustedRandomAccess for Zip --- src/libcore/iter/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index c07a15ebc06d6..3ca37c4090387 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -785,6 +785,17 @@ impl ZipImpl for Zip impl ExactSizeIterator for Zip where A: ExactSizeIterator, B: ExactSizeIterator {} +#[doc(hidden)] +unsafe impl TrustedRandomAccess for Zip + where A: TrustedRandomAccess, + B: TrustedRandomAccess, +{ + unsafe fn get_unchecked(&mut self, i: usize) -> (A::Item, B::Item) { + (self.a.get_unchecked(i), self.b.get_unchecked(i)) + } + +} + /// An iterator that maps the values of `iter` with `f`. /// /// This `struct` is created by the [`map()`] method on [`Iterator`]. See its From 13f8f40961d4e2166a80699a2e21a5dbf2211ecb Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Thu, 21 Apr 2016 21:35:39 +0200 Subject: [PATCH 5/8] specialize zip: TrustedRandomAccess for Enumerate --- src/libcore/iter/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index 3ca37c4090387..1a2d8e57e8684 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -1092,6 +1092,15 @@ impl DoubleEndedIterator for Enumerate where #[stable(feature = "rust1", since = "1.0.0")] impl ExactSizeIterator for Enumerate where I: ExactSizeIterator {} +#[doc(hidden)] +unsafe impl TrustedRandomAccess for Enumerate + where I: TrustedRandomAccess +{ + unsafe fn get_unchecked(&mut self, i: usize) -> (usize, I::Item) { + (self.count + i, self.iter.get_unchecked(i)) + } +} + /// An iterator with a `peek()` that returns an optional reference to the next /// element. /// From c2ef20fd38c5d52e1cf9416606140012ffa20217 Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Thu, 21 Apr 2016 21:35:39 +0200 Subject: [PATCH 6/8] specialize zip: Add codegen test --- src/test/codegen/zip.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/test/codegen/zip.rs diff --git a/src/test/codegen/zip.rs b/src/test/codegen/zip.rs new file mode 100644 index 0000000000000..6c956364bf80f --- /dev/null +++ b/src/test/codegen/zip.rs @@ -0,0 +1,22 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -C no-prepopulate-passes -O + +#![crate_type = "lib"] + +// CHECK-LABEL: @zip_copy +#[no_mangle] +pub fn zip_copy(xs: &[u8], ys: &mut [u8]) { +// CHECK: memcpy + for (x, y) in xs.iter().zip(ys) { + *y = *x; + } +} From 85cd49fc39532a6ad210a4b93dfb59e6fdabdbbe Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Thu, 21 Apr 2016 21:35:39 +0200 Subject: [PATCH 7/8] specialize zip: Add benchmarks --- src/libcoretest/iter.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libcoretest/iter.rs b/src/libcoretest/iter.rs index 54fca291e5ec2..a2848faa105e9 100644 --- a/src/libcoretest/iter.rs +++ b/src/libcoretest/iter.rs @@ -13,6 +13,7 @@ use core::{i8, i16, isize}; use core::usize; use test::Bencher; +use test::black_box; #[test] fn test_lt() { @@ -1030,3 +1031,33 @@ fn bench_max(b: &mut Bencher) { it.map(scatter).max() }) } + +pub fn copy_zip(xs: &[u8], ys: &mut [u8]) { + for (a, b) in ys.iter_mut().zip(xs) { + *a = *b; + } +} + +pub fn add_zip(xs: &[f32], ys: &mut [f32]) { + for (a, b) in ys.iter_mut().zip(xs) { + *a += *b; + } +} + +#[bench] +fn bench_zip_copy(b: &mut Bencher) { + let source = vec![0u8; 16 * 1024]; + let mut dst = black_box(vec![0u8; 16 * 1024]); + b.iter(|| { + copy_zip(&source, &mut dst) + }) +} + +#[bench] +fn bench_zip_add(b: &mut Bencher) { + let source = vec![1.; 16 * 1024]; + let mut dst = vec![0.; 16 * 1024]; + b.iter(|| { + add_zip(&source, &mut dst) + }); +} From 5df05c6e221b718e82c87003e8b078a9cdbefb70 Mon Sep 17 00:00:00 2001 From: Ulrik Sverdrup Date: Fri, 22 Apr 2016 18:47:42 +0200 Subject: [PATCH 8/8] specialize zip: Use associated type for specialized zip struct data The associated type must be 'static to avoid dropck related errors. --- src/libcore/iter/mod.rs | 52 ++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index 1a2d8e57e8684..b866655bbd53d 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -301,6 +301,7 @@ use clone::Clone; use cmp; +use default::Default; use fmt; use iter_private::TrustedRandomAccess; use ops::FnMut; @@ -624,8 +625,7 @@ impl DoubleEndedIterator for Chain where pub struct Zip { a: A, b: B, - index: usize, - len: usize, + spec: <(A, B) as ZipImplData>::Data, } #[stable(feature = "rust1", since = "1.0.0")] @@ -667,6 +667,17 @@ trait ZipImpl { B: DoubleEndedIterator + ExactSizeIterator; } +// Zip specialization data members +#[doc(hidden)] +trait ZipImplData { + type Data: 'static + Clone + Default + fmt::Debug; +} + +#[doc(hidden)] +impl ZipImplData for T { + default type Data = (); +} + // General Zip impl #[doc(hidden)] impl ZipImpl for Zip @@ -677,8 +688,7 @@ impl ZipImpl for Zip Zip { a: a, b: b, - index: 0, // not used in general case - len: 0, + spec: Default::default(), // unused } } @@ -731,6 +741,20 @@ impl ZipImpl for Zip } } +#[doc(hidden)] +#[derive(Default, Debug, Clone)] +struct ZipImplFields { + index: usize, + len: usize, +} + +#[doc(hidden)] +impl ZipImplData for (A, B) + where A: TrustedRandomAccess, B: TrustedRandomAccess +{ + type Data = ZipImplFields; +} + #[doc(hidden)] impl ZipImpl for Zip where A: TrustedRandomAccess, B: TrustedRandomAccess @@ -740,16 +764,18 @@ impl ZipImpl for Zip Zip { a: a, b: b, - index: 0, - len: len, + spec: ZipImplFields { + index: 0, + len: len, + } } } #[inline] fn next(&mut self) -> Option<(A::Item, B::Item)> { - if self.index < self.len { - let i = self.index; - self.index += 1; + if self.spec.index < self.spec.len { + let i = self.spec.index; + self.spec.index += 1; unsafe { Some((self.a.get_unchecked(i), self.b.get_unchecked(i))) } @@ -760,7 +786,7 @@ impl ZipImpl for Zip #[inline] fn size_hint(&self) -> (usize, Option) { - let len = self.len - self.index; + let len = self.spec.len - self.spec.index; (len, Some(len)) } @@ -769,9 +795,9 @@ impl ZipImpl for Zip where A: DoubleEndedIterator + ExactSizeIterator, B: DoubleEndedIterator + ExactSizeIterator { - if self.index < self.len { - self.len -= 1; - let i = self.len; + if self.spec.index < self.spec.len { + self.spec.len -= 1; + let i = self.spec.len; unsafe { Some((self.a.get_unchecked(i), self.b.get_unchecked(i))) }