From cf65417c88752670b56acbad17d8405b3bf01c45 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 12 Mar 2023 04:03:29 -0700 Subject: [PATCH 1/8] Inline currently certain functions --- crates/bevy_ecs/src/component.rs | 5 ++++- crates/bevy_ecs/src/entity/mod.rs | 2 ++ crates/bevy_ecs/src/storage/blob_vec.rs | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/bevy_ecs/src/component.rs b/crates/bevy_ecs/src/component.rs index a7c090e0f4e42..7e3915b730726 100644 --- a/crates/bevy_ecs/src/component.rs +++ b/crates/bevy_ecs/src/component.rs @@ -601,6 +601,7 @@ impl Tick { /// ticks are periodically scanned to ensure their relative values are below this. pub const MAX: Self = Self::new(MAX_CHANGE_AGE); + #[inline] pub const fn new(tick: u32) -> Self { Self { tick } } @@ -617,10 +618,10 @@ impl Tick { self.tick = tick; } - #[inline] /// Returns `true` if this `Tick` occurred since the system's `last_run`. /// /// `this_run` is the current tick of the system, used as a reference to help deal with wraparound. + #[inline] pub fn is_newer_than(self, last_run: Tick, this_run: Tick) -> bool { // This works even with wraparound because the world tick (`this_run`) is always "newer" than // `last_run` and `self.tick`, and we scan periodically to clamp `ComponentTicks` values @@ -634,6 +635,7 @@ impl Tick { } /// Returns a change tick representing the relationship between `self` and `other`. + #[inline] pub(crate) fn relative_to(self, other: Self) -> Self { let tick = self.tick.wrapping_sub(other.tick); Self { tick } @@ -642,6 +644,7 @@ impl Tick { /// Wraps this change tick's value if it exceeds [`Tick::MAX`]. /// /// Returns `true` if wrapping was performed. Otherwise, returns `false`. + #[inline] pub(crate) fn check_tick(&mut self, tick: Tick) -> bool { let age = tick.relative_to(*self); // This comparison assumes that `age` has not overflowed `u32::MAX` before, which will be true diff --git a/crates/bevy_ecs/src/entity/mod.rs b/crates/bevy_ecs/src/entity/mod.rs index 1b1afe3151fc6..cea15024b86ae 100644 --- a/crates/bevy_ecs/src/entity/mod.rs +++ b/crates/bevy_ecs/src/entity/mod.rs @@ -570,6 +570,7 @@ impl Entities { /// Returns the location of an [`Entity`]. /// Note: for pending entities, returns `Some(EntityLocation::INVALID)`. + #[inline] pub fn get(&self, entity: Entity) -> Option { if (entity.index as usize) < self.meta.len() { let meta = &self.meta[entity.index as usize]; @@ -591,6 +592,7 @@ impl Entities { /// - `index` must be a valid entity index. /// - `location` must be valid for the entity at `index` or immediately made valid afterwards /// before handing control to unknown code. + #[inline] pub(crate) unsafe fn set(&mut self, index: u32, location: EntityLocation) { // SAFETY: Caller guarantees that `index` a valid entity index self.meta.get_unchecked_mut(index as usize).location = location; diff --git a/crates/bevy_ecs/src/storage/blob_vec.rs b/crates/bevy_ecs/src/storage/blob_vec.rs index 1f7f636fb6c7f..47719db867fa3 100644 --- a/crates/bevy_ecs/src/storage/blob_vec.rs +++ b/crates/bevy_ecs/src/storage/blob_vec.rs @@ -244,6 +244,7 @@ impl BlobVec { /// Newly added items must be immediately populated with valid values and length must be /// increased. For better unwind safety, call [`BlobVec::set_len`] _after_ populating a new /// value. + #[inline] pub unsafe fn set_len(&mut self, len: usize) { debug_assert!(len <= self.capacity()); self.len = len; From 0a1a084f325e31185c1b7bf8d7d8da55c507d9e9 Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 12 Mar 2023 04:21:39 -0700 Subject: [PATCH 2/8] Inline Archetype changes --- crates/bevy_ecs/src/archetype.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/bevy_ecs/src/archetype.rs b/crates/bevy_ecs/src/archetype.rs index d106a83a4fc9f..2b775077722e3 100644 --- a/crates/bevy_ecs/src/archetype.rs +++ b/crates/bevy_ecs/src/archetype.rs @@ -433,6 +433,7 @@ impl Archetype { /// # Safety /// valid component values must be immediately written to the relevant storages /// `table_row` must be valid + #[inline] pub(crate) unsafe fn allocate( &mut self, entity: Entity, @@ -449,6 +450,7 @@ impl Archetype { } } + #[inline] pub(crate) fn reserve(&mut self, additional: usize) { self.entities.reserve(additional); } @@ -458,6 +460,7 @@ impl Archetype { /// /// # Panics /// This function will panic if `index >= self.len()` + #[inline] pub(crate) fn swap_remove(&mut self, row: ArchetypeRow) -> ArchetypeSwapRemoveResult { let is_last = row.index() == self.entities.len() - 1; let entity = self.entities.swap_remove(row.index()); From 7dbe7d871426ffb5e9c25d81355e9c53e51a17fa Mon Sep 17 00:00:00 2001 From: james7132 Date: Sun, 12 Mar 2023 04:25:14 -0700 Subject: [PATCH 3/8] Inline ArchetypeRow::new --- crates/bevy_ecs/src/archetype.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/bevy_ecs/src/archetype.rs b/crates/bevy_ecs/src/archetype.rs index 2b775077722e3..dfac03dc7e0ed 100644 --- a/crates/bevy_ecs/src/archetype.rs +++ b/crates/bevy_ecs/src/archetype.rs @@ -47,6 +47,7 @@ impl ArchetypeRow { pub const INVALID: ArchetypeRow = ArchetypeRow(u32::MAX); /// Creates a `ArchetypeRow`. + #[inline] pub const fn new(index: usize) -> Self { Self(index as u32) } From f565ed3f85c46b404fa1dd8374cae6eed50e775b Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 14 Mar 2023 02:31:18 -0700 Subject: [PATCH 4/8] Inline init_fetch --- crates/bevy_ecs/src/query/fetch.rs | 6 ++++++ crates/bevy_ecs/src/query/filter.rs | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index 3b4a6a076f143..1e34ccb1a48f2 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -539,6 +539,7 @@ unsafe impl WorldQuery for &T { const IS_ARCHETYPAL: bool = true; + #[inline] unsafe fn init_fetch<'w>( world: &'w World, &component_id: &ComponentId, @@ -684,6 +685,7 @@ unsafe impl<'__w, T: Component> WorldQuery for Ref<'__w, T> { const IS_ARCHETYPAL: bool = true; + #[inline] unsafe fn init_fetch<'w>( world: &'w World, &component_id: &ComponentId, @@ -845,6 +847,7 @@ unsafe impl<'__w, T: Component> WorldQuery for &'__w mut T { const IS_ARCHETYPAL: bool = true; + #[inline] unsafe fn init_fetch<'w>( world: &'w World, &component_id: &ComponentId, @@ -989,6 +992,7 @@ unsafe impl WorldQuery for Option { const IS_ARCHETYPAL: bool = T::IS_ARCHETYPAL; + #[inline] unsafe fn init_fetch<'w>( world: &'w World, state: &T::State, @@ -1093,6 +1097,7 @@ macro_rules! impl_tuple_fetch { )*) } + #[inline] #[allow(clippy::unused_unit)] unsafe fn init_fetch<'w>(_world: &'w World, state: &Self::State, _last_run: Tick, _this_run: Tick) -> Self::Fetch<'w> { let ($($name,)*) = state; @@ -1202,6 +1207,7 @@ macro_rules! impl_anytuple_fetch { )*) } + #[inline] #[allow(clippy::unused_unit)] unsafe fn init_fetch<'w>(_world: &'w World, state: &Self::State, _last_run: Tick, _this_run: Tick) -> Self::Fetch<'w> { let ($($name,)*) = state; diff --git a/crates/bevy_ecs/src/query/filter.rs b/crates/bevy_ecs/src/query/filter.rs index 41f2a7941b3dc..580e496972890 100644 --- a/crates/bevy_ecs/src/query/filter.rs +++ b/crates/bevy_ecs/src/query/filter.rs @@ -50,6 +50,7 @@ unsafe impl WorldQuery for With { fn shrink<'wlong: 'wshort, 'wshort>(_: Self::Item<'wlong>) -> Self::Item<'wshort> {} + #[inline] unsafe fn init_fetch(_world: &World, _state: &ComponentId, _last_run: Tick, _this_run: Tick) {} unsafe fn clone_fetch<'w>(_fetch: &Self::Fetch<'w>) -> Self::Fetch<'w> {} @@ -146,6 +147,7 @@ unsafe impl WorldQuery for Without { fn shrink<'wlong: 'wshort, 'wshort>(_: Self::Item<'wlong>) -> Self::Item<'wshort> {} + #[inline] unsafe fn init_fetch(_world: &World, _state: &ComponentId, _last_run: Tick, _this_run: Tick) {} unsafe fn clone_fetch<'w>(_fetch: &Self::Fetch<'w>) -> Self::Fetch<'w> {} @@ -265,6 +267,7 @@ macro_rules! impl_query_filter_tuple { const IS_ARCHETYPAL: bool = true $(&& $filter::IS_ARCHETYPAL)*; + #[inline] unsafe fn init_fetch<'w>(world: &'w World, state: &Self::State, last_run: Tick, this_run: Tick) -> Self::Fetch<'w> { let ($($filter,)*) = state; ($(OrFetch { @@ -420,6 +423,7 @@ macro_rules! impl_tick_filter { item } + #[inline] unsafe fn init_fetch<'w>(world: &'w World, &id: &ComponentId, last_run: Tick, this_run: Tick) -> Self::Fetch<'w> { Self::Fetch::<'w> { table_ticks: None, From 2459a70fbe2c345f4e2fd5c1c4b902d069278157 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 16 Mar 2023 02:27:05 -0700 Subject: [PATCH 5/8] Inline UnsafeWorldCell's functions --- crates/bevy_ecs/src/world/unsafe_world_cell.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/bevy_ecs/src/world/unsafe_world_cell.rs b/crates/bevy_ecs/src/world/unsafe_world_cell.rs index 99231fe3f32ef..3cec3db5c185b 100644 --- a/crates/bevy_ecs/src/world/unsafe_world_cell.rs +++ b/crates/bevy_ecs/src/world/unsafe_world_cell.rs @@ -81,11 +81,13 @@ unsafe impl Sync for UnsafeWorldCell<'_> {} impl<'w> UnsafeWorldCell<'w> { /// Creates a [`UnsafeWorldCell`] that can be used to access everything immutably + #[inline] pub(crate) fn new_readonly(world: &'w World) -> Self { UnsafeWorldCell(world as *const World as *mut World, PhantomData) } /// Creates [`UnsafeWorldCell`] that can be used to access everything mutably + #[inline] pub(crate) fn new_mutable(world: &'w mut World) -> Self { Self(world as *mut World, PhantomData) } @@ -99,6 +101,7 @@ impl<'w> UnsafeWorldCell<'w> { /// - there must be no other borrows on world /// - returned borrow must not be used in any way if the world is accessed /// through other means than the `&mut World` after this call. + #[inline] pub unsafe fn world_mut(self) -> &'w mut World { // SAFETY: // - caller ensures the created `&mut World` is the only borrow of world @@ -112,6 +115,7 @@ impl<'w> UnsafeWorldCell<'w> { /// - must have permission to access the whole world immutably /// - there must be no live exclusive borrows on world data /// - there must be no live exclusive borrow of world + #[inline] pub unsafe fn world(self) -> &'w World { // SAFETY: // - caller ensures there is no `&mut World` this makes it okay to make a `&World` @@ -128,6 +132,7 @@ impl<'w> UnsafeWorldCell<'w> { /// /// # Safety /// - must only be used to access world metadata + #[inline] pub unsafe fn world_metadata(self) -> &'w World { // SAFETY: caller ensures that returned reference is not used to violate aliasing rules unsafe { self.unsafe_world() } @@ -144,6 +149,7 @@ impl<'w> UnsafeWorldCell<'w> { /// # Safety /// - must not be used in a way that would conflict with any /// live exclusive borrows on world data + #[inline] unsafe fn unsafe_world(self) -> &'w World { // SAFETY: // - caller ensures that the returned `&World` is not used in a way that would conflict @@ -242,6 +248,7 @@ impl<'w> UnsafeWorldCell<'w> { /// Retrieves an [`UnsafeEntityCell`] that exposes read and write operations for the given `entity`. /// Similar to the [`UnsafeWorldCell`], you are in charge of making sure that no aliasing rules are violated. + #[inline] pub fn get_entity(self, entity: Entity) -> Option> { let location = self.entities().get(entity)?; Some(UnsafeEntityCell::new(self, entity, location)) From 5a0d6eaa3870fd9806e58c8fa00642f354fb4c2d Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 16 Mar 2023 02:53:03 -0700 Subject: [PATCH 6/8] Inline SparseSetIndex functions --- crates/bevy_ecs/src/bundle.rs | 1 + crates/bevy_ecs/src/component.rs | 1 + crates/bevy_ecs/src/entity/mod.rs | 2 ++ crates/bevy_ecs/src/storage/sparse_set.rs | 2 ++ crates/bevy_ecs/src/world/identifier.rs | 1 + 5 files changed, 7 insertions(+) diff --git a/crates/bevy_ecs/src/bundle.rs b/crates/bevy_ecs/src/bundle.rs index ae37fdcdccd0e..276074c6fae82 100644 --- a/crates/bevy_ecs/src/bundle.rs +++ b/crates/bevy_ecs/src/bundle.rs @@ -255,6 +255,7 @@ impl SparseSetIndex for BundleId { self.index() } + #[inline] fn get_sparse_set_index(value: usize) -> Self { Self(value) } diff --git a/crates/bevy_ecs/src/component.rs b/crates/bevy_ecs/src/component.rs index 7e3915b730726..43f5b22f31f1a 100644 --- a/crates/bevy_ecs/src/component.rs +++ b/crates/bevy_ecs/src/component.rs @@ -283,6 +283,7 @@ impl SparseSetIndex for ComponentId { self.index() } + #[inline] fn get_sparse_set_index(value: usize) -> Self { Self(value) } diff --git a/crates/bevy_ecs/src/entity/mod.rs b/crates/bevy_ecs/src/entity/mod.rs index cea15024b86ae..ddd1e20875bda 100644 --- a/crates/bevy_ecs/src/entity/mod.rs +++ b/crates/bevy_ecs/src/entity/mod.rs @@ -234,10 +234,12 @@ impl fmt::Debug for Entity { } impl SparseSetIndex for Entity { + #[inline] fn sparse_set_index(&self) -> usize { self.index() as usize } + #[inline] fn get_sparse_set_index(value: usize) -> Self { Entity::from_raw(value as u32) } diff --git a/crates/bevy_ecs/src/storage/sparse_set.rs b/crates/bevy_ecs/src/storage/sparse_set.rs index d15aca7c4ae84..7a19d8f744356 100644 --- a/crates/bevy_ecs/src/storage/sparse_set.rs +++ b/crates/bevy_ecs/src/storage/sparse_set.rs @@ -546,10 +546,12 @@ pub trait SparseSetIndex: Clone + PartialEq + Eq + Hash { macro_rules! impl_sparse_set_index { ($($ty:ty),+) => { $(impl SparseSetIndex for $ty { + #[inline] fn sparse_set_index(&self) -> usize { *self as usize } + #[inline] fn get_sparse_set_index(value: usize) -> Self { value as $ty } diff --git a/crates/bevy_ecs/src/world/identifier.rs b/crates/bevy_ecs/src/world/identifier.rs index 55e2b6a5fa350..854e2f550805b 100644 --- a/crates/bevy_ecs/src/world/identifier.rs +++ b/crates/bevy_ecs/src/world/identifier.rs @@ -69,6 +69,7 @@ impl SparseSetIndex for WorldId { self.0 } + #[inline] fn get_sparse_set_index(value: usize) -> Self { Self(value) } From e66c63c1f0a03d58b3288a437c9c8826967444d4 Mon Sep 17 00:00:00 2001 From: james7132 Date: Thu, 16 Mar 2023 02:55:07 -0700 Subject: [PATCH 7/8] Inline World::as_unsafe_world_cell variants --- crates/bevy_ecs/src/storage/sparse_set.rs | 1 + crates/bevy_ecs/src/world/mod.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/crates/bevy_ecs/src/storage/sparse_set.rs b/crates/bevy_ecs/src/storage/sparse_set.rs index 7a19d8f744356..c124911b27104 100644 --- a/crates/bevy_ecs/src/storage/sparse_set.rs +++ b/crates/bevy_ecs/src/storage/sparse_set.rs @@ -589,6 +589,7 @@ impl SparseSets { } /// Gets a reference to the [`ComponentSparseSet`] of a [`ComponentId`]. + #[inline] pub fn get(&self, component_id: ComponentId) -> Option<&ComponentSparseSet> { self.sets.get(component_id) } diff --git a/crates/bevy_ecs/src/world/mod.rs b/crates/bevy_ecs/src/world/mod.rs index 0832141f4bf02..8d8d73c1ebabe 100644 --- a/crates/bevy_ecs/src/world/mod.rs +++ b/crates/bevy_ecs/src/world/mod.rs @@ -107,11 +107,13 @@ impl World { } /// Creates a new [`UnsafeWorldCell`] view with complete read+write access. + #[inline] pub fn as_unsafe_world_cell(&mut self) -> UnsafeWorldCell<'_> { UnsafeWorldCell::new_mutable(self) } /// Creates a new [`UnsafeWorldCell`] view with only read access to everything. + #[inline] pub fn as_unsafe_world_cell_readonly(&self) -> UnsafeWorldCell<'_> { UnsafeWorldCell::new_readonly(self) } @@ -119,6 +121,7 @@ impl World { /// Creates a new [`UnsafeWorldCell`] with read+write access from a [&World](World). /// This is only a temporary measure until every `&World` that is semantically a [`UnsafeWorldCell`] /// has been replaced. + #[inline] pub(crate) fn as_unsafe_world_cell_migration_internal(&self) -> UnsafeWorldCell<'_> { UnsafeWorldCell::new_readonly(self) } From bdb4218d1bcc388c6fd8ac958b6763df622db5af Mon Sep 17 00:00:00 2001 From: james7132 Date: Tue, 11 Apr 2023 14:06:35 -0700 Subject: [PATCH 8/8] Inline UnsafeEntityCell::new --- crates/bevy_ecs/src/world/unsafe_world_cell.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/bevy_ecs/src/world/unsafe_world_cell.rs b/crates/bevy_ecs/src/world/unsafe_world_cell.rs index fbbdc501fa4f8..b28dbc76d1254 100644 --- a/crates/bevy_ecs/src/world/unsafe_world_cell.rs +++ b/crates/bevy_ecs/src/world/unsafe_world_cell.rs @@ -509,6 +509,7 @@ pub struct UnsafeEntityCell<'w> { } impl<'w> UnsafeEntityCell<'w> { + #[inline] pub(crate) fn new( world: UnsafeWorldCell<'w>, entity: Entity,