Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Add BigNumber Type and syscalls for BPF programs #17082

Closed
wants to merge 59 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
59 commits
Select commit Hold shift + click to select a range
8fea0cc
Add BigNumber and syscalls
FrankC01 May 6, 2021
2cd8645
Removed extra whitespaces
FrankC01 May 6, 2021
03af751
Updated for review comments
FrankC01 May 11, 2021
440aa5e
Merge branch 'master' of https://github.com/solana-labs/solana
FrankC01 May 11, 2021
3035860
BPF test bignum
FrankC01 May 12, 2021
56e833d
Refresh
FrankC01 May 12, 2021
2b1f3e1
BigNumber BPF testing
FrankC01 May 12, 2021
a63450b
Merge remote-tracking branch 'upstream/master'
FrankC01 May 13, 2021
01ed0b9
Addressing white spaces and removed blake3 hash, hash-to-prime and ha…
FrankC01 May 13, 2021
61b56c0
Remove println statements
FrankC01 May 13, 2021
2d8b538
Fix check failures
FrankC01 May 13, 2021
2fffc0a
Fix unused var check failures
FrankC01 May 13, 2021
4ce98c8
Removed hkdf as hashes removed from bignum and correct BPF test
FrankC01 May 13, 2021
a1725b1
Removed hkdf ref
FrankC01 May 13, 2021
ff09a57
Fixes for Cargo.lock
FrankC01 May 13, 2021
952806b
Merge remote-tracking branch 'upstream/master'
FrankC01 May 14, 2021
1d13a79
Fixed BigNumToString syscall buffer overflow
FrankC01 May 14, 2021
6011f78
Fix formatting
FrankC01 May 14, 2021
477359b
Merge remote-tracking branch 'upstream/master'
FrankC01 May 14, 2021
919cdfc
Merge remote-tracking branch 'upstream/master'
FrankC01 May 15, 2021
ca1944e
BigNumber new and from_u32 using vectors instead of Box<BigNum>
FrankC01 May 15, 2021
f5eafc7
BigNumber::from_dec_str using vectors instead of Box
FrankC01 May 15, 2021
3168320
Merge remote-tracking branch 'upstream/master'
FrankC01 May 16, 2021
9b47c24
Merge branch 'master' into bignum-to-vec
FrankC01 May 16, 2021
fba2649
BigNumber::add implemented with vectors
FrankC01 May 16, 2021
cbd738b
Formatting
FrankC01 May 16, 2021
22107d0
BigNumber simple maths using vectors instead of Box
FrankC01 May 16, 2021
47c57b1
BigNumber complex maths-part1 using vector instead of Box
FrankC01 May 17, 2021
5495cc9
Merge remote-tracking branch 'upstream/master'
FrankC01 May 17, 2021
203e619
Merge branch 'master' into bignum-to-vec
FrankC01 May 17, 2021
5e2d397
BigNumber complete conversion to vector from Box
FrankC01 May 17, 2021
069c674
Remove println call
FrankC01 May 17, 2021
7e7bebb
Remove dead code
FrankC01 May 17, 2021
ed924a5
Replace Vec with vec macro as per clippy
FrankC01 May 17, 2021
b173d59
Modernize incrementing by 1
FrankC01 May 17, 2021
aaf90d8
Modernize boolean test
FrankC01 May 17, 2021
0c9258e
remove unused imports
FrankC01 May 17, 2021
0d8a51b
Added assertions to constructor tests
FrankC01 May 17, 2021
8d02300
Merge remote-tracking branch 'upstream/master'
FrankC01 May 18, 2021
af54cae
Fix logging to include sign and refine result allocations
FrankC01 May 18, 2021
35d5c62
Cleanup tests
FrankC01 May 18, 2021
4b7925c
Updated logging for non-bpf execution
FrankC01 May 18, 2021
176e4cc
Merge remote-tracking branch 'upstream/master'
FrankC01 May 19, 2021
bc7eb30
Doubled BigNumber syscall execution base units
FrankC01 May 19, 2021
4f96707
Merged with upstream
FrankC01 May 19, 2021
1f422ab
Fix deadcode errors
FrankC01 May 19, 2021
7a00993
Fix assertion expression
FrankC01 May 19, 2021
ed9cbd5
move Ancestors to its own module (#17316)
jeffwashington May 19, 2021
c20b27b
shink all in parallel on startup (#17308)
jeffwashington May 19, 2021
f1b4a0a
Fix typo (#17326)
ulrichstark May 19, 2021
b5302e7
add calc_stored_meta_us metric (#17318)
jeffwashington May 19, 2021
e7073ec
adds gossip metrics for number of staked nodes (#17330)
behzadnouri May 19, 2021
477898f
Optimize aligned memory used by the runtime (#17324)
jackcmay May 19, 2021
13b032b
removes manual trait impl for contact-info (#17332)
behzadnouri May 19, 2021
32ec834
generate_index inserts ideal initial data (#17247)
jeffwashington May 19, 2021
a544010
don't log shrink metrics on first call (#17328)
jeffwashington May 20, 2021
2ae57c1
Bump bpf-tools version to 1.9
dmakarov May 18, 2021
abe0064
syscall corrections
FrankC01 May 20, 2021
86c9814
Hard reset
FrankC01 May 20, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
generate_index inserts ideal initial data (#17247)
* improve insert into map initially

* rework towards single code path

* rename

* update test
  • Loading branch information
jeffwashington authored May 19, 2021
commit 32ec8341f9fe1328a1ca4de518bde1c11a379656
244 changes: 197 additions & 47 deletions runtime/src/accounts_index.rs
Original file line number Diff line number Diff line change
@@ -179,6 +179,18 @@ impl<T: 'static + Clone + IsCached> WriteAccountMapEntry<T> {
&self.borrow_owned_entry_contents().ref_count
}

// create an entry that is equivalent to this process:
// 1. new empty (refcount=0, slot_list={})
// 2. update(slot, account_info)
// This code is called when the first entry [ie. (slot,account_info)] for a pubkey is inserted into the index.
pub fn new_entry_after_update(slot: Slot, account_info: &T) -> AccountMapEntry<T> {
let ref_count = if account_info.is_cached() { 0 } else { 1 };
Arc::new(AccountMapEntryInner {
ref_count: AtomicU64::new(ref_count),
slot_list: RwLock::new(vec![(slot, account_info.clone())]),
})
}

// Try to update an item in the slot list the given `slot` If an item for the slot
// already exists in the list, remove the older item, add it to `reclaims`, and insert
// the new item.
@@ -496,7 +508,7 @@ pub trait ZeroLamport {
}

type MapType<T> = AccountMap<Pubkey, AccountMapEntry<T>>;
type ReadWriteLockMapType<'a, T> = RwLockWriteGuard<'a, AccountMap<Pubkey, AccountMapEntry<T>>>;
type AccountMapsWriteLock<'a, T> = RwLockWriteGuard<'a, AccountMap<Pubkey, AccountMapEntry<T>>>;

#[derive(Debug)]
pub struct AccountsIndex<T> {
@@ -842,47 +854,55 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
.map(WriteAccountMapEntry::from_account_map_entry)
}

fn new_entry() -> AccountMapEntry<T> {
Arc::new(AccountMapEntryInner {
ref_count: AtomicU64::new(0),
slot_list: RwLock::new(SlotList::with_capacity(1)),
})
}

fn insert_new_entry_if_missing(&self, pubkey: &Pubkey) -> (WriteAccountMapEntry<T>, bool) {
let new_entry = Self::new_entry();
let mut w_account_maps = self.get_account_maps_write_lock();
self.insert_new_entry_if_missing_with_lock(pubkey, &mut w_account_maps, new_entry)
fn insert_new_entry_if_missing(
&self,
pubkey: &Pubkey,
slot: Slot,
info: &T,
w_account_maps: Option<&mut AccountMapsWriteLock<T>>,
) -> Option<WriteAccountMapEntry<T>> {
let new_entry = WriteAccountMapEntry::new_entry_after_update(slot, info);
match w_account_maps {
Some(w_account_maps) => {
self.insert_new_entry_if_missing_with_lock(pubkey, w_account_maps, new_entry)
}
None => {
let mut w_account_maps = self.get_account_maps_write_lock();
self.insert_new_entry_if_missing_with_lock(pubkey, &mut w_account_maps, new_entry)
}
}
}

// return None if item was created new
// if entry for pubkey already existed, return Some(entry). Caller needs to call entry.update.
fn insert_new_entry_if_missing_with_lock(
&self,
pubkey: &Pubkey,
w_account_maps: &mut ReadWriteLockMapType<T>,
w_account_maps: &mut AccountMapsWriteLock<T>,
new_entry: AccountMapEntry<T>,
) -> (WriteAccountMapEntry<T>, bool) {
) -> Option<WriteAccountMapEntry<T>> {
let mut is_newly_inserted = false;
let account_entry = w_account_maps.entry(*pubkey).or_insert_with(|| {
is_newly_inserted = true;
new_entry
});
let w_account_entry = WriteAccountMapEntry::from_account_map_entry(account_entry.clone());
(w_account_entry, is_newly_inserted)
if is_newly_inserted {
None
} else {
Some(WriteAccountMapEntry::from_account_map_entry(
account_entry.clone(),
))
}
}

fn get_account_write_entry_else_create(
&self,
pubkey: &Pubkey,
) -> (WriteAccountMapEntry<T>, bool) {
let mut w_account_entry = self.get_account_write_entry(pubkey);
let mut is_newly_inserted = false;
if w_account_entry.is_none() {
let entry_is_new = self.insert_new_entry_if_missing(pubkey);
w_account_entry = Some(entry_is_new.0);
is_newly_inserted = entry_is_new.1;
}

(w_account_entry.unwrap(), is_newly_inserted)
slot: Slot,
info: &T,
) -> Option<WriteAccountMapEntry<T>> {
let w_account_entry = self.get_account_write_entry(pubkey);
w_account_entry.or_else(|| self.insert_new_entry_if_missing(pubkey, slot, info, None))
}

pub fn handle_dead_keys(
@@ -1159,7 +1179,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
}
}

pub(crate) fn get_account_maps_write_lock(&self) -> ReadWriteLockMapType<T> {
pub(crate) fn get_account_maps_write_lock(&self) -> AccountMapsWriteLock<T> {
self.account_maps.write().unwrap()
}

@@ -1173,15 +1193,16 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
pubkey: &Pubkey,
account_info: T,
reclaims: &mut SlotList<T>,
w_account_maps: &mut ReadWriteLockMapType<T>,
w_account_maps: &mut AccountMapsWriteLock<T>,
) {
let new_entry = Self::new_entry();
let (mut w_account_entry, _is_new) =
self.insert_new_entry_if_missing_with_lock(pubkey, w_account_maps, new_entry);
let account_entry =
self.insert_new_entry_if_missing(pubkey, slot, &account_info, Some(w_account_maps));
if account_info.is_zero_lamport() {
self.zero_lamport_pubkeys.insert(*pubkey);
}
w_account_entry.update(slot, account_info, reclaims);
if let Some(mut w_account_entry) = account_entry {
w_account_entry.update(slot, account_info, reclaims);
}
}

// Updates the given pubkey at the given slot with the new account information.
@@ -1198,8 +1219,8 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
reclaims: &mut SlotList<T>,
) -> bool {
let is_newly_inserted = {
let (mut w_account_entry, is_newly_inserted) =
self.get_account_write_entry_else_create(pubkey);
let w_account_entry =
self.get_account_write_entry_else_create(pubkey, slot, &account_info);
// We don't atomically update both primary index and secondary index together.
// This certainly creates small time window with inconsistent state across the two indexes.
// However, this is acceptable because:
@@ -1214,8 +1235,12 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
if account_info.is_zero_lamport() {
self.zero_lamport_pubkeys.insert(*pubkey);
}
w_account_entry.update(slot, account_info, reclaims);
is_newly_inserted
if let Some(mut w_account_entry) = w_account_entry {
w_account_entry.update(slot, account_info, reclaims);
false
} else {
true
}
};
self.update_secondary_indexes(pubkey, account_owner, account_data, account_indexes);
is_newly_inserted
@@ -2269,29 +2294,154 @@ pub mod tests {
assert_eq!(num, 1);
}

#[test]
fn test_new_entry() {
let slot = 0;
// account_info type that IS cached
let account_info = AccountInfoTest::default();

let new_entry = WriteAccountMapEntry::new_entry_after_update(slot, &account_info);
assert_eq!(new_entry.ref_count.load(Ordering::Relaxed), 0);
assert_eq!(new_entry.slot_list.read().unwrap().capacity(), 1);
assert_eq!(
new_entry.slot_list.read().unwrap().to_vec(),
vec![(slot, account_info)]
);

// account_info type that is NOT cached
let account_info = true;

let new_entry = WriteAccountMapEntry::new_entry_after_update(slot, &account_info);
assert_eq!(new_entry.ref_count.load(Ordering::Relaxed), 1);
assert_eq!(new_entry.slot_list.read().unwrap().capacity(), 1);
assert_eq!(
new_entry.slot_list.read().unwrap().to_vec(),
vec![(slot, account_info)]
);
}

fn test_new_entry_code_paths_helper<
T: 'static + Clone + IsCached + ZeroLamport + std::cmp::PartialEq + std::fmt::Debug,
>(
account_infos: [T; 2],
is_cached: bool,
upsert: bool,
) {
let slot0 = 0;
let slot1 = 1;
let key = Keypair::new().pubkey();

let index = AccountsIndex::<T>::default();
let mut gc = Vec::new();

if upsert {
// insert first entry for pubkey. This will use new_entry_after_update and not call update.
index.upsert(
slot0,
&key,
&Pubkey::default(),
&[],
&AccountSecondaryIndexes::default(),
account_infos[0].clone(),
&mut gc,
);
} else {
let mut lock = index.get_account_maps_write_lock();
index.insert_new_if_missing_into_primary_index(
slot0,
&key,
account_infos[0].clone(),
&mut gc,
&mut lock,
);
}
assert!(gc.is_empty());

// verify the added entry matches expected
{
let entry = index.get_account_read_entry(&key).unwrap();
assert_eq!(
entry.ref_count().load(Ordering::Relaxed),
if is_cached { 0 } else { 1 }
);
let expected = vec![(slot0, account_infos[0].clone())];
assert_eq!(entry.slot_list().to_vec(), expected);
let new_entry = WriteAccountMapEntry::new_entry_after_update(slot0, &account_infos[0]);
assert_eq!(
entry.slot_list().to_vec(),
new_entry.slot_list.read().unwrap().to_vec(),
);
}

// insert second entry for pubkey. This will use update and NOT use new_entry_after_update.
if upsert {
index.upsert(
slot1,
&key,
&Pubkey::default(),
&[],
&AccountSecondaryIndexes::default(),
account_infos[1].clone(),
&mut gc,
);
} else {
let mut lock = index.get_account_maps_write_lock();
index.insert_new_if_missing_into_primary_index(
slot1,
&key,
account_infos[1].clone(),
&mut gc,
&mut lock,
);
}
assert!(gc.is_empty());

{
let entry = index.get_account_read_entry(&key).unwrap();
assert_eq!(
entry.ref_count().load(Ordering::Relaxed),
if is_cached { 0 } else { 2 }
);
assert_eq!(
entry.slot_list().to_vec(),
vec![
(slot0, account_infos[0].clone()),
(slot1, account_infos[1].clone())
]
);

let new_entry = WriteAccountMapEntry::new_entry_after_update(slot1, &account_infos[1]);
assert_eq!(entry.slot_list()[1], new_entry.slot_list.read().unwrap()[0],);
}
}

#[test]
fn test_new_entry_and_update_code_paths() {
for is_upsert in &[false, true] {
// account_info type that IS cached
test_new_entry_code_paths_helper([1.0, 2.0], true, *is_upsert);

// account_info type that is NOT cached
test_new_entry_code_paths_helper([true, false], false, *is_upsert);
}
}

#[test]
fn test_insert_with_lock_no_ancestors() {
let key = Keypair::new();
let index = AccountsIndex::<bool>::default();
let mut gc = Vec::new();
let slot = 0;
let account_info = true;

let new_entry = AccountsIndex::new_entry();
assert_eq!(new_entry.ref_count.load(Ordering::Relaxed), 0);
assert!(new_entry.slot_list.read().unwrap().is_empty());
assert_eq!(new_entry.slot_list.read().unwrap().capacity(), 1);
let new_entry = WriteAccountMapEntry::new_entry_after_update(slot, &account_info);
let mut w_account_maps = index.get_account_maps_write_lock();
let (mut write, insert) = index.insert_new_entry_if_missing_with_lock(
let write = index.insert_new_entry_if_missing_with_lock(
&key.pubkey(),
&mut w_account_maps,
new_entry,
);
assert!(insert);
assert!(write.is_none());
drop(w_account_maps);
let account_info = true;
write.update(slot, account_info, &mut gc);
assert!(gc.is_empty());
drop(write);

let mut ancestors = Ancestors::default();
assert!(index.get(&key.pubkey(), Some(&ancestors), None).is_none());