From b831a5e3b1e9865edd291c4e3ce9b9d731d68cca Mon Sep 17 00:00:00 2001 From: Howard Wu <9260812+howardwu@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:42:11 -0700 Subject: [PATCH] Adds quorum threshold checks before signing --- node/narwhal/src/gateway.rs | 12 +- node/narwhal/src/helpers/channels.rs | 21 ++- node/narwhal/src/helpers/committee.rs | 16 +++ node/narwhal/src/helpers/storage.rs | 64 ++++++--- node/narwhal/src/primary.rs | 186 ++++++++++++++++++-------- node/narwhal/src/worker.rs | 10 +- 6 files changed, 224 insertions(+), 85 deletions(-) diff --git a/node/narwhal/src/gateway.rs b/node/narwhal/src/gateway.rs index e2caca9b50..9fafabfd5e 100644 --- a/node/narwhal/src/gateway.rs +++ b/node/narwhal/src/gateway.rs @@ -380,9 +380,15 @@ impl Gateway { let _ = self.primary_sender().tx_batch_certified.send((peer_ip, batch_certified.certificate)).await; Ok(()) } - Event::CertificateRequest(..) | Event::CertificateResponse(..) => { - // Disconnect as the peer is not following the protocol. - bail!("{CONTEXT} Peer '{peer_ip}' is not following the protocol") + Event::CertificateRequest(certificate_request) => { + // Send the certificate request to the primary. + let _ = self.primary_sender().tx_certificate_request.send((peer_ip, certificate_request)).await; + Ok(()) + } + Event::CertificateResponse(certificate_response) => { + // Send the certificate response to the primary. + let _ = self.primary_sender().tx_certificate_response.send((peer_ip, certificate_response)).await; + Ok(()) } Event::ChallengeRequest(..) | Event::ChallengeResponse(..) => { // Disconnect as the peer is not following the protocol. diff --git a/node/narwhal/src/helpers/channels.rs b/node/narwhal/src/helpers/channels.rs index 1b263ef2d4..34f0c3e154 100644 --- a/node/narwhal/src/helpers/channels.rs +++ b/node/narwhal/src/helpers/channels.rs @@ -12,7 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{BatchPropose, BatchSignature, TransmissionRequest, TransmissionResponse}; +use crate::{ + BatchPropose, + BatchSignature, + CertificateRequest, + CertificateResponse, + TransmissionRequest, + TransmissionResponse, +}; use snarkvm::{ console::network::*, ledger::narwhal::{BatchCertificate, Data, TransmissionID}, @@ -32,12 +39,16 @@ pub fn init_primary_channels() -> (PrimarySender, PrimaryReceiver let (tx_batch_propose, rx_batch_propose) = mpsc::channel(MAX_CHANNEL_SIZE); let (tx_batch_signature, rx_batch_signature) = mpsc::channel(MAX_CHANNEL_SIZE); let (tx_batch_certified, rx_batch_certified) = mpsc::channel(MAX_CHANNEL_SIZE); + let (tx_certificate_request, rx_certificate_request) = mpsc::channel(MAX_CHANNEL_SIZE); + let (tx_certificate_response, rx_certificate_response) = mpsc::channel(MAX_CHANNEL_SIZE); let (tx_unconfirmed_solution, rx_unconfirmed_solution) = mpsc::channel(MAX_CHANNEL_SIZE); let (tx_unconfirmed_transaction, rx_unconfirmed_transaction) = mpsc::channel(MAX_CHANNEL_SIZE); let tx_batch_propose = Arc::new(tx_batch_propose); let tx_batch_signature = Arc::new(tx_batch_signature); let tx_batch_certified = Arc::new(tx_batch_certified); + let tx_certificate_request = Arc::new(tx_certificate_request); + let tx_certificate_response = Arc::new(tx_certificate_response); let tx_unconfirmed_solution = Arc::new(tx_unconfirmed_solution); let tx_unconfirmed_transaction = Arc::new(tx_unconfirmed_transaction); @@ -45,6 +56,8 @@ pub fn init_primary_channels() -> (PrimarySender, PrimaryReceiver tx_batch_propose, tx_batch_signature, tx_batch_certified, + tx_certificate_request, + tx_certificate_response, tx_unconfirmed_solution, tx_unconfirmed_transaction, }; @@ -52,6 +65,8 @@ pub fn init_primary_channels() -> (PrimarySender, PrimaryReceiver rx_batch_propose, rx_batch_signature, rx_batch_certified, + rx_certificate_request, + rx_certificate_response, rx_unconfirmed_solution, rx_unconfirmed_transaction, }; @@ -64,6 +79,8 @@ pub struct PrimarySender { pub tx_batch_propose: Arc)>>, pub tx_batch_signature: Arc)>>, pub tx_batch_certified: Arc>)>>, + pub tx_certificate_request: Arc)>>, + pub tx_certificate_response: Arc)>>, pub tx_unconfirmed_solution: Arc, Data>)>>, pub tx_unconfirmed_transaction: Arc>)>>, } @@ -73,6 +90,8 @@ pub struct PrimaryReceiver { pub rx_batch_propose: mpsc::Receiver<(SocketAddr, BatchPropose)>, pub rx_batch_signature: mpsc::Receiver<(SocketAddr, BatchSignature)>, pub rx_batch_certified: mpsc::Receiver<(SocketAddr, Data>)>, + pub rx_certificate_request: mpsc::Receiver<(SocketAddr, CertificateRequest)>, + pub rx_certificate_response: mpsc::Receiver<(SocketAddr, CertificateResponse)>, pub rx_unconfirmed_solution: mpsc::Receiver<(PuzzleCommitment, Data>)>, pub rx_unconfirmed_transaction: mpsc::Receiver<(N::TransactionID, Data>)>, } diff --git a/node/narwhal/src/helpers/committee.rs b/node/narwhal/src/helpers/committee.rs index 1e99618c44..70f804102a 100644 --- a/node/narwhal/src/helpers/committee.rs +++ b/node/narwhal/src/helpers/committee.rs @@ -15,6 +15,7 @@ use snarkvm::console::{prelude::*, types::Address}; use indexmap::IndexMap; +use std::collections::HashSet; #[derive(Clone, Debug, PartialEq)] pub struct Committee { @@ -68,6 +69,21 @@ impl Committee { self.members.contains_key(&address) } + /// Returns `true` if the combined stake for the given addresses reaches the quorum threshold. + /// This method takes in a `HashSet` to guarantee that the given addresses are unique. + pub fn is_quorum_threshold_reached(&self, addresses: &HashSet>) -> Result { + // Compute the combined stake for the given addresses. + let mut stake = 0u64; + for address in addresses { + stake = match stake.checked_add(self.get_stake(*address)) { + Some(stake) => stake, + None => bail!("Overflow when computing combined stake to check quorum threshold"), + }; + } + // Return whether the combined stake reaches the quorum threshold. + Ok(stake >= self.quorum_threshold()?) + } + /// Returns the amount of stake for the given address. pub fn get_stake(&self, address: Address) -> u64 { self.members.get(&address).copied().unwrap_or_default() diff --git a/node/narwhal/src/helpers/storage.rs b/node/narwhal/src/helpers/storage.rs index ccee9190dc..42ee309df9 100644 --- a/node/narwhal/src/helpers/storage.rs +++ b/node/narwhal/src/helpers/storage.rs @@ -234,7 +234,10 @@ impl Storage { /// This method ensures the following invariants: /// - The certificate ID does not already exist in storage. /// - The batch ID does not already exist in storage. - /// - The certificate is well-formed. + /// - All transmissions declared in the certificate exist in storage (up to GC). + /// - All previous certificates declared in the certificate exist in storage (up to GC). + /// - All previous certificates are for the previous round (i.e. round - 1). + /// - The previous certificates reached the quorum threshold (2f+1). pub fn insert_certificate(&self, certificate: BatchCertificate) -> Result<()> { // Retrieve the round. let round = certificate.round(); @@ -255,30 +258,53 @@ impl Storage { } // TODO (howardwu): Ensure the certificate is well-formed. If not, do not store. - // TODO (howardwu): Ensure the round is within range. If not, do not store. // TODO (howardwu): Ensure the address is in the committee of the specified round. If not, do not store. - // TODO (howardwu): Ensure I have all of the transmissions. If not, do not store. - // TODO (howardwu): Ensure I have all of the previous certificates. If not, do not store. - // TODO (howardwu): Ensure the previous certificates are for round-1. If not, do not store. // TODO (howardwu): Ensure the previous certificates have reached 2f+1. If not, do not store. - // Iterate over the transmission IDs. - for transmission_id in certificate.transmission_ids() { - // Ensure storage contains the declared transmission ID. - if !self.contains_transmission(*transmission_id) { - bail!("Missing transmission {transmission_id} for certificate {certificate_id}"); + // Retrieve the GC round. + let gc_round = self.gc_round(); + // Compute the previous round. + let previous_round = round.saturating_sub(1); + + // Check if the previous round is within range of the GC round. + if previous_round > gc_round { + // Ensure the previous round exists in storage. + if !self.contains_round(previous_round) { + bail!("Missing state for the previous round {previous_round} in storage (gc={gc_round})"); + } + } + + // If the certificate's round is greater than the GC round, ensure the transmissions exists. + if round > gc_round { + // Ensure storage contains all declared transmissions (up to GC). + for transmission_id in certificate.transmission_ids() { + // Ensure storage contains the declared transmission ID. + if !self.contains_transmission(*transmission_id) { + bail!("Missing transmission {transmission_id} for certificate in round {round} (gc={gc_round})"); + } } } - // // Ensure storage contains all declared previous certificates (up to GC). - // for previous_certificate_id in certificate.previous_certificate_ids() { - // // If the certificate's round is greater than the GC round, ensure the previous certificate exists. - // if round > self.gc_round() { - // if !self.certificates.read().contains_key(previous_certificate_id) { - // bail!("Missing previous certificate {previous_certificate_id} for certificate {certificate_id}"); - // } - // } - // } + // If the certificate's *previous* round is greater than the GC round, ensure the previous certificates exists. + if previous_round > gc_round { + // Retrieve the committee for the previous round. + let Some(previous_committee) = self.get_committee_for_round(previous_round) else { + bail!("Missing committee for the previous round {previous_round} in storage (gc={gc_round})"); + }; + // Ensure storage contains all declared previous certificates (up to GC). + for previous_certificate_id in certificate.previous_certificate_ids() { + // Retrieve the previous certificate. + let Some(previous_certificate) = self.get_certificate(*previous_certificate_id) else { + bail!("Missing previous certificate for certificate in round {round} (gc={gc_round})"); + }; + // Ensure the previous certificate is for the previous round. + if previous_certificate.round() != previous_round { + bail!( + "Previous certificate for round {previous_round} found in certificate for round {round} (gc={gc_round})" + ); + } + } + } /* Proceed to store the certificate. */ diff --git a/node/narwhal/src/primary.rs b/node/narwhal/src/primary.rs index 1e7ca33e8a..d904b8ef8a 100644 --- a/node/narwhal/src/primary.rs +++ b/node/narwhal/src/primary.rs @@ -46,7 +46,7 @@ use snarkvm::{ use futures::stream::{FuturesUnordered, StreamExt}; use indexmap::IndexMap; use parking_lot::{Mutex, RwLock}; -use std::{future::Future, net::SocketAddr, sync::Arc}; +use std::{collections::HashSet, future::Future, net::SocketAddr, sync::Arc}; use time::OffsetDateTime; use tokio::{sync::oneshot, task::JoinHandle}; @@ -175,20 +175,14 @@ impl Primary { /// 3. Set the batch in the primary. /// 4. Broadcast the batch to all validators for signing. pub fn propose_batch(&self) -> Result<()> { - // Initialize the RNG. - let mut rng = rand::thread_rng(); - // Initialize a map of the transmissions. let mut transmissions = IndexMap::new(); // Drain the workers. for worker in self.workers.read().iter() { // TODO (howardwu): Perform one final filter against the ledger service. - // Transition the worker to the next round, and add their transmissions to the map. transmissions.extend(worker.drain()); } - // Retrieve the private key. - let private_key = self.gateway.account().private_key(); // Retrieve the current round. let round = self.committee.read().round(); // Compute the previous round. @@ -197,43 +191,36 @@ impl Primary { let previous_certificates = self.storage.get_certificates_for_round(previous_round); // Check if the batch is ready to be proposed. - let mut is_ready = false; - if previous_round == 0 { - // Note: The primary starts at round 1, and round 0 contains no certificates, by definition. - is_ready = true; - } else { - let committee = self.committee.read().clone(); - // TODO (howardwu): Enable this code to turn on dynamic committees. - // } else if let Some(committee) = self.storage.get_committee_for_round(previous_round) { - // Compute the cumulative amount of stake for the previous certificates. - let mut stake = 0u64; - for certificate in previous_certificates.iter() { - stake = stake.saturating_add(committee.get_stake(certificate.author())); - } - // Check if the previous certificates have reached quorum threshold. - if stake >= committee.quorum_threshold()? { + // Note: The primary starts at round 1, and round 0 contains no certificates, by definition. + let mut is_ready = previous_round == 0; + // If the previous round is not 0, check if the previous certificates have reached the quorum threshold. + if let Some(committee) = self.storage.get_committee_for_round(previous_round) { + // Construct a set over the authors. + let authors = previous_certificates.iter().map(BatchCertificate::author).collect(); + // Check if the previous certificates have reached the quorum threshold. + if committee.is_quorum_threshold_reached(&authors)? { is_ready = true; } } + // If the batch is not ready to be proposed, return early. - if !is_ready { - return Ok(()); + match is_ready { + true => debug!("Proposing a batch for round {round}..."), + false => return Ok(()), } /* Proceeding to sign & propose the batch. */ - debug!("Proposing a batch for round {round}..."); - + // Initialize the RNG. + let mut rng = rand::thread_rng(); + // Retrieve the private key. + let private_key = self.gateway.account().private_key(); // Sign the batch. let batch = Batch::new(private_key, round, transmissions, previous_certificates, &mut rng)?; - // Retrieve the batch header. - let header = batch.to_header()?; - + // Broadcast the batch to all validators for signing. + self.gateway.broadcast(Event::BatchPropose(BatchPropose::new(Data::Object(batch.to_header()?)))); // Set the proposed batch. *self.proposed_batch.write() = Some((batch, Default::default())); - - // Broadcast the batch to all validators for signing. - self.gateway.broadcast(Event::BatchPropose(BatchPropose::new(Data::Object(header)))); Ok(()) } @@ -246,8 +233,8 @@ impl Primary { /// - Ensure the timestamp is within range. /// - Ensure we have all of the transmissions. /// - Ensure we have all of the previous certificates. - /// - Ensure the previous certificates are valid. - /// - Ensure the previous certificates have reached quorum threshold. + /// - Ensure the previous certificates are for the previous round (i.e. round - 1). + /// - Ensure the previous certificates have reached the quorum threshold. /// - Ensure we have not already signed the batch ID. /// 2. Sign the batch. /// 3. Broadcast the signature back to the validator. @@ -294,12 +281,42 @@ impl Primary { // bail!("Timestamp {timestamp} for the proposed batch must be after the previous round timestamp") // } - // Ensure the primary has all of the transmissions. - self.fetch_missing_transmissions(peer_ip, &batch_header).await?; + // Compute the previous round. + let previous_round = round.saturating_sub(1); + + if previous_round > 0 { + // Ensure the primary has all of the transmissions. + self.fetch_missing_transmissions(peer_ip, &batch_header).await?; + // Ensure the primary has all of the previous certificates. + self.fetch_missing_certificates(peer_ip, &batch_header).await?; + + // Initialize a set of the previous authors. + let mut previous_authors = HashSet::with_capacity(batch_header.previous_certificate_ids().len()); + + // Retrieve the previous certificates. + for previous_certificate_id in batch_header.previous_certificate_ids() { + // Retrieve the previous certificate. + let Some(previous_certificate) = self.storage.get_certificate(*previous_certificate_id) else { + bail!("Missing previous certificate for a proposed batch from peer {peer_ip} in round {round}"); + }; + // Ensure the previous certificate is for the previous round. + if previous_certificate.round() != previous_round { + bail!("Previous certificate for a proposed batch from peer {peer_ip} is for the wrong round"); + } + // Insert the author of the previous certificate. + previous_authors.insert(previous_certificate.author()); + } + + // Ensure the previous certificates have reached the quorum threshold. + let Some(previous_committee) = self.storage.get_committee_for_round(previous_round) else { + bail!("Missing the committee for the previous round {previous_round}") + }; + // Ensure the previous certificates have reached the quorum threshold. + if !previous_committee.is_quorum_threshold_reached(&previous_authors)? { + bail!("Previous certificates for a proposed batch from peer {peer_ip} did not reach quorum threshold"); + } + } - // TODO (howardwu): Ensure I have all of the previous certificates. If not, request them before signing. - // TODO (howardwu): Ensure the previous certificates are for round-1. If not, do not sign. - // TODO (howardwu): Ensure the previous certificates have reached 2f+1. If not, do not sign. // TODO (howardwu): Ensure I have not signed this batch ID before. If so, do not sign. /* Proceeding to sign the batch. */ @@ -369,25 +386,22 @@ impl Primary { // Check if the batch is ready to be certified. let mut is_ready = false; if let Some((batch, signatures)) = self.proposed_batch.read().as_ref() { - // Compute the cumulative amount of stake, thus far. - let mut stake = 0u64; - for signature in signatures.keys().chain([batch.signature()].into_iter()) { - stake = stake.saturating_add(self.committee.read().get_stake(signature.to_address())); - } - // Check if the batch has reached quorum threshold. - if stake >= self.committee.read().quorum_threshold()? { + // Construct an iterator over the addresses. + let addresses = signatures.keys().chain([batch.signature()].into_iter()).map(Signature::to_address); + // Check if the batch has reached the quorum threshold. + if self.committee.read().is_quorum_threshold_reached(&addresses.collect())? { is_ready = true; } } + // If the batch is not ready to be certified, return early. - if !is_ready { - return Ok(()); + match is_ready { + true => info!("Quorum threshold reached - Preparing to certify our batch..."), + false => return Ok(()), } /* Proceeding to certify the batch. */ - info!("Quorum threshold reached - Preparing to certify our batch..."); - // Retrieve the batch and signatures, clearing the proposed batch. let (batch, signatures) = self.proposed_batch.write().take().unwrap(); @@ -438,14 +452,14 @@ impl Primary { } /// Handles the incoming certificate response. - async fn process_certificate_response(&self, peer_ip: SocketAddr, response: CertificateResponse) -> Result<()> { + fn process_certificate_response(&self, peer_ip: SocketAddr, response: CertificateResponse) -> Result<()> { let certificate_id = response.certificate.certificate_id(); // Check if the peer IP exists in the pending queue for the given certificate ID. if self.pending.get(certificate_id).unwrap_or_default().contains(&peer_ip) { // TODO: Validate the certificate. // TODO: Fetch missing transactions? // Insert the certificate into storage. - // self.storage.insert_certificate(response.certificate)?; + self.storage.insert_certificate(response.certificate)?; // Remove the certificate ID from the pending queue. self.pending.remove(certificate_id); trace!("Primary - Added certificate '{}' from peer '{peer_ip}'", fmt_id(certificate_id)); @@ -461,6 +475,8 @@ impl Primary { mut rx_batch_propose, mut rx_batch_signature, mut rx_batch_certified, + mut rx_certificate_request, + mut rx_certificate_response, mut rx_unconfirmed_solution, mut rx_unconfirmed_transaction, } = receiver; @@ -505,6 +521,24 @@ impl Primary { } }); + // Process the certificate request. + let self_clone = self.clone(); + self.spawn(async move { + while let Some((peer_ip, certificate_request)) = rx_certificate_request.recv().await { + self_clone.process_certificate_request(peer_ip, certificate_request); + } + }); + + // Process the certificate response. + let self_clone = self.clone(); + self.spawn(async move { + while let Some((peer_ip, certificate_response)) = rx_certificate_response.recv().await { + if let Err(e) = self_clone.process_certificate_response(peer_ip, certificate_response) { + warn!("Failed to process a certificate response from peer '{peer_ip}' - {e}"); + } + } + }); + // Process the unconfirmed solutions. let self_clone = self.clone(); self.spawn(async move { @@ -588,15 +622,26 @@ impl Primary { /// - The previous certificates have reached quorum threshold. async fn store_certificate(&self, peer_ip: SocketAddr, certificate: BatchCertificate) -> Result<()> { // TODO (howardwu): Ensure the certificate is well-formed. If not, do not store. - // TODO (howardwu): Ensure the round is within range. If not, do not store. // TODO (howardwu): Ensure the address is in the committee of the specified round. If not, do not store. - // TODO (howardwu): Ensure I have all of the previous certificates. If not, request them before storing. // TODO (howardwu): Ensure the previous certificates are for round-1. If not, do not store. // TODO (howardwu): Ensure the previous certificates have reached 2f+1. If not, do not store. + // Retrieve the GC round. + let gc_round = self.storage.gc_round(); + // If the certificate is for a round less than or equal to the GC round, do not store it. + if certificate.round() <= gc_round { + return Ok(()); + } + // Ensure the primary has all of the transmissions. self.fetch_missing_transmissions(peer_ip, certificate.batch_header()).await?; + // Check if the previous round is above the GC round. + if certificate.round() > gc_round + 1 { + // Ensure the primary has all of the previous certificates. + self.fetch_missing_certificates(peer_ip, certificate.batch_header()).await?; + } + // Store the certificate. self.storage.insert_certificate(certificate)?; // Return success. @@ -643,6 +688,39 @@ impl Primary { Ok(()) } + /// Fetches any missing certificates for the specified batch header from the specified peer. + async fn fetch_missing_certificates(&self, peer_ip: SocketAddr, batch_header: &BatchHeader) -> Result<()> { + // Initialize a list for the missing certificates. + let mut fetch_certificates = FuturesUnordered::new(); + + // Iterate through the certificate IDs. + for certificate_id in batch_header.previous_certificate_ids() { + // If we do not have the certificate, request it. + if !self.storage.contains_certificate(*certificate_id) { + trace!("Primary - Found a new certificate ID '{}' from peer '{peer_ip}'", fmt_id(certificate_id)); + + // Initialize a oneshot channel. + let (callback_sender, callback_receiver) = oneshot::channel(); + // Insert the certificate ID into the pending queue. + self.pending.insert(*certificate_id, peer_ip, Some(callback_sender)); + // TODO (howardwu): Limit the number of open requests we send to a peer. + // Send an certificate request to the peer. + self.send_certificate_request(peer_ip, *certificate_id); + // Push the callback onto the list. + fetch_certificates.push(callback_receiver); + } + } + + // Wait for all of the certificates to be fetched. + while let Some(result) = fetch_certificates.next().await { + if let Err(e) = result { + bail!("Unable to fetch certificate: {e}") + } + } + // Return after receiving all of the certificates. + Ok(()) + } + /// Sends an certificate request to the specified peer. fn send_certificate_request(&self, peer_ip: SocketAddr, certificate_id: Field) { // Send the certificate request to the peer. diff --git a/node/narwhal/src/worker.rs b/node/narwhal/src/worker.rs index 855d613bb7..0bd3095a35 100644 --- a/node/narwhal/src/worker.rs +++ b/node/narwhal/src/worker.rs @@ -132,11 +132,7 @@ impl Worker { } /// Handles the incoming transmission response. - async fn process_transmission_response( - &self, - peer_ip: SocketAddr, - response: TransmissionResponse, - ) -> Result<()> { + fn process_transmission_response(&self, peer_ip: SocketAddr, response: TransmissionResponse) -> Result<()> { // Check if the peer IP exists in the pending queue for the given transmission ID. if self.pending.get(response.transmission_id).unwrap_or_default().contains(&peer_ip) { // TODO: Validate the transmission. @@ -204,7 +200,6 @@ impl Worker { let self_clone = self.clone(); self.spawn(async move { while let Some((peer_ip, transmission_id)) = rx_worker_ping.recv().await { - // Process the ping event. self_clone.process_transmission_id(peer_ip, transmission_id, None); } }); @@ -213,7 +208,6 @@ impl Worker { let self_clone = self.clone(); self.spawn(async move { while let Some((peer_ip, transmission_request)) = rx_transmission_request.recv().await { - // Process the transmission request. self_clone.process_transmission_request(peer_ip, transmission_request); } }); @@ -223,7 +217,7 @@ impl Worker { self.spawn(async move { while let Some((peer_ip, transmission_response)) = rx_transmission_response.recv().await { // Process the transmission response. - if let Err(e) = self_clone.process_transmission_response(peer_ip, transmission_response).await { + if let Err(e) = self_clone.process_transmission_response(peer_ip, transmission_response) { error!( "Worker {} failed to process transmission response from peer '{peer_ip}': {e}", self_clone.id