feat: pallas byron

Cargo.toml

@@ -8,14 +8,12 @@ homepage = "https://github.com/txpipe/oura"
 documentation = "https://docs.rs/oura"
 license = "Apache-2.0"
 readme = "README.md"
-authors = [
-    "Santiago Carmuega <santiago@carmuega.me>"
-]
+authors = ["Santiago Carmuega <santiago@carmuega.me>"]
 
 
 [dependencies]
-pallas = "0.4.0"
-#pallas = { path = "../pallas/pallas" }
+pallas = "0.5.0-alpha"
+# pallas = { path = "../pallas/pallas" }
 hex = "0.4.3"
 net2 = "0.2.37"
 bech32 = "0.8.1"
@@ -38,7 +36,9 @@ file-rotate = { version = "0.6.0", optional = true }
 reqwest = { version = "0.11", optional = true, features = ["blocking", "json"] }
 
 # feature: tuisink
-tui = { version = "0.17", optional = true, default-features = false, features = ["crossterm"] }
+tui = { version = "0.17", optional = true, default-features = false, features = [
+    "crossterm",
+] }
 
 # feature: kafkasink
 kafka = { version = "0.8.0", optional = true }

src/mapper/byron.rs

@@ -0,0 +1,220 @@
+use std::ops::Deref;
+
+use super::map::ToHex;
+use super::EventWriter;
+use crate::model::{BlockRecord, EventData, TransactionRecord, TxInputRecord, TxOutputRecord};
+use crate::{model::EventContext, Error};
+
+use pallas::crypto::hash::Hash;
+use pallas::ledger::primitives::{byron, Fragment};
+
+impl EventWriter {
+    fn to_byron_input_record(&self, source: &byron::TxIn) -> Option<TxInputRecord> {
+        match source {
+            byron::TxIn::Variant0(x) => {
+                let (hash, index) = x.deref();
+
+                Some(TxInputRecord {
+                    tx_id: hash.to_hex(),
+                    index: *index as u64,
+                })
+            }
+            byron::TxIn::Other(a, b) => {
+                log::warn!(
+                    "don't know how to handle byron input: ({}, {})",
+                    a,
+                    b.to_hex()
+                );
+
+                None
+            }
+        }
+    }
+
+    fn collect_byron_input_records(&self, source: &byron::Tx) -> Vec<TxInputRecord> {
+        source
+            .inputs
+            .iter()
+            .filter_map(|x| self.to_byron_input_record(x))
+            .collect()
+    }
+
+    fn to_byron_output_record(&self, source: &byron::TxOut) -> Result<TxOutputRecord, Error> {
+        Ok(TxOutputRecord {
+            address: source.address.to_addr_string()?,
+            amount: source.amount,
+            assets: None,
+        })
+    }
+
+    fn collect_byron_output_records(
+        &self,
+        source: &byron::Tx,
+    ) -> Result<Vec<TxOutputRecord>, Error> {
+        source
+            .outputs
+            .iter()
+            .map(|x| self.to_byron_output_record(x))
+            .collect()
+    }
+
+    fn to_byron_transaction_record(
+        &self,
+        source: &byron::TxPayload,
+        tx_hash: &str,
+    ) -> Result<TransactionRecord, Error> {
+        let input_records = self.collect_byron_input_records(&source.transaction);
+        let output_records = self.collect_byron_output_records(&source.transaction)?;
+
+        let mut record = TransactionRecord {
+            hash: tx_hash.to_owned(),
+            // TODO: we have a problem with here. AFAIK, there's no reference to the tx fee in the
+            // block contents. This leaves us with the two alternative: a) compute the value, b)
+            // omit the value.
+            //
+            // Computing the value is not trivial, the linear policy is easy to
+            // implement, but tracking the parameters for each epoch means hardcoding values or
+            // doing some extra queries.
+            //
+            // Ommiting the value elegantly would require turning the property data type into an
+            // option, which is a breaking change.
+            //
+            // Chossing the lesser evil, going to send a `0` in the field and add a comment to the
+            // docs notifying about this as a known issue to be fixed in v2.
+
+            //fee: source.compute_fee_with_defaults()?,
+            fee: 0,
+            input_count: input_records.len(),
+            output_count: output_records.len(),
+            total_output: output_records.iter().map(|o| o.amount).sum(),
+            ..Default::default()
+        };
+
+        if self.config.include_transaction_details {
+            record.inputs = input_records.into();
+            record.outputs = output_records.into();
+        }
+
+        Ok(record)
+    }
+
+    fn crawl_byron_transaction(
+        &self,
+        source: &byron::TxPayload,
+        tx_hash: &str,
+    ) -> Result<(), Error> {
+        let record = self.to_byron_transaction_record(source, tx_hash)?;
+
+        self.append_from(record.clone())?;
+
+        for (idx, input) in source.transaction.inputs.iter().enumerate() {
+            let child = self.child_writer(EventContext {
+                input_idx: Some(idx),
+                ..EventContext::default()
+            });
+
+            if let Some(record) = self.to_byron_input_record(input) {
+                child.append_from(record)?;
+            }
+        }
+
+        for (idx, output) in source.transaction.outputs.iter().enumerate() {
+            let child = self.child_writer(EventContext {
+                output_idx: Some(idx),
+                ..EventContext::default()
+            });
+
+            if let Ok(record) = self.to_byron_output_record(output) {
+                child.append_from(record)?;
+            }
+        }
+
+        if self.config.include_transaction_end_events {
+            self.append(EventData::TransactionEnd(record))?;
+        }
+
+        Ok(())
+    }
+
+    pub fn to_byron_block_record(
+        &self,
+        source: &byron::MainBlock,
+        hash: &Hash<32>,
+        cbor: &[u8],
+    ) -> Result<BlockRecord, Error> {
+        Ok(BlockRecord {
+            body_size: cbor.len() as usize,
+            issuer_vkey: source.header.consensus_data.1.to_hex(),
+            tx_count: source.body.tx_payload.len(),
+            hash: hash.to_hex(),
+            number: source.header.consensus_data.2[0],
+            slot: source.header.consensus_data.0.to_abs_slot(),
+            previous_hash: source.header.prev_block.to_hex(),
+            cbor_hex: match self.config.include_block_cbor {
+                true => hex::encode(cbor).into(),
+                false => None,
+            },
+        })
+    }
+
+    fn crawl_byron_main_block(
+        &self,
+        block: &byron::MainBlock,
+        hash: &Hash<32>,
+        cbor: &[u8],
+    ) -> Result<(), Error> {
+        let record = self.to_byron_block_record(block, hash, cbor)?;
+
+        self.append(EventData::Block(record.clone()))?;
+
+        for (idx, tx) in block.body.tx_payload.iter().enumerate() {
+            let tx_hash = tx.transaction.to_hash().to_string();
+
+            let child = self.child_writer(EventContext {
+                tx_idx: Some(idx),
+                tx_hash: Some(tx_hash.to_owned()),
+                ..EventContext::default()
+            });
+
+            child.crawl_byron_transaction(tx, &tx_hash)?;
+        }
+
+        if self.config.include_block_end_events {
+            self.append(EventData::BlockEnd(record))?;
+        }
+
+        Ok(())
+    }
+
+    /// Mapper entry-point for decoded Byron blocks
+    ///
+    /// Entry-point to start crawling a blocks for events. Meant to be used when
+    /// we already have a decoded block (for example, N2C). The raw CBOR is also
+    /// passed through in case we need to attach it to outbound events.
+    pub fn crawl_byron_with_cbor(&self, block: &byron::Block, cbor: &[u8]) -> Result<(), Error> {
+        if let byron::Block::MainBlock(block) = block {
+            let hash = block.header.to_hash();
+
+            let child = self.child_writer(EventContext {
+                block_hash: Some(hex::encode(&hash)),
+                block_number: Some(block.header.consensus_data.2[0]),
+                slot: Some(block.header.consensus_data.0.to_abs_slot()),
+                //timestamp: self.compute_timestamp(block.header.header_body.slot),
+                ..EventContext::default()
+            });
+
+            child.crawl_byron_main_block(block, &hash, cbor)?;
+        }
+
+        Ok(())
+    }
+
+    /// Mapper entry-point for raw Byron cbor blocks
+    ///
+    /// Entry-point to start crawling a blocks for events. Meant to be used when
+    /// we haven't decoded the CBOR yet (for example, N2N).
+    pub fn crawl_from_byron_cbor(&self, cbor: &[u8]) -> Result<(), Error> {
+        let block = byron::Block::decode_fragment(cbor)?;
+        self.crawl_byron_with_cbor(&block, cbor)
+    }
+}

src/mapper/cip25.rs

@@ -1,6 +1,6 @@
 use serde_json::Value as JsonValue;
 
-use pallas::ledger::alonzo::Metadatum;
+use pallas::ledger::primitives::alonzo::Metadatum;
 
 use crate::{model::CIP25AssetRecord, Error};
 

src/mapper/collect.rs

@@ -1,4 +1,4 @@
-use pallas::ledger::alonzo::{
+use pallas::ledger::primitives::alonzo::{
     AuxiliaryData, Multiasset, TransactionInput, TransactionOutput, Value,
 };
 

src/mapper/map.rs

@@ -2,11 +2,11 @@ use std::collections::HashMap;
 
 use minicbor::bytes::ByteVec;
 use pallas::crypto::hash::Hash;
-use pallas::ledger::alonzo::{
+use pallas::ledger::primitives::alonzo::{
     self as alonzo, AuxiliaryData, Block, Certificate, InstantaneousRewardSource,
     InstantaneousRewardTarget, Metadatum, Relay, TransactionInput, TransactionOutput, Value,
 };
-use pallas::ledger::alonzo::{NetworkId, TransactionBody, TransactionBodyComponent};
+use pallas::ledger::primitives::alonzo::{NetworkId, TransactionBody, TransactionBodyComponent};
 
 use serde_json::{json, Value as JsonValue};
 

src/mapper/mod.rs

@@ -1,7 +1,8 @@
+mod byron;
 mod cip25;
 mod collect;
-mod crawl;
 mod map;
 mod prelude;
+mod shelley;
 
 pub use prelude::*;

src/mapper/crawl.rs → src/mapper/shelley.rs

@@ -1,6 +1,8 @@
-use pallas::ledger::alonzo::{
-    crypto, AuxiliaryData, Block, Certificate, Metadata, Metadatum, Multiasset, TransactionBody,
-    TransactionBodyComponent, TransactionInput, TransactionOutput, Value,
+use pallas::ledger::primitives::Fragment;
+
+use pallas::ledger::primitives::alonzo::{
+    self, crypto, AuxiliaryData, Block, Certificate, Metadata, Metadatum, Multiasset,
+    TransactionBody, TransactionBodyComponent, TransactionInput, TransactionOutput, Value,
 };
 
 use pallas::crypto::hash::Hash;
@@ -124,7 +126,7 @@ impl EventWriter {
         Ok(())
     }
 
-    fn crawl_transaction(
+    fn crawl_shelley_transaction(
         &self,
         tx: &TransactionBody,
         tx_hash: &str,
@@ -191,7 +193,12 @@ impl EventWriter {
         Ok(())
     }
 
-    fn crawl_block(&self, block: &Block, hash: &Hash<32>, cbor: &[u8]) -> Result<(), Error> {
+    fn crawl_shelley_block(
+        &self,
+        block: &Block,
+        hash: &Hash<32>,
+        cbor: &[u8],
+    ) -> Result<(), Error> {
         let record = self.to_block_record(block, hash, cbor)?;
 
         self.append(EventData::Block(record.clone()))?;
@@ -211,7 +218,7 @@ impl EventWriter {
                 ..EventContext::default()
             });
 
-            child.crawl_transaction(tx, &tx_hash, aux_data)?;
+            child.crawl_shelley_transaction(tx, &tx_hash, aux_data)?;
         }
 
         if self.config.include_block_end_events {
@@ -221,6 +228,7 @@ impl EventWriter {
         Ok(())
     }
 
+    #[deprecated(note = "use crawl_from_shelley_cbor instead")]
     pub fn crawl_with_cbor(&self, block: &Block, cbor: &[u8]) -> Result<(), Error> {
         let hash = crypto::hash_block_header(&block.header);
 
@@ -232,10 +240,10 @@ impl EventWriter {
             ..EventContext::default()
         });
 
-        child.crawl_block(block, &hash, cbor)
+        child.crawl_shelley_block(block, &hash, cbor)
     }
 
-    #[deprecated(note = "use crawl_with_cbor instead")]
+    #[deprecated(note = "use crawl_from_shelley_cbor instead")]
     pub fn crawl(&self, block: &Block) -> Result<(), Error> {
         let hash = crypto::hash_block_header(&block.header);
 
@@ -247,6 +255,38 @@ impl EventWriter {
             ..EventContext::default()
         });
 
-        child.crawl_block(block, &hash, &[])
+        child.crawl_shelley_block(block, &hash, &[])
+    }
+
+    /// Mapper entry-point for decoded Shelley blocks
+    ///
+    /// Entry-point to start crawling a blocks for events. Meant to be used when
+    /// we already have a decoded block (for example, N2C). The raw CBOR is also
+    /// passed through in case we need to attach it to outbound events.
+    pub fn crawl_shelley_with_cbor(&self, block: &Block, cbor: &[u8]) -> Result<(), Error> {
+        let hash = crypto::hash_block_header(&block.header);
+
+        let child = self.child_writer(EventContext {
+            block_hash: Some(hex::encode(&hash)),
+            block_number: Some(block.header.header_body.block_number),
+            slot: Some(block.header.header_body.slot),
+            timestamp: self.compute_timestamp(block.header.header_body.slot),
+            ..EventContext::default()
+        });
+
+        child.crawl_shelley_block(block, &hash, cbor)
+    }
+
+    /// Mapper entry-point for raw Shelley cbor blocks
+    ///
+    /// Entry-point to start crawling a blocks for events. Meant to be used when
+    /// we haven't decoded the CBOR yet (for example, N2N).
+    ///
+    /// We use Alonzo primitives since they are backward compatible with
+    /// Shelley. In this way, we can avoid having to fork the crawling procedure
+    /// for each different hard-fork.
+    pub fn crawl_from_shelley_cbor(&self, cbor: &[u8]) -> Result<(), Error> {
+        let alonzo::BlockWrapper(_, block) = alonzo::BlockWrapper::decode_fragment(cbor)?;
+        self.crawl_shelley_with_cbor(&block, cbor)
     }
 }