diff --git a/Cargo.lock b/Cargo.lock index f90fd7f8daca4d..d37c61fc4953c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6128,6 +6128,19 @@ dependencies = [ "tokio-util 0.7.10", ] +[[package]] +name = "compute-module-expansion-size" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.4.14", + "futures", + "move-binary-format", + "move-core-types", + "rayon", + "tokio", +] + [[package]] name = "concurrent-queue" version = "2.4.0" diff --git a/Cargo.toml b/Cargo.toml index 78792c45d7105e..465c8a0322a607 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -249,6 +249,7 @@ members = [ "third_party/move/tools/move-resource-viewer", "third_party/move/tools/move-unit-test", "tools/calc-dep-sizes", + "tools/compute-module-expansion-size", "types", "vm-validator", ] diff --git a/aptos-move/aptos-vm/src/aptos_vm.rs b/aptos-move/aptos-vm/src/aptos_vm.rs index 2f7f0afd368cd0..e7c1a760d78d08 100644 --- a/aptos-move/aptos-vm/src/aptos_vm.rs +++ b/aptos-move/aptos-vm/src/aptos_vm.rs @@ -50,7 +50,7 @@ use aptos_types::{ move_utils::as_move_value::AsMoveValue, on_chain_config::{ new_epoch_event_key, ApprovedExecutionHashes, ConfigStorage, FeatureFlag, Features, - OnChainConfig, TimedFeatures, + OnChainConfig, TimedFeatureFlag, TimedFeatures, }, randomness::Randomness, state_store::{StateView, TStateView}, @@ -1431,6 +1431,18 @@ impl AptosVM { // TODO: Revisit the order of traversal. Consider switching to alphabetical order. } + if self + .timed_features() + .is_enabled(TimedFeatureFlag::ModuleComplexityCheck) + { + for (module, blob) in modules.iter().zip(bundle.iter()) { + // TODO(Gas): Make budget configurable. + let budget = 2048 + blob.code().len() as u64 * 20; + move_binary_format::check_complexity::check_module_complexity(module, budget) + .map_err(|err| err.finish(Location::Undefined))?; + } + } + // Validate the module bundle self.validate_publish_request(session, modules, expected_modules, allowed_deps)?; diff --git a/aptos-move/aptos-vm/src/lib.rs b/aptos-move/aptos-vm/src/lib.rs index d5f3dd442cefb7..20c3caff2dc296 100644 --- a/aptos-move/aptos-vm/src/lib.rs +++ b/aptos-move/aptos-vm/src/lib.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #![forbid(unsafe_code)] +#![deny(deprecated)] //! # The VM runtime //! diff --git a/aptos-move/aptos-vm/src/verifier/resource_groups.rs b/aptos-move/aptos-vm/src/verifier/resource_groups.rs index caa90bbfaa3919..a7c7e8d193682b 100644 --- a/aptos-move/aptos-vm/src/verifier/resource_groups.rs +++ b/aptos-move/aptos-vm/src/verifier/resource_groups.rs @@ -6,7 +6,6 @@ use aptos_framework::{ResourceGroupScope, RuntimeModuleMetadataV1}; use move_binary_format::{ access::ModuleAccess, errors::{Location, PartialVMError, VMError, VMResult}, - normalized::Struct, CompiledModule, }; use move_core_types::{ @@ -172,7 +171,11 @@ pub(crate) fn extract_resource_group_metadata_from_module( let structs = module .struct_defs() .iter() - .map(|d| Struct::new(&module, d).0.into_string()) + .map(|struct_def| { + let struct_handle = module.struct_handle_at(struct_def.struct_handle); + let name = module.identifier_at(struct_handle.name).to_string(); + name + }) .collect::>(); Ok((groups, members, structs)) } else { diff --git a/aptos-move/framework/src/module_metadata.rs b/aptos-move/framework/src/module_metadata.rs index 7736d60c9bf84b..4b969d1b96eaef 100644 --- a/aptos-move/framework/src/module_metadata.rs +++ b/aptos-move/framework/src/module_metadata.rs @@ -3,22 +3,21 @@ use crate::extended_checks::ResourceGroupScope; use aptos_types::{ - on_chain_config::{FeatureFlag, Features, TimedFeatures}, + on_chain_config::{FeatureFlag, Features, TimedFeatureFlag, TimedFeatures}, transaction::AbortInfo, }; use lru::LruCache; use move_binary_format::{ access::ModuleAccess, file_format::{ - Ability, AbilitySet, CompiledScript, IdentifierIndex, SignatureToken, - StructFieldInformation, TableIndex, + Ability, AbilitySet, CompiledScript, FunctionDefinition, FunctionHandle, IdentifierIndex, + SignatureToken, StructDefinition, StructFieldInformation, StructHandle, TableIndex, }, - normalized::{Function, Struct}, CompiledModule, }; use move_core_types::{ errmap::ErrorDescription, - identifier::Identifier, + identifier::{IdentStr, Identifier}, language_storage::{ModuleId, StructTag}, metadata::Metadata, }; @@ -388,12 +387,12 @@ pub struct AttributeValidationError { } pub fn is_valid_unbiasable_function( - functions: &BTreeMap, + functions: &BTreeMap<&IdentStr, (&FunctionHandle, &FunctionDefinition)>, fun: &str, ) -> Result<(), AttributeValidationError> { if let Ok(ident_fun) = Identifier::new(fun) { - if let Some(f) = functions.get(&ident_fun) { - if f.is_entry && !f.visibility.is_public() { + if let Some((_func_handle, func_def)) = functions.get(ident_fun.as_ident_str()) { + if func_def.is_entry && !func_def.visibility.is_public() { return Ok(()); } } @@ -406,12 +405,14 @@ pub fn is_valid_unbiasable_function( } pub fn is_valid_view_function( - functions: &BTreeMap, + module: &CompiledModule, + functions: &BTreeMap<&IdentStr, (&FunctionHandle, &FunctionDefinition)>, fun: &str, ) -> Result<(), AttributeValidationError> { if let Ok(ident_fun) = Identifier::new(fun) { - if let Some(mod_fun) = functions.get(&ident_fun) { - if !mod_fun.return_.is_empty() { + if let Some((func_handle, _func_def)) = functions.get(ident_fun.as_ident_str()) { + let sig = module.signature_at(func_handle.return_); + if !sig.0.is_empty() { return Ok(()); } } @@ -424,14 +425,18 @@ pub fn is_valid_view_function( } pub fn is_valid_resource_group( - structs: &BTreeMap, + structs: &BTreeMap<&IdentStr, (&StructHandle, &StructDefinition)>, struct_: &str, ) -> Result<(), AttributeValidationError> { if let Ok(ident_struct) = Identifier::new(struct_) { - if let Some(mod_struct) = structs.get(&ident_struct) { - if mod_struct.abilities == AbilitySet::EMPTY - && mod_struct.type_parameters.is_empty() - && mod_struct.fields.len() == 1 + if let Some((struct_handle, struct_def)) = structs.get(ident_struct.as_ident_str()) { + let num_fields = match &struct_def.field_information { + StructFieldInformation::Native => 0, + StructFieldInformation::Declared(fields) => fields.len(), + }; + if struct_handle.abilities == AbilitySet::EMPTY + && struct_handle.type_parameters.is_empty() + && num_fields == 1 { return Ok(()); } @@ -445,12 +450,12 @@ pub fn is_valid_resource_group( } pub fn is_valid_resource_group_member( - structs: &BTreeMap, + structs: &BTreeMap<&IdentStr, (&StructHandle, &StructDefinition)>, struct_: &str, ) -> Result<(), AttributeValidationError> { if let Ok(ident_struct) = Identifier::new(struct_) { - if let Some(mod_struct) = structs.get(&ident_struct) { - if mod_struct.abilities.has_ability(Ability::Key) { + if let Some((struct_handle, _struct_def)) = structs.get(ident_struct.as_ident_str()) { + if struct_handle.abilities.has_ability(Ability::Key) { return Ok(()); } } @@ -465,9 +470,11 @@ pub fn is_valid_resource_group_member( pub fn verify_module_metadata( module: &CompiledModule, features: &Features, - _timed_features: &TimedFeatures, + timed_features: &TimedFeatures, ) -> Result<(), MetaDataValidationError> { - if features.is_enabled(FeatureFlag::SAFER_METADATA) { + if features.is_enabled(FeatureFlag::SAFER_METADATA) + && timed_features.is_enabled(TimedFeatureFlag::ModuleComplexityCheck) + { check_module_complexity(module)?; } @@ -483,13 +490,17 @@ pub fn verify_module_metadata( let functions = module .function_defs .iter() - .map(|func_def| Function::new(module, func_def)) + .map(|func_def| { + let func_handle = module.function_handle_at(func_def.function); + let name = module.identifier_at(func_handle.name); + (name, (func_handle, func_def)) + }) .collect::>(); for (fun, attrs) in &metadata.fun_attributes { for attr in attrs { if attr.is_view_function() { - is_valid_view_function(&functions, fun)?; + is_valid_view_function(module, &functions, fun)?; } else if attr.is_randomness() { is_valid_unbiasable_function(&functions, fun)?; } else { @@ -505,7 +516,11 @@ pub fn verify_module_metadata( let structs = module .struct_defs .iter() - .map(|d| Struct::new(module, d)) + .map(|struct_def| { + let struct_handle = module.struct_handle_at(struct_def.struct_handle); + let name = module.identifier_at(struct_handle.name); + (name, (struct_handle, struct_def)) + }) .collect::>(); for (struct_, attrs) in &metadata.struct_attributes { diff --git a/third_party/move/move-binary-format/src/check_complexity.rs b/third_party/move/move-binary-format/src/check_complexity.rs new file mode 100644 index 00000000000000..b291a2612c0a82 --- /dev/null +++ b/third_party/move/move-binary-format/src/check_complexity.rs @@ -0,0 +1,317 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + binary_views::BinaryIndexedView, + errors::{PartialVMError, PartialVMResult}, + file_format::{ + Bytecode, CodeUnit, CompiledModule, CompiledScript, FieldInstantiationIndex, + FunctionInstantiationIndex, IdentifierIndex, ModuleHandleIndex, SignatureIndex, + SignatureToken, StructDefInstantiationIndex, StructFieldInformation, TableIndex, + }, +}; +use move_core_types::vm_status::StatusCode; +use std::{ + cell::RefCell, + collections::{btree_map, BTreeMap}, +}; + +const COST_PER_TYPE_NODE: u64 = 8; +const COST_PER_IDENT_BYTE: u64 = 1; + +fn safe_get_table(table: &[T], idx: TableIndex) -> PartialVMResult<&T> { + table.get(idx as usize).ok_or_else(|| { + PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR) + .with_message("Index out of bounds while checking binary complexity.".to_string()) + }) +} + +struct BinaryComplexityMeter<'a> { + resolver: BinaryIndexedView<'a>, + + cached_signature_costs: RefCell>, + balance: RefCell, +} + +impl<'a> BinaryComplexityMeter<'a> { + fn charge(&self, amount: u64) -> PartialVMResult<()> { + let mut balance = self.balance.borrow_mut(); + match balance.checked_sub(amount) { + Some(new_balance) => { + *balance = new_balance; + Ok(()) + }, + None => { + *balance = 0; + Err(PartialVMError::new(StatusCode::PROGRAM_TOO_COMPLEX)) + }, + } + } + + fn signature_token_cost(&self, tok: &SignatureToken) -> PartialVMResult { + use SignatureToken::*; + + let mut cost: u64 = 0; + + for node in tok.preorder_traversal() { + cost = cost.saturating_add(COST_PER_TYPE_NODE); + + match node { + Struct(sh_idx) | StructInstantiation(sh_idx, _) => { + let sh = safe_get_table(self.resolver.struct_handles(), sh_idx.0)?; + let mh = safe_get_table(self.resolver.module_handles(), sh.module.0)?; + let struct_name = safe_get_table(self.resolver.identifiers(), sh.name.0)?; + let moduel_name = safe_get_table(self.resolver.identifiers(), mh.name.0)?; + + cost = cost.saturating_add(struct_name.len() as u64 * COST_PER_IDENT_BYTE); + cost = cost.saturating_add(moduel_name.len() as u64 * COST_PER_IDENT_BYTE); + }, + U8 | U16 | U32 | U64 | U128 | U256 | Signer | Address | Bool | Vector(_) + | TypeParameter(_) | Reference(_) | MutableReference(_) => (), + } + } + + Ok(cost) + } + + fn meter_identifier(&self, idx: IdentifierIndex) -> PartialVMResult<()> { + let ident = safe_get_table(self.resolver.identifiers(), idx.0)?; + self.charge(ident.len() as u64 * COST_PER_IDENT_BYTE) + } + + fn meter_signature(&self, idx: SignatureIndex) -> PartialVMResult<()> { + let cost = match self.cached_signature_costs.borrow_mut().entry(idx) { + btree_map::Entry::Occupied(entry) => *entry.into_mut(), + btree_map::Entry::Vacant(entry) => { + let sig = safe_get_table(self.resolver.signatures(), idx.0)?; + + let mut cost: u64 = 0; + for ty in &sig.0 { + cost = cost.saturating_add(self.signature_token_cost(ty)?); + } + + *entry.insert(cost) + }, + }; + + self.charge(cost)?; + + Ok(()) + } + + fn meter_signatures(&self) -> PartialVMResult<()> { + for sig_idx in 0..self.resolver.signatures().len() { + self.meter_signature(SignatureIndex(sig_idx as u16))?; + } + Ok(()) + } + + fn meter_function_instantiation( + &self, + func_inst_idx: FunctionInstantiationIndex, + ) -> PartialVMResult<()> { + let func_inst = safe_get_table(self.resolver.function_instantiations(), func_inst_idx.0)?; + self.meter_signature(func_inst.type_parameters) + } + + fn meter_function_instantiations(&self) -> PartialVMResult<()> { + for func_inst_idx in 0..self.resolver.function_instantiations().len() { + self.meter_function_instantiation(FunctionInstantiationIndex(func_inst_idx as u16))?; + } + Ok(()) + } + + fn meter_struct_instantiation( + &self, + struct_inst_idx: StructDefInstantiationIndex, + ) -> PartialVMResult<()> { + let struct_insts = self.resolver.struct_instantiations().ok_or_else(|| { + PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR) + .with_message("Can't get struct instantiations -- not a module.".to_string()) + })?; + let struct_inst = safe_get_table(struct_insts, struct_inst_idx.0)?; + + self.meter_signature(struct_inst.type_parameters) + } + + fn meter_struct_def_instantiations(&self) -> PartialVMResult<()> { + let struct_insts = self.resolver.struct_instantiations().ok_or_else(|| { + PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR) + .with_message("Can't get struct instantiations -- not a module.".to_string()) + })?; + + for struct_inst_idx in 0..struct_insts.len() { + self.meter_struct_instantiation(StructDefInstantiationIndex(struct_inst_idx as u16))?; + } + Ok(()) + } + + fn meter_field_instantiation( + &self, + field_inst_idx: FieldInstantiationIndex, + ) -> PartialVMResult<()> { + let field_insts = self.resolver.field_instantiations().ok_or_else(|| { + PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR) + .with_message("Can't get field instantiations -- not a module.".to_string()) + })?; + let field_inst = safe_get_table(field_insts, field_inst_idx.0)?; + + self.meter_signature(field_inst.type_parameters) + } + + fn meter_field_instantiations(&self) -> PartialVMResult<()> { + let field_insts = self.resolver.field_instantiations().ok_or_else(|| { + PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR) + .with_message("Can't get field instantiations -- not a module.".to_string()) + })?; + + for field_inst_idx in 0..field_insts.len() { + self.meter_field_instantiation(FieldInstantiationIndex(field_inst_idx as u16))?; + } + Ok(()) + } + + fn meter_module_handle(&self, mh_idx: ModuleHandleIndex) -> PartialVMResult<()> { + let mh = safe_get_table(self.resolver.module_handles(), mh_idx.0)?; + self.meter_identifier(mh.name) + } + + fn meter_function_handles(&self) -> PartialVMResult<()> { + for fh in self.resolver.function_handles() { + self.meter_module_handle(fh.module)?; + self.meter_identifier(fh.name)?; + self.meter_signature(fh.parameters)?; + self.meter_signature(fh.return_)?; + } + Ok(()) + } + + fn meter_struct_handles(&self) -> PartialVMResult<()> { + for sh in self.resolver.struct_handles() { + self.meter_module_handle(sh.module)?; + self.meter_identifier(sh.name)?; + } + Ok(()) + } + + fn meter_struct_defs(&self) -> PartialVMResult<()> { + let struct_defs = self.resolver.struct_defs().ok_or_else(|| { + PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR) + .with_message("Can't get struct defs -- not a module.".to_string()) + })?; + + for sdef in struct_defs { + let fields = match &sdef.field_information { + StructFieldInformation::Native => continue, + StructFieldInformation::Declared(fields) => fields, + }; + + for field in fields { + self.charge(field.signature.0.num_nodes() as u64)?; + } + } + Ok(()) + } + + fn meter_code(&self, code: &CodeUnit) -> PartialVMResult<()> { + use Bytecode::*; + + self.meter_signature(code.locals)?; + + for instr in &code.code { + match instr { + CallGeneric(idx) => { + self.meter_function_instantiation(*idx)?; + }, + PackGeneric(idx) | UnpackGeneric(idx) => { + self.meter_struct_instantiation(*idx)?; + }, + ExistsGeneric(idx) + | MoveFromGeneric(idx) + | MoveToGeneric(idx) + | ImmBorrowGlobalGeneric(idx) + | MutBorrowGlobalGeneric(idx) => { + self.meter_struct_instantiation(*idx)?; + }, + ImmBorrowFieldGeneric(idx) | MutBorrowFieldGeneric(idx) => { + self.meter_field_instantiation(*idx)?; + }, + VecPack(idx, _) + | VecLen(idx) + | VecImmBorrow(idx) + | VecMutBorrow(idx) + | VecPushBack(idx) + | VecPopBack(idx) + | VecUnpack(idx, _) + | VecSwap(idx) => { + self.meter_signature(*idx)?; + }, + + // List out the other options explicitly so there's a compile error if a new + // bytecode gets added. + Pop | Ret | Branch(_) | BrTrue(_) | BrFalse(_) | LdU8(_) | LdU16(_) | LdU32(_) + | LdU64(_) | LdU128(_) | LdU256(_) | LdConst(_) | CastU8 | CastU16 | CastU32 + | CastU64 | CastU128 | CastU256 | LdTrue | LdFalse | Call(_) | Pack(_) + | Unpack(_) | ReadRef | WriteRef | FreezeRef | Add | Sub | Mul | Mod | Div + | BitOr | BitAnd | Xor | Shl | Shr | Or | And | Not | Eq | Neq | Lt | Gt | Le + | Ge | CopyLoc(_) | MoveLoc(_) | StLoc(_) | MutBorrowLoc(_) | ImmBorrowLoc(_) + | MutBorrowField(_) | ImmBorrowField(_) | MutBorrowGlobal(_) + | ImmBorrowGlobal(_) | Exists(_) | MoveTo(_) | MoveFrom(_) | Abort | Nop => (), + } + } + Ok(()) + } + + fn meter_function_defs(&self) -> PartialVMResult<()> { + let func_defs = self.resolver.function_defs().ok_or_else(|| { + PartialVMError::new(StatusCode::UNKNOWN_INVARIANT_VIOLATION_ERROR) + .with_message("Can't get func defs -- not a module.".to_string()) + })?; + + for func_def in func_defs { + if let Some(code) = &func_def.code { + self.meter_code(code)?; + } + } + Ok(()) + } +} + +pub fn check_module_complexity(module: &CompiledModule, budget: u64) -> PartialVMResult { + let meter = BinaryComplexityMeter { + resolver: BinaryIndexedView::Module(module), + cached_signature_costs: RefCell::new(BTreeMap::new()), + balance: RefCell::new(budget), + }; + + meter.meter_signatures()?; + meter.meter_function_instantiations()?; + meter.meter_struct_def_instantiations()?; + meter.meter_field_instantiations()?; + + meter.meter_function_handles()?; + meter.meter_struct_handles()?; + meter.meter_function_defs()?; + meter.meter_struct_defs()?; + + let used = budget - *meter.balance.borrow(); + Ok(used) +} + +pub fn check_script_complexity(script: &CompiledScript, budget: u64) -> PartialVMResult { + let meter = BinaryComplexityMeter { + resolver: BinaryIndexedView::Script(script), + cached_signature_costs: RefCell::new(BTreeMap::new()), + balance: RefCell::new(budget), + }; + + meter.meter_signatures()?; + meter.meter_function_instantiations()?; + + meter.meter_function_handles()?; + meter.meter_struct_handles()?; + meter.meter_code(&script.code)?; + + let used = budget - *meter.balance.borrow(); + Ok(used) +} diff --git a/third_party/move/move-binary-format/src/compatibility.rs b/third_party/move/move-binary-format/src/compatibility.rs index 6954f0c9ee65d5..0f3abccc0c000f 100644 --- a/third_party/move/move-binary-format/src/compatibility.rs +++ b/third_party/move/move-binary-format/src/compatibility.rs @@ -2,6 +2,8 @@ // Copyright (c) The Move Contributors // SPDX-License-Identifier: Apache-2.0 +#![allow(deprecated)] + use crate::{ errors::{PartialVMError, PartialVMResult}, file_format::{AbilitySet, StructTypeParameter, Visibility}, diff --git a/third_party/move/move-binary-format/src/lib.rs b/third_party/move/move-binary-format/src/lib.rs index 3077aa00f25d64..3bbd2285707e63 100644 --- a/third_party/move/move-binary-format/src/lib.rs +++ b/third_party/move/move-binary-format/src/lib.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #![forbid(unsafe_code)] +#![deny(deprecated)] use std::fmt; @@ -12,6 +13,7 @@ pub mod check_bounds; pub mod compatibility; #[macro_use] pub mod errors; +pub mod check_complexity; pub mod constant; pub mod control_flow_graph; pub mod deserializer; diff --git a/third_party/move/move-binary-format/src/normalized.rs b/third_party/move/move-binary-format/src/normalized.rs index 678bb8cc8d7bb0..651bfa9a05900e 100644 --- a/third_party/move/move-binary-format/src/normalized.rs +++ b/third_party/move/move-binary-format/src/normalized.rs @@ -2,6 +2,8 @@ // Copyright (c) The Move Contributors // SPDX-License-Identifier: Apache-2.0 +#![allow(deprecated)] + use crate::{ access::ModuleAccess, file_format::{ @@ -28,6 +30,7 @@ use std::collections::BTreeMap; /// A normalized version of `SignatureToken`, a type expression appearing in struct or function /// declarations. Unlike `SignatureToken`s, `normalized::Type`s from different modules can safely be /// compared. +#[deprecated = "Normalized types are known to have serious performance issues and should be avoided for new use cases."] #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Serialize, Deserialize)] pub enum Type { #[serde(rename = "bool")] @@ -66,6 +69,7 @@ pub enum Type { /// metadata that it is ignored by the VM. The reason: names are important to clients. We would /// want a change from `Account { bal: u64, seq: u64 }` to `Account { seq: u64, bal: u64 }` to be /// marked as incompatible. Not safe to compare without an enclosing `Struct`. +#[deprecated = "Normalized types are known to have serious performance issues and should be avoided for new use cases."] #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct Field { pub name: Identifier, @@ -74,6 +78,7 @@ pub struct Field { /// Normalized version of a `StructDefinition`. Not safe to compare without an associated /// `ModuleId` or `Module`. +#[deprecated = "Normalized types are known to have serious performance issues and should be avoided for new use cases."] #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct Struct { pub abilities: AbilitySet, @@ -83,6 +88,7 @@ pub struct Struct { /// Normalized version of a `FunctionDefinition`. Not safe to compare without an associated /// `ModuleId` or `Module`. +#[deprecated = "Normalized types are known to have serious performance issues and should be avoided for new use cases."] #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Serialize, Deserialize)] pub struct Function { pub visibility: Visibility, @@ -94,6 +100,7 @@ pub struct Function { /// Normalized version of a `CompiledModule`: its address, name, struct declarations, and public /// function declarations. +#[deprecated = "Normalized types are known to have serious performance issues and should be avoided for new use cases."] #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct Module { pub file_format_version: u32, diff --git a/third_party/move/move-binary-format/src/unit_tests/compatibility_tests.rs b/third_party/move/move-binary-format/src/unit_tests/compatibility_tests.rs index b87e4ffe8ae890..5d2db0369afdd8 100644 --- a/third_party/move/move-binary-format/src/unit_tests/compatibility_tests.rs +++ b/third_party/move/move-binary-format/src/unit_tests/compatibility_tests.rs @@ -5,6 +5,7 @@ use crate::{compatibility::Compatibility, file_format::*, normalized}; use move_core_types::{account_address::AccountAddress, identifier::Identifier}; use std::convert::TryFrom; +#[allow(deprecated)] fn mk_module(vis: u8) -> normalized::Module { let (visibility, is_entry) = if vis == Visibility::DEPRECATED_SCRIPT { (Visibility::Public, true) diff --git a/third_party/move/move-core/types/src/vm_status.rs b/third_party/move/move-core/types/src/vm_status.rs index 7abbe6461bcdf7..b5174f041bbb95 100644 --- a/third_party/move/move-core/types/src/vm_status.rs +++ b/third_party/move/move-core/types/src/vm_status.rs @@ -722,10 +722,13 @@ pub enum StatusCode { // This error indicates that unstable bytecode generated by the compiler cannot be published to mainnet UNSTABLE_BYTECODE_REJECTED = 1125, // Reserved error code for future use - RESERVED_VERIFICATION_ERROR_2 = 1126, - RESERVED_VERIFICATION_ERROR_3 = 1127, - RESERVED_VERIFICATION_ERROR_4 = 1128, - RESERVED_VERIFICATION_ERROR_5 = 1129, + PROGRAM_TOO_COMPLEX = 1126, + RESERVED_VERIFICATION_ERROR_1 = 1127, + RESERVED_VERIFICATION_ERROR_2 = 1128, + RESERVED_VERIFICATION_ERROR_3 = 1129, + RESERVED_VERIFICATION_ERROR_4 = 1130, + RESERVED_VERIFICATION_ERROR_5 = 1131, + // These are errors that the VM might raise if a violation of internal // invariants takes place. diff --git a/third_party/move/move-model/src/model.rs b/third_party/move/move-model/src/model.rs index eb485ab3cd5eec..37e0ce6201950b 100644 --- a/third_party/move/move-model/src/model.rs +++ b/third_party/move/move-model/src/model.rs @@ -45,6 +45,8 @@ use itertools::Itertools; #[allow(unused_imports)] use log::{info, warn}; pub use move_binary_format::file_format::{AbilitySet, Visibility}; +#[allow(deprecated)] +use move_binary_format::normalized::Type as MType; use move_binary_format::{ access::ModuleAccess, binary_views::BinaryIndexedView, @@ -53,7 +55,6 @@ use move_binary_format::{ FunctionDefinitionIndex, FunctionHandleIndex, SignatureIndex, SignatureToken, StructDefinitionIndex, }, - normalized::Type as MType, views::{FunctionDefinitionView, FunctionHandleView, StructHandleView}, CompiledModule, }; @@ -1997,6 +1998,7 @@ impl GlobalEnv { } /// Attempt to compute a struct type for (`mid`, `sid`, `ts`). + #[allow(deprecated)] pub fn get_struct_type(&self, mid: ModuleId, sid: StructId, ts: &[Type]) -> Option { let menv = self.get_module(mid); Some(MType::Struct { diff --git a/third_party/move/move-model/src/ty.rs b/third_party/move/move-model/src/ty.rs index db2c37a6db96b2..4e0110e149f688 100644 --- a/third_party/move/move-model/src/ty.rs +++ b/third_party/move/move-model/src/ty.rs @@ -14,10 +14,9 @@ use crate::{ symbol::Symbol, }; use itertools::Itertools; -use move_binary_format::{ - file_format::{Ability, AbilitySet, TypeParameterIndex}, - normalized::Type as MType, -}; +use move_binary_format::file_format::{Ability, AbilitySet, TypeParameterIndex}; +#[allow(deprecated)] +use move_binary_format::normalized::Type as MType; use move_core_types::{ language_storage::{StructTag, TypeTag}, u256::U256, @@ -682,6 +681,7 @@ impl PrimitiveType { } /// Attempt to convert this type into a normalized::Type + #[allow(deprecated)] pub fn into_normalized_type(self) -> Option { use PrimitiveType::*; Some(match self { @@ -1183,6 +1183,7 @@ impl Type { } /// Attempt to convert this type into a normalized::Type + #[allow(deprecated)] pub fn into_struct_type(self, env: &GlobalEnv) -> Option { use Type::*; match self { @@ -1192,6 +1193,7 @@ impl Type { } /// Attempt to convert this type into a normalized::Type + #[allow(deprecated)] pub fn into_normalized_type(self, env: &GlobalEnv) -> Option { use Type::*; match self { diff --git a/third_party/move/move-vm/runtime/src/lib.rs b/third_party/move/move-vm/runtime/src/lib.rs index d8dd7054ca7cdd..770862ddf2fbbd 100644 --- a/third_party/move/move-vm/runtime/src/lib.rs +++ b/third_party/move/move-vm/runtime/src/lib.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #![forbid(unsafe_code)] +#![deny(deprecated)] //! The core Move VM logic. //! diff --git a/third_party/move/move-vm/runtime/src/runtime.rs b/third_party/move/move-vm/runtime/src/runtime.rs index 5731734b8e6820..13e505014bbeba 100644 --- a/third_party/move/move-vm/runtime/src/runtime.rs +++ b/third_party/move/move-vm/runtime/src/runtime.rs @@ -124,7 +124,9 @@ impl VMRuntime { self.loader .load_module(&module_id, data_store, module_store)?; let old_module = old_module_ref.module(); + #[allow(deprecated)] let old_m = normalized::Module::new(old_module); + #[allow(deprecated)] let new_m = normalized::Module::new(module); compat .check(&old_m, &new_m) diff --git a/third_party/move/tools/move-bytecode-utils/src/layout.rs b/third_party/move/tools/move-bytecode-utils/src/layout.rs index fc2a50aed1c1b5..d649a8587b356e 100644 --- a/third_party/move/tools/move-bytecode-utils/src/layout.rs +++ b/third_party/move/tools/move-bytecode-utils/src/layout.rs @@ -2,6 +2,8 @@ // Copyright (c) The Move Contributors // SPDX-License-Identifier: Apache-2.0 +#![allow(deprecated)] + use crate::compiled_module_viewer::CompiledModuleView; use anyhow::{anyhow, bail}; use move_binary_format::{ @@ -196,6 +198,7 @@ impl<'a, T: CompiledModuleView> SerdeLayoutBuilder<'a, T> { declaring_module.borrow().name() ) }); + #[allow(deprecated)] let normalized_struct = Struct::new(declaring_module.borrow(), def).1; assert_eq!( normalized_struct.type_parameters.len(), diff --git a/tools/compute-module-expansion-size/Cargo.toml b/tools/compute-module-expansion-size/Cargo.toml new file mode 100644 index 00000000000000..c87bec5a4b5d1c --- /dev/null +++ b/tools/compute-module-expansion-size/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "compute-module-expansion-size" +version = "0.1.0" + +# Workspace inherited keys +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +futures = { workspace = true } +move-binary-format = { workspace = true } +move-core-types = { workspace = true } +rayon = { workspace = true } +tokio = { workspace = true } diff --git a/tools/compute-module-expansion-size/src/main.rs b/tools/compute-module-expansion-size/src/main.rs new file mode 100644 index 00000000000000..e1b9e131058131 --- /dev/null +++ b/tools/compute-module-expansion-size/src/main.rs @@ -0,0 +1,115 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{format_err, Result}; +use clap::Parser; +use move_binary_format::CompiledModule; +use move_core_types::language_storage::ModuleId; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{fmt::Write, path::PathBuf}; +use tokio::fs; + +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + /// Path to the module directory + #[clap(long, value_parser)] + path: String, +} + +async fn list_files_with_extension( + dir: &str, + extension: &str, +) -> Result, std::io::Error> { + let mut paths = vec![]; + let mut stack = vec![PathBuf::from(dir)]; + + while let Some(curr_dir) = stack.pop() { + let mut entries = fs::read_dir(curr_dir).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.is_file() && path.extension().map_or(false, |ext| ext == extension) { + paths.push(path); + } else if path.is_dir() { + stack.push(path); + } + } + } + + Ok(paths) +} + +async fn read_modules(dir: &str) -> Result>> { + let paths = list_files_with_extension(dir, "mv").await?; + + let reads = paths + .into_iter() + .map(|path| async move { fs::read(path).await }); + + futures::future::join_all(reads) + .await + .into_iter() + .map(|res| res.map_err(|_e| format_err!("failed to read file"))) + .collect() +} + +#[derive(Debug, Clone)] +struct ModuleInfo { + size: usize, + expansion_size: u64, +} + +fn extract_module_info_single(bytes: &[u8]) -> Result<(ModuleId, ModuleInfo)> { + let res = CompiledModule::deserialize(bytes); + let module = res?; + + let expansion_size = + move_binary_format::check_complexity::check_module_complexity(&module, u64::MAX).unwrap(); + + Ok((module.self_id().clone(), ModuleInfo { + size: bytes.len(), + expansion_size, + })) +} + +fn extract_module_info(modules: &Vec>) -> Result> { + Ok(rayon::scope(move |_s| { + modules + .par_iter() + .flat_map(|bytes| extract_module_info_single(bytes).ok()) + }) + .collect()) +} + +fn render_data_csv<'a>( + info: impl IntoIterator, +) -> Result { + let mut s = String::new(); + writeln!(s, "module,size,\"expansion size\"")?; + + for (module_id, info) in info.into_iter() { + writeln!(s, "{},{},{}", module_id, info.size, info.expansion_size)?; + } + + Ok(s) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + + let modules = read_modules(&args.path).await?; + println!("Read {} modules", modules.len()); + + let info = extract_module_info(&modules)?; + println!( + "Deserialized {} out of {} modules", + info.len(), + modules.len() + ); + + let csv = render_data_csv(info.iter().map(|(id, info)| (id, info)))?; + fs::write("modules.csv", csv).await?; + + Ok(()) +} diff --git a/types/src/on_chain_config/timed_features.rs b/types/src/on_chain_config/timed_features.rs index a548c0fee44426..74b9cf568b320a 100644 --- a/types/src/on_chain_config/timed_features.rs +++ b/types/src/on_chain_config/timed_features.rs @@ -14,6 +14,7 @@ pub const END_OF_TIME: u64 = 4102444799000; /* Thursday, December 31, 2099 11:59 pub enum TimedFeatureFlag { DisableInvariantViolationCheckInSwapLoc, LimitTypeTagSize, + ModuleComplexityCheck, } /// Representation of features that are gated by the block timestamps. @@ -41,6 +42,7 @@ impl TimedFeatureOverride { Some(match self { Replay => match flag { LimitTypeTagSize => true, + ModuleComplexityCheck => true, // Add overrides for replay here. _ => return None, }, @@ -58,6 +60,9 @@ impl TimedFeatureFlag { (DisableInvariantViolationCheckInSwapLoc, TESTNET) => NOT_YET_SPECIFIED, (DisableInvariantViolationCheckInSwapLoc, MAINNET) => NOT_YET_SPECIFIED, + (ModuleComplexityCheck, TESTNET) => 1719356400000, /* Tuesday, June 21, 2024 16:00:00 AM GMT-07:00 */ + (ModuleComplexityCheck, MAINNET) => 1720033200000, /* Wednesday, July 3, 2024 12:00:00 AM GMT-07:00 */ + // If unspecified, a timed feature is considered enabled from the very beginning of time. _ => 0, }