move: enable bytecode as dependencies 1/n (#15035)

## Description 

Make it possible to build packages with bytecode dependencies.

This is part of the work to enable compiling against on-chain.

## Test Plan 

There is a unit test and it was tested manually.

---
If your changes are not user-facing and not a breaking change, you can
skip the following section. Otherwise, please indicate what changed, and
then add to the Release Notes section as highlighted during the release
process.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

It is now possible to build packages with dependencies with bytecode
modules.

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/B/Move.toml

@@ -0,0 +1,9 @@
+[package]
+name = "Bar"
+version = "0.0.0"
+
+[addresses]
+B = "0x2"
+
+[dependencies]
+Foo = { local = "../C" }

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/B/sources/Bar.move

@@ -0,0 +1,7 @@
+module B::Bar {
+    use C::Foo;
+
+    public fun foo(): u64 {
+        Foo::bar()
+    }
+}

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/C/Move.toml

@@ -0,0 +1,6 @@
+[package]
+name = "Foo"
+version = "0.0.0"
+
+[addresses]
+C = "0x3"

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/C/sources/Foo.move

@@ -0,0 +1,5 @@
+module C::Foo {
+    public fun bar(): u64 {
+        1
+    }
+}

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/Move.toml

@@ -0,0 +1,10 @@
+[package]
+name = "A"
+version = "0.0.0"
+
+[addresses]
+A = "0x2"
+
+[dependencies]
+Bar = { local = "./B" }
+Foo = { local = "./C" }

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/args.exp

@@ -0,0 +1,13 @@
+Command `build -v -p ./C`:
+BUILDING Foo
+External Command `mv ./C/sources/Foo.move ./C/sources/Foo.move_old`:
+External Command `rm -rf ./C/build/Foo/sources`:
+Command `build -v -p ./B`:
+INCLUDING DEPENDENCY Foo
+BUILDING Bar
+External Command `mv ./B/sources/Bar.move ./C/sources/Bar.move_old`:
+External Command `rm -rf ./B/build/Bar/sources`:
+Command `build -v`:
+INCLUDING DEPENDENCY Bar
+INCLUDING DEPENDENCY Foo
+BUILDING A

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/args.txt

@@ -0,0 +1,7 @@
+build -v -p ./C
+> mv ./C/sources/Foo.move ./C/sources/Foo.move_old
+> rm -rf ./C/build/Foo/sources
+build -v -p ./B
+> mv ./B/sources/Bar.move ./C/sources/Bar.move_old
+> rm -rf ./B/build/Bar/sources
+build -v

external-crates/move/crates/move-cli/tests/build_tests/build_with_bytecode/sources/A.move

@@ -0,0 +1,8 @@
+module A::A {
+    use B::Bar;
+    use C::Foo;
+
+    public fun foo(): u64 {
+        Bar::foo() + Foo::bar()
+    }
+}

external-crates/move/crates/move-compiler/src/command_line/compiler.rs

@@ -677,6 +677,8 @@ fn generate_interface_files_for_deps(
     let interface_files_paths =
         generate_interface_files(deps, interface_files_dir_opt, module_to_named_address, true)?;
     deps.extend(interface_files_paths);
+    // Remove bytecode files
+    deps.retain(|p| !p.path.as_str().ends_with(MOVE_COMPILED_EXTENSION));
     Ok(())
 }
 

external-crates/move/crates/move-compiler/src/interface_generator.rs

@@ -15,6 +15,8 @@ use move_binary_format::{
 use move_core_types::language_storage::ModuleId;
 use std::{collections::BTreeMap, fs};
 
+pub const NATIVE_INTERFACE: &str = "native_interface";
+
 macro_rules! push_line {
     ($s:ident, $e:expr) => {{
         $s = format!("{}{}\n", $s, $e);
@@ -88,6 +90,7 @@ pub fn write_module_to_string(
         members.push(format!("    {}", DISCLAIMER));
     }
     for fdef in externally_visible_funs {
+        members.push(format!(" #[{}]", NATIVE_INTERFACE));
         members.push(write_function_def(&mut context, fdef));
     }
     if has_externally_visible_funs {

external-crates/move/crates/move-package/Cargo.toml

@@ -23,6 +23,7 @@ regex.workspace = true
 treeline.workspace = true
 once_cell.workspace = true
 named-lock.workspace = true
+itertools.workspace = true
 
 move-binary-format.workspace = true
 move-compiler.workspace = true

external-crates/move/crates/move-package/src/compilation/build_plan.rs

@@ -14,7 +14,10 @@ use move_compiler::{
 };
 use std::{collections::BTreeSet, io::Write, path::Path};
 
-use super::package_layout::CompiledPackageLayout;
+use super::{
+    compiled_package::{DependencyInfo, ModuleFormat},
+    package_layout::CompiledPackageLayout,
+};
 
 #[derive(Debug, Clone)]
 pub struct BuildPlan {
@@ -87,19 +90,30 @@ impl BuildPlan {
                     .package_table
                     .get(&package_name)
                     .unwrap();
-                let dep_source_paths = dep_package
+                let mut dep_source_paths = dep_package
                     .get_sources(&self.resolution_graph.build_options)
                     .unwrap();
-                (
-                    package_name,
-                    immediate_dependencies_names.contains(&package_name),
-                    dep_source_paths,
-                    &dep_package.resolved_table,
-                    dep_package.compiler_config(
+                let mut source_available = true;
+                // If source is empty, search bytecode(mv) files
+                if dep_source_paths.is_empty() {
+                    dep_source_paths = dep_package.get_bytecodes().unwrap();
+                    source_available = false;
+                }
+                DependencyInfo {
+                    name: package_name,
+                    is_immediate: immediate_dependencies_names.contains(&package_name),
+                    source_paths: dep_source_paths,
+                    address_mapping: &dep_package.resolved_table,
+                    compiler_config: dep_package.compiler_config(
                         /* is_dependency */ true,
                         &self.resolution_graph.build_options,
                     ),
-                )
+                    module_format: if source_available {
+                        ModuleFormat::Source
+                    } else {
+                        ModuleFormat::Bytecode
+                    },
+                }
             })
             .collect();
 

external-crates/move/crates/move-package/src/compilation/compiled_package.rs

@@ -13,6 +13,7 @@ use crate::{
 };
 use anyhow::{ensure, Result};
 use colored::Colorize;
+use itertools::{Either, Itertools};
 use move_binary_format::file_format::CompiledModule;
 use move_bytecode_source_map::utils::source_map_from_file;
 use move_bytecode_utils::Modules;
@@ -116,6 +117,22 @@ impl CompilationCachingStatus {
     }
 }
 
+#[derive(Debug, Clone)]
+pub enum ModuleFormat {
+    Source,
+    Bytecode,
+}
+
+#[derive(Debug, Clone)]
+pub struct DependencyInfo<'a> {
+    pub name: Symbol,
+    pub is_immediate: bool,
+    pub source_paths: Vec<Symbol>,
+    pub address_mapping: &'a ResolvedTable,
+    pub compiler_config: PackageConfig,
+    pub module_format: ModuleFormat,
+}
+
 impl OnDiskCompiledPackage {
     pub fn from_path(p: &Path) -> Result<Self> {
         let (buf, build_path) = if try_exists(p)? && extension_equals(p, "yaml") {
@@ -409,13 +426,7 @@ impl CompiledPackage {
         w: &mut W,
         project_root: &Path,
         resolved_package: Package,
-        transitive_dependencies: Vec<(
-            /* name */ Symbol,
-            /* is immediate */ bool,
-            /* source paths */ Vec<Symbol>,
-            /* address mapping */ &ResolvedTable,
-            /* compiler config */ PackageConfig,
-        )>,
+        transitive_dependencies: Vec<DependencyInfo>,
         resolution_graph: &ResolvedGraph,
         mut compiler_driver: impl FnMut(
             Compiler,
@@ -424,24 +435,11 @@ impl CompiledPackage {
     ) -> Result<CompiledPackage> {
         let immediate_dependencies = transitive_dependencies
             .iter()
-            .filter(|(_, is_immediate, _, _, _)| *is_immediate)
-            .map(|(name, _, _, _, _)| *name)
-            .collect::<Vec<_>>();
-        let transitive_dependencies = transitive_dependencies
-            .into_iter()
-            .map(
-                |(name, _is_immediate, source_paths, address_mapping, config)| {
-                    (name, source_paths, address_mapping, config)
-                },
-            )
+            .filter(|&dep| dep.is_immediate)
+            .map(|dep| dep.name)
             .collect::<Vec<_>>();
-        for (dep_package_name, _, _, _) in &transitive_dependencies {
-            writeln!(
-                w,
-                "{} {}",
-                "INCLUDING DEPENDENCY".bold().green(),
-                dep_package_name
-            )?;
+        for dep in &transitive_dependencies {
+            writeln!(w, "{} {}", "INCLUDING DEPENDENCY".bold().green(), dep.name)?;
         }
         let root_package_name = resolved_package.source_package.package.name;
         writeln!(w, "{} {}", "BUILDING".bold().green(), root_package_name)?;
@@ -453,11 +451,30 @@ impl CompiledPackage {
             transitive_dependencies,
         )?;
         let flags = resolution_graph.build_options.compiler_flags();
+        // Partition deps_package according whether src is available
+        let (src_deps, bytecode_deps): (Vec<_>, Vec<_>) = deps_package_paths
+            .clone()
+            .into_iter()
+            .partition_map(|(p, b)| match b {
+                ModuleFormat::Source => Either::Left(p),
+                ModuleFormat::Bytecode => Either::Right(p),
+            });
+        // If bytecode dependency is not empty, do not allow renaming
+        if !bytecode_deps.is_empty() {
+            if let Some(pkg_name) = resolution_graph.contains_renaming() {
+                anyhow::bail!(
+                    "Found address renaming in package '{}' when \
+                    building with bytecode dependencies -- this is currently not supported",
+                    pkg_name
+                )
+            }
+        }
+
         // invoke the compiler
-        let mut paths = deps_package_paths.clone();
+        let mut paths = src_deps;
         paths.push(sources_package_paths.clone());
 
-        let compiler = Compiler::from_package_paths(paths, vec![])
+        let compiler = Compiler::from_package_paths(paths, bytecode_deps)
             .unwrap()
             .set_flags(flags);
         let (file_map, all_compiled_units) = compiler_driver(compiler)?;
@@ -481,7 +498,7 @@ impl CompiledPackage {
         if resolution_graph.build_options.generate_docs {
             let model = run_model_builder_with_options(
                 vec![sources_package_paths],
-                deps_package_paths,
+                deps_package_paths.into_iter().map(|(p, _)| p).collect_vec(),
                 ModelBuilderOptions::default(),
                 None,
             )?;
@@ -711,30 +728,29 @@ pub(crate) fn apply_named_address_renaming(
 pub(crate) fn make_source_and_deps_for_compiler(
     resolution_graph: &ResolvedGraph,
     root: &Package,
-    deps: Vec<(
-        /* name */ Symbol,
-        /* source paths */ Vec<Symbol>,
-        /* address mapping */ &ResolvedTable,
-        /* compiler config */ PackageConfig,
-    )>,
+    deps: Vec<DependencyInfo>,
 ) -> Result<(
     /* sources */ PackagePaths,
-    /* deps */ Vec<PackagePaths>,
+    /* deps */ Vec<(PackagePaths, ModuleFormat)>,
 )> {
     let deps_package_paths = deps
         .into_iter()
-        .map(|(name, source_paths, resolved_table, config)| {
-            let paths = source_paths
+        .map(|dep| {
+            let paths = dep
+                .source_paths
                 .into_iter()
                 .collect::<BTreeSet<_>>()
                 .into_iter()
                 .collect::<Vec<_>>();
-            let named_address_map = named_address_mapping_for_compiler(resolved_table);
-            Ok(PackagePaths {
-                name: Some((name, config)),
-                paths,
-                named_address_map,
-            })
+            let named_address_map = named_address_mapping_for_compiler(dep.address_mapping);
+            Ok((
+                PackagePaths {
+                    name: Some((dep.name, dep.compiler_config)),
+                    paths,
+                    named_address_map,
+                },
+                dep.module_format,
+            ))
         })
         .collect::<Result<Vec<_>>>()?;
     let root_named_addrs = apply_named_address_renaming(