Rename PyInit syms to avoid clashes

ci/azure-pipelines-template.yml

@@ -48,7 +48,8 @@ jobs:
     condition: ne( variables['rustup_toolchain'], 'nightly' )
 
   - ${{ if ne(parameters.name, 'Windows') }}:
-    # TODO add `--pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1` once
+    # TODO add `--pip-install appdirs==1.4.3 --pip-install zero-buffer==0.5.1` and
+    # `--pip-install simplejson==3.17.0 --pip-install MarkupSafe==1.1.1` once
     # auto-generated pyoxidizer.bzl support this functionality again.
     - script: |
         cargo run --bin pyoxidizer -- init ~/pyapp

ci/pyapp.py

@@ -10,4 +10,10 @@
 #except AttributeError:
 #    pass
 
+#import markupsafe._speedups
+#import simplejson._speedups
+
+#import markupsafe
+#import simplejson
+
 print("hello, world")

pyoxidizer/Cargo.toml

@@ -20,6 +20,12 @@ path = "src/lib.rs"
 [build-dependencies]
 vergen = "3"
 
+# Contains many fixes
+[dependencies.object]
+git = "https://github.com/gimli-rs/object.git"
+rev = "dc1ca8cddaade970f125da9713a1455f36ba9e85"
+features = ["read", "std", "write", "compression", "wasm"]
+
 [dependencies]
 anyhow = "1.0"
 byteorder = "1.2"

pyoxidizer/src/distutils/_msvccompiler.py

@@ -265,7 +265,7 @@ def initialize(self, plat_name=None):
         # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib
         # later to dynamically link to ucrtbase but not vcruntime.
         self.compile_options = [
-            '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG'
+            '/nologo', '/Ox', '/W3', '/DNDEBUG'
         ]
         self.compile_options.append('/MD' if self._vcruntime_redist else '/MT')
 
@@ -586,6 +586,11 @@ def extension_link_shared_object(self,
         # directory that doesn't outlive this process.
         object_paths = []
         for i, o in enumerate(objects):
+            if 'libffi_msvc' in o:
+                print('Ignored static {}'.format(o))
+                # https://github.com/indygreg/python-build-standalone/issues/23
+                # cffi includes a near replica of CPython's custom libffi.
+                continue
             p = os.path.join(dest_path, '%s.%d.o' % (name, i))
             shutil.copyfile(o, p)
             object_paths.append(p)

pyoxidizer/src/py_packaging/libpython.rs

@@ -14,6 +14,7 @@ use std::path::{Path, PathBuf};
 use super::bytecode::{BytecodeCompiler, CompileMode};
 use super::distribution::{ExtensionModule, LicenseInfo, ParsedPythonDistribution};
 use super::embedded_resource::EmbeddedPythonResources;
+use super::object::rename_init;
 use super::resource::BuiltExtensionModule;
 
 pub const PYTHON_IMPORTER: &[u8] = include_bytes!("memoryimporter.py");
@@ -104,23 +105,44 @@ pub fn make_config_c(
     }
 
     for em in built_extension_modules.values() {
-        lines.push(format!("extern PyObject* {}(void);", em.init_fn));
+        let ambiguous_line = format!("extern PyObject* {}(void);", em.init_fn);
+
+        if lines.contains(&ambiguous_line) {
+            lines.push(format!(
+                "extern PyObject* PyInit_{}(void);",
+                em.name.replace(".", "_")
+            ));
+        } else {
+            lines.push(ambiguous_line);
+        }
     }
 
     lines.push(String::from("struct _inittab _PyImport_Inittab[] = {"));
 
+    let mut ambiguous_init_fns: Vec<String> = Vec::new();
+
     for em in extension_modules.values() {
         if let Some(init_fn) = &em.init_fn {
             if init_fn == "NULL" {
                 continue;
             }
 
             lines.push(format!("{{\"{}\", {}}},", em.module, init_fn));
+            ambiguous_init_fns.push(init_fn.to_string());
         }
     }
 
     for em in built_extension_modules.values() {
-        lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn));
+        if ambiguous_init_fns.contains(&em.init_fn) {
+            lines.push(format!(
+                "{{\"{}\", PyInit_{}}},",
+                em.name,
+                em.name.replace(".", "_")
+            ));
+        } else {
+            lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn));
+            ambiguous_init_fns.push(em.init_fn.clone());
+        }
     }
 
     lines.push(String::from("{0, 0}"));
@@ -275,12 +297,20 @@ pub fn link_libpython(
         // TODO handle static/dynamic libraries.
     }
 
+    let mut ambiguous_init_fns: Vec<String> = Vec::new();
+
     warn!(
         logger,
         "resolving inputs for {} extension modules...",
         extension_modules.len() + built_extension_modules.len()
     );
     for (name, em) in extension_modules {
+        if let Some(init_fn) = &em.init_fn {
+            if init_fn != "NULL" {
+                ambiguous_init_fns.push(init_fn.to_string());
+            }
+        }
+
         if em.builtin_default {
             continue;
         }
@@ -329,10 +359,21 @@ pub fn link_libpython(
             em.object_file_data.len(),
             name
         );
+
         for (i, object_data) in em.object_file_data.iter().enumerate() {
             let out_path = temp_dir_path.join(format!("{}.{}.o", name, i));
 
-            fs::write(&out_path, object_data)?;
+            if i == em.object_file_data.len() - 1 && ambiguous_init_fns.contains(&em.init_fn) {
+                match rename_init(logger, name, object_data) {
+                    Ok(val) => fs::write(&out_path, val)?,
+                    Err(_) => {
+                        fs::write(&out_path, object_data)?
+                    }
+                };
+            } else {
+                fs::write(&out_path, object_data)?;
+            }
+
             build.object(&out_path);
         }
 
@@ -341,6 +382,10 @@ pub fn link_libpython(
             needed_libraries_external.insert(&library);
         }
 
+        if !ambiguous_init_fns.contains(&em.init_fn) {
+            ambiguous_init_fns.push(em.init_fn.clone());
+        }
+
         // TODO do something with library_dirs.
     }
 

pyoxidizer/src/py_packaging/mod.rs

@@ -10,6 +10,7 @@ pub mod distutils;
 pub mod embedded_resource;
 pub mod fsscan;
 pub mod libpython;
+pub mod object;
 pub mod pip;
 pub mod pyembed;
 pub mod resource;

pyoxidizer/src/py_packaging/object.rs

@@ -0,0 +1,196 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use object::{
+    write, Object, ObjectSection, RelocationTarget, SectionKind, SymbolFlags, SymbolKind,
+    SymbolSection,
+};
+use slog::{info, warn};
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt;
+
+#[derive(Debug, Clone)]
+pub struct NoRewriteError;
+
+impl fmt::Display for NoRewriteError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "no object rewriting was performed")
+    }
+}
+
+impl Error for NoRewriteError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        // Generic error, underlying cause isn't tracked.
+        None
+    }
+}
+
+/// Rename object syn PyInit_foo to PyInit_<full_name> to avoid clashes
+pub fn rename_init(
+    logger: &slog::Logger,
+    name: &String,
+    object_data: &[u8],
+) -> Result<Vec<u8>, NoRewriteError> {
+    let mut rewritten = false;
+
+    let name_prefix = name.split('.').next().unwrap();
+
+    let in_object = match object::File::parse(object_data) {
+        Ok(object) => object,
+        Err(err) => {
+            let magic = [
+                object_data[0],
+                object_data[1],
+                object_data[2],
+                object_data[3],
+            ];
+            warn!(
+                logger,
+                "Failed to parse compiled object for {} (magic {:x?}): {}", name, magic, err
+            );
+            return Err(NoRewriteError);
+        }
+    };
+
+    let mut out_object = write::Object::new(in_object.format(), in_object.architecture());
+    out_object.flags = in_object.flags();
+
+    let mut out_sections = HashMap::new();
+    for in_section in in_object.sections() {
+        if in_section.kind() == SectionKind::Metadata {
+            continue;
+        }
+        let section_id = out_object.add_section(
+            in_section.segment_name().unwrap_or("").as_bytes().to_vec(),
+            in_section.name().unwrap_or("").as_bytes().to_vec(),
+            in_section.kind(),
+        );
+        let out_section = out_object.section_mut(section_id);
+        if out_section.is_bss() {
+            out_section.append_bss(in_section.size(), in_section.align());
+        } else {
+            out_section.set_data(in_section.uncompressed_data().into(), in_section.align());
+        }
+        out_section.flags = in_section.flags();
+        out_sections.insert(in_section.index(), section_id);
+    }
+
+    let mut out_symbols = HashMap::new();
+    for (symbol_index, in_symbol) in in_object.symbols() {
+        if in_symbol.kind() == SymbolKind::Null {
+            // This is normal in ELF
+            info!(logger, "object symbol name kind 'null' discarded",);
+            continue;
+        }
+        let in_sym_name = in_symbol.name().unwrap_or("");
+        if in_symbol.kind() == SymbolKind::Unknown {
+            warn!(
+                logger,
+                "object symbol name {} kind 'unknown' encountered", in_sym_name,
+            );
+        }
+        let (section, value) = match in_symbol.section() {
+            SymbolSection::Unknown => panic!("unknown symbol section for {:?}", in_symbol),
+            SymbolSection::Undefined => (write::SymbolSection::Undefined, in_symbol.address()),
+            SymbolSection::Absolute => (write::SymbolSection::Absolute, in_symbol.address()),
+            SymbolSection::Common => (write::SymbolSection::Common, in_symbol.address()),
+            SymbolSection::Section(index) => (
+                write::SymbolSection::Section(*out_sections.get(&index).unwrap()),
+                in_symbol.address() - in_object.section_by_index(index).unwrap().address(),
+            ),
+        };
+        let flags = match in_symbol.flags() {
+            SymbolFlags::None => SymbolFlags::None,
+            SymbolFlags::Elf { st_info, st_other } => SymbolFlags::Elf { st_info, st_other },
+            SymbolFlags::MachO { n_desc } => SymbolFlags::MachO { n_desc },
+            SymbolFlags::CoffSection {
+                selection,
+                associative_section,
+            } => {
+                let associative_section = *out_sections.get(&associative_section).unwrap();
+                SymbolFlags::CoffSection {
+                    selection,
+                    associative_section,
+                }
+            }
+        };
+        let sym_name = if !in_sym_name.starts_with("$")
+            && in_sym_name.contains("PyInit_")
+            && !in_sym_name.contains(name_prefix)
+        {
+            "PyInit_".to_string() + &name.replace(".", "_")
+        } else {
+            String::from(in_sym_name)
+        };
+        if sym_name != in_sym_name {
+            warn!(
+                logger,
+                "renaming object symbol name {} to {}", in_sym_name, sym_name,
+            );
+
+            rewritten = true;
+        }
+
+        let out_symbol = write::Symbol {
+            name: sym_name.as_bytes().to_vec(),
+            value,
+            size: in_symbol.size(),
+            kind: in_symbol.kind(),
+            scope: in_symbol.scope(),
+            weak: in_symbol.is_weak(),
+            section,
+            flags,
+        };
+
+        let symbol_id = out_object.add_symbol(out_symbol);
+        out_symbols.insert(symbol_index, symbol_id);
+        info!(
+            logger,
+            "added object symbol name {} kind {:?}", sym_name, in_symbol,
+        );
+    }
+
+    if !rewritten {
+        warn!(logger, "no symbol name rewriting occurred for {}", name);
+        return Err(NoRewriteError);
+    }
+
+    for in_section in in_object.sections() {
+        if in_section.kind() == SectionKind::Metadata {
+            continue;
+        }
+        let out_section = *out_sections.get(&in_section.index()).unwrap();
+        for (offset, in_relocation) in in_section.relocations() {
+            let symbol = match in_relocation.target() {
+                RelocationTarget::Symbol(symbol) => *out_symbols.get(&symbol).unwrap(),
+                RelocationTarget::Section(section) => {
+                    out_object.section_symbol(*out_sections.get(&section).unwrap())
+                }
+            };
+            let out_relocation = write::Relocation {
+                offset,
+                size: in_relocation.size(),
+                kind: in_relocation.kind(),
+                encoding: in_relocation.encoding(),
+                symbol,
+                addend: in_relocation.addend(),
+            };
+            out_object
+                .add_relocation(out_section, out_relocation)
+                .unwrap();
+        }
+    }
+
+    info!(logger, "serialising object for {} ..", name);
+
+    match out_object.write() {
+        Ok(obj) => Ok(obj),
+        Err(err) => {
+            warn!(logger, "object {} serialisation failed: {}", name, err);
+
+            Err(NoRewriteError)
+        }
+    }
+}