Merge wasmerio#2342

2342: Engine native (slim) r=syrusakbary a=syrusakbary  # Description This PR is the slim version of wasmerio#2272. It advances on: * [x] Stops encoding the endian into the serialized data (rkyv already does that for us since `0.6.1`) * [x] Sets the proper function lengths in the native engine * [x] Catch signal traps only when the pc is originated from a Wasm function, and not in the host (fixing wasmerio#2328)  # Review - [ ] Add a short description of the change to the CHANGELOG.md file Co-authored-by: Syrus <[email protected]> Co-authored-by: Syrus Akbary <[email protected]> Co-authored-by: Mark McCaskey <[email protected]>
Web3-Net · May 26, 2021 · 256f696 · 256f696
2 parents 50f48ff + c867e6e
commit 256f696
Show file tree

Hide file tree

Showing 14 changed files with 178 additions and 78 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/deny.toml b/deny.toml
@@ -169,7 +169,7 @@ deny = [
 ]
 # Certain crates/versions that will be skipped when doing duplicate detection.
 skip = [
-    { name = "cfg-if", version = "=0.1.10" },
+    { name = "cfg-if", version = "0.1.10" },
     { name = "strsim", version = "=0.8.0" },
     { name = "semver", version = "=0.9.0" },
     { name = "semver-parser", version = "=0.7.0" },

diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml
@@ -21,8 +21,8 @@ wasmer-engine = { path = "../engine", version = "1.0.2" }
 wasmer-engine-jit = { path = "../engine-jit", version = "1.0.2", optional = true }
 wasmer-engine-native = { path = "../engine-native", version = "1.0.2", optional = true }
 wasmer-types = { path = "../types", version = "1.0.2" }
-indexmap = { version = "1.4", features = ["serde-1"] }
-cfg-if = "0.1"
+indexmap = { version = "1.6", features = ["serde-1"] }
+cfg-if = "1.0"
 wat = { version = "1.0", optional = true }
 thiserror = "1.0"
 more-asserts = "0.2"

diff --git a/lib/engine-jit/Cargo.toml b/lib/engine-jit/Cargo.toml
@@ -17,7 +17,7 @@ wasmer-vm = { path = "../vm", version = "1.0.2", features = ["enable-rkyv"] }
 wasmer-engine = { path = "../engine", version = "1.0.2" }
 # flexbuffers = { path = "../../../flatbuffers/rust/flexbuffers", version = "0.1.0" }
 region = "2.2"
-cfg-if = "0.1"
+cfg-if = "1.0"
 leb128 = "0.2"
 rkyv = "0.6.1"
 loupe = "0.1"

diff --git a/lib/engine-jit/src/serialize.rs b/lib/engine-jit/src/serialize.rs
@@ -42,35 +42,23 @@ fn to_serialize_error(err: impl std::error::Error) -> SerializeError {
     SerializeError::Generic(format!("{}", err))
 }
 
-cfg_if::cfg_if! {
-    if #[cfg(target_endian = "big")] {
-        const HOST_ENDIAN: u8 = b'b';
-    } else if #[cfg(target_endian = "little")] {
-        const HOST_ENDIAN: u8 = b'l';
-    }
-    else {
-        compile_error!("Endian not supported by the host");
-    }
-}
-
 impl SerializableModule {
     /// Serialize a Module into bytes
     /// The bytes will have the following format:
-    /// RKYV serialization (any length) + POS (8 bytes) + Endian (1 byte)
+    /// RKYV serialization (any length) + POS (8 bytes)
     pub fn serialize(&self) -> Result<Vec<u8>, SerializeError> {
         let mut serializer = SharedSerializerAdapter::new(WriteSerializer::new(vec![]));
         let pos = serializer
             .serialize_value(self)
             .map_err(to_serialize_error)? as u64;
         let mut serialized_data = serializer.into_inner().into_inner();
         serialized_data.extend_from_slice(&pos.to_le_bytes());
-        serialized_data.extend_from_slice(&[HOST_ENDIAN]);
         Ok(serialized_data)
     }
 
     /// Deserialize a Module from a slice.
     /// The slice must have the following format:
-    /// RKYV serialization (any length) + POS (8 bytes) + Endian (1 byte)
+    /// RKYV serialization (any length) + POS (8 bytes)
     ///
     /// # Safety
     ///
@@ -91,26 +79,16 @@ impl SerializableModule {
     unsafe fn archive_from_slice<'a>(
         metadata_slice: &'a [u8],
     ) -> Result<&'a ArchivedSerializableModule, DeserializeError> {
-        if metadata_slice.len() < 9 {
+        if metadata_slice.len() < 8 {
             return Err(DeserializeError::Incompatible(
                 "invalid serialized data".into(),
             ));
         }
         let mut pos: [u8; 8] = Default::default();
-        let endian = metadata_slice[metadata_slice.len() - 1];
-        if endian != HOST_ENDIAN {
-            return Err(DeserializeError::Incompatible(
-                format!(
-                    "incompatible endian. Received {} but expected {}",
-                    endian, HOST_ENDIAN
-                )
-                .into(),
-            ));
-        }
-        pos.copy_from_slice(&metadata_slice[metadata_slice.len() - 9..metadata_slice.len() - 1]);
+        pos.copy_from_slice(&metadata_slice[metadata_slice.len() - 8..metadata_slice.len()]);
         let pos: u64 = u64::from_le_bytes(pos);
         Ok(archived_value::<SerializableModule>(
-            &metadata_slice[..metadata_slice.len() - 9],
+            &metadata_slice[..metadata_slice.len() - 8],
             pos as usize,
         ))
     }

diff --git a/lib/engine-native/Cargo.toml b/lib/engine-native/Cargo.toml
@@ -17,7 +17,7 @@ wasmer-vm = { path = "../vm", version = "1.0.2", features = ["enable-rkyv"] }
 wasmer-engine = { path = "../engine", version = "1.0.2" }
 wasmer-object = { path = "../object", version = "1.0.2" }
 serde = { version = "1.0", features = ["derive", "rc"] }
-cfg-if = "0.1"
+cfg-if = "1.0"
 tracing = "0.1"
 leb128 = "0.2"
 libloading = "0.7"

diff --git a/lib/engine-native/src/artifact.rs b/lib/engine-native/src/artifact.rs
@@ -11,17 +11,23 @@ use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
 #[cfg(feature = "compiler")]
 use std::process::Command;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use tempfile::NamedTempFile;
 #[cfg(feature = "compiler")]
 use tracing::trace;
-use wasmer_compiler::{CompileError, Features, OperatingSystem, Symbol, SymbolRegistry, Triple};
+use wasmer_compiler::{
+    CompileError, CompiledFunctionFrameInfo, Features, FunctionAddressMap, OperatingSystem, Symbol,
+    SymbolRegistry, Triple,
+};
 #[cfg(feature = "compiler")]
 use wasmer_compiler::{
     CompileModuleInfo, Compiler, FunctionBodyData, ModuleEnvironment, ModuleMiddlewareChain,
     ModuleTranslationState,
 };
-use wasmer_engine::{Artifact, DeserializeError, InstantiationError, SerializeError};
+use wasmer_engine::{
+    register_frame_info, Artifact, DeserializeError, FunctionExtent, GlobalFrameInfoRegistration,
+    InstantiationError, SerializeError,
+};
 #[cfg(feature = "compiler")]
 use wasmer_engine::{Engine, Tunables};
 #[cfg(feature = "compiler")]
@@ -49,6 +55,7 @@ pub struct NativeArtifact {
     finished_dynamic_function_trampolines: BoxedSlice<FunctionIndex, FunctionBodyPtr>,
     func_data_registry: Arc<FuncDataRegistry>,
     signatures: BoxedSlice<SignatureIndex, VMSharedSignatureIndex>,
+    frame_info_registration: Mutex<Option<GlobalFrameInfoRegistration>>,
 }
 
 fn to_compile_error(err: impl Error) -> CompileError {
@@ -371,6 +378,7 @@ impl NativeArtifact {
                 .into_boxed_slice(),
             func_data_registry: Arc::new(FuncDataRegistry::new()),
             signatures: signatures.into_boxed_slice(),
+            frame_info_registration: Mutex::new(None),
         })
     }
 
@@ -475,6 +483,7 @@ impl NativeArtifact {
                 .into_boxed_slice(),
             func_data_registry: engine_inner.func_data().clone(),
             signatures: signatures.into_boxed_slice(),
+            frame_info_registration: Mutex::new(None),
         })
     }
 
@@ -593,7 +602,129 @@ impl Artifact for NativeArtifact {
     }
 
     fn register_frame_info(&self) {
-        // Do nothing for now
+        let mut info = self.frame_info_registration.lock().unwrap();
+
+        if info.is_some() {
+            return;
+        }
+
+        // We (reverse) order all the functions by their pointer location.
+        // [f9, f6, f7, f8...] and calculate their potential function body size by
+        // getting the diff in pointers between functions (since they are all located
+        // in the same __text section).
+
+        let min_call_trampolines_pointer = self
+            .finished_function_call_trampolines
+            .values()
+            .map(|t| *t as usize)
+            .min()
+            .unwrap_or(0);
+        let min_dynamic_trampolines_pointer = self
+            .finished_dynamic_function_trampolines
+            .values()
+            .map(|t| **t as usize)
+            .min()
+            .unwrap_or(0);
+
+        let fp = self.finished_functions.clone();
+        let mut function_pointers = fp.into_iter().collect::<Vec<_>>();
+
+        // Sort the keys by the funciton pointer values in reverse order.
+        // This way we can get the maximum function lengths (since functions can't overlap in memory)
+        function_pointers.sort_by(|(_k1, v1), (_k2, v2)| v2.cmp(v1));
+        let mut iter = function_pointers.into_iter();
+        let mut function_pointers = iter
+            .next()
+            .map(|(index, function_pointer)| {
+                let fp = **function_pointer as usize;
+                // In case we are in the first function pointer (the one with the highest pointer)
+                // we try to determine it's bounds (function size) by using the other function trampoline
+                // locations.
+                let current_size_by_ptr = if fp < min_call_trampolines_pointer {
+                    if min_call_trampolines_pointer < min_dynamic_trampolines_pointer
+                        || min_dynamic_trampolines_pointer == 0
+                    {
+                        min_call_trampolines_pointer - fp
+                    } else {
+                        min_dynamic_trampolines_pointer - fp
+                    }
+                } else if fp < min_dynamic_trampolines_pointer {
+                    min_dynamic_trampolines_pointer - fp
+                } else {
+                    // The trampoline pointers are before the function.
+                    // We can safely assume the function will be at least 16 bits of length.
+                    // This is a very hacky assumption, but it makes collisions work perfectly
+                    // Since DLOpen simlinks will always be > 16 bits of difference between
+                    // two different libraries for the same symbol.
+                    // Note: the minmum Mach-O file is 0x1000 bytes and the minimum ELF is 0x0060 bytes
+                    16
+                };
+                let mut prev_pointer = fp;
+                // We choose the minimum between the function size given the pointer diff
+                // and the emitted size by the address map
+                let ptr = function_pointer;
+                let length = current_size_by_ptr;
+                let first = (
+                    index,
+                    FunctionExtent {
+                        ptr: *ptr,
+                        length: length,
+                    },
+                );
+                std::iter::once(first)
+                    .chain(iter.map(|(index, function_pointer)| {
+                        let fp = **function_pointer as usize;
+                        // This assumes we never lay any functions bodies across the usize::MAX..nullptr
+                        // wrapping point.
+                        // Which is generally true on most OSes, but certainly doesn't have to be true.
+                        //
+                        // Further reading: https://lwn.net/Articles/342330/ \
+                        // "There is one little problem with that reasoning, though: NULL (zero) can
+                        // actually be a valid pointer address."
+                        let current_size_by_ptr = prev_pointer - fp;
+
+                        prev_pointer = fp;
+                        // We choose the minimum between the function size given the pointer diff
+                        // and the emitted size by the address map
+                        let ptr = function_pointer;
+                        let length = current_size_by_ptr;
+                        (
+                            index,
+                            FunctionExtent {
+                                ptr: *ptr,
+                                length: length,
+                            },
+                        )
+                    }))
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default();
+
+        // We sort them again, by key this time
+        function_pointers.sort_by(|(k1, _v1), (k2, _v2)| k1.cmp(k2));
+
+        let frame_infos = function_pointers
+            .iter()
+            .map(|(_, extent)| CompiledFunctionFrameInfo {
+                traps: vec![],
+                address_map: FunctionAddressMap {
+                    body_len: extent.length,
+                    ..Default::default()
+                },
+            })
+            .collect::<PrimaryMap<LocalFunctionIndex, _>>();
+
+        let finished_function_extents = function_pointers
+            .into_iter()
+            .map(|(_, function_extent)| function_extent)
+            .collect::<PrimaryMap<LocalFunctionIndex, _>>()
+            .into_boxed_slice();
+
+        *info = register_frame_info(
+            self.metadata.compile_info.module.clone(),
+            &finished_function_extents,
+            frame_infos,
+        );
     }
 
     fn features(&self) -> &Features {

diff --git a/lib/engine-native/src/serialize.rs b/lib/engine-native/src/serialize.rs
@@ -63,11 +63,6 @@ impl ModuleMetadata {
         let pos = serializer.serialize_value(self).map_err(to_compile_error)? as u64;
         let mut serialized_data = serializer.into_inner().into_inner();
         serialized_data.extend_from_slice(&pos.to_le_bytes());
-        if cfg!(target_endian = "big") {
-            serialized_data.extend_from_slice(&[b'b']);
-        } else if cfg!(target_endian = "little") {
-            serialized_data.extend_from_slice(&[b'l']);
-        }
         Ok(serialized_data)
     }
 
@@ -80,16 +75,10 @@ impl ModuleMetadata {
         metadata_slice: &'a [u8],
     ) -> Result<&'a ArchivedModuleMetadata, DeserializeError> {
         let mut pos: [u8; 8] = Default::default();
-        let endian = metadata_slice[metadata_slice.len() - 1];
-        if (cfg!(target_endian = "big") && endian == b'l')
-            || (cfg!(target_endian = "little") && endian == b'b')
-        {
-            return Err(DeserializeError::Incompatible("incompatible endian".into()));
-        }
-        pos.copy_from_slice(&metadata_slice[metadata_slice.len() - 9..metadata_slice.len() - 1]);
+        pos.copy_from_slice(&metadata_slice[metadata_slice.len() - 8..metadata_slice.len()]);
         let pos: u64 = u64::from_le_bytes(pos);
         Ok(archived_value::<ModuleMetadata>(
-            &metadata_slice[..metadata_slice.len() - 9],
+            &metadata_slice[..metadata_slice.len() - 8],
             pos as usize,
         ))
     }

diff --git a/lib/engine-object-file/Cargo.toml b/lib/engine-object-file/Cargo.toml
@@ -17,7 +17,7 @@ wasmer-vm = { path = "../vm", version = "1.0.2" }
 wasmer-engine = { path = "../engine", version = "1.0.2" }
 wasmer-object = { path = "../object", version = "1.0.2" }
 serde = { version = "1.0", features = ["derive", "rc"] }
-cfg-if = "0.1"
+cfg-if = "1.0"
 tracing = "0.1"
 bincode = "1.3"
 leb128 = "0.2"