feat: finish up codegen from ssa ir

- Lowering from kernel to ssa is finished
- Clean up and rework of lowering from ssa to mlir is finished
- Fix up backtrace implementation in rt crate
- Implement a variety of supporting intrinsics
- Provide a few hacked together runtime functions for testing a stripped
  down init
- Implement atom table generation for runtime similar to the intern crate
  used by the compiler

NOTE: No real testing of the generated executables has been done, but
quite complex Erlang source code can now be compiled and linked as
desired, but there are certainly bugs present

Author: bitwalker

Cargo.lock

@@ -13,6 +13,7 @@ default-members = [
 ]
 
 exclude = [
+  "library/macros",
   "liblumen_alloc",
   "liblumen_alloc_macros",
   "native_implemented/*",

Cargo.toml

@@ -12,6 +12,7 @@ default = []
 std = ["anyhow/std"]
 
 [dependencies]
+static_assertions = "1.1"
 paste = "1.0"
 
 [dependencies.anyhow]

compiler/binary/Cargo.toml

@@ -7,6 +7,7 @@
 #![feature(str_internals)]
 #![feature(const_option_ext)]
 #![feature(slice_take)]
+#![feature(arbitrary_enum_discriminant)]
 
 extern crate alloc;
 #[cfg(any(test, feature = "std"))]
@@ -78,7 +79,7 @@ pub use self::traits::{Aligned, Binary, Bitstring, FromEndianBytes, ToEndianByte
 /// machine, or big-endian value on big-endian machine), this is already the case. When reading a value with non-native endianness
 /// though, we need to swap the order of the bytes first.
 ///
-#[repr(C)]
+#[repr(u8)]
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Endianness {
     /// Most-significant bits "first"

compiler/binary/src/lib.rs

@@ -22,6 +22,11 @@ impl<'a> Matcher<'a> {
         }
     }
 
+    #[inline]
+    pub fn bit_size(&self) -> usize {
+        self.selection.bit_size()
+    }
+
     /// Reads a single byte from the current position in the input without
     /// advancing the input.
     pub fn read_byte(&mut self) -> Option<u8> {
@@ -55,6 +60,37 @@ impl<'a> Matcher<'a> {
         true
     }
 
+    /// Reads up to the specified number of bits into the provided buffer.
+    ///
+    /// If the bitsize given is larger than the provided buffer, this function will panic.
+    ///
+    /// If the number of bits requested was read, returns true, otherwise false.
+    pub fn read_bits<const N: usize>(&mut self, buf: &mut [u8; N], bitsize: usize) -> bool {
+        assert!(bitsize <= N * 8);
+
+        match self.selection.take(bitsize) {
+            Ok(s) => {
+                let trailing_bits = bitsize % 8;
+                let needed = (bitsize / 8) + ((trailing_bits > 0) as usize);
+                if let Some(slice) = s.as_bytes() {
+                    let buf2 = &mut buf[..needed];
+                    buf2.copy_from_slice(&slice[..needed]);
+                } else {
+                    let ptr = buf.as_mut_ptr();
+                    let mut idx = 0;
+                    for byte in s.iter().take(needed) {
+                        unsafe {
+                            *ptr.add(idx) = byte;
+                        }
+                        idx += 1;
+                    }
+                }
+                true
+            }
+            Err(_) => false,
+        }
+    }
+
     /// Reads a single number from the current position in the input without
     /// advancing the input. This should be used to build matchers which are
     /// fallible and operate on numeric values.
@@ -80,7 +116,7 @@ impl<'a> Matcher<'a> {
         endianness: Endianness,
     ) -> Option<N>
     where
-        N: FromEndianBytes<S> + core::ops::Shr<usize, Output = N>,
+        N: FromEndianBytes<S> + core::ops::Shr<u32, Output = N>,
     {
         assert!(
             bitsize <= (S * 8),
@@ -91,7 +127,7 @@ impl<'a> Matcher<'a> {
             let mut bytes = [0u8; S];
 
             let mut matcher = Matcher::new(selection);
-            if matcher.read_bytes(&mut bytes) {
+            if matcher.read_bits(&mut bytes, bitsize) {
                 match endianness {
                     Endianness::Native => {
                         let n = N::from_ne_bytes(bytes);
@@ -101,15 +137,15 @@ impl<'a> Matcher<'a> {
                         } else {
                             // We need to shift the bytes right so that the least-significant
                             // bits begin at the correct byte boundary
-                            let shift = 8 - (bitsize % 8);
+                            let shift: u32 = (8 - (bitsize % 8)).try_into().unwrap();
                             Some(n >> shift)
                         }
                     }
                     // Big-endian bytes never require a shift
                     Endianness::Big => Some(N::from_be_bytes(bytes)),
                     // Little-endian bytes always require a shift
                     Endianness::Little => {
-                        let shift = 8 - (bitsize % 8);
+                        let shift: u32 = (8 - (bitsize % 8)).try_into().unwrap();
                         let n = N::from_le_bytes(bytes);
                         Some(n >> shift)
                     }
@@ -165,6 +201,23 @@ impl<'a> Matcher<'a> {
         Some(matched)
     }
 
+    /// Matches any primitive numeric type from the input, of arbitrary bit size up to the
+    /// containing type size, advancing the input if successful
+    pub fn match_ap_number<N, const S: usize>(
+        &mut self,
+        bitsize: usize,
+        endianness: Endianness,
+    ) -> Option<N>
+    where
+        N: FromEndianBytes<S> + core::ops::Shr<u32, Output = N>,
+    {
+        let matched = self.read_ap_number(bitsize, endianness)?;
+
+        self.selection = self.selection.shrink_front(S * 8);
+
+        Some(matched)
+    }
+
     /// Matches a big integer from the input, advancing the input if successful
     pub fn match_bigint(
         &mut self,

compiler/binary/src/matcher.rs

@@ -171,6 +171,16 @@ impl<'a> Selection<'a> {
         }
     }
 
+    /// Creates a new selection covering all of the bits in the provided bitstring
+    pub fn from_bitstring(data: &'a dyn Bitstring) -> Self {
+        let bit_size = data.bit_size();
+        if bit_size == 0 {
+            return Self::Empty;
+        }
+        let bytes = unsafe { data.as_bytes_unchecked() };
+        Self::new(bytes, 0, data.bit_offset(), None, bit_size).unwrap()
+    }
+
     /// This function can be used to select `n` bits from `data`, taking into account both byte
     /// and bit offsets, as well as an optional bit length constraint on `data` (i.e. the buffer
     /// is 8 bytes, but we only want to treat it as if it was 7 1/2 bytes).

compiler/binary/src/select.rs

@@ -1,28 +1,60 @@
+use static_assertions::assert_eq_size;
+
 use crate::Endianness;
 
 /// Represents a binary segment constructor/match specification, e.g. `<<42:8/signed-little-integer>>`
+///
+/// The size and layout of this type is relied upon by our compiler. It can be represented as a single i64
+/// value, and has a layout equivalent to the following struct:
+///
+/// ```rust,ignore
+/// #[repr(C)]
+/// pub struct BinaryEntrySpecifier {
+///   tag: u32,
+///   data: [u32; 1]
+/// }
+/// ```
+///
+/// Where individual variants of the enum are discriminated by the tag and are encoded like:
+///
+/// ```rust,ignore
+/// #[repr(C)]
+/// pub struct BinaryEntrySpecifierInteger {
+///   _tag: [i8; 4],
+///   signed: u8,
+///   endianness: u8,
+///   unit: u8,
+///   _padding: [i8; 1]
+/// }
+/// ```
+///
+/// NOTE: This compact encoding is possible because `Endianness` is `#[repr(u8)]`
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[repr(u32)]
 pub enum BinaryEntrySpecifier {
     Integer {
         signed: bool,
         endianness: Endianness,
-        unit: usize,
-    },
+        unit: u8,
+    } = 0,
     Float {
         endianness: Endianness,
-        unit: usize,
-    },
+        unit: u8,
+    } = 1,
     Binary {
-        unit: usize,
-    },
-    Utf8,
+        unit: u8,
+    } = 2,
+    Utf8 = 3,
     Utf16 {
         endianness: Endianness,
-    },
+    } = 4,
     Utf32 {
         endianness: Endianness,
-    },
+    } = 5,
 }
+
+assert_eq_size!(BinaryEntrySpecifier, u64);
+
 impl BinaryEntrySpecifier {
     pub const DEFAULT: Self = Self::Integer {
         signed: false,
@@ -40,7 +72,7 @@ impl BinaryEntrySpecifier {
     pub fn unit(&self) -> usize {
         match self {
             Self::Integer { unit, .. } | Self::Float { unit, .. } | Self::Binary { unit, .. } => {
-                *unit
+                *unit as usize
             }
             _ => 1,
         }