Merge pull request #425 from qwe661234/Add_LLVM_T2C

Implement tier-2 JIT compiler to accelerate ISS

Author: jserv

.github/workflows/main.yml

@@ -44,6 +44,9 @@ jobs:
             sudo apt-get update -q -y
             sudo apt-get install -q -y libsdl2-dev libsdl2-mixer-dev
             .ci/riscv-toolchain-install.sh
+            wget https://apt.llvm.org/llvm.sh
+            sudo chmod +x ./llvm.sh
+            sudo ./llvm.sh 17
       shell: bash
     - name: default build
       run: make
@@ -65,14 +68,14 @@ jobs:
             make distclean ENABLE_GDBSTUB=1 gdbstub-test
     - name: JIT test
       run: |
-            make clean && make ENABLE_JIT=1 check -j$(nproc)
-            make clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc)
-            make clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc)
-            make clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc)
+            make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check -j$(nproc)
+            make ENABLE_JIT=1 clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc)
+            make ENABLE_JIT=1 clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc)
+            make ENABLE_JIT=1 clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc)
     - name: undefined behavior test
       run: |
             make clean && make ENABLE_UBSAN=1 check -j$(nproc)
-            make clean && make ENABLE_JIT=1 ENABLE_UBSAN=1 check -j$(nproc)
+            make ENABLE_JIT=1 clean clean && make ENABLE_JIT=1 ENABLE_UBSAN=1 check -j$(nproc)
 
   host-arm64:
     needs: [detect-code-related-file-changes]
@@ -91,18 +94,21 @@ jobs:
         # No 'sudo' is available
         install: |
           apt-get update -q -y
-          apt-get install -q -y git build-essential libsdl2-dev libsdl2-mixer-dev
+          apt-get install -q -y git build-essential libsdl2-dev libsdl2-mixer-dev lsb-release wget software-properties-common gnupg
           git config --global --add safe.directory ${{ github.workspace }}
           git config --global --add safe.directory ${{ github.workspace }}/src/softfloat
           git config --global --add safe.directory ${{ github.workspace }}/src/mini-gdbstub
+          wget https://apt.llvm.org/llvm.sh
+          chmod +x ./llvm.sh
+          ./llvm.sh 17
         # Append custom commands here
         run: |
           make -j$(nproc)
           make check -j$(nproc)
-          make clean && make ENABLE_JIT=1 check -j$(nproc)
-          make clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc)
-          make clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc)
-          make clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc)
+          make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check -j$(nproc)
+          make ENABLE_JIT=1 clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc)
+          make ENABLE_JIT=1 clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc)
+          make ENABLE_JIT=1 clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc)
 
   coding-style:
     needs: [detect-code-related-file-changes]
@@ -132,7 +138,8 @@ jobs:
     - name: run scan-build without JIT
       run: make distclean && scan-build -v -o ~/scan-build --status-bugs --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability make ENABLE_EXT_F=0 ENABLE_SDL=0 ENABLE_JIT=0
     - name: run scan-build with JIT
-      run: make distclean && scan-build -v -o ~/scan-build --status-bugs --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability make ENABLE_EXT_F=0 ENABLE_SDL=0 ENABLE_JIT=1
+      run: |
+          make ENABLE_JIT=1 distclean && scan-build -v -o ~/scan-build --status-bugs --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability make ENABLE_EXT_F=0 ENABLE_SDL=0 ENABLE_JIT=1
 
   compliance-test:
     needs: [detect-code-related-file-changes]

Makefile

@@ -125,7 +125,32 @@ endif
 ENABLE_JIT ?= 0
 $(call set-feature, JIT)
 ifeq ($(call has, JIT), 1)
-OBJS_EXT += jit.o
+OBJS_EXT += jit.o 
+# tier-2 JIT compiler powered LLVM
+LLVM_CONFIG = llvm-config-17
+LLVM_CONFIG := $(shell which $(LLVM_CONFIG))
+ifndef LLVM_CONFIG
+# Try Homebrew on macOS
+LLVM_CONFIG = /opt/homebrew/opt/llvm@17/bin/llvm-config
+LLVM_CONFIG := $(shell which $(LLVM_CONFIG))
+ifdef LLVM_CONFIG
+LDFLAGS += -L/opt/homebrew/opt/llvm@17/lib
+endif
+endif
+ifneq ("$(LLVM_CONFIG)", "")
+ifneq ("$(findstring -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS, "$(shell $(LLVM_CONFIG) --cflags)")", "")
+ENABLE_T2C := 1
+$(call set-feature, T2C)
+OBJS_EXT += t2c.o
+CFLAGS += -g $(shell $(LLVM_CONFIG) --cflags)
+LDFLAGS += $(shell $(LLVM_CONFIG) --libs)
+else
+ENABLE_T2C := 0
+$(call set-feature, T2C)
+$(warning No llvm-config-17 installed. Check llvm-config-17 installation in advance)
+endif
+endif
+
 ifneq ($(processor),$(filter $(processor),x86_64 aarch64 arm64))
 $(error JIT mode only supports for x64 and arm64 target currently.)
 endif
@@ -136,6 +161,10 @@ src/rv32_jit.c:
 $(OUT)/jit.o: src/jit.c src/rv32_jit.c
 	$(VECHO) "  CC\t$@\n"
 	$(Q)$(CC) -o $@ $(CFLAGS) -c -MMD -MF $@.d $<
+
+$(OUT)/t2c.o: src/t2c.c src/t2c_template.c
+	$(VECHO) "  CC\t$@\n"
+	$(Q)$(CC) -o $@ $(CFLAGS) -c -MMD -MF $@.d $<
 endif
 # For tail-call elimination, we need a specific set of build flags applied.
 # FIXME: On macOS + Apple Silicon, -fno-stack-protector might have a negative impact.

README.md

@@ -30,7 +30,7 @@ Features:
 * Implementation of commonly used newlib system calls
 * Experimental SDL-based display/event/audio system calls for running video games
 * Support for remote GDB debugging
-* Experimental JIT compiler for performance boost while maintaining a small footprint
+* Tiered JIT compilation for performance boost while maintaining a small footprint
 
 ## Build and Verify
 
@@ -40,6 +40,12 @@ and [SDL2_Mixer library](https://wiki.libsdl.org/SDL2_mixer) installed.
 * macOS: `brew install sdl2 sdl2_mixer`
 * Ubuntu Linux / Debian: `sudo apt install libsdl2-dev libsdl2-mixer-dev`
 
+### JIT compiler
+The tier-2 JIT compiler in `rv32emu` leverages LLVM for powerful optimization. Therefore, the target system must have [`LLVM`](https://llvm.org/) installed, with version 17 recommended. If `LLVM` is not installed, only the tier-1 JIT compiler will be used for performance enhancement.
+
+* macOS: `brew install llvm@17`
+* Ubuntu Linux / Debian: `sudo apt-get install llvm-17`
+
 Build the emulator.
 ```shell
 $ make

src/emulate.c

@@ -304,7 +304,9 @@ static block_t *block_alloc(riscv_t *rv)
 #if RV32_HAS(JIT)
     block->translatable = true;
     block->hot = false;
+    block->hot2 = false;
     block->has_loops = false;
+    block->n_invoke = 0;
     INIT_LIST_HEAD(&block->list);
 #endif
     return block;
@@ -911,8 +913,6 @@ static bool runtime_profiler(riscv_t *rv, block_t *block)
         return true;
     return false;
 }
-
-typedef void (*exec_block_func_t)(riscv_t *rv, uintptr_t);
 #endif
 
 void rv_step(void *arg)
@@ -985,15 +985,31 @@ void rv_step(void *arg)
         }
         last_pc = rv->PC;
 #if RV32_HAS(JIT)
-        /* execute by tier-1 JIT compiler */
+#if RV32_HAS(T2C)
+        /* executed through the tier-2 JIT compiler */
+        if (block->hot2) {
+            ((exec_t2c_func_t) block->func)(rv);
+            prev = NULL;
+            continue;
+        } /* check if the execution path is strong hotspot */
+        if (block->n_invoke >= THRESHOLD) {
+            t2c_compile(block,
+                        (uint64_t) ((memory_t *) PRIV(rv)->mem)->mem_base);
+            ((exec_t2c_func_t) block->func)(rv);
+            prev = NULL;
+            continue;
+        }
+#endif
+        /* executed through the tier-1 JIT compiler */
         struct jit_state *state = rv->jit_state;
         if (block->hot) {
+            block->n_invoke++;
             ((exec_block_func_t) state->buf)(
                 rv, (uintptr_t) (state->buf + block->offset));
             prev = NULL;
             continue;
-        } /* check if using frequency of block exceed threshold */
-        else if (block->translatable && runtime_profiler(rv, block)) {
+        } /* check if the execution path is potential hotspot */
+        if (block->translatable && runtime_profiler(rv, block)) {
             jit_translate(rv, block);
             ((exec_block_func_t) state->buf)(
                 rv, (uintptr_t) (state->buf + block->offset));

src/feature.h

@@ -52,5 +52,10 @@
 #define RV32_FEATURE_JIT 0
 #endif
 
+/* Experimental tier-2 just-in-time compiler */
+#ifndef RV32_FEATURE_T2C
+#define RV32_FEATURE_T2C 0
+#endif
+
 /* Feature test macro */
 #define RV32_HAS(x) RV32_FEATURE_##x

src/jit.h

@@ -48,3 +48,9 @@ struct host_reg {
 struct jit_state *jit_state_init(size_t size);
 void jit_state_exit(struct jit_state *state);
 void jit_translate(riscv_t *rv, block_t *block);
+typedef void (*exec_block_func_t)(riscv_t *rv, uintptr_t);
+
+#if RV32_HAS(T2C)
+void t2c_compile(block_t *block, uint64_t mem_base);
+typedef void (*exec_t2c_func_t)(riscv_t *);
+#endif

src/riscv_private.h

@@ -65,11 +65,14 @@ typedef struct block {
 
     rv_insn_t *ir_head, *ir_tail; /**< the first and last ir for this block */
 #if RV32_HAS(JIT)
-    bool hot; /**< Determine the block is hotspot or not */
-    uint32_t offset;
+    bool hot;  /**< Determine the block is potential hotspot or not */
+    bool hot2; /**< Determine the block is strong hotspot or not */
     bool
         translatable; /**< Determine the block has RV32AF insturctions or not */
     bool has_loops;   /**< Determine the block has loop or not */
+    uint32_t offset;  /**< The machine code offset in T1 code cache */
+    uint32_t n_invoke; /**< The invoking times of T1 machine code */
+    void *func;        /**< The function pointer of T2 machine code */
     struct list_head list;
 #endif
 } block_t;