[GR-48942] Run C extensions natively

PullRequest: truffleruby/4018

Author: eregon

.gitignore

@@ -48,6 +48,8 @@ lib/gems/gems/debug-*/ext/debug/debug_version.h
 src/main/c/*/Makefile
 !src/main/c/spawn-helper/Makefile
 !src/main/c/truffleposix/Makefile
+src/main/c/cext-trampoline/trampoline.c
+src/main/c/cext/wrappers.c
 src/main/c/etc/constdefs.h
 src/main/c/spawn-helper/spawn-helper
 

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 New features:
 
+* C/C++ extensions are now compiled using the system toolchain and executed natively instead of using GraalVM LLVM (Sulong). This leads to faster startup, no warmup, better compatibility and faster installation for C/C++ extensions (#3118, @eregon).
 
 Bug fixes:
 
@@ -11,7 +12,6 @@ Bug fixes:
 
 Compatibility:
 
-* Fix problems with the LLVM toolchain wrappers with Xcode 15 on macOS.
 * Add `Exception#detailed_message` method (#3257, @andrykonchin).
 * Fix `rb_enc_vsprintf` and force String encoding instead of converting it (@andrykonchin).
 * Add `rb_gc_mark_movable` function (@andrykonchin).

ci.jsonnet

@@ -232,14 +232,18 @@ local part_definitions = {
   },
 
   platform: {
-    local common_deps = common.deps.truffleruby + common.deps.sulong,
-    local linux_amd64_extra_deps = {
-      packages+: {
-        binutils: ">=2.30",
-      },
-    },
-
-    linux: common.linux_amd64 + common_deps + linux_amd64_extra_deps + {
+    local devtoolset = { # Until there is a proper object in common.jsonnet for it
+      packages+: if self.os == "linux" then
+        (if self.arch == "aarch64" then {
+          "00:devtoolset": "==10",
+        } else {
+          "00:devtoolset": "==11",
+        })
+      else {},
+    },
+    local common_deps = common.deps.truffleruby + common.deps.sulong + devtoolset,
+
+    linux: common.linux_amd64 + common_deps + {
       platform_name:: "LinuxAMD64",
       "$.cap":: {
         normal_machine: [],
@@ -308,6 +312,7 @@ local part_definitions = {
       run+: jt(["test", "specs", ":all"]) +
             jt(["test", "specs", ":tracepoint"]) +
             jt(["test", "specs", ":next"]) +
+            [["env", "SPEC_CAPI_CXX=true"] + jt(["test", "specs", "--timeout", "180", ":capi"])[0]] +
             jt(["test", "basictest"]),
     },
 
@@ -371,6 +376,13 @@ local part_definitions = {
             jt(["test", "bundle"]),
     },
 
+    test_cexts_sulong: {
+      environment+: {
+        TRUFFLERUBYOPT: "--experimental-options --cexts-sulong",
+      },
+      run+: jt(["test", "specs", ":cext"]),
+    },
+
     testdownstream_aot: { run+: [["mx", "ruby_testdownstream_aot", "$RUBY_BIN"]] },
 
     test_make_standalone_distribution: {
@@ -381,18 +393,12 @@ local part_definitions = {
 
     generate_native_config: {
       setup+: [
-        ["mx", "sforceimports"], # clone the graal repo
-        ["mx", "-p", "../graal/sulong", "build"],
-        ["set-export", "TOOLCHAIN_PATH", ["mx", "-p", "../graal/sulong", "lli", "--print-toolchain-path"]],
+        ["ruby", "-e", "pp RbConfig::MAKEFILE_CONFIG"], # For convenience
       ],
       run+: [
-        ["env",
-          "LD_LIBRARY_PATH=$BUILD_DIR/graal/sulong/mxbuild/" + self.os + "-" + self.arch + "/SULONG_HOME/native/lib:$LD_LIBRARY_PATH", # for finding libc++
-          "PATH=$TOOLCHAIN_PATH:$PATH",
-          "ruby", "tool/generate-native-config.rb"],
+        ["ruby", "tool/generate-native-config.rb"],
         ["cat", "src/main/java/org/truffleruby/platform/" + self.platform_name + "NativeConfiguration.java"],
 
-        # Uses the system compiler as using the toolchain for this does not work on macOS
         ["tool/generate-config-header.sh"],
         ["cat", "lib/cext/include/truffleruby/config_" + self.os + "_" + self.arch + ".h"],
       ],
@@ -537,6 +543,7 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
       "ruby-test-cexts-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.use.sqlite331 + $.run.test_cexts,
       "ruby-test-cexts-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_cexts + { timelimit: "01:30:00" },
       "ruby-test-cexts-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_cexts + { timelimit: "00:40:00" },
+      "ruby-test-cexts-sulong":         $.platform.linux  + $.jdk.stable + $.env.jvm + gate + $.run.test_cexts_sulong + { timelimit: "20:00" },
       "ruby-test-gems-linux-amd64":     $.platform.linux  + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
       "ruby-test-gems-darwin-amd64":    $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
       "ruby-test-gems-darwin-aarch64":  $.platform.darwin_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
@@ -701,11 +708,12 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
 
   manual_builds: {
     local shared = $.use.common + $.cap.manual + { timelimit: "15:00" },
+    local native_config = $.run.generate_native_config + $.run.check_native_config,
 
-    "ruby-generate-native-config-linux-amd64":    $.platform.linux + $.jdk.stable + shared + $.run.generate_native_config,
-    "ruby-generate-native-config-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.stable + shared + $.run.generate_native_config,
-    "ruby-generate-native-config-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.stable + shared + $.run.generate_native_config,
-    "ruby-generate-native-config-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.stable + shared + $.run.generate_native_config,
+    "ruby-generate-native-config-linux-amd64":    $.platform.linux + $.jdk.stable + shared + native_config,
+    "ruby-generate-native-config-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.stable + shared + native_config,
+    "ruby-generate-native-config-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.stable + shared + native_config,
+    "ruby-generate-native-config-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.stable + shared + native_config,
   },
 
   builds:

doc/contributor/how-to-guide.md

@@ -1292,15 +1292,15 @@ When you modify C-files (*.c, *.h) or compilation-related Ruby files
 affect the ABI or increase the ABI *check* version explicitly by modifying one of
 the files:
 
-- lib/cext/ABI_version.txt
+- lib/cext/include/truffleruby/truffleruby-abi-version.h
 - lib/cext/ABI_check.txt
 
 ABI change is:
 - changing headers or compilation flags
 - removing/adding a non-static function
 - implementing already declared non-static functions
 
-In case of doubt, bump `ABI_version.txt`.
+In case of doubt, bump `truffleruby-abi-version.h`.
 
 ## How to choose where to add new specs - in TruffleRuby or in ruby/spec repository
 

doc/contributor/updating-ruby.md

@@ -142,7 +142,7 @@ Also update the list of `provided_executables` in `mx_truffleruby.py` if some la
 Update all of these:
 
 * Update `.ruby-version`, `TruffleRuby.LANGUAGE_VERSION`
-* Reset `lib/cext/ABI_version.txt` and `lib/cext/ABI_check.txt` to `1` if `RUBY_VERSION` was updated.
+* Reset `truffleruby-abi-version.h` to `$RUBY_VERSION.1` and `lib/cext/ABI_check.txt` to `1` if `RUBY_VERSION` was updated.
 * Update `versions.json` (with gem versions provided by `cat ../ruby/gems/bundled_gems | sort`, `ls -l lib/gems/specifications/default` and `grep 'VERSION =' lib/mri/rubygems.rb`)
 * Also update version numbers for `debug` and `rbs` in `src/main/c/Makefile` and in `mx.truffleruby/suite.py`.
 * Copy and paste `-h` and `--help` output to `RubyLauncher` (instructions are in the end of the file `src/launcher/java/org/truffleruby/launcher/RubyLauncher.java`)

lib/cext/ABI_version.txt

@@ -136,9 +136,13 @@ static inline bool BIGNUM_POSITIVE_P(VALUE b);
 static inline bool BIGNUM_NEGATIVE_P(VALUE b);
 static inline void BIGNUM_SET_SIGN(VALUE b, bool sign);
 static inline void BIGNUM_NEGATE(VALUE b);
+#ifndef TRUFFLERUBY
 static inline size_t BIGNUM_LEN(VALUE b);
+#endif
 static inline BDIGIT *BIGNUM_DIGITS(VALUE b);
+#ifndef TRUFFLERUBY
 static inline int BIGNUM_LENINT(VALUE b);
+#endif
 static inline bool BIGNUM_EMBED_P(VALUE b);
 
 RUBY_SYMBOL_EXPORT_BEGIN
@@ -205,6 +209,7 @@ BIGNUM_NEGATE(VALUE b)
     FL_REVERSE_RAW(b, BIGNUM_SIGN_BIT);
 }
 
+#ifndef TRUFFLERUBY
 static inline size_t
 BIGNUM_LEN(VALUE b)
 {
@@ -224,6 +229,7 @@ BIGNUM_LENINT(VALUE b)
 {
     return rb_long2int(BIGNUM_LEN(b));
 }
+#endif
 
 /* LSB:BIGNUM_DIGITS(b)[0], MSB:BIGNUM_DIGITS(b)[BIGNUM_LEN(b)-1] */
 static inline BDIGIT *

lib/cext/include/internal/bignum.h

@@ -25,7 +25,12 @@ st_index_t rb_iseq_cdhash_hash(VALUE a);
 /* iseq.c */
 int rb_vm_insn_addr2insn(const void *);
 int rb_vm_insn_decode(const VALUE encoded);
+
+#ifdef TRUFFLERUBY
+#define ruby_vm_keep_script_lines false
+#else
 extern bool ruby_vm_keep_script_lines;
+#endif
 
 MJIT_SYMBOL_EXPORT_BEGIN
 /* iseq.c (export) */

lib/cext/include/internal/compile.h

@@ -131,8 +131,10 @@ typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t;
 rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt);
 struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc);
 void rb_strterm_mark(VALUE obj);
+#ifndef TRUFFLERUBY
 static inline enum imemo_type imemo_type(VALUE imemo);
 static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type);
+#endif
 static inline bool imemo_throw_data_p(VALUE imemo);
 static inline struct vm_ifunc *rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data);
 #ifdef TRUFFLERUBY
@@ -160,6 +162,7 @@ VALUE rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
 const char *rb_imemo_name(enum imemo_type type);
 RUBY_SYMBOL_EXPORT_END
 
+#ifndef TRUFFLERUBY
 static inline enum imemo_type
 imemo_type(VALUE imemo)
 {
@@ -182,6 +185,7 @@ imemo_type_p(VALUE imemo, enum imemo_type imemo_type)
 }
 
 #define IMEMO_TYPE_P(v, t) imemo_type_p((VALUE)v, t)
+#endif
 
 static inline bool
 imemo_throw_data_p(VALUE imemo)

lib/cext/include/internal/imemo.h

@@ -106,10 +106,8 @@ int rb_long2int(long value);
 #endif
 RBIMPL_SYMBOL_EXPORT_END()
 
-#ifndef TRUFFLERUBY
 RBIMPL_ATTR_CONST_UNLESS_DEBUG()
 RBIMPL_ATTR_CONSTEXPR_UNLESS_DEBUG(CXX14)
-#endif
 RBIMPL_ATTR_ARTIFICIAL()
 /**
  * Converts a C's `long` into an instance of ::rb_cInteger.
@@ -231,10 +229,7 @@ rbimpl_right_shift_is_arithmetic_p(void)
 }
 
 RBIMPL_ATTR_CONST_UNLESS_DEBUG()
-#ifndef TRUFFLERUBY
 RBIMPL_ATTR_CONSTEXPR_UNLESS_DEBUG(CXX14)
-#endif
-
 /**
  * Converts a Fixnum into C's `long`.
  *
@@ -245,16 +240,12 @@ RBIMPL_ATTR_CONSTEXPR_UNLESS_DEBUG(CXX14)
 static inline long
 rb_fix2long(VALUE x)
 {
-#ifdef TRUFFLERUBY
-    return ((long)polyglot_as_i64(rb_tr_unwrap(x)));
-#else
     if /* constexpr */ (rbimpl_right_shift_is_arithmetic_p()) {
         return rbimpl_fix2long_by_shift(x);
     }
     else {
         return rbimpl_fix2long_by_idiv(x);
     }
-#endif
 }
 
 RBIMPL_ATTR_CONST_UNLESS_DEBUG()
@@ -344,11 +335,7 @@ rb_ulong2num_inline(unsigned long v)
     if (RB_POSFIXABLE(v))
         return RB_LONG2FIX(v);
     else
-#ifdef TRUFFLERUBY
-        return rb_tr_wrap(polyglot_invoke(RUBY_CEXT, "rb_ulong2num", (long) v));
-#else
         return rb_uint2big(v);
-#endif
 }
 
 /**