[GR-18163] Fix parsing literal floats when the locale does not use . for the decimal separator

PullRequest: truffleruby/4224

Author: eregon

CHANGELOG.md

@@ -8,6 +8,7 @@ Bug fixes:
 * Fix repeated calling of methods `Dir#{each,each_child,children}` (#3464, @andrykonchin).
 * Fix `IO#{wait,wait_readable,wait_writable}` methods and switch the current thread into a sleep state (@andrykonchin).
 * Fix `rb_global_variable()` for `Float` and bignum values during the `Init_` function (#3478, @eregon).
+* Fix parsing literal floats when the locale does not use `.` for the decimal separator (e.g. `LANG=fr_FR.UTF-8`) (#3512, @eregon).
 
 Compatibility:
 * Move `IO#wait_readable`, `IO#wait_writable`, `IO#wait_priority` and `IO#wait` into core library (@andrykonchin).

doc/contributor/updating-ruby.md

@@ -144,7 +144,7 @@ Update all of these:
 * Update `.ruby-version`, `TruffleRuby.LANGUAGE_VERSION`
 * Reset `truffleruby-abi-version.h` to `$RUBY_VERSION.1` and `lib/cext/ABI_check.txt` to `1` if `RUBY_VERSION` was updated.
 * Update `versions.json` (with gem versions provided by `cat ../ruby/gems/bundled_gems | sort`, `ls -l lib/gems/specifications/default` and `grep 'VERSION =' lib/mri/rubygems.rb`)
-* Also update version numbers for `debug` and `rbs` in `src/main/c/Makefile` and in `mx.truffleruby/suite.py`.
+* Also update version numbers for `debug` and `rbs` in `src/main/c/Makefile`, `mx.truffleruby/suite.py` and `lib/gems/gems/debug-*/ext/debug/extconf.rb`.
 * Copy and paste `-h` and `--help` output to `RubyLauncher` (instructions are in the end of the file `src/launcher/java/org/truffleruby/launcher/RubyLauncher.java`)
 * This is a good time to get `jt build` working.
 * Copy and paste the TruffleRuby `--help` output to `doc/user/options.md` (e.g., with `jt ruby --help | xsel -b`)

lib/cext/ABI_check.txt

@@ -1 +1 @@
-15
+16

lib/gems/gems/debug-1.7.1/ext/debug/extconf.rb

@@ -1,6 +1,6 @@
 require 'mkmf'
-require_relative '../../lib/debug/version'
-File.write("debug_version.h", "#define RUBY_DEBUG_VERSION \"#{DEBUGGER__::VERSION}\"\n")
+# TruffleRuby: hardcode the version here to avoid depending on version.rb which is not copied to src/main/c/debug
+File.write("debug_version.h", "#define RUBY_DEBUG_VERSION \"#{"1.7.1"}\"\n")
 $distcleanfiles << "debug_version.h"
 
 if defined? RubyVM

lib/truffle/rbconfig.rb

@@ -135,10 +135,11 @@ module RbConfig
 
   # Set extra flags needed for --building-core-cexts
   if Truffle::Boot.get_option 'building-core-cexts'
-    libtruffleruby = "#{ruby_home}/src/main/c/cext/libtruffleruby.#{soext}"
-    libtrufflerubytrampoline = "#{ruby_home}/src/main/c/cext-trampoline/libtrufflerubytrampoline.#{soext}"
+    repo = Truffle::System.get_java_property 'truffleruby.repository'
+    libtruffleruby = "#{repo}/src/main/c/cext/libtruffleruby.#{soext}"
+    libtrufflerubytrampoline = "#{repo}/src/main/c/cext-trampoline/libtrufflerubytrampoline.#{soext}"
 
-    relative_debug_paths = " -fdebug-prefix-map=#{ruby_home}=."
+    relative_debug_paths = " -fdebug-prefix-map=#{repo}=."
     cppflags << relative_debug_paths
 
     warnflags << '-Wundef' # Warn for undefined preprocessor macros for core C extensions

mx.truffleruby/mx_truffleruby.py

@@ -11,6 +11,7 @@
 import os
 import pipes
 from os.path import join, exists, basename
+import re
 import shutil
 import sys
 
@@ -48,6 +49,27 @@ def add_ext_suffix(name):
 
 mx_subst.results_substitutions.register_with_arg('extsuffix', add_ext_suffix)
 
+# From org.truffleruby.shared.Platform
+def get_cruby_arch():
+    arch = mx.get_arch()
+    if arch == 'amd64':
+        return 'x86_64'
+    elif arch == 'aarch64':
+        return 'aarch64'
+    else:
+        raise Exception("Unknown platform " + arch)
+
+mx_subst.results_substitutions.register_no_arg('cruby_arch', get_cruby_arch)
+
+def get_truffleruby_abi_version():
+    path = join(root, 'lib/cext/include/truffleruby/truffleruby-abi-version.h')
+    with open(path, "r") as f:
+        contents = f.read()
+    m = re.search('"(.+)"', contents)
+    return m.group(1)
+
+mx_subst.results_substitutions.register_no_arg('truffleruby_abi_version', get_truffleruby_abi_version)
+
 # Utilities
 
 class VerboseMx:
@@ -118,10 +140,10 @@ def contents(self, result):
         if debug_args:
             jvm_args.extend(['-ea', '-esa'])
 
-        jvm_args.append('-Dorg.graalvm.language.ruby.home=' + root)
+        bootstrap_home = mx.distribution('TRUFFLERUBY_BOOTSTRAP_HOME').get_output()
+        jvm_args.append('-Dorg.graalvm.language.ruby.home=' + bootstrap_home)
 
-        libyarpbindings = list(mx.project('org.truffleruby.yarp.bindings').getArchivableResults())[0][0]
-        jvm_args.append('-Dtruffleruby.libyarpbindings=' + libyarpbindings)
+        jvm_args.append('-Dtruffleruby.repository=' + root)
 
         main_class = 'org.truffleruby.launcher.RubyLauncher'
         ruby_options = [

mx.truffleruby/suite.py

@@ -368,7 +368,7 @@
                 "TRUFFLERUBY", # We need this jar to run extconf.rb
                 "TRUFFLERUBY-LAUNCHER", # We need this jar to run extconf.rb
                 "sulong:SULONG_NATIVE", # We need this jar to find the toolchain with Toolchain#getToolPath
-                "org.truffleruby.yarp.bindings", # libyarpbindings.so
+                "TRUFFLERUBY_BOOTSTRAP_HOME", # libyarpbindings.so, librubysignal.so
             ],
             "license": ["EPL-2.0"],
         },
@@ -404,8 +404,8 @@
                 "src/main/c/ripper/<extsuffix:ripper>",
                 "src/main/c/syslog/<extsuffix:syslog>",
                 "src/main/c/zlib/<extsuffix:zlib>",
-                "lib/gems/gems/debug-1.7.1/lib/debug/<extsuffix:debug>",
-                "lib/gems/gems/rbs-2.8.2/lib/<extsuffix:rbs_extension>",
+                "src/main/c/debug/<extsuffix:debug>",
+                "src/main/c/rbs/<extsuffix:rbs_extension>",
             ],
             "license": [
                 "EPL-2.0",          # JRuby (we're choosing EPL out of EPL,GPL,LGPL)
@@ -667,19 +667,39 @@
             "maven": False,
         },
 
+        "TRUFFLERUBY_BOOTSTRAP_HOME": {
+            "description": "TruffleRuby bootstrap home used by a minimal TruffleRuby to run extconf.rb of default & bundled gems C extensions",
+            "native": True,
+            "platformDependent": True,
+            "layout": {
+                "lib/": [
+                    "file:lib/json",
+                    "file:lib/mri",
+                    "file:lib/patches",
+                    "file:lib/truffle",
+                    "dependency:org.truffleruby.yarp.bindings",
+                ],
+                "lib/cext/": [
+                    "file:lib/cext/*.rb",
+                    # libtruffleposix is handled specially in posix.rb to avoid a cyclic dependency between org.truffleruby.cext and TRUFFLERUBY-BOOTSTRAP-LAUNCHER
+                    "dependency:org.truffleruby.librubysignal",
+                ],
+                "lib/cext/include/": [
+                    "file:lib/cext/include/*",
+                ],
+            },
+            "maven": False,
+        },
+
         "TRUFFLERUBY_GRAALVM_SUPPORT_PLATFORM_AGNOSTIC": {
             "description": "Platform-agnostic TruffleRuby home files",
             "fileListPurpose": 'native-image-resources',
             "native": True,
             "platformDependent": False,
-            # The project org.truffleruby.cext touches lib/gems/gems/debug-1.7.1/ext
-            # and lib/gems/extensions/$ARCH-$OS/$ABI/rbs-2.8.2/gem.build_complete.
-            # This causes this layout distribution to be rebuilt even though nothing changes in the result.
-            # To avoid that we force org.truffleruby.cext to complete first.
-            "dependencies": ["org.truffleruby.cext"],
             "layout": {
                 "lib/": [
                     "file:lib/json",
+                    "file:lib/gems",
                     "file:lib/mri",
                     "file:lib/patches",
                     "file:lib/truffle",
@@ -690,23 +710,6 @@
                 "lib/cext/include/": [
                     "file:lib/cext/include/*",
                 ],
-                "lib/gems/": [
-                    {
-                        "source_type": "file",
-                        "path": "lib/gems/*",
-                        "exclude": [
-                            # The debug and rbs gems have native extensions.
-                            # Do not ship ext/ as it includes an unnecessary copy of the .so and intermediate files.
-                            # The .so in lib/ are copied in the platform-specific distribution.
-                            # <extsuffix:...> does not work in exclude, so use .* here (.{so,bundle} does not work either).
-                            "lib/gems/extensions",
-                            "lib/gems/gems/debug-1.7.1/ext",
-                            "lib/gems/gems/debug-1.7.1/lib/debug/debug.*",
-                            "lib/gems/gems/rbs-2.8.2/ext",
-                            "lib/gems/gems/rbs-2.8.2/lib/rbs_extension.*",
-                        ],
-                    },
-                ],
             },
             "license": [
                 "EPL-2.0",          # JRuby (we're choosing EPL out of EPL,GPL,LGPL)
@@ -732,10 +735,15 @@
                     "dependency:org.truffleruby.cext/src/main/c/cext-trampoline/<lib:trufflerubytrampoline>",
                     "dependency:org.truffleruby.librubysignal",
                 ],
-                # The platform-specific files from debug and rbs, see comment above
-                "lib/gems/": "file:lib/gems/extensions",
-                "lib/gems/gems/debug-1.7.1/lib/debug/": "dependency:org.truffleruby.cext/lib/gems/gems/debug-1.7.1/lib/debug/<extsuffix:debug>",
-                "lib/gems/gems/rbs-2.8.2/lib/": "dependency:org.truffleruby.cext/lib/gems/gems/rbs-2.8.2/lib/<extsuffix:rbs_extension>",
+                # Create the complete files to let RubyGems know the gems are fully built
+                "lib/gems/extensions/<cruby_arch>-<os>/<truffleruby_abi_version>/debug-1.7.1/gem.build_complete": "string:",
+                "lib/gems/extensions/<cruby_arch>-<os>/<truffleruby_abi_version>/rbs-2.8.2/gem.build_complete": "string:",
+                "lib/gems/gems/debug-1.7.1/lib/debug/": [
+                    "dependency:org.truffleruby.cext/src/main/c/debug/<extsuffix:debug>",
+                ],
+                "lib/gems/gems/rbs-2.8.2/lib/": [
+                    "dependency:org.truffleruby.cext/src/main/c/rbs/<extsuffix:rbs_extension>",
+                ],
                 "lib/mri/": [
                     "dependency:org.truffleruby.cext/src/main/c/bigdecimal/<extsuffix:bigdecimal>",
                     "dependency:org.truffleruby.cext/src/main/c/date/<extsuffix:date_core>",
@@ -764,9 +772,9 @@
         },
 
         "TRUFFLERUBY_GRAALVM_SUPPORT_NO_NI_RESOURCES": {
+            "description": "TruffleRuby support distribution for the GraalVM, the contents is not included as native image resources.",
             "native": True,
             "platformDependent": True,
-            "description": "TruffleRuby support distribution for the GraalVM, the contents is not included as native image resources.",
             "layout": {
                 "./": [
                     "file:CHANGELOG.md",

src/main/c/Makefile

@@ -32,22 +32,15 @@ BASIC_EXTCONF_DEPS := $(BASIC_DEPS) $(RBCONFIG) $(MKMF)
 # However, if libtruffleruby is recompiled, there is no need to rebuild C extensions, so it's a order-only-prerequisite.
 EXTCONF_DEPS := $(BASIC_EXTCONF_DEPS) | $(LIBTRUFFLERUBY)
 
-DEBUG_GEM := $(ROOT)/lib/gems/gems/debug-1.7.1
-DEBUG_GEM_EXT := $(DEBUG_GEM)/ext/debug
-DEBUG_GEM_EXT_SO := $(DEBUG_GEM_EXT)/debug.$(DLEXT)
-DEBUG_GEM_LIB_SO := $(DEBUG_GEM)/lib/debug/debug.$(DLEXT)
-
-RBS_GEM := $(ROOT)/lib/gems/gems/rbs-2.8.2
-RBS_GEM_EXT := $(RBS_GEM)/ext/rbs_extension
-RBS_GEM_EXT_SO := $(RBS_GEM_EXT)/rbs_extension.$(DLEXT)
-RBS_GEM_LIB_SO := $(RBS_GEM)/lib/rbs_extension.$(DLEXT)
+DEBUG_GEM_EXT := $(ROOT)/lib/gems/gems/debug-1.7.1/ext/debug
+RBS_GEM_EXT := $(ROOT)/lib/gems/gems/rbs-2.8.2/ext/rbs_extension
 
 IF_EXTCONF_FAIL := ( echo "`pwd`/extconf.rb failed:" 1>&2 && cat mkmf.log && false )
 
 all: $(TRAMPOLINE) $(LIBTRUFFLERUBY) openssl/openssl.$(DLEXT) zlib/zlib.$(DLEXT) \
 			psych/psych.$(DLEXT) ripper/ripper.$(DLEXT) syslog/syslog.$(DLEXT) nkf/nkf.$(DLEXT) \
 			bigdecimal/bigdecimal.$(DLEXT) date/date_core.$(DLEXT) io-console/console.$(DLEXT) etc/etc.$(DLEXT) \
-			rbconfig-sizeof/sizeof.$(DLEXT) $(DEBUG_GEM_LIB_SO) $(RBS_GEM_LIB_SO)
+			rbconfig-sizeof/sizeof.$(DLEXT) debug/debug.$(DLEXT) rbs/rbs_extension.$(DLEXT)
 
 clean: clean_cexts clean_truffleposix clean_trampoline
 
@@ -70,8 +63,8 @@ clean_cexts:
 	$(Q) rm -f io-console/Makefile io-console/*.o io-console/console.$(DLEXT)
 	$(Q) rm -f etc/Makefile etc/*.o etc/etc.$(DLEXT) etc/constdefs.h
 	$(Q) rm -f rbconfig-sizeof/Makefile rbconfig-sizeof/*.o rbconfig-sizeof/sizeof.$(DLEXT)
-	$(Q) rm -f $(DEBUG_GEM_EXT)/Makefile $(DEBUG_GEM_EXT)/*.o $(DEBUG_GEM_EXT_SO) $(DEBUG_GEM_LIB_SO)
-	$(Q) rm -f $(RBS_GEM_EXT)/Makefile $(RBS_GEM_EXT)/*.o $(RBS_GEM_EXT_SO) $(RBS_GEM_LIB_SO)
+	$(Q) rm -f debug/Makefile debug/*.o debug/debug.$(DLEXT)
+	$(Q) rm -f rbs/Makefile rbs/*.o rbs/rbs_extension.$(DLEXT)
 
 # spawn-helper
 $(SPAWN_HELPER): spawn-helper/Makefile spawn-helper/spawn-helper.c
@@ -178,23 +171,24 @@ psych/psych.$(DLEXT): psych/Makefile psych/*.c psych/*.h
 	$(Q) cd psych && $(MAKE) $(MKMF_MAKEFILE_SUBPROCESS_FLAGS)
 
 # debug bundled gem
-$(DEBUG_GEM_EXT)/Makefile: $(DEBUG_GEM_EXT)/extconf.rb $(EXTCONF_DEPS)
-	$(Q) cd $(DEBUG_GEM_EXT) && $(RUBY) extconf.rb || $(IF_EXTCONF_FAIL)
+debug/extconf.rb: $(DEBUG_GEM_EXT)/extconf.rb $(DEBUG_GEM_EXT)/*.c
+	$(Q) mkdir -p debug
+	$(Q) cd debug && cp $(DEBUG_GEM_EXT)/*.{rb,c} .
 
-$(DEBUG_GEM_EXT_SO): $(DEBUG_GEM_EXT)/Makefile $(DEBUG_GEM_EXT)/*.c
-	$(Q) cd $(DEBUG_GEM_EXT) && $(MAKE) $(MKMF_MAKEFILE_SUBPROCESS_FLAGS)
+debug/Makefile: debug/extconf.rb $(EXTCONF_DEPS)
+	$(Q) cd debug && $(RUBY) extconf.rb || $(IF_EXTCONF_FAIL)
 
-$(DEBUG_GEM_LIB_SO): $(DEBUG_GEM_EXT_SO)
-	$(Q) cp $< $@
-	$(Q) $(RUBY) $(ROOT)/tool/mark-gem-extension-as-built.rb debug
+debug/debug.$(DLEXT): debug/Makefile
+	$(Q) cd debug && $(MAKE) $(MKMF_MAKEFILE_SUBPROCESS_FLAGS)
 
 # rbs bundled gem
-$(RBS_GEM_EXT)/Makefile: $(RBS_GEM_EXT)/extconf.rb $(EXTCONF_DEPS)
-	$(Q) cd $(RBS_GEM_EXT) && $(RUBY) extconf.rb || $(IF_EXTCONF_FAIL)
+rbs/extconf.rb: $(RBS_GEM_EXT)/extconf.rb $(RBS_GEM_EXT)/*.c $(RBS_GEM_EXT)/*.h
+	$(Q) mkdir -p rbs
+	$(Q) cd rbs && cp $(RBS_GEM_EXT)/*.{rb,c,h} .
+
+rbs/Makefile: rbs/extconf.rb $(EXTCONF_DEPS)
+	$(Q) cd rbs && $(RUBY) extconf.rb || $(IF_EXTCONF_FAIL)
 
-$(RBS_GEM_EXT_SO): $(RBS_GEM_EXT)/Makefile $(RBS_GEM_EXT)/*.c $(RBS_GEM_EXT)/*.h
-	$(Q) cd $(RBS_GEM_EXT) && $(MAKE) $(MKMF_MAKEFILE_SUBPROCESS_FLAGS)
+rbs/rbs_extension.$(DLEXT): rbs/Makefile
+	$(Q) cd rbs && $(MAKE) $(MKMF_MAKEFILE_SUBPROCESS_FLAGS)
 
-$(RBS_GEM_LIB_SO): $(RBS_GEM_EXT_SO)
-	$(Q) cp $< $@
-	$(Q) $(RUBY) $(ROOT)/tool/mark-gem-extension-as-built.rb rbs

src/main/c/rubysignal/src/rubysignal.c

@@ -8,13 +8,19 @@
  * GNU Lesser General Public License version 2.1.
  */
 #include "org_truffleruby_signal_LibRubySignal.h"
+#include <locale.h>
 #include <pthread.h>
 #include <signal.h>
 #include <unistd.h>
 #include <sys/syscall.h>
 
 _Static_assert(sizeof(pthread_t) == sizeof(jlong), "Expected sizeof(pthread_t) == sizeof(jlong)");
 
+JNIEXPORT void JNICALL Java_org_truffleruby_signal_LibRubySignal_setupLocale(JNIEnv *env, jclass clazz) {
+  setlocale(LC_ALL, "C");
+  setlocale(LC_CTYPE, "");
+}
+
 static void empty_handler(int sig) {
 }
 

src/main/java/org/truffleruby/RubyLanguage.java

@@ -816,17 +816,12 @@ private TruffleFile searchRubyHome(Env env) {
     private boolean isRubyHome(TruffleFile path) {
         var lib = path.resolve("lib");
         return lib.resolve("truffle").isDirectory() &&
-                lib.resolve("gems").isDirectory() &&
+                (options.BUILDING_CORE_CEXTS || lib.resolve("gems").isDirectory()) &&
                 lib.resolve("patches").isDirectory();
     }
 
     private void loadLibYARPBindings() {
-        String libyarpbindings;
-        if (options.BUILDING_CORE_CEXTS) {
-            libyarpbindings = System.getProperty("truffleruby.libyarpbindings");
-        } else {
-            libyarpbindings = getRubyHome() + "/lib/libyarpbindings" + Platform.LIB_SUFFIX;
-        }
+        String libyarpbindings = getRubyHome() + "/lib/libyarpbindings" + Platform.LIB_SUFFIX;
         Parser.loadLibrary(libyarpbindings);
     }