[GR-18163] Docker fixes for 21.2

PullRequest: truffleruby/2826

Author: eregon

doc/contributor/docker.md

@@ -22,14 +22,14 @@ jt docker test --graalvm graalvm-ee.tar.gz llvm-toolchain-installable.jar:ruby-i
 jt docker test --standalone truffleruby-linux-amd64.tar.gz --test release_branch
 ```
 
-## Distributions
+To run tests on a specific distribution:
+```bash
+DOCKER=podman jt docker build --ubuntu1804 --standalone ~/Downloads/truffleruby-21.2.0-linux-amd64.tar.gz --test release/graal-vm/21.2
+```
 
-Pick one of:
+## Distributions
 
-* Oracle Linux 7, `--ol7` (default)
-* Ubuntu 18.04 `--ubuntu1804`
-* Ubuntu 16.04 `--ubuntu1604`
-* Fedora 28, `--fedora28`
+Pick one of the distributions in [docker-configs.yaml](../../tool/docker-configs.yaml).
 
 ## Methods of installing
 

tool/docker-configs.yaml

@@ -1,45 +1,66 @@
-rpm_packages: &rpm_packages
-  locale:
+rpm: &rpm
   tar: tar gzip
   specs: which findutils
   zlib: zlib-devel
   openssl: openssl-devel
   cext: gcc make
+  set-locale:
+    - ENV LANG=en_US.UTF-8
 
-deb_packages: &deb_packages
+deb: &deb
   locale: locales
   tar:
   specs: netbase
   zlib: libz-dev
   openssl: libssl-dev
   cext: gcc make
+  set-locale:
+    # Uncomment the en_US.UTF-8 line in /etc/locale.gen
+    - RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen
+    # locale-gen generates locales for all uncommented locales in /etc/locale.gen
+    - RUN locale-gen
+    - ENV LANG=en_US.UTF-8
 
 ol7:
   base: oraclelinux:7-slim
-  set-locale:
-    - ENV LANG=en_US.UTF-8
   install: RUN yum install -y
-  <<: *rpm_packages
+  locale:
+  <<: *rpm
+
+ol8:
+  base: oraclelinux:8-slim
+  install: RUN microdnf install -y
+  locale: glibc-langpack-en
+  <<: *rpm
 
 fedora28:
   base: fedora:28
-  set-locale:
-    - ENV LANG=en_US.UTF-8
   install: RUN dnf install -y
-  <<: *rpm_packages
+  locale:
+  <<: *rpm
+
+fedora34:
+  base: fedora:34
+  install: RUN dnf install -y
+  locale: glibc-langpack-en
+  <<: *rpm
+
+ubuntu2004:
+  base: ubuntu:20.04
+  install: RUN apt-get update && apt-get install -y
+  <<: *deb
 
 ubuntu1804:
   base: ubuntu:18.04
-  set-locale:
-    - RUN locale-gen en_US.UTF-8
-    - ENV LANG=en_US.UTF-8
   install: RUN apt-get update && apt-get install -y
-  <<: *deb_packages
+  <<: *deb
 
 ubuntu1604:
   base: ubuntu:16.04
-  set-locale:
-    - RUN locale-gen en_US.UTF-8
-    - ENV LANG=en_US.UTF-8
   install: RUN apt-get update && apt-get install -y
-  <<: *deb_packages
+  <<: *deb
+
+debian10:
+  base: debian:buster
+  install: RUN apt-get update && apt-get install -y
+  <<: *deb

tool/docker.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+
 class JT
   class Docker
     include Utilities
@@ -18,6 +20,14 @@ def docker(*args)
       end
     end
 
+    private def docker_config
+      @config ||= YAML.load_file(File.join(TRUFFLERUBY_DIR, 'tool', 'docker-configs.yaml'))
+    end
+
+    private def docker_distros
+      docker_config.each_pair.select { |_name, details| details.key?('base') }.map(&:first).map { |distro| "--#{distro}" }
+    end
+
     private def docker_build(*args)
       if args.first.nil? || args.first.start_with?('--')
         image_name = 'truffleruby-test'
@@ -30,7 +40,7 @@ def docker(*args)
     end
 
     private def docker_test(*args)
-      distros = ['--ol7', '--ubuntu1804', '--ubuntu1604', '--fedora28']
+      distros = docker_distros
 
       distros.each do |distro|
         puts '**********************************'
@@ -50,10 +60,9 @@ def docker(*args)
     end
 
     private def dockerfile(*args)
-      require 'yaml'
-      config = @config ||= YAML.load_file(File.join(TRUFFLERUBY_DIR, 'tool', 'docker-configs.yaml'))
+      config = docker_config
 
-      distro = 'ol7'
+      distro_name = 'ol7'
       install_method = nil
       rebuild_images = false
       rebuild_openssl = true
@@ -65,8 +74,8 @@ def docker(*args)
       until args.empty?
         arg = args.shift
         case arg
-        when '--ol7', '--ubuntu1804', '--ubuntu1604', '--fedora28'
-          distro = arg[2..-1]
+        when *docker_distros
+          distro_name = arg[2..-1]
         when '--graalvm'
           install_method = :graalvm
           graalvm_tarball = args.shift
@@ -93,7 +102,7 @@ def docker(*args)
         end
       end
 
-      distro = config.fetch(distro)
+      distro = config.fetch(distro_name)
       run_post_install_hook = rebuild_openssl
 
       packages = []
@@ -107,9 +116,12 @@ def docker(*args)
       packages << distro.fetch('cext')
 
       proxy_vars = []
-      %w[http_proxy https_proxy no_proxy].each do |var|
-        value = ENV[var]
-        proxy_vars << "ENV #{var}=#{value}" if value
+      # There is an issue with dnf + proxy in Fedora 34, install packages outside proxy to workaround
+      unless distro_name == 'fedora34'
+        %w[http_proxy https_proxy no_proxy].each do |var|
+          value = ENV[var]
+          proxy_vars << "ENV #{var}=#{value}" if value
+        end
       end
 
       lines = [
@@ -119,6 +131,9 @@ def docker(*args)
         *distro.fetch('set-locale'),
       ]
 
+      # Check the locale is properly generated
+      lines << 'RUN locale -a | grep en_US.utf8'
+
       lines << 'WORKDIR /test'
 
       lines << 'RUN useradd -ms /bin/bash test'
@@ -181,7 +196,8 @@ def docker(*args)
 
         unless print_only
           chdir(docker_dir) do
-            raw_sh 'git', 'clone', '--branch', test_branch, TRUFFLERUBY_DIR, 'truffleruby-tests'
+            branch_args = test_branch == 'current' ? [] : ['--branch', test_branch]
+            raw_sh 'git', 'clone', *branch_args, TRUFFLERUBY_DIR, 'truffleruby-tests'
             test_files.each do |file|
               FileUtils.cp_r "truffleruby-tests/#{file}", '.'
             end
@@ -241,6 +257,7 @@ def docker(*args)
 
           %w[:command_line :security :language :core :library :capi :library_cext :truffle :truffle_capi].each do |set|
             t_config = c.empty? ? '' : '-T' + c
+            t_config << ' -T--experimental-options -T--pattern-matching'
             t_excludes = excludes.map { |e| '--excl-tag ' + e }.join(' ')
             lines << "RUN ruby spec/mspec/bin/mspec -t #{ruby_bin}/ruby #{t_config} #{t_excludes} #{set}"
           end

tool/jt.rb

@@ -2317,6 +2317,9 @@ def rubocop(*args)
 
   def idea(*args)
     ENV['ECLIPSE_EXE'] ||= install_eclipse
+    if args.empty?
+      args = %w[--dy /substratevm,/tools,/vm]
+    end
     mx(*args, 'intellijinit')
   end