[NFC] Split optz functions out of libc (#17179)

Put them in a new libc_optz library that will override parts of libc with the
size-optimized versions instead. This avoids rebuilding all of libc for optz,
and can save a few minutes on build-linux on CI.

Avoid optz in MAIN_MODULE=1, because this override method (link in
libc_optz first so it overrides libc) conflicts with it. But that is fine, as in
MAIN_MODULE=1 the code size benefit of optz is not noticeable anyhow.

Add a test for MAIN_MODULE=1 + -Oz, which was missing before. That
test would fail without the previous paragraph.

Author: kripken

embuilder.py

@@ -30,8 +30,8 @@
     'libcompiler_rt',
     'libc',
     'libc-debug',
-    'libc-optz',
-    'libc-optz-debug',
+    'libc_optz',
+    'libc_optz-debug',
     'libc++abi',
     'libc++abi-except',
     'libc++abi-noexcept',
@@ -67,6 +67,8 @@
     'libcompiler_rt-wasm-sjlj',
     'libc-mt',
     'libc-mt-debug',
+    'libc_optz-mt',
+    'libc_optz-mt-debug',
     'libc++abi-mt',
     'libc++abi-mt-noexcept',
     'libc++-mt',

src/library.js

@@ -364,9 +364,11 @@ mergeInto(LibraryManager.library, {
 
   // In -Oz builds, we replace memcpy() altogether with a non-unrolled wasm
   // variant, so we should never emit emscripten_memcpy_big() in the build.
-  // In STANDALONE_WASM we aviud the emscripten_memcpy_big dependency so keep
+  // (However, in MAIN_MODULE=1 mode we link in all system libraries, which does
+  // end up adding code that refers to this.)
+  // In STANDALONE_WASM we avoid the emscripten_memcpy_big dependency so keep
   // the wasm file standalone.
-#if SHRINK_LEVEL < 2 && !STANDALONE_WASM
+#if (SHRINK_LEVEL < 2 || MAIN_MODULE == 1) && !STANDALONE_WASM
 
   emscripten_memcpy_big__sig: 'vppp',
 #if MIN_CHROME_VERSION < 45 || MIN_EDGE_VERSION < 14 || MIN_FIREFOX_VERSION < 34 || MIN_IE_VERSION != TARGET_NOT_SUPPORTED || MIN_SAFARI_VERSION < 100101

tests/test_other.py

@@ -12028,9 +12028,10 @@ def test_memory64(self):
       self.do_runf(test_file('hello_world.c'), 'hello, world', emcc_args=['-sMEMORY64', opt])
 
   # Verfy that MAIN_MODULE=1 (which includes all symbols from all libraries)
-  # works with -sPROXY_POSIX_SOCKETS.
-  def test_dylink_proxy_posix_sockets(self):
-    self.do_runf(test_file('hello_world.cpp'), emcc_args=['-lwebsocket.js', '-sMAIN_MODULE=1', '-sPROXY_POSIX_SOCKETS'])
+  # works with -sPROXY_POSIX_SOCKETS and -Oz, both of which affect linking of
+  # system libraries in different ways.
+  def test_dylink_proxy_posix_sockets_oz(self):
+    self.do_runf(test_file('hello_world.cpp'), emcc_args=['-lwebsocket.js', '-sMAIN_MODULE=1', '-sPROXY_POSIX_SOCKETS', '-Oz'])
 
   def test_in_tree_header_usage(self):
     # Using headers directly from where they live in the source tree does not work.

tools/system_libs.py

@@ -529,32 +529,6 @@ def get_default_variation(cls, **kwargs):
     return super().get_default_variation(is_debug=settings.ASSERTIONS, **kwargs)
 
 
-class OptimizedAggressivelyForSizeLibrary(Library):
-  def __init__(self, **kwargs):
-    self.is_optz = kwargs.pop('is_optz')
-    super().__init__(**kwargs)
-
-  def get_base_name(self):
-    name = super().get_base_name()
-    if self.is_optz:
-      name += '-optz'
-    return name
-
-  def get_cflags(self):
-    cflags = super().get_cflags()
-    if self.is_optz:
-      cflags += ['-DEMSCRIPTEN_OPTIMIZE_FOR_OZ']
-    return cflags
-
-  @classmethod
-  def vary_on(cls):
-    return super().vary_on() + ['is_optz']
-
-  @classmethod
-  def get_default_variation(cls, **kwargs):
-    return super().get_default_variation(is_optz=settings.SHRINK_LEVEL >= 2, **kwargs)
-
-
 class Exceptions(IntEnum):
   """
   This represents exception handling mode of Emscripten. Currently there are
@@ -727,7 +701,6 @@ class libnoexit(Library):
 
 class libc(MuslInternalLibrary,
            DebugLibrary,
-           OptimizedAggressivelyForSizeLibrary,
            AsanInstrumentedLibrary,
            MTLibrary):
   name = 'libc'
@@ -782,7 +755,6 @@ def get_libcall_files(self):
       '__math_oflow.c', '__math_oflowf.c',
       '__math_uflow.c', '__math_uflowf.c',
       '__math_invalid.c', '__math_invalidf.c', '__math_invalidl.c',
-      'pow_small.c', 'log_small.c', 'log2_small.c',
       'pow.c', 'pow_data.c', 'log.c', 'log_data.c', 'log2.c', 'log2_data.c'
     ]
     math_files = files_in_path(path='system/lib/libc/musl/src/math', filenames=math_files)
@@ -908,10 +880,10 @@ def get_files(self):
           'proxying_stub.c',
         ])
 
-    if self.is_optz:
-      ignore += ['pow.c', 'pow_data.c', 'log.c', 'log_data.c', 'log2.c', 'log2_data.c']
-    else:
-      ignore += ['pow_small.c', 'log_small.c', 'log2_small.c']
+    # These files are in libc directories, but only built in libc_optz.
+    ignore += [
+      'pow_small.c', 'log_small.c', 'log2_small.c'
+    ]
 
     ignore = set(ignore)
     for dirpath, dirnames, filenames in os.walk(musl_srcdir):
@@ -1041,6 +1013,56 @@ def customize_build_cmd(self, cmd, filename):
     return cmd
 
 
+# Contains the files from libc that are optimized differently in -Oz mode, where
+# we want to aggressively optimize them for size. This is linked in before libc
+# so we can override those specific files, when in -Oz.
+class libc_optz(libc):
+  name = 'libc_optz'
+
+  cflags = ['-Os', '-fno-builtin', '-DEMSCRIPTEN_OPTIMIZE_FOR_OZ']
+
+  def __init__(self, **kwargs):
+    super().__init__(**kwargs)
+    self.non_lto_files = self.get_libcall_files()
+
+  def get_libcall_files(self):
+    # see comments in libc.customize_build_cmd
+
+    # some files also appear in libc, and a #define affects them
+    mem_files = files_in_path(
+      path='system/lib/libc',
+      filenames=['emscripten_memcpy.c', 'emscripten_memset.c',
+                 'emscripten_memmove.c'])
+
+    # some functions have separate files
+    math_files = files_in_path(
+      path='system/lib/libc/musl/src/math',
+      filenames=['pow_small.c', 'log_small.c', 'log2_small.c'])
+
+    return mem_files + math_files
+
+  def get_files(self):
+    libcall_files = self.get_libcall_files()
+
+    # some files also appear in libc, and a #define affects them
+    mem_files = files_in_path(
+      path='system/lib/libc/musl/src/string',
+      filenames=['memcmp.c'])
+
+    return libcall_files + mem_files
+
+  def customize_build_cmd(self, cmd, filename):
+    if filename in self.non_lto_files:
+      # see comments in libc.customize_build_cmd
+      cmd = [a for a in cmd if not a.startswith('-flto')]
+      cmd = [a for a in cmd if not a.startswith('-O')]
+      cmd += ['-O2']
+    return cmd
+
+  def can_use(self):
+    return super(libc_optz, self).can_use() and settings.SHRINK_LEVEL >= 2
+
+
 class libprintf_long_double(libc):
   name = 'libprintf_long_double'
   cflags = ['-DEMSCRIPTEN_PRINTF_LONG_DOUBLE']
@@ -1835,9 +1857,16 @@ def add_library(libname):
   if settings.JS_MATH:
     add_library('libjsmath')
 
-  # to override the normal libc printf, we must come before it
+  # C libraries that override libc must come before it
   if settings.PRINTF_LONG_DOUBLE:
     add_library('libprintf_long_double')
+  # libc_optz is a size optimization, and therefore not really important when
+  # MAIN_MODULE=1 (which links in all system libraries, leading to a large
+  # size far bigger than any savings from libc_optz). Also, libc_optz overrides
+  # parts of libc, which will not link due to --whole-archive in MAIN_MODULE=1
+  # currently.
+  if settings.SHRINK_LEVEL >= 2 and settings.MAIN_MODULE != 1:
+    add_library('libc_optz')
 
   if settings.STANDALONE_WASM:
     add_library('libstandalonewasm')