Merge pull request #14555 from ydb-platform/merge-libs-250214-0050

Author: alexv-smirnov

build/conf/java.conf

@@ -1850,7 +1850,6 @@ MANAGED_PEERS=
 MANAGED_PEERS_CLOSURE=
 MANAGEABLE_PEERS_ROOTS=contrib/java
 HAS_MANAGEABLE_PEERS=no
-PROPAGATES_MANAGEABLE_PEERS=no
 
 # tag:java-specific
 DEPENDENCY_MANAGEMENT_VALUE=

build/conf/linkers/ld.conf

@@ -244,6 +244,7 @@ REAL_LINK_EXEC_DYN_LIB_CMDLINE =\
     $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} \
     ${hide;input:"build/scripts/link_exe.py"} \
     --target $TARGET
+REAL_LINK_EXEC_DYN_LIB_CMDLINE+=--start-plugins ${ext=.pyplugin:SRCS_GLOBAL} --end-plugins
 REAL_LINK_EXEC_DYN_LIB_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_EXEC_DYN_LIB_CMDLINE+=\
     $_ROOT_FLAGS \
@@ -271,6 +272,7 @@ REAL_LINK_DYN_LIB_CMDLINE =\
     $YMAKE_PYTHON ${input:"build/scripts/link_dyn_lib.py"} \
     ${hide;input:"build/scripts/link_exe.py"} \
     --target $TARGET
+REAL_LINK_DYN_LIB_CMDLINE+=--start-plugins ${ext=.pyplugin:SRCS_GLOBAL} --end-plugins
 REAL_LINK_DYN_LIB_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_DYN_LIB_CMDLINE+=\
     ${pre=--whole-archive-peers :WHOLE_ARCHIVE_PEERS} \

build/mapping.conf.json

@@ -498,6 +498,7 @@
         "7914217459": "https://devtools-registry.s3.yandex.net/7914217459",
         "7948644946": "https://devtools-registry.s3.yandex.net/7948644946",
         "7994647367": "https://devtools-registry.s3.yandex.net/7994647367",
+        "8029671029": "https://devtools-registry.s3.yandex.net/8029671029",
         "5486731632": "https://devtools-registry.s3.yandex.net/5486731632",
         "5514350352": "https://devtools-registry.s3.yandex.net/5514350352",
         "5514360398": "https://devtools-registry.s3.yandex.net/5514360398",
@@ -1729,6 +1730,7 @@
         "7914217459": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "7948644946": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "7994647367": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
+        "8029671029": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",

build/platform/test_tool/host.ya.make.inc

@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018525488)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029723475)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018523316)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029717949)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018529985)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029737034)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018521599)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029713125)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018527620)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029729848)
 
 ENDIF()

build/platform/test_tool/host_os.ya.make.inc

@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018808572)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029664104)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018807025)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029660543)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018811524)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029671029)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018805118)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029657728)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8018809911)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8029667517)
 
 ENDIF()

build/scripts/link_dyn_lib.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import sys
 import os
+import json
 import subprocess
 import tempfile
 import collections
@@ -129,8 +130,6 @@ def fix_windows_param(ex):
         return ['/DEF:{}'.format(def_file.name)]
 
 
-MUSL_LIBS = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil'
-
 CUDA_LIBRARIES = {
     '-lcublas_static': '-lcublas',
     '-lcublasLt_static': '-lcublasLt',
@@ -179,14 +178,6 @@ def do_fix(p):
     return sum((do_fix(x) for x in c), [])
 
 
-def fix_cmd_for_musl(cmd):
-    flags = []
-    for flag in cmd:
-        if flag not in MUSL_LIBS:
-            flags.append(flag)
-    return flags
-
-
 def fix_cmd_for_dynamic_cuda(cmd):
     flags = []
     for flag in cmd:
@@ -208,7 +199,7 @@ def fix_blas_resolving(cmd):
     return cmd
 
 
-def parse_args():
+def parse_args(args):
     parser = optparse.OptionParser()
     parser.disable_interspersed_args()
     parser.add_option('--arch')
@@ -218,7 +209,6 @@ def parse_args():
     parser.add_option('--build-root')
     parser.add_option('--fix-elf')
     parser.add_option('--linker-output')
-    parser.add_option('--musl', action='store_true')
     parser.add_option('--dynamic-cuda', action='store_true')
     parser.add_option('--cuda-architectures',
                       help='List of supported CUDA architectures, separated by ":" (e.g. "sm_52:compute_70:lto_90a"')
@@ -229,11 +219,26 @@ def parse_args():
     parser.add_option('--custom-step')
     parser.add_option('--python')
     thinlto_cache.add_options(parser)
-    return parser.parse_args()
+    return parser.parse_args(args)
 
 
 if __name__ == '__main__':
-    opts, args = parse_args()
+    args = sys.argv[1:]
+    plugins = []
+
+    if '--start-plugins' in args:
+        ib = args.index('--start-plugins')
+        ie = args.index('--end-plugins')
+        plugins = args[ib + 1:ie]
+        args = args[:ib] + args[ie + 1:]
+
+    for p in plugins:
+        res = subprocess.check_output([sys.executable, p] + args).decode().strip()
+
+        if res:
+            args = json.loads(res)
+
+    opts, args = parse_args(args)
 
     assert opts.arch
     assert opts.target
@@ -242,8 +247,6 @@ def parse_args():
     cmd = fix_cmd(opts.arch, cmd)
     cmd = fix_py2(cmd)
 
-    if opts.musl:
-        cmd = fix_cmd_for_musl(cmd)
     if opts.dynamic_cuda:
         cmd = fix_cmd_for_dynamic_cuda(cmd)
     else:

build/scripts/link_exe.py

@@ -335,10 +335,13 @@ def parse_args(args):
 
 if __name__ == '__main__':
     args = sys.argv[1:]
-    ib = args.index('--start-plugins')
-    ie = args.index('--end-plugins')
-    plugins = args[ib + 1:ie]
-    args = args[:ib] + args[ie + 1:]
+    plugins = []
+
+    if '--start-plugins' in args:
+        ib = args.index('--start-plugins')
+        ie = args.index('--end-plugins')
+        plugins = args[ib + 1:ie]
+        args = args[:ib] + args[ie + 1:]
 
     for p in plugins:
         res = subprocess.check_output([sys.executable, p] + args).decode().strip()

build/ya.conf.json

@@ -552,6 +552,7 @@
                 "cxx_compiler": "$(CLANG)/bin/clang++",
                 "llvm-symbolizer": "$(CLANG)/bin/llvm-symbolizer",
                 "match_root": "CLANG",
+                "nm": "$(CLANG)/bin/llvm-nm",
                 "objcopy": "$(CLANG)/bin/llvm-objcopy",
                 "profiles": "$(XCODE_TOOLS_ROOT-sbr:799017771)/Xcode/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/CoreSimulator/Profiles",
                 "simctl": "$(XCODE_TOOLS_ROOT-sbr:799017771)/Xcode/SystemRoot/PrivateFrameworks/CoreSimulator.framework/Resources/bin/simctl",
@@ -1229,6 +1230,7 @@
                 "cxx_compiler": "$(CLANG)/bin/clang++",
                 "llvm-symbolizer": "$(CLANG)/bin/llvm-symbolizer",
                 "match_root": "CLANG",
+                "nm": "$(CLANG)/bin/llvm-nm",
                 "objcopy": "$(CLANG)/bin/llvm-objcopy",
                 "profiles": "$(XCODE_TOOLS_ROOT-sbr:799017771)/Xcode/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/CoreSimulator/Profiles",
                 "simctl": "$(XCODE_TOOLS_ROOT-sbr:799017771)/Xcode/SystemRoot/PrivateFrameworks/CoreSimulator.framework/Resources/bin/simctl",

build/ymake.core.conf

@@ -769,7 +769,6 @@ module _BASE_UNIT: _BARE_UNIT {
 
     when ($MUSL == "yes") {
         CFLAGS += -D_musl_
-        LINK_DYN_LIB_FLAGS += --musl
         PEERDIR+=contrib/libs/musl/include
     }
 
@@ -824,8 +823,9 @@ module _BASE_UNIT: _BARE_UNIT {
     DEFAULT(SWIG_LANG python)
     DEFAULT(GP_FLAGS -CtTLANSI-C -Dk* -c)
 
-    when ($NEED_BINUTILS_PEERDIR && $BINUTILS_USED && $NEED_PLATFORM_PEERDIRS == "yes") {
-        PEERDIR+=build/platform/binutils
+    when ($NEED_LLVM_TOOLS_PEERDIR && $NEED_PLATFORM_PEERDIRS == "yes") {
+        PEERDIR+=build/platform/clang
+        LLVM_TOOLS_ROOT=$CLANG18_RESOURCE_GLOBAL
     }
 
     when ($TIDY_ENABLED == "yes") {
@@ -2536,20 +2536,25 @@ when ($BT_MINSIZEREL == "yes" || $LINKER_ICF == "yes") {
 OBJCOPY_TOOL=$OBJCOPY_TOOL_VENDOR
 OBJDUMP_TOOL=$OBJDUMP_TOOL_VENDOR
 STRIP_TOOL=$STRIP_TOOL_VENDOR
-NEED_BINUTILS_PEERDIR=
-BINUTILS_USED=
+NM_TOOL=$NM_TOOL_VENDOR
+NEED_LLVM_TOOLS_PEERDIR=
+LLVM_TOOLS_ROOT=
 
 when (!$OBJCOPY_TOOL_VENDOR) {
-    OBJCOPY_TOOL=$BINUTILS_ROOT_RESOURCE_GLOBAL/bin/objcopy
-    NEED_BINUTILS_PEERDIR=yes
+    OBJCOPY_TOOL=${LLVM_TOOLS_ROOT}/bin/llvm-objcopy
+    NEED_LLVM_TOOLS_PEERDIR=yes
 }
 when (!$OBJDUMP_TOOL_VENDOR) {
-    OBJDUMP_TOOL=$BINUTILS_ROOT_RESOURCE_GLOBAL/bin/objdump
-    NEED_BINUTILS_PEERDIR=yes
+    OBJDUMP_TOOL=${LLVM_TOOLS_ROOT}/bin/llvm-objdump
+    NEED_LLVM_TOOLS_PEERDIR=yes
 }
 when (!$STRIP_TOOL_VENDOR) {
-    STRIP_TOOL=$BINUTILS_ROOT_RESOURCE_GLOBAL/bin/strip
-    NEED_BINUTILS_PEERDIR=yes
+    STRIP_TOOL=${LLVM_TOOLS_ROOT}/bin/llvm-strip
+    NEED_LLVM_TOOLS_PEERDIR=yes
+}
+when (!$NM_TOOL_VENDOR) {
+    NM_TOOL=${LLVM_TOOLS_ROOT}/bin/llvm-nm
+    NEED_LLVM_TOOLS_PEERDIR=yes
 }
 
 SPLIT_DWARF_VALUE=no
@@ -2586,7 +2591,6 @@ when ($SPLIT_DWARF_VALUE == "yes" && $NO_SPLIT_DWARF != "yes" && $NO_DEBUGINFO !
         $OBJCOPY_TOOL --only-keep-debug $TARGET $SPLIT_DWARF_OUTPUT && \
         $STRIP_TOOL --strip-debug $TARGET && \
         $OBJCOPY_TOOL --remove-section=.gnu_debuglink --add-gnu-debuglink $SPLIT_DWARF_OUTPUT $TARGET
-    BINUTILS_USED=yes
 }
 
 ### @usage: EXTRALIBS_STATIC(Libs...)
@@ -3135,10 +3139,19 @@ macro _SRC_f_old(SRC, SRCFLAGS...) {
     .CMD=$YMAKE_PYTHON ${input:"build/scripts/f2c.py"} -t ${tool:"contrib/tools/f2c"} -c ${input:SRC} -o ${output:SRC.c} ${hide;output_include:"f2c.h"} ${hide;kv:"p FT"} ${hide;kv:"pc light-green"}
 }
 
+### @usage: AR_PLUGIN(plugin_name)
+###
+### Register script, which will process module's .a (archive) output
+### Script will receive path to archive, which it should modify in place
 macro AR_PLUGIN(name) {
     SET(_AR_PLUGIN $name.pyplugin)
 }
 
+### @usage: LD_PLUGIN(plugin_name)
+###
+### Register script, which will process all inputs to any link_exe.py call with modules's library
+### Script will receive all arguments to link_exe.py, and can output into stdout preprocessed list
+### of all arguments, in JSON format
 macro LD_PLUGIN(name) {
     SRCS(GLOBAL $name.pyplugin)
 }

build/ymake_conf.py

@@ -1067,6 +1067,7 @@ def __init__(self, build, detector):
         self.objcopy = self.params.get('objcopy')
         self.objdump = self.params.get('objdump')
         self.isystem = self.params.get('isystem')
+        self.nm = self.params.get('nm')
 
         self.dwarf_tool = self.target.find_in_dict(self.params.get('dwarf_tool'))
 
@@ -1665,6 +1666,7 @@ def __init__(self, tc, build):
         self.strip = self.tc.strip
         self.objcopy = self.tc.objcopy
         self.objdump = self.tc.objdump
+        self.nm = self.tc.nm
 
         self.musl = Setting('MUSL', convert=to_bool)
 
@@ -1728,6 +1730,7 @@ def print_linker(self):
         emit('STRIP_TOOL_VENDOR', self.strip)
         emit('OBJCOPY_TOOL_VENDOR', self.objcopy)
         emit('OBJDUMP_TOOL_VENDOR', self.objdump)
+        emit('NM_TOOL_VENDOR', self.nm)
 
         emit('_LD_FLAGS', self.ld_flags)
         emit('LD_SDK_VERSION', self.ld_sdk)
@@ -2013,10 +2016,6 @@ def print_compiler(self):
                 # for msvc compatibility
                 # https://clang.llvm.org/docs/UsersManual.html#microsoft-extensions
                 # '-fdelayed-template-parsing',
-                '-Wno-deprecated-this-capture',
-                '-Wno-c++11-narrowing-const-reference',
-                '-Wno-vla-cxx-extension',  # https://github.com/llvm/llvm-project/issues/62836
-                '-Wno-invalid-offsetof',
             ]
             if target.is_x86:
                 flags.append('-m32')
@@ -2037,12 +2036,16 @@ def print_compiler(self):
                 # Issue a warning if certain overload is hidden due to inheritance
                 '-Woverloaded-virtual',
                 '-Wno-ambiguous-reversed-operator',
+                '-Wno-c++11-narrowing-const-reference',
                 '-Wno-defaulted-function-deleted',
                 '-Wno-deprecated-anon-enum-enum-conversion',
                 '-Wno-deprecated-enum-enum-conversion',
                 '-Wno-deprecated-enum-float-conversion',
+                '-Wno-deprecated-this-capture',
                 '-Wno-deprecated-volatile',
+                '-Wno-invalid-offsetof',
                 '-Wno-undefined-var-template',
+                '-Wno-vla-cxx-extension',  # https://github.com/llvm/llvm-project/issues/62836
             ]
 
         defines.append('/D_WIN32_WINNT={0}'.format(WINDOWS_VERSION_MIN))