Merge pull request #14429 from ydb-platform/merge-libs-250211-1102

Author: alexv-smirnov

build/conf/linkers/ld.conf

@@ -213,6 +213,7 @@ LINK_SCRIPT_EXE_FLAGS=
 
 REAL_LINK_EXE_CMDLINE =\
     $YMAKE_PYTHON ${input:"build/scripts/link_exe.py"} \
+REAL_LINK_EXE_CMDLINE+=--start-plugins ${ext=.pyplugin:SRCS_GLOBAL} --end-plugins
 REAL_LINK_EXE_CMDLINE+=--clang-ver $CLANG_VER
 REAL_LINK_EXE_CMDLINE+=$_LD_LINKER_OUTPUT
 REAL_LINK_EXE_CMDLINE+=\
@@ -347,6 +348,12 @@ LINK_EXEC_DYN_LIB=\
 _LD_LINK_LIB_EXTRA_INPUT=
 _LD_TAIL_LINK_LIB=$AUTO_INPUT $_LD_LINK_LIB_EXTRA_INPUT ${hide;kv:"p AR"} $TOOLCHAIN_ENV ${hide;kv:"pc light-red"} ${hide;kv:"show_out"}
 _LD_ARCHIVER=$YMAKE_PYTHON ${input:"build/scripts/link_lib.py"} ${quo:AR_TOOL} $AR_TYPE $_LD_LLVM_AR_FORMAT $ARCADIA_BUILD_ROOT $_LD_AR_PLUGIN
+when ($MAPSMOBI_BUILD_TARGET == "yes") {
+    _LD_ARCHIVER+= "no-openssl3"
+} 
+otherwise {
+    _LD_ARCHIVER+= "enable-openssl3"
+}
 _LD_LIB_GENERATE_MF=$GENERATE_MF &&
 when ($TIDY == "yes") {
     _LD_ARCHIVER=$YMAKE_PYTHON ${input:"build/scripts/clang_tidy_arch.py"} --source-root $ARCADIA_ROOT --build-root $ARCADIA_BUILD_ROOT --output-file

build/conf/project_specific/yt.conf

@@ -1,5 +1,5 @@
 macro GENERATE_YT_RECORD(Yaml, OUTPUT_INCLUDES[]) {
     .CMD=${tool:"yt/yt/tools/record_codegen"} --input ${input:Yaml} --output-root $ARCADIA_BUILD_ROOT --output-cpp ${output;norel;noext;suf=.record.cpp:Yaml} ${hide;output;norel;noext;suf=.record.h:Yaml} ${pre=--output-include :OUTPUT_INCLUDES} ${hide;output_include:OUTPUT_INCLUDES} ${hide;output_include:"yt/yt/client/table_client/record_codegen_deps.h"} ${hide;kv:"p RC"}
-    .SEM=packages-ITEM && packages-name Python3 && mpackages-ITEM && mpackages-name Python3 && target_commands-ITEM && target_commands-macro add_custom_command && target_commands-args OUTPUT ${output;norel;noext;suf=.record.cpp:Yaml} ${output;norel;noext;suf=.record.h:Yaml} DEPENDS ${input:Yaml} ${input:"yt/yt/tools/record_codegen/__main__.py"} ${input:"yt/python/yt/record_codegen_helpers/__init__.py"} COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=$ENV{PYTHONPATH}:${ARCADIA_ROOT}/yt/python/yt" ${Python3_EXECUTABLE} ${input:"yt/yt/tools/record_codegen/__main__.py"} --input ${input:Yaml} --output-root $ARCADIA_BUILD_ROOT --output-cpp ${output;norel;noext;suf=.record.cpp:Yaml} ${pre=--output-include :OUTPUT_INCLUDES} ${hide;output;norel;noext;suf=.record.h:Yaml}
+    .SEM=custom_runs-ITEM && custom_runs-outputs ${output;norel;noext;suf=.record.cpp:Yaml} ${output;norel;noext;suf=.record.h:Yaml} && custom_runs-depends ${input:Yaml} ${input:"yt/yt/tools/record_codegen/__main__.py"} ${input:"yt/python/yt/record_codegen_helpers/__init__.py"} && custom_runs-env "PYTHONPATH=$ENV{PYTHONPATH}:${ARCADIA_ROOT}/yt/python/yt" && custom_runs-command ${Python3_EXECUTABLE} ${input:"yt/yt/tools/record_codegen/__main__.py"} --input ${input:Yaml} --output-root $ARCADIA_BUILD_ROOT --output-cpp ${output;norel;noext;suf=.record.cpp:Yaml} ${pre=--output-include :OUTPUT_INCLUDES} ${hide;output;norel;noext;suf=.record.h:Yaml} && custom_runs-cmake_packages-ITEM && custom_runs-cmake_packages-name Python3
     PEERDIR(yt/yt/client)
 }

build/export_generators/cmake/dir_packages.jinja

@@ -1,9 +1,9 @@
 {%- macro RenderPackages(target) -%}
 {%-     set packages = [] -%}
-{%-     if (target.packages is defined) and (target.packages|length) -%}
+{%-     if target.packages|length -%}
 {%-         set packages = packages + target.packages -%}
 {%-     endif -%}
-{%-     if (target.mpackages is defined) and (target.mpackages|length) -%}
+{%-     if target.mpackages|length -%}
 {%-         if packages|length -%}
 {#-             Skip duplicating packages -#}
 {%-             set mpackages = target.mpackages|rejectattr('name', 'in', packages|map(attribute='name')) -%}
@@ -14,16 +14,37 @@
 {%-             set packages = packages + target.mpackages -%}
 {%-         endif -%}
 {%-     endif -%}
+{%-     set runs_packages = target.custom_runs|selectattr('cmake_packages')|map(attribute='cmake_packages')|sum -%}
+{%-     if runs_packages|length -%}
+{%-         if packages|length -%}
+{#-             Skip duplicating packages -#}
+{%-             set runs_packages = runs_packages|rejectattr('name', 'in', packages|map(attribute='name')) -%}
+{%-             if runs_packages|length -%}
+{%-                 set packages = packages + runs_packages -%}
+{%-             endif -%}
+{%-         else -%}
+{%-             set packages = packages + runs_packages -%}
+{%-         endif -%}
+{%-     endif -%}
 {%-     if packages|length -%}
-{%-         for package in packages %}
+{%-         set simple_packages = packages|rejectattr('components') -%}
+{%-         set complex_packages = packages|selectattr('components') -%}
+{%-         if simple_packages|length -%}
+{%-             for package_name in simple_packages|map(attribute='name')|unique %}
+find_package({{ package_name }} REQUIRED)
+{%              endfor -%}
+{%-         endif -%}
+{%-         if complex_packages|length -%}
+{%-             for package in complex_packages %}
 find_package({{ package.name }} REQUIRED
-{%-             if package.components|length %} COMPONENTS
-{%-                 for component in package.components %}
+{%-                 if package.components|length %} COMPONENTS
+{%-                     for component in package.components %}
   {{ component }}
-{%                  endfor -%}
-{%-             endif -%}
+{%                      endfor -%}
+{%-                 endif -%}
 )
-{%          endfor -%}
+{%              endfor -%}
+{%-         endif -%}
 {%-     endif -%}
 {%- endmacro -%}
 

build/export_generators/cmake/generator.toml

@@ -169,6 +169,17 @@ target_macroses-merge_args="flag"
 add_test_requirements="list"
 add_ytest_requirements="list"
 
+custom_runs="list"
+custom_runs-depends="list"
+custom_runs-env="list"
+custom_runs-command="list"
+custom_runs-outputs="list"
+custom_runs-cwd="str"
+custom_runs-cmake_packages="list"
+custom_runs-cmake_packages-ITEM="dict"
+custom_runs-cmake_packages-name="str"
+custom_runs-cmake_packages-components="list"
+
 [attrs.induced]
 # Induced packages
 packages="list"

build/export_generators/cmake/target_cmake_lists.jinja

@@ -30,7 +30,7 @@
 {%-             set only_headers_like_sources_library = false -%}
 {%-         endif -%}
 {%-     endif -%}
-{%-     if current_target.target_commands|selectattr('macro', 'eq', 'add_custom_command')|length -%}
+{%-     if current_target.custom_runs|length -%}
 {%-         set only_headers_like_sources_library = false -%}
 {%-     endif -%}
 {%- else -%}

build/export_generators/cmake/target_commands.jinja

@@ -1,11 +1,11 @@
-{%- if current_target.target_commands is defined -%}
+{%- if current_target.target_commands|length -%}
 {%-     for target_command in current_target.target_commands %}
 {{ target_command.macro }}(
 {%-         if target_command.args|length -%}
 {%-             for arg in target_command.args %}
   {{ arg }}
 {%-             endfor -%}
-{%-         endif %}
+{%-         endif -%}
 {%-         if target_command.args_escaped|length %}
   "
 {%-             for arg_escaped in target_command.args_escaped -%}
@@ -17,3 +17,67 @@
 )
 {%      endfor -%}
 {%- endif -%}
+
+{%- if current_target.custom_runs|length -%}
+{%-     for custom_run in current_target.custom_runs %}
+{%-         set first_arg = custom_run.command|first %}
+{%-         set tail_args = custom_run.command|slice(1, custom_run.command|length - 1) %}
+{%-         if (first_arg == "run_antlr") or (first_arg == "run_antlr4") %}
+{{ first_arg }}(
+{%-             if custom_run.outputs|length %}
+  OUTPUT
+{%-                 for output in custom_run.outputs %}
+  {{ output }}
+{%-                 endfor -%}
+{%-             endif %}
+{%-             if (custom_run.cwd is defined) and (custom_run.cwd != "") %}
+  WORKING_DIRECTORY
+  {{ custom_run.cwd }}
+{%-             endif -%}
+{%-             if tail_args|length %}
+  ANTLR_ARGS
+{%-                 for tail_arg in tail_args %}
+  {{ tail_arg }}
+{%-                 endfor -%}
+{%-             endif -%}
+)
+{%-         else %}
+add_custom_command(
+{%-             if custom_run.outputs|length %}
+  OUTPUT
+{%-                 for output in custom_run.outputs %}
+  {{ output }}
+{%-                 endfor -%}
+{%-             endif -%}
+{%-             if custom_run.depends|length %}
+  DEPENDS
+{%-                 for depend in custom_run.depends %}
+  {{ depend }}
+{%-                 endfor -%}
+{%-             endif -%}
+{%-             if (custom_run.cwd is defined ) and (custom_run.cwd != "") %}
+  WORKING_DIRECTORY
+  {{ custom_run.cwd }}
+{%-             endif -%}
+{%-             if custom_run.command|length %}
+  COMMAND
+{%-                 if custom_run.env|length %}
+  ${CMAKE_COMMAND}
+  -E
+  env
+{%-                     for env in custom_run.env %}
+  {{ env|replace('\\', '\\\\')|replace('"', '\\"')|replace(';', '\\;') }}
+{%-                         if not loop.last -%};{%- endif -%}
+{%-                     endfor -%}
+{%-                 endif -%}
+{%-                 for arg in custom_run.command -%}
+{%-                     if loop.first and arg == "python3" -%}
+{%-                         set arg = "Python3::Interpreter" -%}
+{%-                     endif %}
+  {{ arg }}
+{%-                 endfor -%}
+{%-             endif %}
+)
+{%          endif -%}
+{%-     endfor -%}
+{%- endif -%}

build/export_generators/gradle/generator.toml

@@ -73,6 +73,10 @@ custom_runs-env="list"
 custom_runs-command="list"
 custom_runs-outputs="list"
 custom_runs-cwd="str"
+custom_runs-cmake_packages="list"
+custom_runs-cmake_packages-ITEM="dict"
+custom_runs-cmake_packages-name="str"
+custom_runs-cmake_packages-components="list"
 
 javac-flags="list"
 

build/export_generators/ide-gradle/generator.toml

@@ -76,6 +76,10 @@ custom_runs-env="list"
 custom_runs-command="list"
 custom_runs-outputs="list"
 custom_runs-cwd="str"
+custom_runs-cmake_packages="list"
+custom_runs-cmake_packages-ITEM="dict"
+custom_runs-cmake_packages-name="str"
+custom_runs-cmake_packages-components="list"
 
 javac-flags="list"
 

build/scripts/link_exe.py

@@ -2,6 +2,7 @@
 import os
 import os.path
 import sys
+import json
 import subprocess
 import optparse
 import textwrap
@@ -321,7 +322,7 @@ def fix_blas_resolving(cmd):
     return cmd
 
 
-def parse_args():
+def parse_args(args):
     parser = optparse.OptionParser()
     parser.disable_interspersed_args()
     parser.add_option('--musl', action='store_true')
@@ -341,11 +342,23 @@ def parse_args():
     parser.add_option('--whole-archive-libs', action='append')
     parser.add_option('--exclude-libs', action='append')
     thinlto_cache.add_options(parser)
-    return parser.parse_args()
+    return parser.parse_args(args)
 
 
 if __name__ == '__main__':
-    opts, args = parse_args()
+    args = sys.argv[1:]
+    ib = args.index('--start-plugins')
+    ie = args.index('--end-plugins')
+    plugins = args[ib + 1:ie]
+    args = args[:ib] + args[ie + 1:]
+
+    for p in plugins:
+        res = subprocess.check_output([sys.executable, p] + args).decode().strip()
+
+        if res:
+            args = json.loads(res)
+
+    opts, args = parse_args(args)
     args = pcf.skip_markers(args)
 
     cmd = fix_blas_resolving(args)

build/scripts/link_lib.py

@@ -12,8 +12,9 @@ def __init__(self, args):
         self.llvm_ar_format = args[2]
         self.build_root = args[3]
         self.plugin = args[4]
-        self.output = args[5]
-        auto_input = args[6:]
+        self.enable_openssl3 = args[5]
+        self.output = args[6]
+        auto_input = args[7:]
 
         self.need_modify = False
         self.extra_args = []
@@ -162,5 +163,5 @@ def call():
     if exit_code != 0:
         raise Exception('{0} returned non-zero exit code {1}. Stop.'.format(' '.join(cmd), exit_code))
 
-    if os.path.basename(opts.output) in ['libcontrib-libs-openssl.a', 'liblibs-openssl-crypto.a']:
+    if opts.enable_openssl3 != 'no-openssl3' and os.path.basename(opts.output) in ['libcontrib-libs-openssl.a', 'liblibs-openssl-crypto.a']:
         rename_syms(os.path.dirname(opts.archiver) + '/', opts.output)