Merge branch 'rightlib' into merge-libs-250227-0050

Author: alexv-smirnov

build/conf/go.conf

@@ -95,7 +95,6 @@ _GO_CGO1_WRAPPER_FLAGS=--build-prefix=/-B --source-prefix=/-S
 _GO_LINK_EXE_EXT_CMD=
 
 # tag:go-specific
-GO_WITH_MUSL=
 
 # tag:go-specific
 GO_TOOLS_ROOT=${GO_TOOLS_RESOURCE_GLOBAL}
@@ -219,7 +218,7 @@ macro _GO_COMPILE_CGO1(NAME, FLAGS[], FILES...) {
 
 # tag:go-specific
 macro _GO_COMPILE_CGO2(NAME, C_FILES[], S_FILES[], OBJ_FILES[], FILES...) {
-    .CMD=${hide:_CGO_FAKEID} $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} $CGO_CFLAGS_VALUE ${input;tobindir:"_cgo_main.c"} -c -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} && $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} $LDFLAGS $LDFLAGS_GLOBAL $CGO2_LDFLAGS_VALUE ${hide;input:"_cgo_export.h"} ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} ${input;suf=${OBJECT_SUF}:"_cgo_export.c"} ${input;nopath;noext;suf=.cgo2.c${OBJECT_SUF}:FILES} ${input;suf=${OBJECT_SUF}:C_FILES} ${input;suf=.o:S_FILES} ${input:OBJ_FILES} $CGO_LDFLAGS_VALUE && ${GO_TOOLS_ROOT}/pkg/tool/$_GO_TC_PATH/cgo -dynpackage $NAME -dynimport ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} -dynout ${output:"_cgo_import.go"} -dynlinker $GO_CGO2_FLAGS_VALUE $GO_TOOLCHAIN_ENV ${hide;kv:"p go"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"}
+    .CMD=${hide:_CGO_FAKEID} $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} $CGO_CFLAGS_VALUE ${input;tobindir:"_cgo_main.c"} -c -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} && $YMAKE_PYTHON ${input:"build/scripts/link_o.py"} $C_COMPILER $C_FLAGS_PLATFORM ${pre=-I:_C__INCLUDE} -o ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} $LDFLAGS $LDFLAGS_GLOBAL $CGO2_LDFLAGS_VALUE ${hide;input:"_cgo_export.h"} ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_main.c"} ${input;suf=${OBJECT_SUF}:"_cgo_export.c"} ${input;nopath;noext;suf=.cgo2.c${OBJECT_SUF}:FILES} ${input;suf=${OBJECT_SUF}:C_FILES} ${input;suf=.o:S_FILES} ${input:OBJ_FILES} $CGO_LDFLAGS_VALUE && ${GO_TOOLS_ROOT}/pkg/tool/$_GO_TC_PATH/cgo -dynpackage $NAME -dynimport ${tmp;noauto;suf=${OBJECT_SUF}:"_cgo_"} -dynout ${output:"_cgo_import.go"} -dynlinker $GO_CGO2_FLAGS_VALUE $GO_TOOLCHAIN_ENV ${hide;kv:"p go"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"}
 
     _USE_LINKER()
 }
@@ -233,12 +232,12 @@ macro _GO_LINK_LIB_IMPL(CGO_FILES[], EXTRA_INPUTS[], EXTRA_TEXT_INPUTS[], GO_FIL
 
 # tag:go-specific
 macro _GO_LINK_EXE_IMPL(CGO_FILES[], EXTRA_INPUTS[], EXTRA_TEXT_INPUTS[], GO_FILES...) {
-    .CMD=${hide:_GO_FAKEID} $GO_TOOL ${hide;input:EXTRA_INPUTS} ${hide;context=TEXT;input:EXTRA_TEXT_INPUTS} --ya-start-command-file $_GO_TOOL_COMMON_FLAGS ++vcs $VCS_GO $GO_WITH_MUSL $GO_EXTLD ++peers ${rootrel;tags_in=local,GO|local,GO_PROTO|local,GO_PROTO_FROM_SCHEMA|local,GO_FBS:PEERS} ++non-local-peers ${rootrel;tags_in=GO|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS|;tags_out=local:PEERS} ++cgo-peers ${VCS_C_OBJ_RR} ${_EXTRA_OBJS} ${rootrel;tags_out=GO|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS|RESOURCE_LIB:PEERS} --ya-end-command-file ${hide;kv:"p LD"} ${hide;kv:"pc light-red"} ${hide;kv:"show_out"} && $_GO_LINK_EXE_EXT_CMD
+    .CMD=${hide:_GO_FAKEID} $GO_TOOL ${hide;input:EXTRA_INPUTS} ${hide;context=TEXT;input:EXTRA_TEXT_INPUTS} --ya-start-command-file $_GO_TOOL_COMMON_FLAGS ++ld_plugins ${ext=.pyplugin:SRCS_GLOBAL} ++vcs $VCS_GO $GO_EXTLD ++peers ${rootrel;tags_in=local,GO|local,GO_PROTO|local,GO_PROTO_FROM_SCHEMA|local,GO_FBS:PEERS} ++non-local-peers ${rootrel;tags_in=GO|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS|;tags_out=local:PEERS} ++cgo-peers ${VCS_C_OBJ_RR} ${_EXTRA_OBJS} ${rootrel;tags_out=GO|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS|RESOURCE_LIB:PEERS} --ya-end-command-file ${hide;kv:"p LD"} ${hide;kv:"pc light-red"} ${hide;kv:"show_out"} && $_GO_LINK_EXE_EXT_CMD
 }
 
 # tag:go-specific
 macro _GO_LINK_TEST_IMPL(CGO_FILES[], EXTRA_INPUTS[], EXTRA_TEXT_INPUTS[], GO_TEST_FILES[], GO_XTEST_FILES[], GO_FILES...) {
-    .CMD=${hide:_GO_FAKEID} $GO_TOOL ${hide;input:EXTRA_INPUTS} ${hide;context=TEXT;input:EXTRA_TEXT_INPUTS} --ya-start-command-file $_GO_TOOL_COMMON_FLAGS ++vcs $VCS_GO $GO_WITH_MUSL $GO_EXTLD ++test-miner $GO_TEST_MINER ++test-import-path $GO_TEST_IMPORT_PATH ++peers ${rootrel;tags_in=local,GO|local,GO_PROGRAM|local,GO_PROTO|local,GO_PROTO_FROM_SCHEMA|local,GO_FBS:PEERS} ++non-local-peers ${rootrel;tags_in=GO|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS;tags_out=local:PEERS} ++cgo-peers ${VCS_C_OBJ_RR} ${rootrel;tags_out=GO|GO_PROGRAM|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS|RESOURCE_LIB:PEERS} ++test_srcs ${input:GO_TEST_FILES} ++xtest_srcs ${input:GO_XTEST_FILES} ++cover_info $GO_COVER_INFO_VALUE ++skip-tests $_GO_SKIP_TEST_VALUE --ya-end-command-file ${hide;kv:"p GO"} ${hide;kv:"pc light-red"} ${hide;kv:"show_out"}
+    .CMD=${hide:_GO_FAKEID} $GO_TOOL ${hide;input:EXTRA_INPUTS} ${hide;context=TEXT;input:EXTRA_TEXT_INPUTS} --ya-start-command-file $_GO_TOOL_COMMON_FLAGS ++ld_plugins ${ext=.pyplugin:SRCS_GLOBAL} ++vcs $VCS_GO $GO_EXTLD ++test-miner $GO_TEST_MINER ++test-import-path $GO_TEST_IMPORT_PATH ++peers ${rootrel;tags_in=local,GO|local,GO_PROGRAM|local,GO_PROTO|local,GO_PROTO_FROM_SCHEMA|local,GO_FBS:PEERS} ++non-local-peers ${rootrel;tags_in=GO|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS;tags_out=local:PEERS} ++cgo-peers ${VCS_C_OBJ_RR} ${rootrel;tags_out=GO|GO_PROGRAM|GO_PROTO|GO_PROTO_FROM_SCHEMA|GO_FBS|RESOURCE_LIB:PEERS} ++test_srcs ${input:GO_TEST_FILES} ++xtest_srcs ${input:GO_XTEST_FILES} ++cover_info $GO_COVER_INFO_VALUE ++skip-tests $_GO_SKIP_TEST_VALUE --ya-end-command-file ${hide;kv:"p GO"} ${hide;kv:"pc light-red"} ${hide;kv:"show_out"}
 }
 
 # tag:go-specific
@@ -638,7 +637,7 @@ otherwise {
 ###
 ### The base module of all golang modules. Defines common properties, dependencies and rules for go build.
 module _GO_BASE_UNIT: _BASE_UNIT {
-    .EXTS=.go .gosrc .o .obj .sbom .a .symabis .mf
+    .EXTS=.go .gosrc .o .obj .sbom .a .symabis .mf .pyplugin
     .CMD=$TOUCH_UNIT
     .STRUCT_CMD=yes
     .NODE_TYPE=Bundle
@@ -870,10 +869,6 @@ module GO_PROGRAM: _GO_BASE_UNIT {
         PEERDIR += ${GOSTD}/runtime/race
     }
 
-    when ($MUSL == "yes") {
-        GO_WITH_MUSL = ++musl
-    }
-
     when ($HARDENING == "yes") {
         _GO_BUILDMODE=++buildmode=pie
     }

build/conf/licenses.json

@@ -45,6 +45,7 @@
             "GCC-exception-3.1",
             "GDAL",
             "GFDL-1.1-or-later",
+            "GFDL-1.3-only",
             "GFDL-1.3-or-later",
             "Go-License",
             "Google-Patent-License-Webm",

build/conf/ts/node_modules.conf

@@ -36,7 +36,7 @@ _PREPARE_NO_DEPS_CMD=$TOUCH_UNIT \
 
 module _PREPARE_DEPS_BASE: _BARE_UNIT {
     .CMD=_PREPARE_DEPS_CMD
-    .IGNORED=SRCS FILES TS_FILES COPY_FILE RUN_JAVASCRIPT_AFTER_BUILD
+    .IGNORED=SRCS FILES TS_FILES COPY_FILE RUN_JAVASCRIPT_AFTER_BUILD RUN_JAVASCRIPT
     # Propagates peers to related modules
     .PEERDIR_POLICY=as_build_from
     .NODE_TYPE=Bundle

build/conf/ts/ts.conf

@@ -11,7 +11,7 @@ TS_CONFIG_USE_OUTDIR=
 TS_USE_PREBUILT_NOTS_TOOL=yes
 NOTS_TOOL=${tool:"devtools/frontend_build_platform/nots/builder"}
 
-# Additional commands that module can add (with `&&` as delimiter), if set those will be executed 
+# Additional commands that module can add (with `&&` as delimiter), if set those will be executed
 # in build phase just before building the module.
 _TS_PROJECT_SETUP_CMD=
 
@@ -56,6 +56,7 @@ when($AUTOCHECK == "yes") {
 
 _RUN_JAVASCRIPT_AFTER_BUILD_ARGS=
 _RUN_JAVASCRIPT_AFTER_BUILD_INPUTS=
+_RUN_JAVASCRIPT_AFTER_BUILD_TS_PACKAGE_EXTRA=
 
 # Arguments for builders' commands, passed after the command
 # TODO FBP-1978 (remove output.tar output)
@@ -185,7 +186,7 @@ macro _AS_HIDDEN_INPUTS(IN{input}[]) {
   .CMD=${hide;context=TEXT;input:IN}
 }
 
-# Input/output directives for files set by use with 
+# Input/output directives for files set by use with
 # SRCS / TS_LARGE_FILES / TS_FILES / TS_FILES_GLOB
 _TS_FILES_INOUTS=
 
@@ -231,15 +232,25 @@ macro RUN_JAVASCRIPT_AFTER_BUILD(JS_SCRIPT, OUTDIR="", IN{input}[], ARGS...) {
   # --after-build-args uses `arg=val` format because args can start with --, so value will be parsed as another key.
   # With `arg=val` format parser knows that after `=` is a value of the --after-build-args.
   # For --after-build-outdir it is used to handle possible empty string in OUTDIR
-  SET(_RUN_JAVASCRIPT_AFTER_BUILD_ARGS --after-build-js $JS_SCRIPT  "--after-build-args=${join="<~~~>":ARGS}" "--after-build-outdir=$OUTDIR")
+  SET(_RUN_JAVASCRIPT_AFTER_BUILD_ARGS --with-after-build --after-build-js $JS_SCRIPT  "--after-build-args=${join="<~~~>":ARGS}" "--after-build-outdir=$OUTDIR")
   SET(_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS ${IN})
+  # here we need to escape $, otherwise it will be expanded in-place and will end up with empty value
+  SET(_RUN_JAVASCRIPT_AFTER_BUILD_TS_PACKAGE_EXTRA \${hide;output:"output.tar"} \${hide;output:"output.tar.uuid"})
   _RUN_JAVASCRIPT_AFTER_BUILD_ADD_JS_SCRIPT_AS_INPUT($JS_SCRIPT)
 }
 
 macro _RENDER_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS(IN{input}[]) {
   .CMD=${hide;input:IN}
 }
 
+### @usage RUN_JAVASCRIPT(script_path [args...] [IN inputs...] [OUTDIR outdir])
+###
+### Run JS script after build of TS_* module.
+### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/macros#run-javascript-after-build
+macro RUN_JAVASCRIPT(JS_SCRIPT, OUTDIR="", IN{input}[], ARGS...) {
+  MESSAGE(FATAL_ERROR RUN_JAVASCRIPT macro exists only in scope of TS_PACKAGE. For other TS_* modules use RUN_JAVASCRIPT_AFTER_BUILD)
+}
+
 
 @import "${CONF_ROOT}/conf/ts/node_modules.conf"
 @import "${CONF_ROOT}/conf/ts/ts_next.conf"

build/conf/ts/ts_package.conf

@@ -1,7 +1,12 @@
-TS_PACK=$TOUCH_UNIT \
-    && $_TS_PROJECT_SETUP_CMD \
-    && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-package $_NODE_MODULES_INOUTS \
-    $_TS_FILES_INOUTS ${hide;kv:"p TS_PKG"} ${hide;kv:"pc magenta"}
+TS_PACK=$_TS_PROJECT_SETUP_CMD \
+    && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-package \
+    --output-file ${output:TARGET} \
+    $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS \
+    ${hide;kv:"p TS_PKG"} ${hide;kv:"pc magenta"} \
+    $_RUN_JAVASCRIPT_AFTER_BUILD_TS_PACKAGE_EXTRA \
+    $_RUN_JAVASCRIPT_AFTER_BUILD_ARGS \
+    $_RENDER_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS($_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS)
+
 
 ### # internal
 macro _TS_PACKAGE_EPILOGUE() {
@@ -31,9 +36,7 @@ multimodule TS_PACKAGE {
         .CMD=TS_PACK
         .EPILOGUE=_TS_PACKAGE_EPILOGUE
         .ALLOWED=TS_FILES TS_FILES_GLOB
-        # TODO: FBP-2029
-        .RESTRICTED=RUN_JAVASCRIPT_AFTER_BUILD
-        .ALIASES=FILES=TS_FILES SRCS=TS_FILES
+        .ALIASES=FILES=TS_FILES SRCS=TS_FILES RUN_JAVASCRIPT=RUN_JAVASCRIPT_AFTER_BUILD
         .PEERDIRSELF=TS_PREPARE_DEPS
 
         # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case)
@@ -46,9 +49,5 @@ multimodule TS_PACKAGE {
     }
 
     module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
-        # TODO: FBP-2029
-        # You need to restricts macro in all submodules of multimodule
-        # otherwise .RESTRICTED will behave as .IGNORED
-        .RESTRICTED=RUN_JAVASCRIPT_AFTER_BUILD
     }
 }

build/plugins/nots.py

@@ -937,7 +937,6 @@ def __on_ts_files(unit: NotsUnitType, files_in: list[str], files_out: list[str])
     new_items = _build_cmd_input_paths(paths=files_in, hide=True, disable_include_processor=True)
     new_items += _build_cmd_output_paths(paths=files_out, hide=True)
     __set_append(unit, "_TS_FILES_INOUTS", new_items)
-    logger.print_vars("_TS_FILES_INOUTS")
 
 
 @_with_report_configure_error

build/scripts/go_tool.py

@@ -525,12 +525,7 @@ def do_link_exe(args):
     cmd.append('-extld={}'.format(args.extld))
 
     if args.extldflags is not None:
-        filter_musl = bool
-        if args.musl:
-            cmd.append('-linkmode=external')
-            extldflags.append('-static')
-            filter_musl = lambda x: x not in ('-lc', '-ldl', '-lm', '-lpthread', '-lrt')
-        extldflags += [x for x in args.extldflags if filter_musl(x)]
+        extldflags.extend(args.extldflags)
     cgo_peers = []
     if args.cgo_peers is not None and len(args.cgo_peers) > 0:
         is_group = args.targ_os == 'linux'
@@ -545,6 +540,10 @@ def do_link_exe(args):
     except ValueError:
         extldflags.extend(cgo_peers)
     if len(extldflags) > 0:
+        for p in args.ld_plugins:
+            res = subprocess.check_output([sys.executable, p, sys.argv[0]] + extldflags, cwd=args.build_root).decode().strip()
+            if res:
+                extldflags = json.loads(res)[1:]
         cmd.append('-extldflags={}'.format(' '.join(extldflags)))
     cmd.append(compile_args.output)
     call(cmd, args.build_root)
@@ -850,6 +849,7 @@ def find_ydx_file_name(name, flags):
     parser.add_argument('++test_srcs', nargs='*')
     parser.add_argument('++xtest_srcs', nargs='*')
     parser.add_argument('++cover_info', nargs='*')
+    parser.add_argument('++ld_plugins', nargs='*')
     parser.add_argument('++output', nargs='?', default=None)
     parser.add_argument('++source-root', default=None)
     parser.add_argument('++build-root', required=True)
@@ -881,7 +881,6 @@ def find_ydx_file_name(name, flags):
     parser.add_argument('++vet-flags', nargs='*', default=None)
     parser.add_argument('++vet-info-ext', default=vet_info_ext)
     parser.add_argument('++vet-report-ext', default=vet_report_ext)
-    parser.add_argument('++musl', action='store_true')
     parser.add_argument('++skip-tests', nargs='*', default=None)
     parser.add_argument('++ydx-file', default='')
     parser.add_argument('++debug-root-map', default=None)

build/scripts/link_dyn_lib.py

@@ -165,9 +165,6 @@ def parse_args(args):
     parser.add_option('--fix-elf')
     parser.add_option('--linker-output')
     parser.add_option('--dynamic-cuda', action='store_true')
-    parser.add_option('--cuda-architectures',
-                      help='List of supported CUDA architectures, separated by ":" (e.g. "sm_52:compute_70:lto_90a"')
-    parser.add_option('--nvprune-exe')
     parser.add_option('--objcopy-exe')
     parser.add_option('--whole-archive-peers', action='append')
     parser.add_option('--whole-archive-libs', action='append')

build/scripts/link_exe.py

@@ -47,9 +47,6 @@ def parse_args(args):
     parser.add_option('--build-root')
     parser.add_option('--clang-ver')
     parser.add_option('--dynamic-cuda', action='store_true')
-    parser.add_option('--cuda-architectures',
-                      help='List of supported CUDA architectures, separated by ":" (e.g. "sm_52:compute_70:lto_90a"')
-    parser.add_option('--nvprune-exe')
     parser.add_option('--objcopy-exe')
     parser.add_option('--arch')
     parser.add_option('--linker-output')

build/scripts/link_o.py

@@ -0,0 +1,29 @@
+import subprocess
+import sys
+import os
+
+
+def flt(args):
+    for a in args:
+        if a == '-o':
+            yield '-Wl,-r'
+            yield '-fuse-ld=lld'
+            yield '-nodefaultlibs'
+            yield '-nostartfiles'
+            yield '-Wl,-no-pie'
+            yield '-o'
+        elif a.endswith('.o'):
+            yield a
+        elif '--ld' in a:
+            yield a
+        elif '--target' in a:
+            yield a
+
+
+if '-apple-macos' in str(sys.argv):
+    cmd = sys.argv[1:]
+else:
+    cmd = [sys.argv[1]] + list(flt(sys.argv[2:]))
+
+
+subprocess.check_call(cmd)