Merge branch 'rightlib' into mergelibs-241008-2121

Author: alexv-smirnov

build/conf/a.yaml

@@ -51,13 +51,6 @@ when ($IDE_MSVS != "yes") {
         /LIBPATH:\""${_MSVC_TC_VC_ROOT}/lib/$_MSVC_ARCH"\"
 }
 
-when ($WINDOWS_MANIFEST != "") {
-    _FLAGS_MANIFEST=/MANIFEST:EMBED /MANIFESTINPUT:${input:WINDOWS_MANIFEST}
-}
-otherwise {
-    _FLAGS_MANIFEST=
-}
-
 _FLAGS_COMMON =/NOLOGO /ERRORREPORT:PROMPT /SUBSYSTEM:CONSOLE /TLBID:1 $MSVC_DYNAMICBASE /NXCOMPAT
 _FLAGS_COMMON+=$_FLAGS_IGNORE
 _FLAGS_COMMON+=$_FLAGS_MACHINE
@@ -107,7 +100,7 @@ LINK_STDLIBS=\
     ws2_32.lib
 
 LDFLAGS_GLOBAL=
-LDFLAGS=$_FLAGS_MANIFEST
+LDFLAGS=
 OBJADDE=
 OBJADDE_LIB=
 OBJADDE_LIB_GLOBAL=

build/conf/linkers/msvc_linker.conf

@@ -183,18 +183,36 @@ _TS_FILES_COPY_CMD=
 ### TS_FILES(Files...)
 ###
 ### Adds files to output as is. Similar to FILES but works for TS build modules
+### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-files
 macro TS_FILES(Files...) {
+    # TODO: FBP-1795
     _TS_FILES($Files)
 }
 
 ### TS_FILES_GLOB(Glob...)
 ###
 ### Adds files to output by glob, e.g. TS_FILES_GLOB(**/*.css)
+### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-files-glob
 macro TS_FILES_GLOB(Glob...) {
     _GLOB(FILES_BY_GLOB ${Glob})
     TS_FILES(${FILES_BY_GLOB})
 }
 
+### @usage TS_LARGE_FILES(DESTINATION dest_dir Files...)
+###
+### Use large file ether from working copy or from remote storage via placeholder <File>.external
+### If <File> is present locally (and not a symlink!) it will be copied to build directory.
+### Otherwise macro will try to locate <File>.external, parse it retrieve ot during build phase.
+###
+### Then file will be copied to DESTINATION folder preserving file structure.
+### Copied file becomes output of TS_PACKAGE
+### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-large-files
+macro TS_LARGE_FILES(DESTINATION="~~required~~", FILES...) {
+    LARGE_FILES($FILES)
+    # TODO: FBP-1795
+    _TS_LARGE_FILES($DESTINATION $FILES)
+}
+
 @import "${CONF_ROOT}/conf/ts/node_modules.conf"
 @import "${CONF_ROOT}/conf/ts/ts_next.conf"
 @import "${CONF_ROOT}/conf/ts/ts_package.conf"

build/conf/ts/ts.conf

@@ -28,6 +28,8 @@
     "reset": "49",
 }
 
+REQUIRED_MISSING = "~~required~~"
+
 
 class ConsoleColors(dict):
     def __init__(self, color_codes):
@@ -884,6 +886,32 @@ def on_ts_files(unit, *files):
     unit.set(["_TS_FILES_COPY_CMD", " && ".join(new_cmds)])
 
 
+@_with_report_configure_error
+def on_ts_large_files(unit, destination: str, *files: list[str]):
+    if destination == REQUIRED_MISSING:
+        ymake.report_configure_error(
+            "Macro TS_LARGE_FILES() requires to use DESTINATION parameter.\n"
+            "   TS_LARGE_FILES(\n"
+            "       DESTINATION some_dir\n"
+            "       large/file1\n"
+            "       large/file2\n"
+            "   )\n"
+            "Docs: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-large-files."
+        )
+        return
+
+    # TODO: FBP-1795
+    # ${BINDIR} prefix for input is important to resove to result of LARGE_FILES and not to SOURCEDIR
+    new_cmds = [
+        '$COPY_CMD ${{input;context=TEXT:"${{BINDIR}}/{0}"}} ${{output;noauto:"{1}/{0}"}}'.format(f, destination)
+        for f in files
+    ]
+    all_cmds = unit.get("_TS_FILES_COPY_CMD")
+    if all_cmds:
+        new_cmds.insert(0, all_cmds)
+    unit.set(["_TS_FILES_COPY_CMD", " && ".join(new_cmds)])
+
+
 @_with_report_configure_error
 def on_ts_package_check_files(unit):
     ts_files = unit.get("_TS_FILES_COPY_CMD")

build/plugins/nots.py

@@ -5697,20 +5697,19 @@ when (!$TENSORRT_VERSION) {
 ANDROID_APK_TEST_ACTIVITY_VALUE=com.yandex.test.unittests/.RunTestsActivity
 
 # tag:windows-specific
-WINDOWS_MANIFEST=${ARCADIA_ROOT}/build/long-path.manifest
+WINDOWS_MANIFEST=
 macro WINDOWS_MANIFEST(Manifest) {
     SET(WINDOWS_MANIFEST $Manifest)
 }
 
-# tag:deprecated
 # https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation#enable-long-paths-in-windows-10-version-1607-and-later
 macro WINDOWS_LONG_PATH_MANIFEST() {
-    ENABLE(UNUSED_MACRO)
+    SET(WINDOWS_MANIFEST build/long-path.manifest)
 }
 
 # tag:windows-specific
-macro DISABLE_WINDOWS_MANIFEST() {
-    SET(WINDOWS_MANIFEST)
+when ($MSVC == "yes" && $WINDOWS_MANIFEST) {
+    LDFLAGS+=/MANIFEST:EMBED /MANIFESTINPUT:${input:WINDOWS_MANIFEST}
 }
 
 # tag:cpp-specific

build/ymake.core.conf

@@ -0,0 +1,7 @@
+Files in this directory come from Bob Ippolito's py2app.
+
+License: Any components of the py2app suite may be distributed under
+the MIT or PSF open source licenses.
+
+This is version 1.0, SVN revision 789, from 2006/01/25.
+The main repository is http://svn.red-bean.com/bob/macholib/trunk/macholib/

contrib/tools/python3/Lib/ctypes/macholib/README.ctypes

@@ -0,0 +1,46 @@
+
+
+About
+=====
+
+_decimal.c is a wrapper for the libmpdec library. libmpdec is a fast C
+library for correctly-rounded arbitrary precision decimal floating point
+arithmetic. It is a complete implementation of Mike Cowlishaw/IBM's
+General Decimal Arithmetic Specification.
+
+
+Build process for the module
+============================
+
+As usual, the build process for _decimal.so is driven by setup.py in the top
+level directory. setup.py autodetects the following build configurations:
+
+   1) x64         - 64-bit Python, x86_64 processor (AMD, Intel)
+
+   2) uint128     - 64-bit Python, compiler provides __uint128_t (gcc)
+
+   3) ansi64      - 64-bit Python, ANSI C
+
+   4) ppro        - 32-bit Python, x86 CPU, PentiumPro or later
+
+   5) ansi32      - 32-bit Python, ANSI C
+
+   6) ansi-legacy - 32-bit Python, compiler without uint64_t
+
+   7) universal   - Mac OS only (multi-arch)
+
+
+It is possible to override autodetection by exporting:
+
+   PYTHON_DECIMAL_WITH_MACHINE=value, where value is one of the above options.
+
+
+NOTE
+====
+
+decimal.so is not built from a static libmpdec.a since doing so led to
+failures on AIX (user report) and Windows (mixing static and dynamic CRTs
+causes locale problems and more).
+
+
+

contrib/tools/python3/Modules/_decimal/README.txt

@@ -0,0 +1,88 @@
+
+
+libmpdec
+========
+
+libmpdec is a fast C/C++ library for correctly-rounded arbitrary precision
+decimal floating point arithmetic. It is a complete implementation of
+Mike Cowlishaw/IBM's General Decimal Arithmetic Specification.
+
+
+Files required for the Python _decimal module
+=============================================
+
+  Core files for small and medium precision arithmetic
+  ----------------------------------------------------
+
+    basearith.{c,h}  ->  Core arithmetic in base 10**9 or 10**19.
+    bits.h           ->  Portable detection of least/most significant one-bit.
+    constants.{c,h}  ->  Constants that are used in multiple files.
+    context.c        ->  Context functions.
+    io.{c,h}         ->  Conversions between mpd_t and ASCII strings,
+                         mpd_t formatting (allows UTF-8 fill character).
+    mpalloc.{c,h}    ->  Allocation handlers with overflow detection
+                         and functions for switching between static
+                         and dynamic mpd_t.
+    mpdecimal.{c,h}  ->  All (quiet) functions of the specification.
+    typearith.h      ->  Fast primitives for double word multiplication,
+                         division etc.
+
+    Visual Studio only:
+    ~~~~~~~~~~~~~~~~~~~
+      vcdiv64.asm   ->  Double word division used in typearith.h. VS 2008 does
+                        not allow inline asm for x64. Also, it does not provide
+                        an intrinsic for double word division.
+
+  Files for bignum arithmetic:
+  ----------------------------
+
+    The following files implement the Fast Number Theoretic Transform
+    used for multiplying coefficients with more than 1024 words (see
+    mpdecimal.c: _mpd_fntmul()).
+
+      umodarith.h        ->  Fast low level routines for unsigned modular arithmetic.
+      numbertheory.{c,h} ->  Routines for setting up the Number Theoretic Transform.
+      difradix2.{c,h}    ->  Decimation in frequency transform, used as the
+                             "base case" by the following three files:
+
+        fnt.{c,h}        ->  Transform arrays up to 4096 words.
+        sixstep.{c,h}    ->  Transform larger arrays of length 2**n.
+        fourstep.{c,h}   ->  Transform larger arrays of length 3 * 2**n.
+
+      convolute.{c,h}    ->  Fast convolution using one of the three transform
+                             functions.
+      transpose.{c,h}    ->  Transpositions needed for the sixstep algorithm.
+      crt.{c,h}          ->  Chinese Remainder Theorem: use information from three
+                             transforms modulo three different primes to get the
+                             final result.
+
+
+Pointers to literature, proofs and more
+=======================================
+
+  literature/
+  -----------
+
+    REFERENCES.txt  ->  List of relevant papers.
+    bignum.txt      ->  Explanation of the Fast Number Theoretic Transform (FNT).
+    fnt.py          ->  Verify constants used in the FNT; Python demo for the
+                        O(N**2) discrete transform.
+
+    matrix-transform.txt -> Proof for the Matrix Fourier Transform used in
+                            fourstep.c.
+    six-step.txt         -> Show that the algorithm used in sixstep.c is
+                            a variant of the Matrix Fourier Transform.
+    mulmod-64.txt        -> Proof for the mulmod64 algorithm from
+                            umodarith.h.
+    mulmod-ppro.txt      -> Proof for the x87 FPU modular multiplication
+                            from umodarith.h.
+    umodarith.lisp       -> ACL2 proofs for many functions from umodarith.h.
+
+
+Library Author
+==============
+
+  Stefan Krah <skrah@bytereef.org>
+
+
+

contrib/tools/python3/Modules/_decimal/libmpdec/README.txt

@@ -0,0 +1,29 @@
+# Algorithm implementations used by the `hashlib` module.
+
+This code comes from the
+[HACL\*](https://github.com/hacl-star/hacl-star/) project.
+
+HACL\* is a cryptographic library that has been formally verified for memory
+safety, functional correctness, and secret independence.
+
+## Updating HACL*
+
+Use the `refresh.sh` script in this directory to pull in a new upstream code
+version.  The upstream git hash used for the most recent code pull is recorded
+in the script.  Modify the script as needed to bring in more if changes are
+needed based on upstream code refactoring.
+
+Never manually edit HACL\* files. Always add transformation shell code to the
+`refresh.sh` script to perform any necessary edits. If there are serious code
+changes needed, work with the upstream repository.
+
+## Local files
+
+1. `./include/python_hacl_namespaces.h`
+1. `./README.md`
+1. `./refresh.sh`
+
+## ACKS
+
+* Jonathan Protzenko aka [@msprotz on Github](https://github.com/msprotz)
+contributed our HACL\* based builtin code.

contrib/tools/python3/Modules/_hacl/README.md

@@ -0,0 +1,56 @@
+Fuzz Tests for CPython
+======================
+
+These fuzz tests are designed to be included in Google's `oss-fuzz`_ project.
+
+oss-fuzz works against a library exposing a function of the form
+``int LLVMFuzzerTestOneInput(const uint8_t* data, size_t length)``. We provide
+that library (``fuzzer.c``), and include a ``_fuzz`` module for testing with
+some toy values -- no fuzzing occurs in Python's test suite.
+
+oss-fuzz will regularly pull from CPython, discover all the tests in
+``fuzz_tests.txt``, and run them -- so adding a new test here means it will
+automatically be run in oss-fuzz, while also being smoke-tested as part of
+CPython's test suite.
+
+Adding a new fuzz test
+----------------------
+
+Add the test name on a new line in ``fuzz_tests.txt``.
+
+In ``fuzzer.c``, add a function to be run::
+
+    int $test_name (const char* data, size_t size) {
+        ...
+        return 0;
+    }
+
+
+And invoke it from ``LLVMFuzzerTestOneInput``::
+
+    #if _Py_FUZZ_YES(fuzz_builtin_float)
+        rv |= _run_fuzz(data, size, fuzz_builtin_float);
+    #endif
+
+``LLVMFuzzerTestOneInput`` will run in oss-fuzz, with each test in
+``fuzz_tests.txt`` run separately.
+
+Seed data (corpus) for the test can be provided in a subfolder called
+``<test_name>_corpus`` such as ``fuzz_json_loads_corpus``. A wide variety
+of good input samples allows the fuzzer to more easily explore a diverse
+set of paths and provides a better base to find buggy input from.
+
+Dictionaries of tokens (see oss-fuzz documentation for more details) can
+be placed in the ``dictionaries`` folder with the name of the test.
+For example, ``dictionaries/fuzz_json_loads.dict`` contains JSON tokens
+to guide the fuzzer.
+
+What makes a good fuzz test
+---------------------------
+
+Libraries written in C that might handle untrusted data are worthwhile. The
+more complex the logic (e.g. parsing), the more likely this is to be a useful
+fuzz test. See the existing examples for reference, and refer to the
+`oss-fuzz`_ docs.
+
+.. _oss-fuzz: https://github.com/google/oss-fuzz