#896: Add template-python3.12 flavor

exaudfclient/base/BUILD

@@ -2,7 +2,8 @@
 package(default_visibility = ["//visibility:public"])
 exports_files(["filter_swig_code.py", "build_integrated.py",
 	       "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh",
-	       "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh"])
+	       "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh",
+	       "replace_swig_import_helper.py"])
 
 load("//:variables.bzl", "VM_ENABLED_DEFINES")
 

exaudfclient/base/exaudflib/swig/swig_common.h

@@ -1,6 +1,7 @@
 #ifndef SWIG_COMMON_H
 #define SWIG_COMMON_H
 
+#include <cstdint>
 #include <vector>
 #include <string>
 #include <cstdint>

exaudfclient/base/python/exascript_python_preset_core.py

@@ -6,14 +6,14 @@
 from exascript_python import *
 import decimal
 import datetime
-import imp
+import types
 
 class ExaUDFError(Exception):
     pass 
 
 def clean_stacktrace_line(line):
     import re
-    match = re.match("""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line)
+    match = re.match(r"""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line)
     if match is not None:
         filename=match.group(1)
         lineno=match.group(2)
@@ -126,7 +126,7 @@ def import_script(self, script):
             modobj = self.__modules[str(code)]
         else:
             print("%%% new code", modname, repr(code), code in self.__modules)
-            modobj = imp.new_module(modname)
+            modobj = types.ModuleType(modname)
             modobj.__file__ = "<%s>" % modname
             modobj.__dict__['exa'] = self
             self.__modules[str(code)] = modobj

exaudfclient/base/python/python3/BUILD

@@ -8,9 +8,12 @@ genrule(
         cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib
         cd build_exascript_python_tmp_cc
         swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i
+        cd ..
+        python3 "$(location //base:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)"
         """,
     outs = ["exascript_python_tmp.cc", "exascript_python.py"],
-    srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"]
+    srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"],
+    tools = ["//base:replace_swig_import_helper.py"],
 )
 
 genrule(

exaudfclient/base/replace_swig_import_helper.py

@@ -0,0 +1,43 @@
+import sys
+from pathlib import Path
+
+old_function = """
+    def swig_import_helper():
+        from os.path import dirname
+        import imp
+        fp = None
+        try:
+            fp, pathname, description = imp.find_module('_exascript_python', [dirname(__file__)])
+        except ImportError:
+            import _exascript_python
+            return _exascript_python
+        if fp is not None:
+            try:
+                _mod = imp.load_module('_exascript_python', fp, pathname, description)
+            finally:
+                fp.close()
+            return _mod
+    _exascript_python = swig_import_helper()
+    del swig_import_helper
+"""
+
+new_function = """
+    import _exascript_python
+"""
+
+def replace_swig_import_helper(target, source):
+    source_code = Path(source).read_text()
+
+    assert old_function in source_code
+
+    new_source_code = source_code.replace(old_function, new_function)
+
+    with open(target, 'w') as f:
+        f.write(new_source_code)
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print('Usage: replace_swig_import_helper.py target source')
+        sys.exit(1)
+    print(f"replacing function swig_import_helper() using source={sys.argv[2]} and target={sys.argv[1]}")
+    replace_swig_import_helper(sys.argv[1], sys.argv[2])

ext/scripts/install_scripts/install_via_pip.pl

@@ -40,6 +40,7 @@ =head1 SYNOPSIS
 my $use_deprecated_legacy_resolver = 0;
 my $ancestor_pip_package_root_path = '';
 my $install_build_tools_ephemerally = 0;
+my $pip_needs_break_system_packages = 0;
 GetOptions (
             "help" => \$help,
             "dry-run" => \$dry_run,
@@ -51,7 +52,8 @@ =head1 SYNOPSIS
             "use-deprecated-legacy-resolver" => \$use_deprecated_legacy_resolver,
             "python-binary=s" => \$python_binary,
             "ancestor-pip-package-root-path=s" => \$ancestor_pip_package_root_path,
-            "install-build-tools-ephemerally" => \$install_build_tools_ephemerally
+            "install-build-tools-ephemerally" => \$install_build_tools_ephemerally,
+            "pip-needs-break-system-packages" => \$pip_needs_break_system_packages
 
           ) or package_mgmt_utils::print_usage_and_abort(__FILE__,"Error in command line arguments",2);
 package_mgmt_utils::print_usage_and_abort(__FILE__,"",0) if $help;
@@ -79,6 +81,10 @@ =head1 SYNOPSIS
   push @pip_parameters, "--use-deprecated=legacy-resolver";
 }
 
+if($pip_needs_break_system_packages){
+  push @pip_parameters, "--break-system-packages";
+}
+
 my $pip_parameters_str = join( ' ', @pip_parameters);
 my $element_separator = '\\|';
 my $combining_template = "$python_binary -m pip install $pip_parameters_str --no-cache-dir <<<<0>>>>";

ext/scripts/tests/install_scripts/run_pip_tests.sh

@@ -111,5 +111,10 @@ TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file te
 assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install  --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'"
 echo
 
+echo ./install_via_pip.pl with pip-needs-break-system-packages
+TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --pip-needs-break-system-packages --python-binary python3 "$DRY_RUN_OPTION")
+assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly' 'requests'"
+echo
+
 check_for_failed_tests
 echo "All pip tests passed"

flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile

@@ -13,7 +13,7 @@ RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfc
 
 RUN addgroup --gid 1000 exasolution
 RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone"
-RUN addgroup --gid 500 exausers 
+RUN addgroup --gid 500 exausers
 RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone"
 
 ENV LANG en_US.UTF-8

flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md

@@ -0,0 +1,14 @@
+# Details for template-Exasol-all-python-3.12
+
+## Packages
+
+**Remarks:** The packages are grouped by there main usage and their type.
+
+- [UDFclient dependencies](flavor_base/udfclient_deps/packages/apt_get_packages)
+- [Language dependencies](flavor_base/language_deps/packages/apt_get_packages)
+- Flavor packages
+  - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages)
+  - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages)
+- Customization
+  - [Ubuntu packages](flavor_customization/packages/apt_get_packages)
+  - [Python3 pip packages](flavor_customization/packages/python3_pip_packages)

flavors/template-Exasol-all-python-3.12/ci.json

@@ -0,0 +1,32 @@
+{
+  "build_runner": "ubuntu-24.04",
+  "test_config": {
+    "default_test_runner": "ubuntu-24.04",
+    "test_sets": [
+      {
+        "name": "python",
+        "folders": ["python3/all"],
+        "goal": "release",
+        "generic_language_tests": []
+      },
+      {
+        "name": "pandas",
+        "folders": ["pandas/all", "pandas/pandas2"],
+        "goal": "release",
+        "generic_language_tests": []
+      },
+      {
+        "name": "linker_namespace",
+        "folders": ["linker_namespace_sanity"],
+        "goal": "base_test_build_run",
+        "generic_language_tests": []
+      },
+      {
+        "name": "generic",
+        "folders": [],
+        "goal": "release",
+        "generic_language_tests": ["python3"]
+      }
+    ]
+  }
+}

flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile

@@ -0,0 +1,41 @@
+FROM {{language_deps}}
+
+RUN mkdir /conf /buckets
+
+COPY --from={{base_test_deps}} /usr /usr
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{base_test_deps}} /lib /lib
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{base_test_deps}} /bin /bin
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{base_test_deps}} /opt /opt
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{base_test_deps}} /etc /etc
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{base_test_deps}} /env /env
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+
+RUN ldconfig
+
+RUN mkdir /exaudfclient /exaudf
+COPY /exaudfclient/ /exaudfclient/
+
+WORKDIR /exaudfclient/
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"]
+RUN cp -r -L bazel-bin/* /exaudf
+
+WORKDIR /exaudfclient/base
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+RUN ./test_udfclient.sh /exaudf/exaudfclient_static
+
+WORKDIR /
+RUN mkdir /exasol_emulator
+COPY emulator/ /exasol_emulator
+COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator
+RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=.

flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{build_deps}}
+
+RUN mkdir -p /build_info/packages
+COPY base_test_deps/packages /build_info/packages/base_test_deps
+
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions

flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages

@@ -0,0 +1,5 @@
+gdb|15.0.50.20240403-0ubuntu1
+valgrind|1:3.22.0-0ubuntu3
+binutils|2.42-4ubuntu2.5
+patchelf|0.18.0-1.1build1
+strace|6.8-0ubuntu2

flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:24.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV ARCHIVE_UBUNTU_PREFIX=""
+RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list
+
+COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
+COPY scripts /scripts
+
+RUN mkdir -p /build_info/packages
+COPY build_deps/packages /build_info/packages/build_deps
+
+ENV BAZEL_PACKAGE_VERSION="7.2.1"
+ENV BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb"
+ENV BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE"
+
+RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/build_deps/apt_get_packages --with-versions
+
+RUN apt-get -y update && \
+    curl -L --output "$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" && \
+    apt-get install -y "./$BAZEL_PACKAGE_FILE" && \
+    rm "$BAZEL_PACKAGE_FILE" && \
+    apt-get -y clean && \
+    apt-get -y autoremove
+
+RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \
+    tar zxf swig-2.0.4.tar.gz && \
+    (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \
+    rm -rf swig-2.0.4 swig-2.0.4.tar.gz
+
+RUN locale-gen en_US.UTF-8 && \
+    update-locale LC_ALL=en_US.UTF-8 && \
+    ldconfig
+
+RUN touch /env && \
+    echo "export PROTOBUF_BIN=/usr/bin/protoc" >> /env && \
+    echo "export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64" >> /env

flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages

@@ -0,0 +1,9 @@
+coreutils|9.4-3ubuntu6
+tar|1.35+dfsg-3build1
+curl|8.5.0-2ubuntu10.6
+openjdk-11-jdk|11.0.27+6~us1-0ubuntu1~24.04
+build-essential|12.10ubuntu1
+libpcre3-dev|2:8.39-15build1
+protobuf-compiler|3.21.12-8.2ubuntu0.1
+chrpath|0.16-2build1
+locales|2.39-0ubuntu8.4

flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile

@@ -0,0 +1,43 @@
+FROM {{language_deps}}
+
+COPY --from={{build_deps}} /usr /usr
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{build_deps}} /lib /lib
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{build_deps}} /bin /bin
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{build_deps}} /opt /opt
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{build_deps}} /etc /etc
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+COPY --from={{build_deps}} /env /env
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+
+RUN ldconfig
+
+RUN mkdir /exaudfclient /exaudf
+COPY exaudfclient/ /exaudfclient/
+
+WORKDIR /exaudfclient/
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"]
+RUN cp -r -L bazel-bin/* /exaudf
+RUN rm -r /root/.cache/bazel #Delete bazel cache as it contains java rules with special characters which will cause a problem for Bucketfs
+
+WORKDIR /exaudfclient/base
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+
+WORKDIR /exaudf
+
+RUN rm -r /exaudfclient
+
+COPY --from={{build_deps}} /build_info /build_info
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+
+RUN mkdir /conf /buckets