diff --git a/exaudfclient/base/BUILD b/exaudfclient/base/BUILD index 357c34da9..9d6728354 100644 --- a/exaudfclient/base/BUILD +++ b/exaudfclient/base/BUILD @@ -2,7 +2,8 @@ package(default_visibility = ["//visibility:public"]) exports_files(["filter_swig_code.py", "build_integrated.py", "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh", - "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh"]) + "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh", + "replace_swig_import_helper.py"]) load("//:variables.bzl", "VM_ENABLED_DEFINES") diff --git a/exaudfclient/base/exaudflib/swig/swig_common.h b/exaudfclient/base/exaudflib/swig/swig_common.h index b4e463fed..8080e5916 100644 --- a/exaudfclient/base/exaudflib/swig/swig_common.h +++ b/exaudfclient/base/exaudflib/swig/swig_common.h @@ -1,6 +1,7 @@ #ifndef SWIG_COMMON_H #define SWIG_COMMON_H +#include #include #include #include diff --git a/exaudfclient/base/python/exascript_python_preset_core.py b/exaudfclient/base/python/exascript_python_preset_core.py index 550b31949..b2dcc7766 100644 --- a/exaudfclient/base/python/exascript_python_preset_core.py +++ b/exaudfclient/base/python/exascript_python_preset_core.py @@ -6,14 +6,14 @@ from exascript_python import * import decimal import datetime -import imp +import types class ExaUDFError(Exception): pass def clean_stacktrace_line(line): import re - match = re.match("""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line) + match = re.match(r"""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line) if match is not None: filename=match.group(1) lineno=match.group(2) @@ -126,7 +126,7 @@ def import_script(self, script): modobj = self.__modules[str(code)] else: print("%%% new code", modname, repr(code), code in self.__modules) - modobj = imp.new_module(modname) + modobj = types.ModuleType(modname) modobj.__file__ = "<%s>" % modname modobj.__dict__['exa'] = self self.__modules[str(code)] = modobj diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index 9748ef7a1..6c74d1ff2 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -8,9 +8,12 @@ genrule( cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib cd build_exascript_python_tmp_cc swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i + cd .. + python3 "$(location //base:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" """, outs = ["exascript_python_tmp.cc", "exascript_python.py"], - srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"] + srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"], + tools = ["//base:replace_swig_import_helper.py"], ) genrule( diff --git a/exaudfclient/base/replace_swig_import_helper.py b/exaudfclient/base/replace_swig_import_helper.py new file mode 100644 index 000000000..51cc5f6b6 --- /dev/null +++ b/exaudfclient/base/replace_swig_import_helper.py @@ -0,0 +1,43 @@ +import sys +from pathlib import Path + +old_function = """ + def swig_import_helper(): + from os.path import dirname + import imp + fp = None + try: + fp, pathname, description = imp.find_module('_exascript_python', [dirname(__file__)]) + except ImportError: + import _exascript_python + return _exascript_python + if fp is not None: + try: + _mod = imp.load_module('_exascript_python', fp, pathname, description) + finally: + fp.close() + return _mod + _exascript_python = swig_import_helper() + del swig_import_helper +""" + +new_function = """ + import _exascript_python +""" + +def replace_swig_import_helper(target, source): + source_code = Path(source).read_text() + + assert old_function in source_code + + new_source_code = source_code.replace(old_function, new_function) + + with open(target, 'w') as f: + f.write(new_source_code) + +if __name__ == "__main__": + if len(sys.argv) != 3: + print('Usage: replace_swig_import_helper.py target source') + sys.exit(1) + print(f"replacing function swig_import_helper() using source={sys.argv[2]} and target={sys.argv[1]}") + replace_swig_import_helper(sys.argv[1], sys.argv[2]) diff --git a/ext/scripts/install_scripts/install_via_pip.pl b/ext/scripts/install_scripts/install_via_pip.pl index da93abea6..95ecdbf6b 100755 --- a/ext/scripts/install_scripts/install_via_pip.pl +++ b/ext/scripts/install_scripts/install_via_pip.pl @@ -40,6 +40,7 @@ =head1 SYNOPSIS my $use_deprecated_legacy_resolver = 0; my $ancestor_pip_package_root_path = ''; my $install_build_tools_ephemerally = 0; +my $pip_needs_break_system_packages = 0; GetOptions ( "help" => \$help, "dry-run" => \$dry_run, @@ -51,7 +52,8 @@ =head1 SYNOPSIS "use-deprecated-legacy-resolver" => \$use_deprecated_legacy_resolver, "python-binary=s" => \$python_binary, "ancestor-pip-package-root-path=s" => \$ancestor_pip_package_root_path, - "install-build-tools-ephemerally" => \$install_build_tools_ephemerally + "install-build-tools-ephemerally" => \$install_build_tools_ephemerally, + "pip-needs-break-system-packages" => \$pip_needs_break_system_packages ) or package_mgmt_utils::print_usage_and_abort(__FILE__,"Error in command line arguments",2); package_mgmt_utils::print_usage_and_abort(__FILE__,"",0) if $help; @@ -79,6 +81,10 @@ =head1 SYNOPSIS push @pip_parameters, "--use-deprecated=legacy-resolver"; } +if($pip_needs_break_system_packages){ + push @pip_parameters, "--break-system-packages"; +} + my $pip_parameters_str = join( ' ', @pip_parameters); my $element_separator = '\\|'; my $combining_template = "$python_binary -m pip install $pip_parameters_str --no-cache-dir <<<<0>>>>"; diff --git a/ext/scripts/tests/install_scripts/run_pip_tests.sh b/ext/scripts/tests/install_scripts/run_pip_tests.sh index b507202c5..c5c87954c 100644 --- a/ext/scripts/tests/install_scripts/run_pip_tests.sh +++ b/ext/scripts/tests/install_scripts/run_pip_tests.sh @@ -111,5 +111,10 @@ TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file te assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'" echo +echo ./install_via_pip.pl with pip-needs-break-system-packages +TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --pip-needs-break-system-packages --python-binary python3 "$DRY_RUN_OPTION") +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly' 'requests'" +echo + check_for_failed_tests echo "All pip tests passed" diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile index a18d69b23..992344671 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile @@ -13,7 +13,7 @@ RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfc RUN addgroup --gid 1000 exasolution RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" -RUN addgroup --gid 500 exausers +RUN addgroup --gid 500 exausers RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" ENV LANG en_US.UTF-8 diff --git a/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md b/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md new file mode 100644 index 000000000..7398e1a51 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md @@ -0,0 +1,14 @@ +# Details for template-Exasol-all-python-3.12 + +## Packages + +**Remarks:** The packages are grouped by there main usage and their type. + +- [UDFclient dependencies](flavor_base/udfclient_deps/packages/apt_get_packages) +- [Language dependencies](flavor_base/language_deps/packages/apt_get_packages) +- Flavor packages + - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages) + - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages) +- Customization + - [Ubuntu packages](flavor_customization/packages/apt_get_packages) + - [Python3 pip packages](flavor_customization/packages/python3_pip_packages) diff --git a/flavors/template-Exasol-all-python-3.12/ci.json b/flavors/template-Exasol-all-python-3.12/ci.json new file mode 100644 index 000000000..1bda959cd --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/ci.json @@ -0,0 +1,32 @@ +{ + "build_runner": "ubuntu-24.04", + "test_config": { + "default_test_runner": "ubuntu-24.04", + "test_sets": [ + { + "name": "python", + "folders": ["python3/all"], + "goal": "release", + "generic_language_tests": [] + }, + { + "name": "pandas", + "folders": ["pandas/all", "pandas/pandas2"], + "goal": "release", + "generic_language_tests": [] + }, + { + "name": "linker_namespace", + "folders": ["linker_namespace_sanity"], + "goal": "base_test_build_run", + "generic_language_tests": [] + }, + { + "name": "generic", + "folders": [], + "goal": "release", + "generic_language_tests": ["python3"] + } + ] + } +} diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile new file mode 100644 index 000000000..b87c2bfc5 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile @@ -0,0 +1,41 @@ +FROM {{language_deps}} + +RUN mkdir /conf /buckets + +COPY --from={{base_test_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /env /env +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY /exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"] +RUN cp -r -L bazel-bin/* /exaudf + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient +RUN ./test_udfclient.sh /exaudf/exaudfclient_static + +WORKDIR / +RUN mkdir /exasol_emulator +COPY emulator/ /exasol_emulator +COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile new file mode 100644 index 000000000..3a8ca3f61 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile @@ -0,0 +1,6 @@ +FROM {{build_deps}} + +RUN mkdir -p /build_info/packages +COPY base_test_deps/packages /build_info/packages/base_test_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages new file mode 100644 index 000000000..04a04253f --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages @@ -0,0 +1,5 @@ +gdb|15.0.50.20240403-0ubuntu1 +valgrind|1:3.22.0-0ubuntu3 +binutils|2.42-4ubuntu2.5 +patchelf|0.18.0-1.1build1 +strace|6.8-0ubuntu2 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile new file mode 100644 index 000000000..93d58af5d --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive + +ENV ARCHIVE_UBUNTU_PREFIX="" +RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + +RUN mkdir -p /build_info/packages +COPY build_deps/packages /build_info/packages/build_deps + +ENV BAZEL_PACKAGE_VERSION="7.2.1" +ENV BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" +ENV BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/build_deps/apt_get_packages --with-versions + +RUN apt-get -y update && \ + curl -L --output "$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" && \ + apt-get install -y "./$BAZEL_PACKAGE_FILE" && \ + rm "$BAZEL_PACKAGE_FILE" && \ + apt-get -y clean && \ + apt-get -y autoremove + +RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \ + tar zxf swig-2.0.4.tar.gz && \ + (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \ + rm -rf swig-2.0.4 swig-2.0.4.tar.gz + +RUN locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + ldconfig + +RUN touch /env && \ + echo "export PROTOBUF_BIN=/usr/bin/protoc" >> /env && \ + echo "export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64" >> /env diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages new file mode 100644 index 000000000..d92f66436 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages @@ -0,0 +1,9 @@ +coreutils|9.4-3ubuntu6 +tar|1.35+dfsg-3build1 +curl|8.5.0-2ubuntu10.6 +openjdk-11-jdk|11.0.27+6~us1-0ubuntu1~24.04 +build-essential|12.10ubuntu1 +libpcre3-dev|2:8.39-15build1 +protobuf-compiler|3.21.12-8.2ubuntu0.1 +chrpath|0.16-2build1 +locales|2.39-0ubuntu8.4 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile new file mode 100644 index 000000000..1d1524f16 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile @@ -0,0 +1,43 @@ +FROM {{language_deps}} + +COPY --from={{build_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /env /env +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"] +RUN cp -r -L bazel-bin/* /exaudf +RUN rm -r /root/.cache/bazel #Delete bazel cache as it contains java rules with special characters which will cause a problem for Bucketfs + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient + +WORKDIR /exaudf + +RUN rm -r /exaudfclient + +COPY --from={{build_deps}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN mkdir /conf /buckets diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py b/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py new file mode 100644 index 000000000..d6a10c28f --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py @@ -0,0 +1,147 @@ +from typing import Dict + +from exasol.slc.internal.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask + + +class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "udfclient_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeLanguageDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "language_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"scripts": "ext/scripts"} + + def requires_tasks(self): + return {"udfclient_deps": AnalyzeUDFClientDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "build_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "build_run" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps, + "language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "exaudfclient"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "base_test_deps" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "base_test_build_run" + + def requires_tasks(self): + return {"base_test_deps": AnalyzeBaseTestDeps, + "language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "exaudfclient", "emulator": "emulator"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeFlavorBaseDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_base_deps" + + def requires_tasks(self): + return {"language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self): + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeFlavorCustomization(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_customization" + + def requires_tasks(self): + return {"flavor_base_deps": AnalyzeFlavorBaseDeps} + + +class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_test_build_run" + + def requires_tasks(self): + return {"flavor_customization": AnalyzeFlavorCustomization, + "base_test_build_run": AnalyzeBaseTestBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeRelease(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "release" + + def requires_tasks(self): + return {"flavor_customization": AnalyzeFlavorCustomization, + "build_run": AnalyzeBuildRun, + "language_deps": AnalyzeLanguageDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class SecurityScan(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "security_scan" + + def requires_tasks(self): + return {"release": AnalyzeRelease} + + def get_path_in_flavor(self): + return "flavor_base" diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from b/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from new file mode 100644 index 000000000..fb0760244 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from @@ -0,0 +1 @@ +flavors/template-Exasol-all-python-3.10 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile new file mode 100644 index 000000000..33c65c6ee --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile @@ -0,0 +1,8 @@ +FROM {{ language_deps }} + +RUN mkdir -p /build_info/packages +COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions + +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages new file mode 100644 index 000000000..3b66a1381 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages @@ -0,0 +1,4 @@ +unzip|6.0-28ubuntu4 +git|1:2.43.0-1ubuntu7 +libcurl4-openssl-dev|8.5.0-2ubuntu10.6 +build-essential|12.10ubuntu1 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/python3_pip_packages new file mode 100644 index 000000000..e69de29bb diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile new file mode 100644 index 000000000..32230a4d9 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile @@ -0,0 +1,25 @@ +FROM {{base_test_build_run}} + +RUN mkdir -p /conf /buckets + +COPY --from={{flavor_customization}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition new file mode 100644 index 000000000..0576fe703 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition @@ -0,0 +1 @@ +PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile new file mode 100644 index 000000000..f1ca7dfc2 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile @@ -0,0 +1,13 @@ +FROM {{udfclient_deps}} + +RUN mkdir -p /build_info/packages/language_deps +COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions + +COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages + +ENV PYTHON3_PREFIX /usr +ENV PYTHON3_VERSION python3.12 + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages new file mode 100644 index 000000000..8bad8824a --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages @@ -0,0 +1,4 @@ +ca-certificates|20240203 +python3.12-dev|3.12.3-1ubuntu0.7 +curl|8.5.0-2ubuntu10.6 +python3-pip|24.0+dfsg-1ubuntu1.1 \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages new file mode 100644 index 000000000..63697c0e3 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages @@ -0,0 +1,3 @@ +pandas|2.2.2 +numpy|1.26.4 +pyarrow|16.0.0 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile new file mode 100644 index 000000000..b5e1d4708 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:24.04 +RUN mkdir /conf /buckets + +COPY --from={{language_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /var /var +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /scripts /scripts +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +COPY --from={{flavor_customization}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /var /var +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +COPY --from={{build_run}} /exaudf /exaudf +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_run}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN mkdir -p /build_info/actual_installed_packages/release && \ + /scripts/list_installed_scripts/list_installed_apt.sh > /build_info/actual_installed_packages/release/apt_get_packages && \ + /scripts/list_installed_scripts/list_installed_pip.sh python3.12 > /build_info/actual_installed_packages/release/python3_pip_packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/.trivyignore b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/.trivyignore new file mode 100644 index 000000000..e69de29bb diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile new file mode 100644 index 000000000..5dabc1ff6 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile @@ -0,0 +1,15 @@ +FROM {{release}} +ENV DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /build_info/packages +COPY security_scan/packages /build_info/packages/security_scan +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages_trivy_deps +RUN curl -s https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add - && \ + /scripts/install_scripts/install_ppa.pl --ppa 'deb https://aquasecurity.github.io/trivy-repo/deb noble main' --out-file trivy.list +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages + +ENV SECURITY_SCANNERS="trivy" +COPY /security_scan/.trivyignore /.trivyignore +COPY /security_scan/trivy.rego /trivy.rego + +ENTRYPOINT ["/scripts/security_scan/run.sh"] \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages new file mode 100644 index 000000000..6bd628d4c --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages @@ -0,0 +1 @@ +trivy diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps new file mode 100644 index 000000000..8920661bf --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps @@ -0,0 +1,2 @@ +apt-transport-https +gnupg \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego new file mode 100644 index 000000000..ea531bfaf --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego @@ -0,0 +1,10 @@ +package trivy + +import data.lib.trivy + +default ignore = false + +ignore { + input.PkgName == "linux-libc-dev" + regex.match("^(kernel:|In the Linux kernel)", input.Title) +} \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig b/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig new file mode 100644 index 000000000..d1bdb8113 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig @@ -0,0 +1,2 @@ +generic_language_tests=python3 +test_folders=python3/all pandas/all pandas/pandas2 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile new file mode 100644 index 000000000..c67b999ec --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive + +ENV ARCHIVE_UBUNTU_PREFIX="" +RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + +RUN mkdir -p /build_info/packages +COPY udfclient_deps/packages /build_info/packages/udfclient_deps +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions + +RUN addgroup --gid 1001 exasolution +RUN adduser --disabled-login --uid 1001 --gid 1001 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 501 exausers +RUN adduser --disabled-login --uid 501 --gid 501 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +ENV PROTOBUF_LIBRARY_PREFIX=/usr/lib/ +ENV PROTOBUF_INCLUDE_PREFIX=/usr/include/ +ENV ZMQ_LIBRARY_PREFIX=/usr/lib +ENV ZMQ_INCLUDE_PREFIX=/usr/include diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages new file mode 100644 index 000000000..8d9e27f08 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages @@ -0,0 +1,8 @@ +coreutils|9.4-3ubuntu6 +adduser|3.137ubuntu1 +locales|2.39-0ubuntu8.4 +libnss-db|2.2.3pre1-9build3 +libzmq3-dev|4.3.5-1build2 +cppzmq-dev|4.10.0-1build1 +libprotobuf-dev|3.21.12-8.2ubuntu0.1 +libssl-dev|3.0.13-0ubuntu3.5 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile new file mode 100644 index 000000000..e83813dbe --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile @@ -0,0 +1,77 @@ +############################################################################################ +############################################################################################ +# This Dockerfile allows you to extend this flavor by installing packages or adding files. +# IF you didn't change the lines below, you can add packages and their version to the +# files in ./packages and they get automatically installed. +############################################################################################ +############################################################################################ + +####################################################################### +####################################################################### +# Do not change the following lines unless you know what you are doing +####################################################################### +####################################################################### + +FROM {{flavor_base_deps}} + +RUN mkdir -p /build_info/packages/flavor_customization + +COPY flavor_customization/packages/apt_get_packages /build_info/packages/flavor_customization +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_customization/apt_get_packages --with-versions + +COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3 --with-versions --allow-no-version + + +########################################################################## +########################################################################## +# Below this text you can add any Dockerfile commands except of FROM. +# However only commands which change the filesystem will be +# reflected to the final script-language container. +# We recommend to use only RUN, COPY and ADD. For example, WORKDIR, USER +# and ENV will be not carried over into the final container. For more +# information about Dockerfile commands, please check the reference +# https://docs.docker.com/engine/reference/builder/#dockerfile-reference +########################################################################## +########################################################################## + + +########### +########### +# Examples: +########### +########### + +################################################# +# Execute a command during the build of the image +################################################# + +# With RUN you can excute any command you usally would run in a bash shell. +# It usually is a good idea to cleanup downloaded files or packages +# caches in the same in the same RUN statement to keep the image size +# small. You can execute multiple shell commands in one RUN statement +# by combine them with +# - `&&` (only execute if the previous command was successfull), +# - `||` (only execute if the previous command was unsuccesfull) +# - `;` (execute regardless of the success of the previous command) +# To format multiple command per RUN statement you can add a line break with `\`. +# If you need to execute complex sequences of commands you should execute +# scripts which you can copy into the image. +# ------------------------------------------------------------------------------- + +# RUN git clone https://github.com/exasol/data-science-examples +# RUN curl -o data-science-examples.zip https://codeload.github.com/exasol/data-science-examples/zip/master && \ +# unzip data-science-examples.zip && \ +# rm data-science-examples.zip + +##################################### +# Copy a file from ./ into the image. +##################################### + +# In case of exaslct it is important to prefix source files or directories with `flavor_customization` +# if you want to copy files or directories located in the same directory as this Dockerfile +# ----------------------------------------------------------------------------------------------------- + +# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization +# ADD flavor_customization/code.tar.gz /code + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages new file mode 100644 index 000000000..d78cd6de0 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages @@ -0,0 +1,9 @@ +# This file specifies the package list which gets installed via apt. +# You must specify the the package and its version separated by a |. +# We recommend here the usage of package versions, to ensure that the container +# builds are reproducible. However, we allow also packages without version. +# As you can see, this file can contain comments which start with #. +# If a line starts with # the whole line is a comment, however you can +# also start a comment after the package definition. + +# p7zip-full|16.02+dfsg-6 # you can check which versions of a package are available with apt-cache policy diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages new file mode 100644 index 000000000..a9f22f993 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages @@ -0,0 +1,9 @@ +# This file specifies the package list which gets installed via pip for python3. +# You must specify the the package and its version separated by a |. +# We recommend here the usage of package versions, to ensure that the container +# builds are reproducible. However, we allow also packages without version. +# As you can see, this file can contain comments which start with #. +# If a line starts with # the whole line is a comment, however you can +# also start a comment after the package definition. + +#tensorflow-probability|0.9.0