Support python 311 (#619)

Companion Open3D PR: isl-org/Open3D#6288
- Add support for python3.11.
-️ Upgrade pytorch to 2.0.1 with cuda 11.7
- Upgrade tensorflow / tensorboard to 2.13. 
- Disable building tensorflow ops on Linux. Keep old cxx11_abi.
- Update CI to use requirements files instead of manually specified versions.
- Add instructions to build with new cxx11_abi with tensorflow ops support.

Author: ssheorey

.github/workflows/ubuntu.yml

@@ -16,7 +16,7 @@ jobs:
       NPROC: 2
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup cache
         uses: actions/cache@v3
         with:
@@ -33,7 +33,7 @@ jobs:
       - name: Set up Python version
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       # Pre-installed  packages: https://github.com/actions/runner-images/tree/main/images
       - name: Install ccache
         run: |

README.md

@@ -1,54 +1,42 @@
 #!/usr/bin/env bash
-#
-# The following environment variables are required:
-# - NPROC
-#
-TENSORFLOW_VER="2.8.2"
-TORCH_GLNX_VER="1.13.1+cpu"
-# OPENVINO_DEV_VER="2021.4.2"  # Numpy version conflict with TF 2.8.2
-PIP_VER="21.1.1"
-WHEEL_VER="0.38.4"
-STOOLS_VER="67.3.2"
-YAPF_VER="0.30.0"
-PYTEST_VER="7.1.2"
-PYTEST_RANDOMLY_VER="3.8.0"
 
 set -euo pipefail
 
+NPROC=${NPROC:?'env var must be set to number of available CPUs.'}
+PIP_VER="23.2.1"
+
 echo 1. Prepare the Open3D-ML repo and install dependencies
 echo
-export PATH_TO_OPEN3D_ML=$(pwd)
+export PATH_TO_OPEN3D_ML="$PWD"
+echo "$PATH_TO_OPEN3D_ML"
 # the build system of the main repo expects a master branch. make sure master exists
 git checkout -b master || true
-python -m pip install -U pip==$PIP_VER \
-    wheel=="$WHEEL_VER" \
-    setuptools=="$STOOLS_VER" \
-    yapf=="$YAPF_VER" \
-    pytest=="$PYTEST_VER" \
-    pytest-randomly=="$PYTEST_RANDOMLY_VER"
-
-python -m pip install -r requirements.txt
-echo $PATH_TO_OPEN3D_ML
+python -m pip install -U pip==$PIP_VER
+python -m pip install -r requirements.txt \
+    -r requirements-torch.txt
+# -r requirements-tensorflow.txt # TF disabled on Linux (Open3D PR#6288)
+# -r requirements-openvino.txt # Numpy version conflict with TF 2.8.2
 cd ..
 python -m pip install -U Cython
 
 echo 2. clone Open3D and install dependencies
 echo
-git clone --recursive --branch master https://github.com/isl-org/Open3D.git
+git clone --branch master https://github.com/isl-org/Open3D.git
 
 ./Open3D/util/install_deps_ubuntu.sh assume-yes
-python -m pip install -U tensorflow-cpu==$TENSORFLOW_VER \
-    torch==${TORCH_GLNX_VER} --extra-index-url https://download.pytorch.org/whl/cpu/
-# openvino-dev=="$OPENVINO_DEV_VER"
+python -m pip install -r Open3D/python/requirements.txt \
+    -r Open3D/python/requirements_style.txt \
+    -r Open3D/python/requirements_test.txt
 
 echo 3. Configure for bundling the Open3D-ML part
 echo
 mkdir Open3D/build
 pushd Open3D/build
+# TF disabled on Linux (Open3D PR#6288)
 cmake -DBUNDLE_OPEN3D_ML=ON \
-    -DOPEN3D_ML_ROOT=$PATH_TO_OPEN3D_ML \
+    -DOPEN3D_ML_ROOT="${PATH_TO_OPEN3D_ML}" \
     -DGLIBCXX_USE_CXX11_ABI=OFF \
-    -DBUILD_TENSORFLOW_OPS=ON \
+    -DBUILD_TENSORFLOW_OPS=OFF \
     -DBUILD_PYTORCH_OPS=ON \
     -DBUILD_GUI=ON \
     -DBUILD_UNIT_TESTS=OFF \
@@ -66,12 +54,13 @@ echo
 popd
 mkdir test_workdir
 pushd test_workdir
-mv $PATH_TO_OPEN3D_ML/tests .
+mv "$PATH_TO_OPEN3D_ML/tests" .
 echo Add --randomly-seed=SEED to the test command to reproduce test order.
 python -m pytest tests
 
-echo ... now do the same but in dev mode by setting OPEN3D_ML_ROOT
-export OPEN3D_ML_ROOT=$PATH_TO_OPEN3D_ML
+echo "... now do the same but in dev mode by setting OPEN3D_ML_ROOT"
+echo
+export OPEN3D_ML_ROOT="$PATH_TO_OPEN3D_ML"
 echo Add --randomly-seed=SEED to the test command to reproduce test order.
 python -m pytest tests
 unset OPEN3D_ML_ROOT

ci/run_ci.sh

@@ -1,2 +1 @@
-tensorflow~=2.8.4 ; sys_platform != 'darwin' or platform_machine != 'arm64'
-tensorflow-macos==2.8.0 ; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow~=2.13.0

requirements-tensorflow.txt

@@ -1,5 +1,5 @@
 -f https://download.pytorch.org/whl/torch/
-torch==1.13.1+cu116
+torch==2.0.1+cu117
 -f https://download.pytorch.org/whl/torchvision/
-torchvision==0.14.1+cu116
+torchvision==0.15.2+cu117
 tensorboard

requirements-torch-cuda.txt

@@ -1,6 +1,6 @@
 --extra-index-url https://download.pytorch.org/whl/cpu/
-torch==1.13.1+cpu ; sys_platform != 'darwin'
-torchvision==0.14.1+cpu ; sys_platform != 'darwin'
-torch==1.13.1 ; sys_platform == 'darwin'
-torchvision==0.14.1 ; sys_platform == 'darwin'
+torch==2.0.1+cpu ; sys_platform != 'darwin'
+torchvision==0.15.2+cpu ; sys_platform != 'darwin'
+torch==2.0.1 ; sys_platform == 'darwin'
+torchvision==0.15.2 ; sys_platform == 'darwin'
 tensorboard

requirements-torch.txt

@@ -1,7 +1,7 @@
 addict
 pillow>=9.3.0
 matplotlib>=3
-numpy>1.15
+numpy>1.18
 pandas>=1.0
 pyyaml>=5.4.1
 scikit-learn>=0.21

requirements.txt

@@ -1,5 +1,6 @@
 import pytest
 import os
+import open3d as o3d
 
 if 'PATH_TO_OPEN3D_ML' in os.environ.keys():
     base = os.environ['PATH_TO_OPEN3D_ML']
@@ -8,6 +9,7 @@
     # base = '../Open3D-ML'
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_PYTORCH_OPS']")
 def test_integration_torch():
     import torch
     import open3d.ml.torch as ml3d
@@ -25,6 +27,7 @@ def test_integration_torch():
     print(model)
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_TENSORFLOW_OPS']")
 def test_integration_tf():
     import tensorflow as tf
     import open3d.ml.tf as ml3d
@@ -45,15 +48,10 @@ def test_integration_tf():
 def test_integration_openvino():
     try:
         from openvino.inference_engine import IECore
-        openvino_available = True
-    except:
-        openvino_available = False
-
-    if not openvino_available:
+    except ImportError:
         return
 
-    from open3d.ml.torch.models import OpenVINOModel
-    from open3d.ml.tf.models import OpenVINOModel
-
-
-test_integration_torch()
+    if o3d._build_config['BUILD_TORCH_OPS']:
+        from open3d.ml.torch.models import OpenVINOModel
+    if o3d._build_config['BUILD_TENSORFLOW_OPS']:
+        from open3d.ml.tf.models import OpenVINOModel

tests/test_integration.py

@@ -1,22 +1,28 @@
 import pytest
 import os
 import numpy as np
-import torch
-import tensorflow as tf
+import open3d as o3d
+try:
+    import torch
+except ImportError:
+    torch = None
 
 if 'PATH_TO_OPEN3D_ML' in os.environ.keys():
     base = os.environ['PATH_TO_OPEN3D_ML']
 else:
     base = '.'
 
-gpus = tf.config.list_physical_devices('GPU')
-if gpus:
-    # Use first GPU and restrict memory growth.
-    try:
+try:
+    import tensorflow as tf
+    gpus = tf.config.list_physical_devices('GPU')
+    if gpus:
+        # Use first GPU and restrict memory growth.
         tf.config.experimental.set_visible_devices(gpus[0], 'GPU')
-        tf.config.set_memory_growth(gpu[0], True)
-    except RuntimeError as e:
-        print(e)
+        tf.config.set_memory_growth(gpus[0], True)
+except RuntimeError as e:
+    print(e)
+except ImportError:
+    tf = None
 
 try:
     from open3d.ml.torch.models import OpenVINOModel
@@ -25,6 +31,7 @@
     openvino_available = False
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_PYTORCH_OPS']")
 def test_randlanet_torch():
     import open3d.ml.torch as ml3d
 
@@ -66,6 +73,7 @@ def test_randlanet_torch():
     assert out.shape == (1, 5000, 10)
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_TENSORFLOW_OPS']")
 def test_randlanet_tf():
     import open3d.ml.tf as ml3d
 
@@ -106,6 +114,7 @@ def test_randlanet_tf():
         assert np.max(np.abs(ov_out - out)) < 1e-6
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_PYTORCH_OPS']")
 def test_kpconv_torch():
     import open3d.ml.torch as ml3d
 
@@ -144,6 +153,7 @@ def test_kpconv_torch():
         assert np.max(np.abs(ov_out - out)) < 1e-7
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_TENSORFLOW_OPS']")
 def test_kpconv_tf():
     import open3d.ml.tf as ml3d
 
@@ -191,6 +201,7 @@ def test_kpconv_tf():
         assert np.max(np.abs(ov_out - out)) < 1e-5
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_PYTORCH_OPS']")
 def test_pointpillars_torch():
     import open3d.ml.torch as ml3d
     from open3d.ml.utils import Config
@@ -225,6 +236,7 @@ def test_pointpillars_torch():
             assert torch.max(torch.abs(out - ref)) < 1e-5
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_TENSORFLOW_OPS']")
 def test_pointpillars_tf():
     import open3d.ml.tf as ml3d
     from open3d.ml.utils import Config