Merge branch 'develop' into 'master'

Release 4.3.0

See merge request remi.cresson/otbtf!108

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 variables:
-  OTBTF_VERSION: 4.2.2
+  OTBTF_VERSION: 4.3.0
   OTB_BUILD: /src/otb/build/OTB/build  # Local OTB build directory
   OTBTF_SRC: /src/otbtf  # Local OTBTF source directory
   OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary  # OTB testing directory

Dockerfile

@@ -10,23 +10,15 @@ WORKDIR /tmp
 ### System packages
 COPY tools/docker/build-deps-*.txt ./
 ARG DEBIAN_FRONTEND=noninteractive
-# CLI
 RUN apt-get update -y && apt-get upgrade -y \
  && cat build-deps-cli.txt | xargs apt-get install --no-install-recommends -y \
  && apt-get clean && rm -rf /var/lib/apt/lists/*
-# Optional GUI
-ARG GUI=false
-RUN if $GUI; then \
-      apt-get update -y \
-      && cat build-deps-gui.txt | xargs apt-get install --no-install-recommends -y \
-      && apt-get clean && rm -rf /var/lib/apt/lists/* ; fi
-
 ### Python3 links and pip packages
 RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip
 # Upgrade pip
 RUN pip install --no-cache-dir pip --upgrade
-# NumPy version is conflicting with system's gdal dep and may require venv
-ARG NUMPY_SPEC="==1.22.*"
+# In case NumPy version is conflicting with system's gdal dep and may require venv
+ARG NUMPY_SPEC=""
 # This is to avoid https://github.com/tensorflow/tensorflow/issues/61551
 ARG PROTO_SPEC="==4.23.*"
 RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging requests \
@@ -35,7 +27,7 @@ RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$N
 # ----------------------------------------------------------------------------
 # Tmp builder stage - dangling cache should persist until "docker builder prune"
 FROM otbtf-base AS builder
-# A smaller value may be required to avoid OOM errors when building OTB GUI
+# A smaller value may be required to avoid OOM errors when building OTB
 ARG CPU_RATIO=1
 
 RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib/python3
@@ -45,7 +37,7 @@ RUN git config --global advice.detachedHead false
 
 ### TF
 
-ARG TF=v2.12.0
+ARG TF=v2.14.0
 ARG TENSORRT
 
 # Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default)
@@ -79,6 +71,7 @@ RUN cd tensorflow \
       && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" '
 
 # Installation
+RUN apt update && apt install -y patchelf
 RUN cd tensorflow \
  && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \
  && pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \
@@ -97,8 +90,7 @@ RUN cd tensorflow \
 
 ### OTB
 
-ARG GUI=false
-ARG OTB=d74ab47d4308591db4ed5a5ea3b820cef73a39fe
+ARG OTB=release-9.0
 ARG OTBTESTS=false
 
 RUN mkdir /src/otb
@@ -110,27 +102,26 @@ RUN apt-get update -y \
  && apt-get install --reinstall ca-certificates -y \
  && update-ca-certificates \
  && git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \
- && cd otb && git checkout $OTB \
+ && cd otb && git checkout $OTB
+
 # <---------------------------------------- Begin dirty hack
 # This is a dirty hack for release 4.0.0alpha
 # We have to wait that OTB moves from C++14 to C++17
 # See https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2338
+RUN cd /src/otb/otb \
  && sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt \
- && echo "" > Modules/Filtering/ImageManipulation/test/CMakeLists.txt \
- && echo "" > Modules/Segmentation/Conversion/test/CMakeLists.txt \
- && echo "" > Modules/Radiometry/Indices/test/CMakeLists.txt \
- && echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \
- && echo "" > Modules/Feature/Edge/test/CMakeLists.txt \
+ && echo "" > Modules/Core/ImageManipulation/test/CMakeLists.txt \
+ && echo "" > Modules/Core/Conversion/test/CMakeLists.txt \
+ && echo "" > Modules/Core/Indices/test/CMakeLists.txt \
+ && echo "" > Modules/Core/Edge/test/CMakeLists.txt \
  && echo "" > Modules/Core/ImageBase/test/CMakeLists.txt \
+ && echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \
 # <---------------------------------------- End dirty hack
  && cd .. \
  && mkdir -p build \
  && cd build \
  && if $OTBTESTS; then \
       echo "-DBUILD_TESTING=ON" >> ../build-flags-otb.txt; fi \
- # Set GL/Qt build flags
- && if $GUI; then \
-      sed -i -r "s/-DOTB_USE_(QT|OPENGL|GL[UFE][WT])=OFF/-DOTB_USE_\1=ON/" ../build-flags-otb.txt; fi \
  # Possible ENH: superbuild-all-dependencies switch, with separated build-deps-minimal.txt and build-deps-otbcli.txt)
  #&& if $OTB_SUPERBUILD_ALL; then sed -i -r "s/-DUSE_SYSTEM_([A-Z0-9]*)=ON/-DUSE_SYSTEM_\1=OFF/ " ../build-flags-otb.txt; fi \
  && OTB_FLAGS=$(cat "../build-flags-otb.txt") \
@@ -139,7 +130,6 @@ RUN apt-get update -y \
 
 ### OTBTF - copy (without .git/) or clone repository
 COPY . /src/otbtf
-#RUN git clone https://github.com/remicres/otbtf.git /src/otbtf
 RUN ln -s /src/otbtf /src/otb/otb/Modules/Remote/otbtf
 
 # Rebuild OTB with module
@@ -152,12 +142,10 @@ RUN cd /src/otb/build/OTB/build \
       -DOTB_WRAP_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 \
       -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
       -Dtensorflow_include_dir=/opt/otbtf/include/tf \
-      # Forcing TF>=2, this Dockerfile hasn't been tested with v1 + missing link for libtensorflow_framework.so in the wheel
       -DTENSORFLOW_CC_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_cc.so.2 \
       -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_framework.so.2 \
  && make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \
  # Cleaning
- && ( $GUI || rm -rf /opt/otbtf/bin/otbgui* ) \
  && ( $KEEP_SRC_OTB || rm -rf /src/otb ) \
  && rm -rf /root/.cache /tmp/*
 
@@ -206,4 +194,3 @@ RUN python -c "import tensorflow"
 RUN python -c "import otbtf, tricks"
 RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')"
 RUN python -c "from osgeo import gdal"
-

RELEASE_NOTES.txt

@@ -1,3 +1,10 @@
+Version 4.3.0 (02 jan 2024)
+----------------------------------------------------------------
+* Add compute capability 8.0 and 9.0
+* Remove OTB GUI dependencies
+* Move OTB to version 9.0.0
+* Move TensorFlow to version 2.14
+
 Version 4.2.2 (19 oct 2023)
 ----------------------------------------------------------------
 * Add a test to ensure that planetary computer works in the OTBTF docker image

doc/docker_use.md

@@ -25,12 +25,12 @@ Since OTBTF >= 3.2.1 you can find the latest docker images on
 
 | Name                                                                               | Os            | TF    | OTB   | Description            | Dev files | Compute capability |
 |------------------------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:4.2.2-cpu**                                                         | Ubuntu Jammy  | r2.12 | d74ab | CPU, no optimization   | no        | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.2.2-cpu-dev**                                                     | Ubuntu Jammy  | r2.12 | d74ab | CPU, no optimization (dev) |  yes  | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.2.2-gpu**                                                         | Ubuntu Jammy  | r2.12 | d74ab | GPU, no optimization   | no        | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.2.2-gpu-dev**                                                     | Ubuntu Jammy  | r2.12 | d74ab | GPU, no optimization (dev) | yes   | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt**     | Ubuntu Jammy  | r2.12 | d74ab | GPU with opt.          | no        | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt-dev** | Ubuntu Jammy  | r2.12 | d74ab | GPU with opt. (dev)    | yes       | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.0-cpu**                                                         | Ubuntu Jammy  | r2.14 | 9.0.0 | CPU, no optimization   | no        | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.0-cpu-dev**                                                     | Ubuntu Jammy  | r2.14 | 9.0.0 | CPU, no optimization (dev) |  yes  | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.0-gpu**                                                         | Ubuntu Jammy  | r2.14 | 9.0.0 | GPU, no optimization   | no        | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.0-gpu-dev**                                                     | Ubuntu Jammy  | r2.14 | 9.0.0 | GPU, no optimization (dev) | yes   | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.0-gpu-opt**     | Ubuntu Jammy  | r2.14 | 9.0.0 | GPU with opt.          | no        | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.0-gpu-opt-dev** | Ubuntu Jammy  | r2.14 | 9.0.0 | GPU with opt. (dev)    | yes       | 5.2,6.1,7.0,7.5,8.6|
 
 The list of older releases is available [here](#older-images).
 
@@ -184,4 +184,9 @@ Here you can find the list of older releases of OTBTF:
 | **mdl4eo/otbtf:4.2.1-gpu-dev**                                                     | Ubuntu Jammy  | r2.12 | 8.1.2 | GPU, no optimization (dev) | yes   | 5.2,6.1,7.0,7.5,8.6|
 | **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.1-gpu-opt**     | Ubuntu Jammy  | r2.12 | 8.1.2 | GPU with opt.          | no        | 5.2,6.1,7.0,7.5,8.6|
 | **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.1-gpu-opt-dev** | Ubuntu Jammy  | r2.12 | 8.1.2 | GPU with opt. (dev)    | yes       | 5.2,6.1,7.0,7.5,8.6|
-
+| **mdl4eo/otbtf:4.2.2-cpu**                                                         | Ubuntu Jammy  | r2.12 | d74ab | CPU, no optimization   | no        | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.2.2-cpu-dev**                                                     | Ubuntu Jammy  | r2.12 | d74ab | CPU, no optimization (dev) |  yes  | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.2.2-gpu**                                                         | Ubuntu Jammy  | r2.12 | d74ab | GPU, no optimization   | no        | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.2.2-gpu-dev**                                                     | Ubuntu Jammy  | r2.12 | d74ab | GPU, no optimization (dev) | yes   | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt**     | Ubuntu Jammy  | r2.12 | d74ab | GPU with opt.          | no        | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.2-gpu-opt-dev** | Ubuntu Jammy  | r2.12 | d74ab | GPU with opt. (dev)    | yes       | 5.2,6.1,7.0,7.5,8.6|

setup.py

@@ -6,7 +6,7 @@
 
 setuptools.setup(
     name="otbtf",
-    version="4.2.2",
+    version="4.3.0",
     author="Remi Cresson",
     author_email="[email protected]",
     description="OTBTF: Orfeo ToolBox meets TensorFlow",

tools/docker/build-env-tf.sh

@@ -20,6 +20,7 @@ export TF_NEED_OPENCL=0
 export TF_NEED_OPENCL_SYCL=0
 export TF_NEED_VERBS=0
 export TF_SET_ANDROID_WORKSPACE=0
+export TF_NEED_CLANG=0
 # For MKL support BZL_CONFIGS+=" --config=mkl"
 #export TF_DOWNLOAD_MKL=1
 #export TF_NEED_MKL=0
@@ -46,7 +47,7 @@ if  [ ! -z $CUDA_TOOLKIT_PATH ] ; then
     fi
     export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CUDA_TOOLKIT_PATH/lib64:$CUDA_TOOLKIT_PATH/lib64/stubs"
     export TF_CUDA_VERSION=$(echo $CUDA_TOOLKIT_PATH | sed -r 's/.*\/cuda-(.*)/\1/')
-    export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5,8.6"
+    export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5,8.0,8.6,9.0"
     export TF_NEED_CUDA=1
     export TF_CUDA_CLANG=0
     export TF_NEED_TENSORRT=0

tools/docker/build-flags-otb.txt

@@ -1,19 +1,8 @@
--DUSE_SYSTEM_BOOST=ON
--DUSE_SYSTEM_CURL=ON
--DUSE_SYSTEM_EXPAT=ON
--DUSE_SYSTEM_FFTW=ON
--DUSE_SYSTEM_FREETYPE=ON
--DUSE_SYSTEM_GDAL=OFF
--DUSE_SYSTEM_GEOS=ON
--DUSE_SYSTEM_GEOTIFF=OFF
--DUSE_SYSTEM_GSL=ON
--DUSE_SYSTEM_ITK=ON
--DUSE_SYSTEM_LIBKML=ON
--DUSE_SYSTEM_LIBSVM=ON
--DUSE_SYSTEM_MUPARSER=ON
--DUSE_SYSTEM_MUPARSERX=ON
--DUSE_SYSTEM_OPENCV=ON
--DUSE_SYSTEM_PNG=ON
--DUSE_SYSTEM_TINYXML=ON
--DUSE_SYSTEM_ZLIB=ON
--DUSE_SYSTEM_SWIG=ON
+-DOTB_BUILD_FeaturesExtraction=ON
+-DOTB_BUILD_Hyperspectral=ON
+-DOTB_BUILD_Learning=ON
+-DOTB_BUILD_Miscellaneous=ON
+-DOTB_BUILD_RemoteModules=ON
+-DOTB_BUILD_SAR=ON
+-DOTB_BUILD_Segmentation=ON
+-DOTB_BUILD_StereoProcessing=ON