Upgrade to support and default to clang 18 for the OSS compiler

PiperOrigin-RevId: 616889174
tensorflow · Apr 16, 2024 · da444d5 · da444d5
1 parent d8ad46b
commit da444d5
Show file tree

Hide file tree

Showing 30 changed files with 245 additions and 240 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -251,7 +251,7 @@ build:cuda_clang_official --action_env=TF_CUDA_VERSION="12"
 build:cuda_clang_official --action_env=TF_CUDNN_VERSION="8"
 build:cuda_clang_official --action_env=CUDA_TOOLKIT_PATH="/usr/local/cuda-12.3"
 build:cuda_clang_official --action_env=GCC_HOST_COMPILER_PATH="/dt9/usr/bin/gcc"
-build:cuda_clang_official --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-17/bin/clang"
+build:cuda_clang_official --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-18/bin/clang"
 build:cuda_clang_official --action_env=LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64"
 build:cuda_clang_official --crosstool_top="@sigbuild-r2.17-clang_config_cuda//crosstool:toolchain"
 
@@ -602,15 +602,12 @@ try-import %workspace%/.bazelrc.user
 # Build TensorFlow v2.
 test:release_base --test_size_filters=small,medium
 
-# Ensure release_base is set on linux
-build:release_linux_base --config=release_base
-
-# Target the AVX instruction set
-build:release_linux_base --config=avx_linux
-
 # Enable support for all targets
 build:release_base --config=cpu_cross
 
+# Ensure release_base is set on linux
+build:release_linux_base --config=release_base
+
 # Disable clang extension that rejects type definitions within offsetof.
 # This was added in clang-16 by https://reviews.llvm.org/D133574.
 # Can be removed once upb is updated, since a type definition is used within
@@ -633,8 +630,8 @@ build:release_linux_base --action_env PYTHON_BIN_PATH="/usr/bin/python3"
 build:release_linux_base --action_env PYTHON_LIB_PATH="/usr/lib/tf_python"
 build:release_linux_base --python_path="/usr/bin/python3"
 # Set Clang as compiler. Use the actual path to clang installed in container.
-build:release_cpu_linux_base --repo_env=CC="/usr/lib/llvm-17/bin/clang"
-build:release_cpu_linux_base --repo_env=BAZEL_COMPILER="/usr/lib/llvm-17/bin/clang"
+build:release_cpu_linux_base --repo_env=CC="/usr/lib/llvm-18/bin/clang"
+build:release_cpu_linux_base --repo_env=BAZEL_COMPILER="/usr/lib/llvm-18/bin/clang"
 # Test-related settings below this point.
 test:release_linux_base --build_tests_only --keep_going --test_output=errors --verbose_failures=true
 test:release_linux_base --local_test_jobs=HOST_CPUS
@@ -645,6 +642,8 @@ test:release_linux_base --test_summary=short
 # Use the Clang toolchain to compile
 build:release_cpu_linux --config=release_linux_base
 build:release_cpu_linux --crosstool_top="@sigbuild-r2.17-clang_config_cuda//crosstool:toolchain"
+# Target the AVX instruction set
+build:release_cpu_linux --config=avx_linux
 
 build:release_gpu_linux --config=release_cpu_linux
 # Set up compilation CUDA version and paths and use the CUDA Clang toolchain.

diff --git a/ci/official/containers/linux_arm64/Dockerfile b/ci/official/containers/linux_arm64/Dockerfile
@@ -1,5 +1,5 @@
 ################################################################################
-FROM ubuntu:20.04 as builder
+FROM ubuntu:22.04 as builder
 ################################################################################
 
 # Install devtoolset build dependencies
@@ -17,16 +17,10 @@ COPY builder.devtoolset/stringop_trunc.patch /stringop_trunc.patch
 
 RUN /build_devtoolset.sh devtoolset-10 /dt10
 
-# Build later version of patchelf that is not so buggy
-COPY builder.patchelf/build_patchelf.sh /build_patchelf.sh
-COPY apt.conf /etc/apt/
-RUN /build_patchelf.sh
-
 ################################################################################
 FROM nvidia/cuda:12.3.1-devel-ubuntu20.04 as devel
 ################################################################################
 COPY --from=builder /dt10 /dt10
-COPY --from=builder /patchelf/patchelf_0.14.3-1_arm64.deb /patchelf/patchelf_0.14.3-1_arm64.deb
 
 # Install devtoolset devel dependencies
 COPY setup.sources.sh /setup.sources.sh
@@ -49,7 +43,6 @@ RUN wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --trie
 RUN groupadd -g 1001 buildslave && useradd -m -u 1001 -g buildslave buildslave
 RUN mkdir -p /tf/venv
 RUN chown -R buildslave:buildslave /tf
-RUN dpkg -i /patchelf/patchelf_0.14.3-1_arm64.deb
 
 # All lines past this point are reset when $CACHEBUSTER is set. We need this
 # for Python specifically because we install some nightly packages which are

diff --git a/ci/official/containers/linux_arm64/apt.conf b/ci/official/containers/linux_arm64/apt.conf
@@ -12,4 +12,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-APT::Default-Release "focal";
+APT::Default-Release "jammy";
diff --git a/ci/official/containers/linux_arm64/cuda.packages.txt b/ci/official/containers/linux_arm64/cuda.packages.txt
@@ -1,6 +1,8 @@
 # CuDNN: https://docs.nvidia.com/deeplearning/sdk/cudnn-install/index.html#ubuntu-network-installation
 libcudnn8=8.9.6.50-1+cuda12.2
 libcudnn8-dev=8.9.6.50-1+cuda12.2
+libnvinfer-headers-dev=8.6.1.6-1+cuda12.0
+libnvinfer-headers-plugin-dev=8.6.1.6-1+cuda12.0
 
 # This can be removed once NVIDIA publishes a cuda-12.3.2 Docker image.
 # For now it ensures that we install at least version 12.3.107 of PTXAS,

diff --git a/ci/official/containers/linux_arm64/devel.packages.txt b/ci/official/containers/linux_arm64/devel.packages.txt
@@ -3,10 +3,10 @@ autoconf
 automake
 build-essential
 ca-certificates
-llvm-17
-clang-17
+llvm-18
+clang-18
 clang-format-12
-lld-17
+lld-18
 colordiff
 curl
 ffmpeg
@@ -18,16 +18,16 @@ libcurl3-dev
 libcurl4-openssl-dev
 libfreetype6-dev
 libhdf5-serial-dev
-libomp-17-dev
+libomp-18-dev
 libssl-dev
 libtool
 libxml2-dev
 libxslt1-dev
 libzmq3-dev
 mlocate
 moreutils
-openjdk-11-jdk
-openjdk-11-jre-headless
+openjdk-21-jdk
+openjdk-21-jre-headless
 openssl
 patchelf
 pkg-config

diff --git a/ci/official/containers/linux_arm64/setup.sources.sh b/ci/official/containers/linux_arm64/setup.sources.sh
@@ -36,10 +36,10 @@ apt-key adv --fetch-keys https://apt.llvm.org/llvm-snapshot.gpg.key
 # Set up custom sources
 cat >/etc/apt/sources.list.d/custom.list <<SOURCES
 # More Python versions: Deadsnakes
-deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main
-deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main
+deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
+deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
 
-# LLVM/Clang 17 repository
-deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main
-deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main
+# LLVM/Clang 18 repository
+deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
+deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
 SOURCES
diff --git a/ci/official/envs/ci_default b/ci/official/envs/ci_default
@@ -40,7 +40,8 @@ TFCI_DOCKER_ENABLE=
 TFCI_DOCKER_IMAGE=
 TFCI_DOCKER_PULL_ENABLE=
 TFCI_DOCKER_REBUILD_ARGS=
-TFCI_DOCKER_REBUILD_ENABLE=
+# DO_NOT_SUBMIT - This should be disabled before merge
+TFCI_DOCKER_REBUILD_ENABLE=1
 TFCI_DOCKER_REBUILD_UPLOAD_ENABLE=
 TFCI_GIT_DIR=
 TFCI_INDEX_HTML_ENABLE=

diff --git a/configure.py b/configure.py
@@ -608,9 +608,12 @@ def prompt_loop_or_load_from_env(environ_cp,
 
 def set_clang_cuda_compiler_path(environ_cp):
   """Set CLANG_CUDA_COMPILER_PATH."""
-  default_clang_path = '/usr/lib/llvm-17/bin/clang'
+  # Upon clang 19 drop the check for 16
+  default_clang_path = '/usr/lib/llvm-18/bin/clang'
   if not os.path.exists(default_clang_path):
-    default_clang_path = '/usr/lib/llvm-16/bin/clang'
+    default_clang_path = '/usr/lib/llvm-17/bin/clang'
+    if not os.path.exists(default_clang_path):
+      default_clang_path = '/usr/lib/llvm-16/bin/clang'
     if not os.path.exists(default_clang_path):
       default_clang_path = which('clang') or ''
 
@@ -832,10 +835,13 @@ def set_clang_compiler_path(environ_cp):
   Returns:
     string value for clang_compiler_path.
   """
-  # Default path if clang-16 is installed by using apt-get install
-  default_clang_path = '/usr/lib/llvm-17/bin/clang'
+  # Default path if clang-18 is installed by using apt-get install
+  # remove 16 logic upon release of 19
+  default_clang_path = '/usr/lib/llvm-18/bin/clang'
   if not os.path.exists(default_clang_path):
-    default_clang_path = '/usr/lib/llvm-16/bin/clang'
+    default_clang_path = '/usr/lib/llvm-17/bin/clang'
+    if not os.path.exists(default_clang_path):
+      default_clang_path = '/usr/lib/llvm-16/bin/clang'
     if not os.path.exists(default_clang_path):
       default_clang_path = which('clang') or ''
 

diff --git a/tensorflow/lite/tools/build_aar_with_docker.sh b/tensorflow/lite/tools/build_aar_with_docker.sh
@@ -113,7 +113,7 @@ else
     'N'
     'N'
     'Y'
-    '/usr/lib/llvm-17/bin/clang'
+    '/usr/lib/llvm-18/bin/clang'
     '-Wno-sign-compare -Wno-c++20-designator -Wno-gnu-inline-cpp-without-extern'
     'y'
     '/android/sdk'

diff --git a/..._arm64/builder.patchelf/build_patchelf.sh → ...ools/ci_build/install/install_clang_18.sh b/..._arm64/builder.patchelf/build_patchelf.sh → ...ools/ci_build/install/install_clang_18.sh
@@ -14,15 +14,18 @@
 # limitations under the License.
 # ==============================================================================
 
-# Need a newer version of patchelf as the installed version is buggy in 20.04
-# so get patchelf source from 22.04 ie 'jammy' and build it to avoid dependency
-# problems that would occur with a binary package
+# LLVM/Clang: https://apt.llvm.org/
+apt-key adv --fetch-keys https://apt.llvm.org/llvm-snapshot.gpg.key
 
-mkdir -p /patchelf
-cd /patchelf
-echo deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy universe>>/etc/apt/sources.list
-apt-get update
-apt-get -y build-dep patchelf/jammy
-apt-get -b source patchelf/jammy
+# Set up custom sources
+cat >/etc/apt/sources.list.d/custom.list <<SOURCES
 
-# This will leave a .deb file for installation in a later stage
+# LLVM/Clang repository
+deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
+deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
+SOURCES
+
+apt-get update && apt-get install -y \
+    llvm-18 \
+    clang-18 \
+    lld-18
diff --git a/tensorflow/tools/tf_sig_build_dockerfiles/Dockerfile b/tensorflow/tools/tf_sig_build_dockerfiles/Dockerfile
@@ -29,7 +29,7 @@ COPY devel.packages.txt /devel.packages.txt
 RUN /setup.sources.sh && /setup.packages.sh /devel.packages.txt && /setup.cuda.sh
 
 # Make sure clang is on the path
-RUN ln -s /usr/lib/llvm-17/bin/clang /usr/bin/clang
+RUN ln -s /usr/lib/llvm-18/bin/clang /usr/bin/clang
 
 # Install various tools.
 # - bats: bash unit testing framework

diff --git a/tensorflow/tools/tf_sig_build_dockerfiles/devel.packages.txt b/tensorflow/tools/tf_sig_build_dockerfiles/devel.packages.txt
@@ -38,10 +38,10 @@ autoconf
 automake
 build-essential
 ca-certificates
-llvm-17
-clang-17
-clang-tidy-17
-lld-17
+llvm-18
+clang-18
+clang-tidy-18
+lld-18
 clang-format-12
 colordiff
 curl

diff --git a/tensorflow/tools/tf_sig_build_dockerfiles/setup.sources.sh b/tensorflow/tools/tf_sig_build_dockerfiles/setup.sources.sh
@@ -48,6 +48,6 @@ deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
 deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
 
 # LLVM/Clang repository
-deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
-deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
+deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
+deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
 SOURCES
diff --git a/tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl b/tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl
@@ -74,22 +74,23 @@ def aarch64_compiler_configure():
             "ml2014_clang_aarch64-python3.9": "docker://localhost/tensorflow-build-aarch64:latest-python3.9",
             "ml2014_clang_aarch64-python3.10": "docker://localhost/tensorflow-build-aarch64:latest-python3.10",
             "ml2014_clang_aarch64-python3.11": "docker://localhost/tensorflow-build-aarch64:latest-python3.11",
+            "ml2014_clang_aarch64-python3.12": "docker://localhost/tensorflow-build-aarch64:latest-python3.12",
         },
         env = {
             "ABI_LIBC_VERSION": "glibc_2.17",
             "ABI_VERSION": "gcc",
-            "BAZEL_COMPILER": "/usr/lib/llvm-17/bin/clang",
+            "BAZEL_COMPILER": "/usr/lib/llvm-18/bin/clang",
             "BAZEL_HOST_SYSTEM": "aarch64-unknown-linux-gnu",
             "BAZEL_TARGET_CPU": "generic",
             "BAZEL_TARGET_LIBC": "glibc_2.17",
             "BAZEL_TARGET_SYSTEM": "aarch64-unknown-linux-gnu",
-            "CC": "/usr/lib/llvm-17/bin/clang",
+            "CC": "/usr/lib/llvm-18/bin/clang",
             "CC_TOOLCHAIN_NAME": "linux_llvm_aarch64",
             "CLEAR_CACHE": "1",
             "CUDNN_INSTALL_PATH": "",
-            "CLANG_COMPILER_PATH": "/usr/lib/llvm-17/bin/clang",
-            "HOST_CXX_COMPILER": "/usr/lib/llvm-17/bin/clang",
-            "HOST_C_COMPILER": "/usr/lib/llvm-17/bin/clang",
+            "CLANG_COMPILER_PATH": "/usr/lib/llvm-18/bin/clang",
+            "HOST_CXX_COMPILER": "/usr/lib/llvm-18/bin/clang",
+            "HOST_C_COMPILER": "/usr/lib/llvm-18/bin/clang",
             "PYTHON_BIN_PATH": "/usr/local/bin/python3",
             "TENSORRT_INSTALL_PATH": "",
             "TF_CUDA_CLANG": "0",