[go: nahoru, domu]
Skip to content

Commit

Permalink
Upgrade to support and default to clang 18 for the OSS compiler
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 616889174
  • Loading branch information
@MichaelHudgins @tensorflower-gardener
MichaelHudgins authored and tensorflower-gardener committed Jun 12, 2024
1 parent 1be97ef commit 1582d02
Show file tree
Hide file tree
Showing 32 changed files with 272 additions and 225 deletions.
17 changes: 8 additions & 9 deletions .bazelrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ build:cuda_clang_official --action_env=TF_CUDA_VERSION="12"
build:cuda_clang_official --action_env=TF_CUDNN_VERSION="8"
build:cuda_clang_official --action_env=CUDA_TOOLKIT_PATH="/usr/local/cuda-12.3"
build:cuda_clang_official --action_env=GCC_HOST_COMPILER_PATH="/dt9/usr/bin/gcc"
build:cuda_clang_official --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-17/bin/clang"
build:cuda_clang_official --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-18/bin/clang"
build:cuda_clang_official --action_env=LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64"
build:cuda_clang_official --crosstool_top="@sigbuild-r2.17-clang_config_cuda//crosstool:toolchain"

Expand Down Expand Up @@ -598,15 +598,12 @@ try-import %workspace%/.bazelrc.user
# Build TensorFlow v2.
test:release_base --test_size_filters=small,medium

# Ensure release_base is set on linux
build:release_linux_base --config=release_base

# Target the AVX instruction set
build:release_linux_base --config=avx_linux

# Enable support for all targets
build:release_base --config=cpu_cross

# Ensure release_base is set on linux
build:release_linux_base --config=release_base

# Disable clang extension that rejects type definitions within offsetof.
# This was added in clang-16 by https://reviews.llvm.org/D133574.
# Can be removed once upb is updated, since a type definition is used within
Expand All @@ -629,8 +626,8 @@ build:release_linux_base --action_env PYTHON_BIN_PATH="/usr/bin/python3"
build:release_linux_base --action_env PYTHON_LIB_PATH="/usr/lib/tf_python"
build:release_linux_base --python_path="/usr/bin/python3"
# Set Clang as compiler. Use the actual path to clang installed in container.
build:release_cpu_linux_base --repo_env=CC="/usr/lib/llvm-17/bin/clang"
build:release_cpu_linux_base --repo_env=BAZEL_COMPILER="/usr/lib/llvm-17/bin/clang"
build:release_cpu_linux_base --repo_env=CC="/usr/lib/llvm-18/bin/clang"
build:release_cpu_linux_base --repo_env=BAZEL_COMPILER="/usr/lib/llvm-18/bin/clang"
# Test-related settings below this point.
test:release_linux_base --build_tests_only --keep_going --test_output=errors --verbose_failures=true
test:release_linux_base --local_test_jobs=HOST_CPUS
Expand All @@ -641,6 +638,8 @@ test:release_linux_base --test_summary=short
# Use the Clang toolchain to compile
build:release_cpu_linux --config=release_linux_base
build:release_cpu_linux --crosstool_top="@sigbuild-r2.17-clang_config_cuda//crosstool:toolchain"
# Target the AVX instruction set
build:release_cpu_linux --config=avx_linux

build:release_gpu_linux --config=release_cpu_linux
# Set up compilation CUDA version and paths and use the CUDA Clang toolchain.
Expand Down
12 changes: 6 additions & 6 deletions ci/official/containers/linux_arm64/devel.packages.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ autoconf
automake
build-essential
ca-certificates
llvm-17
clang-17
llvm-18
clang-18
clang-format-12
lld-17
lld-18
colordiff
curl
ffmpeg
Expand All @@ -18,16 +18,16 @@ libcurl3-dev
libcurl4-openssl-dev
libfreetype6-dev
libhdf5-serial-dev
libomp-17-dev
libomp-18-dev
libssl-dev
libtool
libxml2-dev
libxslt1-dev
libzmq3-dev
mlocate
moreutils
openjdk-11-jdk
openjdk-11-jre-headless
openjdk-21-jdk
openjdk-21-jre-headless
openssl
patchelf
pkg-config
Expand Down
6 changes: 3 additions & 3 deletions ci/official/containers/linux_arm64/setup.sources.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ cat >/etc/apt/sources.list.d/custom.list <<SOURCES
deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main
deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main
# LLVM/Clang 17 repository
deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main
deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main
# LLVM/Clang 18 repository
deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
SOURCES
3 changes: 2 additions & 1 deletion ci/official/envs/ci_default
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ TFCI_DOCKER_ENABLE=
TFCI_DOCKER_IMAGE=
TFCI_DOCKER_PULL_ENABLE=
TFCI_DOCKER_REBUILD_ARGS=
TFCI_DOCKER_REBUILD_ENABLE=
# DO_NOT_SUBMIT - This should be disabled before merge
TFCI_DOCKER_REBUILD_ENABLE=1
TFCI_DOCKER_REBUILD_UPLOAD_ENABLE=
TFCI_GIT_DIR=
TFCI_INDEX_HTML_ENABLE=
Expand Down
16 changes: 11 additions & 5 deletions configure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -608,9 +608,12 @@ def prompt_loop_or_load_from_env(environ_cp,

def set_clang_cuda_compiler_path(environ_cp):
"""Set CLANG_CUDA_COMPILER_PATH."""
default_clang_path = '/usr/lib/llvm-17/bin/clang'
# Upon clang 19 drop the check for 16
default_clang_path = '/usr/lib/llvm-18/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
default_clang_path = '/usr/lib/llvm-17/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = which('clang') or ''

Expand Down Expand Up @@ -832,10 +835,13 @@ def set_clang_compiler_path(environ_cp):
Returns:
string value for clang_compiler_path.
"""
# Default path if clang-16 is installed by using apt-get install
default_clang_path = '/usr/lib/llvm-17/bin/clang'
# Default path if clang-18 is installed by using apt-get install
# remove 16 logic upon release of 19
default_clang_path = '/usr/lib/llvm-18/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
default_clang_path = '/usr/lib/llvm-17/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = which('clang') or ''

Expand Down
2 changes: 1 addition & 1 deletion tensorflow/lite/tools/build_aar_with_docker.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ else
'N'
'N'
'Y'
'/usr/lib/llvm-17/bin/clang'
'/usr/lib/llvm-18/bin/clang'
'-Wno-sign-compare -Wno-c++20-designator -Wno-gnu-inline-cpp-without-extern'
'y'
'/android/sdk'
Expand Down
3 changes: 3 additions & 0 deletions ...low/tools/ci_build/Dockerfile.rbe.cuda12.3-cudnn8.9-ubuntu20.04-manylinux2014-multipython
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,6 @@ RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.11" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.12" "jax"

COPY install/install_clang_18.sh /install/
RUN /install/install_clang_18.sh
3 changes: 3 additions & 0 deletions ...low/tools/ci_build/Dockerfile.rbe.cuda12.3-cudnn8.9-ubuntu22.04-manylinux2014-multipython
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,6 @@ RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.11" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.12" "jax"

COPY install/install_clang_18.sh /install/
RUN /install/install_clang_18.sh
3 changes: 3 additions & 0 deletions ...low/tools/ci_build/Dockerfile.rbe.cuda12.3-cudnn9.1-ubuntu20.04-manylinux2014-multipython
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,6 @@ RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.11" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.12" "jax"

COPY install/install_clang_18.sh /install/
RUN /install/install_clang_18.sh
31 changes: 31 additions & 0 deletions tensorflow/tools/ci_build/install/install_clang_18.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash -eu
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

# LLVM/Clang: https://apt.llvm.org/
apt-key adv --fetch-keys https://apt.llvm.org/llvm-snapshot.gpg.key

# Set up custom sources
cat >/etc/apt/sources.list.d/custom.list <<SOURCES
# LLVM/Clang repository
deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
SOURCES

apt-get update && apt-get install -y \
llvm-18 \
clang-18 \
lld-18
2 changes: 1 addition & 1 deletion tensorflow/tools/tf_sig_build_dockerfiles/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ COPY devel.packages.txt /devel.packages.txt
RUN /setup.sources.sh && /setup.packages.sh /devel.packages.txt && /setup.cuda.sh

# Make sure clang is on the path
RUN ln -s /usr/lib/llvm-17/bin/clang /usr/bin/clang
RUN ln -s /usr/lib/llvm-18/bin/clang /usr/bin/clang

# Install various tools.
# - bats: bash unit testing framework
Expand Down
8 changes: 4 additions & 4 deletions tensorflow/tools/tf_sig_build_dockerfiles/devel.packages.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ autoconf
automake
build-essential
ca-certificates
llvm-17
clang-17
clang-tidy-17
lld-17
llvm-18
clang-18
clang-tidy-18
lld-18
clang-format-12
colordiff
curl
Expand Down
4 changes: 2 additions & 2 deletions tensorflow/tools/tf_sig_build_dockerfiles/setup.sources.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,6 @@ deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
# LLVM/Clang repository
deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
SOURCES
11 changes: 6 additions & 5 deletions tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,23 @@ def aarch64_compiler_configure():
"ml2014_clang_aarch64-python3.9": "docker://localhost/tensorflow-build-aarch64:latest-python3.9",
"ml2014_clang_aarch64-python3.10": "docker://localhost/tensorflow-build-aarch64:latest-python3.10",
"ml2014_clang_aarch64-python3.11": "docker://localhost/tensorflow-build-aarch64:latest-python3.11",
"ml2014_clang_aarch64-python3.12": "docker://localhost/tensorflow-build-aarch64:latest-python3.12",
},
env = {
"ABI_LIBC_VERSION": "glibc_2.17",
"ABI_VERSION": "gcc",
"BAZEL_COMPILER": "/usr/lib/llvm-17/bin/clang",
"BAZEL_COMPILER": "/usr/lib/llvm-18/bin/clang",
"BAZEL_HOST_SYSTEM": "aarch64-unknown-linux-gnu",
"BAZEL_TARGET_CPU": "generic",
"BAZEL_TARGET_LIBC": "glibc_2.17",
"BAZEL_TARGET_SYSTEM": "aarch64-unknown-linux-gnu",
"CC": "/usr/lib/llvm-17/bin/clang",
"CC": "/usr/lib/llvm-18/bin/clang",
"CC_TOOLCHAIN_NAME": "linux_llvm_aarch64",
"CLEAR_CACHE": "1",
"CUDNN_INSTALL_PATH": "",
"CLANG_COMPILER_PATH": "/usr/lib/llvm-17/bin/clang",
"HOST_CXX_COMPILER": "/usr/lib/llvm-17/bin/clang",
"HOST_C_COMPILER": "/usr/lib/llvm-17/bin/clang",
"CLANG_COMPILER_PATH": "/usr/lib/llvm-18/bin/clang",
"HOST_CXX_COMPILER": "/usr/lib/llvm-18/bin/clang",
"HOST_C_COMPILER": "/usr/lib/llvm-18/bin/clang",
"PYTHON_BIN_PATH": "/usr/local/bin/python3",
"TENSORRT_INSTALL_PATH": "",
"TF_CUDA_CLANG": "0",
Expand Down
64 changes: 32 additions & 32 deletions tensorflow/tools/toolchains/cross_compile/cc/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,15 @@ cc_toolchain_config(
cpu = "k8",
cxx_builtin_include_directories = [
"/dt9/",
"/usr/lib/llvm-17/include/",
"/usr/lib/llvm-17/lib/clang/17/include",
"/usr/lib/llvm-18/include/",
"/usr/lib/llvm-18/lib/clang/18/include",
],
dbg_compile_flags = ["-g"],
host_system_name = "linux",
link_flags = [
"--target=x86_64-unknown-linux-gnu",
"-fuse-ld=lld",
"--ld-path=/usr/lib/llvm-17/bin/ld.lld",
"--ld-path=/usr/lib/llvm-18/bin/ld.lld",
"-Wl,--undefined-version",
],
link_libs = [
Expand All @@ -90,14 +90,14 @@ cc_toolchain_config(
target_libc = "",
target_system_name = "x86_64-unknown-linux-gnu",
tool_paths = {
"gcc": "/usr/lib/llvm-17/bin/clang",
"ld": "/usr/lib/llvm-17/bin/ld.lld",
"ar": "/usr/lib/llvm-17/bin/llvm-ar",
"cpp": "/usr/lib/llvm-17/bin/clang++",
"llvm-cov": "/usr/lib/llvm-17/bin/llvm-cov",
"nm": "/usr/lib/llvm-17/bin/llvm-nm",
"objdump": "/usr/lib/llvm-17/bin/llvm-objdump",
"strip": "/usr/lib/llvm-17/bin/llvm-strip",
"gcc": "/usr/lib/llvm-18/bin/clang",
"ld": "/usr/lib/llvm-18/bin/ld.lld",
"ar": "/usr/lib/llvm-18/bin/llvm-ar",
"cpp": "/usr/lib/llvm-18/bin/clang++",
"llvm-cov": "/usr/lib/llvm-18/bin/llvm-cov",
"nm": "/usr/lib/llvm-18/bin/llvm-nm",
"objdump": "/usr/lib/llvm-18/bin/llvm-objdump",
"strip": "/usr/lib/llvm-18/bin/llvm-strip",
},
toolchain_identifier = "linux_x86_toolchain",
unfiltered_compile_flags = [
Expand Down Expand Up @@ -148,15 +148,15 @@ cc_toolchain_config(
cpu = "aarch64",
cxx_builtin_include_directories = [
"/dt10/",
"/usr/lib/llvm-17/include/",
"/usr/lib/llvm-17/lib/clang/17/include",
"/usr/lib/llvm-18/include/",
"/usr/lib/llvm-18/lib/clang/18/include",
],
dbg_compile_flags = ["-g"],
host_system_name = "linux",
link_flags = [
"--target=aarch64-unknown-linux-gnu",
"-fuse-ld=lld",
"--ld-path=/usr/lib/llvm-17/bin/ld.lld",
"--ld-path=/usr/lib/llvm-18/bin/ld.lld",
"-Wl,--undefined-version",
],
link_libs = [
Expand All @@ -176,14 +176,14 @@ cc_toolchain_config(
target_libc = "",
target_system_name = "aarch64-unknown-linux-gnu",
tool_paths = {
"gcc": "/usr/lib/llvm-17/bin/clang",
"ld": "/usr/lib/llvm-17/bin/ld.lld",
"ar": "/usr/lib/llvm-17/bin/llvm-ar",
"cpp": "/usr/lib/llvm-17/bin/clang++",
"llvm-cov": "/usr/lib/llvm-17/bin/llvm-cov",
"nm": "/usr/lib/llvm-17/bin/llvm-nm",
"objdump": "/usr/lib/llvm-17/bin/llvm-objdump",
"strip": "/usr/lib/llvm-17/bin/llvm-strip",
"gcc": "/usr/lib/llvm-18/bin/clang",
"ld": "/usr/lib/llvm-18/bin/ld.lld",
"ar": "/usr/lib/llvm-18/bin/llvm-ar",
"cpp": "/usr/lib/llvm-18/bin/clang++",
"llvm-cov": "/usr/lib/llvm-18/bin/llvm-cov",
"nm": "/usr/lib/llvm-18/bin/llvm-nm",
"objdump": "/usr/lib/llvm-18/bin/llvm-objdump",
"strip": "/usr/lib/llvm-18/bin/llvm-strip",
},
toolchain_identifier = "linux_aarch64_toolchain",
unfiltered_compile_flags = [
Expand Down Expand Up @@ -238,8 +238,8 @@ cc_toolchain_config(
cpu = "darwin",
cxx_builtin_include_directories = [
"%sysroot%/usr/include",
"/usr/lib/llvm-17/include/",
"/usr/lib/llvm-17/lib/clang/17/include",
"/usr/lib/llvm-18/include/",
"/usr/lib/llvm-18/lib/clang/18/include",
"%sysroot%/System/Library/Frameworks/Security.framework/Headers",
"%sysroot%/System/Library/Frameworks/CoreFoundation.framework/Headers",
"%sysroot%/System/Library/Frameworks/SystemConfiguration.framework/Headers",
Expand All @@ -250,7 +250,7 @@ cc_toolchain_config(
"--target=x86_64-apple-darwin",
"-lSystem",
"-fuse-ld=lld",
"--ld-path=/usr/lib/llvm-17/bin/ld64.lld",
"--ld-path=/usr/lib/llvm-18/bin/ld64.lld",
"-headerpad_max_install_names",
"-Wl,-undefined,dynamic_lookup",
# Target Catalina as the minimum supported OS
Expand All @@ -274,13 +274,13 @@ cc_toolchain_config(
target_system_name = "x86_64-apple-macosx10.15",
tool_paths = {
"gcc": "cc_wrapper.sh",
"ld": "/usr/lib/llvm-17/bin/ld64.lld",
"ar": "/usr/lib/llvm-17/bin/llvm-libtool-darwin",
"cpp": "/usr/lib/llvm-17/bin/clang++",
"llvm-cov": "/usr/lib/llvm-17/bin/llvm-cov",
"nm": "/usr/lib/llvm-17/bin/llvm-nm",
"objdump": "/usr/lib/llvm-17/bin/llvm-objdump",
"strip": "/usr/lib/llvm-17/bin/llvm-strip",
"ld": "/usr/lib/llvm-18/bin/ld64.lld",
"ar": "/usr/lib/llvm-18/bin/llvm-libtool-darwin",
"cpp": "/usr/lib/llvm-18/bin/clang++",
"llvm-cov": "/usr/lib/llvm-18/bin/llvm-cov",
"nm": "/usr/lib/llvm-18/bin/llvm-nm",
"objdump": "/usr/lib/llvm-18/bin/llvm-objdump",
"strip": "/usr/lib/llvm-18/bin/llvm-strip",
},
toolchain_identifier = "macos_x86_toolchain",
unfiltered_compile_flags = [
Expand Down
Loading

0 comments on commit 1582d02

Please sign in to comment.