[go: nahoru, domu]
Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Upgrade to support and default to clang 18 for the OSS compiler #65084

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Upgrade to support and default to clang 18 for the OSS compiler #65084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Upgrade to support and default to clang 18 for the OSS compiler 
PiperOrigin-RevId: 616889174
  • Loading branch information
@MichaelHudgins @tensorflower-gardener
MichaelHudgins authored and tensorflower-gardener committed Jun 12, 2024
commit 1582d029735203d0eb8cd967b4cebe9cc951de1b
17 changes: 8 additions & 9 deletions .bazelrc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ build:cuda_clang_official --action_env=TF_CUDA_VERSION="12"
build:cuda_clang_official --action_env=TF_CUDNN_VERSION="8"
build:cuda_clang_official --action_env=CUDA_TOOLKIT_PATH="/usr/local/cuda-12.3"
build:cuda_clang_official --action_env=GCC_HOST_COMPILER_PATH="/dt9/usr/bin/gcc"
build:cuda_clang_official --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-17/bin/clang"
build:cuda_clang_official --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-18/bin/clang"
build:cuda_clang_official --action_env=LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64"
build:cuda_clang_official --crosstool_top="@sigbuild-r2.17-clang_config_cuda//crosstool:toolchain"

Expand Down Expand Up @@ -598,15 +598,12 @@ try-import %workspace%/.bazelrc.user
# Build TensorFlow v2.
test:release_base --test_size_filters=small,medium

# Ensure release_base is set on linux
build:release_linux_base --config=release_base

# Target the AVX instruction set
build:release_linux_base --config=avx_linux

# Enable support for all targets
build:release_base --config=cpu_cross

# Ensure release_base is set on linux
build:release_linux_base --config=release_base

# Disable clang extension that rejects type definitions within offsetof.
# This was added in clang-16 by https://reviews.llvm.org/D133574.
# Can be removed once upb is updated, since a type definition is used within
Expand All @@ -629,8 +626,8 @@ build:release_linux_base --action_env PYTHON_BIN_PATH="/usr/bin/python3"
build:release_linux_base --action_env PYTHON_LIB_PATH="/usr/lib/tf_python"
build:release_linux_base --python_path="/usr/bin/python3"
# Set Clang as compiler. Use the actual path to clang installed in container.
build:release_cpu_linux_base --repo_env=CC="/usr/lib/llvm-17/bin/clang"
build:release_cpu_linux_base --repo_env=BAZEL_COMPILER="/usr/lib/llvm-17/bin/clang"
build:release_cpu_linux_base --repo_env=CC="/usr/lib/llvm-18/bin/clang"
build:release_cpu_linux_base --repo_env=BAZEL_COMPILER="/usr/lib/llvm-18/bin/clang"
# Test-related settings below this point.
test:release_linux_base --build_tests_only --keep_going --test_output=errors --verbose_failures=true
test:release_linux_base --local_test_jobs=HOST_CPUS
Expand All @@ -641,6 +638,8 @@ test:release_linux_base --test_summary=short
# Use the Clang toolchain to compile
build:release_cpu_linux --config=release_linux_base
build:release_cpu_linux --crosstool_top="@sigbuild-r2.17-clang_config_cuda//crosstool:toolchain"
# Target the AVX instruction set
build:release_cpu_linux --config=avx_linux

build:release_gpu_linux --config=release_cpu_linux
# Set up compilation CUDA version and paths and use the CUDA Clang toolchain.
Expand Down
12 changes: 6 additions & 6 deletions ci/official/containers/linux_arm64/devel.packages.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ autoconf
automake
build-essential
ca-certificates
llvm-17
clang-17
llvm-18
clang-18
clang-format-12
lld-17
lld-18
colordiff
curl
ffmpeg
Expand All @@ -18,16 +18,16 @@ libcurl3-dev
libcurl4-openssl-dev
libfreetype6-dev
libhdf5-serial-dev
libomp-17-dev
libomp-18-dev
libssl-dev
libtool
libxml2-dev
libxslt1-dev
libzmq3-dev
mlocate
moreutils
openjdk-11-jdk
openjdk-11-jre-headless
openjdk-21-jdk
openjdk-21-jre-headless
openssl
patchelf
pkg-config
Expand Down
6 changes: 3 additions & 3 deletions ci/official/containers/linux_arm64/setup.sources.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ cat >/etc/apt/sources.list.d/custom.list <<SOURCES
deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main
deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main

# LLVM/Clang 17 repository
deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main
deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main
# LLVM/Clang 18 repository
deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
SOURCES
3 changes: 2 additions & 1 deletion ci/official/envs/ci_default
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ TFCI_DOCKER_ENABLE=
TFCI_DOCKER_IMAGE=
TFCI_DOCKER_PULL_ENABLE=
TFCI_DOCKER_REBUILD_ARGS=
TFCI_DOCKER_REBUILD_ENABLE=
# DO_NOT_SUBMIT - This should be disabled before merge
TFCI_DOCKER_REBUILD_ENABLE=1
TFCI_DOCKER_REBUILD_UPLOAD_ENABLE=
TFCI_GIT_DIR=
TFCI_INDEX_HTML_ENABLE=
Expand Down
16 changes: 11 additions & 5 deletions configure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,9 +608,12 @@ def prompt_loop_or_load_from_env(environ_cp,

def set_clang_cuda_compiler_path(environ_cp):
"""Set CLANG_CUDA_COMPILER_PATH."""
default_clang_path = '/usr/lib/llvm-17/bin/clang'
# Upon clang 19 drop the check for 16
default_clang_path = '/usr/lib/llvm-18/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
default_clang_path = '/usr/lib/llvm-17/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = which('clang') or ''

Expand Down Expand Up @@ -832,10 +835,13 @@ def set_clang_compiler_path(environ_cp):
Returns:
string value for clang_compiler_path.
"""
# Default path if clang-16 is installed by using apt-get install
default_clang_path = '/usr/lib/llvm-17/bin/clang'
# Default path if clang-18 is installed by using apt-get install
# remove 16 logic upon release of 19
default_clang_path = '/usr/lib/llvm-18/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
default_clang_path = '/usr/lib/llvm-17/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = '/usr/lib/llvm-16/bin/clang'
if not os.path.exists(default_clang_path):
default_clang_path = which('clang') or ''

Expand Down
2 changes: 1 addition & 1 deletion tensorflow/lite/tools/build_aar_with_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ else
'N'
'N'
'Y'
'/usr/lib/llvm-17/bin/clang'
'/usr/lib/llvm-18/bin/clang'
'-Wno-sign-compare -Wno-c++20-designator -Wno-gnu-inline-cpp-without-extern'
'y'
'/android/sdk'
Expand Down
3 changes: 3 additions & 0 deletions ...low/tools/ci_build/Dockerfile.rbe.cuda12.3-cudnn8.9-ubuntu20.04-manylinux2014-multipython
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,6 @@ RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.11" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.12" "jax"

COPY install/install_clang_18.sh /install/
RUN /install/install_clang_18.sh
3 changes: 3 additions & 0 deletions ...low/tools/ci_build/Dockerfile.rbe.cuda12.3-cudnn8.9-ubuntu22.04-manylinux2014-multipython
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,6 @@ RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.11" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.12" "jax"

COPY install/install_clang_18.sh /install/
RUN /install/install_clang_18.sh
3 changes: 3 additions & 0 deletions ...low/tools/ci_build/Dockerfile.rbe.cuda12.3-cudnn9.1-ubuntu20.04-manylinux2014-multipython
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,6 @@ RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.11" "jax"
RUN SETUPTOOLS_USE_DISTUTILS=stdlib /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.12" "jax"

COPY install/install_clang_18.sh /install/
RUN /install/install_clang_18.sh
31 changes: 31 additions & 0 deletions tensorflow/tools/ci_build/install/install_clang_18.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash -eu
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

# LLVM/Clang: https://apt.llvm.org/
apt-key adv --fetch-keys https://apt.llvm.org/llvm-snapshot.gpg.key

# Set up custom sources
cat >/etc/apt/sources.list.d/custom.list <<SOURCES

# LLVM/Clang repository
deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
SOURCES

apt-get update && apt-get install -y \
llvm-18 \
clang-18 \
lld-18
2 changes: 1 addition & 1 deletion tensorflow/tools/tf_sig_build_dockerfiles/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ COPY devel.packages.txt /devel.packages.txt
RUN /setup.sources.sh && /setup.packages.sh /devel.packages.txt && /setup.cuda.sh

# Make sure clang is on the path
RUN ln -s /usr/lib/llvm-17/bin/clang /usr/bin/clang
RUN ln -s /usr/lib/llvm-18/bin/clang /usr/bin/clang

# Install various tools.
# - bats: bash unit testing framework
Expand Down
8 changes: 4 additions & 4 deletions tensorflow/tools/tf_sig_build_dockerfiles/devel.packages.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ autoconf
automake
build-essential
ca-certificates
llvm-17
clang-17
clang-tidy-17
lld-17
llvm-18
clang-18
clang-tidy-18
lld-18
clang-format-12
colordiff
curl
Expand Down
4 changes: 2 additions & 2 deletions tensorflow/tools/tf_sig_build_dockerfiles/setup.sources.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,6 @@ deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main

# LLVM/Clang repository
deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
SOURCES
11 changes: 6 additions & 5 deletions tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,23 @@ def aarch64_compiler_configure():
"ml2014_clang_aarch64-python3.9": "docker://localhost/tensorflow-build-aarch64:latest-python3.9",
"ml2014_clang_aarch64-python3.10": "docker://localhost/tensorflow-build-aarch64:latest-python3.10",
"ml2014_clang_aarch64-python3.11": "docker://localhost/tensorflow-build-aarch64:latest-python3.11",
"ml2014_clang_aarch64-python3.12": "docker://localhost/tensorflow-build-aarch64:latest-python3.12",
},
env = {
"ABI_LIBC_VERSION": "glibc_2.17",
"ABI_VERSION": "gcc",
"BAZEL_COMPILER": "/usr/lib/llvm-17/bin/clang",
"BAZEL_COMPILER": "/usr/lib/llvm-18/bin/clang",
"BAZEL_HOST_SYSTEM": "aarch64-unknown-linux-gnu",
"BAZEL_TARGET_CPU": "generic",
"BAZEL_TARGET_LIBC": "glibc_2.17",
"BAZEL_TARGET_SYSTEM": "aarch64-unknown-linux-gnu",
"CC": "/usr/lib/llvm-17/bin/clang",
"CC": "/usr/lib/llvm-18/bin/clang",
"CC_TOOLCHAIN_NAME": "linux_llvm_aarch64",
"CLEAR_CACHE": "1",
"CUDNN_INSTALL_PATH": "",
"CLANG_COMPILER_PATH": "/usr/lib/llvm-17/bin/clang",
"HOST_CXX_COMPILER": "/usr/lib/llvm-17/bin/clang",
"HOST_C_COMPILER": "/usr/lib/llvm-17/bin/clang",
"CLANG_COMPILER_PATH": "/usr/lib/llvm-18/bin/clang",
"HOST_CXX_COMPILER": "/usr/lib/llvm-18/bin/clang",
"HOST_C_COMPILER": "/usr/lib/llvm-18/bin/clang",
"PYTHON_BIN_PATH": "/usr/local/bin/python3",
"TENSORRT_INSTALL_PATH": "",
"TF_CUDA_CLANG": "0",
Expand Down
64 changes: 32 additions & 32 deletions tensorflow/tools/toolchains/cross_compile/cc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,15 @@ cc_toolchain_config(
cpu = "k8",
cxx_builtin_include_directories = [
"/dt9/",
"/usr/lib/llvm-17/include/",
"/usr/lib/llvm-17/lib/clang/17/include",
"/usr/lib/llvm-18/include/",
"/usr/lib/llvm-18/lib/clang/18/include",
],
dbg_compile_flags = ["-g"],
host_system_name = "linux",
link_flags = [
"--target=x86_64-unknown-linux-gnu",
"-fuse-ld=lld",
"--ld-path=/usr/lib/llvm-17/bin/ld.lld",
"--ld-path=/usr/lib/llvm-18/bin/ld.lld",
"-Wl,--undefined-version",
],
link_libs = [
Expand All @@ -90,14 +90,14 @@ cc_toolchain_config(
target_libc = "",
target_system_name = "x86_64-unknown-linux-gnu",
tool_paths = {
"gcc": "/usr/lib/llvm-17/bin/clang",
"ld": "/usr/lib/llvm-17/bin/ld.lld",
"ar": "/usr/lib/llvm-17/bin/llvm-ar",
"cpp": "/usr/lib/llvm-17/bin/clang++",
"llvm-cov": "/usr/lib/llvm-17/bin/llvm-cov",
"nm": "/usr/lib/llvm-17/bin/llvm-nm",
"objdump": "/usr/lib/llvm-17/bin/llvm-objdump",
"strip": "/usr/lib/llvm-17/bin/llvm-strip",
"gcc": "/usr/lib/llvm-18/bin/clang",
"ld": "/usr/lib/llvm-18/bin/ld.lld",
"ar": "/usr/lib/llvm-18/bin/llvm-ar",
"cpp": "/usr/lib/llvm-18/bin/clang++",
"llvm-cov": "/usr/lib/llvm-18/bin/llvm-cov",
"nm": "/usr/lib/llvm-18/bin/llvm-nm",
"objdump": "/usr/lib/llvm-18/bin/llvm-objdump",
"strip": "/usr/lib/llvm-18/bin/llvm-strip",
},
toolchain_identifier = "linux_x86_toolchain",
unfiltered_compile_flags = [
Expand Down Expand Up @@ -148,15 +148,15 @@ cc_toolchain_config(
cpu = "aarch64",
cxx_builtin_include_directories = [
"/dt10/",
"/usr/lib/llvm-17/include/",
"/usr/lib/llvm-17/lib/clang/17/include",
"/usr/lib/llvm-18/include/",
"/usr/lib/llvm-18/lib/clang/18/include",
],
dbg_compile_flags = ["-g"],
host_system_name = "linux",
link_flags = [
"--target=aarch64-unknown-linux-gnu",
"-fuse-ld=lld",
"--ld-path=/usr/lib/llvm-17/bin/ld.lld",
"--ld-path=/usr/lib/llvm-18/bin/ld.lld",
"-Wl,--undefined-version",
],
link_libs = [
Expand All @@ -176,14 +176,14 @@ cc_toolchain_config(
target_libc = "",
target_system_name = "aarch64-unknown-linux-gnu",
tool_paths = {
"gcc": "/usr/lib/llvm-17/bin/clang",
"ld": "/usr/lib/llvm-17/bin/ld.lld",
"ar": "/usr/lib/llvm-17/bin/llvm-ar",
"cpp": "/usr/lib/llvm-17/bin/clang++",
"llvm-cov": "/usr/lib/llvm-17/bin/llvm-cov",
"nm": "/usr/lib/llvm-17/bin/llvm-nm",
"objdump": "/usr/lib/llvm-17/bin/llvm-objdump",
"strip": "/usr/lib/llvm-17/bin/llvm-strip",
"gcc": "/usr/lib/llvm-18/bin/clang",
"ld": "/usr/lib/llvm-18/bin/ld.lld",
"ar": "/usr/lib/llvm-18/bin/llvm-ar",
"cpp": "/usr/lib/llvm-18/bin/clang++",
"llvm-cov": "/usr/lib/llvm-18/bin/llvm-cov",
"nm": "/usr/lib/llvm-18/bin/llvm-nm",
"objdump": "/usr/lib/llvm-18/bin/llvm-objdump",
"strip": "/usr/lib/llvm-18/bin/llvm-strip",
},
toolchain_identifier = "linux_aarch64_toolchain",
unfiltered_compile_flags = [
Expand Down Expand Up @@ -238,8 +238,8 @@ cc_toolchain_config(
cpu = "darwin",
cxx_builtin_include_directories = [
"%sysroot%/usr/include",
"/usr/lib/llvm-17/include/",
"/usr/lib/llvm-17/lib/clang/17/include",
"/usr/lib/llvm-18/include/",
"/usr/lib/llvm-18/lib/clang/18/include",
"%sysroot%/System/Library/Frameworks/Security.framework/Headers",
"%sysroot%/System/Library/Frameworks/CoreFoundation.framework/Headers",
"%sysroot%/System/Library/Frameworks/SystemConfiguration.framework/Headers",
Expand All @@ -250,7 +250,7 @@ cc_toolchain_config(
"--target=x86_64-apple-darwin",
"-lSystem",
"-fuse-ld=lld",
"--ld-path=/usr/lib/llvm-17/bin/ld64.lld",
"--ld-path=/usr/lib/llvm-18/bin/ld64.lld",
"-headerpad_max_install_names",
"-Wl,-undefined,dynamic_lookup",
# Target Catalina as the minimum supported OS
Expand All @@ -274,13 +274,13 @@ cc_toolchain_config(
target_system_name = "x86_64-apple-macosx10.15",
tool_paths = {
"gcc": "cc_wrapper.sh",
"ld": "/usr/lib/llvm-17/bin/ld64.lld",
"ar": "/usr/lib/llvm-17/bin/llvm-libtool-darwin",
"cpp": "/usr/lib/llvm-17/bin/clang++",
"llvm-cov": "/usr/lib/llvm-17/bin/llvm-cov",
"nm": "/usr/lib/llvm-17/bin/llvm-nm",
"objdump": "/usr/lib/llvm-17/bin/llvm-objdump",
"strip": "/usr/lib/llvm-17/bin/llvm-strip",
"ld": "/usr/lib/llvm-18/bin/ld64.lld",
"ar": "/usr/lib/llvm-18/bin/llvm-libtool-darwin",
"cpp": "/usr/lib/llvm-18/bin/clang++",
"llvm-cov": "/usr/lib/llvm-18/bin/llvm-cov",
"nm": "/usr/lib/llvm-18/bin/llvm-nm",
"objdump": "/usr/lib/llvm-18/bin/llvm-objdump",
"strip": "/usr/lib/llvm-18/bin/llvm-strip",
},
toolchain_identifier = "macos_x86_toolchain",
unfiltered_compile_flags = [
Expand Down
Loading
Loading