use hermetic Python in Tensorflow.

See ci/official/requirements_updater/README.md for details. PiperOrigin-RevId: 546059481
tensorflow · Jul 6, 2023 · e85860e · e85860e
1 parent 2c0af64
commit e85860e
Show file tree

Hide file tree

Showing 65 changed files with 3,938 additions and 330 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -0,0 +1,17 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# The requirements updater has its own WORKSPACE file
+ci/official/requirements_updater
diff --git a/WORKSPACE b/WORKSPACE
@@ -1,5 +1,70 @@
 workspace(name = "org_tensorflow")
 
+# We must initialize hermetic python first.
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "rules_python",
+    sha256 = "29a801171f7ca190c543406f9894abf2d483c206e14d6acbd695623662320097",
+    strip_prefix = "rules_python-0.18.1",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.18.1/rules_python-0.18.1.tar.gz",
+)
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+load(
+    "//tensorflow/tools/toolchains/python:python_repo.bzl",
+    "python_repository",
+)
+
+python_repository(name = "python_version_repo")
+
+load("@python_version_repo//:py_version.bzl", "HERMETIC_PYTHON_VERSION")
+
+python_register_toolchains(
+    name = "python",
+    ignore_root_user_error = True,
+    python_version = HERMETIC_PYTHON_VERSION,
+)
+
+load("@python//:defs.bzl", "interpreter")
+load("@rules_python//python:pip.bzl", "package_annotation", "pip_parse")
+
+NUMPY_ANNOTATIONS = {
+    "numpy": package_annotation(
+        additive_build_content = """\
+filegroup(
+    name = "includes",
+    srcs = glob(["site-packages/numpy/core/include/**/*.h"]),
+)
+cc_library(
+    name = "numpy_headers",
+    hdrs = [":includes"],
+    strip_include_prefix="site-packages/numpy/core/include/",
+)
+""",
+    ),
+}
+
+pip_parse(
+    name = "pypi",
+    annotations = NUMPY_ANNOTATIONS,
+    python_interpreter_target = interpreter,
+    requirements = "//:requirements_lock_" + HERMETIC_PYTHON_VERSION.replace(".", "_") + ".txt",
+)
+
+load("@pypi//:requirements.bzl", "install_deps")
+
+install_deps()
+
 # Initialize the TensorFlow repository and all dependencies.
 #
 # The cascade of load() statements and tf_workspace?() calls works around the

diff --git a/ci/official/requirements_updater/BUILD.bazel b/ci/official/requirements_updater/BUILD.bazel
@@ -0,0 +1,47 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+load("@python//3.8:defs.bzl", compile_pip_requirements_3_8 = "compile_pip_requirements")
+load("@python//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements")
+load("@python//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements")
+load("@python//3.11:defs.bzl", compile_pip_requirements_3_11 = "compile_pip_requirements")
+
+compile_pip_requirements_3_8(
+    name = "requirements_3_8",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_8.txt",
+)
+
+compile_pip_requirements_3_9(
+    name = "requirements_3_9",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_9.txt",
+)
+
+compile_pip_requirements_3_10(
+    name = "requirements_3_10",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_10.txt",
+)
+
+compile_pip_requirements_3_11(
+    name = "requirements_3_11",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_11.txt",
+)
diff --git a/ci/official/requirements_updater/README.md b/ci/official/requirements_updater/README.md
@@ -0,0 +1,64 @@
+### Hermetic Python
+
+Hermetic Python allows us not to rely on system-installed python and
+system-installed python packages, instead we register our own python toolchain.
+See https://github.com/bazelbuild/rules_python/ for more details.
+
+#### Hermetic Python toolchain details
+
+To set a version for hermetic Python toolchain, use `TF_PYTHON_VERSION`
+environment variable, e.g.
+
+```
+export TF_PYTHON_VERSION=3.10
+```
+
+To set a version from argument line, add to your command
+
+```
+--repo_env=TF_PYTHON_VERSION=3.10
+```
+
+### Requirements updater
+
+Requirements updater is a standalone tool intended to simplify process of
+updating requirements for multiple versions of Python.
+
+#### How to update/add requirements
+
+Add or update requirements in `requirements.in` file, then run the updater.
+
+#### How to run the updater
+
+```
+bash updater.sh
+```
+
+### How to add a new Python version
+
+1) In the `WORKSPACE` file add a new version to `python_versions` argument of
+the `python_register_multi_toolchains` function.
+
+2) In `BUILD.bazel` file add a load statement for the new version, e.g.
+
+```
+load("@python//3.11:defs.bzl",
+     compile_pip_requirements_3_11 = "compile_pip_requirements")
+```
+
+Add a new entry for the loaded `compile_pip_requirements`, e.g.
+
+```
+compile_pip_requirements_3_11(
+    name = "requirements_3_11",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_11.txt",
+)
+```
+
+3) Add the version to `SUPPORTED_VERSIONS` in `updater.sh`, after that run the
+ requirements updater tool.
+
+4) As a result, a new `requirements_lock_3_11.txt` file should appear under the
+root of tensorflow directory.
diff --git a/ci/official/requirements_updater/WORKSPACE b/ci/official/requirements_updater/WORKSPACE
@@ -0,0 +1,39 @@
+workspace(name = "requirements_updater")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "rules_python",
+    sha256 = "29a801171f7ca190c543406f9894abf2d483c206e14d6acbd695623662320097",
+    strip_prefix = "rules_python-0.18.1",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.18.1/rules_python-0.18.1.tar.gz",
+)
+
+load("@rules_python//python:repositories.bzl", "python_register_multi_toolchains")
+load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
+
+default_python_version = "3.10"
+
+python_register_multi_toolchains(
+    name = "python",
+    default_version = default_python_version,
+    ignore_root_user_error = True,
+    python_versions = [
+        "3.8",
+        "3.9",
+        "3.10",
+        "3.11",
+    ],
+    register_coverage_tool = True,
+)
+
+pip_install_dependencies()
diff --git a/ci/official/requirements_updater/requirements.in b/ci/official/requirements_updater/requirements.in
@@ -0,0 +1,21 @@
+numpy >= 1.22
+wheel ~= 0.38.1
+h5py >= 2.9.0
+lit ~= 16.0.5.post0
+
+# Install tensorboard and estimator and keras
+# Note that here we want the latest version that matches TF major.minor version
+# Note that we must use nightly here as these are used in nightly jobs
+# For release jobs, we will pin these on the release branch
+keras-nightly ~= 2.14.0.dev
+tb-nightly ~= 2.14.0.a
+tf-estimator-nightly ~= 2.14.0.dev
+
+# Test dependencies
+grpcio >= 1.24.3, < 2.0
+portpicker == 1.5.2
+scipy == 1.10.1
+requests == 2.31.0
+packaging==23.1
+setuptools==67.6.1
+jax==0.4.7
diff --git a/ci/official/requirements_updater/updater.sh b/ci/official/requirements_updater/updater.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# script to run pip-compile for each requirement.
+# if there is a change in requirements.in then all lock files will be updated
+# accordingly
+
+
+mv BUILD.bazel BUILD
+
+SUPPORTED_VERSIONS=("3_8" "3_9" "3_10" "3_11")
+
+for VERSION in "${SUPPORTED_VERSIONS[@]}"
+do
+  touch "requirements_lock_$VERSION.txt"
+  bazel run --experimental_convenience_symlinks=ignore //:requirements_"$VERSION".update
+  sed -i '/^#/d' requirements_lock_"$VERSION".txt
+  mv requirements_lock_"$VERSION".txt ../../../requirements_lock_"$VERSION".txt
+done
+
+mv BUILD BUILD.bazel