build: update protobuf to 3.19.6 and grpc to 1.48.2 (#6147)

Fixes #5708 and #5703. This updates our protobuf dependency to 3.19.6 in an attempt to address #5708, and provide a cleaner solution to #5703. The choice of 3.19.6 is meant to satisfy two competing constraints: - Current Python protobuf runtimes (the 4.x series) only support generated code from protoc versions 3.19.0+, as discussed in https://protobuf.dev/news/2022-05-06/. As a result, prior to this change, TensorBoard's pip package had to force its pip package dependency to `protobuf < 4` to avoid the errors seen in #5703. This PR lifts that restriction. - Current TensorFlow is still stuck on protobuf 3.x, the same as we have been, and as a result pins its pip package dependency using `protobuf < 3.20` (this could presumably be relaxed to `< 4` but that would require new TF releases). As a result, we must support at least one protobuf runtime version that also works with TF's constraints. Our previous attempt at this upgrade (to ~3.18 or so) caused test failures for Keras (which depends on TB, via TF, for the summary API code), apparently due to a protobuf runtime that was too old for our generated code. At the time, this was puzzling because they were pip-installing a protobuf runtime version that should have been recent enough - but I suspect now that this was a red herring, and bazel test was actually getting the protobuf runtime from the protobuf build dependency, not from the installed Python packages. If we see this failure mode again, we'll have to get Keras to update the protobuf Python runtime available in bazel tests. Lastly, this upgrade lets us clean up some additional issues we had to work around: - We can also upgrade gRPC now, to 1.48.2. I selected this version since it appears to be the most recent version prior to gRPC adopting protobuf 4.x (see grpc/grpc@41ec08c) - We can revert the backported fixes to protobuf and grpc from #5793, since the upgraded dependencies don't require patching - We can back out rules_apple reintroduction from #5561 that we only needed for gRPC
tensorflow · Jan 19, 2023 · 78f97c2 · 78f97c2
1 parent 15aa812
commit 78f97c2
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 125 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -2,15 +2,6 @@ workspace(name = "org_tensorflow_tensorboard")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-http_archive(
-    name = "build_bazel_rules_apple",
-    sha256 = "0052d452af7742c8f3a4e0929763388a66403de363775db7e90adecb2ba4944b",
-    urls = [
-        "http://mirror.tensorflow.org/github.com/bazelbuild/rules_apple/releases/download/0.31.3/rules_apple.0.31.3.tar.gz",
-        "https://github.com/bazelbuild/rules_apple/releases/download/0.31.3/rules_apple.0.31.3.tar.gz",
-    ],
-)
-
 http_archive(
     name = "bazel_skylib",
     sha256 = "07b4117379dde7ab382345c3b0f5edfc6b7cff6c93756eac63da121e0bbcc5de",
@@ -121,74 +112,43 @@ load("@io_bazel_rules_sass//:defs.bzl", "sass_repositories")
 
 sass_repositories()
 
-# Always bump the requirements.txt protobuf dep to be >= the version here.
-# Keep this version to be in sync with TensorFlow:
-# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/workspace2.bzl#L446
-# For other projects (e.g. Keras) depending on tb-nightly, generating protobuf code at
-# a more recent version than the protobuf runtime supplied by TF's bazel build tooling
-# might lead to test failures.
+# This dependency specifies the version of protobuf that will be used to compile
+# protos as part of TensorBoard's build (i.e., the protoc version).
+#
+# This version must always be <= the protobuf runtime version, which is the version of
+# the "protobuf" pip package as specified in our requirements.txt file.
+#
+# NOTE: This dependency currently cannot be advanced past 3.19.x. This is because
+# TF is currently unable to use a runtime any greater than 3.19.x, see details here:
+# https://github.com/tensorflow/tensorflow/blob/9d22f4a0a9499c8e10a4312503e63e0da35ccd94/tensorflow/tools/pip_package/setup.py#L100-L107
+#
+# As a result of TF's constraint and the above <= requirement, 3.19.x is the most recent
+# possible protoc we can use while remaining cross-compatible with TF. At the same time,
+# 3.19.x is the minimum possible protoc that will generate compiled proto code that *is*
+# compatible with protobuf runtimes >= 4, as discussed here:
+# https://developers.google.com/protocol-buffers/docs/news/2022-05-06
 http_archive(
     name = "com_google_protobuf",
-    patch_args = ["-p1"],
-    patches = [
-        # To maintain compatibility with python 3.10 and greater, we need to patch
-        # in the following protobuf change:
-        # https://github.com/grpc/grpc/commit/9d61eada0f47d7be793983638c4a29707b192d0c
-        #
-        # To reproduce the patch:
-        # ```
-        # $ git clone https://github.com/protocolbuffers/protobuf.git
-        # $ cd protobuf
-        # $ git checkout tags/v3.9.2 -b my-patch
-        # $ git cherry-pick 9d61eada0f47d7be793983638c4a29707b192d0c
-        # $ git diff HEAD~1 > protobuf.patch
-        # # Remove trailing whitespace to satisify whitespace_hygiene_test.py.
-        # $ sed -i 's/[[:space:]]*$//' protobuf.patch
-        # ```
-        "//third_party:protobuf.patch",
-    ],
-    sha256 = "1fbf1c2962af287607232b2eddeaec9b4f4a7a6f5934e1a9276e9af76952f7e0",
-    strip_prefix = "protobuf-3.9.2",
+    sha256 = "9a301cf94a8ddcb380b901e7aac852780b826595075577bb967004050c835056",
+    strip_prefix = "protobuf-3.19.6",
     urls = [
-        "http://mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.tar.gz",
-        "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.tar.gz",  # 2019-09-23
+        "http://mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.19.6.tar.gz",
+        "https://github.com/protocolbuffers/protobuf/archive/v3.19.6.tar.gz",  # 2022-09-29
     ],
 )
 
 # gRPC.
+#
+# NOTE: The version used here must be cross-compatible with our protobuf version.
+# As 2023-01-13, 1.48.2 is the most recent gRPC release that was still using a 3.19.x
+# version of protobuf in its own builds (more recent releases move to 3.21.x).
 http_archive(
     name = "com_github_grpc_grpc",
-    patch_args = ["-p1"],
-    patches = [
-        # To maintain compatibility with python 3.10 and greater, we need to patch
-        # in the following grpc change:
-        # https://github.com/grpc/grpc/commit/dbe73c9004e483d24168c220cd589fe1824e72bc
-        #
-        # To reproduce the patch:
-        # ```
-        # $ git clone https://github.com/grpc/grpc.git
-        # $ cd grpc
-        # $ git checkout b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd -b my-patch
-        # $ git cherry-pick dbe73c9004e483d24168c220cd589fe1824e72bc
-        # $ git diff HEAD~1 > grpc.patch
-        # ```
-        #
-        # Note that we choose b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd as the
-        # base since it matches the archive (the commit number is in the archive
-        # file's name). There is no exact corresponding tag to use but the
-        # nearest might be v1.27.0-pre1.
-        "//third_party:grpc.patch",
-    ],
-    sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f",
-    strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd",
+    sha256 = "bdb8e98145469d58c69ab9f2c9e0bd838c2836a99b5760bc0ebf658623768f52",
+    strip_prefix = "grpc-1.48.2",
     urls = [
-        # Same as TF: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/workspace2.bzl#L492
-        # Currently we can't upgrade gRPC past 1.30.0 without also bumping protobuf to 3.12.0+:
-        # https://github.com/grpc/grpc/issues/23311.
-        #
-        # Inspecting the contents of this archive, the version is v1.27.0-dev.
-        "http://mirror.tensorflow.org/github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz",
-        "https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz",
+        "http://mirror.tensorflow.org/github.com/grpc/grpc/archive/v1.48.2.tar.gz",
+        "https://github.com/grpc/grpc/archive/v1.48.2.tar.gz",  # 2022-09-21
     ],
 )
 

diff --git a/tensorboard/data/server/descriptor.bin b/tensorboard/data/server/descriptor.bin
diff --git a/tensorboard/pip_package/requirements.txt b/tensorboard/pip_package/requirements.txt
@@ -16,16 +16,17 @@
 # Non-vendored runtime dependencies of TensorBoard.
 
 absl-py >= 0.4
-grpcio >= 1.24.3
+# NOTE: this version should be >= the grpc version in our WORKSPACE file.
+grpcio >= 1.48.2
 google-auth >= 1.6.3, < 3
 google-auth-oauthlib >= 0.4.1, < 0.5
 markdown >= 2.6.8
 numpy >= 1.12.0
-# Protobuf 4.0 is incompatible with TF. Force < 4 until they unblock upgrade.
-# See: http://b/182876485
-# See: https://github.com/protocolbuffers/protobuf/issues/9954#issuecomment-1128283911
-# See: https://cs.opensource.google/tensorflow/tensorflow/+/master:tensorflow/tools/pip_package/setup.py?q=protobuf
-protobuf >= 3.9.2, < 4
+# NOTE: this version must be >= the protoc version in our WORKSPACE file.
+# At the same time, any constraints we specify here must allow at least some
+# version to be installed that is also compatible with TensorFlow's constraints:
+# https://github.com/tensorflow/tensorflow/blob/9d22f4a0a9499c8e10a4312503e63e0da35ccd94/tensorflow/tools/pip_package/setup.py#L100-L107
+protobuf >= 3.19.6
 requests >= 2.21.0, < 3
 setuptools >= 41.0.0
 tensorboard-data-server >= 0.6.0, < 0.7.0

diff --git a/third_party/grpc.patch b/third_party/grpc.patch
diff --git a/third_party/protobuf.patch b/third_party/protobuf.patch
diff --git a/third_party/protobuf/BUILD b/third_party/protobuf/BUILD