Build for TensorFlow 2.11

WORKSPACE.in

@@ -36,20 +36,20 @@ http_archive(
 
 http_archive(
     name = "boringssl",
-    sha256 = "1188e29000013ed6517168600fc35a010d58c5d321846d6a6dfee74e4c788b45",
-    strip_prefix = "boringssl-7f634429a04abc48e2eb041c81c5235816c96514",
+    sha256 = "1c5c5bca468c2bc98d02a035f9e10171cdcbd47b8e517a5e06ec9e0fae8bf983",
+    strip_prefix = "boringssl-3ccef042c8390706182b81141c5fdd7dfebcfab8",
     urls = [
-        "http://mirror.tensorflow.org/github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz",
-        "https://github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz",
+        "http://mirror.tensorflow.org/github.com/google/boringssl/archive/3ccef042c8390706182b81141c5fdd7dfebcfab8.tar.gz",
+        "https://github.com/google/boringssl/archive/3ccef042c8390706182b81141c5fdd7dfebcfab8.tar.gz",
     ],
 )
 
 http_archive(
     name = "bazel_skylib",
-    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
+    sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz",
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
     ],
 )
 

configure.sh

@@ -42,7 +42,7 @@ function write_action_env_to_bazelrc() {
 [ -e .bazelrc ] && rm .bazelrc
 
 write_to_bazelrc "build -c opt"
-write_to_bazelrc 'build --cxxopt="-std=c++11"'
+write_to_bazelrc 'build --cxxopt="-std=c++17"'
 write_to_bazelrc 'build --auto_output_filter=subpackages'
 write_to_bazelrc 'build --copt="-Wall" --copt="-Wno-sign-compare"'
 write_to_bazelrc 'build --linkopt="-lrt -lm"'
@@ -51,7 +51,7 @@ TF_NEED_CUDA=0
 # Check if it's installed
 TF_CFLAGS=""
 TF_LFLAGS=""
-if ${PIP} list | grep "tensorflow \|tensorflow-gpu\|tensorflow-cpu\|tf-nightly" >/dev/null ; then
+if ${PIP} list | grep "tensorflow \|tensorflow-gpu\|tensorflow-cpu\|tf-nightly\|tensorflow-metal" >/dev/null ; then
   echo 'Using installed tensorflow'
   TF_CFLAGS=( $(${PYTHON} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
   TF_LFLAGS="$(${PYTHON} -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))')"
@@ -82,16 +82,18 @@ if [[ "$PIP_MANYLINUX2010" == "1" ]]; then
   write_to_bazelrc "test --config=manylinux2010"
 fi
 
-export TF_VERSION="${TF_VERSION:-2.5.0}"
+export TF_VERSION="${TF_VERSION:-2.11.0}"
 export TF_VERSION_UNDERSCORE=$(echo $TF_VERSION | sed 's/\./_/g')
 export TF_VERSION_DASH=$(echo $TF_VERSION | sed 's/\./-/g')
 
 cat WORKSPACE.in | sed "s/TF_VERSION/${TF_VERSION_UNDERSCORE}/" > WORKSPACE
 cat pip_pkg_scripts/setup.py.in | sed "s/TF_VERSION/${TF_VERSION_DASH}/" > pip_pkg_scripts/setup.py
 
-if [[ ${TF_VERSION} == '1.14.0' ]]; then
-  write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1"'
-else
-  write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"'
-fi
-
+case "$TF_VERSION" in
+  1.14.0 | 2.9.* | 2.10.* | 2.11.*)
+    write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1"'
+    ;;
+  *)
+    write_to_bazelrc 'build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"'
+    ;;
+esac

pip_pkg_scripts/build.Dockerfile

@@ -1,16 +1,11 @@
-FROM tensorflow/tensorflow:custom-op-ubuntu16
+FROM tensorflow/tensorflow:2.3.0-custom-op-gpu-ubuntu16
 
 ENV PYTHON_VERSION="3"
 ENV PYTHON_MINOR_VERSION=""
 ENV PIP_MANYLINUX2010="1"
-ENV TF_VERSION="2.6.0"
-RUN wget https://github.com/bazelbuild/bazel/releases/download/4.0.0/bazel-4.0.0-installer-linux-x86_64.sh > /dev/null
-RUN bash bazel-4.0.0-installer-linux-x86_64.sh
-# There are some problems with the python3 installation from custom-op-ubuntu16.
-# Remove it and install new ones.
-RUN apt-get remove --purge -y python3.5 python3.6
-RUN rm -f /etc/apt/sources.list.d/jonathonf-ubuntu-python-3_6-xenial.list
-RUN apt-key del F06FC659
+ENV TF_VERSION="2.11.0"
+RUN wget https://github.com/bazelbuild/bazel/releases/download/5.3.2/bazel-5.3.2-installer-linux-x86_64.sh > /dev/null
+RUN bash bazel-5.3.2-installer-linux-x86_64.sh
 
 # Deadsnakes PPA no longer supports 16.04
 # https://github.com/deadsnakes/issues/issues/195
@@ -20,7 +15,11 @@ RUN apt-get update
 RUN apt-get install -y apt-utils
 RUN apt-get install -y build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libssl-dev zlib1g-dev openssl libffi-dev
 
-RUN for v in 3.6.15 3.7.12 3.8.12 3.9.10; do \
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test
+RUN apt-get update
+RUN apt-get install
+
+RUN for v in 3.7.12 3.8.12 3.9.10; do \
     wget "https://www.python.org/ftp/python/$v/Python-${v}.tar.xz" && \
     tar xvf "Python-${v}.tar.xz" -C /tmp/python && \
     cd "/tmp/python/Python-${v}" && \
@@ -31,7 +30,7 @@ RUN for v in 3.6.15 3.7.12 3.8.12 3.9.10; do \
 
 RUN apt-get install -y libopenexr-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN for python in python3.6 python3.7 python3.8 python3.9; do \
+RUN for python in python3.7 python3.8 python3.9; do \
       $python get-pip.py && \
       $python -m pip install --upgrade pip setuptools auditwheel && \
       $python -m pip install --upgrade grpcio>=1.24.3; \

pip_pkg_scripts/build.sh

@@ -25,7 +25,7 @@ set -e -x
 export PYTHON_VERSION="${PYTHON_VERSION:-3}"
 export PYTHON_MINOR_VERSION="${PYTHON_MINOR_VERSION}"
 export PIP_MANYLINUX2010="${PIP_MANYLINUX2010:-0}"
-export TF_VERSION="${TF_VERSION:-2.5.0}"
+export TF_VERSION="${TF_VERSION:-2.11.0}"
 
 if [[ -z "${PYTHON_MINOR_VERSION}" ]]; then
   PYTHON="python${PYTHON_VERSION}"
@@ -48,6 +48,15 @@ case "$TF_VERSION" in
   2.8.*)
     export KERAS_VERSION='2.8.0'
     ;;
+  2.9.*)
+    export KERAS_VERSION='2.9.0'
+    ;;
+  2.10.*)
+    export KERAS_VERSION='2.10.0'
+    ;;
+  2.11.*)
+    export KERAS_VERSION='2.11.0'
+    ;;
   *)
     export KERAS_VERSION='nightly'
 esac
@@ -74,4 +83,3 @@ rm -rf "$DST_DIR" || true
 if [[ "${PIP_MANYLINUX2010}" == "1" ]]; then
   find "$DST_DIR" -name *.whl | xargs $PYTHON ./third_party/auditwheel_wrapper.py repair --plat manylinux2010_x86_64 -w "$DST_DIR"
 fi
-

pip_pkg_scripts/docker_build_all.sh

@@ -20,10 +20,9 @@ docker build --tag=open_dataset_pip -f pip_pkg_scripts/build.Dockerfile .
 mkdir /tmp/artifacts | true
 mkdir /tmp/artifacts_all | true
 
-for pyv in 7 8 9; do
-  for tfv in 6; do
+for pyv in 7 8 9 10; do
+  for tfv in 11; do
     docker run --mount type=bind,source=/tmp/artifacts,target=/tmp/artifacts -e "PYTHON_VERSION=3" -e "PYTHON_MINOR_VERSION=${pyv}" -e "PIP_MANYLINUX2010=1" -e "TF_VERSION=2.${tfv}.0" open_dataset_pip
     cp -f /tmp/artifacts/* /tmp/artifacts_all
   done
 done
-

tf/workspace_tf2_10_0.bzl

@@ -0,0 +1,51 @@
+"""Install dependencies."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def load_tf_version_specific_dependencies():
+    """Load TF version specific dependencies."""
+    http_archive(
+        name = "com_google_absl",
+        sha256 = "94aef187f688665dc299d09286bfa0d22c4ecb86a80b156dff6aabadc5a5c26d",
+        strip_prefix = "abseil-cpp-273292d1cfc0a94a65082ee350509af1d113344d",
+        urls = [
+            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz",
+        ],
+    )
+
+    http_archive(
+        name = "zlib",
+        build_file = "//third_party:zlib.BUILD",
+        sha256 = "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
+        strip_prefix = "zlib-1.2.12",
+        urls = [
+            "http://mirror.tensorflow.org/zlib.net/zlib-1.2.12.tar.gz",
+            "https://zlib.net/zlib-1.2.12.tar.gz",
+        ],
+    )
+
+    PROTOBUF_URLS = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
+        "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
+    ]
+
+    PROTOBUF_SHA256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b"
+
+    PROTOBUF_STRIP_PREFIX = "protobuf-3.9.2"
+
+    # We need to import the protobuf library under the names com_google_protobuf
+    # and com_google_protobuf_cc to enable proto_library support in bazel.
+    http_archive(
+        name = "com_google_protobuf",
+        sha256 = PROTOBUF_SHA256,
+        strip_prefix = PROTOBUF_STRIP_PREFIX,
+        urls = PROTOBUF_URLS,
+    )
+
+    http_archive(
+        name = "com_google_protobuf_cc",
+        sha256 = PROTOBUF_SHA256,
+        strip_prefix = PROTOBUF_STRIP_PREFIX,
+        urls = PROTOBUF_URLS,
+    )

tf/workspace_tf2_11_0.bzl

@@ -0,0 +1,51 @@
+"""Install dependencies."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def load_tf_version_specific_dependencies():
+    """Load TF version specific dependencies."""
+    http_archive(
+        name = "com_google_absl",
+        sha256 = "94aef187f688665dc299d09286bfa0d22c4ecb86a80b156dff6aabadc5a5c26d",
+        strip_prefix = "abseil-cpp-273292d1cfc0a94a65082ee350509af1d113344d",
+        urls = [
+            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/273292d1cfc0a94a65082ee350509af1d113344d.tar.gz",
+        ],
+    )
+
+    http_archive(
+        name = "zlib",
+        build_file = "//third_party:zlib.BUILD",
+        sha256 = "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
+        strip_prefix = "zlib-1.2.12",
+        urls = [
+            "http://mirror.tensorflow.org/zlib.net/zlib-1.2.12.tar.gz",
+            "https://zlib.net/zlib-1.2.12.tar.gz",
+        ],
+    )
+
+    PROTOBUF_URLS = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
+        "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
+    ]
+
+    PROTOBUF_SHA256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b"
+
+    PROTOBUF_STRIP_PREFIX = "protobuf-3.9.2"
+
+    # We need to import the protobuf library under the names com_google_protobuf
+    # and com_google_protobuf_cc to enable proto_library support in bazel.
+    http_archive(
+        name = "com_google_protobuf",
+        sha256 = PROTOBUF_SHA256,
+        strip_prefix = PROTOBUF_STRIP_PREFIX,
+        urls = PROTOBUF_URLS,
+    )
+
+    http_archive(
+        name = "com_google_protobuf_cc",
+        sha256 = PROTOBUF_SHA256,
+        strip_prefix = PROTOBUF_STRIP_PREFIX,
+        urls = PROTOBUF_URLS,
+    )