m1-mac-source-builds-tensorflow-tf_io-tf_addons-tf_text.sh

#!/usr/bin/env bash

########################################################################################################################
# This is a build script for misc. TensorFlow dependencies which do NOT have pre-compiled packages available for
# M-series (Apple Silicon) macs.
#
# Who: team members who use M1 macs and would like to run the project
#
# Caution: This script will use all available processors (rendering the machine relatively unusable) and will take a
# significant amount of time (about 2h 0m 15s on an M1 mac).
#
# Please set aside sufficient time to run this build script.
########################################################################################################################

# WARNING: MUST run prior to enabling bash strict mode or else it will fail
source deactivate || true # deactivate any current venvs

# Set Bash "strict" mode
# see: http://redsymbol.net/articles/unofficial-bash-strict-mode/
set -euo pipefail
set -x # Print command traces for debugging (feel free to comment this out if it is too noisy for you)
IFS=$'\n\t'

# Variables used throughout script
PROJECT_ROOT="$(git rev-parse --show-toplevel || pwd)"
SRC_BUILD_FOLDER_NAME="source_builds"
SRC_BUILDS_ROOT="${PROJECT_ROOT}/${SRC_BUILD_FOLDER_NAME}"

# In Monterey, there is an issue with OS versions not being properly inferred for built wheels, resulting in wheels
# using older platform specifiers. When attempting to install these wheels, they may erroneously be reported as,
# "... *.whl is not a supported wheel on this platform." However, they *are* supported, and the error is due solely to
# metadata mismatch (i.e., if you manually changed the metadata, it would install and work just fine).
# Manually set this C-compiler flag so wheels are correctly built for the host platform.
MACOSX_MAJOR_VERSION="$(sw_vers -productVersion | awk -F. '{print $1}')" # e.g., 12
export MACOSX_DEPLOYMENT_TARGET="${MACOSX_MAJOR_VERSION}.0"              # e.g., 12.0
ARCH="$(arch)"                                                           #i.e., arm64

# Install Bazel build tool for tensorflow ecosystem projects source builds
brew install bazelisk
export USE_BAZEL_VERSION="4.2.2" # the max version known to work with the below TF versions combination as of 2021/11/13

# TF ecosystem versions as of 2021/11/13
TF_VERSION="2.7.0"
TF_TEXT_VERSION="2.7.3"
TF_IO_VERSION="0.22.0"     # built against TF version 2.7.0
TF_ADDONS_VERSION="0.15.0" # built against TF version 2.7.0

## Main build logic

main() {
  pushd "$(pwd)" # save the current directory to restore after the build process completes
  sudo -v        # login at start of script so that remainder can run without interruption

  build_dependencies_from_source

  sudo -k # logout of sudo
  popd
  printf "\n************************************************************************************"
  printf "\nSource packages built:\n\n%s" "$(ls "$(get_source_build_wheel_output_root)")"
  printf "\n************************************************************************************"
}

build_dependencies_from_source() {
  setup_src_build_dir_venv_and_enter

  build_tensorflow
  build_tensorflow_io # depends on tensorflow

  build_tensorflow_addons # depends on tensorflow, tensorflow-io
  build_tensorflow_text   # depends on tensorflow, tensorflow-io
}

## Build commands
setup_src_build_dir_venv_and_enter() {
  mkdir -p "${SRC_BUILDS_ROOT}"
  cd "${SRC_BUILDS_ROOT}"
  python -m venv .venv
  source .venv/bin/activate
  pip install pip wheel numpy
  pip install keras_preprocessing --no-deps
}

build_tensorflow() {
  clone_setup_and_enter_tf_proj_dir "tensorflow" "v${TF_VERSION}"
  yes '' | ./configure || true #fallback to true to prevent non-zero exit code of configuration script from making the script exit early

  local WHEEL_OUTPUT_ROOT="./tmp/tensorflow_pkg"
  # sudo or else won't pick up the darwin_arm64 cc toolchain...
  sudo bazel build --config="macos_${ARCH}" //tensorflow/tools/pip_package:build_pip_package
  ./bazel-bin/tensorflow/tools/pip_package/build_pip_package "${WHEEL_OUTPUT_ROOT}"

  copy_wheel_to_source_builds_packages_dir "${WHEEL_OUTPUT_ROOT}/tensorflow-${TF_VERSION}-"*.whl
}

build_tensorflow_io() {
  # https://github.com/tensorflow/io/issues/1298#issuecomment-958610412
  clone_setup_and_enter_tf_proj_dir "io" "v${TF_IO_VERSION}"
  python setup.py -q bdist_wheel --project tensorflow_io_gcs_filesystem
  copy_wheel_to_source_builds_packages_dir ./dist/tensorflow_io_gcs_filesystem-*.whl
}

build_tensorflow_addons() {
  # https://github.com/tensorflow/addons
  clone_setup_and_enter_tf_proj_dir "addons" "v${TF_ADDONS_VERSION}"

  local WHEEL_OUTPUT_ROOT="./tmp/tensorflow_addon_pkg"
  # Install Tensorflow IO & Tensorflow needed for text (built in the previous steps)
  # Note: TF depends on TF IO, so must be installed in this order
  pip install "$(get_source_build_wheel_output_root)/tensorflow_io_gcs_filesystem-${TF_IO_VERSION}-"*"${ARCH}.whl"
  pip install "$(get_source_build_wheel_output_root)/tensorflow-${TF_VERSION}-"*"${ARCH}.whl"
  python ./configure.py

  # Update to compatible platform extension [OPTIONAL; here for conformity ]
  sed -i '' "s/11_0_arm64/${MACOSX_DEPLOYMENT_TARGET}-$(arch)/g" "./build_deps/build_pip_pkg.sh"
  bazel build build_pip_pkg
  bazel-bin/build_pip_pkg "${WHEEL_OUTPUT_ROOT}"

  copy_wheel_to_source_builds_packages_dir "${WHEEL_OUTPUT_ROOT}/tensorflow_addons-"*.whl
}

build_tensorflow_text() {

  # Note: built bazel bin exported to path, so must restore old path upon function exit
  local OLD_PATH="${PATH}"
  local ORIG_BAZEL="$(which bazel)"
  local ORIG_BAZEL_VER="$(get_bazel_version)"

  _build_bazel3_7_2

  ## https://github.com/tensorflow/text
  clone_setup_and_enter_tf_proj_dir "text" "v${TF_TEXT_VERSION}"
  # Install deps needed for text (built in the previous steps)
  pip install "$(get_source_build_wheel_output_root)/tensorflow"*"${ARCH}.whl"

  # Update to compatible platform extension [MANDATORY]
  sed -i '' "s/10.9-x86_64/${MACOSX_DEPLOYMENT_TARGET}-$(arch)/g" "./oss_scripts/pip_package/build_pip_package.sh"
  ./oss_scripts/run_build.sh
  copy_wheel_to_source_builds_packages_dir ./tensorflow_text-*.whl

  rm .bazelversion
  export PATH="${OLD_PATH}"
  if [ "${ORIG_BAZEL_VER}" != "$(get_bazel_version)" ]; then
    echo "Original bazel on path (${ORIG_BAZEL}) not the same bazel that is on path now! ($(which bazel)"
    exit 2
  fi
}

## Bazel 3.7.2 needed for tensorflow-text
_build_bazel3_7_2() {
  cd "${SRC_BUILDS_ROOT}"

  local BAZEL_VER="3.7.2"
  local BAZEL_RELEASES_URL="https://github.com/bazelbuild/bazel/releases/download"
  local BAZEL_DIST_ARCHIVE="bazel-${BAZEL_VER}-dist.zip"
  local BAZEL_DIST_ARCHIVE_CHECKSUM="${BAZEL_DIST_ARCHIVE}.sha256"

  local BAZEL_DIST_PATH="${SRC_BUILDS_ROOT}/bazel"
  mkdir -p "${BAZEL_DIST_PATH}"

  # WARNING: do NOT try to be clever and pipe `curl` with `-L` directly to `tar`; the redirect will grab the GitHub source
  # tree which is NOT what we want since only the distribution archive is configured to boostrap Bazel.
  # https://docs.bazel.build/versions/main/install-compile-source.html#download-distfile
  wget "${BAZEL_RELEASES_URL}/${BAZEL_VER}/${BAZEL_DIST_ARCHIVE}"
  wget "${BAZEL_RELEASES_URL}/${BAZEL_VER}/${BAZEL_DIST_ARCHIVE_CHECKSUM}"
  sha256sum -c "${BAZEL_DIST_ARCHIVE_CHECKSUM}"
  unzip "${BAZEL_DIST_ARCHIVE}" -d "${BAZEL_DIST_PATH}"
  rm "${BAZEL_DIST_ARCHIVE}" "${BAZEL_DIST_ARCHIVE_CHECKSUM}"
  cd "${BAZEL_DIST_PATH}"

  # https://github.com/bazelbuild/bazel/issues/11399#issuecomment-628945756
  brew install openjdk@11
  export JAVA_HOME="$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home"

  # Specify EMBED_LABEL to override auto-generated (potentially-broken) bazel version imputed on dist builds
  env \
    EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" \
    EMBED_LABEL="${BAZEL_VER}" \
    bash ./compile.sh

  # Note: would ideally like to just return `BAZEL_BIN_PATH`
  # and perform the below in the calling function, but for some reason
  # this doesn't make it out of the function to be captured; leaving
  # it here is the only way to make it work
  local BAZEL_BIN_PATH="$(pwd)/output"
  export PATH="${BAZEL_BIN_PATH}:${PATH}"
  if [ "${BAZEL_VER}" != "$(get_bazel_version)" ]; then
    echo "bazel on path ($(which bazel)) not the same bazel that was just built! (${BAZEL_BIN_PATH}/bazel)"
    exit 1
  fi
  bazel --version # Log bazel version as a sanity check
}

## Helper commands
clone_setup_and_enter_tf_proj_dir() {
  local TF_PROJ="${1}"
  local GIT_TAG="${2}"
  clone_and_enter_proj_dir "tensorflow/${TF_PROJ}"
  git checkout "${GIT_TAG}"
  get_bazel_version >.bazelversion # Use the newest bazel version; versions < 4.0 not supported on M1 and tensorflow sets this to version 3.7.1 as of 2021/11/13
}

clone_and_enter_proj_dir() {
  local GIT_REPO="${1}"
  cd "${SRC_BUILDS_ROOT}"
  local PROJ_DIR="$(basename ${GIT_REPO})"
  rm -rf "${PROJ_DIR}"
  git clone "https://github.com/${GIT_REPO}.git" && cd "${PROJ_DIR}"
}

get_bazel_version() {
  bazel --version | awk '{print $2}'
}

copy_wheel_to_source_builds_packages_dir() {
  local WHEEL_PATH="${1}"
  cp -v "${WHEEL_PATH}" "$(get_source_build_wheel_output_root)"
}

get_source_build_wheel_output_root() {
  local WHEEL_OUTPUT_ROOT="${PROJECT_ROOT}/packages/${SRC_BUILD_FOLDER_NAME}"
  mkdir -p "${WHEEL_OUTPUT_ROOT}"
  echo "${WHEEL_OUTPUT_ROOT}"
}

## Execute build process

main

@TeoZosa - thanks for the script.
But I get this error . I think tensorflow compiled fine but io package build failed. Before I start fixing myself , any thoughts ?
ERROR: tensorflow_io_gcs_filesystem-0.22.0-*arm64.whl is not a valid wheel filename.

@TeoZosa - thanks for the script. But I get this error . I think tensorflow compiled fine but io package build failed. Before I start fixing myself , any thoughts ? ERROR: tensorflow_io_gcs_filesystem-0.22.0-*arm64.whl is not a valid wheel filename.

@ashsha21: Hm, sounds like it. That error message is probably from the build_tensorflow_addons step which sounds like the wheel didn't build and thus didn't get copied to the expected wheel output folder.

1. Do you have any other error logs?
2. Are any wheels built into the io/dist folder?
3. If (1) & (2) don't provide any useful clues, I just updated the script to include command traces and make copy_wheel_to_source_builds_packages_dir more verbose. You can try running the script again and seeing if any useful error messages pop up near that failed build step.
   • if Tensorflow already built sucessfully, then you can comment out build_tensorflow on line 66 to start compiling at the Tensorflow IO step.

@TeoZosa thanks for the reply Teo. No , no other error messages. But for now I could use the prebuilt wheel from here https://github.com/sun1638650145/Libraries-and-Extensions-for-TensorFlow-for-Apple-Silicon/releases/tag/v2.8 . It worked to my surprise. The io and tf wheels were built but i guess add-on build failed because of "*" in the filename. Fo some reason script could not expand it.