Merge pull request #240 from google/sr

load tf versioned so file to avoid abi incompatibility

tensorflow/make_py_wheel.sh

@@ -30,6 +30,33 @@ run_docker() {
   docker stop tf_sentencepiece
 }
 
+build_tf_wrapper() {
+  if [ "$1" != "" ]; then
+    pkg_name="==$1"
+  fi
+
+  # Builds _sentencepiece_processor_ops.so
+  pip install tensorflow${pkg_name} --upgrade
+
+  TF_CFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
+  TF_LFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') )
+  TF_VERSION=( $(python -c 'import tensorflow as tf; print(tf.__version__)') )
+
+  g++ -std=c++11 -shared \
+    -I../../src \
+    -fPIC ${TF_CFLAGS[@]} -O2 \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -Wl,--whole-archive \
+    /usr/local/lib/libprotobuf.a \
+    /usr/local/lib/libsentencepiece.a \
+    -Wl,--no-whole-archive \
+    sentencepiece_processor_ops.cc \
+    -o tf_sentencepiece/_sentencepiece_processor_ops.so.${TF_VERSION} \
+    ${TF_LFLAGS[@]}
+
+  strip tf_sentencepiece/_sentencepiece_processor_ops.so.${TF_VERSION}
+}
+
 build() {
   TRG=$1
   rm -fr build
@@ -55,23 +82,12 @@ build() {
   make install
   cd ..
 
-  # Builds _sentencepiece_processor_ops.so
-#  pip install tensorflow
-  TF_CFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
-  TF_LFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') )
-
-  g++ -std=c++11 -shared \
-    -I../../src \
-    -fPIC ${TF_CFLAGS[@]} -O2 \
-    -D_GLIBCXX_USE_CXX11_ABI=0 \
-    -Wl,--whole-archive \
-    /usr/local/lib/libprotobuf.a \
-    /usr/local/lib/libsentencepiece.a \
-    -Wl,--no-whole-archive \
-    sentencepiece_processor_ops.cc \
-    -o tf_sentencepiece/_sentencepiece_processor_ops.so \
-    ${TF_LFLAGS[@]}
-  strip tf_sentencepiece/_sentencepiece_processor_ops.so
+  build_tf_wrapper ""
+  build_tf_wrapper "1.11.0"
+  build_tf_wrapper "1.10.0"
+  build_tf_wrapper "1.9.0"
+  build_tf_wrapper "1.8.0"
+  build_tf_wrapper "1.7.0"
 
   # Builds Python manylinux wheel package.
   python setup.py bdist_wheel --universal --plat-name=manylinux1_x86_64

tensorflow/setup.py

@@ -27,12 +27,6 @@
 with codecs.open(os.path.join('..', 'VERSION'), 'r', 'utf-8') as f:
   version = f.read()
 
-tf_version = tf.__version__
-if tf_version[0] == '.':
-  tf_version = tf_version[1:]
-
-version = tf_version + '.' + version
-
 setup(name = 'tf_sentencepiece',
       author = 'Taku Kudo',
       author_email='[email protected]',
@@ -42,7 +36,7 @@
       license = 'Apache',
       platforms = 'Unix',
       packages=find_packages(exclude=['test']),
-      package_data={'tf_sentencepiece':  ['_sentencepiece_processor_ops.so']},
+      package_data={'tf_sentencepiece':  ['_sentencepiece_processor_ops.so*']},
       classifiers = [
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',

tensorflow/tf_sentencepiece/sentencepiece_processor_ops.py

@@ -19,12 +19,28 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from distutils.version import StrictVersion
 
+import warnings
+import glob
+import re
 import os
 import tensorflow as tf
 
-_gen_sentencepiece_processor_op = tf.load_op_library(
-    os.path.join(os.path.dirname(__file__), '_sentencepiece_processor_ops.so'))
+so_base = os.path.join(os.path.dirname(__file__), '_sentencepiece_processor_ops.so')
+so_file = so_base + '.' + tf.__version__
+
+if not os.path.exists(so_file):
+  versions = [re.search('[0-9]+\.[0-9\.]+$', n).group(0)
+              for n in glob.glob(so_base + '.*')]
+  latest = sorted(versions, key=StrictVersion)[-1]
+  warnings.warn('No so file is found for [%s] from [%s]' % (tf.__version__,
+                                                            ', '.join(versions)))
+  warnings.warn('use the latest version %s' % (latest))
+  so_file = so_base + '.' + latest
+
+
+_gen_sentencepiece_processor_op = tf.load_op_library(so_file)
 
 
 def piece_size(model_file=None, model_proto=None, name=None):