Merge pull request #225 from google/sr

pushed new nfkc_cf.tsv
google · Oct 28, 2018 · d55372c · d55372c
2 parents 42ba256 + 465a419
commit d55372c
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 14 deletions.
diff --git a/data/nfkc_cf.tsv b/data/nfkc_cf.tsv
@@ -1,7 +1,3 @@
-9	20	# 	 =>  
-A	20	#   =>  
-C	20	#  =>  
-D	20	#   =>  
 3C 338	226E	# ≮ => ≮
 3D 338	2260	# ≠ => ≠
 3E 338	226F	# ≯ => ≯
@@ -57024,7 +57020,6 @@ FB9	F90 FB5	# ྐྵ => ྐྵ
 13FB	13F3	# ᏻ => Ᏻ
 13FC	13F4	# ᏼ => Ᏼ
 13FD	13F5	# ᏽ => Ᏽ
-1680	20	#   =>  
 1B05 1B35	1B06	# ᬆ => ᬆ
 1B07 1B35	1B08	# ᬈ => ᬈ
 1B09 1B35	1B0A	# ᬊ => ᬊ
@@ -57948,16 +57943,11 @@ FB9	F90 FB5	# ྐྵ => ྐྵ
 2008	20	#   =>  
 2009	20	#   =>  
 200A	20	#   =>  
-200B	20	#  =>  
-200E	20	# ‎ =>  
-200F	20	# ‏ =>  
 2011	2010	# ‑ => ‐
 2017	20 333	# ‗ =>  ̳
 2024	2E	# ․ => .
 2025	2E 2E	# ‥ => ..
 2026	2E 2E 2E	# … => ...
-2028	20	#   =>  
-2029	20	#   =>  
 202F	20	#   =>  
 2033	2032 2032	# ″ => ′′
 2034	2032 2032 2032	# ‴ => ′′′
@@ -59598,7 +59588,6 @@ FB9	F90 FB5	# ྐྵ => ྐྵ
 24E9 331	1E95	# ⓩ̱ => ẕ
 24E9 341	17A	# ⓩ́ => ź
 24EA	30	# ⓪ => 0
-2581	20	# ▁ =>  
 2A0C	222B 222B 222B 222B	# ⨌ => ∫∫∫∫
 2A74	3A 3A 3D	# ⩴ => ::=
 2A75	3D 3D	# ⩵ => ==
@@ -157901,7 +157890,6 @@ FEF9	644 625	# ﻹ => لإ
 FEFA	644 625	# ﻺ => لإ
 FEFB	644 627	# ﻻ => لا
 FEFC	644 627	# ﻼ => لا
-FEFF	20	#  =>  
 FF01	21	# ！ => !
 FF02	22	# ＂ => "
 FF03	23	# ＃ => #
@@ -158584,6 +158572,7 @@ FF5A 341	17A	# ｚ́ => ź
 FF5B	7B	# ｛ => {
 FF5C	7C	# ｜ => |
 FF5D	7D	# ｝ => }
+FF5E	7E	# ～ => ~
 FF5F	2985	# ｟ => ⦅
 FF60	2986	# ｠ => ⦆
 FF61	3002	# ｡ => 。
@@ -213835,7 +213824,6 @@ FFEB 338	219B	# ￫̸ => ↛
 FFEC	2193	# ￬ => ↓
 FFED	25A0	# ￭ => ■
 FFEE	25CB	# ￮ => ○
-FFFD	20	# � =>  
 10400	10428	# 𐐀 => 𐐨
 10401	10429	# 𐐁 => 𐐩
 10402	1042A	# 𐐂 => 𐐪

diff --git a/src/compile_charsmap_main.cc b/src/compile_charsmap_main.cc
@@ -160,7 +160,7 @@ int main(int argc, char **argv) {
       std::pair<std::string, std::function<Status(Builder::CharsMap *)>>>
       kRuleList = {{"nfkc", Builder::BuildNFKCMap},
                    {"nmt_nfkc", Builder::BuildNmtNFKCMap},
-                   {"nfkc_cf", Builder::BuildNmtNFKC_CFMap},
+                   {"nfkc_cf", Builder::BuildNFKC_CFMap},
                    {"nmt_nfkc_cf", Builder::BuildNmtNFKC_CFMap}};
 
   std::vector<std::pair<std::string, std::string>> data;

diff --git a/tensorflow/setup.py b/tensorflow/setup.py
@@ -16,6 +16,7 @@
 
 from setuptools import setup
 from setuptools import find_packages
+import tensorflow as tf
 import codecs
 import string
 import sys
@@ -26,6 +27,12 @@
 with codecs.open(os.path.join('..', 'VERSION'), 'r', 'utf-8') as f:
   version = f.read()
 
+tf_version = tf.__version__
+if tf_version[0] == '.':
+  tf_version = tf_version[1:]
+
+version = tf_version + '.' + version
+
 setup(name = 'tf_sentencepiece',
       author = 'Taku Kudo',
       author_email='[email protected]',