pip

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 asahi417
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -113,7 +113,7 @@ Words tagged with O are outside of named entities and the I-XXX tag is used for
 named entity of type XXX. Whenever two entities of type XXX are immediately next to each other, the
 first word of the second entity will be tagged B-XXX in order to show that it starts another entity.
 The custom dataset should has `train.txt` and `valid.txt` file in a same folder. 
-Please take a look [sample custom data](examples/custom_dataset_sample).
+Please take a look [sample custom data](asset/custom_dataset_sample).
 
 ### Model Finetuning
 Language model finetuning can be done with a few lines:

setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md

setup.py

@@ -1,34 +1,53 @@
 from setuptools import setup, find_packages
 
-VERSION = '0.0.0'
-NAME = 'tner'
-IS_RELEASED = False
-
-with open('README.md') as f:
+with open('README.md', 'r') as f:
     readme = f.read()
 
 setup(
-    name=NAME,
-    version=VERSION,
-    description='A library for language model finetuning on named entity recognition and model evaluation over cross-domain datasets.',
+    name='tner',
+    packages=find_packages(exclude=["asset", "examples", "static", "templates", "tests"]),
+    version='0.0.0',
+    license='MIT',
+    description='Transformer-based named entity recognition',
+    url='https://github.com/asahi417/tner',
+    download_url="https://github.com/asahi417/tner/archive/0.0.0.tar.gz",
+    keywords=['ner', 'nlp', 'language-model'],
     long_description=readme,
+    long_description_content_type="text/markdown",
     author='Asahi Ushio',
     author_email='[email protected]',
-    packages=find_packages(),
+    classifiers=[
+        'Development Status :: 4 - Beta',       # Chose either "3 - Alpha", "4 - Beta" or "5 - Production/Stable" as the current state of your package
+        'Intended Audience :: Developers',      # Define that your audience are developers
+        'Intended Audience :: Science/Research',
+        'Topic :: Scientific/Engineering',
+        'License :: OSI Approved :: MIT License',   # Again, pick a license
+        'Programming Language :: Python :: 3',      #Specify which pyhton versions that you want to support
+    ],
     include_package_data=True,
+    test_suite='tests',
     install_requires=[
-		'Pillow>=7.1.0',
-		'mecab-python3==0.996.5',  # this version can only work
-		'uvicorn==0.11.8',
-		'jinja2==2.11.2',
-		'aiofiles==0.5.0',
-		'fastapi==0.61.0',
-		'matplotlib==3.3.1',
-		'toml',
-		'tensorboard',
-		'torch',
-		'transformers',
-		'seqeval',
-		'segtok'
-	]
-)
+        'Pillow>=7.1.0',
+        # 'mecab-python3==0.996.5',  # this version can only work
+        'mecab-python3',  # this version can only work
+        'uvicorn==0.11.8',
+        'jinja2==2.11.2',
+        'aiofiles==0.5.0',
+        'fastapi==0.61.0',
+        'matplotlib==3.3.1',
+        'toml',
+        'tensorboard',
+        'torch',
+        'transformers',
+        'seqeval',
+        'segtok'
+    ],
+    python_requires='>=3.6',
+    entry_points={
+        'console_scripts': [
+            'tner-train = tner_cl.train:main',
+            'tner-test = tner_cl.test:main',
+            'tner-predict = tner_cl.predict:main'
+        ],
+    }
+)

tner/get_dataset.py

@@ -172,14 +172,27 @@ def get_dataset_ner_single(data_name: str = 'wnut2017',
         files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
         if not os.path.exists(data_path):
             os.makedirs(data_path, exist_ok=True)
-            os.system('wget -O {0}/data.tar.gz https://github.com/swiseman/neighbor-tagging/raw/master/data.tar.gz'.
+            os.system('wget -O {0}/data.tar.gz https://github.com/asahi417/neighbor-tagging/blob/master/data.tar.gz'.
                       format(CACHE_DIR))
             os.system('tar -xzf {0}/data.tar.gz -C {0}'.format(CACHE_DIR))
             for i in ['train', 'dev', 'test']:
                 conll_formatting(
                     file_token=os.path.join(CACHE_DIR, 'data/conll2003/conll2003-{}.words'.format(i)),
                     file_tag=os.path.join(CACHE_DIR, 'data/conll2003/conll2003-{}.nertags'.format(i)),
                     output_file=os.path.join(data_path, '{}.txt'.format(i)))
+    elif data_name == 'ontonotes5':
+        files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
+        if not os.path.exists(data_path):
+            raise ValueError('please download Ontonotes5 from https://catalog.ldc.upenn.edu/LDC2013T19')
+            # os.makedirs(data_path, exist_ok=True)
+            # os.system('wget -O {0}/data.tar.gz https://github.com/asahi417/neighbor-tagging/blob/master/data.tar.gz'.
+            #           format(CACHE_DIR))
+            # os.system('tar -xzf {0}/data.tar.gz -C {0}'.format(CACHE_DIR))
+            # for i in ['train', 'dev', 'test']:
+            #     conll_formatting(
+            #         file_token=os.path.join(CACHE_DIR, 'data/onto/{}.words'.format(i)),
+            #         file_tag=os.path.join(CACHE_DIR, 'data/onto/{}.ner'.format(i)),
+            #         output_file=os.path.join(data_path, '{}.txt'.format(i)))
     elif data_name == 'bc5cdr':
         files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
         if not os.path.exists(data_path):
@@ -229,7 +242,6 @@ def convert_to_iob(path, export):
         convert_to_iob('CDR_DevelopmentSet.PubTator.txt', 'dev.txt')
         convert_to_iob('CDR_TestSet.PubTator.txt', 'test.txt')
         convert_to_iob('CDR_TrainingSet.PubTator.txt', 'train.txt')
-
     elif data_name == 'bionlp2004':  # https://www.aclweb.org/anthology/W04-1213.pdf
         files_info = {'train': 'Genia4ERtask1.iob2', 'valid': 'Genia4EReval1.iob2'}
         if not os.path.exists(data_path):
@@ -261,18 +273,6 @@ def convert_to_iob(path, export):
             os.system('wget -O {0} https://groups.csail.mit.edu/sls/downloads/restaurant/restauranttest.bio'.format(
                 os.path.join(data_path, 'valid.txt')))
         entity_first = True
-    elif data_name == 'ontonotes5':
-        files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
-        if not os.path.exists(data_path):
-            os.makedirs(data_path, exist_ok=True)
-            os.system('wget -O {0}/data.tar.gz https://github.com/swiseman/neighbor-tagging/raw/master/data.tar.gz'.
-                      format(CACHE_DIR))
-            os.system('tar -xzf {0}/data.tar.gz -C {0}'.format(CACHE_DIR))
-            for i in ['train', 'dev', 'test']:
-                conll_formatting(
-                    file_token=os.path.join(CACHE_DIR, 'data/onto/{}.words'.format(i)),
-                    file_tag=os.path.join(CACHE_DIR, 'data/onto/{}.ner'.format(i)),
-                    output_file=os.path.join(data_path, '{}.txt'.format(i)))
     elif data_name == 'mit_movie_trivia':
         files_info = {'train': 'train.txt', 'valid': 'valid.txt'}
         if not os.path.exists(data_path):

examples/example_inference.py → tner_cl/predict.py

@@ -11,7 +11,7 @@ def get_options():
     return parser.parse_args()
 
 
-if __name__ == '__main__':
+def main():
     opt = get_options()
     classifier = TransformersNER(checkpoint=opt.checkpoint)
     test_sentences = [

tner_cl/test.py

@@ -0,0 +1,30 @@
+""" Fine-tune transformers on NER dataset """
+import argparse
+from tner import TrainTransformersNER
+
+
+def get_options():
+    parser = argparse.ArgumentParser(description='Fine-tune transformers on NER dataset')
+    parser.add_argument('-c', '--checkpoint', help='checkpoint to load', default=None, type=str)
+    parser.add_argument('--checkpoint-dir', help='checkpoint directory', default=None, type=str)
+    parser.add_argument('--lower-case', help='lower case all the data', action='store_true')
+    parser.add_argument('--test-data', help='test dataset (if not specified, use trained set)', default=None, type=str)
+    parser.add_argument('--test-lower-case', help='lower case all the test data', action='store_true')
+    parser.add_argument('--test-entity-span', help='evaluate entity span', action='store_true')
+    return parser.parse_args()
+
+
+def main():
+    opt = get_options()
+    # train model
+    trainer = TrainTransformersNER(
+        checkpoint=opt.checkpoint,
+        checkpoint_dir=opt.checkpoint_dir
+    )
+    if not trainer.is_trained:
+        raise ValueError('checkpoints not found at {}'.format(trainer.checkpoint))
+    test_data = [None] if opt.test_data is None else opt.test_data.split(',')
+    for i in test_data:
+        trainer.test(test_dataset=i, entity_span_prediction=opt.test_entity_span, lower_case=opt.test_lower_case)
+
+

examples/example_train_eval.py → tner_cl/train.py

@@ -1,6 +1,5 @@
 """ Fine-tune transformers on NER dataset """
 import argparse
-from glob import glob
 from tner import VALID_DATASET
 from tner import TrainTransformersNER
 
@@ -23,13 +22,10 @@ def get_options():
     parser.add_argument('--fp16', help='fp16', action='store_true')
     parser.add_argument('--monitor-validation', help='display validation after each epoch', action='store_true')
     parser.add_argument('--lower-case', help='lower case all the data', action='store_true')
-    parser.add_argument('--test-data', help='test dataset (if not specified, use trained set)', default=None, type=str)
-    parser.add_argument('--test-lower-case', help='lower case all the test data', action='store_true')
-    parser.add_argument('--test-entity-span', help='evaluate entity span', action='store_true')
     return parser.parse_args()
 
 
-if __name__ == '__main__':
+def main():
     opt = get_options()
     # train model
     trainer = TrainTransformersNER(
@@ -48,11 +44,6 @@ def get_options():
         max_grad_norm=opt.max_grad_norm,
         lower_case=opt.lower_case
     )
-    if not trainer.is_trained:
-        trainer.train(monitor_validation=opt.monitor_validation)
-    else:
-        test_data = [None] if opt.test_data is None else opt.test_data.split(',')
-        for i in test_data:
-            trainer.test(test_dataset=i, entity_span_prediction=opt.test_entity_span, lower_case=opt.test_lower_case)
-
-
+    if trainer.is_trained:
+        raise ValueError('checkpoint exists at {}'.format(trainer.checkpoint))
+    trainer.train(monitor_validation=opt.monitor_validation)