Skip to content

Commit

Permalink
Import order matters.
Browse files Browse the repository at this point in the history
  • Loading branch information
Narsil committed Mar 11, 2024
1 parent 20f60bf commit 732d0ca
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 23 deletions.
46 changes: 23 additions & 23 deletions bindings/python/py_src/tokenizers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,5 @@
from enum import Enum
from typing import List, Tuple, Union
from .tokenizers import (
AddedToken,
Encoding,
NormalizedString,
PreTokenizedString,
Regex,
Token,
Tokenizer,
decoders,
models,
normalizers,
pre_tokenizers,
processors,
trainers,
__version__,
)
from .implementations import (
BertWordPieceTokenizer,
ByteLevelBPETokenizer,
CharBPETokenizer,
SentencePieceBPETokenizer,
SentencePieceUnigramTokenizer,
)


Offsets = Tuple[int, int]
Expand Down Expand Up @@ -98,3 +75,26 @@ class SplitDelimiterBehavior(Enum):
CONTIGUOUS = "contiguous"


from .tokenizers import (
AddedToken,
Encoding,
NormalizedString,
PreTokenizedString,
Regex,
Token,
Tokenizer,
decoders,
models,
normalizers,
pre_tokenizers,
processors,
trainers,
__version__,
)
from .implementations import (
BertWordPieceTokenizer,
ByteLevelBPETokenizer,
CharBPETokenizer,
SentencePieceBPETokenizer,
SentencePieceUnigramTokenizer,
)
2 changes: 2 additions & 0 deletions bindings/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,6 @@ lint.ignore = [
"F401",
# Raw type equality is required in asserts
"E721",
# Import order
"E402",
]

0 comments on commit 732d0ca

Please sign in to comment.