Skip to content

Commit

Permalink
Remove unnecessary name-mangling.
Browse files Browse the repository at this point in the history
  • Loading branch information
sultur committed Sep 22, 2023
1 parent c226966 commit cacf8fa
Showing 1 changed file with 10 additions and 11 deletions.
21 changes: 10 additions & 11 deletions src/tokenizer/tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -878,34 +878,33 @@ def Split_Sentence(t: Optional[Tok] = None) -> Tok:


class TokenStream:

"""Wrapper for token iterator allowing lookahead."""

def __init__(self, token_it: Iterator[Tok], *, lookahead_size: int = 2):
"""Initialize from token iterator."""
self.__it: Iterator[Tok] = token_it
self._it: Iterator[Tok] = token_it
if lookahead_size <= 0:
lookahead_size = 1
self.__lookahead: Deque[Tok] = deque(maxlen=lookahead_size)
self.__max_lookahead: int = lookahead_size
self._lookahead: Deque[Tok] = deque(maxlen=lookahead_size)
self._max_lookahead: int = lookahead_size

def __next__(self) -> Tok:
if self.__lookahead:
return self.__lookahead.popleft()
return next(self.__it)
if self._lookahead:
return self._lookahead.popleft()
return next(self._it)

def __iter__(self):
return self

def __getitem__(self, i: int) -> Optional[Tok]:
if 0 <= i < self.__max_lookahead:
l = len(self.__lookahead)
if 0 <= i < self._max_lookahead:
l = len(self._lookahead)
try:
while l <= i:
# Extend deque to lookahead
self.__lookahead.append(next(self.__it))
self._lookahead.append(next(self._it))
l += 1
return self.__lookahead[i]
return self._lookahead[i]
except StopIteration:
pass
return None
Expand Down

0 comments on commit cacf8fa

Please sign in to comment.