Add Stopword Tala 2003, Lru Cache, and Support List Tokens

.travis.yml

@@ -5,11 +5,18 @@ python:
   - "3.4"
   - "3.5"
 sudo: false
+# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs
+matrix:
+  include:
+    - python: 3.7
+      dist: xenial
+      sudo: true
 install:
   - pip install python-coveralls
   - pip install coveralls
+  - pip install cachetools
 script: nosetests tests --verbose --with-coverage
 after_success:
   - coveralls
 notifications:
-  email: false
+  email: false

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.linting.pylintEnabled": true
+}

README.md

@@ -1,58 +1,4 @@
-Sastrawi Python
-===============
+# sastrawi
+Indonesian stemmer. Python port of PHP Sastrawi project.
 
-Sastrawi Python is a simple python library which allows you to reduce inflected words in Indonesian Language (Bahasa Indonesia) to their base form ([stem](http://en.wikipedia.org/wiki/Stemming)).
-This is Python port of the original [Sastrawi](https://github.com/sastrawi/sastrawi) project written in PHP (credits goes to the original author and contributors of Sastrawi PHP).
-
-
-[![Build Status](https://travis-ci.org/har07/PySastrawi.svg?branch=master)](https://travis-ci.org/har07/PySastrawi)
-[![Coverage Status](https://coveralls.io/repos/github/har07/PySastrawi/badge.svg?branch=master)](https://coveralls.io/github/har07/PySastrawi?branch=master)
-[![PyPI version](https://badge.fury.io/py/PySastrawi.svg)](https://badge.fury.io/py/PySastrawi)
-
-Cara Install
--------------
-
-Sastrawi dapat di-*install* menggunakan [pip](https://docs.python.org/3.6/installing/index.html), dengan menjalankan perintah berikut di terminal/command prompt : `pip install PySastrawi`
-
-Penggunaan
------------
-
-Jalankan baris-baris kode berikut di *Python interactive terminal* :
-
-```python
-# import StemmerFactory class
-from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
-
-# create stemmer
-factory = StemmerFactory()
-stemmer = factory.create_stemmer()
-
-# stemming process
-sentence = 'Perekonomian Indonesia sedang dalam pertumbuhan yang membanggakan'
-output   = stemmer.stem(sentence)
-
-print(output)
-# ekonomi indonesia sedang dalam tumbuh yang bangga
-
-print(stemmer.stem('Mereka meniru-nirukannya'))
-# mereka tiru
-```
-
-Demo
---------
-
-Live demo URL : https://pysastrawi-demo.appspot.com/
-
-Repository : https://github.com/har07/pystastrawi-demo
-
-Lisensi
---------
-
-Lisensi Sastrawi Python adalah MIT License (MIT).
-
-Project ini mengandung kamus kata dasar yang berasal dari Kateglo dengan lisensi [CC-BY-NC-SA 3.0](http://creativecommons.org/licenses/by-nc-sa/3.0/).
-
-Informasi Lebih Lanjut
-----------------------
-
-- [Sastrawi PHP Repository page](https://github.com/sastrawi/sastrawi)
+[![Coverage Status](https://coveralls.io/repos/har07/sastrawi/badge.svg?branch=development&service=github)](https://coveralls.io/github/har07/sastrawi?branch=development)

src/Sastrawi/Dictionary/ArrayDictionary.py

@@ -1,10 +1,17 @@
-class ArrayDictionary(object):
+from Sastrawi.Dictionary.DictionaryInterface import DictionaryInterface
+
+class ArrayDictionary(DictionaryInterface):
     """description of class"""
 
     def __init__(self, words=None):
-        self.words = {}
-        if words:
+        if words is None:
+            self.words = {}
+        elif type(words) is dict:
+            self.words = words
+        elif type(words) is list:
             self.add_words(words)
+        else:
+            self.words = {}
 
     def contains(self, word):
         return word in self.words
@@ -20,9 +27,4 @@ def add(self, word):
         """Add a word to the dictionary"""
         if not word or word.strip() == '':
             return
-        self.words[word]=word
-
-
-
-
-
+        self.words[word] = word

src/Sastrawi/Dictionary/DictionaryInterface.py

@@ -1,5 +1,12 @@
-class DictionaryInterface(object):
+# @update_by Mufid Jamaluddin
+# @update_date 16/03/2019
+
+from abc import ABCMeta, abstractmethod
+
+class DictionaryInterface:
     """Interface definition of dictionary"""
+    __metaclass__ = ABCMeta
 
+    @abstractmethod
     def contains(self, word):
-        raise NotImplementedError('you must implement this method manually')
+        pass

src/Sastrawi/Stemmer/Context/Context.py

@@ -160,5 +160,4 @@ def restore_prefix(self):
 
         for removal in self.removals:
             if removal.get_affix_type() == 'DP':
-                self.removals.remove(removal)
-
+                self.removals.remove(removal)

src/Sastrawi/Stemmer/Context/ContextInterface.py

@@ -1,30 +1,40 @@
-class ContextInterface(object):
-    """description of class"""
+# @update_by Mufid Jamaluddin
+# @update_date 16/03/2019
 
+from abc import ABCMeta, abstractmethod
+
+class ContextInterface:
+    """description of abs class"""
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
     def getOriginalWord(self):
         pass
 
+    @abstractmethod
     def setCurrentWord(self, word):
         pass
 
+    @abstractmethod
     def getCurrentWord(self):
         pass
 
+    @abstractmethod
     def getDictionary(self):
         pass
 
+    @abstractmethod
     def stopProcess(self):
         pass
 
+    @abstractmethod
     def processIsStopped(self):
         pass
 
+    @abstractmethod
     def addRemoval(self, removal):
         pass
 
+    @abstractmethod
     def getRemovals(self):
-        pass
-
-
-
-
+        pass

src/Sastrawi/Stemmer/Context/RemovalInterface.py

@@ -1,18 +1,29 @@
-class RemovalInterface(object):
+# @update_by Mufid Jamaluddin
+# @update_date 16/03/2019
+
+from abc import ABCMeta, abstractmethod
+
+class RemovalInterface:
     """description of class"""
+    __metaclass__ = ABCMeta
 
+    @abstractmethod
     def get_visitor(self):
         pass
 
+    @abstractmethod
     def get_subject(self):
         pass
 
+    @abstractmethod
     def get_result(self):
         pass
 
+    @abstractmethod
     def get_removed_part(self):
         pass
 
+    @abstractmethod
     def get_affix_type(self):
         pass
 

src/Sastrawi/Stemmer/Stemmer.py

@@ -2,6 +2,7 @@
 from Sastrawi.Stemmer.Context.Visitor.VisitorProvider import VisitorProvider
 from Sastrawi.Stemmer.Filter import TextNormalizer
 from Sastrawi.Stemmer.Context.Context import Context
+from cachetools import cached, LRUCache
 
 class Stemmer(object):
     """Indonesian Stemmer.
@@ -35,6 +36,16 @@ def stem_word(self, word):
         else:
             return self.stem_singular_word(word)
 
+    # Stemming word in Tokens
+    # @author Mufid Jamaluddin <[email protected]>
+    def stem_tokens(self, tokens):
+        stemmed_tokens = []
+        for token in tokens:
+            if not token or token.strip() == '':
+                continue
+            stemmed_tokens.append(self.stem_word(token))
+        return stemmed_tokens
+
     def is_plural(self, word):
         #-ku|-mu|-nya
         #nikmat-Ku, etc

src/Sastrawi/Stemmer/StemmerFactory.py

@@ -1,43 +1,37 @@
 import os
+from cachetools import cached, LRUCache
 from Sastrawi.Dictionary.ArrayDictionary import ArrayDictionary
 from Sastrawi.Stemmer.Stemmer import Stemmer
-from Sastrawi.Stemmer.CachedStemmer import CachedStemmer
-from Sastrawi.Stemmer.Cache.ArrayCache import ArrayCache
 
 class StemmerFactory(object):
     """ Stemmer factory helps creating pre-configured stemmer """
-    APC_KEY = 'sastrawi_cache_dictionary'
 
     def create_stemmer(self, isDev=False):
         """ Returns Stemmer instance """
+        if isDev:
+            words = self.get_words_from_file()
+            dictionary = ArrayDictionary(words)
+        else:
+            dictionary = self.get_prod_words_dictionary()
 
-        words = self.get_words(isDev)
-        dictionary = ArrayDictionary(words)
         stemmer = Stemmer(dictionary)
 
-        resultCache = ArrayCache()
-        cachedStemmer = CachedStemmer(resultCache, stemmer)
-
-        return cachedStemmer
+        return stemmer
 
-    def get_words(self, isDev=False):
-        #if isDev or callable(getattr(self, 'apc_fetch')):
-        #    words = self.getWordsFromFile()
-        #else:
-        #    words = apc_fetch(self.APC_KEY)
-        #    if not words:
-        #        words = self.getWordsFromFile()
-        #        apc_store(self.APC_KEY, words)
-        return self.get_words_from_file()
+    @cached(cache=LRUCache(maxsize=32))
+    def get_prod_words_dictionary(self):
+        words = self.get_words_from_file()
+        dictionary = ArrayDictionary(words)
+        return dictionary
 
     def get_words_from_file(self):
         current_dir = os.path.dirname(os.path.realpath(__file__))
         dictionaryFile = current_dir + '/data/kata-dasar.txt'
+
         if not os.path.isfile(dictionaryFile):
             raise RuntimeError('Dictionary file is missing. It seems that your installation is corrupted.')
 
-        dictionaryContent = ''
+        text = ''
         with open(dictionaryFile, 'r') as f:
-            dictionaryContent = f.read()
-
-        return dictionaryContent.split('\n')
+            text = f.read()
+        return text.split('\n')

src/Sastrawi/StopWordRemover/StopWordRemover.py

@@ -14,6 +14,8 @@ def remove(self, text):
 
         return ' '.join(stopped_words)
 
-
-
-
+    # Remove Stopword in Tokens
+    # @author Mufid Jamaluddin <[email protected]>
+    def remove_tokens(self, tokens):
+        clean_tokens = [token for token in tokens if not self.dictionary.contains(token)]
+        return clean_tokens