Update __init__.py

sloved unicode error in some charset like gb2312 etc.
add gettitle  getdate method and remove getimage.
add segement load source from local file.

src/boilerpipe/extract/__init__.py

@@ -3,6 +3,7 @@
 import socket
 import charade
 import threading
+import re
 
 socket.setdefaulttimeout(15)
 lock = threading.Lock()
@@ -18,7 +19,8 @@ class Extractor(object):
     being one of the boilerpipe extractors:
     - DefaultExtractor
     - ArticleExtractor
-    - ArticleSentencesExtractor
+    - ArticleSentencesExtractor                      file='/home/mj/t20150528_653893.htm'
+
     - KeepEverythingExtractor
     - KeepEverythingWithMinKWordsExtractor
     - LargestContentExtractor
@@ -29,7 +31,7 @@ class Extractor(object):
     source    = None
     data      = None
     headers   = {'User-Agent': 'Mozilla/5.0'}
-    
+
     def __init__(self, extractor='DefaultExtractor', **kwargs):
         if kwargs.get('url'):
             request     = urllib2.Request(kwargs['url'], headers=self.headers)
@@ -38,11 +40,34 @@ def __init__(self, extractor='DefaultExtractor', **kwargs):
             encoding    = connection.headers['content-type'].lower().split('charset=')[-1]
             if encoding.lower() == 'text/html':
                 encoding = charade.detect(self.data)['encoding']
-            self.data = unicode(self.data, encoding)
+            # self.data = unicode(self.data, 'gbk')
+            #self.data = self.data.decode(encoding, 'ignore')
+            try:
+                self.data = unicode(self.data, charade.detect(self.data)['encoding'])
+            except UnicodeError:
+                encoding = charade.detect(self.data)['encoding']
+                self.data = self.data.decode(encoding, 'ignore')
         elif kwargs.get('html'):
             self.data = kwargs['html']
             if not isinstance(self.data, unicode):
-                self.data = unicode(self.data, charade.detect(self.data)['encoding'])
+                # self.data = unicode(self.data, charade.detect(self.data)['encoding'])
+                try:
+                    self.data = unicode(self.data, charade.detect(self.data)['encoding'])
+                except UnicodeError:
+                    encoding = charade.detect(self.data)['encoding']
+                    self.data = self.data.decode(encoding, 'ignore')
+        ## Extractor(extractor='ArticleExtractor',file='/tmp/a.html')
+        elif kwargs.get('file'):
+            Path = kwargs['file']
+            f = open(Path, 'r')
+            self.data = f.read()
+            if not isinstance(self.data, unicode):
+                try:
+                    self.data = unicode(self.data, charade.detect(self.data)['encoding'])
+                except UnicodeError:
+                    encoding = charade.detect(self.data)['encoding']
+                    self.data = self.data.decode(encoding, 'ignore')
+
         else:
             raise Exception('No text or url provided')
 
@@ -52,35 +77,26 @@ def __init__(self, extractor='DefaultExtractor', **kwargs):
                 if jpype.isThreadAttachedToJVM() == False:
                     jpype.attachThreadToJVM()
             lock.acquire()
-            
+
             self.extractor = jpype.JClass(
                 "de.l3s.boilerpipe.extractors."+extractor).INSTANCE
         finally:
             lock.release()
-    
+
         reader = StringReader(self.data)
         self.source = BoilerpipeSAXInput(InputSource(reader)).getTextDocument()
         self.extractor.process(self.source)
-    
+
     def getText(self):
         return self.source.getContent()
-
+
+    def getTitle(self):
+        return self.source.getTitle()
+
     def getHTML(self):
         highlighter = HTMLHighlighter.newExtractingInstance()
         return highlighter.process(self.source, self.data)
-
-    def getImages(self):
-        extractor = jpype.JClass(
-            "de.l3s.boilerpipe.sax.ImageExtractor").INSTANCE
-        images = extractor.process(self.source, self.data)
-        jpype.java.util.Collections.sort(images)
-        images = [
-            {
-                'src'   : image.getSrc(),
-                'width' : image.getWidth(),
-                'height': image.getHeight(),
-                'alt'   : image.getAlt(),
-                'area'  : image.getArea()
-            } for image in images
-        ]
-        return images
+
+    def getDate(self):
+        r='(19[7-9][0-9]|20[0-1][0-9])-(0[1-9]|1[0-2])-([1-2][0-9]|0[1-9]|3[0-1]) ([0-1][0-9]|2[0-4]):([0-5][0-9]):([0-5][0-9])'
+        return re.search(r,self.data).group()