add more debug info if json format is wrong

MacHu-GWU · Jan 3, 2019 · 2ef6e0f · 2ef6e0f
1 parent df509c7
commit 2ef6e0f
Show file tree

Hide file tree

Showing 12 changed files with 114 additions and 69 deletions.
diff --git a/README.rst b/README.rst
@@ -1,37 +1,17 @@
 
-.. image:: https://readthedocs.org/projects/afwf_fts_anything/badge/?version=latest
-    :target: https://afwf_fts_anything.readthedocs.io/?badge=latest
-    :alt: Documentation Status
-
 .. image:: https://travis-ci.org/MacHu-GWU/afwf_fts_anything-project.svg?branch=master
     :target: https://travis-ci.org/MacHu-GWU/afwf_fts_anything-project?branch=master
 
 .. image:: https://codecov.io/gh/MacHu-GWU/afwf_fts_anything-project/branch/master/graph/badge.svg
   :target: https://codecov.io/gh/MacHu-GWU/afwf_fts_anything-project
 
-.. image:: https://img.shields.io/pypi/v/afwf_fts_anything.svg
-    :target: https://pypi.python.org/pypi/afwf_fts_anything
-
-.. image:: https://img.shields.io/pypi/l/afwf_fts_anything.svg
-    :target: https://pypi.python.org/pypi/afwf_fts_anything
-
-.. image:: https://img.shields.io/pypi/pyversions/afwf_fts_anything.svg
-    :target: https://pypi.python.org/pypi/afwf_fts_anything
-
 .. image:: https://img.shields.io/badge/STAR_Me_on_GitHub!--None.svg?style=social
     :target: https://github.com/MacHu-GWU/afwf_fts_anything-project
 
 ------
 
-
 .. image:: https://img.shields.io/badge/Link-Document-blue.svg
-      :target: https://afwf_fts_anything.readthedocs.io/index.html
-
-.. image:: https://img.shields.io/badge/Link-API-blue.svg
-      :target: https://afwf_fts_anything.readthedocs.io/py-modindex.html
-
-.. image:: https://img.shields.io/badge/Link-Source_Code-blue.svg
-      :target: https://afwf_fts_anything.readthedocs.io/py-modindex.html
+      :target: https://github.com/MacHu-GWU/afwf_fts_anything-project
 
 .. image:: https://img.shields.io/badge/Link-Install-blue.svg
       :target: `install`_
@@ -46,7 +26,7 @@
       :target: https://github.com/MacHu-GWU/afwf_fts_anything-project/issues
 
 .. image:: https://img.shields.io/badge/Link-Download-blue.svg
-      :target: https://pypi.org/pypi/afwf_fts_anything#files
+      :target: https://github.com/MacHu-GWU/afwf_fts_anything-project/releases
 
 
 The Alfred Workflow: Full Text Search Anything
@@ -117,8 +97,14 @@ Search Setting (content of ``movie-setting.json``):
     }
 
 
-Installation
+Note: ``fts.anything`` support comment in json.
+
+
+.. _install:
+
+Install
 ------------------------------------------------------------------------------
+Go to `Release <https://github.com/MacHu-GWU/afwf_fts_anything-project/releases>`_, download the latest ``Full-Text-Search-Anything.alfredworkflow``. And double click to install to alfred.
 
 
 Usage
@@ -172,7 +158,7 @@ It is a dictonary with 6 fields:
                 "type_is_keyword": true
             }
         ],
-        "title_field": "title", // title on Alfred drop down menu
+        "title_field": "{title} ({genres})", // title on Alfred drop down menu
         "subtitle_field": "description", // subtitle on Alfred drop down menu
         "arg_field": "movie_id", // argument for other workflow component
         "autocomplete_field": "{movie_id} - {title}", // tab auto complete behavior
@@ -210,7 +196,7 @@ column setting template:
 
 ``title_field``, ``subtitle_field``, ``arg_field``, ``autocomplete_field``, ``icon_field`` defines how you want to construct drop down items. By default, everything is None. Let's use ``title_field`` as an example:
 
-1. if ``title_field`` is not defined, use the ``"title"`` field in record, this **will raise error** if ``"title"`` field not exist.
+1. if ``title_field`` is not defined, use the ``"title"`` field in the record, this **will raise error** if ``"title"`` field not exist.
 2. if ``title_field`` is a string, let's say it is ``"movie_title"``, test if it is one of columns fields, if true, then use that field (``"movie_title"``)for title.
 3. if ``title_field`` is a str, but not in columns fields, it must be a `Python String Format Template <https://docs.python.org/3/library/string.html#format-examples>`_. For example: ``{movie_id} - {title}``.
 
@@ -220,3 +206,6 @@ FAQ
 
 - Q: Why use json, why not CSV?
 - A: json provides more flexibility and compatible with multi-line text, which CSV usually not.
+
+- Q: Why it still returns old data after I updated the dataset?
+- A: Just delete the ``${HOME}/.alfred-fts/<dataname>-whoosh_index`` directory.
diff --git a/afwf_fts_anything/__init__.py b/afwf_fts_anything/__init__.py
@@ -11,4 +11,4 @@
 __author_email__ = "[email protected]"
 __maintainer__ = "Sanhe Hu"
 __maintainer_email__ = "[email protected]"
-__github_username__ = "MacHu-GWU"
+__github_username__ = "MacHu-GWU"
diff --git a/afwf_fts_anything/dataset.py b/afwf_fts_anything/dataset.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import attr
+import shutil
 from attrs_mate import AttrsClass
 from whoosh import index, qparser
 from pathlib_mate import PathCls as Path
@@ -37,13 +38,15 @@ def update_data_from_file(self):
         if self.data is None:
             data_file = self.get_data_file_path()
             with open(data_file.abspath, "rb") as f:
-                self.data = json.loads(f.read().decode("utf-8"), ignore_comments=True)
+                self.data = json.loads(f.read().decode(
+                    "utf-8"), ignore_comments=True)
 
     def update_setting_from_file(self):
         if not self.setting.columns:
             setting_file = self.get_setting_file_path()
             with open(setting_file.abspath, "rb") as f:
-                setting_data = json.loads(f.read().decode("utf-8"), ignore_comments=True)
+                setting_data = json.loads(
+                    f.read().decode("utf-8"), ignore_comments=True)
                 self.setting = Setting(**setting_data)
 
     def get_data_file_path(self):
@@ -81,6 +84,12 @@ def build_index(self, idx):
             writer.add_document(**doc)
         writer.commit()
 
+    def remove_index(self):
+        """
+        Remove whoosh index dir.
+        """
+        shutil.rmtree(self.get_index_dir_path().abspath)
+
     def search(self, query_str, limit=20):
         """
         Use full text search for result.
@@ -92,5 +101,6 @@ def search(self, query_str, limit=20):
             schema=schema,
         ).parse(query_str)
         with idx.searcher() as searcher:
-            result = [hit.fields() for hit in searcher.search(query, limit=limit)]
+            result = [hit.fields()
+                      for hit in searcher.search(query, limit=limit)]
         return result
diff --git a/afwf_fts_anything/docs/__init__.py b/afwf_fts_anything/docs/__init__.py
@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 
-doc_data = dict()
+doc_data = dict()
diff --git a/afwf_fts_anything/fts_setting.py b/afwf_fts_anything/fts_setting.py
@@ -29,7 +29,8 @@ class ColumnSetting(AttrsClass):
     keyword_commas = attr.ib(default=True)
 
     def __attrs_post_init__(self):
-        flag = self.type_is_store + self.type_is_ngram + self.type_is_phrase + self.type_is_keyword
+        flag = self.type_is_store + self.type_is_ngram + \
+            self.type_is_phrase + self.type_is_keyword
         if flag == 1:
             pass
         elif flag < 1:
@@ -151,7 +152,8 @@ def convert_to_item(self, doc):
         """
         # whoosh 所返回的 doc 中并不一定所有项都有, 有的项可能没有, 我们先为这些
         # 没有的项赋值 None
-        doc = {c_setting.name: doc.get(c_setting.name) for c_setting in self.columns}
+        doc = {c_setting.name: doc.get(c_setting.name)
+               for c_setting in self.columns}
         item_data = dict()
 
         # find corresponding value for every workflow item field

diff --git a/afwf_fts_anything/handlers.py b/afwf_fts_anything/handlers.py
@@ -27,20 +27,56 @@ def main(wf, args=None):
     elif n_args >= 2:
         dataset_name = args[0]
         dataset = DataSet(dataset_name)
-        dataset.update_setting_from_file()
+
+        try:
+            dataset.update_setting_from_file()
+        except Exception as e:
+            wf.add_item(
+                title="{}-setting.json format is broken!".format(dataset_name),
+                subtitle="please check for unexpected trailing comma!",
+                valid=True,
+            )
+            return wf
+
         index_dir = dataset.get_index_dir_path()
         if index_dir.exists():
             pass
         else:
             idx = dataset.get_index()
-            dataset.update_data_from_file()
-            dataset.build_index(idx)
+            try:
+                dataset.update_data_from_file()
+            except Exception as e:
+                dataset.remove_index()
+                wf.add_item(
+                    title="{}.json format is broken!".format(dataset_name),
+                    subtitle="please check for unexpected trailing comma!",
+                    valid=True,
+                )
+                return wf
+            try:
+                dataset.build_index(idx)
+            except Exception as e:
+                dataset.remove_index()
+                wf.add_item(
+                    title="data is not compatible with your settings!".format(
+                        dataset_name),
+                    valid=True,
+                )
+                return wf
         query_str = " ".join(args[1:])
         result = dataset.search(query_str)
         if len(result):
             for doc in result:
-                item = dataset.setting.convert_to_item(doc)
-                wf.add_item(valid=True, **item.to_dict())
+                try:
+                    item = dataset.setting.convert_to_item(doc)
+                    wf.add_item(valid=True, **item.to_dict())
+                except Exception as e:
+                    wf.add_item(
+                        title="unable to convert this record to item",
+                        subtitle=str(doc),
+                        valid=True,
+                    )
+
         else:
             wf.add_item(
                 title=MSG_FOUND_NOTHING,

diff --git a/afwf_fts_anything/helpers.py b/afwf_fts_anything/helpers.py
@@ -3,7 +3,7 @@
 import hashlib
 
 
-def md5_file(path):
+def md5_file(path):  # pragma: no cover
     """
     Get md5 check sum of a file.
     """

diff --git a/alfred-readme.txt b/alfred-readme.txt
@@ -0,0 +1,5 @@
+``fts.anything`` is an `Alfred Workflow <https://www.alfredapp.com/workflows/>`_ allow you to **custom full-text search on your own dataset**. You can easily define **which fields you want to search**, **how you want the data to be matched** and **send the result to other workflow to process**.
+
+User manual and Document can be found at:
+
+https://github.com/MacHu-GWU/afwf_fts_anything-project
diff --git a/tests/all.py b/tests/all.py
@@ -4,4 +4,4 @@
 if __name__ == "__main__":
     import pytest
 
-    pytest.main(["-s", "--tb=native"])
+    pytest.main(["-s", "--tb=native"])
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
@@ -61,7 +61,8 @@ def setup_class(cls):
         cls.data = movie_data
         cls.setting = movie_setting
 
-        dataset = DataSet(name=dataset_name, data=None, setting=Setting(skip_post_init=True))
+        dataset = DataSet(name=dataset_name, data=None,
+                          setting=Setting(skip_post_init=True))
         data_file_path = dataset.get_data_file_path()
         setting_file_path = dataset.get_setting_file_path()
         index_dir = dataset.get_index_dir_path()

diff --git a/tests/test_fts_setting.py b/tests/test_fts_setting.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
 import pytest
 from pathlib_mate import PathCls as Path
 from afwf_fts_anything.constant import ALFRED_FTS
@@ -12,7 +13,7 @@ def test_type(self):
             ColumnSetting(name="a")
         with pytest.raises(ValueError):
             ColumnSetting(name="a",
-                type_is_ngram=True, type_is_phrase=True, type_is_keyword=True)
+                          type_is_ngram=True, type_is_phrase=True, type_is_keyword=True)
 
 
 class TestSetting(object):
@@ -30,40 +31,41 @@ class TestSetting(object):
         icon_field=Path(ALFRED_FTS, "movie-icon.png").abspath,
     )
     setting2 = Setting.from_dict({
-            "columns": [
-                {
-                    "name": "movie_id",
-                    "type_is_store": True,
-                },
-                {
-                    "name": "title",
-                    "type_is_ngram": True,
-                    "ngram_minsize": 2,
-                    "ngram_maxsize": 10,
-                },
-                {
-                    "name": "description",
-                    "type_is_phrase": True,
-                },
-                {
-                    "name": "genres",
-                    "type_is_keyword": True,
-                    "keyword_lowercase": True,
-                },
-            ],
-            "title_field": "title",
-            "subtitle_field": "description",
-            "arg_field": "movie_id",
-            "autocomplete_field": "{movie_id} - {title}",
-            "icon_field": Path(ALFRED_FTS, "movie-icon.png").abspath,
+        "columns": [
+            {
+                "name": "movie_id",
+                "type_is_store": True,
+            },
+            {
+                "name": "title",
+                "type_is_ngram": True,
+                "ngram_minsize": 2,
+                "ngram_maxsize": 10,
+            },
+            {
+                "name": "description",
+                "type_is_phrase": True,
+            },
+            {
+                "name": "genres",
+                "type_is_keyword": True,
+                "keyword_lowercase": True,
+            },
+        ],
+        "title_field": "title",
+        "subtitle_field": "description",
+        "arg_field": "movie_id",
+        "autocomplete_field": "{movie_id} - {title}",
+        "icon_field": Path(ALFRED_FTS, "movie-icon.png").abspath,
     })
 
     def test_columns_property(self):
         assert self.setting1.store_columns == ["movie_id", ]
         assert self.setting1.ngram_columns == ["title", ]
         assert self.setting1.phrase_columns == ["description", ]
         assert self.setting1.keyword_columns == ["genres", ]
-        assert self.setting1.searchable_columns == ["title", "description", "genres"]
+        assert self.setting1.searchable_columns == [
+            "title", "description", "genres"]
 
     def test_create_whoosh_schema(self):
         schema = self.setting1.create_whoosh_schema()

diff --git a/tests/test_import.py b/tests/test_import.py
@@ -14,4 +14,4 @@ def test():
     import os
 
     basename = os.path.basename(__file__)
-    pytest.main([basename, "-s", "--tb=native"])
+    pytest.main([basename, "-s", "--tb=native"])