diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..02c1e6d --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,9 @@ +[bumpversion] +current_version = 0.0.12 +commit = False +tag = False + +[bumpversion:file:setup.py] + +[bumpversion:file:termfeed/feed.py] + diff --git a/.gitignore b/.gitignore index ba74660..a8c42eb 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ docs/_build/ # PyBuilder target/ + +#IDE's +.idea/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4476759 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ + + +help: + cat Makefile + +install: + pip install . + +.PHONY: tests +tests: + nosetests tests/ + +feed: + python -m feed + +bumpversion_patch: + bumpversion patch + +bumpversion_minor: + bumpversion minor + +bumpversion_mayor: + bumpversion mayor diff --git a/README.md b/README.md index 4cc5935..c037269 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,31 @@ To read, preview, open, store, or delete your favorite RSS feeds from the comman If 1) you are a terminal addict, and 2) you want to stay up to date with the outside world by reading quick feed and summaries WITHOUT having to leave your terminal; then TermFeed is for you. These are the main reasons I created TermFeed. +# Changes agains original +```Usage: feed [OPTIONS] COMMAND [ARGS]... + +Options: + -l, --label TEXT + -n, --dry-run + -v, --verbose Levels: -v:INFO, -vvv:DEBUG + -h, --help Show this message and exit. + +Commands: + add + browse + edit + load + remove + show + ``` + +Config file is a termdeef/db.yaml +Modifications are made to support git repos. + + + +# Old Readme ### Usage @@ -24,9 +48,9 @@ If 1) you are a terminal addict, and 2) you want to stay up to date with the out - browse latest feed from the single link `` provided. - e.g. `$ feed https://news.ycombinator.com/rss` -`$ feed -b` +`$ feed -b []` -- browse latest feeds by category of your library. +- browse latest feeds by of your library. If is missing select input appear. `$ feed -t` @@ -51,9 +75,9 @@ If 1) you are a terminal addict, and 2) you want to stay up to date with the out `$ feed -D ` - Remove entire category (with its URLs) from library. -`$ feed -R` +`$ feed -R []` -- rebuild the library from `urls.py` +- rebuild the library. Default `rss.yaml` ### Features (what you can do?) diff --git a/setup.py b/setup.py index 6fe0463..12ca621 100644 --- a/setup.py +++ b/setup.py @@ -8,15 +8,23 @@ setup( name='TermFeed', - description=('Browse, read, and open your favorite rss feed in the terminal (without curses).'), + description=( + 'Browse, read, and open your favorite rss feed in the terminal (without curses).'), author='Aziz Alto', url='https://github.com/iamaziz/TermFeed', download_url='https://github.com/iamaziz/TermFeed/archive/master.zip', - license = "MIT", + license="MIT", author_email='iamaziz.alto@gmail.com', - version='0.0.11', - install_requires=['feedparser'], - packages=['termfeed', 'termfeed.support'], + version='0.0.12', + install_requires=[ + 'feedparser', + 'pyyaml', + 'docopt', + 'plumbum', + 'arrow', + 'cached-property>=1.3.0', + ], + packages=['termfeed'], scripts=[], entry_points={ 'console_scripts': [ diff --git a/termfeed/__main__.py b/termfeed/__main__.py new file mode 100644 index 0000000..d4ddb77 --- /dev/null +++ b/termfeed/__main__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- + +from .feed import main + +if __name__ == '__main__': + + main() diff --git a/termfeed/database.py b/termfeed/database.py new file mode 100644 index 0000000..7f2d553 --- /dev/null +++ b/termfeed/database.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- + +""" +database operations. + +database.py manipulate database add, update, delete +""" + +import yaml, json, re +from plumbum import local +from plumbum import colors as c +from os import path +from cached_property import cached_property +from pathlib import Path +import logging +logger = logging.getLogger(__name__) + +log_info = logger.info +log_debug = logger.debug + + +class DataBase: + + file = local.env.home / '.termfeed.json' + __data = None + dry_run = False + + @cached_property + def data(self): + # The following will only called once + log_info('Load library') + if not self.__data: + if not self.file.exists(): + file = local.path(__file__).dirname / 'db.yaml' + if not file.exists(): + raise FileNotFoundError(file) + with open(file, 'r') as f: + log_info('Open yaml') + return yaml.load(f) + else: + with open(self.file, 'r') as f: + log_info('Open json') + self.__data = json.load(f) + return self.__data.copy() # ensure copy, for comp in __del__ + + # def set_data(self, data): + # verify_data(data) + # self.__data = data + # del self.data + # with self: + # pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # type, value, traceback + verify_data(self.data) + if not self.dry_run: + print('Backup changed library in {}.'.format(self.file)) + with open(self.file, 'w') as f: + json.dump(self.data, f) + log_info('Write to json') + log_debug(json.dumps(self.data)) + log_debug(self.as_yaml) + else: + print('Normaly would write the following to file {}: '.format(self.file)) + print(c.highlight | c.black | json.dumps(self.data)) + + debug_flag = False + + @property + def labels(self): + labels = set() + for _, val in self.data.items(): + labels |= set(val['label']) + return labels + + @property + def as_yaml(self): + return yaml.dump(self.data) # , default_flow_style=False + + def link_for_label(self, *labels): + return [link for link, val in self.data.items() if any(True for l in labels if l in val['label'])] + + @property + def links(self): + return self.data.keys() + + def browse_links(self, label): + if label in self.labels: + links = self(label) + print('{} resources:'.format(label)) + for link in links: + print('\t{}'.format(link)) + else: + print('no category named {}'.format(label)) + print(self) + + def __str__(self): + out = 'available lables: \n\t' + '\n\t'.join(self.labels) + return(out) + + def print_labels(self): + print(self) + + def link_as_yaml(self, link): + return yaml.dump({link:self.data[link]}) + + def add_link(self, link, *labels, flag = None, title=''): + if link not in self.data: + if not flag: + flag = [] + template = dict(title=title, flag=flag, label=list(labels)) + self.data[link] = verify_entry(template, link) + print('Added:') + print(self.link_as_yaml(link)) + if logger.level <= logging.INFO: + print(self.as_yaml) + elif not title == '' or not title == self.data[link]['title']: + self.data[link]['label'] = list(set(self.data[link]['label']) | set(labels)) + self.data[link]['title'] = title + log_info('Title has changed') + print(self.as_yaml) + + elif set(labels) | set(self.data[link]['label']) == set(self.data[link]['label']): + print('{} already exists and has all labels: {}!!'.format(link, labels)) + print(self.as_yaml) + else: + self.data[link]['label'] = list(set(self.data[link]['label']) | set(labels)) + # print('Created new category .. {}'.format(topic)) + print(self.link_as_yaml(link)) + if logger.level <= logging.INFO: + print(self.as_yaml) + + def remove_link(self, link): + done = False + if link in self.data: + del self.data[link] + print('removed: {}'.format(link)) + else: + print('URL not found: {}'.format(link)) + + def delete_topic(self, label): + if label == '': + print('Default topic "General" cannot be removed.') + exit() + try: + for link in self.data: + if label in self.data[link]['label']: + self.data[link]['label'] = list(set(self.data[link]['label']) - set(label)) + print('Removed "{}" from your library.'.format(label)) + except KeyError: + print('"{}" is not in your library!'.format(label)) + exit() + + +def verify_entry(entry, link): + allowed_keys = {'label', 'flag', 'title'} + if not entry: + entry = dict() + if not (entry.keys() | allowed_keys) == allowed_keys: + print('The url {} has invalid keys: '.format(link), entry) + exit() + if not isinstance(entry.setdefault('title', ''), str): + print('The url {} has invalid title member: {}'.format(link, entry['title'])) + exit() + if not isinstance(entry.setdefault('flag', []), list): + print('The url {} has invalid flag type: {}'.format(link, entry['flag'])) + exit() + if not all([isinstance(f, str) for f in entry['flag']]): + print('The url {} has invalid flag member: {}'.format(link, entry['flag'])) + exit() + if not isinstance(entry.setdefault('label', []), list): + print('The url {} has invalid flag type: {}'.format(link, entry['label'])) + exit() + if not all([isinstance(l, str) for l in entry['label']]): + print('The url {} has invalid flag member: {}'.format(link, entry['label'])) + exit() + return entry + + +def verify_data(data): + for link in data: + data[link] = verify_entry(data[link], link) + +# if __name__ == '__main__': + +# for l in read('News'): +# print(l) + +# remove_link('http://rt.com/rss/') + +# add_link('http://rt.com/rss/', 'News') + +# for l in read('News'): +# print(l) diff --git a/termfeed/db.yaml b/termfeed/db.yaml new file mode 100644 index 0000000..e808937 --- /dev/null +++ b/termfeed/db.yaml @@ -0,0 +1,13 @@ +https://github.com/boeddeker/TermFeed/commits/master.atom: + flag: [git, github] + label: [github] + title: TermFeed +https://github.com/boeddeker/TermFeed/commits/master.atom: + flag: [git, github] + label: [github] + title: TermFeed +http://blog.jupyter.org/rss/: + flag: [] + label: [github] + title: blog.jupyter + diff --git a/termfeed/dbinit.py b/termfeed/dbinit.py deleted file mode 100644 index c98fcb9..0000000 --- a/termfeed/dbinit.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -# This should be exectued once to initialize the db from urls.py - -import shelve -from os import path - -from termfeed.urls import rss - -homedir = path.expanduser('~') - -# initiate database datafile -d = shelve.open(path.join(homedir, '.termfeed')) - - -# dump urls.py into rss_shelf.db -for topic in rss: - links = rss[topic] - d[topic] = [link for link in links] - -d.close() diff --git a/termfeed/dbop.py b/termfeed/dbop.py deleted file mode 100644 index 6c40d4a..0000000 --- a/termfeed/dbop.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- - -""" -database operations. - -dbop.py manipulate database add, update, delete -""" - -import shelve -from os import path - -homedir = path.expanduser('~') - -def rebuild_library(): - import termfeed.dbinit - print('created ".termfeed.db" in {}'.format(homedir)) - -# instantiate db if it's not created yet -if not path.exists(homedir + '/.termfeed.db'): - rebuild_library() - - -# connect to db -d = shelve.open(path.join(homedir, '.termfeed'), 'w') - - -def topics(): - return d.keys() - - -def read(topic): - if topic in d.keys(): - return d[topic] - else: - return None - - -def browse_links(topic): - if topic in d.keys(): - links = d[topic] - print('{} resources:'.format(topic)) - for link in links: - print('\t{}'.format(link)) - else: - print('no category named {}'.format(topic)) - print_topics() - - -def print_topics(): - print('available topics: ') - for t in topics(): - print('\t{}'.format(t)) - - -def add_link(link, topic='General'): - - if topic in d.keys(): - if link not in d[topic]: - # to add a new url: copy, mutates, store back - temp = d[topic] - temp.append(link) - d[topic] = temp - print('Updated .. {}'.format(topic)) - else: - print('{} already exists in {}!!'.format(link, topic)) - else: - print('Created new category .. {}'.format(topic)) - d[topic] = [link] - - -def remove_link(link): - - done = False - for topic in topics(): - if link in d[topic]: - d[topic] = [l for l in d[topic] if l != link] - print('removed: {}\nfrom: {}'.format(link, topic)) - done = True - - if not done: - print('URL not found: {}'.format(link)) - - -def delete_topic(topic): - if topic == 'General': - print('Default topic "General" cannot be removed.') - exit() - try: - del d[topic] - print('Removed "{}" from your library.'.format(topic)) - except KeyError: - print('"{}" is not in your library!'.format(topic)) - exit() - - -# if __name__ == '__main__': - -# for l in read('News'): -# print(l) - -# remove_link('http://rt.com/rss/') - -# add_link('http://rt.com/rss/', 'News') - -# for l in read('News'): -# print(l) diff --git a/termfeed/feed.py b/termfeed/feed.py index b1f9d92..44f8c33 100755 --- a/termfeed/feed.py +++ b/termfeed/feed.py @@ -1,57 +1,31 @@ #!/usr/bin/env python -#-*- coding: utf-8 -*- - -"""TermFeed 0.0.11 - -Usage: - feed - feed - feed -b - feed -a [] - feed -d - feed -t [] - feed -D - feed -R - feed (-h | --help) - feed --version - -Options: - List feeds from the default category 'General' of your library. - List feeds from the provided url source. - -b Browse feed by category avaialble in the database file. - -a URL Add new url to database under [] (or 'General' otherwise). - -d URL Delete from the database file. - -t See the stored categories in your library, or list the URLs stored under in your library. - -D TOPIC Remove entire cateogry (and its urls) from your library. - -R Rebuild the library from the url.py - -h --help Show this screen. - -""" - - -from __future__ import print_function -import sys + +import logging +import os +import re import webbrowser + +import arrow +import click +import dateutil.parser import feedparser -import re +import yaml +yaml.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list) +yaml.add_representer(feedparser.FeedParserDict, + yaml.representer.SafeRepresenter.represent_dict) -try: - from urllib import urlopen -except ImportError: - from urllib.request import urlopen +from bs4 import BeautifulSoup +from plumbum import colors as c +from tabulate import tabulate +from urllib.request import urlopen +import termfeed.database -import termfeed.dbop as dbop +logger = logging.getLogger(__name__) +log_info = logger.info +log_debug = logger.debug -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' +dbop = termfeed.database.DataBase() def _connected(): @@ -66,62 +40,123 @@ def _connected(): def open_page(url, title): - print(bcolors.WARNING + - '\topening ... {}\n'.format(title.encode('utf8')) + bcolors.ENDC) + with c.info: + print('\topening ... {}\n'.format(title.encode('utf8'))) # open page in browser webbrowser.open(url) def print_feed(zipped): + # keys() + # dict_keys(['id', 'links', 'summary', 'author', 'guidislink', + # 'author_detail', 'link', 'summary_detail', 'published', 'content', + # 'authors', 'published_parsed', 'title', 'title_detail']) + + def parse_time(post): + try: + return c.info | arrow.get(dateutil.parser.parse(post.published)).humanize() + except: + return c.info | arrow.get(dateutil.parser.parse(post.updated)).humanize() - for num, post in zipped.items(): - print(bcolors.OKGREEN + '[{}] '.format(num) + bcolors.ENDC, end='') - print('{}'.format(post.title.encode('utf8'))) + def parse_author(post): + try: + if post.author_detail.name: + return post.author_detail.name + else: + return post.author_detail.email.split('@')[0] + except AttributeError: + return 'unknown' + except BaseException as e: + print(c.red | yaml.dump(post)) + import sys + raise e + + # r = re.compile(r'(\w+)(?:/commits/\w+\.atom|\.git)') + + def repo(post): + try: + return dbop.data[post.title_detail.base]['title'] + except: + print('Keys: ', dbop.data.keys()) + raise + + # try: + table = [[c.green | '[{}]'.format(num), + repo(post), + parse_time(post), + c.dark_gray | parse_author(post), + post.title, + ] for num, post in reversed(list(zipped.items()))] + # except AttributeError as e: + # print('Bug:', post.keys()) + # print(post.published) + # print(c.magenta | yaml.dump(post)) + # print(post.title) + # print(post.title_detail) + # raise e + # else: + # print('Bug:', dir(post), post.keys()) + + print(tabulate(table, tablefmt="plain")) # , tablefmt="plain")) + + +def print_desc(topic, txt, post): + with c.info: + try: + print('\n\nTitle : {}:'.format(topic)) -def print_desc(topic, txt): - try: - print(bcolors.WARNING + '\n\n{}:'.format(topic) + bcolors.ENDC) - except UnicodeEncodeError: - pass - print(bcolors.BOLD + '\n\t{}'.format(txt.encode('utf8')) + bcolors.ENDC) + except UnicodeEncodeError: + pass + with c.dark_gray: + yaml.add_representer( + tuple, yaml.representer.SafeRepresenter.represent_list) + yaml.add_representer(feedparser.FeedParserDict, + yaml.representer.SafeRepresenter.represent_dict) + + import copy + post_copy = copy.deepcopy(post) + + # post_copy = post.copy() + for key in list(post_copy.keys()): + if '_parsed' in key \ + or '_detail' in key \ + or key in ('guidislink', 'link', 'links', 'id', 'summary', 'authors', 'title'): + del post_copy[key] + if 'content' is key: + for i in range(len(post_copy[key])): + post_copy[key][i] = clean_txt(post_copy[key][i]['value']) + print(yaml.dump(post_copy)) def open_it(): - try: - txt = '\n\n\t Open it in browser ? [y/n] ' - try: - q = raw_input(txt) # python 2 - except NameError: - q = input(txt) # python 3 + if os.environ.get('DISPLAY'): + return click.confirm('\n\n\t Open it in browser ?') - print('\n') - if q == 'y': - return True - except KeyboardInterrupt: - print('\n') + elif click.confirm('No display aviable, do you want to continue?', default=True, show_default=False): + log_info('Confirm True') return False + else: + log_info('Confirm False') + exit() + return False + + def clean_txt(txt): """clean txt from e.g. html tags""" - cleaned = re.sub(r'<.*?>', '', txt) # remove html - cleaned = cleaned.replace('<', '<').replace('>', '>') # retain html code tags - cleaned = cleaned.replace('"', '"') - cleaned = cleaned.replace('’', "'") - cleaned = cleaned.replace(' ', ' ') # italized text - return cleaned + clean_text = BeautifulSoup(txt, "html.parser").text + return clean_text + def _continue(): try: - msg = """\n\nPress: Enter to continue, ... [NUM] for short description / open a page, ... or CTRL-C to exit: """ - print(bcolors.FAIL + msg + bcolors.ENDC, end='') - # kb is the pressed keyboard key - try: - kb = raw_input() - except NameError: - kb = input() - return kb + msg = """\nPress: Enter to continue, ... [NUM] for short description / open a page, ... or CTRL-C to exit""" + with c.warn: + return click.prompt(msg, type=str, default='', show_default=False) + # click.confirm('No display aviaable, do you want to continue?', + # default=True, show_default=False): except KeyboardInterrupt: # return False @@ -129,7 +164,6 @@ def _continue(): def parse_feed(url): - d = feedparser.parse(url) # validate rss URL @@ -137,151 +171,258 @@ def parse_feed(url): return d else: print("INVALID URL feed: {}".format(url)) + # exit() return None def fetch_feeds(urls): + # feeds source + l = len(urls) - 1 + + feeds = [parse_feed(url) for url in urls] + for f, u in zip(feeds, urls): + if not f: + print(c.warn | 'ERROR with {}'.format(u)) + feeds.remove(f) + + zipped = [] + for i, d in enumerate(feeds): + # title = url if d.feed.title else d.feed.title + print(c.magenta | " {}/{} SOURCE>> {}".format(i, l, d.feed.title)) + zipped += d.entries + + # https://wiki.python.org/moin/HowTo/Sorting#The_Old_Way_Using_Decorate-Sort-Undecorate + try: + decorated = [(dateutil.parser.parse(post.published), i, post) + for i, post in enumerate(zipped)] + except: + decorated = [(dateutil.parser.parse(post.updated), i, post) + for i, post in enumerate(zipped)] + decorated.sort(reverse=True) + zipped = [post for time, i, post in decorated] # undecorate - for i, url in enumerate(urls): - - d = parse_feed(url) - - if d is None: - continue # to next url + zipped = dict(enumerate(zipped)) - # feeds source - l = len(urls) - 1 - print( - bcolors.HEADER + "\n {}/{} SOURCE>> {}\n".format(i, l, url) + bcolors.ENDC) + def recurse(zipped): - # print out feeds - zipped = dict(enumerate(d.entries)) + print_feed(zipped) - def recurse(zipped): + kb = _continue() # keystroke listener - print_feed(zipped) + if kb: + user_selected = kb is not '' and kb in str(zipped.keys()) + if user_selected: + # to open page in browser + link = zipped[int(kb)].link + title = zipped[int(kb)].title + try: + desc = zipped[int(kb)].description + desc = clean_txt(desc) + print_desc(title, desc, zipped[int(kb)]) + except AttributeError: + print('\n\tNo description available!!') - kb = _continue() # keystroke listener + if open_it(): + open_page(link, title) + else: + with c.bold: + print('Invalid entry ... {} '.format(kb)) + # repeat with same feeds and listen to kb again + recurse(zipped) - if kb: - user_selected = kb is not '' and kb in str(zipped.keys()) - if user_selected: - # to open page in browser - link = zipped[int(kb)].link - title = zipped[int(kb)].title - try: - desc = zipped[int(kb)].description - desc = clean_txt(desc) - print_desc(title, desc) - except AttributeError: - print('\n\tNo description available!!') + recurse(zipped) - if open_it(): - open_page(link, title) - else: - print( - bcolors.BOLD + 'Invalid entry ... {} '.format(kb) + bcolors.ENDC) - # repeat with same feeds and listen to kb again - recurse(zipped) - recurse(zipped) +def topic_choice(browse, *labels): + log_info(labels) + labels = set(labels) & dbop.labels -def topic_choice(browse): + log_info(labels) - if browse: - topics = dbop.topics() + if not labels: + if browse: - tags = {} + tags = {} - for i, tag in enumerate(topics): - tags[i] = tag - print("{}) {}".format(i, tags[i])) + for i, tag in enumerate(dbop.labels | {'all'}): + tags[i] = tag + print("{}) {}".format(i, tags[i])) - try: - m = '\nChoose the topic (number)? : ' - try: # python 2 - uin = raw_input(m) - except NameError: # python 3 - uin = input(m) - uin = int(uin) - topic = tags[uin] - except: # catch all exceptions - print('\nInvalid choice!') - topic = 'General' + uin = click.prompt('\nChoose the topic (number)? ', type=int) + try: + labels = tags[uin] + except: # catch all exceptions + print('\nInvalid choice!') + labels = [] + else: + labels = [] + if labels == 'all': + urls = dbop.links else: - topic = 'General' - urls = dbop.read(topic) + urls = dbop.link_for_label(labels) return urls def validate_feed(url): if parse_feed(url): - return url + return True else: - exit() + return False -from .support.docopt import docopt +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -def main(): - args = docopt( - __doc__, version="TermFeed 0.0.11 (with pleasure by: Aziz Alto)") - - # parse args - browse = args['-b'] - external = args[''] - add_link = args['-a'] - category = args[''] - delete = args['-d'] - remove = args['-D'] - tags = args['-t'] - rebuild = args['-R'] - - fetch = True - - # get rss urls - if external: - urls = [validate_feed(external)] - else: - urls = topic_choice(browse) +@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS) +@click.pass_context +@click.option('-l', '--label', multiple=True) +@click.option('-n', '--dry-run', is_flag=True) +# @click.option('--debug/--no-debug', default=False) +@click.option('-v', '--verbose', count=True, help='Levels: -v:INFO, -vvv:DEBUG') +def cli(ctx, verbose, label, dry_run): + logging.basicConfig(format=(c.dark_gray | logging.BASIC_FORMAT)) - # if not listing feeds - if add_link or delete or category or tags or rebuild or remove: - fetch = False + logger_pkg = logging.getLogger(__package__) + if verbose >= 3: + logger_pkg.setLevel(logging.DEBUG) + elif verbose >= 1: + logger_pkg.setLevel(logging.INFO) - # updating URLs library - if add_link: - url = validate_feed(add_link) - if category: - dbop.add_link(url, category) - else: - dbop.add_link(url) - if delete: - dbop.remove_link(delete) - - if remove: - dbop.delete_topic(remove) - # display resource contents - if tags: - if category: - dbop.browse_links(category) + if dry_run: + dbop.dry_run = True + + if ctx.invoked_subcommand is None: + log_info('I was invoked without subcommand') + if not label: + fetch_feeds(dbop.links) else: - dbop.print_topics() + fetch_feeds(dbop.link_for_label(*label)) + else: + log_info('I am about to invoke %s' % ctx.invoked_subcommand) + + +@cli.command() +@click.argument('url', nargs=1) +@click.argument('label', nargs=-1) # , help='One or more labels for this url.' +@click.option('--title', default='') +@click.option('-f', '--flag', multiple=True) +def add(url, label, title, flag): + + regex = re.compile( + r'(https://github\.com/|git@github\.com:)(\w+)/([\w-]+)\.git') + if regex.match(url): + (r_prefix, r_user, r_name), = regex.findall(url) + + url_offer = 'https://github.com/' + r_user + \ + '/' + r_name + '/commits/master.atom' + + with c.info: + if click.confirm(''' + It seams, that you try to add a github repo. + But the url seams to be wrong. + Do you mean {}?'''.format(url_offer), default=True): + url = url_offer + flag = list(set(flag) | {'github', 'git'}) + elif 'git' in url and click.confirm(''' + It seams, that your url is a git url, add Flag? + '''.format(url_offer), default=True): + flag = list(set(flag) | set('git')) + if title == '': + title = r_name + + if validate_feed(url): + with dbop as db: + db.add_link(url, *label, title=title, flag=flag) + print("Add URL feed: {}".format(url)) + + +@cli.command() +@click.argument('name') # , help='namme must be a url or a label' +@click.argument('url', nargs=1) +def remove(url): + if url in dbop.links: + # with dbop as dbop: + dbop.remove_link(url) + print("Removed URL feed: {}".format(url)) + else: + print("Could not find URL feed: {}".format(url)) + + +@cli.command() +@click.option('--label/--no-label', default=False) +def show(label): + if label: + print('Labels: ', dbop.labels) + print(dbop.as_yaml) + + +@cli.command() +@click.argument('label', nargs=-1) # , help='One or more labels for this url.' +def browse(label): + urls = topic_choice(browse, *label) + fetch_feeds(urls) + + +@cli.command() +@click.pass_context +def edit(ctx): + import tempfile + from contextlib import suppress + from subprocess import call + + EDITOR = os.environ.get('EDITOR', 'dav') # that easy! + #EDITOR = 'dav' + if EDITOR == 'suplemon': + os.environ['TERM'] = 'xterm-256color' + + initial_message = dbop.as_yaml + with tempfile.NamedTemporaryFile(suffix=".tmp", mode='w+') as tf: + tf.write(initial_message) + tf.flush() + with suppress(KeyboardInterrupt): + call([EDITOR, tf.name]) + + # do the parsing with `tf` using regular File operations. + # for instance: + if False: + tf.seek(0) + edited_message = tf.read() + print(edited_message) + ctx.invoke(load, file=tf.name) + + +@cli.command() +@click.argument('file', type=click.File('r')) +def load(file): + with open(file) as f: + data = yaml.load(f.read()) + print('Loaded: ', file, os.path.exists(file)) + print(yaml.dump(data)) + #log_debug('data keys: ', data.keys()) + #log_debug('data keys: ', data) + + verification = [validate_feed(link) for link in data] + if not all(verification): + for link in data: + if not validate_feed(link): + print("INVALID URL feed: {}".format(link)) + exit() - if rebuild: - dbop.rebuild_library() + with dbop as db: + db.data = data - if fetch: - fetch_feeds(urls) + print(yaml.dump(data)) -# start -if __name__ == '__main__': +def main(): if not _connected(): print('No Internet Connection!') exit() + cli() + +# start +if __name__ == '__main__': main() diff --git a/termfeed/rss.yaml b/termfeed/rss.yaml new file mode 100644 index 0000000..80abcb1 --- /dev/null +++ b/termfeed/rss.yaml @@ -0,0 +1,66 @@ +AI: +- http://cervisia.org/rss_feed.pl/machine_learning_news.rss +- http://feeds.feedburner.com/FeaturedBlogPosts-DataScienceCentral +- http://feeds.feedburner.com/ResourcesDiscussions-DataScienceCentral +- http://feeds.feedburner.com/ResearchDiscussions-DataScienceCentral +- http://feeds.feedburner.com/FeaturedBlogPosts-Bigdatanews +- http://cacm.acm.org/browse-by-subject/artificial-intelligence.rss +- http://www.deepstuff.org/feed/ +- http://feeds.feedburner.com/miriblog +- http://www.csail.mit.edu/csailnews/rss +CS: +- http://feeds.sciencedaily.com/sciencedaily/computers_math/computer_science +- http://phys.org/rss-feed/technology-news/computer-sciences/ +- http://newsoffice.mit.edu/rss/topic/computers +- http://www.nature.com/subjects/computer-science.rss +- http://www.nature.com/subjects/mathematics-and-computing.rss +- http://newsoffice.mit.edu/rss/topic/robotics +- http://cacm.acm.org/news.rss +GEEK: +- http://feeds.feedburner.com/hacker-news-feed-200?format=xml +- http://feeds.feedburner.com/TheHackersNews +- https://news.ycombinator.com/rss +- http://www.reddit.com/r/learnprogramming/ +- http://www.reddit.com/r/algorithms/ +- http://www.reddit.com/r/programming/ +- http://feeds.feedburner.com/SingularityBlog +General: +- http://feeds.feedburner.com/PythonCentral +- http://cacm.acm.org/browse-by-subject/artificial-intelligence.rss +- feed:https://news.ycombinator.com/rss +ML: +- http://hunch.net/?feed=rss2 +- http://research.microsoft.com/rss/downloads.xml +- http://mlg.eng.cam.ac.uk/?feed=rss2 +News: +- feed://www.aljazeera.com/xml/rss/all.xml +- feed://america.aljazeera.com/content/ajam/articles.rss +- http://rt.com/rss/ +- http://feeds.feedburner.com/japantimes +- http://www.japantoday.com/feed +- http://api.breakingnews.com/api/v1/item/?format=rss +Python: +- feed://changelog.com/tagged/python/feed/ +- http://feeds.feedburner.com/PythonCentral +- feed://blog.jupyter.org/rss/ +- http://planetpython.org/rss20.xml +- https://pypi.python.org/pypi?%3Aaction=packages_rss +Research: +- http://www.nsf.gov/rss/rss_www_funding_pgm_annc_inf.xml +- http://www.nsf.gov/statistics/rss/srs_rss.xml +- http://www.darpa.mil/rss +- http://feeds.feedburner.com/blogspot/gJZg +Science: +- https://www.sciencenews.org/feeds/headlines.rss +- http://feeds.reuters.com/reuters/scienceNews +- http://feeds.sciencedaily.com/sciencedaily/top_news +- http://phys.org/rss-feed/space-news/space-exploration/ +- http://phys.org/rss-feed/space-news/astronomy/ +- http://phys.org/rss-feed/physics-news/quantum-physics/ +- http://phys.org/rss-feed/physics-news/physics/ +Stack: +- http://stackoverflow.com/questions/tagged/python +- http://stackoverflow.com/questions/tagged/python+numpy +- http://stackoverflow.com/feeds +- http://stackoverflow.com/questions/tagged/python+django +- http://stackoverflow.com/questions/tagged/python+django+javascript diff --git a/termfeed/support/__init__.py b/termfeed/support/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/termfeed/support/docopt.py b/termfeed/support/docopt.py deleted file mode 100644 index 4d59f1f..0000000 --- a/termfeed/support/docopt.py +++ /dev/null @@ -1,581 +0,0 @@ -"""Pythonic command-line interface parser that will make you smile. - - * http://docopt.org - * Repository and issue-tracker: https://github.com/docopt/docopt - * Licensed under terms of MIT license (see LICENSE-MIT) - * Copyright (c) 2013 Vladimir Keleshev, vladimir@keleshev.com - -""" -import sys -import re - - -__all__ = ['docopt'] -__version__ = '0.6.1' - - -class DocoptLanguageError(Exception): - - """Error in construction of usage-message by developer.""" - - -class DocoptExit(SystemExit): - - """Exit in case user invoked program with incorrect arguments.""" - - usage = '' - - def __init__(self, message=''): - SystemExit.__init__(self, (message + '\n' + self.usage).strip()) - - -class Pattern(object): - - def __eq__(self, other): - return repr(self) == repr(other) - - def __hash__(self): - return hash(repr(self)) - - def fix(self): - self.fix_identities() - self.fix_repeating_arguments() - return self - - def fix_identities(self, uniq=None): - """Make pattern-tree tips point to same object if they are equal.""" - if not hasattr(self, 'children'): - return self - uniq = list(set(self.flat())) if uniq is None else uniq - for i, child in enumerate(self.children): - if not hasattr(child, 'children'): - assert child in uniq - self.children[i] = uniq[uniq.index(child)] - else: - child.fix_identities(uniq) - - def fix_repeating_arguments(self): - """Fix elements that should accumulate/increment values.""" - either = [list(child.children) for child in transform(self).children] - for case in either: - for e in [child for child in case if case.count(child) > 1]: - if type(e) is Argument or type(e) is Option and e.argcount: - if e.value is None: - e.value = [] - elif type(e.value) is not list: - e.value = e.value.split() - if type(e) is Command or type(e) is Option and e.argcount == 0: - e.value = 0 - return self - - -def transform(pattern): - """Expand pattern into an (almost) equivalent one, but with single Either. - - Example: ((-a | -b) (-c | -d)) => (-a -c | -a -d | -b -c | -b -d) - Quirks: [-a] => (-a), (-a...) => (-a -a) - - """ - result = [] - groups = [[pattern]] - while groups: - children = groups.pop(0) - parents = [Required, Optional, OptionsShortcut, Either, OneOrMore] - if any(t in map(type, children) for t in parents): - child = [c for c in children if type(c) in parents][0] - children.remove(child) - if type(child) is Either: - for c in child.children: - groups.append([c] + children) - elif type(child) is OneOrMore: - groups.append(child.children * 2 + children) - else: - groups.append(child.children + children) - else: - result.append(children) - return Either(*[Required(*e) for e in result]) - - -class LeafPattern(Pattern): - - """Leaf/terminal node of a pattern tree.""" - - def __init__(self, name, value=None): - self.name, self.value = name, value - - def __repr__(self): - return '%s(%r, %r)' % (self.__class__.__name__, self.name, self.value) - - def flat(self, *types): - return [self] if not types or type(self) in types else [] - - def match(self, left, collected=None): - collected = [] if collected is None else collected - pos, match = self.single_match(left) - if match is None: - return False, left, collected - left_ = left[:pos] + left[pos + 1:] - same_name = [a for a in collected if a.name == self.name] - if type(self.value) in (int, list): - if type(self.value) is int: - increment = 1 - else: - increment = ([match.value] if type(match.value) is str - else match.value) - if not same_name: - match.value = increment - return True, left_, collected + [match] - same_name[0].value += increment - return True, left_, collected - return True, left_, collected + [match] - - -class BranchPattern(Pattern): - - """Branch/inner node of a pattern tree.""" - - def __init__(self, *children): - self.children = list(children) - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, - ', '.join(repr(a) for a in self.children)) - - def flat(self, *types): - if type(self) in types: - return [self] - return sum([child.flat(*types) for child in self.children], []) - - -class Argument(LeafPattern): - - def single_match(self, left): - for n, pattern in enumerate(left): - if type(pattern) is Argument: - return n, Argument(self.name, pattern.value) - return None, None - - @classmethod - def parse(class_, source): - name = re.findall('(<\S*?>)', source)[0] - value = re.findall('\[default: (.*)\]', source, flags=re.I) - return class_(name, value[0] if value else None) - - -class Command(Argument): - - def __init__(self, name, value=False): - self.name, self.value = name, value - - def single_match(self, left): - for n, pattern in enumerate(left): - if type(pattern) is Argument: - if pattern.value == self.name: - return n, Command(self.name, True) - else: - break - return None, None - - -class Option(LeafPattern): - - def __init__(self, short=None, long=None, argcount=0, value=False): - assert argcount in (0, 1) - self.short, self.long, self.argcount = short, long, argcount - self.value = None if value is False and argcount else value - - @classmethod - def parse(class_, option_description): - short, long, argcount, value = None, None, 0, False - options, _, description = option_description.strip().partition(' ') - options = options.replace(',', ' ').replace('=', ' ') - for s in options.split(): - if s.startswith('--'): - long = s - elif s.startswith('-'): - short = s - else: - argcount = 1 - if argcount: - matched = re.findall('\[default: (.*)\]', description, flags=re.I) - value = matched[0] if matched else None - return class_(short, long, argcount, value) - - def single_match(self, left): - for n, pattern in enumerate(left): - if self.name == pattern.name: - return n, pattern - return None, None - - @property - def name(self): - return self.long or self.short - - def __repr__(self): - return 'Option(%r, %r, %r, %r)' % (self.short, self.long, - self.argcount, self.value) - - -class Required(BranchPattern): - - def match(self, left, collected=None): - collected = [] if collected is None else collected - l = left - c = collected - for pattern in self.children: - matched, l, c = pattern.match(l, c) - if not matched: - return False, left, collected - return True, l, c - - -class Optional(BranchPattern): - - def match(self, left, collected=None): - collected = [] if collected is None else collected - for pattern in self.children: - m, left, collected = pattern.match(left, collected) - return True, left, collected - - -class OptionsShortcut(Optional): - - """Marker/placeholder for [options] shortcut.""" - - -class OneOrMore(BranchPattern): - - def match(self, left, collected=None): - assert len(self.children) == 1 - collected = [] if collected is None else collected - l = left - c = collected - l_ = None - matched = True - times = 0 - while matched: - # could it be that something didn't match but changed l or c? - matched, l, c = self.children[0].match(l, c) - times += 1 if matched else 0 - if l_ == l: - break - l_ = l - if times >= 1: - return True, l, c - return False, left, collected - - -class Either(BranchPattern): - - def match(self, left, collected=None): - collected = [] if collected is None else collected - outcomes = [] - for pattern in self.children: - matched, _, _ = outcome = pattern.match(left, collected) - if matched: - outcomes.append(outcome) - if outcomes: - return min(outcomes, key=lambda outcome: len(outcome[1])) - return False, left, collected - - -class Tokens(list): - - def __init__(self, source, error=DocoptExit): - self += source.split() if hasattr(source, 'split') else source - self.error = error - - @staticmethod - def from_pattern(source): - source = re.sub(r'([\[\]\(\)\|]|\.\.\.)', r' \1 ', source) - source = [s for s in re.split('\s+|(\S*<.*?>)', source) if s] - return Tokens(source, error=DocoptLanguageError) - - def move(self): - return self.pop(0) if len(self) else None - - def current(self): - return self[0] if len(self) else None - - -def parse_long(tokens, options): - """long ::= '--' chars [ ( ' ' | '=' ) chars ] ;""" - long, eq, value = tokens.move().partition('=') - assert long.startswith('--') - value = None if eq == value == '' else value - similar = [o for o in options if o.long == long] - if tokens.error is DocoptExit and similar == []: # if no exact match - similar = [o for o in options if o.long and o.long.startswith(long)] - if len(similar) > 1: # might be simply specified ambiguously 2+ times? - raise tokens.error('%s is not a unique prefix: %s?' % - (long, ', '.join(o.long for o in similar))) - elif len(similar) < 1: - argcount = 1 if eq == '=' else 0 - o = Option(None, long, argcount) - options.append(o) - if tokens.error is DocoptExit: - o = Option(None, long, argcount, value if argcount else True) - else: - o = Option(similar[0].short, similar[0].long, - similar[0].argcount, similar[0].value) - if o.argcount == 0: - if value is not None: - raise tokens.error('%s must not have an argument' % o.long) - else: - if value is None: - if tokens.current() in [None, '--']: - raise tokens.error('%s requires argument' % o.long) - value = tokens.move() - if tokens.error is DocoptExit: - o.value = value if value is not None else True - return [o] - - -def parse_shorts(tokens, options): - """shorts ::= '-' ( chars )* [ [ ' ' ] chars ] ;""" - token = tokens.move() - assert token.startswith('-') and not token.startswith('--') - left = token.lstrip('-') - parsed = [] - while left != '': - short, left = '-' + left[0], left[1:] - similar = [o for o in options if o.short == short] - if len(similar) > 1: - raise tokens.error('%s is specified ambiguously %d times' % - (short, len(similar))) - elif len(similar) < 1: - o = Option(short, None, 0) - options.append(o) - if tokens.error is DocoptExit: - o = Option(short, None, 0, True) - else: # why copying is necessary here? - o = Option(short, similar[0].long, - similar[0].argcount, similar[0].value) - value = None - if o.argcount != 0: - if left == '': - if tokens.current() in [None, '--']: - raise tokens.error('%s requires argument' % short) - value = tokens.move() - else: - value = left - left = '' - if tokens.error is DocoptExit: - o.value = value if value is not None else True - parsed.append(o) - return parsed - - -def parse_pattern(source, options): - tokens = Tokens.from_pattern(source) - result = parse_expr(tokens, options) - if tokens.current() is not None: - raise tokens.error('unexpected ending: %r' % ' '.join(tokens)) - return Required(*result) - - -def parse_expr(tokens, options): - """expr ::= seq ( '|' seq )* ;""" - seq = parse_seq(tokens, options) - if tokens.current() != '|': - return seq - result = [Required(*seq)] if len(seq) > 1 else seq - while tokens.current() == '|': - tokens.move() - seq = parse_seq(tokens, options) - result += [Required(*seq)] if len(seq) > 1 else seq - return [Either(*result)] if len(result) > 1 else result - - -def parse_seq(tokens, options): - """seq ::= ( atom [ '...' ] )* ;""" - result = [] - while tokens.current() not in [None, ']', ')', '|']: - atom = parse_atom(tokens, options) - if tokens.current() == '...': - atom = [OneOrMore(*atom)] - tokens.move() - result += atom - return result - - -def parse_atom(tokens, options): - """atom ::= '(' expr ')' | '[' expr ']' | 'options' - | long | shorts | argument | command ; - """ - token = tokens.current() - result = [] - if token in '([': - tokens.move() - matching, pattern = {'(': [')', Required], '[': [']', Optional]}[token] - result = pattern(*parse_expr(tokens, options)) - if tokens.move() != matching: - raise tokens.error("unmatched '%s'" % token) - return [result] - elif token == 'options': - tokens.move() - return [OptionsShortcut()] - elif token.startswith('--') and token != '--': - return parse_long(tokens, options) - elif token.startswith('-') and token not in ('-', '--'): - return parse_shorts(tokens, options) - elif token.startswith('<') and token.endswith('>') or token.isupper(): - return [Argument(tokens.move())] - else: - return [Command(tokens.move())] - - -def parse_argv(tokens, options, options_first=False): - """Parse command-line argument vector. - - If options_first: - argv ::= [ long | shorts ]* [ argument ]* [ '--' [ argument ]* ] ; - else: - argv ::= [ long | shorts | argument ]* [ '--' [ argument ]* ] ; - - """ - parsed = [] - while tokens.current() is not None: - if tokens.current() == '--': - return parsed + [Argument(None, v) for v in tokens] - elif tokens.current().startswith('--'): - parsed += parse_long(tokens, options) - elif tokens.current().startswith('-') and tokens.current() != '-': - parsed += parse_shorts(tokens, options) - elif options_first: - return parsed + [Argument(None, v) for v in tokens] - else: - parsed.append(Argument(None, tokens.move())) - return parsed - - -def parse_defaults(doc): - defaults = [] - for s in parse_section('options:', doc): - # FIXME corner case "bla: options: --foo" - _, _, s = s.partition(':') # get rid of "options:" - split = re.split('\n[ \t]*(-\S+?)', '\n' + s)[1:] - split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])] - options = [Option.parse(s) for s in split if s.startswith('-')] - defaults += options - return defaults - - -def parse_section(name, source): - pattern = re.compile('^([^\n]*' + name + '[^\n]*\n?(?:[ \t].*?(?:\n|$))*)', - re.IGNORECASE | re.MULTILINE) - return [s.strip() for s in pattern.findall(source)] - - -def formal_usage(section): - _, _, section = section.partition(':') # drop "usage:" - pu = section.split() - return '( ' + ' '.join(') | (' if s == pu[0] else s for s in pu[1:]) + ' )' - - -def extras(help, version, options, doc): - if help and any((o.name in ('-h', '--help')) and o.value for o in options): - print(doc.strip("\n")) - sys.exit() - if version and any(o.name == '--version' and o.value for o in options): - print(version) - sys.exit() - - -class Dict(dict): - def __repr__(self): - return '{%s}' % ',\n '.join('%r: %r' % i for i in sorted(self.items())) - - -def docopt(doc, argv=None, help=True, version=None, options_first=False): - """Parse `argv` based on command-line interface described in `doc`. - - `docopt` creates your command-line interface based on its - description that you pass as `doc`. Such description can contain - --options, , commands, which could be - [optional], (required), (mutually | exclusive) or repeated... - - Parameters - ---------- - doc : str - Description of your command-line interface. - argv : list of str, optional - Argument vector to be parsed. sys.argv[1:] is used if not - provided. - help : bool (default: True) - Set to False to disable automatic help on -h or --help - options. - version : any object - If passed, the object will be printed if --version is in - `argv`. - options_first : bool (default: False) - Set to True to require options precede positional arguments, - i.e. to forbid options and positional arguments intermix. - - Returns - ------- - args : dict - A dictionary, where keys are names of command-line elements - such as e.g. "--verbose" and "", and values are the - parsed values of those elements. - - Example - ------- - >>> from docopt import docopt - >>> doc = ''' - ... Usage: - ... my_program tcp [--timeout=] - ... my_program serial [--baud=] [--timeout=] - ... my_program (-h | --help | --version) - ... - ... Options: - ... -h, --help Show this screen and exit. - ... --baud= Baudrate [default: 9600] - ... ''' - >>> argv = ['tcp', '127.0.0.1', '80', '--timeout', '30'] - >>> docopt(doc, argv) - {'--baud': '9600', - '--help': False, - '--timeout': '30', - '--version': False, - '': '127.0.0.1', - '': '80', - 'serial': False, - 'tcp': True} - - See also - -------- - * For video introduction see http://docopt.org - * Full documentation is available in README.rst as well as online - at https://github.com/docopt/docopt#readme - - """ - argv = sys.argv[1:] if argv is None else argv - - usage_sections = parse_section('usage:', doc) - if len(usage_sections) == 0: - raise DocoptLanguageError('"usage:" (case-insensitive) not found.') - if len(usage_sections) > 1: - raise DocoptLanguageError('More than one "usage:" (case-insensitive).') - DocoptExit.usage = usage_sections[0] - - options = parse_defaults(doc) - pattern = parse_pattern(formal_usage(DocoptExit.usage), options) - # [default] syntax for argument is disabled - #for a in pattern.flat(Argument): - # same_name = [d for d in arguments if d.name == a.name] - # if same_name: - # a.value = same_name[0].value - argv = parse_argv(Tokens(argv), list(options), options_first) - pattern_options = set(pattern.flat(Option)) - for options_shortcut in pattern.flat(OptionsShortcut): - doc_options = parse_defaults(doc) - options_shortcut.children = list(set(doc_options) - pattern_options) - #if any_options: - # options_shortcut.children += [Option(o.short, o.long, o.argcount) - # for o in argv if type(o) is Option] - extras(help, version, argv, doc) - matched, left, collected = pattern.fix().match(argv) - if matched and left == []: # better error message if left? - return Dict((a.name, a.value) for a in (pattern.flat() + collected)) - raise DocoptExit() \ No newline at end of file diff --git a/tests/feed_test.py b/tests/feed_test.py new file mode 100644 index 0000000..2f21983 --- /dev/null +++ b/tests/feed_test.py @@ -0,0 +1,13 @@ +import click +from click.testing import CliRunner + + +from termfeed.feed import * + +def test_feed(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(cli, input='y') + #print(result.output) + #assert result.output == 'Hello World!\n' + # assert result.exit_code == 0