spell_checker.py

#!/usr/bin/python


# Dealing with non-auto-fixed lines:
# ----------------------------------
#     1. Get the matching egrep command:
#            $> egrep -rl "^         [\*]{70}\/$" src
#     2. Open all matching files in vim:
#            $> CMD='egrep -rl "^         [\*]{70}\/$" src'
#            $> v `eval ${CMD} | ol`
#     3. Get the matching vim search command (it'll be very similar to the egrep
#        command, except that '{' & '}' would need to be escaped as well:
#            /^         [\*]\{70\}\/$
#     4. Record a suitable macro, say to @q (ensure that 'n' is the first
#        character recorded, so that the macro can be run many number of times
#        with it refusing to run if there are no more matches)
#     5. Set hidden, so that you can change buffers without saving
#            :se hidden
#     6. Run the macro many times in each buffer
#            :bufdo normal 20@q
#     7. Save all buffers
#            :wa 


####################################################################################################
#
#   Imports
#
####################################################################################################

from enchant         import Dict, DictWithPWL
from enchant.checker import SpellChecker

import argparse
import getpass
import fnmatch
import os
import re
import sys



####################################################################################################
#
#   Checks for possible spelling mistakes in the comments of a file.
#
#   Params:
#       filename = file to analyse
#       checker  = object used to check spellings
#
#   Returns:
#       a set containing all potential spelling mistakes
#
####################################################################################################

def analyse_file(filename, checker):
    f = open(filename, 'r')
    contents = f.read()

    errors = set()

    if filename.endswith(".c") or filename.endswith(".cc") or filename.endswith(".cpp"):
        pattern = re.compile(
            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
            re.DOTALL | re.MULTILINE
        )
        text_list = re.findall(pattern, contents)
    elif filename.endswith(".d"):
        pattern = re.compile(
            r'/\+.*?\+/|//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
            re.DOTALL | re.MULTILINE
        )
        text_list = re.findall(pattern, contents)
    elif filename.endswith(".py") or filename.endswith(".sh") or filename == "Makefile":
        pattern = re.compile(r'#.*')
        text_list = re.findall(pattern, contents)
    elif filename.endswith(".vim"):
        pattern = re.compile(r'^\s*?".*?$', re.MULTILINE)
        text_list = re.findall(pattern, contents)
    elif filename.endswith(".txt") or filename.endswith(".markdown") or filename.endswith(".md") or filename.endswith(".rst"):
        text_list = contents.split()
    else:
        return errors

    for text in text_list:
        checker.set_text(text)
        for error in checker:
            errors.add(error.word)

    return errors


####################################################################################################
#
#   Displays a progress bar to indicate the status of the program.
#
#   Params:
#       progress = a floating point integer between 0 and 1 indicating how much of the program is
#                  done (0 implies nothing is done, 1 implies the program is complete)
#
#   Returns:
#       a set containing all errors (to be thought of as a set of suggestions)
#
####################################################################################################

def update_progress(progress):
    bar_len = 80 # Modify this to change the length of the progress bar

    if not isinstance(progress, float):
        return

    if progress < 0:
        progress = 0
    elif progress >= 1:
        progress = 1

    block = int(round(bar_len * progress))
    text = "Status: [{0}] {1}%".format( "="*(block-1) + ">" + " "*(bar_len-block-1), int(progress*100))

    sys.stdout.write("\r")
    sys.stdout.write("\033[K") # Clear to the end of line
    sys.stdout.write(text)
    sys.stdout.flush()


####################################################################################################
#
#   Gets all valid words.
#
#   Params:
#       dictionary = dictionary to use
#       whitelist = file containing words that are to be considered valid (optional)
#
#   Returns:
#       a 'Dict' object containing all valid words
#
####################################################################################################

def get_valid_words(dictionary, whitelist=""):
    if whitelist:
        d = DictWithPWL(dictionary, whitelist)
    else:
        d = Dict(dictionary)

    return d


####################################################################################################
#
#   Filters a list of files to get only supported files
#
#   Params:
#       all_files = list of files to be filtered
#
#   Returns:
#       a list containing only supported files
#
####################################################################################################

def filter_supported_files(all_files):
    supported_extensionless_files = [ 'Makefile' ]
    supported_extensioned_file_types = [ 'c', 'cc', 'cpp', 'd', 'markdown', 'md', 'py', 'rst', 'sh',
                                         'txt', 'vim' ]

    filtered_files = []

    for filename in all_files:
        if filename in supported_extensionless_files:
            filtered_files.append(filename)
        else:
            dummy, extension = os.path.splitext(filename)
            if extension[1:] in supported_extensioned_file_types:
                filtered_files.append(filename)

    return filtered_files


####################################################################################################
#
#    Execution starts here! [main :)]
#
####################################################################################################

parser = argparse.ArgumentParser(description='Spell Checker')
parser.add_argument('-k', '--skip-count', nargs='?',
                    required = False, default = "",
                    help = 'Number of files to skip')
parser.add_argument('-n', '--non-interactive', action='store_true',
                    required = False, default = False,
                    help = 'Run for all files without waiting for user input')
parser.add_argument('-s', '--suggest', action='store_true',
                    required = False, default = False,
                    help = 'Provide suggestions for possible errors')
parser.add_argument('-u', '--us-english', action='store_true',
                    required = False, default = False,
                    help = 'Use American English')
parser.add_argument('-w', '--whitelist', nargs='?',
                    required = False, default = "",
                    help = 'Provide a whitelist file')
args = vars(parser.parse_args())

cwd = os.getcwd()

files = []

for root, subdirs, filenames in os.walk(cwd):
    for filename in filter_supported_files(filenames):
        files.append(os.path.join(root, filename))

total_files = len(files)

if (total_files == 0):
    print("No supported files to process. Aborting.")
    sys.exit(1)

print("Files to analyse : " + str(total_files))

files_to_skip = args['skip_count']

if not files_to_skip:
    files_to_skip = 0
else:
    files_to_skip = int(files_to_skip)

print("Files to skip    : " + str(files_to_skip))
print("")

if files_to_skip > total_files:
    print("Files to skip more than total files. Aborting.")
    sys.exit(2)

# Note 'us_english' instead of 'us-english' since the hyphen is automatically converted to an
# underscore
dictionary = "en_US" if args['us_english'] else "en_GB"

# Get the collection of all valid words
valid_words = get_valid_words(dictionary, args['whitelist'])

# Make a spell checker object using all the valid words
checker = SpellChecker(valid_words)

files_analysed = 0

files_with_errors = 0

for f in files:
    update_progress(files_analysed / float(total_files))

    errors = set()

    errors = analyse_file(f, checker)

    files_analysed += 1

    if (len(errors)):
        files_with_errors += 1

        # Get rid of the progress bar
        sys.stdout.write("\r")
        sys.stdout.write("\033[K") # Clear to the end of line
        sys.stdout.flush()

        if files_to_skip >= files_with_errors:
            print(str(files_with_errors) + ". " + f + " -- skipped")
            if files_to_skip == files_with_errors:
                print("")
            continue

        print(str(files_with_errors) + ". " + f + ":")
        for e in errors:
            print("    * " + e)
            if args['suggest']:
                print("        ",)
                print(valid_words.suggest(e))
        print("")

        # Note 'non_interactive' instead of 'non-interactive' since the hyphen is automatically
        # converted to an underscore
        if not args['non_interactive']:
            dummy = getpass.getpass("")

update_progress(1.0)

print("")