diff --git a/scripts/filter-doc-log.sh b/scripts/filter-doc-log.sh index 4b48457d..fbbd4db0 100755 --- a/scripts/filter-doc-log.sh +++ b/scripts/filter-doc-log.sh @@ -33,8 +33,7 @@ fi if [ -s "${LOG_FILE}" ]; then echo "run $KI_SCRIPT" - #python3 $KI_SCRIPT --config-dir ${CONFIG_DIR} ${LOG_FILE} > ${BUILDDIR}/doc.warnings 2>&1 - $KI_SCRIPT --config-dir ${CONFIG_DIR} ${LOG_FILE} > ${BUILDDIR}/doc.warnings + $KI_SCRIPT --config-dir ${CONFIG_DIR} ${LOG_FILE} -o ${BUILDDIR}/doc.warnings 2>&1 if [ -s ${BUILDDIR}/doc.warnings ]; then echo echo -e "${red}New errors/warnings found, please fix them:" diff --git a/scripts/filter-known-issues.py b/scripts/filter-known-issues.py index 79f56fd1..b69b584e 100755 --- a/scripts/filter-known-issues.py +++ b/scripts/filter-known-issues.py @@ -123,133 +123,105 @@ def config_import(paths): config_import_path(path) -arg_parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) -arg_parser.add_argument("-v", "--verbosity", action="count", default=0, - help="increase verbosity") -arg_parser.add_argument("-q", "--quiet", action="count", default=0, - help="decrease verbosity") -arg_parser.add_argument("-e", "--errors", action="store", default=None, - help="file where to store errors") -arg_parser.add_argument("-w", "--warnings", action="store", default=None, - help="file where to store warnings") -arg_parser.add_argument("-c", "--config-dir", action="append", nargs="?", - default=[".known-issues/"], - help="configuration directory (multiple can be " - "given; if none given, clears the current list) " - "%(default)s") -arg_parser.add_argument("FILENAMEs", nargs="+", - help="files to filter") -args = arg_parser.parse_args() - -logging.basicConfig(level=40 - 10 * (args.verbosity - args.quiet), - format="%(levelname)s: %(message)s") - -path = ".known-issues/" -logging.debug("Reading configuration from directory `%s`", path) -config_import(args.config_dir) - -exclude_ranges = [] - -if args.warnings: - warnings = open(args.warnings, "w") -else: - warnings = None -if args.errors: - errors = open(args.errors, "w") -else: - errors = None - - -def report_error(data): - sys.stdout.write(data) - if errors: - errors.write(data) - - -def report_warning(data): - sys.stderr.write(data) - if warnings: - warnings.write(data) - - -if args.warnings: - warnings = open(args.warnings, "w") -else: - warnings = None -if args.errors: - errors = open(args.errors, "w") -else: - errors = None - - -def report_error(data): - sys.stdout.write(data.decode('utf-8')) - if errors: - errors.write(data) - - -def report_warning(data): - sys.stderr.write(data) - if warnings: - warnings.write(data) - - -for filename in args.FILENAMEs: - if os.stat(filename).st_size == 0: - continue # skip empty log files - try: - with open(filename, "r+b") as f: - logging.info("%s: filtering", filename) - # Yeah, this should be more protected in case of exception - # and such, but this is a short running program... - mm = mmap.mmap(f.fileno(), 0) - for ex, origin, flags in exclude_regexs: - logging.info("%s: searching from %s: %s", - filename, origin, ex.pattern) - for m in re.finditer(ex.pattern, mm, re.MULTILINE): - logging.info("%s: %s-%s: match from from %s %s", - filename, m.start(), m.end(), origin, flags) - if 'warning' in flags: - exclude_ranges.append((m.start(), m.end(), True)) - else: - exclude_ranges.append((m.start(), m.end(), False)) - - exclude_ranges = sorted(exclude_ranges, key=lambda r: r[0]) - logging.warning( - "%s: ranges excluded: %s", - filename, - exclude_ranges) - - # Decide what to do with what has been filtered; warnings - # go to stderr and warnings file, errors to stdout, what - # is ignored is just dumped. - offset = 0 - for b, e, warning in exclude_ranges: +def print_error(f, str): + tmp = str.decode("utf-8") + f.write(tmp) + +def filter_log(args): + exclude_ranges = [] + + output_file = args.output + fout = open(output_file, "w") + + for filename in args.FILENAMEs: + if os.stat(filename).st_size == 0: + continue # skip empty log files + try: + with open(filename, "r+b") as f: + logging.info("%s: filtering", filename) + # Yeah, this should be more protected in case of exception + # and such, but this is a short running program... + mm = mmap.mmap(f.fileno(), 0) + for ex, origin, flags in exclude_regexs: + logging.info("%s: searching from %s: %s", + filename, origin, ex.pattern) + for m in re.finditer(ex.pattern, mm, re.MULTILINE): + logging.info("%s: %s-%s: match from from %s %s", + filename, m.start(), m.end(), origin, flags) + if 'warning' in flags: + exclude_ranges.append((m.start(), m.end(), True)) + else: + exclude_ranges.append((m.start(), m.end(), False)) + + exclude_ranges = sorted(exclude_ranges, key=lambda r: r[0]) + logging.warning( + "%s: ranges excluded: %s", + filename, + exclude_ranges) + + # Decide what to do with what has been filtered; warnings + # go to stderr and warnings file, errors to stdout, what + # is ignored is just dumped. + offset = 0 + for b, e, warning in exclude_ranges: + mm.seek(offset) + if b > offset: + # We have something not caught by a filter, an error + logging.info("%s: error range (%d, %d), from %d %dB", + filename, offset, b, offset, b - offset) + print_error(fout, mm.read(b - offset)) + mm.seek(b) + if warning == True: # A warning, print it + mm.seek(b) + logging.info("%s: warning range (%d, %d), from %d %dB", + filename, b, e, offset, e - b) + print_error(fout, mm.read(e - b)) + else: # Exclude, ignore it + d = b - offset + logging.info("%s: exclude range (%d, %d), from %d %dB", + filename, b, e, offset, d) + offset = e mm.seek(offset) - if b > offset: - # We have something not caught by a filter, an error - logging.info("%s: error range (%d, %d), from %d %dB", - filename, offset, b, offset, b - offset) - report_error(mm.read(b - offset)) - mm.seek(b) - if warning == True: # A warning, print it - mm.seek(b) - logging.info("%s: warning range (%d, %d), from %d %dB", - filename, b, e, offset, e - b) - report_warning(mm.read(e - b)) - else: # Exclude, ignore it - d = b - offset - logging.info("%s: exclude range (%d, %d), from %d %dB", - filename, b, e, offset, d) - offset = e - mm.seek(offset) - if len(mm) != offset: - logging.info("%s: error final range from %d %dB", - filename, offset, len(mm)) - report_error(mm.read(len(mm) - offset - 1)) - del mm - except Exception as e: - logging.error("%s: cannot load: %s", filename, e) - raise + if len(mm) != offset: + logging.info("%s: error final range from %d %dB", + filename, offset, len(mm)) + print_error(fout, mm.read(len(mm) - offset - 1)) + del mm + f.close() + except Exception as e: + logging.error("%s: cannot load: %s", filename, e) + f.close() + raise + + +if __name__=="__main__": + arg_parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + arg_parser.add_argument("-v", "--verbosity", action="count", default=0, + help="increase verbosity") + arg_parser.add_argument("-q", "--quiet", action="count", default=0, + help="decrease verbosity") + arg_parser.add_argument("-e", "--errors", action="store", default=None, + help="file where to store errors") + arg_parser.add_argument("-w", "--warnings", action="store", default=None, + help="file where to store warnings") + arg_parser.add_argument("-c", "--config-dir", action="append", nargs="?", + default=[".known-issues/"], + help="configuration directory (multiple can be " + "given; if none given, clears the current list) " + "%(default)s") + arg_parser.add_argument("FILENAMEs", nargs="+", + help="files to filter") + arg_parser.add_argument("-o", "--output", action="store", default="doc.warnings", + help="file where to store warnings") + args = arg_parser.parse_args() + + logging.basicConfig(level=40 - 10 * (args.verbosity - args.quiet), + format="%(levelname)s: %(message)s") + + path = ".known-issues/" + logging.debug("Reading configuration from directory `%s`", path) + config_import(args.config_dir) + + filter_log(args) \ No newline at end of file