diff --git a/Dockerfile b/Dockerfile index 2ab0fa6..aa0c919 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update -qq \ golang-go \ python3 \ python3-click \ + python3-unidiff \ && apt-get autoclean && apt-get clean && apt-get -y autoremove \ && update-ca-certificates \ && rm -rf /var/lib/apt/lists/* @@ -34,6 +35,7 @@ RUN git clone https://github.com/reviewdog/reviewdog \ COPY entrypoint.sh /opt/antmicro/entrypoint.sh COPY action.py /opt/antmicro/action.py +COPY rdf_gen.py /opt/antmicro/rdf_gen.py WORKDIR /opt/antmicro ENTRYPOINT ["/opt/antmicro/entrypoint.sh"] diff --git a/entrypoint.sh b/entrypoint.sh index 419a078..d1aab90 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -16,45 +16,42 @@ fi touch "$INPUT_LOG_FILE" export REVIEWDOG_GITHUB_API_TOKEN="$INPUT_GITHUB_TOKEN" -patch=$(mktemp) - -/opt/antmicro/action.py \ - --conf-file "$INPUT_CONFIG_FILE" \ - --extra-opts "$INPUT_EXTRA_ARGS" \ - --exclude-paths "$INPUT_EXCLUDE_PATHS" \ - --log-file "$INPUT_LOG_FILE" \ - --patch "$patch" \ - "$INPUT_PATHS" || exitcode=$? - -# If posing both change suggestions and review -# first remove the fixed parts from (INPUT_LOG_FILE) -# in order not to double-report -if [ "$INPUT_SUGGEST_FIXES" = "true" ] && [ "$INPUT_REVIEWDOG_REPORTER" = "github-pr-review" ] -then - # remove every line containing "(fixed)" and the preceding line - perl -i -ne 'push @lines, $_; - splice @lines, 0, 2 if /\(fixed\)/; - print shift @lines if @lines > 1 - }{ print @lines;' "$INPUT_LOG_FILE" - - echo "posting autofix results" - "$GOBIN"/reviewdog -name="verible-verilog-lint" \ - -f=diff -f.diff.strip=1 \ - -reporter="github-pr-review" \ - -filter-mode="diff_context" \ - -level="info" \ - -diff="$diff_cmd" \ - -fail-on-error="false" <"$patch" || true +rdf_log=$(mktemp) +if [ "$INPUT_SUGGEST_FIXES" = "true" ]; then + echo "suggesting fixes" + patch=$(mktemp) + /opt/antmicro/action.py \ + --conf-file "$INPUT_CONFIG_FILE" \ + --extra-opts "$INPUT_EXTRA_ARGS" \ + --exclude-paths "$INPUT_EXCLUDE_PATHS" \ + --log-file "$INPUT_LOG_FILE" \ + --patch "$patch" \ + "$INPUT_PATHS" || exitcode=$? + + /opt/antmicro/rdf_gen.py \ + --efm-file "$INPUT_LOG_FILE" \ + --diff-file "$patch" > "$rdf_log" + rm "$patch" +else + echo "not suggesting fixes" + /opt/antmicro/action.py \ + --conf-file "$INPUT_CONFIG_FILE" \ + --extra-opts "$INPUT_EXTRA_ARGS" \ + --exclude-paths "$INPUT_EXCLUDE_PATHS" \ + --log-file "$INPUT_LOG_FILE" \ + "$INPUT_PATHS" || exitcode=$? + + /opt/antmicro/rdf_gen.py \ + --efm-file "$INPUT_LOG_FILE" > "$rdf_log" fi -rm "$patch" echo "Running reviewdog" -"$GOBIN"/reviewdog -efm="%f:%l:%c: %m" \ +"$GOBIN"/reviewdog -f=rdjson \ -reporter="$INPUT_REVIEWDOG_REPORTER" \ -fail-on-error="false" \ -name="verible-verilog-lint" \ - -diff="$diff_cmd" < "$INPUT_LOG_FILE" || cat "$INPUT_LOG_FILE" + -diff="$diff_cmd" < "$rdf_log" || cat "$INPUT_LOG_FILE" if [ -f "$event_file" ]; then git checkout - diff --git a/rdf_gen.py b/rdf_gen.py new file mode 100755 index 0000000..ef8617a --- /dev/null +++ b/rdf_gen.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +import re +import click +import json +from unidiff import PatchSet + + +class Fix: + def __init__(self, filename, text, start_line, end_line): + self.filename = filename + self.text = text + self.start_line = start_line + self.end_line = end_line + + def __repr__(self): + return 'Fix: ' + self.filename + + +class ErrorMessage: + def __init__(self, filename, line, column, message): + self.filename = filename + self.line = line + self.column = column + self.message = message + self.suggestion = None + + def __repr__(self): + return f'{self.filename}:{self.line}:{self.column}: {self.message}' + + def fix(self, suggestion): + '''Adds a change suggestion to an existing ErrorMessage''' + self.suggestion = suggestion + + def as_rdf_dict(self): + '''Creates a dictionary with data used as a component + in Reviewdog Diagnostic Format + + The result is a dict for a single element of 'diagnostics' node in RDF + implements this structure: + https://github.com/reviewdog/reviewdog/blob/master/proto/rdf/reviewdog.proto#L39 + ''' + result = { + 'message': self.message, + 'location': { + 'path': self.filename, + 'range': { + 'start': {'line': self.line, 'column': self.column} + } + }, + 'severity': 'WARNING', + # rule code is embedded in the message, but we can move it here: + # 'code': + } + + if self.suggestion: + result['suggestions'] = [{ + 'range': { + 'start': {'line': self.line, 'column': self.column}, + 'end': {'line': self.suggestion.end_line} + }, + 'text': self.suggestion.text + }] + + return result + + +def error_messages_to_rdf(messages): + '''Create a dictionary structured as Reviewdog Diagnostic Format + using ErrorMessages + + implements this structure: + https://github.com/reviewdog/reviewdog/blob/master/proto/rdf/reviewdog.proto#L23 + + Returns + ------- + a dictionary ready to be json-dumped to look like this: + https://github.com/reviewdog/reviewdog/tree/master/proto/rdf#rdjson + + ''' + result = { + 'source': {'name': 'verible-verilog-lint', + 'url': 'https://github.com/chipsalliance/verible'}, + 'severity': 'WARNING' + } + result['diagnostics'] = tuple([msg.as_rdf_dict() for msg in messages]) + return result + + +def read_efm(filename): + '''Reads errorformat-ed log from linter + + the syntax of each line should be: "%f:%l:%c: %m" + the fields are documented here: + https://vim-jp.org/vimdoc-en/quickfix.html#error-file-format + all non-matching lines are skipped + + Returns + ------- + a list of ErrorMessage, an instance for each error is created + ''' + with open(filename, 'r') as f: + lines = f.readlines() + + messages = [] + for line in lines: + data = re.split(':', line) + if len(data) < 4: + # skip this line, it's not errorformat + continue + if len(data) > 4: + # there are ':' inside the message part + # merge the message part into one string + data = data[0:3] + [''.join(data[3:])] + + # now the data has 4 elements + data = [elem.strip() for elem in data] + messages.append( + ErrorMessage(data[0], int(data[1]), int(data[2]), data[3]) + ) + + return messages + + +def read_diff(filename): + '''Read unified diff file with code changes + + Returns + ------- + a list of Fix, an instance for each hunk in the diff file is created + ''' + patch_set = PatchSet.from_filename(filename, encoding='utf-8') + fixes = [] + + # iterating over a PatchSet returns consecutive lines + # indexing [] a PatchSet returns patches for consecutive files + for file_no in range(len(patch_set)): + patch = patch_set[file_no] + path = patch.path + for hunk in patch: + removed_lines = [ + line[1:] for line in hunk.source if line.startswith('-') + ] + added_lines = [ + line[1:] for line in hunk.target if line.startswith('+') + ] + start_line = hunk.source_start + 1 + end_line = hunk.source_start + 1 + len(removed_lines) + + # if the fix only deletes text, + # added_lines will be empty as expected + # if the fix only adds text + # start_line == end_line as expected + fixes.append( + Fix(path, ''.join(added_lines), start_line, end_line) + ) + + return fixes + + +def apply_fixes(err_messages, fixes): + '''Add change suggestions to ErrorMessages using Fix objects + + this function matches Fixes with their corresponding ErrorMessages + using file names and line numbers, then applies the Fixes + + Prints a message if a Fix doesn't match any of the ErrorMessages + the Fix is skipped in this case + + Returns + ------- + None + ''' + for fix in fixes: + filtered_msgs = [ + msg for msg in err_messages + if msg.filename == fix.filename and msg.line == fix.start_line + ] + + if not filtered_msgs: + print(f'Did not find any errors to be solved by fix: {fix}') + continue + + filtered_msgs[0].fix(fix) + + +@click.command() +@click.option('--efm-file', '-e', type=click.Path(exists=True), required=True, + help='name of a file containing linter output in errorformat') +@click.option('--diff-file', '-d', type=click.Path(exists=True), + required=False, help='name of a file containing ' + 'change suggestions in diff format') +def main(efm_file, diff_file): + '''Generate Reviewdog Diagnostic Format file, + using a log file from a linter (errorformat) and optionally a patch with + fix suggestions + ''' + messages = read_efm(efm_file) + + if diff_file: + fixes = read_diff(diff_file) + apply_fixes(messages, fixes) + + print(json.dumps(error_messages_to_rdf(messages))) + + +if __name__ == '__main__': + main()