From ca59fff509f93ac48394f163cefa60a16352b362 Mon Sep 17 00:00:00 2001 From: Anil Vishnoi Date: Fri, 13 Sep 2024 01:33:27 -0700 Subject: [PATCH] Add python, markdown and github action linter to ensure the code maintenance. Thanks to docling project- took some good bits around python linting from the project. These checks are also added in the markdown, so that user can run these checks locally to recreate the errors. Signed-off-by: Anil Vishnoi --- .github/actions/setup-poetry/action.yml | 19 ++++++++++++++ .github/workflows/actionlint.yml | 23 +++++++++++++++++ .github/workflows/checks.yml | 34 +++++++++++++++++++++++++ .gitignore | 1 + .markdownlint-cli2.yaml | 6 +++++ CODE_OF_CONDUCT.md | 4 +-- CONTRIBUTING.md | 28 +++++++++----------- Makefile | 26 +++++++++++++++++-- README.md | 1 - 9 files changed, 121 insertions(+), 21 deletions(-) create mode 100644 .github/actions/setup-poetry/action.yml create mode 100644 .github/workflows/actionlint.yml create mode 100644 .github/workflows/checks.yml create mode 100644 .markdownlint-cli2.yaml diff --git a/.github/actions/setup-poetry/action.yml b/.github/actions/setup-poetry/action.yml new file mode 100644 index 0000000..e9ce697 --- /dev/null +++ b/.github/actions/setup-poetry/action.yml @@ -0,0 +1,19 @@ +name: 'Set up Poetry and install' +description: 'Set up a specific version of Poetry and install dependencies using caching.' +inputs: + python-version: + description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax." + default: '3.11' +runs: + using: 'composite' + steps: + - name: Install poetry + run: pipx install poetry==1.8.3 + shell: bash + - uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + shell: bash diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml new file mode 100644 index 0000000..c2ae5b4 --- /dev/null +++ b/.github/workflows/actionlint.yml @@ -0,0 +1,23 @@ +name: Lint GitHub Actions workflows +on: + push: + branches: ["main"] + paths: + - '.github/**' + pull_request: + branches: ["main"] + paths: + - '.github/**' + +jobs: + actionlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download actionlint + id: get_actionlint + run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + shell: bash + - name: Check workflow files + run: PATH=".:$PATH" make action-lint + shell: bash diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..eecf60e --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,34 @@ +name: Run linter checks +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + py-lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.11'] + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-poetry + with: + python-version: ${{ matrix.python-version }} + - name: Run styling check + run: poetry run pre-commit run --all-files + + markdown-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: markdownlint-cli2-action + uses: DavidAnson/markdownlint-cli2-action@v16 + with: + globs: "**/*.md" + diff --git a/.gitignore b/.gitignore index 1369e39..694b749 100644 --- a/.gitignore +++ b/.gitignore @@ -441,3 +441,4 @@ pip-selfcheck.json # Makefile .action-lint +.markdown-lint diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000..78913e3 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,6 @@ +config: + line-length: false + no-emphasis-as-header: false + first-line-heading: false +globs: + - "**/*.md" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c7d5ea1..9d68f51 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -115,7 +115,7 @@ the community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org)], version 2.0, available at [https://www.contributor-covenant.org/version/2/0/code_of_conduct.html](https://www.contributor-covenant.org/version/2/0/code_of_conduct.html). @@ -126,4 +126,4 @@ Homepage: [https://www.contributor-covenant.org](https://www.contributor-covenan For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq). Translations are available at -[https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations). \ No newline at end of file +[https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0dfb26..e86e8a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,5 @@ ## Contributing In General + Our project welcomes external contributions. If you have an itch, please feel free to scratch it. @@ -34,14 +35,13 @@ maintainers of each component affected. For a list of the maintainers, see the [MAINTAINERS.md](MAINTAINERS.md) page. - ## Legal Each source file must include a license header for the MIT Software. Using the SPDX format is the simplest approach. e.g. -``` +```text /* Copyright IBM Inc. All rights reserved. @@ -60,54 +60,54 @@ must include a sign-off statement in the commit message. Here is an example Signed-off-by line, which indicates that the submitter accepts the DCO: -``` +```text Signed-off-by: John Doe ``` You can include this automatically when you commit a change to your local git repository using the following command: -``` +```text git commit -s ``` - ## Communication Please feel free to connect with us using the [discussion section](https://github.com/DS4SD/docling-serve/discussions). - - ## Developing ### Usage of Poetry We use Poetry to manage dependencies. - #### Install -To install, see the documentation here: https://python-poetry.org/docs/master/#installing-with-the-official-installer +To install, see the documentation here: 1. Install the Poetry globally in your machine + ```bash curl -sSL https://install.python-poetry.org | python3 - ``` + The installation script will print the installation bin folder `POETRY_BIN` which you need in the next steps. 2. Make sure Poetry is in your `$PATH` - for `zsh` + ```sh echo 'export PATH="POETRY_BIN:$PATH"' >> ~/.zshrc ``` + - for `bash` + ```sh echo 'export PATH="POETRY_BIN:$PATH"' >> ~/.bashrc ``` 3. The official guidelines linked above include useful details on the configuration of autocomplete for most shell environments, e.g. Bash and Zsh. - #### Create a Virtual Environment and Install Dependencies To activate the Virtual Environment, run: @@ -122,7 +122,7 @@ To spawn a shell with the Virtual Environment activated. If the Virtual Environm poetry install ``` -**(Advanced) Use a Specific Python Version** +#### (Advanced) Use a Specific Python Version If for whatever reason you need to work in a specific (older) version of Python, run: @@ -132,7 +132,6 @@ poetry env use $(which python3.10) This creates a Virtual Environment with Python 3.10. For other versions, replace `$(which python3.10)` by the path to the interpreter (e.g., `/usr/bin/python3.8`) or use `$(which pythonX.Y)`. - #### Add a new dependency ```bash @@ -146,7 +145,6 @@ We use the following tools to enforce code style: - iSort, to sort imports - Black, to format code - We run a series of checks on the code base on every commit, using `pre-commit`. To install the hooks, run: ```bash @@ -155,10 +153,8 @@ pre-commit install To run the checks on-demand, run: -``` +```shell pre-commit run --all-files ``` Note: Checks like `Black` and `isort` will "fail" if they modify files. This is because `pre-commit` doesn't like to see files modified by their Hooks. In these cases, `git add` the modified files and `git commit` again. - - diff --git a/Makefile b/Makefile index 1bca857..6525afd 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,12 @@ endif TAG=$(shell git rev-parse HEAD) -lint: +action-lint-file: $(CMD_PREFIX) touch .action-lint +md-lint-file: + $(CMD_PREFIX) touch .markdown-lint + .PHONY: docling-serve-cpu-image docling-serve-cpu-image: Containerfile ## Build docling-serve "cpu only" continaer image $(ECHO_PREFIX) printf " %-12s Containerfile\n" "[docling-serve CPU ONLY]" @@ -37,7 +40,7 @@ docling-serve-gpu-image: Containerfile ## Build docling-serve continaer image wi .PHONY: action-lint action-lint: .action-lint ## Lint GitHub Action workflows -.action-lint: $(shell find .github -type f) | lint +.action-lint: $(shell find .github -type f) | action-lint-file $(ECHO_PREFIX) printf " %-12s .github/...\n" "[ACTION LINT]" $(CMD_PREFIX) if ! which actionlint $(PIPE_DEV_NULL) ; then \ echo "Please install actionlint." ; \ @@ -51,3 +54,22 @@ action-lint: .action-lint ## Lint GitHub Action workflows fi $(CMD_PREFIX) actionlint -color $(CMD_PREFIX) touch $@ + +.PHONY: md-lint +md-lint: .md-lint ## Lint markdown files +.md-lint: $(wildcard */**/*.md) | md-lint-file + $(ECHO_PREFIX) printf " %-12s ./...\n" "[MD LINT]" + $(CMD_PREFIX) docker run --rm -v $$(pwd):/workdir davidanson/markdownlint-cli2:v0.14.0 "**/*.md" + $(CMD_PREFIX) touch $@ + + +.PHONY: py-Lint +py-lint: ## Lint Python files + $(ECHO_PREFIX) printf " %-12s ./...\n" "[PY LINT]" + $(CMD_PREFIX) if ! which poetry $(PIPE_DEV_NULL) ; then \ + echo "Please install poetry." ; \ + echo "pip install poetry" ; \ + exit 1 ; \ + fi + $(CMD_PREFIX) poetry install --all-extras + $(CMD_PREFIX) poetry run pre-commit run --all-files diff --git a/README.md b/README.md index c73c5cc..a8d1790 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ > [!NOTE] > This is an unstable draft implementation which will quickly evolve. - ## Development Install the dependencies